From 1e076a83b69f147d6346651440be11ea3df944f2 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 20:24:52 +0800 Subject: [PATCH 01/51] split transcribe.cpp into library-level and ASR dispatchers Pure move of the ASR C API (sessions, open/close, run, batch, streaming, stream text policy, capabilities, limits, tokenize, timings, result and batch accessors, ASR struct inits) into src/transcribe-asr.cpp. src/transcribe.cpp keeps status, version, ABI sizes, logging, backends, devices, model load/free and model queries. Non-move edits: - api_guard_* move to src/transcribe-api-guard.h (namespace transcribe) - enum_field_raw moves to src/transcribe-abi.h - four ASR log sites call transcribe::log_msg(level, "%s", buf) instead of the library-private emitter (same output) - file header comments, includes, using-declarations; AGENTS.md and two comments point at the new locations No behavior change: exported symbols identical, abihash unchanged, ctest unchanged. --- AGENTS.md | 2 +- src/CMakeLists.txt | 1 + src/transcribe-abi.h | 25 +- src/transcribe-api-guard.h | 63 + src/transcribe-asr.cpp | 2465 ++++++++++++++++++++++++++++++++++ src/transcribe-model.h | 2 +- src/transcribe.cpp | 2608 ++---------------------------------- 7 files changed, 2637 insertions(+), 2529 deletions(-) create mode 100644 src/transcribe-api-guard.h create mode 100644 src/transcribe-asr.cpp diff --git a/AGENTS.md b/AGENTS.md index 3c547af58..89a74ccf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ scripts/ci/clang-format.sh --check # verify, no changes No C++ exception may escape a public entry point. - A new public entry point must either route through an `api_guard_*` - wrapper (`src/transcribe.cpp`) or be nothrow by construction. Device and + wrapper (`src/transcribe-api-guard.h`) or be nothrow by construction. Device and registry queries are not pure reads; guard them. - Entry points with ownership out-params enforce "non-OK => `*out == NULL`, nothing leaked" in their forwarders on every error return. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5b0931f71..532ad8441 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(transcribe transcribe.cpp + transcribe-asr.cpp transcribe-loader.cpp transcribe-arch.cpp transcribe-meta.cpp diff --git a/src/transcribe-abi.h b/src/transcribe-abi.h index dc792feb1..05d38cb39 100644 --- a/src/transcribe-abi.h +++ b/src/transcribe-abi.h @@ -1,9 +1,9 @@ // transcribe-abi.h - internal helpers for the size-aware public ABI. // -// Shared by the central dispatcher (transcribe.cpp) and per-family public -// accessors (e.g. arch/whisper/public.cpp) so the struct_size validation -// and copy-out truncation logic lives in exactly one place. Not part of -// the public API. +// Shared by the public dispatchers (transcribe.cpp, transcribe-asr.cpp) and +// per-family public accessors (e.g. arch/whisper/public.cpp) so the raw enum +// reads, struct_size validation and copy-out truncation logic live in +// exactly one place. Not part of the public API. #pragma once @@ -14,6 +14,23 @@ namespace transcribe { +// Raw enum reads at the public ABI boundary +// +// C callers can store ANY int in an enum-typed ABI field; in C++ loading an +// out-of-range value through an enum-typed lvalue is UB (UBSan traps it). So +// every public entry point reads caller-supplied enum values as raw bytes +// first, validates the raw int, and only then converts to the enum type. All +// public enums are int-sized; the static_assert below pins one. + +inline int enum_field_raw(const void * field) { + int raw; + std::memcpy(&raw, field, sizeof(raw)); + return raw; +} + +static_assert(sizeof(transcribe_backend_request) == sizeof(int), + "public enums must be int-sized for raw boundary reads"); + // Strict size check for every caller-owned struct (inputs AND outputs): // struct_size smaller than the prefix the library relies on is rejected. // `0` is invalid — defaults are reached by passing NULL where the entry diff --git a/src/transcribe-api-guard.h b/src/transcribe-api-guard.h new file mode 100644 index 000000000..27cd2e4d1 --- /dev/null +++ b/src/transcribe-api-guard.h @@ -0,0 +1,63 @@ +// transcribe-api-guard.h - exception guards for public C ABI entry points. +// +// Shared by every public dispatcher (transcribe.cpp, transcribe-asr.cpp) so +// the mapping from C++ exceptions to status codes lives in one place. Not +// part of the public API. + +#pragma once + +#include "transcribe-log.h" +#include "transcribe.h" + +#include +#include + +// Public C ABI entry points must not leak C++ exceptions. The guarded +// paths below map bad_alloc to OOM, other exceptions to BACKEND, and make +// teardown best-effort/no-throw. MSVC builds use /EHs so exceptions thrown +// by ggml C-ABI frames remain catchable here. + +namespace transcribe { + +template transcribe_status api_guard_status(const char * fn_name, Fn && body) { + try { + return body(); + } catch (const std::bad_alloc &) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: out of memory", fn_name); + return TRANSCRIBE_ERR_OOM; + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); + return TRANSCRIBE_ERR_BACKEND; + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); + return TRANSCRIBE_ERR_BACKEND; + } +} + +// Value-returning variant: `on_throw` is the entry point's documented +// hard-failure value (0 devices, false, INT_MIN, ...). +template T api_guard_value(const char * fn_name, T on_throw, Fn && body) { + try { + return body(); + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); + return on_throw; + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); + return on_throw; + } +} + +// Teardown variant: log and swallow. Frees must never fail. +template void api_guard_void(const char * fn_name, Fn && body) { + try { + body(); + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught backend exception during teardown: %s", fn_name, + e.what()); + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught unknown backend exception during teardown", fn_name); + } +} + +} // namespace transcribe diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp new file mode 100644 index 000000000..f1ac4cc16 --- /dev/null +++ b/src/transcribe-asr.cpp @@ -0,0 +1,2465 @@ +// transcribe-asr.cpp - ASR public C API entry points + central dispatch. +// +// What lives here: +// - Session lifecycle (transcribe_session_init / _free, transcribe_open / +// _close) and the run, batch and streaming dispatchers that validate +// params and hand off to the per-family Arch hooks. +// - The stream text policy (committed / tentative text view). +// - ASR capabilities, session limits, tokenize, timings, and the result +// and batch-result accessors. +// - The init functions for the ASR params / output structs. +// +// What does NOT live here: library-level entry points (status, version, +// logging, backends, devices, model load / free and model queries; +// transcribe.cpp) and per-family load / context / run code +// (src/arch//...). The accessors read the base structs in +// transcribe-model.h / transcribe-session.h directly. + +#include "transcribe-abi.h" +#include "transcribe-api-guard.h" +#include "transcribe-arch.h" +#include "transcribe-batch-util.h" +#include "transcribe-log.h" +#include "transcribe-model.h" +#include "transcribe-prompting.h" +#include "transcribe-session.h" +#include "transcribe-tokenizer.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using transcribe::api_guard_status; +using transcribe::api_guard_value; +using transcribe::api_guard_void; +using transcribe::enum_field_raw; + +namespace { + +// Ordered rank for timestamp granularities (NONE < SEGMENT < WORD < TOKEN), +// used to compare a request against a family's advertised maximum. AUTO and +// any unknown value get rank -1. +int timestamp_rank(transcribe_timestamp_kind k) { + switch (k) { + case TRANSCRIBE_TIMESTAMPS_NONE: + return 0; + case TRANSCRIBE_TIMESTAMPS_SEGMENT: + return 1; + case TRANSCRIBE_TIMESTAMPS_WORD: + return 2; + case TRANSCRIBE_TIMESTAMPS_TOKEN: + return 3; + case TRANSCRIBE_TIMESTAMPS_AUTO: + return -1; + } + return -1; +} + +// Shared run-params validation used by transcribe_run and +// transcribe_stream_begin. Both call sites have already validated +// that session, session->model, and params are non-null. The translate-task +// rejection differs between the two (run mirrors supports_translate, +// streaming-begin rejects unconditionally in v1), so each caller +// applies its own translate check before reaching this helper. +transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params); + +transcribe_status validate_run_params_common(const transcribe_session * session, const transcribe_run_params * params) { + // Raw-validate every enum field before its first enum-typed load (see + // enum_field_raw). Once a field passes here, downstream typed reads — + // including the per-family handlers' — are defined. + switch (enum_field_raw(¶ms->task)) { + case TRANSCRIBE_TASK_TRANSCRIBE: + case TRANSCRIBE_TASK_TRANSLATE: + case TRANSCRIBE_TASK_INSTRUCT: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->timestamps)) { + case TRANSCRIBE_TIMESTAMPS_NONE: + case TRANSCRIBE_TIMESTAMPS_AUTO: + case TRANSCRIBE_TIMESTAMPS_SEGMENT: + case TRANSCRIBE_TIMESTAMPS_WORD: + case TRANSCRIBE_TIMESTAMPS_TOKEN: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->pnc)) { + case TRANSCRIBE_PNC_MODE_DEFAULT: + case TRANSCRIBE_PNC_MODE_OFF: + case TRANSCRIBE_PNC_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->itn)) { + case TRANSCRIBE_ITN_MODE_DEFAULT: + case TRANSCRIBE_ITN_MODE_OFF: + case TRANSCRIBE_ITN_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->diarize)) { + case TRANSCRIBE_DIARIZE_MODE_DEFAULT: + case TRANSCRIBE_DIARIZE_MODE_OFF: + case TRANSCRIBE_DIARIZE_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { + const int req_rank = timestamp_rank(params->timestamps); + const int max_rank = timestamp_rank(session->model->caps.max_timestamp_kind); + if (req_rank > max_rank) { + return TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS; + } + } + if (params->language != nullptr && session->model->caps.n_languages > 0 && + session->model->caps.languages != nullptr) { + bool found = false; + for (int i = 0; i < session->model->caps.n_languages; ++i) { + const char * entry = session->model->caps.languages[i]; + if (entry != nullptr && std::strcmp(entry, params->language) == 0) { + found = true; + break; + } + } + if (!found) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + } + // Translation-target gate: for a TRANSLATE request naming a target + // language, reject up front when the model advertises a target set + // (n > 0) that does not include it. Mirrors the source-language check + // above and returns the same code. An empty/unadvertised set (old + // GGUFs, ASR-only models) makes this inert; family-level target/pair + // checks (e.g. canary's pivot pairs) still apply on top. + if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->target_language != nullptr && + session->model->caps.n_translate_target_languages > 0 && + session->model->caps.translate_target_languages != nullptr) { + bool found = false; + for (int i = 0; i < session->model->caps.n_translate_target_languages; ++i) { + const char * entry = session->model->caps.translate_target_languages[i]; + if (entry != nullptr && std::strcmp(entry, params->target_language) == 0) { + found = true; + break; + } + } + if (!found) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + } + // Translation-pair gate: when a GGUF advertises exact src>dst pairs and + // the caller supplied both sides, reject unsupported directions before + // family dispatch. A missing source hint keeps this inert because most + // families do not perform generic language detection here. + if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->language != nullptr && + params->target_language != nullptr && + !session->model->allows_translation_pair(params->language, params->target_language)) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + return validate_prompting(session->model, params); +} + +// Shape and hard-gate checks for the generic prompting fields, on a +// normalized view. Soft inputs a model ignores are removed later by +// prepare_prompting. +transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params) { + auto reject = [](transcribe_status st, const char * why) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_run: %s", why); + return st; + }; + if (params->n_vocabulary < 0 || (params->n_vocabulary > 0 && params->vocabulary == nullptr)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary is NULL or n_vocabulary is negative"); + } + for (int32_t i = 0; i < params->n_vocabulary; ++i) { + if (params->vocabulary[i] == nullptr) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary has a NULL entry"); + } + } + const bool has_prefix = transcribe::prompting::has_text(params->prefix); + if (params->task == TRANSCRIBE_TASK_INSTRUCT) { + if (!transcribe::has_feature(model, TRANSCRIBE_FEATURE_INSTRUCT)) { + return reject(TRANSCRIBE_ERR_UNSUPPORTED_TASK, + "this model does not support TRANSCRIBE_TASK_INSTRUCT (TRANSCRIBE_FEATURE_INSTRUCT)"); + } + // Output is free text: no target language and no alignment. A prefix + // as answer prefill is untested on every INSTRUCT family. + if (!transcribe::prompting::has_text(params->prompt)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT requires a non-empty prompt"); + } + if (params->target_language != nullptr) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT does not take a target_language"); + } + if (params->timestamps != TRANSCRIBE_TIMESTAMPS_NONE && params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT supports timestamps NONE or AUTO only"); + } + if (has_prefix) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, + "a transcript prefix is not supported with TRANSCRIBE_TASK_INSTRUCT"); + } + } + // Ignoring a prefix would make the output repeat the prefix's words and + // silently break callers that stitch text together, so it is a hard gate. + if (has_prefix && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, + "this model does not support a transcript prefix (TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)"); + } + return TRANSCRIBE_OK; +} + +// Rejects control-token literals in the prompting text before the result +// snapshot is cleared. Runs after strip_ignored_prompting, so ignored inputs +// are not rejected. +transcribe_status check_prompting_text(const transcribe_model * model, const transcribe_run_params * params) { + const transcribe::Tokenizer * tok = model->tokenizer(); + if (tok == nullptr) { + return TRANSCRIBE_OK; + } + for (int32_t i = 0; i < params->n_vocabulary; ++i) { + if (const transcribe_status st = + transcribe::prompting::check_plain_text(*tok, params->vocabulary[i], "vocabulary"); + st != TRANSCRIBE_OK) { + return st; + } + } + if (params->prompt != nullptr) { + if (const transcribe_status st = transcribe::prompting::check_plain_text(*tok, params->prompt, "prompt"); + st != TRANSCRIBE_OK) { + return st; + } + } + if (params->prefix != nullptr) { + return transcribe::prompting::check_plain_text(*tok, params->prefix, "prefix"); + } + return TRANSCRIBE_OK; +} + +// Full-size copy of a caller's run params: defaults first, then only the +// prefix the caller's struct_size covers, so every trailing field is +// readable (NULL/0 for an older caller). struct_size is preserved so +// has_field() gating still sees the caller's true layout. Idempotent. +void normalize_run_params(const transcribe_run_params * in, transcribe_run_params * out) { + transcribe_run_params_init(out); + std::memcpy(out, in, static_cast(std::min(in->struct_size, sizeof(*out)))); +} + +// Warn about, then remove, the soft prompting inputs this model ignores, so +// a family only ever sees inputs it should act on. Idempotent: the batch +// serial fallback re-enters run_one_inner per utterance. +void strip_ignored_prompting(const transcribe_model * model, transcribe_run_params * params) { + const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; + const bool instruct = params->task == TRANSCRIBE_TASK_INSTRUCT; + if (params->n_vocabulary > 0 && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_VOCABULARY)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "transcribe_run: model '%s' does not support vocabulary; ignoring %d term(s). Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_VOCABULARY) to pre-check.", + arch_name, params->n_vocabulary); + params->vocabulary = nullptr; + params->n_vocabulary = 0; + } + if (!instruct && transcribe::prompting::has_text(params->prompt) && + !transcribe::has_feature(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "transcribe_run: model '%s' has no context-prompt slot; ignoring prompt. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT) to pre-check, or " + "TRANSCRIBE_TASK_INSTRUCT on models with TRANSCRIBE_FEATURE_INSTRUCT.", + arch_name); + params->prompt = nullptr; + } + if (!transcribe::prompting::has_text(params->prompt)) { + params->prompt = nullptr; + } + if (!transcribe::prompting::has_text(params->prefix)) { + params->prefix = nullptr; + } +} + +// The pre-clear prompting step every entry point shares, on a validated +// normalized view. +transcribe_status prepare_prompting(const transcribe_model * model, transcribe_run_params * params) { + strip_ignored_prompting(model, params); + return check_prompting_text(model, params); +} + +} // namespace + +// Params init functions +// +// See transcribe.cpp for the zero-fill + struct_size convention. + +extern "C" void transcribe_session_params_init(struct transcribe_session_params * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + p->spec_k_drafts = -1; // family default + // Default to AUTO (richest output compatible with the model and selected + // run tasks, resolved per-family) rather than the memset NONE. + p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; +} + +extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +// Output struct init functions + +extern "C" void transcribe_capabilities_init(struct transcribe_capabilities * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_session_limits_init(struct transcribe_session_limits * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_stream_update_init(struct transcribe_stream_update * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_stream_text_init(struct transcribe_stream_text * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_segment_init(struct transcribe_segment * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_word_init(struct transcribe_word * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +extern "C" void transcribe_token_init(struct transcribe_token * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); +} + +// Whisper telemetry + run-extension init and chunk-trace accessors live in +// arch/whisper/public.cpp so this dispatcher stays family-agnostic. Whisper +// run knobs are in transcribe_whisper_run_ext (via run_params::family); +// sensevoice/funasr_nano use_itn and canary pnc use the generic run_params +// itn/pnc enums. + +namespace { + +// Minimum struct_size accepted on each caller-owned input/output struct. +// Sized to the prefix the library currently relies on: any field the +// library writes on a given call path must lie inside this prefix. New +// fields appended at the end of the public struct without growing the +// library-side prefix do NOT raise this value. +#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) + +constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); +// run_params is the one 0.2.0 exception to the append-only rule: `diarize` +// was inserted mid-struct, shifting every field from `language` on by 8 +// bytes. A 0.1-layout caller's sizeof (64) equals FIELD_END(family) in the +// NEW layout, so a `family`-prefix minimum would admit stale callers and +// read their pointers as enums. Require through spec_k_drafts so every +// pre-0.2 caller that bypasses the SONAME/abihash checks (dlopen by path, +// stale static link, hand-rolled FFI) gets BAD_STRUCT_SIZE instead. +constexpr size_t k_min_run_params_size = TRANSCRIBE_FIELD_END(transcribe_run_params, spec_k_drafts); +constexpr size_t k_min_stream_params_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, family); +constexpr size_t k_stream_params_commit_policy_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, commit_policy); +constexpr size_t k_stream_params_agreement_n_size = + TRANSCRIBE_FIELD_END(transcribe_stream_params, stable_prefix_agreement_n); +constexpr size_t k_min_stream_update_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, buffered_ms); +constexpr size_t k_stream_update_committed_changed_size = + TRANSCRIBE_FIELD_END(transcribe_stream_update, committed_changed); +constexpr size_t k_stream_update_tentative_changed_size = + TRANSCRIBE_FIELD_END(transcribe_stream_update, tentative_changed); +constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); +constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); +constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); +constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); +constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); +constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); +constexpr size_t k_min_speaker_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); +constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); +// k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with +// the chunk-trace accessor that uses it. + +#undef TRANSCRIBE_FIELD_END + +// Size-aware ABI helpers (check_struct_size / check_input_struct_size / +// copy_out_prefix) live in transcribe-abi.h so per-family public +// accessors (arch/whisper/public.cpp) share one definition. Pull them +// into this TU's unqualified scope so existing call sites are unchanged. +using transcribe::check_input_struct_size; +using transcribe::check_struct_size; +using transcribe::copy_out_prefix; + +static bool has_field(uint64_t struct_size, size_t field_end) { + return struct_size >= static_cast(field_end); +} + +// Takes the RAW integer, not the enum: a C caller can store any int in the +// struct's enum-typed field, and in C++ loading an out-of-range value through +// the enum lvalue is UB (UBSan trips on it). Callers memcpy the bytes into an +// int and validate before the first enum-typed load. +static bool valid_stream_commit_policy(int policy) { + switch (policy) { + case TRANSCRIBE_STREAM_COMMIT_AUTO: + case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: + case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: + return true; + } + return false; +} + +} // namespace + +namespace { + +enum class StreamStablePrefixImpl { + GenericTextAgreement, + FamilyTokenAgreement, + FamilyNativeCommit, +}; + +static StreamStablePrefixImpl stream_stable_prefix_impl_for_arch(const transcribe::Arch * arch) { + // Single source of truth for the stable-prefix implementation used by + // AUTO and explicit STABLE_PREFIX. Add new streaming families here + // deliberately; otherwise they use the generic text-agreement fallback. + struct Entry { + const char * arch_name; + StreamStablePrefixImpl impl; + }; + + static constexpr Entry k_defaults[] = { + // Parakeet publishes native committed chunks; agreement_n does not + // add useful evidence on top of the family-provided boundary. + { "parakeet", StreamStablePrefixImpl::FamilyNativeCommit }, + // Moonshine re-decodes the full prefix and its family boundary is + // token-id agreement; the family applies stable_prefix_agreement_n. + { "moonshine_streaming", StreamStablePrefixImpl::FamilyTokenAgreement }, + }; + + const char * name = arch != nullptr && arch->name != nullptr ? arch->name : ""; + for (const Entry & entry : k_defaults) { + if (std::strcmp(name, entry.arch_name) == 0) { + return entry.impl; + } + } + return StreamStablePrefixImpl::GenericTextAgreement; +} + +static size_t utf8_floor_boundary(const std::string & s, size_t pos) { + if (pos >= s.size()) { + return s.size(); + } + while (pos > 0 && (static_cast(s[pos]) & 0xC0u) == 0x80u) { + --pos; + } + return pos; +} + +static size_t common_prefix_bytes(const std::string & a, const std::string & b) { + const size_t n = std::min(a.size(), b.size()); + size_t i = 0; + while (i < n && a[i] == b[i]) { + ++i; + } + return utf8_floor_boundary(a, i); +} + +static bool starts_with_bytes(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && s.compare(0, prefix.size(), prefix) == 0; +} + +static size_t token_prefix_raw_bytes(const transcribe_session * session, int n_tokens) { + if (session == nullptr || n_tokens <= 0 || session->tokens.empty()) { + return 0; + } + const int capped = std::min(n_tokens, static_cast(session->tokens.size())); + std::string prefix; + prefix.reserve(session->full_text.size()); + for (int i = 0; i < capped; ++i) { + prefix += session->tokens[static_cast(i)].text; + } + if (starts_with_bytes(session->full_text, prefix)) { + return prefix.size(); + } + if (!prefix.empty() && prefix.front() == ' ' && (session->full_text.empty() || session->full_text.front() != ' ')) { + const std::string normalized_prefix = prefix.substr(1); + if (starts_with_bytes(session->full_text, normalized_prefix)) { + return normalized_prefix.size(); + } + return common_prefix_bytes(session->full_text, normalized_prefix); + } + return common_prefix_bytes(session->full_text, prefix); +} + +static size_t family_candidate_raw_prefix_bytes(const transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + if (session->n_committed_tokens > 0) { + return token_prefix_raw_bytes(session, session->n_committed_tokens); + } + if (session->n_committed_words > 0 && session->n_committed_words >= static_cast(session->words.size())) { + return session->full_text.size(); + } + if (session->n_committed_segments > 0 && + session->n_committed_segments >= static_cast(session->segments.size())) { + return session->full_text.size(); + } + return 0; +} + +static size_t generic_text_stable_prefix_candidate_raw_bytes(transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + // Library default when the caller leaves stable_prefix_agreement_n at + // 0; documented as "currently 3" on transcribe_stream_params. + static constexpr uint32_t k_default_stable_prefix_agreement_n = 3; + uint32_t agreement_n = session->stream_stable_prefix_agreement_n; + if (agreement_n == 0) { + agreement_n = k_default_stable_prefix_agreement_n; + } + if (agreement_n <= 1) { + return session->full_text.size(); + } + + session->stream_raw_history.push_back(session->full_text); + while (session->stream_raw_history.size() > agreement_n) { + session->stream_raw_history.pop_front(); + } + if (session->stream_raw_history.size() < agreement_n) { + return 0; + } + + size_t prefix_n = session->stream_raw_history.front().size(); + for (const auto & text : session->stream_raw_history) { + prefix_n = std::min(prefix_n, common_prefix_bytes(session->stream_raw_history.front(), text)); + } + return utf8_floor_boundary(session->full_text, prefix_n); +} + +static size_t selected_stable_prefix_candidate_raw_bytes(transcribe_session * session) { + if (session == nullptr) { + return 0; + } + const transcribe::Arch * arch = session->model != nullptr ? session->model->arch : nullptr; + switch (stream_stable_prefix_impl_for_arch(arch)) { + case StreamStablePrefixImpl::FamilyTokenAgreement: + case StreamStablePrefixImpl::FamilyNativeCommit: + return family_candidate_raw_prefix_bytes(session); + case StreamStablePrefixImpl::GenericTextAgreement: + return generic_text_stable_prefix_candidate_raw_bytes(session); + } + return 0; +} + +static bool append_committed_raw_prefix(transcribe_session * session, size_t candidate_bytes) { + if (session == nullptr || !session->has_result) { + return false; + } + const std::string & raw = session->full_text; + const size_t old_boundary = utf8_floor_boundary( + raw, std::min(static_cast(session->stream_raw_tentative_start_bytes), raw.size())); + const size_t candidate = utf8_floor_boundary(raw, std::min(candidate_bytes, raw.size())); + if (candidate <= old_boundary) { + return false; + } + if (old_boundary != session->stream_committed_text.size()) { + return false; + } + if (raw.compare(0, old_boundary, session->stream_committed_text.data(), old_boundary) != 0) { + return false; + } + session->stream_committed_text.append(raw.data() + old_boundary, candidate - old_boundary); + session->stream_raw_tentative_start_bytes = static_cast(candidate); + return true; +} + +static bool finalize_committed_text(transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return false; + } + if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE || + session->stream_committed_text.empty()) { + const bool changed = session->stream_committed_text != session->full_text; + session->stream_committed_text = session->full_text; + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return changed; + } + if (!starts_with_bytes(session->full_text, session->stream_committed_text)) { + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return false; + } + const size_t old_n = session->stream_committed_text.size(); + if (session->full_text.size() > old_n) { + session->stream_committed_text.append(session->full_text.data() + old_n, session->full_text.size() - old_n); + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return true; + } + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return false; +} + +struct StreamTextDelta { + bool committed_changed = false; + bool tentative_changed = false; +}; + +static StreamTextDelta apply_stream_text_policy(transcribe_session * session, bool is_finalize) { + StreamTextDelta delta; + if (session == nullptr) { + return delta; + } + const std::string prev_committed = session->stream_committed_text; + const std::string prev_tentative = session->stream_tentative_text; + + if (is_finalize) { + delta.committed_changed = finalize_committed_text(session); + session->stream_tentative_text.clear(); + } else if (session->has_result) { + size_t candidate = 0; + switch (session->stream_commit_policy) { + case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: + candidate = 0; + break; + case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: + case TRANSCRIBE_STREAM_COMMIT_AUTO: + default: + candidate = selected_stable_prefix_candidate_raw_bytes(session); + break; + } + delta.committed_changed = append_committed_raw_prefix(session, candidate); + + const size_t boundary = utf8_floor_boundary( + session->full_text, std::min(static_cast(session->stream_raw_tentative_start_bytes), + session->full_text.size())); + session->stream_tentative_text.assign(session->full_text.data() + boundary, + session->full_text.size() - boundary); + } else { + session->stream_tentative_text.clear(); + } + + delta.committed_changed = delta.committed_changed || session->stream_committed_text != prev_committed; + delta.tentative_changed = session->stream_tentative_text != prev_tentative; + return delta; +} + +static void publish_stream_update_tail(transcribe_stream_update * update, + bool committed_changed, + bool tentative_changed) { + if (update == nullptr) { + return; + } + if (has_field(update->struct_size, k_stream_update_committed_changed_size)) { + update->committed_changed = committed_changed; + } + if (has_field(update->struct_size, k_stream_update_tentative_changed_size)) { + update->tentative_changed = tentative_changed; + } +} + +// Compute the observable-change verdict, advance the revision counter +// when the result moved, and publish revision / result_changed / +// committed_changed / tentative_changed onto the caller's update. Shared +// verbatim by feed and finalize so the result_changed semantics live in +// exactly one place; the path-specific work (ON_FINALIZE counter zeroing +// and audio_committed_ms on feed, is_final and the lifecycle transition +// on finalize) stays in the callers. +static void publish_observable_delta(transcribe_session * session, + transcribe_stream_update * update, + int32_t prev_revision, + const std::string & prev_full_text, + bool prev_has_result, + const StreamTextDelta & text_delta) { + const bool raw_changed = session->has_result != prev_has_result || session->full_text != prev_full_text; + const bool observable_changed = raw_changed || text_delta.committed_changed || text_delta.tentative_changed || + (update != nullptr && update->result_changed); + if (observable_changed && session->stream_revision == prev_revision) { + session->stream_revision += 1; + } + if (update != nullptr) { + update->result_changed = + update->result_changed || observable_changed || session->stream_revision != prev_revision; + update->revision = session->stream_revision; + publish_stream_update_tail(update, text_delta.committed_changed, text_delta.tentative_changed); + } +} + +// Advisory pnc/itn warning. Emits a WARN when a non-DEFAULT request hits a +// model that does not support runtime control of that axis, then returns so +// the dispatcher proceeds with best-effort semantics. The reserved +// TRANSCRIBE_ERR_UNSUPPORTED_PNC / _ITN codes are NOT returned today +// (placeholders for a future opt-in strict mode). The message includes the +// arch + variant strings to pinpoint which model dropped the request. +void warn_unsupported_advisory(const struct transcribe_model * model, const struct transcribe_run_params * rp) { + if (model == nullptr || rp == nullptr) { + return; + } + const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; + const char * variant = model->variant.c_str(); + if (variant == nullptr || variant[0] == '\0') { + variant = "(unknown)"; + } + + // Defense in depth: keep raw reads here even though every dispatcher + // validates before warning. A future call-site reorder must not turn a + // malformed C enum into a typed C++ load. + const int pnc_raw = enum_field_raw(&rp->pnc); + const int itn_raw = enum_field_raw(&rp->itn); + const int diarize_raw = enum_field_raw(&rp->diarize); + + char buf[512]; + if (pnc_raw != TRANSCRIBE_PNC_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_PNC)) { + const char * req = (pnc_raw == TRANSCRIBE_PNC_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested pnc=%s but model '%s' " + "(variant '%s') does not support pnc control; output will use " + "the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_PNC) to " + "pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } + if (itn_raw != TRANSCRIBE_ITN_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_ITN)) { + const char * req = (itn_raw == TRANSCRIBE_ITN_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested itn=%s but model '%s' " + "(variant '%s') does not support itn control; output will use " + "the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_ITN) to " + "pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } + if (diarize_raw != TRANSCRIBE_DIARIZE_MODE_DEFAULT && + !transcribe::has_feature(model, TRANSCRIBE_FEATURE_DIARIZATION)) { + const char * req = (diarize_raw == TRANSCRIBE_DIARIZE_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested diarize=%s but model '%s' " + "(variant '%s') does not support diarization control; output " + "will use the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION) " + "to pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } +} + +} // namespace + +// Lifecycle + +static transcribe_status transcribe_session_init_impl(struct transcribe_model * model, + const struct transcribe_session_params * params, + struct transcribe_session ** out_session) { + if (out_session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out_session = nullptr; + if (model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // NULL params means "all defaults" (see transcribe_model_load_file). + struct transcribe_session_params params_defaults; + transcribe_session_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_context_params_size); st != TRANSCRIBE_OK) { + return st; + } + // n_threads contract from include/transcribe.h: 0 means "library + // picks a sensible default", positive means "use this many." A + // negative value is undefined input, not a documented sentinel — + // reject it here so individual family handlers don't have to + // re-check the same condition. + if (params->n_threads < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Raw-validate kv_type before the families' first enum-typed load of it + // (see enum_field_raw): an unknown value from a C caller is a clean + // error here, never UB downstream. + switch (enum_field_raw(¶ms->kv_type)) { + case TRANSCRIBE_KV_TYPE_AUTO: + case TRANSCRIBE_KV_TYPE_F32: + case TRANSCRIBE_KV_TYPE_F16: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + // n_ctx is a trailing field (appended after kv_type), so only read it + // when the caller's struct_size actually covers it; an older caller's + // smaller struct leaves it at the default 0 = "model max". A negative + // value is undefined input, not a documented sentinel. The family that + // honors n_ctx clamps a too-large value down to the model maximum, so + // only the negative case is rejected here. See include/transcribe.h. + if (has_field(params->struct_size, offsetof(struct transcribe_session_params, n_ctx) + sizeof(params->n_ctx)) && + params->n_ctx < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // The model carries its own arch dispatch pointer. A model that + // came from a load() call always has arch set; the null check is + // defensive against a hypothetical malformed model object. + if (model->arch == nullptr || model->arch->init_context == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + return model->arch->init_context(model, params, out_session); +} + +static void transcribe_session_free_impl(struct transcribe_session * session) { + // Free the session and, if it owns its model (transcribe_open path), + // free the model too. The owns_model flag is set only by + // transcribe_open; sessions created via the two-step + // transcribe_session_init borrow the model and leave it alone here. + // + // Capture before the session is destroyed; reading session->model + // after delete would be use-after-free. + if (session == nullptr) { + return; + } + struct transcribe_model * owned = session->owns_model ? session->model : nullptr; + delete session; + transcribe_model_free(owned); // NULL is a no-op +} + +// Convenience: open / close / get_model +// +// transcribe_open bundles load + session_init and hands back a session that +// owns its model (owns_model = true). transcribe_close is a thin alias for +// transcribe_session_free; both honor owns_model and free the owned model +// after the session. + +static transcribe_status transcribe_open_impl(const char * path, + const struct transcribe_model_load_params * load_params, + const struct transcribe_session_params * session_params, + struct transcribe_session ** out_session) { + if (out_session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out_session = nullptr; + + // NULL params are forwarded as-is: the callees already treat NULL as + // "all defaults", so no substitution is needed here. + struct transcribe_model * model = nullptr; + if (const auto st = transcribe_model_load_file(path, load_params, &model); st != TRANSCRIBE_OK) { + return st; + } + + struct transcribe_session * session = nullptr; + if (const auto st = transcribe_session_init(model, session_params, &session); st != TRANSCRIBE_OK) { + // Load succeeded but session init failed: own the cleanup so the + // caller sees the same all-or-nothing contract as the two-step + // API. No partial state leaks. + transcribe_model_free(model); + return st; + } + + // The convenience session owns the model it loaded; either + // transcribe_session_free or transcribe_close (its alias) will free + // both. This is the only place owns_model is set. + session->owns_model = true; + *out_session = session; + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_close(struct transcribe_session * session) { + // Alias for transcribe_session_free, kept for source compatibility + // with the prior split-API shape. Both honor owns_model. + transcribe_session_free(session); +} + +extern "C" const struct transcribe_model * transcribe_get_model(const struct transcribe_session * session) { + if (session == nullptr) { + return nullptr; + } + return session->model; +} + +extern "C" void transcribe_set_abort_callback(struct transcribe_session * session, + transcribe_abort_callback cb, + void * user_data) { + if (session == nullptr) { + return; + } + session->abort_cb = cb; + session->abort_userdata = user_data; +} + +extern "C" bool transcribe_was_aborted(const struct transcribe_session * session) { + if (session == nullptr) { + return false; + } + return session->was_aborted; +} + +extern "C" bool transcribe_was_truncated(const struct transcribe_session * session) { + if (session == nullptr) { + return false; + } + return session->was_truncated; +} + +// Streaming dispatcher +// +// State transitions are managed entirely here; hooks see ACTIVE on entry to +// begin/feed/finalize and never observe transitions. Hooks own the +// per-utterance result data (tokens/words/segments, committed counts, audio +// cursors, stream_revision); the dispatcher owns lifecycle state, +// last_status, and the public committed/tentative text view. clear_result() +// wipes the snapshot but never touches stream_state. + +static transcribe_status transcribe_stream_begin_impl(struct transcribe_session * session, + const struct transcribe_run_params * run_params, + const struct transcribe_stream_params * stream_params) { + if (session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // NULL run/stream params each mean "all defaults". + struct transcribe_run_params run_params_defaults; + transcribe_run_params_init(&run_params_defaults); + struct transcribe_stream_params stream_params_defaults; + transcribe_stream_params_init(&stream_params_defaults); + if (run_params == nullptr) { + run_params = &run_params_defaults; + } + if (stream_params == nullptr) { + stream_params = &stream_params_defaults; + } + if (const auto st = check_input_struct_size(run_params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + // Full-size view (see normalize_run_params); the family hook gets a + // further copy whose strings the library owns, built below. + struct transcribe_run_params run_params_view; + normalize_run_params(run_params, &run_params_view); + run_params = &run_params_view; + if (const auto st = check_input_struct_size(stream_params->struct_size, k_min_stream_params_size); + st != TRANSCRIBE_OK) { + return st; + } + // Read the caller's commit_policy as raw bytes and validate BEFORE the + // first load through the enum-typed lvalue: C callers can legally store + // any int there, and an out-of-range enum load is UB in C++. + int commit_policy_raw = TRANSCRIBE_STREAM_COMMIT_AUTO; + uint32_t stable_prefix_agreement_n = 0; + if (has_field(stream_params->struct_size, k_stream_params_commit_policy_size)) { + static_assert(sizeof(stream_params->commit_policy) == sizeof(int), + "commit_policy must be int-sized for the raw read"); + std::memcpy(&commit_policy_raw, &stream_params->commit_policy, sizeof(commit_policy_raw)); + } + if (!valid_stream_commit_policy(commit_policy_raw)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const transcribe_stream_commit_policy commit_policy = + static_cast(commit_policy_raw); + if (has_field(stream_params->struct_size, k_stream_params_agreement_n_size)) { + stable_prefix_agreement_n = stream_params->stable_prefix_agreement_n; + if (stable_prefix_agreement_n > 32) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + // Caller must finalize or reset before starting a new stream. + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->model == nullptr || session->model->arch == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + // Capability gate: the model must advertise streaming AND the + // family must wire the full required hook set. begin / feed / + // finalize come as a triple — a partially-wired family that + // accepts begin would otherwise let the caller enter ACTIVE and + // then get stuck on NOT_IMPLEMENTED at the first feed. stream_reset + // remains optional; the dispatcher's clear_result + state wipe is + // sufficient when a family does not need to release per-utterance + // buffers explicitly. + if (!session->model->caps.supports_streaming || session->model->arch->stream_begin == nullptr || + session->model->arch->stream_feed == nullptr || session->model->arch->stream_finalize == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + if (const transcribe_status st = validate_run_params_common(session, run_params); st != TRANSCRIBE_OK) { + return st; + } + + // v1 rejects TRANSLATE unconditionally for streaming. A future + // family that supports streaming translate would loosen this in + // its stream_begin hook, but the central dispatcher refuses + // upfront so partially-wired callers fail fast. + if (run_params->task == TRANSCRIBE_TASK_TRANSLATE || run_params->task == TRANSCRIBE_TASK_INSTRUCT) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + // A prefix is forced decoder text for one utterance's opening; a stream + // has no fixed opening to force it onto. + if (transcribe::prompting::has_text(run_params->prefix)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_stream_begin: a transcript prefix is not accepted " + "for streaming"); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + if (stream_params->family != nullptr) { + if (stream_params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_STREAM, + stream_params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + + // Advisory warn for pnc/itn requests against models that don't + // expose the corresponding runtime toggle. Emitted before + // clear_result so the pre-hook "snapshot preserved on rejection" + // contract is undisturbed. + warn_unsupported_advisory(session->model, run_params); + if (const transcribe_status st = prepare_prompting(session->model, &run_params_view); st != TRANSCRIBE_OK) { + return st; + } + + // Optional family preflight: validates extension field values + // (e.g. parakeet's (L, C, R) menu) without mutating state. On + // non-OK return the previous snapshot and lifecycle are preserved + // so a caller-side typo does not destroy the prior utterance. + if (session->model->arch->stream_validate != nullptr) { + if (const transcribe_status st = session->model->arch->stream_validate(session, run_params, stream_params); + st != TRANSCRIBE_OK) { + return st; + } + } + + // All checks pass — clear the previous snapshot and hand off. + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + session->stream_state = TRANSCRIBE_STREAM_ACTIVE; + session->stream_commit_policy = commit_policy; + session->stream_stable_prefix_agreement_n = stable_prefix_agreement_n; + + // Hand the family hook a params view whose pointers the LIBRARY owns. + // Families may capture `*run_params` for the stream's lifetime + // (parakeet re-reads .language on every feed), and the public contract + // lets the caller free every params pointer once begin returns — so the + // strings are copied into session storage here and the view repointed + // at it. `family` (the run-slot extension) is nulled in the view: no + // family reads it on the stream path today, and per the ext copy-out + // contract a retained pointer to it would dangle; a future family that + // wants a run-slot ext at stream begin must plumb it deliberately. + session->stream_language_owned = run_params->language != nullptr ? run_params->language : ""; + session->stream_target_language_owned = run_params->target_language != nullptr ? run_params->target_language : ""; + // Copied from the normalized view (never the caller's struct, whose + // allocation may end before the trailing fields); the view keeps the + // caller's struct_size, so has_field() gating still sees its layout. + struct transcribe_run_params run_params_owned = run_params_view; + run_params_owned.language = run_params->language != nullptr ? session->stream_language_owned.c_str() : nullptr; + run_params_owned.target_language = + run_params->target_language != nullptr ? session->stream_target_language_owned.c_str() : nullptr; + run_params_owned.family = nullptr; + // Generic prompting strings, same ownership rule. Only non-empty terms + // are kept, so the view's count matches the owned array. + session->stream_vocabulary_owned = transcribe::prompting::terms(run_params); + session->stream_vocabulary_ptrs.clear(); + for (const std::string & term : session->stream_vocabulary_owned) { + session->stream_vocabulary_ptrs.push_back(term.c_str()); + } + session->stream_prompt_owned = run_params->prompt != nullptr ? run_params->prompt : ""; + run_params_owned.vocabulary = + session->stream_vocabulary_ptrs.empty() ? nullptr : session->stream_vocabulary_ptrs.data(); + run_params_owned.n_vocabulary = static_cast(session->stream_vocabulary_ptrs.size()); + run_params_owned.prompt = run_params->prompt != nullptr ? session->stream_prompt_owned.c_str() : nullptr; + run_params_owned.prefix = nullptr; + + const transcribe_status st = session->model->arch->stream_begin(session, &run_params_owned, stream_params); + if (st != TRANSCRIBE_OK) { + // Family hook rejected the begin (config it does not understand, + // memory allocation failure, etc.). Roll lifecycle back to + // FAILED so the caller can read last_status; leave the cleared + // result snapshot in place. + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + } + return st; +} + +// Helper: validate the caller's stream_update struct_size and reset +// every field except struct_size to a clean state before handing the +// update buffer to the family hook. Returns OK when update is NULL +// (the update buffer is optional) or when the size passes validation. +static transcribe_status prepare_stream_update(struct transcribe_stream_update * update) { + if (update == nullptr) { + return TRANSCRIBE_OK; + } + if (const auto st = check_struct_size(update->struct_size, k_min_stream_update_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = update->struct_size; + transcribe_stream_update staged{}; + staged.struct_size = caller_size; + copy_out_prefix(update, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * session, + const float * pcm, + int n_samples, + struct transcribe_stream_update * update) { + if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { + return st; + } + if (session == nullptr || pcm == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Zero-length feed is rejected: feed exists to consume audio, + // and callers that just want to inspect state use the stream + // accessors. n_samples < 0 is plain garbage. + if (n_samples <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // begin already confirmed model/arch/hook; we re-check defensively + // so a malformed context (never happens in practice from a real + // begin) does not deref a null function pointer. + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->stream_feed == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + const int32_t prev_revision = session->stream_revision; + const std::string prev_full_text = session->full_text; + const bool prev_has_result = session->has_result; + + const transcribe_status st = session->model->arch->stream_feed(session, pcm, n_samples, update); + if (st != TRANSCRIBE_OK) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + return st; + } + const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/false); + if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { + session->n_committed_tokens = 0; + session->n_committed_words = 0; + session->n_committed_segments = 0; + } + publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); + if (update != nullptr && session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { + update->audio_committed_ms = 0; + } + return st; +} + +static transcribe_status transcribe_stream_finalize_impl(struct transcribe_session * session, + struct transcribe_stream_update * update) { + if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { + return st; + } + if (session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->model == nullptr || session->model->arch == nullptr || + session->model->arch->stream_finalize == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + const int32_t prev_revision = session->stream_revision; + const std::string prev_full_text = session->full_text; + const bool prev_has_result = session->has_result; + + const transcribe_status st = session->model->arch->stream_finalize(session, update); + if (update != nullptr) { + // Force is_final true regardless of family hook return; the + // marker describes the call site, not the result. Set after + // the hook so the family cannot accidentally clear it. + update->is_final = true; + } + if (st != TRANSCRIBE_OK) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + return st; + } + const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/true); + publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); + session->stream_state = TRANSCRIBE_STREAM_FINISHED; + return TRANSCRIBE_OK; +} + +static void transcribe_stream_reset_impl(struct transcribe_session * session) { + if (session == nullptr) { + return; + } + // Family hook releases per-utterance state and clears any buffered + // audio contents while keeping the allocations. A family without + // streaming just has no hook installed; reset becomes a pure + // dispatcher state wipe. + if (session->model != nullptr && session->model->arch != nullptr && session->model->arch->stream_reset != nullptr) { + session->model->arch->stream_reset(session); + } + session->clear_result(); + session->stream_state = TRANSCRIBE_STREAM_IDLE; + // was_aborted is per-stream; reset re-arms it the same way begin + // does so a caller that resets after an abort starts clean. + session->was_aborted = false; +} + +extern "C" enum transcribe_stream_state transcribe_stream_get_state(const struct transcribe_session * session) { + if (session == nullptr) { + return TRANSCRIBE_STREAM_IDLE; + } + return session->stream_state; +} + +extern "C" int transcribe_stream_revision(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->stream_revision; +} + +extern "C" int transcribe_stream_n_committed_segments(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_segments; +} + +extern "C" int transcribe_stream_n_committed_words(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_words; +} + +extern "C" int transcribe_stream_n_committed_tokens(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_tokens; +} + +extern "C" transcribe_status transcribe_stream_last_status(const struct transcribe_session * session) { + if (session == nullptr) { + return TRANSCRIBE_OK; + } + return session->stream_last_status; +} + +extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_session * session, + struct transcribe_stream_text * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_stream_text_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_stream_text staged{}; + staged.struct_size = caller_size; + if (session != nullptr && session->has_result) { + staged.full_text = session->full_text.c_str(); + staged.full_text_bytes = static_cast(session->full_text.size()); + staged.committed_text = session->stream_committed_text.c_str(); + staged.committed_text_bytes = static_cast(session->stream_committed_text.size()); + staged.tentative_text = session->stream_tentative_text.c_str(); + staged.tentative_text_bytes = static_cast(session->stream_tentative_text.size()); + staged.raw_tentative_start_bytes = + std::min(session->stream_raw_tentative_start_bytes, staged.full_text_bytes); + } else { + staged.full_text = ""; + staged.committed_text = ""; + staged.tentative_text = ""; + } + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +// Scope guard that calls transcribe_session::release_scratch on exit once +// armed. Both offline entry points use it so release also happens when a +// family hook throws and the api_guard unwinds the stack, including the path +// where memory pressure matters most. release_scratch is noexcept, so running +// it during unwinding is safe. +namespace { + +struct scratch_release_guard { + transcribe_session * session = nullptr; + bool armed = false; + + ~scratch_release_guard() { + if (armed && session != nullptr) { + session->release_scratch(); + } + } +}; + +} // namespace + +// Shared one-utterance run body. Does NOT touch session->batch_results, so +// the batch dispatcher can call it once per utterance inside a loop without +// erasing already-accumulated entries; the public transcribe_run wrapper +// below clears batch_results once before delegating here. Every early +// return preserves the previous result snapshot exactly as the original +// transcribe_run contract documented (see the inline comments). +// `committed` (optional) is set true once the call passes the pre-clear +// gates and commits to replacing the result. The caller uses it to decide +// whether compute scratch needs releasing. +static transcribe_status run_one_inner(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params, + bool * committed = nullptr) { + // Parameter-shape validation runs first and does not touch session + // state. A caller that passes NULL pointers or a non-positive sample + // count gets ERR_INVALID_ARG back without any visible side effect + // — including the session's result fields, which are preserved + // across a malformed call. This is the narrower half of the + // "transcribe_run replaces the previous result" contract: the + // replacement happens when the call is well-formed enough that + // the dispatcher is willing to dereference session. A call that + // isn't well-formed isn't considered to have "run". + if (session == nullptr || pcm == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // n_samples must be strictly positive, matching transcribe_stream_feed. + // There is no meaningful "transcribe zero samples" operation; a + // zero-length batch is treated as a caller error, not an empty run. + if (n_samples <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // NULL params means "all defaults" (transcribe vs translate, no + // timestamps, etc.). A well-formed default run is not a malformed + // call, so it proceeds and replaces the previous result. + struct transcribe_run_params params_defaults; + transcribe_run_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + // Everything downstream reads this full-size view, never the caller's + // struct: an older caller's allocation may end before the prompting + // fields. Strings stay caller-owned; the call is synchronous. + struct transcribe_run_params params_view; + normalize_run_params(params, ¶ms_view); + params = ¶ms_view; + // A run cannot replace an active stream's results — that would + // strand the in-flight stream's per-family state. Caller must + // finalize or reset first. FINISHED and FAILED both fall through; + // the run path below resets stream_state to IDLE. + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + + // Pre-clear family-extension SHAPE validation: a run ext with a bad + // transcribe_ext header size or a kind the model does not accept must + // NOT wipe the previous result snapshot (mirrors the + // transcribe_stream_begin contract). When model is null we skip the + // pre-clear check and let the post-clear NOT_IMPLEMENTED path handle it. + // + // This guarantee covers ext shape (size/kind) and the run-param checks + // below. A family that defers deeper value validation to run() (whisper + // does this for prompt semantics) can still reject a correctly-shaped + // ext after the snapshot is cleared; full pre-clear safety requires + // validating values in run_validate. + if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + + // Validate before advisory warning so typed enum reads are proven safe. + // A rejection preserves the previous snapshot. Skipped when model is + // null — the post-clear NOT_IMPLEMENTED path signals that directly. + if (session->model != nullptr) { + // Pre-clear run-param validation. A caller-side param bug + // (out-of-range enum, timestamp granularity finer than the + // model's max, unsupported language, or TRANSLATE against a + // model that doesn't support it) must NOT wipe the previous + // result snapshot — the same contract transcribe_stream_begin + // honors for its run params. These checks read model->caps and + // are therefore guarded by model != null; the degenerate + // model-null case falls through to the post-clear + // NOT_IMPLEMENTED path, which keeps its existing snapshot-wipe + // behavior. + if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { + return st; + } + warn_unsupported_advisory(session->model, params); + if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { + return st; + } + + // Family run-ext validation (the _RUN analogue of stream_validate), + // the final pre-clear gate. Runs AFTER the run-param checks above, + // mirroring transcribe_stream_begin's ordering (dispatcher param + // checks, then the family validate hook, then clear). The family + // alone knows its per-kind minimum struct size; whisper enforces + // that here. A non-OK return preserves the previous snapshot. A + // family with no _RUN ext leaves this hook NULL. Called regardless + // of whether params->family is set (whisper's hook treats a NULL + // family as "defaults", returning OK). + if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { + if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + // Result-replacement contract: everything past this point either + // succeeds and writes a fresh result, or fails and leaves the + // context in the documented "no result" sentinel state. Clear + // eagerly so the one remaining downstream rejection path — + // NOT_IMPLEMENTED on an incomplete arch — inherits the sentinel + // without having to remember to call clear_result() itself. + // Caller-param rejections (enum range, timestamp ceiling, + // language, TRANSLATE support) are validated ABOVE, before this + // clear, so a malformed call preserves the previous snapshot + // exactly as transcribe_stream_begin does. + // + // Per-family run() handlers call clear_result() again after + // their own front-matter checks succeed; that call is now + // redundant but idempotent, and removing it is a refactor + // deferred to a later pass. + if (committed != nullptr) { + *committed = true; + } + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + // Force stream_state to IDLE: clear_result deliberately preserves + // lifecycle state, but a well-formed transcribe_run subsumes any + // prior FINISHED/FAILED stream — after a one-shot run the context + // is no longer meaningfully in a streaming lifecycle. + session->stream_state = TRANSCRIBE_STREAM_IDLE; + + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + return session->model->arch->run(session, pcm, n_samples, params); +} + +static transcribe_status transcribe_run_impl(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params) { + // A well-formed single run has the same result view as a one-item batch: + // reset batch_results so the transcribe_batch_* accessors fall back to + // reading the scratch slot as utterance 0. + // + // Guard the deref with the SAME pre-deref conditions run_one_inner uses + // (non-null session + non-null pcm + positive n_samples). A malformed + // call must not touch the session at all — the API smoke test probes + // this with a fake (session *)0x1 and a NULL pcm / non-positive + // n_samples, expecting INVALID_ARG with no dereference. + if (session != nullptr && pcm != nullptr && n_samples > 0) { + session->batch_results.clear(); + } + // run_one_inner arms the guard at its commit point, so a pre-clear + // rejection never releases and everything after (family error, abort, + // throw) always does. + scratch_release_guard scratch_release; + scratch_release.session = session; + return run_one_inner(session, pcm, n_samples, params, &scratch_release.armed); +} + +// Batch run (offline) +// +// transcribe_run_batch validates the shared run_params ONCE, then either +// delegates to the family's batched run_batch() fast path or falls back to +// run_one_inner() per utterance. Either way the result is N entries in +// session->batch_results read back via the transcribe_batch_* accessors. + +namespace { + +// Pad batch_results out to `n` entries with explicit aborted failures. +// Called only on an aborted batch: synthesized slots carry +// TRANSCRIBE_ERR_ABORTED ("did not complete because the batch was aborted"), +// so the result-set view has one slot per input utterance +// (transcribe_batch_n_results() == n) regardless of family or path. +void pad_batch_results_aborted(transcribe_session * s, int n) { + while (s->batch_results.size() < static_cast(n)) { + transcribe_session::ResultSet rs; + rs.status = TRANSCRIBE_ERR_ABORTED; + s->batch_results.push_back(std::move(rs)); + } +} + +// Restore the scratch slot from a ResultSet so the single-result accessors +// stay coherent after a batch run (they reflect utterance 0). This is the +// inverse of transcribe_session::capture_result minus the per-run timing +// accumulators — keep the result fields in sync with that method when adding +// one (raw_text was once missed here). +void restore_scratch_from_result(transcribe_session * s, const transcribe_session::ResultSet & rs) { + s->tokens = rs.tokens; + s->words = rs.words; + s->segments = rs.segments; + s->speaker_segments = rs.speaker_segments; + s->full_text = rs.full_text; + s->raw_text = rs.raw_text; + s->detected_language = rs.detected_language; + s->result_kind = rs.result_kind; + s->has_result = rs.has_result; +} + +} // namespace + +static transcribe_status transcribe_run_batch_impl(struct transcribe_session * session, + const float * const * pcm, + const int * n_samples, + int n, + const struct transcribe_run_params * params) { + // Top-level argument shape. The arrays themselves must be present and + // n strictly positive; an individual malformed utterance (pcm[i] NULL + // or n_samples[i] <= 0) is a per-utterance failure handled in the loop, + // not a whole-batch error. As with transcribe_run, these checks do not + // mutate the session, so a malformed call preserves the prior result. + if (session == nullptr || pcm == nullptr || n_samples == nullptr || n <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + struct transcribe_run_params params_defaults; + transcribe_run_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + struct transcribe_run_params params_view; + normalize_run_params(params, ¶ms_view); + params = ¶ms_view; + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // One shared params across different audio: a transcript prefix can + // only describe one of them. + if (transcribe::prompting::has_text(params->prefix)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_run_batch: a transcript prefix is per-utterance " + "and is not accepted in a batch"); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + // Shared-param validation, ONCE, mirroring transcribe_run's pre-clear + // gates (ext shape/kind, pnc/itn advisory, enum range, timestamp + // ceiling, language, TRANSLATE support, run_validate). A rejection here + // preserves the previous result snapshot — nothing has been cleared. + if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + if (session->model != nullptr) { + if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { + return st; + } + warn_unsupported_advisory(session->model, params); + if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { + return st; + } + if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { + if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + // Past this point we commit to producing a fresh batch result. + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + session->stream_state = TRANSCRIBE_STREAM_IDLE; + session->batch_results.clear(); + + // Release the compute scratch once the batch has run, whichever path it + // took (see transcribe_session::release_scratch). Once per call, not per + // utterance, so the serial fallback keeps its workspace across the loop. + scratch_release_guard scratch_release; + scratch_release.session = session; + scratch_release.armed = true; + + // Fast path: a family with a batched compute graph owns the whole loop. + if (session->model->arch->run_batch != nullptr) { + const transcribe_status st = session->model->arch->run_batch(session, pcm, n_samples, n, params); + // On abort the hook may retain only completed results; synthesize any + // missing slots so the result-set view always exposes n entries. + if (st == TRANSCRIBE_ERR_ABORTED) { + pad_batch_results_aborted(session, n); + } + // Keep the single accessors coherent with utterance 0. + if (!session->batch_results.empty()) { + restore_scratch_from_result(session, session->batch_results.front()); + } + return st; + } + + // Generic serial fallback: run each utterance in turn and snapshot it. + // Correct for every family; only the per-dispatch device throughput of + // a real run_batch() is forgone. run_one_inner re-validates the shared + // params (idempotent) before the family run(). + session->batch_results.reserve(static_cast(n)); + const transcribe_status batch_status = transcribe::run_batch_serial( + session, pcm, n_samples, n, [&](const float * p, int ns) { return run_one_inner(session, p, ns, params); }); + + // On abort the loop can break early, leaving fewer than n entries; + // synthesize any missing slots so the result-set view always exposes n + // entries (same invariant the fast path holds). + if (batch_status == TRANSCRIBE_ERR_ABORTED) { + pad_batch_results_aborted(session, n); + } + + // Restore the scratch slot to mirror utterance 0 for the single-result + // accessors (n >= 1 guarantees at least one entry, but be defensive). + if (!session->batch_results.empty()) { + restore_scratch_from_result(session, session->batch_results.front()); + } + return batch_status; +} + +// Model introspection +// +// These accessors read directly from the base struct; per-family load() +// fills caps / variant / backend before returning success. + +extern "C" transcribe_status transcribe_model_get_capabilities(const struct transcribe_model * model, + struct transcribe_capabilities * out_caps) { + if (model == nullptr || out_caps == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out_caps->struct_size, k_min_capabilities_size); st != TRANSCRIBE_OK) { + return st; + } + // Preserve the caller-declared size, then write only the prefix + // that fits in both the caller's buffer and this library's view; + // any tail bytes stay as the caller initialized them (zero, by + // the init function's zero-fill contract). + // + // Pointer fields written into the caller buffer (e.g. languages) + // remain model-owned and valid until transcribe_model_free(). + const uint64_t caller_size = out_caps->struct_size; + transcribe_capabilities staged = model->caps; + staged.struct_size = caller_size; + copy_out_prefix(out_caps, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_session_get_limits(const struct transcribe_session * session, + struct transcribe_session_limits * out) { + if (session == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_session_limits_size); st != TRANSCRIBE_OK) { + return st; + } + + transcribe_session_limits staged; + transcribe_session_limits_init(&staged); + + const transcribe_model * model = session->model; + const auto & lb = model->limits; + + if (lb.has_context_cap && lb.model_max_ctx > 0) { + // effective_n_ctx = model max, lowered (never raised) by the session + // n_ctx cap. This mirrors the per-family context_ceiling helpers. + int32_t eff = lb.model_max_ctx; + if (session->n_ctx > 0 && session->n_ctx < eff) { + eff = session->n_ctx; + } + staged.effective_n_ctx = eff; + + // effective_max_audio_ms: for families whose audio tokens consume the + // decoder context, invert the input gate at the effective ceiling so + // the audio bound tracks n_ctx. For families whose audio bound is the + // encoder (audio_from_caps), the audio limit is independent of the + // decoder context, so report caps.max_audio_ms unchanged. Either way + // it is advisory (representative prompt), not an exact per-call bound. + if (lb.audio_from_caps) { + staged.effective_max_audio_ms = model->caps.max_audio_ms; + } else { + const int64_t audio_tokens = (int64_t) eff - lb.prompt_overhead - lb.gen_reserve; + staged.effective_max_audio_ms = (audio_tokens > 0 && lb.ms_per_audio_token > 0.0) ? + (int64_t) ((double) audio_tokens * lb.ms_per_audio_token) : + 0; + } + + // max_kv_bytes: worst-case single-utterance KV allocation at the + // effective ceiling, exact for the session's kv_type. The families + // resolve AUTO (and F16) to f16 for the KV cache and use f32 only for + // an explicit F32 request, so the byte size is 4/elem for F32 and + // 2/elem otherwise. This is the ceiling for one utterance, not the + // per-run allocation (the cache grows to fit input); transcribe_run_batch + // allocates roughly batch_size x this. + const int64_t kv_bytes_per_elem = (session->kv_type == TRANSCRIBE_KV_TYPE_F32) ? 4 : 2; + staged.max_kv_bytes = lb.kv_elems_per_ctx_token * (int64_t) eff * kv_bytes_per_elem; + } else { + // Unbounded / soft-window family: no decoder context cap. Report the + // soft window (if any) from caps.max_audio_ms, independent of n_ctx. + staged.effective_n_ctx = 0; + staged.effective_max_audio_ms = model->caps.max_audio_ms; + staged.max_kv_bytes = 0; + } + + const uint64_t caller_size = out->struct_size; + staged.struct_size = caller_size; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static int transcribe_tokenize_impl(const struct transcribe_model * model, + const char * text, + int32_t * tokens, + size_t n_max) { + if (model == nullptr || text == nullptr) { + return INT_MIN; + } + const transcribe::Tokenizer * tok = model->tokenizer(); + if (tok == nullptr) { + return INT_MIN; + } + + std::vector ids; + if (tok->encode(std::string(text), ids) != TRANSCRIBE_OK) { + return INT_MIN; + } + + const size_t n = ids.size(); + if (n > static_cast(INT_MAX)) { + return INT_MIN; + } + if (n > n_max) { + // Negative-of-N retry contract (mirrors whisper.cpp). Guard + // against -n overflowing int range for pathologically large + // vocabularies; fall back to INT_MIN + 1 so the caller still + // gets a hard-failure code rather than a stale pointer. + if (n > static_cast(INT_MAX)) { + return INT_MIN; + } + return -static_cast(n); + } + if (tokens == nullptr && n > 0) { + return INT_MIN; + } + for (size_t i = 0; i < n; ++i) { + tokens[i] = ids[i]; + } + return static_cast(n); +} + +// Timings +// +// Accumulators live on the base classes (transcribe_model::t_load_us and +// transcribe_session::t_{mel,encode,decode}_us), populated by per-family +// load() / run(). These accessors convert microseconds to milliseconds. + +extern "C" transcribe_status transcribe_get_timings(const struct transcribe_session * session, + struct transcribe_timings * out_timings) { + if (session == nullptr || out_timings == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out_timings->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out_timings->struct_size; + transcribe_timings staged{}; + staged.struct_size = caller_size; + if (session->model != nullptr) { + staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; + } + staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; + staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; + staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; + copy_out_prefix(out_timings, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_print_timings(const struct transcribe_session * session) { + if (session == nullptr) { + return; + } + struct transcribe_timings t; + transcribe_timings_init(&t); + (void) transcribe_get_timings(session, &t); + char buf[256]; + + std::snprintf(buf, sizeof(buf), + "timings: load=%.2f ms mel=%.2f ms " + "encode=%.2f ms decode=%.2f ms", + t.load_ms, t.mel_ms, t.encode_ms, t.decode_ms); + // Surfaced even in the common dev case (transcribe-cli without an + // installed sink) via the stderr fallback; an explicit + // transcribe_log_set(NULL, ...) silences it like everything else. + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s", buf); +} + +extern "C" void transcribe_reset_timings(struct transcribe_session * session) { + if (session == nullptr) { + return; + } + // load_us is intentionally left alone — it's a model-scoped fact, + // not a per-call accumulator. + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; +} + +// Result accessors +// +// These read from the base context's result storage (tokens / words / +// segments / full_text / result_kind / has_result), populated by the +// per-family run() driver. Out-of-range indices and pre-run access return +// safe sentinels per the public contract. + +extern "C" const char * transcribe_full_text(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->full_text.c_str(); +} + +extern "C" const char * transcribe_raw_text(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->raw_text.c_str(); +} + +extern "C" const char * transcribe_detected_language(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->detected_language.c_str(); +} + +extern "C" transcribe_timestamp_kind transcribe_returned_timestamp_kind(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return TRANSCRIBE_TIMESTAMPS_NONE; + } + return session->result_kind; +} + +extern "C" int transcribe_n_segments(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->segments.size()); +} + +extern "C" int transcribe_n_words(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->words.size()); +} + +extern "C" int transcribe_n_tokens(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->tokens.size()); +} + +extern "C" int transcribe_n_speaker_segments(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->speaker_segments.size()); +} + +// Result accessors - per-item rows +// +// Copy-out accessors backed by the context's segments / words / tokens +// vectors, writing only the prefix that fits the caller's struct_size. +// Out-of-range index or pre-run access leaves the caller's struct zero-init +// and still returns OK, so callers can use text!=NULL as the "row present" +// signal without branching on status. +// +// `text` pointers in the staged structs alias the context's std::string +// storage; the library treats that as session-owned, valid until the next +// transcribe_run / transcribe_stream_begin / transcribe_stream_reset / +// transcribe_session_free on the same context (see the public header). + +extern "C" transcribe_status transcribe_get_segment(const struct transcribe_session * session, + int i, + struct transcribe_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->segments.size()) { + return TRANSCRIBE_OK; + } + const auto & s = session->segments[static_cast(i)]; + transcribe_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.first_word = s.first_word; + staged.n_words = s.n_words; + staged.first_token = s.first_token; + staged.n_tokens = s.n_tokens; + staged.text = s.text.c_str(); + staged.speaker_id = s.speaker_id; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_word(const struct transcribe_session * session, + int i, + struct transcribe_word * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_word zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->words.size()) { + return TRANSCRIBE_OK; + } + const auto & w = session->words[static_cast(i)]; + transcribe_word staged{}; + staged.struct_size = caller_size; + staged.t0_ms = w.t0_ms; + staged.t1_ms = w.t1_ms; + staged.seg_index = w.seg_index; + staged.first_token = w.first_token; + staged.n_tokens = w.n_tokens; + staged.text = w.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_token(const struct transcribe_session * session, + int i, + struct transcribe_token * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_token zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->tokens.size()) { + return TRANSCRIBE_OK; + } + const auto & t = session->tokens[static_cast(i)]; + transcribe_token staged{}; + staged.struct_size = caller_size; + staged.id = t.id; + staged.p = t.p; + staged.t0_ms = t.t0_ms; + staged.t1_ms = t.t1_ms; + staged.seg_index = t.seg_index; + staged.word_index = t.word_index; + staged.text = t.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_speaker_segment(const struct transcribe_session * session, + int i, + struct transcribe_speaker_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_speaker_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || + static_cast(i) >= session->speaker_segments.size()) { + return TRANSCRIBE_OK; + } + const auto & s = session->speaker_segments[static_cast(i)]; + transcribe_speaker_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.speaker_id = s.speaker_id; + staged.p = s.p; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +// Batch result accessors +// +// These index session->batch_results when a batch run populated it, and +// otherwise synthesize utterance 0 from the scratch slot after +// transcribe_run. They share the staging shape of the single-result +// accessors above; only the source vector differs. + +namespace { + +// Non-owning view of one utterance's result. valid == false means "no such +// utterance / empty result" and the accessors return their safe sentinels. +struct BatchResultView { + const std::vector * tokens = nullptr; + const std::vector * words = nullptr; + const std::vector * segments = nullptr; + const std::vector * speaker_segments = nullptr; + const std::string * full_text = nullptr; + const std::string * raw_text = nullptr; + const std::string * detected_language = nullptr; + transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; + bool valid = false; +}; + +int batch_result_count(const transcribe_session * s) { + if (s == nullptr) { + return 0; + } + if (!s->batch_results.empty()) { + return static_cast(s->batch_results.size()); + } + return s->has_result ? 1 : 0; +} + +BatchResultView batch_result_view(const transcribe_session * s, int i) { + BatchResultView v; + if (s == nullptr || i < 0) { + return v; + } + if (!s->batch_results.empty()) { + if (static_cast(i) >= s->batch_results.size()) { + return v; + } + const auto & rs = s->batch_results[static_cast(i)]; + if (!rs.has_result) { + return v; // individually-failed / empty utterance + } + v.tokens = &rs.tokens; + v.words = &rs.words; + v.segments = &rs.segments; + v.speaker_segments = &rs.speaker_segments; + v.full_text = &rs.full_text; + v.raw_text = &rs.raw_text; + v.detected_language = &rs.detected_language; + v.result_kind = rs.result_kind; + v.valid = true; + return v; + } + if (i == 0 && s->has_result) { + v.tokens = &s->tokens; + v.words = &s->words; + v.segments = &s->segments; + v.speaker_segments = &s->speaker_segments; + v.full_text = &s->full_text; + v.raw_text = &s->raw_text; + v.detected_language = &s->detected_language; + v.result_kind = s->result_kind; + v.valid = true; + } + return v; +} + +} // namespace + +extern "C" int transcribe_batch_n_results(const struct transcribe_session * session) { + return batch_result_count(session); +} + +extern "C" transcribe_status transcribe_batch_status(const struct transcribe_session * session, int i) { + if (session == nullptr || i < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (!session->batch_results.empty()) { + if (static_cast(i) >= session->batch_results.size()) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + return session->batch_results[static_cast(i)].status; + } + if (i == 0 && session->has_result) { + return TRANSCRIBE_OK; + } + return TRANSCRIBE_ERR_INVALID_ARG; +} + +extern "C" const char * transcribe_batch_full_text(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.full_text->c_str() : ""; +} + +extern "C" const char * transcribe_batch_raw_text(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.raw_text->c_str() : ""; +} + +extern "C" const char * transcribe_batch_detected_language(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.detected_language->c_str() : ""; +} + +extern "C" transcribe_timestamp_kind transcribe_batch_returned_timestamp_kind(const struct transcribe_session * session, + int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.result_kind : TRANSCRIBE_TIMESTAMPS_NONE; +} + +extern "C" int transcribe_batch_n_segments(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.segments->size()) : 0; +} + +extern "C" int transcribe_batch_n_words(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.words->size()) : 0; +} + +extern "C" int transcribe_batch_n_tokens(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.tokens->size()) : 0; +} + +extern "C" transcribe_status transcribe_batch_get_segment(const struct transcribe_session * session, + int i, + int j, + struct transcribe_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.segments->size()) { + return TRANSCRIBE_OK; + } + const auto & s = (*v.segments)[static_cast(j)]; + transcribe_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.first_word = s.first_word; + staged.n_words = s.n_words; + staged.first_token = s.first_token; + staged.n_tokens = s.n_tokens; + staged.text = s.text.c_str(); + staged.speaker_id = s.speaker_id; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_batch_get_word(const struct transcribe_session * session, + int i, + int j, + struct transcribe_word * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_word zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.words->size()) { + return TRANSCRIBE_OK; + } + const auto & w = (*v.words)[static_cast(j)]; + transcribe_word staged{}; + staged.struct_size = caller_size; + staged.t0_ms = w.t0_ms; + staged.t1_ms = w.t1_ms; + staged.seg_index = w.seg_index; + staged.first_token = w.first_token; + staged.n_tokens = w.n_tokens; + staged.text = w.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_batch_get_token(const struct transcribe_session * session, + int i, + int j, + struct transcribe_token * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_token zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.tokens->size()) { + return TRANSCRIBE_OK; + } + const auto & t = (*v.tokens)[static_cast(j)]; + transcribe_token staged{}; + staged.struct_size = caller_size; + staged.id = t.id; + staged.p = t.p; + staged.t0_ms = t.t0_ms; + staged.t1_ms = t.t1_ms; + staged.seg_index = t.seg_index; + staged.word_index = t.word_index; + staged.text = t.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" int transcribe_batch_n_speaker_segments(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.speaker_segments->size()) : 0; +} + +extern "C" transcribe_status transcribe_batch_get_speaker_segment(const struct transcribe_session * session, + int i, + int j, + struct transcribe_speaker_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_speaker_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.speaker_segments->size()) { + return TRANSCRIBE_OK; + } + const auto & s = (*v.speaker_segments)[static_cast(j)]; + transcribe_speaker_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.speaker_id = s.speaker_id; + staged.p = s.p; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_batch_get_timings(const struct transcribe_session * session, + int i, + struct transcribe_timings * out) { + if (session == nullptr || out == nullptr || i < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + int64_t mel_us = 0, enc_us = 0, dec_us = 0; + if (!session->batch_results.empty()) { + if (static_cast(i) >= session->batch_results.size()) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const auto & rs = session->batch_results[static_cast(i)]; + mel_us = rs.t_mel_us; + enc_us = rs.t_encode_us; + dec_us = rs.t_decode_us; + } else if (i == 0) { + mel_us = session->t_mel_us; + enc_us = session->t_encode_us; + dec_us = session->t_decode_us; + } else { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const uint64_t caller_size = out->struct_size; + transcribe_timings staged{}; + staged.struct_size = caller_size; + if (session->model != nullptr) { + staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; + } + staged.mel_ms = static_cast(mel_us) / 1000.0f; + staged.encode_ms = static_cast(enc_us) / 1000.0f; + staged.decode_ms = static_cast(dec_us) / 1000.0f; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +// C ABI forwarders. Entry points that allocate, reach ggml/driver state, +// or transfer ownership route through api_guard_* here. Other public symbols +// above are kept nothrow by construction: pure POD/c_str reads, prefix +// copies, plain stores, or contained callback emission. + +extern "C" transcribe_status transcribe_session_init(struct transcribe_model * model, + const struct transcribe_session_params * params, + struct transcribe_session ** out_session) { + const transcribe_status st = api_guard_status( + "transcribe_session_init", [&] { return transcribe_session_init_impl(model, params, out_session); }); + // Boundary-owned postcondition: failure => *out_session == NULL. + if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { + transcribe_session_free(*out_session); + *out_session = nullptr; + } + return st; +} + +extern "C" void transcribe_session_free(struct transcribe_session * session) { + api_guard_void("transcribe_session_free", [&] { transcribe_session_free_impl(session); }); +} + +extern "C" transcribe_status transcribe_open(const char * path, + const struct transcribe_model_load_params * load_params, + const struct transcribe_session_params * session_params, + struct transcribe_session ** out_session) { + const transcribe_status st = api_guard_status( + "transcribe_open", [&] { return transcribe_open_impl(path, load_params, session_params, out_session); }); + // Boundary-owned postcondition: failure => *out_session == NULL. + if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { + transcribe_session_free(*out_session); + *out_session = nullptr; + } + return st; +} + +extern "C" transcribe_status transcribe_run(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params) { + return api_guard_status("transcribe_run", [&] { return transcribe_run_impl(session, pcm, n_samples, params); }); +} + +extern "C" transcribe_status transcribe_run_batch(struct transcribe_session * session, + const float * const * pcm, + const int * n_samples, + int n, + const struct transcribe_run_params * params) { + return api_guard_status("transcribe_run_batch", + [&] { return transcribe_run_batch_impl(session, pcm, n_samples, n, params); }); +} + +extern "C" transcribe_status transcribe_stream_begin(struct transcribe_session * session, + const struct transcribe_run_params * run_params, + const struct transcribe_stream_params * stream_params) { + return api_guard_status("transcribe_stream_begin", + [&] { return transcribe_stream_begin_impl(session, run_params, stream_params); }); +} + +extern "C" transcribe_status transcribe_stream_feed(struct transcribe_session * session, + const float * pcm, + int n_samples, + struct transcribe_stream_update * update) { + return api_guard_status("transcribe_stream_feed", + [&] { return transcribe_stream_feed_impl(session, pcm, n_samples, update); }); +} + +extern "C" transcribe_status transcribe_stream_finalize(struct transcribe_session * session, + struct transcribe_stream_update * update) { + return api_guard_status("transcribe_stream_finalize", + [&] { return transcribe_stream_finalize_impl(session, update); }); +} + +extern "C" void transcribe_stream_reset(struct transcribe_session * session) { + api_guard_void("transcribe_stream_reset", [&] { transcribe_stream_reset_impl(session); }); +} + +extern "C" int transcribe_tokenize(const struct transcribe_model * model, + const char * text, + int32_t * tokens, + size_t n_max) { + return api_guard_value("transcribe_tokenize", INT_MIN, + [&] { return transcribe_tokenize_impl(model, text, tokens, n_max); }); +} diff --git a/src/transcribe-model.h b/src/transcribe-model.h index ce522cd9b..9f6ad7b7b 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -85,7 +85,7 @@ struct transcribe_model { // Basis for the session-level limits query (transcribe_session_get_limits). // A hard-context-cap family fills this at load() — the same place it - // computes caps.max_audio_ms — so the generic query in transcribe.cpp can + // computes caps.max_audio_ms — so the generic query in transcribe-asr.cpp can // recompute the effective limits for any session n_ctx without a // per-family hook. Left zero by unbounded / soft-window families (which // have no decoder context cap): zero model_max_ctx => the query reports diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 852702d60..e28d692e5 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -1,17 +1,19 @@ -// transcribe.cpp - public C API entry points + central dispatch. +// transcribe.cpp - library-level public C API entry points + model dispatch. // // What lives here: -// - The C entry points declared in include/transcribe.h. -// - The single dispatch site mapping an opened GGUF to its per-family -// Arch handler (transcribe_model_load_file). +// - Status strings, version, and ABI struct-size metadata. // - The log-sink publication / emission helpers. -// - The factory functions that initialize each public params struct. +// - Backend module loading and device discovery. +// - The single dispatch site mapping an opened GGUF to its per-family +// Arch handler (transcribe_model_load_file), model free, and the +// model-level queries (arch / variant / metadata / backend / device, +// extension-kind and feature probes). +// - The init functions for the params / output structs shared by every +// role (model load params, device info, timings, speaker segments). // -// What does NOT live here: GGUF reading (transcribe-loader), per-family -// load / context / run code (src/arch//...), and ggml-specific -// code. The introspection accessors read the base structs in -// transcribe-model.h / transcribe-session.h directly; per-family code owns -// load(), init_context(), and run(). +// What does NOT live here: the ASR session API (sessions, run, batch, +// streaming, results; transcribe-asr.cpp), GGUF reading (transcribe-loader), +// and per-family load / context / run code (src/arch//...). #include "transcribe.h" @@ -19,17 +21,13 @@ #include "ggml-backend.h" #include "ggml.h" // ggml_log_set: route ggml diagnostics into our sink #include "transcribe-abi.h" +#include "transcribe-api-guard.h" #include "transcribe-arch.h" #include "transcribe-backend.h" -#include "transcribe-batch-util.h" #include "transcribe-loader.h" #include "transcribe-log.h" #include "transcribe-model.h" #include "transcribe-path.h" -#include "transcribe-prompting.h" -#include "transcribe-session.h" -#include "transcribe-tokenizer.h" -#include "transcribe/whisper.h" #if defined(TRANSCRIBE_GGML_BACKEND_DL) && defined(_WIN32) # ifndef WIN32_LEAN_AND_MEAN @@ -62,55 +60,10 @@ #include #include -// Public C ABI entry points must not leak C++ exceptions. The guarded -// paths below map bad_alloc to OOM, other exceptions to BACKEND, and make -// teardown best-effort/no-throw. MSVC builds use /EHs so exceptions thrown -// by ggml C-ABI frames remain catchable here. - -namespace { - -template transcribe_status api_guard_status(const char * fn_name, Fn && body) { - try { - return body(); - } catch (const std::bad_alloc &) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: out of memory", fn_name); - return TRANSCRIBE_ERR_OOM; - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); - return TRANSCRIBE_ERR_BACKEND; - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); - return TRANSCRIBE_ERR_BACKEND; - } -} - -// Value-returning variant: `on_throw` is the entry point's documented -// hard-failure value (0 devices, false, INT_MIN, ...). -template T api_guard_value(const char * fn_name, T on_throw, Fn && body) { - try { - return body(); - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); - return on_throw; - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); - return on_throw; - } -} - -// Teardown variant: log and swallow. Frees must never fail. -template void api_guard_void(const char * fn_name, Fn && body) { - try { - body(); - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught backend exception during teardown: %s", fn_name, - e.what()); - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught unknown backend exception during teardown", fn_name); - } -} - -} // namespace +using transcribe::api_guard_status; +using transcribe::api_guard_value; +using transcribe::api_guard_void; +using transcribe::enum_field_raw; // Status @@ -181,23 +134,6 @@ extern "C" const char * transcribe_version_commit(void) { return TRANSCRIBE_COMMIT; } -// Raw enum reads at the public ABI boundary -// -// C callers can store ANY int in an enum-typed ABI field; in C++ loading an -// out-of-range value through an enum-typed lvalue is UB (UBSan traps it). So -// every public entry point reads caller-supplied enum values as raw bytes -// first, validates the raw int, and only then converts to the enum type. All -// public enums are int-sized; the static_assert below pins one. - -static inline int enum_field_raw(const void * field) { - int raw; - std::memcpy(&raw, field, sizeof(raw)); - return raw; -} - -static_assert(sizeof(transcribe_backend_request) == sizeof(int), - "public enums must be int-sized for raw boundary reads"); - // ABI metadata: sizeof/alignof for the public structs, so a binding can // verify its layout against this build. Keep these switches in sync with the // transcribe_abi_struct enum. @@ -274,258 +210,6 @@ extern "C" size_t transcribe_abi_struct_align(transcribe_abi_struct which) { return 0; } -namespace { - -// Ordered rank for timestamp granularities (NONE < SEGMENT < WORD < TOKEN), -// used to compare a request against a family's advertised maximum. AUTO and -// any unknown value get rank -1. -int timestamp_rank(transcribe_timestamp_kind k) { - switch (k) { - case TRANSCRIBE_TIMESTAMPS_NONE: - return 0; - case TRANSCRIBE_TIMESTAMPS_SEGMENT: - return 1; - case TRANSCRIBE_TIMESTAMPS_WORD: - return 2; - case TRANSCRIBE_TIMESTAMPS_TOKEN: - return 3; - case TRANSCRIBE_TIMESTAMPS_AUTO: - return -1; - } - return -1; -} - -// Shared run-params validation used by transcribe_run and -// transcribe_stream_begin. Both call sites have already validated -// that session, session->model, and params are non-null. The translate-task -// rejection differs between the two (run mirrors supports_translate, -// streaming-begin rejects unconditionally in v1), so each caller -// applies its own translate check before reaching this helper. -transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params); - -transcribe_status validate_run_params_common(const transcribe_session * session, const transcribe_run_params * params) { - // Raw-validate every enum field before its first enum-typed load (see - // enum_field_raw). Once a field passes here, downstream typed reads — - // including the per-family handlers' — are defined. - switch (enum_field_raw(¶ms->task)) { - case TRANSCRIBE_TASK_TRANSCRIBE: - case TRANSCRIBE_TASK_TRANSLATE: - case TRANSCRIBE_TASK_INSTRUCT: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->timestamps)) { - case TRANSCRIBE_TIMESTAMPS_NONE: - case TRANSCRIBE_TIMESTAMPS_AUTO: - case TRANSCRIBE_TIMESTAMPS_SEGMENT: - case TRANSCRIBE_TIMESTAMPS_WORD: - case TRANSCRIBE_TIMESTAMPS_TOKEN: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->pnc)) { - case TRANSCRIBE_PNC_MODE_DEFAULT: - case TRANSCRIBE_PNC_MODE_OFF: - case TRANSCRIBE_PNC_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->itn)) { - case TRANSCRIBE_ITN_MODE_DEFAULT: - case TRANSCRIBE_ITN_MODE_OFF: - case TRANSCRIBE_ITN_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->diarize)) { - case TRANSCRIBE_DIARIZE_MODE_DEFAULT: - case TRANSCRIBE_DIARIZE_MODE_OFF: - case TRANSCRIBE_DIARIZE_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { - const int req_rank = timestamp_rank(params->timestamps); - const int max_rank = timestamp_rank(session->model->caps.max_timestamp_kind); - if (req_rank > max_rank) { - return TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS; - } - } - if (params->language != nullptr && session->model->caps.n_languages > 0 && - session->model->caps.languages != nullptr) { - bool found = false; - for (int i = 0; i < session->model->caps.n_languages; ++i) { - const char * entry = session->model->caps.languages[i]; - if (entry != nullptr && std::strcmp(entry, params->language) == 0) { - found = true; - break; - } - } - if (!found) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - } - // Translation-target gate: for a TRANSLATE request naming a target - // language, reject up front when the model advertises a target set - // (n > 0) that does not include it. Mirrors the source-language check - // above and returns the same code. An empty/unadvertised set (old - // GGUFs, ASR-only models) makes this inert; family-level target/pair - // checks (e.g. canary's pivot pairs) still apply on top. - if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->target_language != nullptr && - session->model->caps.n_translate_target_languages > 0 && - session->model->caps.translate_target_languages != nullptr) { - bool found = false; - for (int i = 0; i < session->model->caps.n_translate_target_languages; ++i) { - const char * entry = session->model->caps.translate_target_languages[i]; - if (entry != nullptr && std::strcmp(entry, params->target_language) == 0) { - found = true; - break; - } - } - if (!found) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - } - // Translation-pair gate: when a GGUF advertises exact src>dst pairs and - // the caller supplied both sides, reject unsupported directions before - // family dispatch. A missing source hint keeps this inert because most - // families do not perform generic language detection here. - if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->language != nullptr && - params->target_language != nullptr && - !session->model->allows_translation_pair(params->language, params->target_language)) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - return validate_prompting(session->model, params); -} - -// Shape and hard-gate checks for the generic prompting fields, on a -// normalized view. Soft inputs a model ignores are removed later by -// prepare_prompting. -transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params) { - auto reject = [](transcribe_status st, const char * why) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_run: %s", why); - return st; - }; - if (params->n_vocabulary < 0 || (params->n_vocabulary > 0 && params->vocabulary == nullptr)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary is NULL or n_vocabulary is negative"); - } - for (int32_t i = 0; i < params->n_vocabulary; ++i) { - if (params->vocabulary[i] == nullptr) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary has a NULL entry"); - } - } - const bool has_prefix = transcribe::prompting::has_text(params->prefix); - if (params->task == TRANSCRIBE_TASK_INSTRUCT) { - if (!transcribe::has_feature(model, TRANSCRIBE_FEATURE_INSTRUCT)) { - return reject(TRANSCRIBE_ERR_UNSUPPORTED_TASK, - "this model does not support TRANSCRIBE_TASK_INSTRUCT (TRANSCRIBE_FEATURE_INSTRUCT)"); - } - // Output is free text: no target language and no alignment. A prefix - // as answer prefill is untested on every INSTRUCT family. - if (!transcribe::prompting::has_text(params->prompt)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT requires a non-empty prompt"); - } - if (params->target_language != nullptr) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT does not take a target_language"); - } - if (params->timestamps != TRANSCRIBE_TIMESTAMPS_NONE && params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT supports timestamps NONE or AUTO only"); - } - if (has_prefix) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, - "a transcript prefix is not supported with TRANSCRIBE_TASK_INSTRUCT"); - } - } - // Ignoring a prefix would make the output repeat the prefix's words and - // silently break callers that stitch text together, so it is a hard gate. - if (has_prefix && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, - "this model does not support a transcript prefix (TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)"); - } - return TRANSCRIBE_OK; -} - -// Rejects control-token literals in the prompting text before the result -// snapshot is cleared. Runs after strip_ignored_prompting, so ignored inputs -// are not rejected. -transcribe_status check_prompting_text(const transcribe_model * model, const transcribe_run_params * params) { - const transcribe::Tokenizer * tok = model->tokenizer(); - if (tok == nullptr) { - return TRANSCRIBE_OK; - } - for (int32_t i = 0; i < params->n_vocabulary; ++i) { - if (const transcribe_status st = - transcribe::prompting::check_plain_text(*tok, params->vocabulary[i], "vocabulary"); - st != TRANSCRIBE_OK) { - return st; - } - } - if (params->prompt != nullptr) { - if (const transcribe_status st = transcribe::prompting::check_plain_text(*tok, params->prompt, "prompt"); - st != TRANSCRIBE_OK) { - return st; - } - } - if (params->prefix != nullptr) { - return transcribe::prompting::check_plain_text(*tok, params->prefix, "prefix"); - } - return TRANSCRIBE_OK; -} - -// Full-size copy of a caller's run params: defaults first, then only the -// prefix the caller's struct_size covers, so every trailing field is -// readable (NULL/0 for an older caller). struct_size is preserved so -// has_field() gating still sees the caller's true layout. Idempotent. -void normalize_run_params(const transcribe_run_params * in, transcribe_run_params * out) { - transcribe_run_params_init(out); - std::memcpy(out, in, static_cast(std::min(in->struct_size, sizeof(*out)))); -} - -// Warn about, then remove, the soft prompting inputs this model ignores, so -// a family only ever sees inputs it should act on. Idempotent: the batch -// serial fallback re-enters run_one_inner per utterance. -void strip_ignored_prompting(const transcribe_model * model, transcribe_run_params * params) { - const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; - const bool instruct = params->task == TRANSCRIBE_TASK_INSTRUCT; - if (params->n_vocabulary > 0 && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_VOCABULARY)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, - "transcribe_run: model '%s' does not support vocabulary; ignoring %d term(s). Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_VOCABULARY) to pre-check.", - arch_name, params->n_vocabulary); - params->vocabulary = nullptr; - params->n_vocabulary = 0; - } - if (!instruct && transcribe::prompting::has_text(params->prompt) && - !transcribe::has_feature(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, - "transcribe_run: model '%s' has no context-prompt slot; ignoring prompt. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT) to pre-check, or " - "TRANSCRIBE_TASK_INSTRUCT on models with TRANSCRIBE_FEATURE_INSTRUCT.", - arch_name); - params->prompt = nullptr; - } - if (!transcribe::prompting::has_text(params->prompt)) { - params->prompt = nullptr; - } - if (!transcribe::prompting::has_text(params->prefix)) { - params->prefix = nullptr; - } -} - -// The pre-clear prompting step every entry point shares, on a validated -// normalized view. -transcribe_status prepare_prompting(const transcribe_model * model, transcribe_run_params * params) { - strip_ignored_prompting(model, params); - return check_prompting_text(model, params); -} - -} // namespace - // Logging // // Two-atomic log sink. Publication stores userdata (relaxed) then cb @@ -693,34 +377,6 @@ extern "C" void transcribe_model_load_params_init(struct transcribe_model_load_p p->struct_size = sizeof(*p); } -extern "C" void transcribe_session_params_init(struct transcribe_session_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - p->spec_k_drafts = -1; // family default - // Default to AUTO (richest output compatible with the model and selected - // run tasks, resolved per-family) rather than the memset NONE. - p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; -} - -extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - // Output struct init functions // // Output structs are caller-allocated buffers the library writes into, @@ -729,22 +385,6 @@ extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * // struct_size is the correct empty state. struct_size == 0 is rejected by // the accessors as a "you forgot to init the buffer" error. -extern "C" void transcribe_capabilities_init(struct transcribe_capabilities * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_session_limits_init(struct transcribe_session_limits * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - extern "C" void transcribe_timings_init(struct transcribe_timings * p) { if (p == nullptr) { return; @@ -753,30 +393,6 @@ extern "C" void transcribe_timings_init(struct transcribe_timings * p) { p->struct_size = sizeof(*p); } -extern "C" void transcribe_stream_update_init(struct transcribe_stream_update * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_stream_text_init(struct transcribe_stream_text * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_segment_init(struct transcribe_segment * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - extern "C" void transcribe_device_info_init(struct transcribe_device_info * p) { if (p == nullptr) { return; @@ -785,22 +401,6 @@ extern "C" void transcribe_device_info_init(struct transcribe_device_info * p) { p->struct_size = sizeof(*p); } -extern "C" void transcribe_word_init(struct transcribe_word * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_token_init(struct transcribe_token * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - extern "C" void transcribe_speaker_segment_init(struct transcribe_speaker_segment * p) { if (p == nullptr) { return; @@ -809,12 +409,6 @@ extern "C" void transcribe_speaker_segment_init(struct transcribe_speaker_segmen p->struct_size = sizeof(*p); } -// Whisper telemetry + run-extension init and chunk-trace accessors live in -// arch/whisper/public.cpp so this dispatcher stays family-agnostic. Whisper -// run knobs are in transcribe_whisper_run_ext (via run_params::family); -// sensevoice/funasr_nano use_itn and canary pnc use the generic run_params -// itn/pnc enums. - // Extension helpers extern "C" transcribe_status transcribe_ext_check(const struct transcribe_ext * ext, @@ -863,36 +457,8 @@ namespace { // library-side prefix do NOT raise this value. #define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) -constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); -constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); -// run_params is the one 0.2.0 exception to the append-only rule: `diarize` -// was inserted mid-struct, shifting every field from `language` on by 8 -// bytes. A 0.1-layout caller's sizeof (64) equals FIELD_END(family) in the -// NEW layout, so a `family`-prefix minimum would admit stale callers and -// read their pointers as enums. Require through spec_k_drafts so every -// pre-0.2 caller that bypasses the SONAME/abihash checks (dlopen by path, -// stale static link, hand-rolled FFI) gets BAD_STRUCT_SIZE instead. -constexpr size_t k_min_run_params_size = TRANSCRIBE_FIELD_END(transcribe_run_params, spec_k_drafts); -constexpr size_t k_min_stream_params_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, family); -constexpr size_t k_stream_params_commit_policy_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, commit_policy); -constexpr size_t k_stream_params_agreement_n_size = - TRANSCRIBE_FIELD_END(transcribe_stream_params, stable_prefix_agreement_n); -constexpr size_t k_min_stream_update_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, buffered_ms); -constexpr size_t k_stream_update_committed_changed_size = - TRANSCRIBE_FIELD_END(transcribe_stream_update, committed_changed); -constexpr size_t k_stream_update_tentative_changed_size = - TRANSCRIBE_FIELD_END(transcribe_stream_update, tentative_changed); -constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); -constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); -constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); -constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); -constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); -constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); -constexpr size_t k_min_speaker_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); -constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); -constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); -// k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with -// the chunk-trace accessor that uses it. +constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); +constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); #undef TRANSCRIBE_FIELD_END @@ -904,24 +470,6 @@ using transcribe::check_input_struct_size; using transcribe::check_struct_size; using transcribe::copy_out_prefix; -static bool has_field(uint64_t struct_size, size_t field_end) { - return struct_size >= static_cast(field_end); -} - -// Takes the RAW integer, not the enum: a C caller can store any int in the -// struct's enum-typed field, and in C++ loading an out-of-range value through -// the enum lvalue is UB (UBSan trips on it). Callers memcpy the bytes into an -// int and validate before the first enum-typed load. -static bool valid_stream_commit_policy(int policy) { - switch (policy) { - case TRANSCRIBE_STREAM_COMMIT_AUTO: - case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: - case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: - return true; - } - return false; -} - } // namespace // Backend modules and device discovery @@ -1217,417 +765,80 @@ static bool transcribe_backend_available_impl(int raw) { return false; } -namespace { +// Lifecycle -enum class StreamStablePrefixImpl { - GenericTextAgreement, - FamilyTokenAgreement, - FamilyNativeCommit, -}; - -static StreamStablePrefixImpl stream_stable_prefix_impl_for_arch(const transcribe::Arch * arch) { - // Single source of truth for the stable-prefix implementation used by - // AUTO and explicit STABLE_PREFIX. Add new streaming families here - // deliberately; otherwise they use the generic text-agreement fallback. - struct Entry { - const char * arch_name; - StreamStablePrefixImpl impl; - }; - - static constexpr Entry k_defaults[] = { - // Parakeet publishes native committed chunks; agreement_n does not - // add useful evidence on top of the family-provided boundary. - { "parakeet", StreamStablePrefixImpl::FamilyNativeCommit }, - // Moonshine re-decodes the full prefix and its family boundary is - // token-id agreement; the family applies stable_prefix_agreement_n. - { "moonshine_streaming", StreamStablePrefixImpl::FamilyTokenAgreement }, - }; - - const char * name = arch != nullptr && arch->name != nullptr ? arch->name : ""; - for (const Entry & entry : k_defaults) { - if (std::strcmp(name, entry.arch_name) == 0) { - return entry.impl; - } +static transcribe_status transcribe_model_load_file_impl(const char * path, + const struct transcribe_model_load_params * params, + struct transcribe_model ** out_model) { + // Set up ggml's timer before any family's load-timing ggml_time_us() runs + // (Windows/MSVC ÷0 otherwise — see ensure_ggml_time_init). + ensure_ggml_time_init(); + if (out_model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; } - return StreamStablePrefixImpl::GenericTextAgreement; -} - -static size_t utf8_floor_boundary(const std::string & s, size_t pos) { - if (pos >= s.size()) { - return s.size(); + *out_model = nullptr; + if (path == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; } - while (pos > 0 && (static_cast(s[pos]) & 0xC0u) == 0x80u) { - --pos; + // NULL params means "all defaults". This is the version-proof + // spelling of defaults — it carries no struct_size, so it always + // matches the running library. A zeroed struct ({0}) is a different + // thing: it claims struct_size == 0 and is rejected below. + struct transcribe_model_load_params params_defaults; + transcribe_model_load_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; } - return pos; -} - -static size_t common_prefix_bytes(const std::string & a, const std::string & b) { - const size_t n = std::min(a.size(), b.size()); - size_t i = 0; - while (i < n && a[i] == b[i]) { - ++i; + if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { + return st; } - return utf8_floor_boundary(a, i); -} -static bool starts_with_bytes(const std::string & s, const std::string & prefix) { - return s.size() >= prefix.size() && s.compare(0, prefix.size(), prefix) == 0; -} + // device is validated where the registry is available, in + // load_common::init_backends. NULL means automatic selection; a non-NULL + // handle selects that exact registered device or fails. -static size_t token_prefix_raw_bytes(const transcribe_session * session, int n_tokens) { - if (session == nullptr || n_tokens <= 0 || session->tokens.empty()) { - return 0; - } - const int capped = std::min(n_tokens, static_cast(session->tokens.size())); - std::string prefix; - prefix.reserve(session->full_text.size()); - for (int i = 0; i < capped; ++i) { - prefix += session->tokens[static_cast(i)].text; + // Raw-validate the backend request before the families' first + // enum-typed load of it (see enum_field_raw). init_backends re-checks + // as defense in depth; this boundary check is what makes every + // downstream `params->backend` read defined. + switch (enum_field_raw(¶ms->backend)) { + case TRANSCRIBE_BACKEND_AUTO: + case TRANSCRIBE_BACKEND_CPU: + case TRANSCRIBE_BACKEND_METAL: + case TRANSCRIBE_BACKEND_VULKAN: + case TRANSCRIBE_BACKEND_CPU_ACCEL: + case TRANSCRIBE_BACKEND_CUDA: + case TRANSCRIBE_BACKEND_ROCM: + break; + default: + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: invalid backend request %d", + enum_field_raw(¶ms->backend)); + return TRANSCRIBE_ERR_INVALID_ARG; } - if (starts_with_bytes(session->full_text, prefix)) { - return prefix.size(); + + // Format sniff by reading the first four bytes (not the file extension, + // which HF/users mislabel): GGUF magic 0x46554747 is the canonical path, + // ggml magic 0x67676d6c is a legacy whisper.cpp .bin. Public contract: + // missing path -> ERR_FILE_NOT_FOUND; every other failure -> ERR_GGUF. + if (!transcribe::path_is_present(path)) { + return TRANSCRIBE_ERR_FILE_NOT_FOUND; } - if (!prefix.empty() && prefix.front() == ' ' && (session->full_text.empty() || session->full_text.front() != ' ')) { - const std::string normalized_prefix = prefix.substr(1); - if (starts_with_bytes(session->full_text, normalized_prefix)) { - return normalized_prefix.size(); - } - return common_prefix_bytes(session->full_text, normalized_prefix); - } - return common_prefix_bytes(session->full_text, prefix); -} - -static size_t family_candidate_raw_prefix_bytes(const transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - if (session->n_committed_tokens > 0) { - return token_prefix_raw_bytes(session, session->n_committed_tokens); - } - if (session->n_committed_words > 0 && session->n_committed_words >= static_cast(session->words.size())) { - return session->full_text.size(); - } - if (session->n_committed_segments > 0 && - session->n_committed_segments >= static_cast(session->segments.size())) { - return session->full_text.size(); - } - return 0; -} - -static size_t generic_text_stable_prefix_candidate_raw_bytes(transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - // Library default when the caller leaves stable_prefix_agreement_n at - // 0; documented as "currently 3" on transcribe_stream_params. - static constexpr uint32_t k_default_stable_prefix_agreement_n = 3; - uint32_t agreement_n = session->stream_stable_prefix_agreement_n; - if (agreement_n == 0) { - agreement_n = k_default_stable_prefix_agreement_n; - } - if (agreement_n <= 1) { - return session->full_text.size(); - } - - session->stream_raw_history.push_back(session->full_text); - while (session->stream_raw_history.size() > agreement_n) { - session->stream_raw_history.pop_front(); - } - if (session->stream_raw_history.size() < agreement_n) { - return 0; - } - - size_t prefix_n = session->stream_raw_history.front().size(); - for (const auto & text : session->stream_raw_history) { - prefix_n = std::min(prefix_n, common_prefix_bytes(session->stream_raw_history.front(), text)); - } - return utf8_floor_boundary(session->full_text, prefix_n); -} - -static size_t selected_stable_prefix_candidate_raw_bytes(transcribe_session * session) { - if (session == nullptr) { - return 0; - } - const transcribe::Arch * arch = session->model != nullptr ? session->model->arch : nullptr; - switch (stream_stable_prefix_impl_for_arch(arch)) { - case StreamStablePrefixImpl::FamilyTokenAgreement: - case StreamStablePrefixImpl::FamilyNativeCommit: - return family_candidate_raw_prefix_bytes(session); - case StreamStablePrefixImpl::GenericTextAgreement: - return generic_text_stable_prefix_candidate_raw_bytes(session); - } - return 0; -} - -static bool append_committed_raw_prefix(transcribe_session * session, size_t candidate_bytes) { - if (session == nullptr || !session->has_result) { - return false; - } - const std::string & raw = session->full_text; - const size_t old_boundary = utf8_floor_boundary( - raw, std::min(static_cast(session->stream_raw_tentative_start_bytes), raw.size())); - const size_t candidate = utf8_floor_boundary(raw, std::min(candidate_bytes, raw.size())); - if (candidate <= old_boundary) { - return false; - } - if (old_boundary != session->stream_committed_text.size()) { - return false; - } - if (raw.compare(0, old_boundary, session->stream_committed_text.data(), old_boundary) != 0) { - return false; - } - session->stream_committed_text.append(raw.data() + old_boundary, candidate - old_boundary); - session->stream_raw_tentative_start_bytes = static_cast(candidate); - return true; -} - -static bool finalize_committed_text(transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return false; - } - if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE || - session->stream_committed_text.empty()) { - const bool changed = session->stream_committed_text != session->full_text; - session->stream_committed_text = session->full_text; - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return changed; - } - if (!starts_with_bytes(session->full_text, session->stream_committed_text)) { - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return false; - } - const size_t old_n = session->stream_committed_text.size(); - if (session->full_text.size() > old_n) { - session->stream_committed_text.append(session->full_text.data() + old_n, session->full_text.size() - old_n); - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return true; - } - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return false; -} - -struct StreamTextDelta { - bool committed_changed = false; - bool tentative_changed = false; -}; - -static StreamTextDelta apply_stream_text_policy(transcribe_session * session, bool is_finalize) { - StreamTextDelta delta; - if (session == nullptr) { - return delta; - } - const std::string prev_committed = session->stream_committed_text; - const std::string prev_tentative = session->stream_tentative_text; - - if (is_finalize) { - delta.committed_changed = finalize_committed_text(session); - session->stream_tentative_text.clear(); - } else if (session->has_result) { - size_t candidate = 0; - switch (session->stream_commit_policy) { - case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: - candidate = 0; - break; - case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: - case TRANSCRIBE_STREAM_COMMIT_AUTO: - default: - candidate = selected_stable_prefix_candidate_raw_bytes(session); - break; - } - delta.committed_changed = append_committed_raw_prefix(session, candidate); - - const size_t boundary = utf8_floor_boundary( - session->full_text, std::min(static_cast(session->stream_raw_tentative_start_bytes), - session->full_text.size())); - session->stream_tentative_text.assign(session->full_text.data() + boundary, - session->full_text.size() - boundary); - } else { - session->stream_tentative_text.clear(); - } - - delta.committed_changed = delta.committed_changed || session->stream_committed_text != prev_committed; - delta.tentative_changed = session->stream_tentative_text != prev_tentative; - return delta; -} - -static void publish_stream_update_tail(transcribe_stream_update * update, - bool committed_changed, - bool tentative_changed) { - if (update == nullptr) { - return; - } - if (has_field(update->struct_size, k_stream_update_committed_changed_size)) { - update->committed_changed = committed_changed; - } - if (has_field(update->struct_size, k_stream_update_tentative_changed_size)) { - update->tentative_changed = tentative_changed; - } -} - -// Compute the observable-change verdict, advance the revision counter -// when the result moved, and publish revision / result_changed / -// committed_changed / tentative_changed onto the caller's update. Shared -// verbatim by feed and finalize so the result_changed semantics live in -// exactly one place; the path-specific work (ON_FINALIZE counter zeroing -// and audio_committed_ms on feed, is_final and the lifecycle transition -// on finalize) stays in the callers. -static void publish_observable_delta(transcribe_session * session, - transcribe_stream_update * update, - int32_t prev_revision, - const std::string & prev_full_text, - bool prev_has_result, - const StreamTextDelta & text_delta) { - const bool raw_changed = session->has_result != prev_has_result || session->full_text != prev_full_text; - const bool observable_changed = raw_changed || text_delta.committed_changed || text_delta.tentative_changed || - (update != nullptr && update->result_changed); - if (observable_changed && session->stream_revision == prev_revision) { - session->stream_revision += 1; - } - if (update != nullptr) { - update->result_changed = - update->result_changed || observable_changed || session->stream_revision != prev_revision; - update->revision = session->stream_revision; - publish_stream_update_tail(update, text_delta.committed_changed, text_delta.tentative_changed); - } -} - -// Advisory pnc/itn warning. Emits a WARN when a non-DEFAULT request hits a -// model that does not support runtime control of that axis, then returns so -// the dispatcher proceeds with best-effort semantics. The reserved -// TRANSCRIBE_ERR_UNSUPPORTED_PNC / _ITN codes are NOT returned today -// (placeholders for a future opt-in strict mode). The message includes the -// arch + variant strings to pinpoint which model dropped the request. -void warn_unsupported_advisory(const struct transcribe_model * model, const struct transcribe_run_params * rp) { - if (model == nullptr || rp == nullptr) { - return; - } - const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; - const char * variant = model->variant.c_str(); - if (variant == nullptr || variant[0] == '\0') { - variant = "(unknown)"; - } - - // Defense in depth: keep raw reads here even though every dispatcher - // validates before warning. A future call-site reorder must not turn a - // malformed C enum into a typed C++ load. - const int pnc_raw = enum_field_raw(&rp->pnc); - const int itn_raw = enum_field_raw(&rp->itn); - const int diarize_raw = enum_field_raw(&rp->diarize); - - char buf[512]; - if (pnc_raw != TRANSCRIBE_PNC_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_PNC)) { - const char * req = (pnc_raw == TRANSCRIBE_PNC_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested pnc=%s but model '%s' " - "(variant '%s') does not support pnc control; output will use " - "the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_PNC) to " - "pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } - if (itn_raw != TRANSCRIBE_ITN_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_ITN)) { - const char * req = (itn_raw == TRANSCRIBE_ITN_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested itn=%s but model '%s' " - "(variant '%s') does not support itn control; output will use " - "the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_ITN) to " - "pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } - if (diarize_raw != TRANSCRIBE_DIARIZE_MODE_DEFAULT && - !transcribe::has_feature(model, TRANSCRIBE_FEATURE_DIARIZATION)) { - const char * req = (diarize_raw == TRANSCRIBE_DIARIZE_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested diarize=%s but model '%s' " - "(variant '%s') does not support diarization control; output " - "will use the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION) " - "to pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } -} - -} // namespace - -// Lifecycle - -static transcribe_status transcribe_model_load_file_impl(const char * path, - const struct transcribe_model_load_params * params, - struct transcribe_model ** out_model) { - // Set up ggml's timer before any family's load-timing ggml_time_us() runs - // (Windows/MSVC ÷0 otherwise — see ensure_ggml_time_init). - ensure_ggml_time_init(); - if (out_model == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_model = nullptr; - if (path == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults". This is the version-proof - // spelling of defaults — it carries no struct_size, so it always - // matches the running library. A zeroed struct ({0}) is a different - // thing: it claims struct_size == 0 and is rejected below. - struct transcribe_model_load_params params_defaults; - transcribe_model_load_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { - return st; - } - - // device is validated where the registry is available, in - // load_common::init_backends. NULL means automatic selection; a non-NULL - // handle selects that exact registered device or fails. - - // Raw-validate the backend request before the families' first - // enum-typed load of it (see enum_field_raw). init_backends re-checks - // as defense in depth; this boundary check is what makes every - // downstream `params->backend` read defined. - switch (enum_field_raw(¶ms->backend)) { - case TRANSCRIBE_BACKEND_AUTO: - case TRANSCRIBE_BACKEND_CPU: - case TRANSCRIBE_BACKEND_METAL: - case TRANSCRIBE_BACKEND_VULKAN: - case TRANSCRIBE_BACKEND_CPU_ACCEL: - case TRANSCRIBE_BACKEND_CUDA: - case TRANSCRIBE_BACKEND_ROCM: - break; - default: - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: invalid backend request %d", - enum_field_raw(¶ms->backend)); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Format sniff by reading the first four bytes (not the file extension, - // which HF/users mislabel): GGUF magic 0x46554747 is the canonical path, - // ggml magic 0x67676d6c is a legacy whisper.cpp .bin. Public contract: - // missing path -> ERR_FILE_NOT_FOUND; every other failure -> ERR_GGUF. - if (!transcribe::path_is_present(path)) { - return TRANSCRIBE_ERR_FILE_NOT_FOUND; - } - uint32_t magic = 0; - { - std::ifstream fin(transcribe::path_from_utf8(path), std::ios::binary); - if (!fin) { - // The existence pre-check said the path is reachable but - // the open failed — permissions / type issue. Treat as a generic - // load failure rather than FILE_NOT_FOUND. - return TRANSCRIBE_ERR_GGUF; - } - fin.read(reinterpret_cast(&magic), sizeof(magic)); - if (!fin || fin.gcount() != sizeof(magic)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_model_load_file: short read on file " - "magic for %s", - path); - return TRANSCRIBE_ERR_GGUF; + uint32_t magic = 0; + { + std::ifstream fin(transcribe::path_from_utf8(path), std::ios::binary); + if (!fin) { + // The existence pre-check said the path is reachable but + // the open failed — permissions / type issue. Treat as a generic + // load failure rather than FILE_NOT_FOUND. + return TRANSCRIBE_ERR_GGUF; + } + fin.read(reinterpret_cast(&magic), sizeof(magic)); + if (!fin || fin.gcount() != sizeof(magic)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_model_load_file: short read on file " + "magic for %s", + path); + return TRANSCRIBE_ERR_GGUF; } } @@ -1684,991 +895,10 @@ static void transcribe_model_free_impl(struct transcribe_model * model) { delete model; } -static transcribe_status transcribe_session_init_impl(struct transcribe_model * model, - const struct transcribe_session_params * params, - struct transcribe_session ** out_session) { - if (out_session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_session = nullptr; - if (model == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults" (see transcribe_model_load_file). - struct transcribe_session_params params_defaults; - transcribe_session_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_context_params_size); st != TRANSCRIBE_OK) { - return st; - } - // n_threads contract from include/transcribe.h: 0 means "library - // picks a sensible default", positive means "use this many." A - // negative value is undefined input, not a documented sentinel — - // reject it here so individual family handlers don't have to - // re-check the same condition. - if (params->n_threads < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // Raw-validate kv_type before the families' first enum-typed load of it - // (see enum_field_raw): an unknown value from a C caller is a clean - // error here, never UB downstream. - switch (enum_field_raw(¶ms->kv_type)) { - case TRANSCRIBE_KV_TYPE_AUTO: - case TRANSCRIBE_KV_TYPE_F32: - case TRANSCRIBE_KV_TYPE_F16: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - // n_ctx is a trailing field (appended after kv_type), so only read it - // when the caller's struct_size actually covers it; an older caller's - // smaller struct leaves it at the default 0 = "model max". A negative - // value is undefined input, not a documented sentinel. The family that - // honors n_ctx clamps a too-large value down to the model maximum, so - // only the negative case is rejected here. See include/transcribe.h. - if (has_field(params->struct_size, offsetof(struct transcribe_session_params, n_ctx) + sizeof(params->n_ctx)) && - params->n_ctx < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // The model carries its own arch dispatch pointer. A model that - // came from a load() call always has arch set; the null check is - // defensive against a hypothetical malformed model object. - if (model->arch == nullptr || model->arch->init_context == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - return model->arch->init_context(model, params, out_session); -} - -static void transcribe_session_free_impl(struct transcribe_session * session) { - // Free the session and, if it owns its model (transcribe_open path), - // free the model too. The owns_model flag is set only by - // transcribe_open; sessions created via the two-step - // transcribe_session_init borrow the model and leave it alone here. - // - // Capture before the session is destroyed; reading session->model - // after delete would be use-after-free. - if (session == nullptr) { - return; - } - struct transcribe_model * owned = session->owns_model ? session->model : nullptr; - delete session; - transcribe_model_free(owned); // NULL is a no-op -} - -// Convenience: open / close / get_model -// -// transcribe_open bundles load + session_init and hands back a session that -// owns its model (owns_model = true). transcribe_close is a thin alias for -// transcribe_session_free; both honor owns_model and free the owned model -// after the session. - -static transcribe_status transcribe_open_impl(const char * path, - const struct transcribe_model_load_params * load_params, - const struct transcribe_session_params * session_params, - struct transcribe_session ** out_session) { - if (out_session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_session = nullptr; - - // NULL params are forwarded as-is: the callees already treat NULL as - // "all defaults", so no substitution is needed here. - struct transcribe_model * model = nullptr; - if (const auto st = transcribe_model_load_file(path, load_params, &model); st != TRANSCRIBE_OK) { - return st; - } - - struct transcribe_session * session = nullptr; - if (const auto st = transcribe_session_init(model, session_params, &session); st != TRANSCRIBE_OK) { - // Load succeeded but session init failed: own the cleanup so the - // caller sees the same all-or-nothing contract as the two-step - // API. No partial state leaks. - transcribe_model_free(model); - return st; - } - - // The convenience session owns the model it loaded; either - // transcribe_session_free or transcribe_close (its alias) will free - // both. This is the only place owns_model is set. - session->owns_model = true; - *out_session = session; - return TRANSCRIBE_OK; -} - -extern "C" void transcribe_close(struct transcribe_session * session) { - // Alias for transcribe_session_free, kept for source compatibility - // with the prior split-API shape. Both honor owns_model. - transcribe_session_free(session); -} - -extern "C" const struct transcribe_model * transcribe_get_model(const struct transcribe_session * session) { - if (session == nullptr) { - return nullptr; - } - return session->model; -} - -extern "C" void transcribe_set_abort_callback(struct transcribe_session * session, - transcribe_abort_callback cb, - void * user_data) { - if (session == nullptr) { - return; - } - session->abort_cb = cb; - session->abort_userdata = user_data; -} - -extern "C" bool transcribe_was_aborted(const struct transcribe_session * session) { - if (session == nullptr) { - return false; - } - return session->was_aborted; -} - -extern "C" bool transcribe_was_truncated(const struct transcribe_session * session) { - if (session == nullptr) { - return false; - } - return session->was_truncated; -} - -// Streaming dispatcher -// -// State transitions are managed entirely here; hooks see ACTIVE on entry to -// begin/feed/finalize and never observe transitions. Hooks own the -// per-utterance result data (tokens/words/segments, committed counts, audio -// cursors, stream_revision); the dispatcher owns lifecycle state, -// last_status, and the public committed/tentative text view. clear_result() -// wipes the snapshot but never touches stream_state. - -static transcribe_status transcribe_stream_begin_impl(struct transcribe_session * session, - const struct transcribe_run_params * run_params, - const struct transcribe_stream_params * stream_params) { - if (session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL run/stream params each mean "all defaults". - struct transcribe_run_params run_params_defaults; - transcribe_run_params_init(&run_params_defaults); - struct transcribe_stream_params stream_params_defaults; - transcribe_stream_params_init(&stream_params_defaults); - if (run_params == nullptr) { - run_params = &run_params_defaults; - } - if (stream_params == nullptr) { - stream_params = &stream_params_defaults; - } - if (const auto st = check_input_struct_size(run_params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - // Full-size view (see normalize_run_params); the family hook gets a - // further copy whose strings the library owns, built below. - struct transcribe_run_params run_params_view; - normalize_run_params(run_params, &run_params_view); - run_params = &run_params_view; - if (const auto st = check_input_struct_size(stream_params->struct_size, k_min_stream_params_size); - st != TRANSCRIBE_OK) { - return st; - } - // Read the caller's commit_policy as raw bytes and validate BEFORE the - // first load through the enum-typed lvalue: C callers can legally store - // any int there, and an out-of-range enum load is UB in C++. - int commit_policy_raw = TRANSCRIBE_STREAM_COMMIT_AUTO; - uint32_t stable_prefix_agreement_n = 0; - if (has_field(stream_params->struct_size, k_stream_params_commit_policy_size)) { - static_assert(sizeof(stream_params->commit_policy) == sizeof(int), - "commit_policy must be int-sized for the raw read"); - std::memcpy(&commit_policy_raw, &stream_params->commit_policy, sizeof(commit_policy_raw)); - } - if (!valid_stream_commit_policy(commit_policy_raw)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const transcribe_stream_commit_policy commit_policy = - static_cast(commit_policy_raw); - if (has_field(stream_params->struct_size, k_stream_params_agreement_n_size)) { - stable_prefix_agreement_n = stream_params->stable_prefix_agreement_n; - if (stable_prefix_agreement_n > 32) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - // Caller must finalize or reset before starting a new stream. - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->model == nullptr || session->model->arch == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - // Capability gate: the model must advertise streaming AND the - // family must wire the full required hook set. begin / feed / - // finalize come as a triple — a partially-wired family that - // accepts begin would otherwise let the caller enter ACTIVE and - // then get stuck on NOT_IMPLEMENTED at the first feed. stream_reset - // remains optional; the dispatcher's clear_result + state wipe is - // sufficient when a family does not need to release per-utterance - // buffers explicitly. - if (!session->model->caps.supports_streaming || session->model->arch->stream_begin == nullptr || - session->model->arch->stream_feed == nullptr || session->model->arch->stream_finalize == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - if (const transcribe_status st = validate_run_params_common(session, run_params); st != TRANSCRIBE_OK) { - return st; - } - - // v1 rejects TRANSLATE unconditionally for streaming. A future - // family that supports streaming translate would loosen this in - // its stream_begin hook, but the central dispatcher refuses - // upfront so partially-wired callers fail fast. - if (run_params->task == TRANSCRIBE_TASK_TRANSLATE || run_params->task == TRANSCRIBE_TASK_INSTRUCT) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - // A prefix is forced decoder text for one utterance's opening; a stream - // has no fixed opening to force it onto. - if (transcribe::prompting::has_text(run_params->prefix)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_stream_begin: a transcript prefix is not accepted " - "for streaming"); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - if (stream_params->family != nullptr) { - if (stream_params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_STREAM, - stream_params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - - // Advisory warn for pnc/itn requests against models that don't - // expose the corresponding runtime toggle. Emitted before - // clear_result so the pre-hook "snapshot preserved on rejection" - // contract is undisturbed. - warn_unsupported_advisory(session->model, run_params); - if (const transcribe_status st = prepare_prompting(session->model, &run_params_view); st != TRANSCRIBE_OK) { - return st; - } - - // Optional family preflight: validates extension field values - // (e.g. parakeet's (L, C, R) menu) without mutating state. On - // non-OK return the previous snapshot and lifecycle are preserved - // so a caller-side typo does not destroy the prior utterance. - if (session->model->arch->stream_validate != nullptr) { - if (const transcribe_status st = session->model->arch->stream_validate(session, run_params, stream_params); - st != TRANSCRIBE_OK) { - return st; - } - } - - // All checks pass — clear the previous snapshot and hand off. - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - session->stream_state = TRANSCRIBE_STREAM_ACTIVE; - session->stream_commit_policy = commit_policy; - session->stream_stable_prefix_agreement_n = stable_prefix_agreement_n; - - // Hand the family hook a params view whose pointers the LIBRARY owns. - // Families may capture `*run_params` for the stream's lifetime - // (parakeet re-reads .language on every feed), and the public contract - // lets the caller free every params pointer once begin returns — so the - // strings are copied into session storage here and the view repointed - // at it. `family` (the run-slot extension) is nulled in the view: no - // family reads it on the stream path today, and per the ext copy-out - // contract a retained pointer to it would dangle; a future family that - // wants a run-slot ext at stream begin must plumb it deliberately. - session->stream_language_owned = run_params->language != nullptr ? run_params->language : ""; - session->stream_target_language_owned = run_params->target_language != nullptr ? run_params->target_language : ""; - // Copied from the normalized view (never the caller's struct, whose - // allocation may end before the trailing fields); the view keeps the - // caller's struct_size, so has_field() gating still sees its layout. - struct transcribe_run_params run_params_owned = run_params_view; - run_params_owned.language = run_params->language != nullptr ? session->stream_language_owned.c_str() : nullptr; - run_params_owned.target_language = - run_params->target_language != nullptr ? session->stream_target_language_owned.c_str() : nullptr; - run_params_owned.family = nullptr; - // Generic prompting strings, same ownership rule. Only non-empty terms - // are kept, so the view's count matches the owned array. - session->stream_vocabulary_owned = transcribe::prompting::terms(run_params); - session->stream_vocabulary_ptrs.clear(); - for (const std::string & term : session->stream_vocabulary_owned) { - session->stream_vocabulary_ptrs.push_back(term.c_str()); - } - session->stream_prompt_owned = run_params->prompt != nullptr ? run_params->prompt : ""; - run_params_owned.vocabulary = - session->stream_vocabulary_ptrs.empty() ? nullptr : session->stream_vocabulary_ptrs.data(); - run_params_owned.n_vocabulary = static_cast(session->stream_vocabulary_ptrs.size()); - run_params_owned.prompt = run_params->prompt != nullptr ? session->stream_prompt_owned.c_str() : nullptr; - run_params_owned.prefix = nullptr; - - const transcribe_status st = session->model->arch->stream_begin(session, &run_params_owned, stream_params); - if (st != TRANSCRIBE_OK) { - // Family hook rejected the begin (config it does not understand, - // memory allocation failure, etc.). Roll lifecycle back to - // FAILED so the caller can read last_status; leave the cleared - // result snapshot in place. - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - } - return st; -} - -// Helper: validate the caller's stream_update struct_size and reset -// every field except struct_size to a clean state before handing the -// update buffer to the family hook. Returns OK when update is NULL -// (the update buffer is optional) or when the size passes validation. -static transcribe_status prepare_stream_update(struct transcribe_stream_update * update) { - if (update == nullptr) { - return TRANSCRIBE_OK; - } - if (const auto st = check_struct_size(update->struct_size, k_min_stream_update_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = update->struct_size; - transcribe_stream_update staged{}; - staged.struct_size = caller_size; - copy_out_prefix(update, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * session, - const float * pcm, - int n_samples, - struct transcribe_stream_update * update) { - if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { - return st; - } - if (session == nullptr || pcm == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // Zero-length feed is rejected: feed exists to consume audio, - // and callers that just want to inspect state use the stream - // accessors. n_samples < 0 is plain garbage. - if (n_samples <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // begin already confirmed model/arch/hook; we re-check defensively - // so a malformed context (never happens in practice from a real - // begin) does not deref a null function pointer. - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->stream_feed == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - const int32_t prev_revision = session->stream_revision; - const std::string prev_full_text = session->full_text; - const bool prev_has_result = session->has_result; - - const transcribe_status st = session->model->arch->stream_feed(session, pcm, n_samples, update); - if (st != TRANSCRIBE_OK) { - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - return st; - } - const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/false); - if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { - session->n_committed_tokens = 0; - session->n_committed_words = 0; - session->n_committed_segments = 0; - } - publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); - if (update != nullptr && session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { - update->audio_committed_ms = 0; - } - return st; -} - -static transcribe_status transcribe_stream_finalize_impl(struct transcribe_session * session, - struct transcribe_stream_update * update) { - if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { - return st; - } - if (session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->model == nullptr || session->model->arch == nullptr || - session->model->arch->stream_finalize == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - const int32_t prev_revision = session->stream_revision; - const std::string prev_full_text = session->full_text; - const bool prev_has_result = session->has_result; - - const transcribe_status st = session->model->arch->stream_finalize(session, update); - if (update != nullptr) { - // Force is_final true regardless of family hook return; the - // marker describes the call site, not the result. Set after - // the hook so the family cannot accidentally clear it. - update->is_final = true; - } - if (st != TRANSCRIBE_OK) { - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - return st; - } - const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/true); - publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); - session->stream_state = TRANSCRIBE_STREAM_FINISHED; - return TRANSCRIBE_OK; -} - -static void transcribe_stream_reset_impl(struct transcribe_session * session) { - if (session == nullptr) { - return; - } - // Family hook releases per-utterance state and clears any buffered - // audio contents while keeping the allocations. A family without - // streaming just has no hook installed; reset becomes a pure - // dispatcher state wipe. - if (session->model != nullptr && session->model->arch != nullptr && session->model->arch->stream_reset != nullptr) { - session->model->arch->stream_reset(session); - } - session->clear_result(); - session->stream_state = TRANSCRIBE_STREAM_IDLE; - // was_aborted is per-stream; reset re-arms it the same way begin - // does so a caller that resets after an abort starts clean. - session->was_aborted = false; -} - -extern "C" enum transcribe_stream_state transcribe_stream_get_state(const struct transcribe_session * session) { - if (session == nullptr) { - return TRANSCRIBE_STREAM_IDLE; - } - return session->stream_state; -} - -extern "C" int transcribe_stream_revision(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->stream_revision; -} - -extern "C" int transcribe_stream_n_committed_segments(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_segments; -} - -extern "C" int transcribe_stream_n_committed_words(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_words; -} - -extern "C" int transcribe_stream_n_committed_tokens(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_tokens; -} - -extern "C" transcribe_status transcribe_stream_last_status(const struct transcribe_session * session) { - if (session == nullptr) { - return TRANSCRIBE_OK; - } - return session->stream_last_status; -} - -extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_session * session, - struct transcribe_stream_text * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_stream_text_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_stream_text staged{}; - staged.struct_size = caller_size; - if (session != nullptr && session->has_result) { - staged.full_text = session->full_text.c_str(); - staged.full_text_bytes = static_cast(session->full_text.size()); - staged.committed_text = session->stream_committed_text.c_str(); - staged.committed_text_bytes = static_cast(session->stream_committed_text.size()); - staged.tentative_text = session->stream_tentative_text.c_str(); - staged.tentative_text_bytes = static_cast(session->stream_tentative_text.size()); - staged.raw_tentative_start_bytes = - std::min(session->stream_raw_tentative_start_bytes, staged.full_text_bytes); - } else { - staged.full_text = ""; - staged.committed_text = ""; - staged.tentative_text = ""; - } - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -// Scope guard that calls transcribe_session::release_scratch on exit once -// armed. Both offline entry points use it so release also happens when a -// family hook throws and the api_guard unwinds the stack, including the path -// where memory pressure matters most. release_scratch is noexcept, so running -// it during unwinding is safe. -namespace { - -struct scratch_release_guard { - transcribe_session * session = nullptr; - bool armed = false; - - ~scratch_release_guard() { - if (armed && session != nullptr) { - session->release_scratch(); - } - } -}; - -} // namespace - -// Shared one-utterance run body. Does NOT touch session->batch_results, so -// the batch dispatcher can call it once per utterance inside a loop without -// erasing already-accumulated entries; the public transcribe_run wrapper -// below clears batch_results once before delegating here. Every early -// return preserves the previous result snapshot exactly as the original -// transcribe_run contract documented (see the inline comments). -// `committed` (optional) is set true once the call passes the pre-clear -// gates and commits to replacing the result. The caller uses it to decide -// whether compute scratch needs releasing. -static transcribe_status run_one_inner(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params, - bool * committed = nullptr) { - // Parameter-shape validation runs first and does not touch session - // state. A caller that passes NULL pointers or a non-positive sample - // count gets ERR_INVALID_ARG back without any visible side effect - // — including the session's result fields, which are preserved - // across a malformed call. This is the narrower half of the - // "transcribe_run replaces the previous result" contract: the - // replacement happens when the call is well-formed enough that - // the dispatcher is willing to dereference session. A call that - // isn't well-formed isn't considered to have "run". - if (session == nullptr || pcm == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // n_samples must be strictly positive, matching transcribe_stream_feed. - // There is no meaningful "transcribe zero samples" operation; a - // zero-length batch is treated as a caller error, not an empty run. - if (n_samples <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults" (transcribe vs translate, no - // timestamps, etc.). A well-formed default run is not a malformed - // call, so it proceeds and replaces the previous result. - struct transcribe_run_params params_defaults; - transcribe_run_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - // Everything downstream reads this full-size view, never the caller's - // struct: an older caller's allocation may end before the prompting - // fields. Strings stay caller-owned; the call is synchronous. - struct transcribe_run_params params_view; - normalize_run_params(params, ¶ms_view); - params = ¶ms_view; - // A run cannot replace an active stream's results — that would - // strand the in-flight stream's per-family state. Caller must - // finalize or reset first. FINISHED and FAILED both fall through; - // the run path below resets stream_state to IDLE. - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Pre-clear family-extension SHAPE validation: a run ext with a bad - // transcribe_ext header size or a kind the model does not accept must - // NOT wipe the previous result snapshot (mirrors the - // transcribe_stream_begin contract). When model is null we skip the - // pre-clear check and let the post-clear NOT_IMPLEMENTED path handle it. - // - // This guarantee covers ext shape (size/kind) and the run-param checks - // below. A family that defers deeper value validation to run() (whisper - // does this for prompt semantics) can still reject a correctly-shaped - // ext after the snapshot is cleared; full pre-clear safety requires - // validating values in run_validate. - if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { - if (params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - - // Validate before advisory warning so typed enum reads are proven safe. - // A rejection preserves the previous snapshot. Skipped when model is - // null — the post-clear NOT_IMPLEMENTED path signals that directly. - if (session->model != nullptr) { - // Pre-clear run-param validation. A caller-side param bug - // (out-of-range enum, timestamp granularity finer than the - // model's max, unsupported language, or TRANSLATE against a - // model that doesn't support it) must NOT wipe the previous - // result snapshot — the same contract transcribe_stream_begin - // honors for its run params. These checks read model->caps and - // are therefore guarded by model != null; the degenerate - // model-null case falls through to the post-clear - // NOT_IMPLEMENTED path, which keeps its existing snapshot-wipe - // behavior. - if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { - return st; - } - warn_unsupported_advisory(session->model, params); - if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { - return st; - } - - // Family run-ext validation (the _RUN analogue of stream_validate), - // the final pre-clear gate. Runs AFTER the run-param checks above, - // mirroring transcribe_stream_begin's ordering (dispatcher param - // checks, then the family validate hook, then clear). The family - // alone knows its per-kind minimum struct size; whisper enforces - // that here. A non-OK return preserves the previous snapshot. A - // family with no _RUN ext leaves this hook NULL. Called regardless - // of whether params->family is set (whisper's hook treats a NULL - // family as "defaults", returning OK). - if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { - if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { - return st; - } - } - } - - // Result-replacement contract: everything past this point either - // succeeds and writes a fresh result, or fails and leaves the - // context in the documented "no result" sentinel state. Clear - // eagerly so the one remaining downstream rejection path — - // NOT_IMPLEMENTED on an incomplete arch — inherits the sentinel - // without having to remember to call clear_result() itself. - // Caller-param rejections (enum range, timestamp ceiling, - // language, TRANSLATE support) are validated ABOVE, before this - // clear, so a malformed call preserves the previous snapshot - // exactly as transcribe_stream_begin does. - // - // Per-family run() handlers call clear_result() again after - // their own front-matter checks succeed; that call is now - // redundant but idempotent, and removing it is a refactor - // deferred to a later pass. - if (committed != nullptr) { - *committed = true; - } - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - // Force stream_state to IDLE: clear_result deliberately preserves - // lifecycle state, but a well-formed transcribe_run subsumes any - // prior FINISHED/FAILED stream — after a one-shot run the context - // is no longer meaningfully in a streaming lifecycle. - session->stream_state = TRANSCRIBE_STREAM_IDLE; - - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - return session->model->arch->run(session, pcm, n_samples, params); -} - -static transcribe_status transcribe_run_impl(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params) { - // A well-formed single run has the same result view as a one-item batch: - // reset batch_results so the transcribe_batch_* accessors fall back to - // reading the scratch slot as utterance 0. - // - // Guard the deref with the SAME pre-deref conditions run_one_inner uses - // (non-null session + non-null pcm + positive n_samples). A malformed - // call must not touch the session at all — the API smoke test probes - // this with a fake (session *)0x1 and a NULL pcm / non-positive - // n_samples, expecting INVALID_ARG with no dereference. - if (session != nullptr && pcm != nullptr && n_samples > 0) { - session->batch_results.clear(); - } - // run_one_inner arms the guard at its commit point, so a pre-clear - // rejection never releases and everything after (family error, abort, - // throw) always does. - scratch_release_guard scratch_release; - scratch_release.session = session; - return run_one_inner(session, pcm, n_samples, params, &scratch_release.armed); -} - -// Batch run (offline) -// -// transcribe_run_batch validates the shared run_params ONCE, then either -// delegates to the family's batched run_batch() fast path or falls back to -// run_one_inner() per utterance. Either way the result is N entries in -// session->batch_results read back via the transcribe_batch_* accessors. - -namespace { - -// Pad batch_results out to `n` entries with explicit aborted failures. -// Called only on an aborted batch: synthesized slots carry -// TRANSCRIBE_ERR_ABORTED ("did not complete because the batch was aborted"), -// so the result-set view has one slot per input utterance -// (transcribe_batch_n_results() == n) regardless of family or path. -void pad_batch_results_aborted(transcribe_session * s, int n) { - while (s->batch_results.size() < static_cast(n)) { - transcribe_session::ResultSet rs; - rs.status = TRANSCRIBE_ERR_ABORTED; - s->batch_results.push_back(std::move(rs)); - } -} - -// Restore the scratch slot from a ResultSet so the single-result accessors -// stay coherent after a batch run (they reflect utterance 0). This is the -// inverse of transcribe_session::capture_result minus the per-run timing -// accumulators — keep the result fields in sync with that method when adding -// one (raw_text was once missed here). -void restore_scratch_from_result(transcribe_session * s, const transcribe_session::ResultSet & rs) { - s->tokens = rs.tokens; - s->words = rs.words; - s->segments = rs.segments; - s->speaker_segments = rs.speaker_segments; - s->full_text = rs.full_text; - s->raw_text = rs.raw_text; - s->detected_language = rs.detected_language; - s->result_kind = rs.result_kind; - s->has_result = rs.has_result; -} - -} // namespace - -static transcribe_status transcribe_run_batch_impl(struct transcribe_session * session, - const float * const * pcm, - const int * n_samples, - int n, - const struct transcribe_run_params * params) { - // Top-level argument shape. The arrays themselves must be present and - // n strictly positive; an individual malformed utterance (pcm[i] NULL - // or n_samples[i] <= 0) is a per-utterance failure handled in the loop, - // not a whole-batch error. As with transcribe_run, these checks do not - // mutate the session, so a malformed call preserves the prior result. - if (session == nullptr || pcm == nullptr || n_samples == nullptr || n <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - struct transcribe_run_params params_defaults; - transcribe_run_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - struct transcribe_run_params params_view; - normalize_run_params(params, ¶ms_view); - params = ¶ms_view; - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // One shared params across different audio: a transcript prefix can - // only describe one of them. - if (transcribe::prompting::has_text(params->prefix)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_run_batch: a transcript prefix is per-utterance " - "and is not accepted in a batch"); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Shared-param validation, ONCE, mirroring transcribe_run's pre-clear - // gates (ext shape/kind, pnc/itn advisory, enum range, timestamp - // ceiling, language, TRANSLATE support, run_validate). A rejection here - // preserves the previous result snapshot — nothing has been cleared. - if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { - if (params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - if (session->model != nullptr) { - if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { - return st; - } - warn_unsupported_advisory(session->model, params); - if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { - return st; - } - if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { - if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { - return st; - } - } - } - - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - // Past this point we commit to producing a fresh batch result. - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - session->stream_state = TRANSCRIBE_STREAM_IDLE; - session->batch_results.clear(); - - // Release the compute scratch once the batch has run, whichever path it - // took (see transcribe_session::release_scratch). Once per call, not per - // utterance, so the serial fallback keeps its workspace across the loop. - scratch_release_guard scratch_release; - scratch_release.session = session; - scratch_release.armed = true; - - // Fast path: a family with a batched compute graph owns the whole loop. - if (session->model->arch->run_batch != nullptr) { - const transcribe_status st = session->model->arch->run_batch(session, pcm, n_samples, n, params); - // On abort the hook may retain only completed results; synthesize any - // missing slots so the result-set view always exposes n entries. - if (st == TRANSCRIBE_ERR_ABORTED) { - pad_batch_results_aborted(session, n); - } - // Keep the single accessors coherent with utterance 0. - if (!session->batch_results.empty()) { - restore_scratch_from_result(session, session->batch_results.front()); - } - return st; - } - - // Generic serial fallback: run each utterance in turn and snapshot it. - // Correct for every family; only the per-dispatch device throughput of - // a real run_batch() is forgone. run_one_inner re-validates the shared - // params (idempotent) before the family run(). - session->batch_results.reserve(static_cast(n)); - const transcribe_status batch_status = transcribe::run_batch_serial( - session, pcm, n_samples, n, [&](const float * p, int ns) { return run_one_inner(session, p, ns, params); }); - - // On abort the loop can break early, leaving fewer than n entries; - // synthesize any missing slots so the result-set view always exposes n - // entries (same invariant the fast path holds). - if (batch_status == TRANSCRIBE_ERR_ABORTED) { - pad_batch_results_aborted(session, n); - } - - // Restore the scratch slot to mirror utterance 0 for the single-result - // accessors (n >= 1 guarantees at least one entry, but be defensive). - if (!session->batch_results.empty()) { - restore_scratch_from_result(session, session->batch_results.front()); - } - return batch_status; -} - // Model introspection // // These accessors read directly from the base struct; per-family load() -// fills caps / variant / backend before returning success. - -extern "C" transcribe_status transcribe_model_get_capabilities(const struct transcribe_model * model, - struct transcribe_capabilities * out_caps) { - if (model == nullptr || out_caps == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out_caps->struct_size, k_min_capabilities_size); st != TRANSCRIBE_OK) { - return st; - } - // Preserve the caller-declared size, then write only the prefix - // that fits in both the caller's buffer and this library's view; - // any tail bytes stay as the caller initialized them (zero, by - // the init function's zero-fill contract). - // - // Pointer fields written into the caller buffer (e.g. languages) - // remain model-owned and valid until transcribe_model_free(). - const uint64_t caller_size = out_caps->struct_size; - transcribe_capabilities staged = model->caps; - staged.struct_size = caller_size; - copy_out_prefix(out_caps, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_session_get_limits(const struct transcribe_session * session, - struct transcribe_session_limits * out) { - if (session == nullptr || out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_session_limits_size); st != TRANSCRIBE_OK) { - return st; - } - - transcribe_session_limits staged; - transcribe_session_limits_init(&staged); - - const transcribe_model * model = session->model; - const auto & lb = model->limits; - - if (lb.has_context_cap && lb.model_max_ctx > 0) { - // effective_n_ctx = model max, lowered (never raised) by the session - // n_ctx cap. This mirrors the per-family context_ceiling helpers. - int32_t eff = lb.model_max_ctx; - if (session->n_ctx > 0 && session->n_ctx < eff) { - eff = session->n_ctx; - } - staged.effective_n_ctx = eff; - - // effective_max_audio_ms: for families whose audio tokens consume the - // decoder context, invert the input gate at the effective ceiling so - // the audio bound tracks n_ctx. For families whose audio bound is the - // encoder (audio_from_caps), the audio limit is independent of the - // decoder context, so report caps.max_audio_ms unchanged. Either way - // it is advisory (representative prompt), not an exact per-call bound. - if (lb.audio_from_caps) { - staged.effective_max_audio_ms = model->caps.max_audio_ms; - } else { - const int64_t audio_tokens = (int64_t) eff - lb.prompt_overhead - lb.gen_reserve; - staged.effective_max_audio_ms = (audio_tokens > 0 && lb.ms_per_audio_token > 0.0) ? - (int64_t) ((double) audio_tokens * lb.ms_per_audio_token) : - 0; - } - - // max_kv_bytes: worst-case single-utterance KV allocation at the - // effective ceiling, exact for the session's kv_type. The families - // resolve AUTO (and F16) to f16 for the KV cache and use f32 only for - // an explicit F32 request, so the byte size is 4/elem for F32 and - // 2/elem otherwise. This is the ceiling for one utterance, not the - // per-run allocation (the cache grows to fit input); transcribe_run_batch - // allocates roughly batch_size x this. - const int64_t kv_bytes_per_elem = (session->kv_type == TRANSCRIBE_KV_TYPE_F32) ? 4 : 2; - staged.max_kv_bytes = lb.kv_elems_per_ctx_token * (int64_t) eff * kv_bytes_per_elem; - } else { - // Unbounded / soft-window family: no decoder context cap. Report the - // soft window (if any) from caps.max_audio_ms, independent of n_ctx. - staged.effective_n_ctx = 0; - staged.effective_max_audio_ms = model->caps.max_audio_ms; - staged.max_kv_bytes = 0; - } - - const uint64_t caller_size = out->struct_size; - staged.struct_size = caller_size; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} +// fills variant / backend / meta before returning success. extern "C" const char * transcribe_model_arch_string(const struct transcribe_model * model) { // The dispatch token's name field has static lifetime and is the @@ -2698,46 +928,6 @@ extern "C" const char * transcribe_model_meta_val_str(const struct transcribe_mo return it != model->meta.end() ? it->second.c_str() : ""; } -static int transcribe_tokenize_impl(const struct transcribe_model * model, - const char * text, - int32_t * tokens, - size_t n_max) { - if (model == nullptr || text == nullptr) { - return INT_MIN; - } - const transcribe::Tokenizer * tok = model->tokenizer(); - if (tok == nullptr) { - return INT_MIN; - } - - std::vector ids; - if (tok->encode(std::string(text), ids) != TRANSCRIBE_OK) { - return INT_MIN; - } - - const size_t n = ids.size(); - if (n > static_cast(INT_MAX)) { - return INT_MIN; - } - if (n > n_max) { - // Negative-of-N retry contract (mirrors whisper.cpp). Guard - // against -n overflowing int range for pathologically large - // vocabularies; fall back to INT_MIN + 1 so the caller still - // gets a hard-failure code rather than a stale pointer. - if (n > static_cast(INT_MAX)) { - return INT_MIN; - } - return -static_cast(n); - } - if (tokens == nullptr && n > 0) { - return INT_MIN; - } - for (size_t i = 0; i < n; ++i) { - tokens[i] = ids[i]; - } - return static_cast(n); -} - extern "C" const char * transcribe_model_backend(const struct transcribe_model * model) { // Empty string means "no runtime backend bound" — see the public header // for the full semantic. @@ -2758,554 +948,6 @@ static transcribe_device_t transcribe_model_device_impl(const struct transcribe_ return reinterpret_cast(dev); } -// Timings -// -// Accumulators live on the base classes (transcribe_model::t_load_us and -// transcribe_session::t_{mel,encode,decode}_us), populated by per-family -// load() / run(). These accessors convert microseconds to milliseconds. - -extern "C" transcribe_status transcribe_get_timings(const struct transcribe_session * session, - struct transcribe_timings * out_timings) { - if (session == nullptr || out_timings == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out_timings->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out_timings->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; - staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; - staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; - copy_out_prefix(out_timings, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" void transcribe_print_timings(const struct transcribe_session * session) { - if (session == nullptr) { - return; - } - struct transcribe_timings t; - transcribe_timings_init(&t); - (void) transcribe_get_timings(session, &t); - char buf[256]; - - std::snprintf(buf, sizeof(buf), - "timings: load=%.2f ms mel=%.2f ms " - "encode=%.2f ms decode=%.2f ms", - t.load_ms, t.mel_ms, t.encode_ms, t.decode_ms); - // Surfaced even in the common dev case (transcribe-cli without an - // installed sink) via the stderr fallback; an explicit - // transcribe_log_set(NULL, ...) silences it like everything else. - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_INFO, buf); -} - -extern "C" void transcribe_reset_timings(struct transcribe_session * session) { - if (session == nullptr) { - return; - } - // load_us is intentionally left alone — it's a model-scoped fact, - // not a per-call accumulator. - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; -} - -// Result accessors -// -// These read from the base context's result storage (tokens / words / -// segments / full_text / result_kind / has_result), populated by the -// per-family run() driver. Out-of-range indices and pre-run access return -// safe sentinels per the public contract. - -extern "C" const char * transcribe_full_text(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->full_text.c_str(); -} - -extern "C" const char * transcribe_raw_text(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->raw_text.c_str(); -} - -extern "C" const char * transcribe_detected_language(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->detected_language.c_str(); -} - -extern "C" transcribe_timestamp_kind transcribe_returned_timestamp_kind(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return TRANSCRIBE_TIMESTAMPS_NONE; - } - return session->result_kind; -} - -extern "C" int transcribe_n_segments(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->segments.size()); -} - -extern "C" int transcribe_n_words(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->words.size()); -} - -extern "C" int transcribe_n_tokens(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->tokens.size()); -} - -extern "C" int transcribe_n_speaker_segments(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->speaker_segments.size()); -} - -// Result accessors - per-item rows -// -// Copy-out accessors backed by the context's segments / words / tokens -// vectors, writing only the prefix that fits the caller's struct_size. -// Out-of-range index or pre-run access leaves the caller's struct zero-init -// and still returns OK, so callers can use text!=NULL as the "row present" -// signal without branching on status. -// -// `text` pointers in the staged structs alias the context's std::string -// storage; the library treats that as session-owned, valid until the next -// transcribe_run / transcribe_stream_begin / transcribe_stream_reset / -// transcribe_session_free on the same context (see the public header). - -extern "C" transcribe_status transcribe_get_segment(const struct transcribe_session * session, - int i, - struct transcribe_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->segments.size()) { - return TRANSCRIBE_OK; - } - const auto & s = session->segments[static_cast(i)]; - transcribe_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.first_word = s.first_word; - staged.n_words = s.n_words; - staged.first_token = s.first_token; - staged.n_tokens = s.n_tokens; - staged.text = s.text.c_str(); - staged.speaker_id = s.speaker_id; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_word(const struct transcribe_session * session, - int i, - struct transcribe_word * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_word zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->words.size()) { - return TRANSCRIBE_OK; - } - const auto & w = session->words[static_cast(i)]; - transcribe_word staged{}; - staged.struct_size = caller_size; - staged.t0_ms = w.t0_ms; - staged.t1_ms = w.t1_ms; - staged.seg_index = w.seg_index; - staged.first_token = w.first_token; - staged.n_tokens = w.n_tokens; - staged.text = w.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_token(const struct transcribe_session * session, - int i, - struct transcribe_token * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_token zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->tokens.size()) { - return TRANSCRIBE_OK; - } - const auto & t = session->tokens[static_cast(i)]; - transcribe_token staged{}; - staged.struct_size = caller_size; - staged.id = t.id; - staged.p = t.p; - staged.t0_ms = t.t0_ms; - staged.t1_ms = t.t1_ms; - staged.seg_index = t.seg_index; - staged.word_index = t.word_index; - staged.text = t.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_speaker_segment(const struct transcribe_session * session, - int i, - struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || - static_cast(i) >= session->speaker_segments.size()) { - return TRANSCRIBE_OK; - } - const auto & s = session->speaker_segments[static_cast(i)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -// Batch result accessors -// -// These index session->batch_results when a batch run populated it, and -// otherwise synthesize utterance 0 from the scratch slot after -// transcribe_run. They share the staging shape of the single-result -// accessors above; only the source vector differs. - -namespace { - -// Non-owning view of one utterance's result. valid == false means "no such -// utterance / empty result" and the accessors return their safe sentinels. -struct BatchResultView { - const std::vector * tokens = nullptr; - const std::vector * words = nullptr; - const std::vector * segments = nullptr; - const std::vector * speaker_segments = nullptr; - const std::string * full_text = nullptr; - const std::string * raw_text = nullptr; - const std::string * detected_language = nullptr; - transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; - bool valid = false; -}; - -int batch_result_count(const transcribe_session * s) { - if (s == nullptr) { - return 0; - } - if (!s->batch_results.empty()) { - return static_cast(s->batch_results.size()); - } - return s->has_result ? 1 : 0; -} - -BatchResultView batch_result_view(const transcribe_session * s, int i) { - BatchResultView v; - if (s == nullptr || i < 0) { - return v; - } - if (!s->batch_results.empty()) { - if (static_cast(i) >= s->batch_results.size()) { - return v; - } - const auto & rs = s->batch_results[static_cast(i)]; - if (!rs.has_result) { - return v; // individually-failed / empty utterance - } - v.tokens = &rs.tokens; - v.words = &rs.words; - v.segments = &rs.segments; - v.speaker_segments = &rs.speaker_segments; - v.full_text = &rs.full_text; - v.raw_text = &rs.raw_text; - v.detected_language = &rs.detected_language; - v.result_kind = rs.result_kind; - v.valid = true; - return v; - } - if (i == 0 && s->has_result) { - v.tokens = &s->tokens; - v.words = &s->words; - v.segments = &s->segments; - v.speaker_segments = &s->speaker_segments; - v.full_text = &s->full_text; - v.raw_text = &s->raw_text; - v.detected_language = &s->detected_language; - v.result_kind = s->result_kind; - v.valid = true; - } - return v; -} - -} // namespace - -extern "C" int transcribe_batch_n_results(const struct transcribe_session * session) { - return batch_result_count(session); -} - -extern "C" transcribe_status transcribe_batch_status(const struct transcribe_session * session, int i) { - if (session == nullptr || i < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (!session->batch_results.empty()) { - if (static_cast(i) >= session->batch_results.size()) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - return session->batch_results[static_cast(i)].status; - } - if (i == 0 && session->has_result) { - return TRANSCRIBE_OK; - } - return TRANSCRIBE_ERR_INVALID_ARG; -} - -extern "C" const char * transcribe_batch_full_text(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.full_text->c_str() : ""; -} - -extern "C" const char * transcribe_batch_raw_text(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.raw_text->c_str() : ""; -} - -extern "C" const char * transcribe_batch_detected_language(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.detected_language->c_str() : ""; -} - -extern "C" transcribe_timestamp_kind transcribe_batch_returned_timestamp_kind(const struct transcribe_session * session, - int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.result_kind : TRANSCRIBE_TIMESTAMPS_NONE; -} - -extern "C" int transcribe_batch_n_segments(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.segments->size()) : 0; -} - -extern "C" int transcribe_batch_n_words(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.words->size()) : 0; -} - -extern "C" int transcribe_batch_n_tokens(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.tokens->size()) : 0; -} - -extern "C" transcribe_status transcribe_batch_get_segment(const struct transcribe_session * session, - int i, - int j, - struct transcribe_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.segments->size()) { - return TRANSCRIBE_OK; - } - const auto & s = (*v.segments)[static_cast(j)]; - transcribe_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.first_word = s.first_word; - staged.n_words = s.n_words; - staged.first_token = s.first_token; - staged.n_tokens = s.n_tokens; - staged.text = s.text.c_str(); - staged.speaker_id = s.speaker_id; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_word(const struct transcribe_session * session, - int i, - int j, - struct transcribe_word * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_word zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.words->size()) { - return TRANSCRIBE_OK; - } - const auto & w = (*v.words)[static_cast(j)]; - transcribe_word staged{}; - staged.struct_size = caller_size; - staged.t0_ms = w.t0_ms; - staged.t1_ms = w.t1_ms; - staged.seg_index = w.seg_index; - staged.first_token = w.first_token; - staged.n_tokens = w.n_tokens; - staged.text = w.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_token(const struct transcribe_session * session, - int i, - int j, - struct transcribe_token * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_token zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.tokens->size()) { - return TRANSCRIBE_OK; - } - const auto & t = (*v.tokens)[static_cast(j)]; - transcribe_token staged{}; - staged.struct_size = caller_size; - staged.id = t.id; - staged.p = t.p; - staged.t0_ms = t.t0_ms; - staged.t1_ms = t.t1_ms; - staged.seg_index = t.seg_index; - staged.word_index = t.word_index; - staged.text = t.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" int transcribe_batch_n_speaker_segments(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.speaker_segments->size()) : 0; -} - -extern "C" transcribe_status transcribe_batch_get_speaker_segment(const struct transcribe_session * session, - int i, - int j, - struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.speaker_segments->size()) { - return TRANSCRIBE_OK; - } - const auto & s = (*v.speaker_segments)[static_cast(j)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_timings(const struct transcribe_session * session, - int i, - struct transcribe_timings * out) { - if (session == nullptr || out == nullptr || i < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { - return st; - } - int64_t mel_us = 0, enc_us = 0, dec_us = 0; - if (!session->batch_results.empty()) { - if (static_cast(i) >= session->batch_results.size()) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const auto & rs = session->batch_results[static_cast(i)]; - mel_us = rs.t_mel_us; - enc_us = rs.t_encode_us; - dec_us = rs.t_decode_us; - } else if (i == 0) { - mel_us = session->t_mel_us; - enc_us = session->t_encode_us; - dec_us = session->t_decode_us; - } else { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const uint64_t caller_size = out->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(mel_us) / 1000.0f; - staged.encode_ms = static_cast(enc_us) / 1000.0f; - staged.decode_ms = static_cast(dec_us) / 1000.0f; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - // C ABI forwarders. Entry points that allocate, reach ggml/driver state, // or transfer ownership route through api_guard_* here. Other public symbols // above are kept nothrow by construction: pure POD/c_str reads, prefix @@ -3359,87 +1001,7 @@ extern "C" void transcribe_model_free(struct transcribe_model * model) { api_guard_void("transcribe_model_free", [&] { transcribe_model_free_impl(model); }); } -extern "C" transcribe_status transcribe_session_init(struct transcribe_model * model, - const struct transcribe_session_params * params, - struct transcribe_session ** out_session) { - const transcribe_status st = api_guard_status( - "transcribe_session_init", [&] { return transcribe_session_init_impl(model, params, out_session); }); - // Boundary-owned postcondition: failure => *out_session == NULL. - if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { - transcribe_session_free(*out_session); - *out_session = nullptr; - } - return st; -} - -extern "C" void transcribe_session_free(struct transcribe_session * session) { - api_guard_void("transcribe_session_free", [&] { transcribe_session_free_impl(session); }); -} - -extern "C" transcribe_status transcribe_open(const char * path, - const struct transcribe_model_load_params * load_params, - const struct transcribe_session_params * session_params, - struct transcribe_session ** out_session) { - const transcribe_status st = api_guard_status( - "transcribe_open", [&] { return transcribe_open_impl(path, load_params, session_params, out_session); }); - // Boundary-owned postcondition: failure => *out_session == NULL. - if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { - transcribe_session_free(*out_session); - *out_session = nullptr; - } - return st; -} - -extern "C" transcribe_status transcribe_run(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params) { - return api_guard_status("transcribe_run", [&] { return transcribe_run_impl(session, pcm, n_samples, params); }); -} - -extern "C" transcribe_status transcribe_run_batch(struct transcribe_session * session, - const float * const * pcm, - const int * n_samples, - int n, - const struct transcribe_run_params * params) { - return api_guard_status("transcribe_run_batch", - [&] { return transcribe_run_batch_impl(session, pcm, n_samples, n, params); }); -} - -extern "C" transcribe_status transcribe_stream_begin(struct transcribe_session * session, - const struct transcribe_run_params * run_params, - const struct transcribe_stream_params * stream_params) { - return api_guard_status("transcribe_stream_begin", - [&] { return transcribe_stream_begin_impl(session, run_params, stream_params); }); -} - -extern "C" transcribe_status transcribe_stream_feed(struct transcribe_session * session, - const float * pcm, - int n_samples, - struct transcribe_stream_update * update) { - return api_guard_status("transcribe_stream_feed", - [&] { return transcribe_stream_feed_impl(session, pcm, n_samples, update); }); -} - -extern "C" transcribe_status transcribe_stream_finalize(struct transcribe_session * session, - struct transcribe_stream_update * update) { - return api_guard_status("transcribe_stream_finalize", - [&] { return transcribe_stream_finalize_impl(session, update); }); -} - -extern "C" void transcribe_stream_reset(struct transcribe_session * session) { - api_guard_void("transcribe_stream_reset", [&] { transcribe_stream_reset_impl(session); }); -} - extern "C" transcribe_device_t transcribe_model_device(const struct transcribe_model * model) { return api_guard_value("transcribe_model_device", static_cast(nullptr), [&] { return transcribe_model_device_impl(model); }); } - -extern "C" int transcribe_tokenize(const struct transcribe_model * model, - const char * text, - int32_t * tokens, - size_t n_max) { - return api_guard_value("transcribe_tokenize", INT_MIN, - [&] { return transcribe_tokenize_impl(model, text, tokens, n_max); }); -} From f587fe1922d2d5746de0b7a0f742c4bf6c31f051 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 20:37:27 +0800 Subject: [PATCH 02/51] roles: SessionCore base, model role bits, load-time role validation Internal groundwork for per-role sessions; no public API change. - transcribe::SessionCore (src/transcribe-session-core.h) holds the role-agnostic session members: model, n_threads, stage timings, abort callback / poll_abort, compute scratch and release_scratch. transcribe_session derives from it; families keep ctx->sched etc. - SpeakerSegmentEntry is hoisted to transcribe::SpeakerSegmentEntry, with an alias on transcribe_session so existing references compile unchanged. - transcribe_model::roles (internal k_role_asr / k_role_diarize bits). Families that leave it 0 and have ASR hooks default to ASR, so no family changes. - Arch gains a trailing `diarize` ops-table pointer (DiarizeOps is defined with the diarize role). ASR hooks stay flat. - transcribe::resolve_roles runs after every successful load (GGUF and whisper .bin) and fails the load with NOT_IMPLEMENTED when the mask is empty, has unknown bits, or names a role the arch cannot serve. Covered by tests/role_resolve_unit.cpp. Verified: exported symbols and abihash unchanged; ctest green. On CPU, transcribe-cli output matches main (timing lines excluded) for whisper (GGUF, .bin, initial prompt), parakeet (offline, -o, batch, batch-jsonl, cache-aware and buffered streaming), sortformer (speaker segments via batch-jsonl), multitalker --diarize, moonshine streaming, qwen3_asr (vocabulary + prompt), canary --no-pnc, sensevoice --no-itn, gigaam, and cohere with --n-ctx / --kv-type. Real-model ctest: sortformer_stream_ext_unit and the other configured smokes pass; the failures seen are identical on main. --- src/transcribe-arch.cpp | 35 +++++++++ src/transcribe-arch.h | 13 ++++ src/transcribe-model.cpp | 8 ++- src/transcribe-model.h | 12 ++++ src/transcribe-session-core.h | 131 ++++++++++++++++++++++++++++++++++ src/transcribe-session.h | 112 +++-------------------------- src/transcribe.cpp | 17 +++-- tests/CMakeLists.txt | 17 +++++ tests/role_resolve_unit.cpp | 112 +++++++++++++++++++++++++++++ 9 files changed, 346 insertions(+), 111 deletions(-) create mode 100644 src/transcribe-session-core.h create mode 100644 tests/role_resolve_unit.cpp diff --git a/src/transcribe-arch.cpp b/src/transcribe-arch.cpp index fb768cbac..166b2c638 100644 --- a/src/transcribe-arch.cpp +++ b/src/transcribe-arch.cpp @@ -6,6 +6,9 @@ #include "transcribe-arch.h" +#include "transcribe-log.h" +#include "transcribe-model.h" + #include namespace transcribe { @@ -112,4 +115,36 @@ const Arch * find_arch(const char * name) { return nullptr; } +transcribe_status resolve_roles(transcribe_model * model) { + if (model == nullptr || model->arch == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + const Arch & arch = *model->arch; + const char * name = arch.name != nullptr ? arch.name : "(unknown)"; + const bool has_asr = arch.init_context != nullptr && arch.run != nullptr; + const bool has_diarize = arch.diarize != nullptr; + + if (model->roles == 0 && has_asr) { + model->roles = k_role_asr; + } + + const uint32_t known = k_role_asr | k_role_diarize; + const char * why = nullptr; + if (model->roles == 0) { + why = "serves no role"; + } else if ((model->roles & ~known) != 0) { + why = "sets an unknown role bit"; + } else if ((model->roles & k_role_asr) != 0 && !has_asr) { + why = "sets the ASR role without init_context / run hooks"; + } else if ((model->roles & k_role_diarize) != 0 && !has_diarize) { + why = "sets the DIARIZE role without a diarize ops table"; + } + if (why != nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: arch '%s' %s (roles 0x%x)", name, why, + static_cast(model->roles)); + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + return TRANSCRIBE_OK; +} + } // namespace transcribe diff --git a/src/transcribe-arch.h b/src/transcribe-arch.h index c88d63801..d2a4f28d4 100644 --- a/src/transcribe-arch.h +++ b/src/transcribe-arch.h @@ -14,6 +14,7 @@ namespace transcribe { class Loader; +struct DiarizeOps; // Per-family trait. Function pointers may be null when an entry point is // not yet implemented; the central dispatch converts null entries into @@ -129,10 +130,22 @@ struct Arch { // out of the guarantee). NULL is skipped; the generic header-size + kind // checks remain in force. transcribe_status (*run_validate)(const struct transcribe_session * ctx, const transcribe_run_params * params); + + // Per-role ops tables for the non-ASR roles. nullptr = the arch does not + // implement the role. The ASR hooks above stay flat on the trait. + const DiarizeOps * diarize; }; // Look up an architecture by name. Returns nullptr if no registered // family matches. const Arch * find_arch(const char * name); +// Called once on a freshly loaded model, before it reaches the caller. +// Defaults roles to ASR when the family left it 0 and the arch has ASR hooks, +// then checks that the mask is non-zero, has no unknown bits, and that every +// set role is backed by the arch (init_context + run for ASR, a non-null ops +// table otherwise). A violation is a family wiring bug: it is logged and the +// load fails with TRANSCRIBE_ERR_NOT_IMPLEMENTED. +transcribe_status resolve_roles(struct transcribe_model * model); + } // namespace transcribe diff --git a/src/transcribe-model.cpp b/src/transcribe-model.cpp index ea14c46ec..8bf79f23c 100644 --- a/src/transcribe-model.cpp +++ b/src/transcribe-model.cpp @@ -12,16 +12,18 @@ transcribe_model::~transcribe_model() = default; // Scheduler first, then the compute context (release_compute_scratch keeps -// that order). Runs after the derived destructor; see the header note. -transcribe_session::~transcribe_session() { +// that order). Runs after the derived destructors; see the header note. +transcribe::SessionCore::~SessionCore() { transcribe::release_compute_scratch(sched, compute_ctx); } -void transcribe_session::release_scratch() noexcept { +void transcribe::SessionCore::release_scratch() noexcept { transcribe::release_compute_scratch(sched, compute_ctx); on_scratch_released(); } +transcribe_session::~transcribe_session() = default; + void transcribe_session::clear_result() { tokens.clear(); words.clear(); diff --git a/src/transcribe-model.h b/src/transcribe-model.h index 9f6ad7b7b..5dcc93cb7 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -23,6 +23,12 @@ namespace transcribe { struct Arch; class Tokenizer; +// Role bits for transcribe_model::roles: the kinds of work a loaded model +// can do. Internal until the public transcribe_role enum lands; the values +// match it. +constexpr uint32_t k_role_asr = 1u << 0; +constexpr uint32_t k_role_diarize = 1u << 1; + // Transparent comparator keeps transcribe_model_meta_val_str allocation-free. // Loader and model must use the same map type for assignment. using MetaMap = std::map>; @@ -41,6 +47,12 @@ struct transcribe_model { // for per-call dispatch (init_context, run). const transcribe::Arch * arch = nullptr; + // Bitmask of transcribe::k_role_* this loaded model serves. Set by the + // family's load() when it serves anything other than ASR alone; left 0, + // it defaults to ASR for an arch with ASR hooks. Validated against the + // arch at load (transcribe::resolve_roles). Immutable after load. + uint32_t roles = 0; + // Identification, both surfaced via the public string accessors. // variant is whatever the family decided (loader leaves it empty if // stt.variant was absent; the family supplies a default). diff --git a/src/transcribe-session-core.h b/src/transcribe-session-core.h new file mode 100644 index 000000000..b23a22241 --- /dev/null +++ b/src/transcribe-session-core.h @@ -0,0 +1,131 @@ +// transcribe-session-core.h - internal base shared by every role session. +// +// SessionCore holds what is not specific to any role: the borrowed model, +// the thread count, the per-run ggml compute scratch, the abort callback and +// the stage timings. transcribe_session (ASR) derives from it, and so does +// each role's session type, so scratch release (Handy #2000), cancellation +// and timings behave the same for every role. Families keep reaching these +// members through their derived context (ctx->sched, ctx->poll_abort(), ...). + +#pragma once + +#include "transcribe.h" + +#include +#include + +struct transcribe_model; + +// ggml handle types for the base-owned compute scratch. Forward-declared so +// this internal header does not pull the ggml headers into every includer +// (the public transcribe.h never sees them); transcribe-backend.h uses the +// same pattern for ggml_backend_t. +struct ggml_context; +struct ggml_backend_sched; +typedef struct ggml_backend_sched * ggml_backend_sched_t; + +namespace transcribe { + +// "Who spoke when" rows (diarization). Populated only when a run +// resolves diarize ON for a supporting family; may overlap in time. +// t0_ms == t1_ms == 0 means the family attributes text but has no +// timing information for the turn. +struct SpeakerSegmentEntry { + int64_t t0_ms = 0; + int64_t t1_ms = 0; + int32_t speaker_id = 0; // 1-based + float p = NAN; +}; + +struct SessionCore { + // The model this session was constructed from. Borrowed pointer: + // the caller is required (per the public threading contract) to keep + // the model alive for the lifetime of every derived session. + transcribe_model * model = nullptr; + + // Cached n_threads value the caller passed at init time. 0 means + // "library picks a sensible default" (matches the factory). + int n_threads = 0; + + // Per-call timings, populated by the most recent transcribe_run. + // Surfaced via the public transcribe_get_timings accessor; reset + // by transcribe_reset_timings. + int64_t t_mel_us = 0; + int64_t t_encode_us = 0; + int64_t t_decode_us = 0; + + // Abort / cancellation (set via transcribe_set_abort_callback). + // run() drivers call poll_abort() at chunk / decode-step boundaries; + // a callback returning true sets was_aborted and the run returns + // TRANSCRIBE_ERR_ABORTED with partial segments preserved. was_aborted + // is cleared at the top of every transcribe_run, NOT by clear_result + // (the partial result may be deliberately retained). + transcribe_abort_callback abort_cb = nullptr; + void * abort_userdata = nullptr; + bool was_aborted = false; + + bool poll_abort() { + if (abort_cb != nullptr && abort_cb(abort_userdata)) { + was_aborted = true; + return true; + } + return false; + } + + // Per-run ggml compute scratch, owned by the base so every family + // releases it the same way and none can forget to: the backend + // scheduler (whose graph allocator only ever grows) and the no_alloc + // graph context. Families create both lazily inside their run / stream + // hooks and use them directly; the base frees them in release_scratch + // and in its destructor (scheduler first, then context). + ggml_backend_sched_t sched = nullptr; + ggml_context * compute_ctx = nullptr; + + // Release the per-run ggml compute scratch (sched, then compute_ctx; + // see transcribe::release_compute_scratch), then let the family drop + // pointers that lived in them (on_scratch_released). + // The dispatcher calls this after every offline transcribe_run or + // transcribe_run_batch that passes pre-clear validation and reaches its + // commit point, whether family execution succeeds, fails, or throws. + // Without this, a single long utterance would pin the scheduler's + // high-water mark in backend compute memory for the session's lifetime + // (Handy #2000). Families re-create the scheduler lazily on the next run. + // The measured recreation cost is about 1 ms per run on Metal and up to + // about 10 ms on CPU for families that reserve a worst-case decoder + // workspace each run, such as Canary and Cohere. + // + // Host-side vectors that scale with input length, such as mel buffers, + // encoder host copies, and positional banks, deliberately retain their + // capacity. Family KV caches sized by the last batch are also retained; + // this hook releases only the ggml scheduler and compute context. + // + // Streaming entry points do not invoke this hook. A streaming session + // keeps its scheduler until a later offline run or session destruction. + // Parakeet and Voxtral Realtime use per-chunk bounded stream workspaces; + // Moonshine Streaming's decode graph cross-attends over the complete + // committed stream, so its workspace grows with total stream length. + // Must not throw. Non-virtual: a family cannot opt out of the release. + void release_scratch() noexcept; + + SessionCore() = default; + // Frees sched / compute_ctx after the derived destructors have run. The + // scheduler owns only its own allocator buffers and references + // model-owned backends, so freeing it after the family's KV caches and + // stream buffers is order-independent. + virtual ~SessionCore(); + + SessionCore(const SessionCore &) = delete; + SessionCore & operator=(const SessionCore &) = delete; + SessionCore(SessionCore &&) = delete; + SessionCore & operator=(SessionCore &&) = delete; + + protected: + // Family hook, called by release_scratch after sched / compute_ctx are + // freed: null out tensors borrowed from the freed context (encoder_out) + // or reset capacity bookkeeping (whisper compute_ctx_size). Most + // families need nothing. Not called from the base destructor because + // derived members are already destroyed by then. Must not throw. + virtual void on_scratch_released() noexcept {} +}; + +} // namespace transcribe diff --git a/src/transcribe-session.h b/src/transcribe-session.h index e3984b7e3..8f4ff9646 100644 --- a/src/transcribe-session.h +++ b/src/transcribe-session.h @@ -10,25 +10,15 @@ #pragma once +#include "transcribe-session-core.h" #include "transcribe.h" -#include #include #include #include #include #include -struct transcribe_model; - -// ggml handle types for the base-owned compute scratch. Forward-declared so -// this internal header does not pull the ggml headers into every includer -// (the public transcribe.h never sees them); transcribe-backend.h uses the -// same pattern for ggml_backend_t. -struct ggml_context; -struct ggml_backend_sched; -typedef struct ggml_backend_sched * ggml_backend_sched_t; - // Read transcribe_session_params::n_ctx with a struct_size guard. n_ctx is // a trailing field appended after kv_type, so an older caller's smaller // struct may not include it; in that case (or a NULL params) the default 0 @@ -47,12 +37,9 @@ inline int32_t transcribe_session_params_n_ctx(const struct transcribe_session_p return params->n_ctx; } -struct transcribe_session { - // The model this session was constructed from. Borrowed pointer: - // the caller is required (per the public threading contract) to keep - // the model alive for the lifetime of every derived session. - transcribe_model * model = nullptr; - +// Common members (model, n_threads, timings, abort callback, compute +// scratch) live on transcribe::SessionCore; see transcribe-session-core.h. +struct transcribe_session : transcribe::SessionCore { // True only for sessions created via transcribe_open(), which loads // and therefore owns its model. Both transcribe_session_free() and // transcribe_close() (now an alias) read this flag and free the @@ -61,10 +48,6 @@ struct transcribe_session { // freed independently by the caller via transcribe_model_free(). bool owns_model = false; - // Cached n_threads value the caller passed at init time. 0 means - // "library picks a sensible default" (matches the factory). - int n_threads = 0; - // Cached n_ctx value the caller passed at init time (decoder context // cap in tokens). 0 means "use the model's true maximum from GGUF". // A positive value lowers the ceiling to bound KV-cache memory and is @@ -81,13 +64,6 @@ struct transcribe_session { // for byte accounting only F32 differs (4 bytes/elem vs 2). transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; - // Per-call timings, populated by the most recent transcribe_run. - // Surfaced via the public transcribe_get_timings accessor; reset - // by transcribe_reset_timings. - int64_t t_mel_us = 0; - int64_t t_encode_us = 0; - int64_t t_decode_us = 0; - // Result storage (family-agnostic; populated by per-family run()). // A flat backing array per level (segments / words / tokens) with // forward/backward indices between levels. result_kind is the @@ -125,16 +101,9 @@ struct transcribe_session { int32_t speaker_id = 0; // 1-based; 0 = no attribution }; - // "Who spoke when" rows (diarization). Populated only when a run - // resolves diarize ON for a supporting family; may overlap in time. - // t0_ms == t1_ms == 0 means the family attributes text but has no - // timing information for the turn. - struct SpeakerSegmentEntry { - int64_t t0_ms = 0; - int64_t t1_ms = 0; - int32_t speaker_id = 0; // 1-based - float p = NAN; - }; + // Hoisted to transcribe::SpeakerSegmentEntry so every session type + // that reports speaker segments shares one row type. + using SpeakerSegmentEntry = transcribe::SpeakerSegmentEntry; std::vector tokens; std::vector words; @@ -218,24 +187,6 @@ struct transcribe_session { transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; bool has_result = false; - // Abort / cancellation (set via transcribe_set_abort_callback). - // run() drivers call poll_abort() at chunk / decode-step boundaries; - // a callback returning true sets was_aborted and the run returns - // TRANSCRIBE_ERR_ABORTED with partial segments preserved. was_aborted - // is cleared at the top of every transcribe_run, NOT by clear_result - // (the partial result may be deliberately retained). - transcribe_abort_callback abort_cb = nullptr; - void * abort_userdata = nullptr; - bool was_aborted = false; - - bool poll_abort() { - if (abort_cb != nullptr && abort_cb(abort_userdata)) { - was_aborted = true; - return true; - } - return false; - } - // Set by a run() driver when decode stops at the model's context / // position cap before end-of-stream (output truncated; partial result // retained). Surfaced via transcribe_was_truncated(); cleared at the @@ -306,58 +257,11 @@ struct transcribe_session { void clear_result(); - // Per-run ggml compute scratch, owned by the base so every family - // releases it the same way and none can forget to: the backend - // scheduler (whose graph allocator only ever grows) and the no_alloc - // graph context. Families create both lazily inside their run / stream - // hooks and use them directly; the base frees them in release_scratch - // and in its destructor (scheduler first, then context). - ggml_backend_sched_t sched = nullptr; - ggml_context * compute_ctx = nullptr; - - // Release the per-run ggml compute scratch (sched, then compute_ctx; - // see transcribe::release_compute_scratch), then let the family drop - // pointers that lived in them (on_scratch_released). - // The dispatcher calls this after every offline transcribe_run or - // transcribe_run_batch that passes pre-clear validation and reaches its - // commit point, whether family execution succeeds, fails, or throws. - // Without this, a single long utterance would pin the scheduler's - // high-water mark in backend compute memory for the session's lifetime - // (Handy #2000). Families re-create the scheduler lazily on the next run. - // The measured recreation cost is about 1 ms per run on Metal and up to - // about 10 ms on CPU for families that reserve a worst-case decoder - // workspace each run, such as Canary and Cohere. - // - // Host-side vectors that scale with input length, such as mel buffers, - // encoder host copies, and positional banks, deliberately retain their - // capacity. Family KV caches sized by the last batch are also retained; - // this hook releases only the ggml scheduler and compute context. - // - // Streaming entry points do not invoke this hook. A streaming session - // keeps its scheduler until a later offline run or session destruction. - // Parakeet and Voxtral Realtime use per-chunk bounded stream workspaces; - // Moonshine Streaming's decode graph cross-attends over the complete - // committed stream, so its workspace grows with total stream length. - // Must not throw. Non-virtual: a family cannot opt out of the release. - void release_scratch() noexcept; - transcribe_session() = default; - // Frees sched / compute_ctx after the derived destructor has run. The - // scheduler owns only its own allocator buffers and references - // model-owned backends, so freeing it after the family's KV caches and - // stream buffers is order-independent. - virtual ~transcribe_session(); + ~transcribe_session() override; transcribe_session(const transcribe_session &) = delete; transcribe_session & operator=(const transcribe_session &) = delete; transcribe_session(transcribe_session &&) = delete; transcribe_session & operator=(transcribe_session &&) = delete; - - protected: - // Family hook, called by release_scratch after sched / compute_ctx are - // freed: null out tensors borrowed from the freed context (encoder_out) - // or reset capacity bookkeeping (whisper compute_ctx_size). Most - // families need nothing. Not called from the base destructor because - // derived members are already destroyed by then. Must not throw. - virtual void on_scratch_released() noexcept {} }; diff --git a/src/transcribe.cpp b/src/transcribe.cpp index e28d692e5..f89c22be7 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -846,7 +846,11 @@ static transcribe_status transcribe_model_load_file_impl(const char * // to the whisper .bin adapter, which validates the hparams as // whisper-shaped (rejecting unrelated ggml-magic files like Silero VAD). if (magic == 0x67676d6cu) { - return transcribe::whisper::load_from_bin(path, params, out_model); + const transcribe_status st = transcribe::whisper::load_from_bin(path, params, out_model); + if (st != TRANSCRIBE_OK || *out_model == nullptr) { + return st; + } + return transcribe::resolve_roles(*out_model); } // Header-only GGUF inspection. The Loader is stack-allocated; if @@ -881,10 +885,15 @@ static transcribe_status transcribe_model_load_file_impl(const char * // here instead of in every per-family handler. `variant` is owned by the // family (it may default it when stt.variant was absent) and stays on its // own accessor; this map is the generic general.* / display surface. - if (st == TRANSCRIBE_OK && out_model != nullptr && *out_model != nullptr) { - (*out_model)->meta = loader.meta(); + if (st != TRANSCRIBE_OK || *out_model == nullptr) { + return st; } - return st; + (*out_model)->meta = loader.meta(); + + // Role bits are validated against the arch here, for every family, so a + // model never reaches the caller with a role it cannot serve. On failure + // the forwarder frees the model. + return transcribe::resolve_roles(*out_model); } static void transcribe_model_free_impl(struct transcribe_model * model) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 893375542..93da81287 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -318,6 +318,23 @@ transcribe_apply_warnings(transcribe_run_dispatch_unit) add_test(NAME transcribe_run_dispatch_unit COMMAND transcribe_run_dispatch_unit) +# ----------------------------------------------------------------------------- +# Load-time role validation +# ----------------------------------------------------------------------------- + +add_executable(transcribe_role_resolve_unit + role_resolve_unit.cpp) + +target_link_libraries(transcribe_role_resolve_unit PRIVATE transcribe) + +target_include_directories(transcribe_role_resolve_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_role_resolve_unit) + +add_test(NAME transcribe_role_resolve_unit + COMMAND transcribe_role_resolve_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- diff --git a/tests/role_resolve_unit.cpp b/tests/role_resolve_unit.cpp new file mode 100644 index 000000000..cb49bda21 --- /dev/null +++ b/tests/role_resolve_unit.cpp @@ -0,0 +1,112 @@ +// role_resolve_unit.cpp - load-time role validation (transcribe::resolve_roles). + +#include "transcribe-arch.h" +#include "transcribe-model.h" +#include "transcribe.h" + +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +transcribe_status fake_init_context(transcribe_model *, const transcribe_session_params *, transcribe_session **) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; +} + +transcribe_status fake_run(transcribe_session *, const float *, int, const transcribe_run_params *) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; +} + +const transcribe::Arch k_asr_arch = { + /* .name = */ "fake_asr", + /* .load = */ nullptr, + /* .init_context = */ fake_init_context, + /* .run = */ fake_run, +}; + +// init_context without run: not a usable ASR arch. +const transcribe::Arch k_half_asr_arch = { + /* .name = */ "fake_half_asr", + /* .load = */ nullptr, + /* .init_context = */ fake_init_context, +}; + +const transcribe::Arch k_no_role_arch = { + /* .name = */ "fake_no_role", +}; + +transcribe_status resolve(const transcribe::Arch & arch, uint32_t roles, uint32_t * out_roles) { + transcribe_model model; + model.arch = &arch; + model.roles = roles; + const transcribe_status st = transcribe::resolve_roles(&model); + *out_roles = model.roles; + return st; +} + +void test_asr_default() { + uint32_t roles = 0; + CHECK(resolve(k_asr_arch, 0, &roles) == TRANSCRIBE_OK); + CHECK(roles == transcribe::k_role_asr); +} + +void test_asr_explicit() { + uint32_t roles = 0; + CHECK(resolve(k_asr_arch, transcribe::k_role_asr, &roles) == TRANSCRIBE_OK); + CHECK(roles == transcribe::k_role_asr); +} + +void test_no_role_rejected() { + uint32_t roles = 0; + CHECK(resolve(k_no_role_arch, 0, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_half_asr_arch, 0, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); +} + +void test_unbacked_role_rejected() { + uint32_t roles = 0; + CHECK(resolve(k_half_asr_arch, transcribe::k_role_asr, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_asr_arch, transcribe::k_role_diarize, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_asr_arch, transcribe::k_role_asr | transcribe::k_role_diarize, &roles) == + TRANSCRIBE_ERR_NOT_IMPLEMENTED); +} + +void test_unknown_bit_rejected() { + uint32_t roles = 0; + CHECK(resolve(k_asr_arch, transcribe::k_role_asr | (1u << 31), &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); +} + +void test_null_model() { + CHECK(transcribe::resolve_roles(nullptr) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + transcribe_model model; + CHECK(transcribe::resolve_roles(&model) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); // the rejections log at ERROR + + test_asr_default(); + test_asr_explicit(); + test_no_role_rejected(); + test_unbacked_role_rejected(); + test_unknown_bit_rejected(); + test_null_model(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("role_resolve_unit: ok\n"); + return EXIT_SUCCESS; +} From c57038d671ba3576e860b7fc29b218be606e3731 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 20:50:08 +0800 Subject: [PATCH 03/51] cli: split transcribe-cli into main.cpp + asr.cpp Pure move. main.cpp keeps argument parsing, usage, --list-devices, the log sink and -o setup; asr.cpp gets the ASR JSON helpers, apply_prompting, and the batch and single-file bodies of main(), moved verbatim into transcribe_cli::run_asr_batch / run_asr_file. cli.h holds cli_args and the shared json_escape / write_output_file declarations. Later roles add their own driver file next to asr.cpp. Non-move edits: cli.h, include / using lines, the two function wrappers (each declares the output_ok local main() used to), main()'s two calls, and clang-format reflowing one printf after the de-indent. Verified: --help and --list-devices identical to main; on CPU, output matches main (timing lines excluded) for whisper (GGUF, .bin, initial prompt), parakeet (offline, -o, batch, batch-jsonl, cache-aware and buffered streaming), sortformer (batch-jsonl speaker segments), multitalker --diarize, moonshine streaming, qwen3_asr, canary --no-pnc, sensevoice --no-itn --raw-tokens, gigaam, cohere --n-ctx/--kv-type, and the bad-flag / bad-stream-param error paths. --- examples/cli/CMakeLists.txt | 1 + examples/cli/asr.cpp | 951 +++++++++++++++++++++++++++++++ examples/cli/cli.h | 119 ++++ examples/cli/main.cpp | 1065 ++--------------------------------- 4 files changed, 1109 insertions(+), 1027 deletions(-) create mode 100644 examples/cli/asr.cpp create mode 100644 examples/cli/cli.h diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 2b1284cf8..441a0ba27 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -6,6 +6,7 @@ add_executable(transcribe-cli main.cpp + asr.cpp ) target_link_libraries(transcribe-cli diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp new file mode 100644 index 000000000..78e37541e --- /dev/null +++ b/examples/cli/asr.cpp @@ -0,0 +1,951 @@ +// asr.cpp - transcribe-cli ASR driver: batch and single-file transcription +// through the public ASR API (transcribe_run, transcribe_run_batch, +// transcribe_stream_*), plus the JSONL / text output for them. + +#include "cli.h" +#include "transcribe.h" +#include "transcribe/parakeet.h" +#include "transcribe/voxtral_realtime.h" +#include "transcribe/whisper.h" +#include "wav.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using transcribe_cli::cli_args; +using transcribe_cli::json_escape; +using transcribe_cli::write_output_file; + +namespace { + +// Shared row formatter for segments_json / batch_segments_json below. +std::string segment_row_json(const struct transcribe_segment & seg) { + std::string out; + char head[128]; + if (seg.speaker_id > 0) { + std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"text\":\"", + static_cast(seg.t0_ms), static_cast(seg.t1_ms), + static_cast(seg.speaker_id)); + } else { + std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"text\":\"", + static_cast(seg.t0_ms), static_cast(seg.t1_ms)); + } + out += head; + out += json_escape(seg.text != nullptr ? seg.text : ""); + out += "\"}"; + return out; +} + +// Build the ",\"segments\":[...]" fragment when the context has real +// segment rows: segment timestamps, or speaker-attributed turns (which +// may carry no timing — granite's speaker-attribution task). Returns an +// empty string otherwise (the library still populates a single dummy +// entry when the kind is NONE, but with zero timing — don't pollute the +// JSON with it). +std::string segments_json(const transcribe_session * ctx) { + if (transcribe_returned_timestamp_kind(ctx) == TRANSCRIBE_TIMESTAMPS_NONE && + transcribe_n_speaker_segments(ctx) <= 0) { + return {}; + } + const int n_seg = transcribe_n_segments(ctx); + if (n_seg <= 0) { + return {}; + } + std::string out = ",\"segments\":["; + for (int s = 0; s < n_seg; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_get_segment(ctx, s, &seg); + out += segment_row_json(seg); + } + out += "]"; + return out; +} + +// Batch variant: segments JSON for utterance `i` of a transcribe_run_batch +// result, using the indexed transcribe_batch_* accessors. +std::string batch_segments_json(const transcribe_session * ctx, int i) { + if (transcribe_batch_returned_timestamp_kind(ctx, i) == TRANSCRIBE_TIMESTAMPS_NONE && + transcribe_batch_n_speaker_segments(ctx, i) <= 0) { + return {}; + } + const int n_seg = transcribe_batch_n_segments(ctx, i); + if (n_seg <= 0) { + return {}; + } + std::string out = ",\"segments\":["; + for (int s = 0; s < n_seg; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_batch_get_segment(ctx, i, s, &seg); + out += segment_row_json(seg); + } + out += "]"; + return out; +} + +// ",\"raw_text\":\"...\"" fragment: the model's pre-cleanup decode +// (transcribe_raw_text). Emitted only when it differs from the clean text, +// so JSONL stays compact for families with no post-processing. +std::string raw_text_json(const char * raw, const char * clean) { + if (raw == nullptr || raw[0] == '\0' || (clean != nullptr && std::strcmp(raw, clean) == 0)) { + return {}; + } + std::string out = ",\"raw_text\":\""; + out += json_escape(raw); + out += "\""; + return out; +} + +// A decode cut short before end-of-stream (budget or repetition stop): non-OK, +// but the partial transcript is preserved (see docs/input-limits.md). +bool is_cut_short(transcribe_status st) { + return st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED || st == TRANSCRIBE_ERR_OUTPUT_REPETITION; +} + +const char * cut_short_label(transcribe_status st) { + return st == TRANSCRIBE_ERR_OUTPUT_REPETITION ? "stopped repeating" : "truncated"; +} + +// ",\"speakers\":[...]" fragment: the "who spoke when" rows. Emitted only +// when the run produced speaker segments. p is omitted unless finite +// (NaN — "model provides no confidence" — is not representable in JSON). +std::string speaker_row_json(const struct transcribe_speaker_segment & row) { + char buf[160]; + if (std::isfinite(row.p)) { + std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"p\":%.4f}", + static_cast(row.t0_ms), static_cast(row.t1_ms), + static_cast(row.speaker_id), static_cast(row.p)); + } else { + std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d}", + static_cast(row.t0_ms), static_cast(row.t1_ms), + static_cast(row.speaker_id)); + } + return buf; +} + +std::string speakers_json(const transcribe_session * ctx) { + const int n = transcribe_n_speaker_segments(ctx); + if (n <= 0) { + return {}; + } + std::string out = ",\"speakers\":["; + for (int s = 0; s < n; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + (void) transcribe_get_speaker_segment(ctx, s, &row); + out += speaker_row_json(row); + } + out += "]"; + return out; +} + +std::string batch_speakers_json(const transcribe_session * ctx, int i) { + const int n = transcribe_batch_n_speaker_segments(ctx, i); + if (n <= 0) { + return {}; + } + std::string out = ",\"speakers\":["; + for (int s = 0; s < n; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + (void) transcribe_batch_get_speaker_segment(ctx, i, s, &row); + out += speaker_row_json(row); + } + out += "]"; + return out; +} + +// Point rp's generic prompting fields at args' storage; vocabulary_ptrs +// backs rp.vocabulary and must outlive the run. +void apply_prompting(const cli_args & args, transcribe_run_params & rp, std::vector & vocabulary_ptrs) { + if (args.instruct) { + rp.task = TRANSCRIBE_TASK_INSTRUCT; + } + vocabulary_ptrs.clear(); + for (const std::string & term : args.vocabulary) { + vocabulary_ptrs.push_back(term.c_str()); + } + rp.vocabulary = vocabulary_ptrs.empty() ? nullptr : vocabulary_ptrs.data(); + rp.n_vocabulary = static_cast(vocabulary_ptrs.size()); + rp.prompt = args.prompt.empty() ? nullptr : args.prompt.c_str(); + rp.prefix = args.prefix.empty() ? nullptr : args.prefix.c_str(); +} + +} // namespace + +namespace transcribe_cli { + +int run_asr_batch(const cli_args & args, std::ofstream * output) { + bool output_ok = true; + if (args.model_path.empty()) { + std::fprintf(stderr, "error: --batch requires --model\n"); + return EXIT_FAILURE; + } + + std::vector wav_paths; + { + std::ifstream fin(args.batch_file); + if (!fin) { + std::fprintf(stderr, "error: cannot open batch file %s\n", args.batch_file.c_str()); + return EXIT_FAILURE; + } + std::string line; + while (std::getline(fin, line)) { + while (!line.empty() && + (line.back() == '\n' || line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { + line.pop_back(); + } + if (!line.empty()) { + wav_paths.push_back(line); + } + } + } + if (wav_paths.empty()) { + std::fprintf(stderr, "error: batch file is empty\n"); + return EXIT_FAILURE; + } + if (!args.batch_jsonl) { + std::fprintf(stderr, "batch: %zu files\n", wav_paths.size()); + } + + struct transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } + struct transcribe_model * model = nullptr; + const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); + if (load_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "model load: %s\n", transcribe_status_string(load_st)); + return EXIT_FAILURE; + } + + struct transcribe_session_params cp; + transcribe_session_params_init(&cp); + cp.n_threads = args.n_threads; + cp.n_ctx = args.n_ctx; + cp.kv_type = args.kv_type; + struct transcribe_session * ctx = nullptr; + const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); + if (init_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + struct transcribe_run_params rp; + transcribe_run_params_init(&rp); + if (args.translate) { + rp.task = TRANSCRIBE_TASK_TRANSLATE; + } + std::vector vocabulary_ptrs; + apply_prompting(args, rp, vocabulary_ptrs); + if (!args.language.empty()) { + rp.language = args.language.c_str(); + } + if (!args.target_language.empty()) { + rp.target_language = args.target_language.c_str(); + } + rp.timestamps = args.timestamps; + rp.spec_k_drafts = args.spec_k_drafts; + + if (args.itn_set) { + rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + } + if (args.canary_pnc_set) { + rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + } + if (args.diarize_set) { + rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + } + + // Whisper run extension. Allocated outside rp's scope so its + // bytes outlive the per-file loop below; the library copies + // initial_prompt/prompt_tokens before transcribe_run returns, + // but rp aliases &wx.ext for the run call itself. + struct transcribe_whisper_run_ext wx; + transcribe_whisper_run_ext_init(&wx); + if (args.whisper_set) { + if (!args.initial_prompt.empty()) { + wx.initial_prompt = args.initial_prompt.c_str(); + } + wx.condition_on_prev_tokens = args.condition_on_prev_tokens; + if (args.temperature_set) { + wx.temperature = args.temperature; + } + wx.prompt_condition = args.prompt_condition; + if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { + rp.family = &wx.ext; + } + } + + if (args.keep_special_tags) { + rp.keep_special_tags = true; + } + + // Emit a batch header line once, before any per-file output. Carries + // the one-shot load time so downstream WER tooling can record it + // without parsing stderr. Per-file lines follow on subsequent lines. + if (args.batch_jsonl) { + struct transcribe_timings load_tm; + transcribe_timings_init(&load_tm); + (void) transcribe_get_timings(ctx, &load_tm); + std::printf("{\"type\":\"batch_header\",\"load_ms\":%.1f}\n", (double) load_tm.load_ms); + std::fflush(stdout); + } + + int n_ok = 0; + int n_truncated = 0; // result-bearing: hit the generation cap, partial hyp emitted + int n_repeating = 0; // result-bearing: stopped when the output looped, partial hyp emitted + int n_fail = 0; // no usable result (wav load / backend / unsupported / whole-batch) + + // Offline batched path: group up to batch_size utterances into one + // transcribe_run_batch call. Mutually exclusive with the streaming + // path (--stream-chunk-ms), which is per-utterance by construction. + // Per-file JSONL output is byte-identical in shape to the serial + // path so the WER harness (scripts/wer/run.py) consumes either. + const bool use_batched = args.batch_size > 1 && args.stream_chunk_ms <= 0; + if (use_batched) { + const size_t total = wav_paths.size(); + const size_t group = static_cast(args.batch_size); + for (size_t base = 0; base < total; base += group) { + const size_t end = std::min(total, base + group); + + // Load each wav in the group. A wav that fails to load is + // emitted as an error line and excluded from the batch, so + // its failure cannot abort its neighbours. + std::vector> pcms; + std::vector pcm_ptrs; + std::vector n_samps; + std::vector src_index; // -> wav_paths + for (size_t i = base; i < end; ++i) { + std::vector pcm; + std::string wav_err; + if (!transcribe_cli::load_wav_mono_16k(wav_paths[i], pcm, wav_err)) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav_paths[i].c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"wav: %s\"}\n", + file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); + } else { + std::fprintf(stderr, "SKIP %s: %s\n", wav_paths[i].c_str(), wav_err.c_str()); + } + ++n_fail; + std::fflush(stdout); + continue; + } + pcms.push_back(std::move(pcm)); + src_index.push_back(i); + } + for (auto & p : pcms) { + pcm_ptrs.push_back(p.data()); + n_samps.push_back(static_cast(p.size())); + } + if (pcms.empty()) { + continue; + } + + const transcribe_status bst = + transcribe_run_batch(ctx, pcm_ptrs.data(), n_samps.data(), static_cast(pcm_ptrs.size()), &rp); + if (bst != TRANSCRIBE_OK && transcribe_batch_n_results(ctx) == 0) { + // Whole-batch failure (no per-utterance results): emit an + // error line per file in the group and continue. + for (size_t k = 0; k < src_index.size(); ++k) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav_paths[src_index[k]].c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"%s\"}\n", + file_esc.c_str(), json_escape(transcribe_status_string(bst)).c_str()); + } + ++n_fail; + } + std::fflush(stdout); + continue; + } + + for (size_t k = 0; k < src_index.size(); ++k) { + const std::string & wav = wav_paths[src_index[k]]; + const transcribe_status ust = transcribe_batch_status(ctx, static_cast(k)); + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the + // partial transcript is preserved and readable via + // transcribe_batch_full_text (see transcribe.h). Emit it as the hyp + // so downstream tooling scores the partial rather than an empty + // string; the error field below still tags it so the stop stays + // visible. + const bool result_present = ust == TRANSCRIBE_OK || is_cut_short(ust); + const char * text = ""; + if (result_present) { + const char * t = transcribe_batch_full_text(ctx, static_cast(k)); + if (t && *t) { + text = t; + } + } + if (ust == TRANSCRIBE_OK) { + ++n_ok; + } else if (ust == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + ++n_truncated; + } else if (ust == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + ++n_repeating; + } else { + ++n_fail; + } + if (args.batch_jsonl) { + const std::string escaped = json_escape(text); + std::string segments = batch_segments_json(ctx, static_cast(k)); + segments += batch_speakers_json(ctx, static_cast(k)); + segments += raw_text_json(transcribe_batch_raw_text(ctx, static_cast(k)), text); + std::string err_field; + if (ust != TRANSCRIBE_OK) { + err_field = ",\"error\":\""; + err_field += json_escape(transcribe_status_string(ust)); + err_field += "\""; + } + // Per-utterance timings: the batched encoder time is + // amortized across the batch, decode is real per-utt, + // so summing across utterances gives the true encode vs + // decode split for the whole batched run. + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_batch_get_timings(ctx, static_cast(k), &tm); + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"%s\"%s," + "\"mel_ms\":%.1f,\"encode_ms\":%.1f," + "\"decode_ms\":%.1f%s}\n", + file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, + (double) tm.decode_ms, err_field.c_str()); + } else { + std::printf("[%zu/%zu] %s", src_index[k] + 1, total, wav.c_str()); + if (ust == TRANSCRIBE_OK) { + std::printf("\n text: %s\n", text); + } else if (is_cut_short(ust)) { + std::printf(" (%s)\n text: %s\n", cut_short_label(ust), text); + } else { + std::printf(" ERROR: %s\n", transcribe_status_string(ust)); + } + } + output_ok = write_output_file(output, args.output_path, text) && output_ok; + std::fflush(stdout); + } + } + } else { + for (size_t i = 0; i < wav_paths.size(); ++i) { + const std::string & wav = wav_paths[i]; + + std::vector pcm; + std::string wav_err; + if (!transcribe_cli::load_wav_mono_16k(wav, pcm, wav_err)) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"wav: %s\"}\n", + file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); + } else { + std::fprintf(stderr, "SKIP %s: %s\n", wav.c_str(), wav_err.c_str()); + } + ++n_fail; + std::fflush(stdout); + continue; + } + + // Run. When --stream-chunk-ms > 0, drive the streaming API + // for this utterance (begin/feed/finalize) so the WER + // harness can measure cache-aware streaming output. + transcribe_status run_st = TRANSCRIBE_OK; + if (args.stream_chunk_ms > 0) { + struct transcribe_stream_params sp; + transcribe_stream_params_init(&sp); + struct transcribe_parakeet_stream_ext pkt_sp; + transcribe_parakeet_stream_ext_init(&pkt_sp); + struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; + transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); + struct transcribe_voxtral_realtime_stream_ext vx_sp; + transcribe_voxtral_realtime_stream_ext_init(&vx_sp); + const bool want_cache_aware = (args.stream_att_right >= 0); + const bool want_buffered = + args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; + if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { + pkt_sp.att_context_right = args.stream_att_right; + sp.family = &pkt_sp.ext; + } else if (want_buffered && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { + pkt_buf_sp.left_ms = args.stream_buf_left_ms; + pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; + pkt_buf_sp.right_ms = args.stream_buf_right_ms; + sp.family = &pkt_buf_sp.ext; + } else if (args.stream_voxtral_delay != -1 && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { + vx_sp.num_delay_tokens = args.stream_voxtral_delay; + sp.family = &vx_sp.ext; + } + run_st = transcribe_stream_begin(ctx, &rp, &sp); + if (run_st == TRANSCRIBE_OK) { + const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); + size_t pos = 0; + while (pos < pcm.size()) { + const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); + struct transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); + if (run_st != TRANSCRIBE_OK) { + break; + } + pos += take; + } + if (run_st == TRANSCRIBE_OK) { + struct transcribe_stream_update fin_upd; + transcribe_stream_update_init(&fin_upd); + run_st = transcribe_stream_finalize(ctx, &fin_upd); + } + } + } else { + run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + } + + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the partial + // transcript is preserved and readable via transcribe_full_text (see + // transcribe.h). Emit it as the hyp so downstream tooling scores the + // partial rather than an empty string; the error field below still + // tags it so the stop stays visible. + const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st); + const char * text = ""; + if (result_present) { + const char * t = transcribe_full_text(ctx); + if (t && *t) { + text = t; + } + } + if (run_st == TRANSCRIBE_OK) { + ++n_ok; + } else if (run_st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + ++n_truncated; + } else if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + ++n_repeating; + } else { + ++n_fail; + } + + // Output. When run_st != OK we still emit a JSON line so + // batch consumers see one record per input wav, but we tag + // it with the error string. Without this the wav-load error + // path (above) reports errors but transcribe_run failures + // (e.g. unsupported language) produce empty hyp_text with no + // error tag, so downstream tools count them as silent + // successes. + if (args.batch_jsonl) { + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_get_timings(ctx, &tm); + const std::string escaped = json_escape(text); + std::string segments = segments_json(ctx); + segments += speakers_json(ctx); + segments += raw_text_json(transcribe_raw_text(ctx), text); + std::string err_field; + if (run_st != TRANSCRIBE_OK) { + err_field = ",\"error\":\""; + err_field += json_escape(transcribe_status_string(run_st)); + err_field += "\""; + } + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"%s\"%s," + "\"mel_ms\":%.1f,\"encode_ms\":%.1f," + "\"decode_ms\":%.1f%s}\n", + file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, + (double) tm.decode_ms, err_field.c_str()); + } else { + std::printf("[%zu/%zu] %s", i + 1, wav_paths.size(), wav.c_str()); + if (run_st == TRANSCRIBE_OK) { + std::printf("\n text: %s\n", text); + } else if (is_cut_short(run_st)) { + std::printf(" (%s)\n text: %s\n", cut_short_label(run_st), text); + } else { + std::printf(" ERROR: %s\n", transcribe_status_string(run_st)); + } + } + output_ok = write_output_file(output, args.output_path, text) && output_ok; + std::fflush(stdout); + } + } + + if (!args.batch_jsonl) { + std::fprintf(stderr, "batch: %d ok, %d truncated, %d stopped repeating, %d failed out of %zu\n", n_ok, + n_truncated, n_repeating, n_fail, wav_paths.size()); + } + + transcribe_session_free(ctx); + transcribe_model_free(model); + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing and do not fail the batch, but + // hard per-utterance failures must remain visible to automation. + return n_fail > 0 || !output_ok ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int run_asr_file(const cli_args & args, std::ofstream * output) { + bool output_ok = true; + // Single-file mode. + std::vector pcm; + std::string load_err; + if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { + std::fprintf(stderr, "wav: %s\n", load_err.c_str()); + return EXIT_FAILURE; + } + + const double duration_s = static_cast(pcm.size()) / 16000.0; + std::printf("audio: %s\n", args.wav_path.c_str()); + std::printf(" samples: %zu\n", pcm.size()); + std::printf(" duration: %.3f s\n", duration_s); + std::printf(" sample rate 16000 Hz mono float32\n"); + + if (!args.model_path.empty()) { + struct transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } + struct transcribe_model * model = nullptr; + const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); + std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); + if (st != TRANSCRIBE_OK) { + return EXIT_FAILURE; + } + std::printf(" backend: %s\n", transcribe_model_backend(model)); + if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { + std::printf(" name: %s\n", dn); + } + if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { + std::printf(" license: %s\n", lic); + } + + struct transcribe_session_params cp; + transcribe_session_params_init(&cp); + cp.n_threads = args.n_threads; + cp.n_ctx = args.n_ctx; + cp.kv_type = args.kv_type; + struct transcribe_session * ctx = nullptr; + const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); + if (init_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + // Surface the effective input-length limit so it's obvious how much + // audio this session accepts (reflects --n-ctx). 0 means "no practical + // limit", which covers two different families: one that chunks long + // audio internally (FEATURE_LONG_FORM) and one that is genuinely + // unbounded and encodes the clip in a single pass. Distinguish them + // rather than asserting the first. See docs/input-limits.md. + { + struct transcribe_session_limits lim; + transcribe_session_limits_init(&lim); + if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { + if (lim.effective_max_audio_ms > 0) { + std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); + if (lim.effective_n_ctx > 0) { + std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, + (long long) (lim.max_kv_bytes >> 20)); + } + std::printf("\n"); + } else if (lim.effective_n_ctx > 0) { + // Capped family whose context is too small to fit any audio + // plus a prompt (e.g. an aggressively low --n-ctx). + std::printf( + " max audio: ~0 s (context %d tok too small for " + "audio + prompt)\n", + lim.effective_n_ctx); + } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { + std::printf(" max audio: unbounded (long audio chunked internally)\n"); + } else { + // No context cap and no chunker: the family encodes the + // whole clip in one pass (e.g. block-local attention, + // where cost is linear in audio length). + std::printf(" max audio: unbounded (whole clip in one pass)\n"); + } + } + } + + struct transcribe_run_params rp; + transcribe_run_params_init(&rp); + if (args.translate) { + rp.task = TRANSCRIBE_TASK_TRANSLATE; + } + std::vector vocabulary_ptrs; + apply_prompting(args, rp, vocabulary_ptrs); + if (!args.language.empty()) { + rp.language = args.language.c_str(); + } + if (!args.target_language.empty()) { + rp.target_language = args.target_language.c_str(); + } + rp.timestamps = args.timestamps; + rp.spec_k_drafts = args.spec_k_drafts; + + if (args.itn_set) { + rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + } + if (args.canary_pnc_set) { + rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + } + if (args.diarize_set) { + rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + } + + struct transcribe_whisper_run_ext wx; + transcribe_whisper_run_ext_init(&wx); + if (args.whisper_set) { + if (!args.initial_prompt.empty()) { + wx.initial_prompt = args.initial_prompt.c_str(); + } + wx.condition_on_prev_tokens = args.condition_on_prev_tokens; + if (args.temperature_set) { + wx.temperature = args.temperature; + } + wx.prompt_condition = args.prompt_condition; + if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { + rp.family = &wx.ext; + } + } + + if (args.keep_special_tags) { + rp.keep_special_tags = true; + } + + // Streaming demo: drive transcribe_stream_begin/feed/finalize + // with fixed-size PCM chunks. Families with true per-feed + // partial decoding (moonshine_streaming) flip + // update.result_changed whenever the transcript advances; the + // CLI prints the live tentative text on each such feed. + // Families that only commit at finalize keep result_changed + // false until the finalize call. + transcribe_status run_st = TRANSCRIBE_OK; + if (args.stream_chunk_ms > 0) { + struct transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); + if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { + std::fprintf(stderr, + "stream: model does not advertise " + "supports_streaming; use a streaming-capable " + "model or drop --stream-chunk-ms\n"); + transcribe_session_free(ctx); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); + std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); + + struct transcribe_stream_params sp; + transcribe_stream_params_init(&sp); + struct transcribe_parakeet_stream_ext pkt_sp; + transcribe_parakeet_stream_ext_init(&pkt_sp); + struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; + transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); + struct transcribe_voxtral_realtime_stream_ext vx_sp; + transcribe_voxtral_realtime_stream_ext_init(&vx_sp); + const bool want_cache_aware = (args.stream_att_right >= 0); + const bool want_buffered = + args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; + if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { + pkt_sp.att_context_right = args.stream_att_right; + sp.family = &pkt_sp.ext; + std::printf("stream: att_context_right=%d\n", args.stream_att_right); + } else if (want_buffered && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { + pkt_buf_sp.left_ms = args.stream_buf_left_ms; + pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; + pkt_buf_sp.right_ms = args.stream_buf_right_ms; + sp.family = &pkt_buf_sp.ext; + std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, + args.stream_buf_chunk_ms, args.stream_buf_right_ms); + } else if (args.stream_voxtral_delay != -1 && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { + vx_sp.num_delay_tokens = args.stream_voxtral_delay; + sp.family = &vx_sp.ext; + std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); + } + run_st = transcribe_stream_begin(ctx, &rp, &sp); + if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); + } else { + size_t pos = 0; + int feed_n = 0; + while (pos < pcm.size()) { + const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); + struct transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); + if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); + break; + } + pos += take; + std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, + (long long) upd.input_received_ms, (long long) upd.buffered_ms); + if (upd.result_changed) { + const char * partial = transcribe_full_text(ctx); + std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); + } + std::printf("\n"); + ++feed_n; + } + if (run_st == TRANSCRIBE_OK) { + struct transcribe_stream_update fin_upd; + transcribe_stream_update_init(&fin_upd); + run_st = transcribe_stream_finalize(ctx, &fin_upd); + std::printf( + " finalize: status=%s " + "revision=%d input=%lld ms committed=%lld ms\n", + transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, + (long long) fin_upd.audio_committed_ms); + } + } + } else { + // --repeat N runs transcribe_run() N times for steady-state + // perf measurements. + for (int r = 0; r < args.repeat; ++r) { + run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + if (run_st != TRANSCRIBE_OK) { + break; + } + } + } + std::printf("run: %s\n", transcribe_status_string(run_st)); + // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but + // preserve the partial transcript (see docs/input-limits.md), so show + // the result for them too — just flagged. + const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; + if (result_present) { + const char * text = transcribe_full_text(ctx); + std::printf("text: %s\n", (text && *text) ? text : "(empty)"); + output_ok = write_output_file(output, args.output_path, text) && output_ok; + + // A decode cut short before end-of-stream; the text above is + // incomplete. + if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + std::printf( + " note: decode stopped when the output began repeating " + "itself (repeats dropped); transcript is incomplete\n"); + } else if (transcribe_was_truncated(ctx)) { + std::printf( + " note: output truncated (hit the model's " + "context/generation cap before end-of-stream); " + "transcript is incomplete\n"); + } + + const char * dl = transcribe_detected_language(ctx); + if (dl && *dl) { + std::printf("detected-language: %s\n", dl); + } + + const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); + const int n_seg = transcribe_n_segments(ctx); + const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; + if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { + std::printf("segments: %d\n", n_seg); + for (int i = 0; i < n_seg; ++i) { + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_get_segment(ctx, i, &seg); + char spk[16] = ""; + if (seg.speaker_id > 0) { + std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); + } + if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { + std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, + (seg.text != nullptr) ? seg.text : ""); + } else { + // Speaker-attributed turns without timing (granite SAA). + std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); + } + } + } + if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_wrd = transcribe_n_words(ctx); + std::printf("words: %d\n", n_wrd); + for (int i = 0; i < n_wrd; ++i) { + struct transcribe_word wrd; + transcribe_word_init(&wrd); + (void) transcribe_get_word(ctx, i, &wrd); + std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, + (wrd.text != nullptr) ? wrd.text : ""); + } + } + if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_tok = transcribe_n_tokens(ctx); + std::printf("tokens: %d\n", n_tok); + for (int i = 0; i < n_tok; ++i) { + struct transcribe_token tok; + transcribe_token_init(&tok); + (void) transcribe_get_token(ctx, i, &tok); + std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, + (tok.text != nullptr) ? tok.text : ""); + } + } + } + + transcribe_print_timings(ctx); + + { + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_get_timings(ctx, &tm); + const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; + if (total_ms > 0.0 && duration_s > 0.0) { + std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, + duration_s); + } + } + + transcribe_session_free(ctx); + transcribe_model_free(model); + + if (run_st != TRANSCRIBE_OK || !output_ok) { + return EXIT_FAILURE; + } + } else { + std::printf("model: (none specified, skipping load)\n"); + } + + return EXIT_SUCCESS; +} + +} // namespace transcribe_cli diff --git a/examples/cli/cli.h b/examples/cli/cli.h new file mode 100644 index 000000000..dbb477e12 --- /dev/null +++ b/examples/cli/cli.h @@ -0,0 +1,119 @@ +// cli.h - shared declarations for the transcribe-cli example. +// +// main.cpp parses arguments and owns process setup (log sink, output file); +// the per-role drivers (asr.cpp) do the work. Shared helpers live in +// namespace transcribe_cli next to the WAV loader in examples/common. + +#pragma once + +#include "transcribe.h" +#include "transcribe/whisper.h" + +#include +#include +#include + +namespace transcribe_cli { + +struct cli_args { + std::string wav_path; + std::string model_path; + std::string language; + std::string target_language; // --target-language: target lang for translation + std::string batch_file; // --batch: one wav path per line + int batch_size = 0; // --batch-size: >1 groups utterances into + // transcribe_run_batch calls (offline only). + // 0/1 keeps the per-file serial loop. + bool translate = false; + bool instruct = false; // --task instruct + bool quiet = false; + bool list_devices = false; // --list-devices: print devices and exit + bool batch_jsonl = false; // --batch-jsonl: output JSONL + std::string output_path; // -o/--output: write raw text here + int repeat = 1; + int n_threads = 0; // 0 = library default (all cores) + int n_ctx = 0; // 0 = model's true max; >0 lowers the cap + transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; + transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; + int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; + + // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). + std::vector vocabulary; // --vocabulary TERMS / --vocabulary-file PATH + std::string prompt; // --prompt TEXT + std::string prefix; // --prefix TEXT + + // Whisper-family knobs. Ignored for non-Whisper models. + std::string initial_prompt; // --initial-prompt TEXT + bool whisper_set = false; + bool temperature_set = false; // --temperature F + float temperature = 0.0f; // tier-0 sampling temp + bool condition_on_prev_tokens = false; // --condition-on-prev-tokens + enum transcribe_whisper_prompt_condition prompt_condition = + TRANSCRIBE_WHISPER_PROMPT_FIRST_SEGMENT; // --prompt-condition first|all + + // SenseVoice / FunASR-Nano family knobs. The `--itn` flag is shared: + // it routes to whichever family the loaded model belongs to. Ignored + // by non-ITN-aware families. Unset leaves the library default in place, + // which differs per family (sensevoice: on; funasr-nano: off), so the + // initializer here is only read once --itn / --no-itn has been seen. + bool use_itn = false; // --itn / --no-itn + bool itn_set = false; + bool keep_special_tags = false; // --raw-tokens + + // Canary family knobs. Ignored by non-Canary families. + bool canary_pnc = true; // default: punctuation+caps on + bool canary_pnc_set = false; // --pnc / --no-pnc set this + + // Speaker diarization toggle (moss / granite-plus). Unset = library + // default (OFF). --diarize / --no-diarize set this. + bool diarize = true; + bool diarize_set = false; + + // Streaming demo: when > 0, the single-file path feeds the WAV + // through transcribe_stream_begin/feed/finalize in fixed-size + // ms-aligned chunks instead of one transcribe_run call. Requires + // the loaded model to advertise supports_streaming. Set by + // --stream-chunk-ms N. + int stream_chunk_ms = 0; + // Parakeet streaming: pick a right-context (lookahead) setting + // from the model's training menu. -1 = model default (max + // accuracy / max latency); 0/1/6/13 select the published + // nemotron-speech-streaming-en-0.6b settings. Set by + // --stream-att-right N. Ignored when stream_chunk_ms == 0. + int stream_att_right = -1; + // Parakeet buffered streaming (parakeet-unified-en-0.6b): override + // the (L, C, R) attention context tuple in milliseconds. -1 = use + // the model's default (highest-accuracy row of the training menu). + // Frame-aligned: the lib rounds each value down to the nearest + // post-subsample frame (80ms at 4x subsampling). Ignored when + // stream_chunk_ms == 0 or when the model is not buffered-streaming. + int stream_buf_left_ms = -1; + int stream_buf_chunk_ms = -1; + int stream_buf_right_ms = -1; + // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio + // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by + // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when + // the model is not voxtral_realtime. + int stream_voxtral_delay = -1; + // Speculative-decode draft length passed through to + // transcribe_run_params::spec_k_drafts on the offline path. -1 = family + // default (each family picks its tuned K). 0 = explicitly off. >0 = + // explicit K. Silently ignored by families without + // supports_spec_decode. Set by --spec-k-drafts N. + int spec_k_drafts = -1; +}; + +// Minimal JSON string escape (main.cpp). +std::string json_escape(const char * s); + +// Append `text` plus a trailing newline to the -o/--output file, when one is +// open. Returns false on a write error (main.cpp). +bool write_output_file(std::ofstream * output, const std::string & path, const char * text); + +// ASR drivers (asr.cpp). Each loads the model, runs, prints, and returns the +// process exit code. +int run_asr_batch(const cli_args & args, std::ofstream * output); +int run_asr_file(const cli_args & args, std::ofstream * output); + +} // namespace transcribe_cli diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 1867af445..b740ff2f2 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -5,6 +5,7 @@ // and batch modes, offline and streaming paths, and per-family knobs. // Run with --help for the full option list. +#include "cli.h" #include "transcribe.h" #include "transcribe/parakeet.h" #include "transcribe/voxtral_realtime.h" @@ -24,6 +25,8 @@ #include #include +using transcribe_cli::cli_args; + namespace { bool parse_device_index(const char * text, int & out) { @@ -40,289 +43,6 @@ bool parse_device_index(const char * text, int & out) { return true; } -// Minimal JSON string escape: covers the characters MUST be escaped by -// the JSON spec (quote, backslash, control chars). Transcribed text is -// short UTF-8 in practice; we don't need unicode escaping. -std::string json_escape(const char * s) { - std::string out; - for (const char * p = s ? s : ""; *p; ++p) { - const unsigned char c = static_cast(*p); - if (c == '"') { - out += "\\\""; - } else if (c == '\\') { - out += "\\\\"; - } else if (c == '\n') { - out += "\\n"; - } else if (c == '\r') { - out += "\\r"; - } else if (c == '\t') { - out += "\\t"; - } else if (c < 0x20) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", c); - out += buf; - } else { - out += static_cast(c); - } - } - return out; -} - -// Shared row formatter for segments_json / batch_segments_json below. -std::string segment_row_json(const struct transcribe_segment & seg) { - std::string out; - char head[128]; - if (seg.speaker_id > 0) { - std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"text\":\"", - static_cast(seg.t0_ms), static_cast(seg.t1_ms), - static_cast(seg.speaker_id)); - } else { - std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"text\":\"", - static_cast(seg.t0_ms), static_cast(seg.t1_ms)); - } - out += head; - out += json_escape(seg.text != nullptr ? seg.text : ""); - out += "\"}"; - return out; -} - -// Build the ",\"segments\":[...]" fragment when the context has real -// segment rows: segment timestamps, or speaker-attributed turns (which -// may carry no timing — granite's speaker-attribution task). Returns an -// empty string otherwise (the library still populates a single dummy -// entry when the kind is NONE, but with zero timing — don't pollute the -// JSON with it). -std::string segments_json(const transcribe_session * ctx) { - if (transcribe_returned_timestamp_kind(ctx) == TRANSCRIBE_TIMESTAMPS_NONE && - transcribe_n_speaker_segments(ctx) <= 0) { - return {}; - } - const int n_seg = transcribe_n_segments(ctx); - if (n_seg <= 0) { - return {}; - } - std::string out = ",\"segments\":["; - for (int s = 0; s < n_seg; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_get_segment(ctx, s, &seg); - out += segment_row_json(seg); - } - out += "]"; - return out; -} - -// Batch variant: segments JSON for utterance `i` of a transcribe_run_batch -// result, using the indexed transcribe_batch_* accessors. -std::string batch_segments_json(const transcribe_session * ctx, int i) { - if (transcribe_batch_returned_timestamp_kind(ctx, i) == TRANSCRIBE_TIMESTAMPS_NONE && - transcribe_batch_n_speaker_segments(ctx, i) <= 0) { - return {}; - } - const int n_seg = transcribe_batch_n_segments(ctx, i); - if (n_seg <= 0) { - return {}; - } - std::string out = ",\"segments\":["; - for (int s = 0; s < n_seg; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_batch_get_segment(ctx, i, s, &seg); - out += segment_row_json(seg); - } - out += "]"; - return out; -} - -// ",\"raw_text\":\"...\"" fragment: the model's pre-cleanup decode -// (transcribe_raw_text). Emitted only when it differs from the clean text, -// so JSONL stays compact for families with no post-processing. -std::string raw_text_json(const char * raw, const char * clean) { - if (raw == nullptr || raw[0] == '\0' || (clean != nullptr && std::strcmp(raw, clean) == 0)) { - return {}; - } - std::string out = ",\"raw_text\":\""; - out += json_escape(raw); - out += "\""; - return out; -} - -// A decode cut short before end-of-stream (budget or repetition stop): non-OK, -// but the partial transcript is preserved (see docs/input-limits.md). -bool is_cut_short(transcribe_status st) { - return st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED || st == TRANSCRIBE_ERR_OUTPUT_REPETITION; -} - -const char * cut_short_label(transcribe_status st) { - return st == TRANSCRIBE_ERR_OUTPUT_REPETITION ? "stopped repeating" : "truncated"; -} - -// ",\"speakers\":[...]" fragment: the "who spoke when" rows. Emitted only -// when the run produced speaker segments. p is omitted unless finite -// (NaN — "model provides no confidence" — is not representable in JSON). -std::string speaker_row_json(const struct transcribe_speaker_segment & row) { - char buf[160]; - if (std::isfinite(row.p)) { - std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"p\":%.4f}", - static_cast(row.t0_ms), static_cast(row.t1_ms), - static_cast(row.speaker_id), static_cast(row.p)); - } else { - std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d}", - static_cast(row.t0_ms), static_cast(row.t1_ms), - static_cast(row.speaker_id)); - } - return buf; -} - -std::string speakers_json(const transcribe_session * ctx) { - const int n = transcribe_n_speaker_segments(ctx); - if (n <= 0) { - return {}; - } - std::string out = ",\"speakers\":["; - for (int s = 0; s < n; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - (void) transcribe_get_speaker_segment(ctx, s, &row); - out += speaker_row_json(row); - } - out += "]"; - return out; -} - -std::string batch_speakers_json(const transcribe_session * ctx, int i) { - const int n = transcribe_batch_n_speaker_segments(ctx, i); - if (n <= 0) { - return {}; - } - std::string out = ",\"speakers\":["; - for (int s = 0; s < n; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - (void) transcribe_batch_get_speaker_segment(ctx, i, s, &row); - out += speaker_row_json(row); - } - out += "]"; - return out; -} - -struct cli_args { - std::string wav_path; - std::string model_path; - std::string language; - std::string target_language; // --target-language: target lang for translation - std::string batch_file; // --batch: one wav path per line - int batch_size = 0; // --batch-size: >1 groups utterances into - // transcribe_run_batch calls (offline only). - // 0/1 keeps the per-file serial loop. - bool translate = false; - bool instruct = false; // --task instruct - bool quiet = false; - bool list_devices = false; // --list-devices: print devices and exit - bool batch_jsonl = false; // --batch-jsonl: output JSONL - std::string output_path; // -o/--output: write raw text here - int repeat = 1; - int n_threads = 0; // 0 = library default (all cores) - int n_ctx = 0; // 0 = model's true max; >0 lowers the cap - transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; - transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device - transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; - - // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). - std::vector vocabulary; // --vocabulary TERMS / --vocabulary-file PATH - std::string prompt; // --prompt TEXT - std::string prefix; // --prefix TEXT - - // Whisper-family knobs. Ignored for non-Whisper models. - std::string initial_prompt; // --initial-prompt TEXT - bool whisper_set = false; - bool temperature_set = false; // --temperature F - float temperature = 0.0f; // tier-0 sampling temp - bool condition_on_prev_tokens = false; // --condition-on-prev-tokens - enum transcribe_whisper_prompt_condition prompt_condition = - TRANSCRIBE_WHISPER_PROMPT_FIRST_SEGMENT; // --prompt-condition first|all - - // SenseVoice / FunASR-Nano family knobs. The `--itn` flag is shared: - // it routes to whichever family the loaded model belongs to. Ignored - // by non-ITN-aware families. Unset leaves the library default in place, - // which differs per family (sensevoice: on; funasr-nano: off), so the - // initializer here is only read once --itn / --no-itn has been seen. - bool use_itn = false; // --itn / --no-itn - bool itn_set = false; - bool keep_special_tags = false; // --raw-tokens - - // Canary family knobs. Ignored by non-Canary families. - bool canary_pnc = true; // default: punctuation+caps on - bool canary_pnc_set = false; // --pnc / --no-pnc set this - - // Speaker diarization toggle (moss / granite-plus). Unset = library - // default (OFF). --diarize / --no-diarize set this. - bool diarize = true; - bool diarize_set = false; - - // Streaming demo: when > 0, the single-file path feeds the WAV - // through transcribe_stream_begin/feed/finalize in fixed-size - // ms-aligned chunks instead of one transcribe_run call. Requires - // the loaded model to advertise supports_streaming. Set by - // --stream-chunk-ms N. - int stream_chunk_ms = 0; - // Parakeet streaming: pick a right-context (lookahead) setting - // from the model's training menu. -1 = model default (max - // accuracy / max latency); 0/1/6/13 select the published - // nemotron-speech-streaming-en-0.6b settings. Set by - // --stream-att-right N. Ignored when stream_chunk_ms == 0. - int stream_att_right = -1; - // Parakeet buffered streaming (parakeet-unified-en-0.6b): override - // the (L, C, R) attention context tuple in milliseconds. -1 = use - // the model's default (highest-accuracy row of the training menu). - // Frame-aligned: the lib rounds each value down to the nearest - // post-subsample frame (80ms at 4x subsampling). Ignored when - // stream_chunk_ms == 0 or when the model is not buffered-streaming. - int stream_buf_left_ms = -1; - int stream_buf_chunk_ms = -1; - int stream_buf_right_ms = -1; - // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio - // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by - // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when - // the model is not voxtral_realtime. - int stream_voxtral_delay = -1; - // Speculative-decode draft length passed through to - // transcribe_run_params::spec_k_drafts on the offline path. -1 = family - // default (each family picks its tuned K). 0 = explicitly off. >0 = - // explicit K. Silently ignored by families without - // supports_spec_decode. Set by --spec-k-drafts N. - int spec_k_drafts = -1; -}; - -// Point rp's generic prompting fields at args' storage; vocabulary_ptrs -// backs rp.vocabulary and must outlive the run. -void apply_prompting(const cli_args & args, transcribe_run_params & rp, std::vector & vocabulary_ptrs) { - if (args.instruct) { - rp.task = TRANSCRIBE_TASK_INSTRUCT; - } - vocabulary_ptrs.clear(); - for (const std::string & term : args.vocabulary) { - vocabulary_ptrs.push_back(term.c_str()); - } - rp.vocabulary = vocabulary_ptrs.empty() ? nullptr : vocabulary_ptrs.data(); - rp.n_vocabulary = static_cast(vocabulary_ptrs.size()); - rp.prompt = args.prompt.empty() ? nullptr : args.prompt.c_str(); - rp.prefix = args.prefix.empty() ? nullptr : args.prefix.c_str(); -} - void print_usage(const char * argv0) { std::fprintf(stderr, "usage: %s [options] audio.wav\n" @@ -827,6 +547,38 @@ void log_cb(transcribe_log_level level, const char * msg, void * userdata) { std::fprintf(stderr, "%s %s%s", prefix, msg, (msg && *msg && msg[std::strlen(msg) - 1] == '\n') ? "" : "\n"); } +} // namespace + +namespace transcribe_cli { + +// Minimal JSON string escape: covers the characters MUST be escaped by +// the JSON spec (quote, backslash, control chars). Transcribed text is +// short UTF-8 in practice; we don't need unicode escaping. +std::string json_escape(const char * s) { + std::string out; + for (const char * p = s ? s : ""; *p; ++p) { + const unsigned char c = static_cast(*p); + if (c == '"') { + out += "\\\""; + } else if (c == '\\') { + out += "\\\\"; + } else if (c == '\n') { + out += "\\n"; + } else if (c == '\r') { + out += "\\r"; + } else if (c == '\t') { + out += "\\t"; + } else if (c < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + return out; +} + bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { if (output == nullptr) { return true; @@ -844,7 +596,7 @@ bool write_output_file(std::ofstream * output, const std::string & path, const c return true; } -} // namespace +} // namespace transcribe_cli int main(int argc, char ** argv) { cli_args args; @@ -879,755 +631,14 @@ int main(int argc, char ** argv) { } output = &output_file; } - bool output_ok = true; // Batch mode: --batch reads a file list, one wav path per line. Loads // the model ONCE and reuses the context across all files. Outputs one // JSONL line per file to stdout when --batch-jsonl is set, otherwise // the same human-readable format as single-file mode. if (!args.batch_file.empty()) { - if (args.model_path.empty()) { - std::fprintf(stderr, "error: --batch requires --model\n"); - return EXIT_FAILURE; - } - - std::vector wav_paths; - { - std::ifstream fin(args.batch_file); - if (!fin) { - std::fprintf(stderr, "error: cannot open batch file %s\n", args.batch_file.c_str()); - return EXIT_FAILURE; - } - std::string line; - while (std::getline(fin, line)) { - while (!line.empty() && - (line.back() == '\n' || line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { - line.pop_back(); - } - if (!line.empty()) { - wav_paths.push_back(line); - } - } - } - if (wav_paths.empty()) { - std::fprintf(stderr, "error: batch file is empty\n"); - return EXIT_FAILURE; - } - if (!args.batch_jsonl) { - std::fprintf(stderr, "batch: %zu files\n", wav_paths.size()); - } - - struct transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; - if (args.device_index >= 0 && mp.device == nullptr) { - std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); - return EXIT_FAILURE; - } - struct transcribe_model * model = nullptr; - const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); - if (load_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "model load: %s\n", transcribe_status_string(load_st)); - return EXIT_FAILURE; - } - - struct transcribe_session_params cp; - transcribe_session_params_init(&cp); - cp.n_threads = args.n_threads; - cp.n_ctx = args.n_ctx; - cp.kv_type = args.kv_type; - struct transcribe_session * ctx = nullptr; - const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); - if (init_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - struct transcribe_run_params rp; - transcribe_run_params_init(&rp); - if (args.translate) { - rp.task = TRANSCRIBE_TASK_TRANSLATE; - } - std::vector vocabulary_ptrs; - apply_prompting(args, rp, vocabulary_ptrs); - if (!args.language.empty()) { - rp.language = args.language.c_str(); - } - if (!args.target_language.empty()) { - rp.target_language = args.target_language.c_str(); - } - rp.timestamps = args.timestamps; - rp.spec_k_drafts = args.spec_k_drafts; - - if (args.itn_set) { - rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; - } - if (args.canary_pnc_set) { - rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; - } - if (args.diarize_set) { - rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; - } - - // Whisper run extension. Allocated outside rp's scope so its - // bytes outlive the per-file loop below; the library copies - // initial_prompt/prompt_tokens before transcribe_run returns, - // but rp aliases &wx.ext for the run call itself. - struct transcribe_whisper_run_ext wx; - transcribe_whisper_run_ext_init(&wx); - if (args.whisper_set) { - if (!args.initial_prompt.empty()) { - wx.initial_prompt = args.initial_prompt.c_str(); - } - wx.condition_on_prev_tokens = args.condition_on_prev_tokens; - if (args.temperature_set) { - wx.temperature = args.temperature; - } - wx.prompt_condition = args.prompt_condition; - if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { - rp.family = &wx.ext; - } - } - - if (args.keep_special_tags) { - rp.keep_special_tags = true; - } - - // Emit a batch header line once, before any per-file output. Carries - // the one-shot load time so downstream WER tooling can record it - // without parsing stderr. Per-file lines follow on subsequent lines. - if (args.batch_jsonl) { - struct transcribe_timings load_tm; - transcribe_timings_init(&load_tm); - (void) transcribe_get_timings(ctx, &load_tm); - std::printf("{\"type\":\"batch_header\",\"load_ms\":%.1f}\n", (double) load_tm.load_ms); - std::fflush(stdout); - } - - int n_ok = 0; - int n_truncated = 0; // result-bearing: hit the generation cap, partial hyp emitted - int n_repeating = 0; // result-bearing: stopped when the output looped, partial hyp emitted - int n_fail = 0; // no usable result (wav load / backend / unsupported / whole-batch) - - // Offline batched path: group up to batch_size utterances into one - // transcribe_run_batch call. Mutually exclusive with the streaming - // path (--stream-chunk-ms), which is per-utterance by construction. - // Per-file JSONL output is byte-identical in shape to the serial - // path so the WER harness (scripts/wer/run.py) consumes either. - const bool use_batched = args.batch_size > 1 && args.stream_chunk_ms <= 0; - if (use_batched) { - const size_t total = wav_paths.size(); - const size_t group = static_cast(args.batch_size); - for (size_t base = 0; base < total; base += group) { - const size_t end = std::min(total, base + group); - - // Load each wav in the group. A wav that fails to load is - // emitted as an error line and excluded from the batch, so - // its failure cannot abort its neighbours. - std::vector> pcms; - std::vector pcm_ptrs; - std::vector n_samps; - std::vector src_index; // -> wav_paths - for (size_t i = base; i < end; ++i) { - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav_paths[i], pcm, wav_err)) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav_paths[i].c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"wav: %s\"}\n", - file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); - } else { - std::fprintf(stderr, "SKIP %s: %s\n", wav_paths[i].c_str(), wav_err.c_str()); - } - ++n_fail; - std::fflush(stdout); - continue; - } - pcms.push_back(std::move(pcm)); - src_index.push_back(i); - } - for (auto & p : pcms) { - pcm_ptrs.push_back(p.data()); - n_samps.push_back(static_cast(p.size())); - } - if (pcms.empty()) { - continue; - } - - const transcribe_status bst = - transcribe_run_batch(ctx, pcm_ptrs.data(), n_samps.data(), static_cast(pcm_ptrs.size()), &rp); - if (bst != TRANSCRIBE_OK && transcribe_batch_n_results(ctx) == 0) { - // Whole-batch failure (no per-utterance results): emit an - // error line per file in the group and continue. - for (size_t k = 0; k < src_index.size(); ++k) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav_paths[src_index[k]].c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"%s\"}\n", - file_esc.c_str(), json_escape(transcribe_status_string(bst)).c_str()); - } - ++n_fail; - } - std::fflush(stdout); - continue; - } - - for (size_t k = 0; k < src_index.size(); ++k) { - const std::string & wav = wav_paths[src_index[k]]; - const transcribe_status ust = transcribe_batch_status(ctx, static_cast(k)); - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the - // partial transcript is preserved and readable via - // transcribe_batch_full_text (see transcribe.h). Emit it as the hyp - // so downstream tooling scores the partial rather than an empty - // string; the error field below still tags it so the stop stays - // visible. - const bool result_present = ust == TRANSCRIBE_OK || is_cut_short(ust); - const char * text = ""; - if (result_present) { - const char * t = transcribe_batch_full_text(ctx, static_cast(k)); - if (t && *t) { - text = t; - } - } - if (ust == TRANSCRIBE_OK) { - ++n_ok; - } else if (ust == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { - ++n_truncated; - } else if (ust == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - ++n_repeating; - } else { - ++n_fail; - } - if (args.batch_jsonl) { - const std::string escaped = json_escape(text); - std::string segments = batch_segments_json(ctx, static_cast(k)); - segments += batch_speakers_json(ctx, static_cast(k)); - segments += raw_text_json(transcribe_batch_raw_text(ctx, static_cast(k)), text); - std::string err_field; - if (ust != TRANSCRIBE_OK) { - err_field = ",\"error\":\""; - err_field += json_escape(transcribe_status_string(ust)); - err_field += "\""; - } - // Per-utterance timings: the batched encoder time is - // amortized across the batch, decode is real per-utt, - // so summing across utterances gives the true encode vs - // decode split for the whole batched run. - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_batch_get_timings(ctx, static_cast(k), &tm); - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"%s\"%s," - "\"mel_ms\":%.1f,\"encode_ms\":%.1f," - "\"decode_ms\":%.1f%s}\n", - file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, - (double) tm.encode_ms, (double) tm.decode_ms, err_field.c_str()); - } else { - std::printf("[%zu/%zu] %s", src_index[k] + 1, total, wav.c_str()); - if (ust == TRANSCRIBE_OK) { - std::printf("\n text: %s\n", text); - } else if (is_cut_short(ust)) { - std::printf(" (%s)\n text: %s\n", cut_short_label(ust), text); - } else { - std::printf(" ERROR: %s\n", transcribe_status_string(ust)); - } - } - output_ok = write_output_file(output, args.output_path, text) && output_ok; - std::fflush(stdout); - } - } - } else { - for (size_t i = 0; i < wav_paths.size(); ++i) { - const std::string & wav = wav_paths[i]; - - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav, pcm, wav_err)) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"wav: %s\"}\n", - file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); - } else { - std::fprintf(stderr, "SKIP %s: %s\n", wav.c_str(), wav_err.c_str()); - } - ++n_fail; - std::fflush(stdout); - continue; - } - - // Run. When --stream-chunk-ms > 0, drive the streaming API - // for this utterance (begin/feed/finalize) so the WER - // harness can measure cache-aware streaming output. - transcribe_status run_st = TRANSCRIBE_OK; - if (args.stream_chunk_ms > 0) { - struct transcribe_stream_params sp; - transcribe_stream_params_init(&sp); - struct transcribe_parakeet_stream_ext pkt_sp; - transcribe_parakeet_stream_ext_init(&pkt_sp); - struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; - transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); - struct transcribe_voxtral_realtime_stream_ext vx_sp; - transcribe_voxtral_realtime_stream_ext_init(&vx_sp); - const bool want_cache_aware = (args.stream_att_right >= 0); - const bool want_buffered = - args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; - if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { - pkt_sp.att_context_right = args.stream_att_right; - sp.family = &pkt_sp.ext; - } else if (want_buffered && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { - pkt_buf_sp.left_ms = args.stream_buf_left_ms; - pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; - pkt_buf_sp.right_ms = args.stream_buf_right_ms; - sp.family = &pkt_buf_sp.ext; - } else if (args.stream_voxtral_delay != -1 && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { - vx_sp.num_delay_tokens = args.stream_voxtral_delay; - sp.family = &vx_sp.ext; - } - run_st = transcribe_stream_begin(ctx, &rp, &sp); - if (run_st == TRANSCRIBE_OK) { - const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); - size_t pos = 0; - while (pos < pcm.size()) { - const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); - struct transcribe_stream_update upd; - transcribe_stream_update_init(&upd); - run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); - if (run_st != TRANSCRIBE_OK) { - break; - } - pos += take; - } - if (run_st == TRANSCRIBE_OK) { - struct transcribe_stream_update fin_upd; - transcribe_stream_update_init(&fin_upd); - run_st = transcribe_stream_finalize(ctx, &fin_upd); - } - } - } else { - run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); - } - - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the partial - // transcript is preserved and readable via transcribe_full_text (see - // transcribe.h). Emit it as the hyp so downstream tooling scores the - // partial rather than an empty string; the error field below still - // tags it so the stop stays visible. - const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st); - const char * text = ""; - if (result_present) { - const char * t = transcribe_full_text(ctx); - if (t && *t) { - text = t; - } - } - if (run_st == TRANSCRIBE_OK) { - ++n_ok; - } else if (run_st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { - ++n_truncated; - } else if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - ++n_repeating; - } else { - ++n_fail; - } - - // Output. When run_st != OK we still emit a JSON line so - // batch consumers see one record per input wav, but we tag - // it with the error string. Without this the wav-load error - // path (above) reports errors but transcribe_run failures - // (e.g. unsupported language) produce empty hyp_text with no - // error tag, so downstream tools count them as silent - // successes. - if (args.batch_jsonl) { - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_get_timings(ctx, &tm); - const std::string escaped = json_escape(text); - std::string segments = segments_json(ctx); - segments += speakers_json(ctx); - segments += raw_text_json(transcribe_raw_text(ctx), text); - std::string err_field; - if (run_st != TRANSCRIBE_OK) { - err_field = ",\"error\":\""; - err_field += json_escape(transcribe_status_string(run_st)); - err_field += "\""; - } - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"%s\"%s," - "\"mel_ms\":%.1f,\"encode_ms\":%.1f," - "\"decode_ms\":%.1f%s}\n", - file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, - (double) tm.decode_ms, err_field.c_str()); - } else { - std::printf("[%zu/%zu] %s", i + 1, wav_paths.size(), wav.c_str()); - if (run_st == TRANSCRIBE_OK) { - std::printf("\n text: %s\n", text); - } else if (is_cut_short(run_st)) { - std::printf(" (%s)\n text: %s\n", cut_short_label(run_st), text); - } else { - std::printf(" ERROR: %s\n", transcribe_status_string(run_st)); - } - } - output_ok = write_output_file(output, args.output_path, text) && output_ok; - std::fflush(stdout); - } - } - - if (!args.batch_jsonl) { - std::fprintf(stderr, "batch: %d ok, %d truncated, %d stopped repeating, %d failed out of %zu\n", n_ok, - n_truncated, n_repeating, n_fail, wav_paths.size()); - } - - transcribe_session_free(ctx); - transcribe_model_free(model); - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing and do not fail the batch, but - // hard per-utterance failures must remain visible to automation. - return n_fail > 0 || !output_ok ? EXIT_FAILURE : EXIT_SUCCESS; + return transcribe_cli::run_asr_batch(args, output); } - // Single-file mode. - std::vector pcm; - std::string load_err; - if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { - std::fprintf(stderr, "wav: %s\n", load_err.c_str()); - return EXIT_FAILURE; - } - - const double duration_s = static_cast(pcm.size()) / 16000.0; - std::printf("audio: %s\n", args.wav_path.c_str()); - std::printf(" samples: %zu\n", pcm.size()); - std::printf(" duration: %.3f s\n", duration_s); - std::printf(" sample rate 16000 Hz mono float32\n"); - - if (!args.model_path.empty()) { - struct transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; - if (args.device_index >= 0 && mp.device == nullptr) { - std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); - return EXIT_FAILURE; - } - struct transcribe_model * model = nullptr; - const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); - std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); - if (st != TRANSCRIBE_OK) { - return EXIT_FAILURE; - } - std::printf(" backend: %s\n", transcribe_model_backend(model)); - if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { - std::printf(" name: %s\n", dn); - } - if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { - std::printf(" license: %s\n", lic); - } - - struct transcribe_session_params cp; - transcribe_session_params_init(&cp); - cp.n_threads = args.n_threads; - cp.n_ctx = args.n_ctx; - cp.kv_type = args.kv_type; - struct transcribe_session * ctx = nullptr; - const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); - if (init_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - // Surface the effective input-length limit so it's obvious how much - // audio this session accepts (reflects --n-ctx). 0 means "no practical - // limit", which covers two different families: one that chunks long - // audio internally (FEATURE_LONG_FORM) and one that is genuinely - // unbounded and encodes the clip in a single pass. Distinguish them - // rather than asserting the first. See docs/input-limits.md. - { - struct transcribe_session_limits lim; - transcribe_session_limits_init(&lim); - if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { - if (lim.effective_max_audio_ms > 0) { - std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); - if (lim.effective_n_ctx > 0) { - std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, - (long long) (lim.max_kv_bytes >> 20)); - } - std::printf("\n"); - } else if (lim.effective_n_ctx > 0) { - // Capped family whose context is too small to fit any audio - // plus a prompt (e.g. an aggressively low --n-ctx). - std::printf( - " max audio: ~0 s (context %d tok too small for " - "audio + prompt)\n", - lim.effective_n_ctx); - } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { - std::printf(" max audio: unbounded (long audio chunked internally)\n"); - } else { - // No context cap and no chunker: the family encodes the - // whole clip in one pass (e.g. block-local attention, - // where cost is linear in audio length). - std::printf(" max audio: unbounded (whole clip in one pass)\n"); - } - } - } - - struct transcribe_run_params rp; - transcribe_run_params_init(&rp); - if (args.translate) { - rp.task = TRANSCRIBE_TASK_TRANSLATE; - } - std::vector vocabulary_ptrs; - apply_prompting(args, rp, vocabulary_ptrs); - if (!args.language.empty()) { - rp.language = args.language.c_str(); - } - if (!args.target_language.empty()) { - rp.target_language = args.target_language.c_str(); - } - rp.timestamps = args.timestamps; - rp.spec_k_drafts = args.spec_k_drafts; - - if (args.itn_set) { - rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; - } - if (args.canary_pnc_set) { - rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; - } - if (args.diarize_set) { - rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; - } - - struct transcribe_whisper_run_ext wx; - transcribe_whisper_run_ext_init(&wx); - if (args.whisper_set) { - if (!args.initial_prompt.empty()) { - wx.initial_prompt = args.initial_prompt.c_str(); - } - wx.condition_on_prev_tokens = args.condition_on_prev_tokens; - if (args.temperature_set) { - wx.temperature = args.temperature; - } - wx.prompt_condition = args.prompt_condition; - if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { - rp.family = &wx.ext; - } - } - - if (args.keep_special_tags) { - rp.keep_special_tags = true; - } - - // Streaming demo: drive transcribe_stream_begin/feed/finalize - // with fixed-size PCM chunks. Families with true per-feed - // partial decoding (moonshine_streaming) flip - // update.result_changed whenever the transcript advances; the - // CLI prints the live tentative text on each such feed. - // Families that only commit at finalize keep result_changed - // false until the finalize call. - transcribe_status run_st = TRANSCRIBE_OK; - if (args.stream_chunk_ms > 0) { - struct transcribe_capabilities caps; - transcribe_capabilities_init(&caps); - const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); - if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { - std::fprintf(stderr, - "stream: model does not advertise " - "supports_streaming; use a streaming-capable " - "model or drop --stream-chunk-ms\n"); - transcribe_session_free(ctx); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); - std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); - - struct transcribe_stream_params sp; - transcribe_stream_params_init(&sp); - struct transcribe_parakeet_stream_ext pkt_sp; - transcribe_parakeet_stream_ext_init(&pkt_sp); - struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; - transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); - struct transcribe_voxtral_realtime_stream_ext vx_sp; - transcribe_voxtral_realtime_stream_ext_init(&vx_sp); - const bool want_cache_aware = (args.stream_att_right >= 0); - const bool want_buffered = - args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; - if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { - pkt_sp.att_context_right = args.stream_att_right; - sp.family = &pkt_sp.ext; - std::printf("stream: att_context_right=%d\n", args.stream_att_right); - } else if (want_buffered && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { - pkt_buf_sp.left_ms = args.stream_buf_left_ms; - pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; - pkt_buf_sp.right_ms = args.stream_buf_right_ms; - sp.family = &pkt_buf_sp.ext; - std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, - args.stream_buf_chunk_ms, args.stream_buf_right_ms); - } else if (args.stream_voxtral_delay != -1 && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { - vx_sp.num_delay_tokens = args.stream_voxtral_delay; - sp.family = &vx_sp.ext; - std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); - } - run_st = transcribe_stream_begin(ctx, &rp, &sp); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); - } else { - size_t pos = 0; - int feed_n = 0; - while (pos < pcm.size()) { - const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); - struct transcribe_stream_update upd; - transcribe_stream_update_init(&upd); - run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); - break; - } - pos += take; - std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, - (long long) upd.input_received_ms, (long long) upd.buffered_ms); - if (upd.result_changed) { - const char * partial = transcribe_full_text(ctx); - std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); - } - std::printf("\n"); - ++feed_n; - } - if (run_st == TRANSCRIBE_OK) { - struct transcribe_stream_update fin_upd; - transcribe_stream_update_init(&fin_upd); - run_st = transcribe_stream_finalize(ctx, &fin_upd); - std::printf( - " finalize: status=%s " - "revision=%d input=%lld ms committed=%lld ms\n", - transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, - (long long) fin_upd.audio_committed_ms); - } - } - } else { - // --repeat N runs transcribe_run() N times for steady-state - // perf measurements. - for (int r = 0; r < args.repeat; ++r) { - run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); - if (run_st != TRANSCRIBE_OK) { - break; - } - } - } - std::printf("run: %s\n", transcribe_status_string(run_st)); - // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but - // preserve the partial transcript (see docs/input-limits.md), so show - // the result for them too — just flagged. - const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; - if (result_present) { - const char * text = transcribe_full_text(ctx); - std::printf("text: %s\n", (text && *text) ? text : "(empty)"); - output_ok = write_output_file(output, args.output_path, text) && output_ok; - - // A decode cut short before end-of-stream; the text above is - // incomplete. - if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - std::printf( - " note: decode stopped when the output began repeating " - "itself (repeats dropped); transcript is incomplete\n"); - } else if (transcribe_was_truncated(ctx)) { - std::printf( - " note: output truncated (hit the model's " - "context/generation cap before end-of-stream); " - "transcript is incomplete\n"); - } - - const char * dl = transcribe_detected_language(ctx); - if (dl && *dl) { - std::printf("detected-language: %s\n", dl); - } - - const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); - const int n_seg = transcribe_n_segments(ctx); - const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; - if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { - std::printf("segments: %d\n", n_seg); - for (int i = 0; i < n_seg; ++i) { - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_get_segment(ctx, i, &seg); - char spk[16] = ""; - if (seg.speaker_id > 0) { - std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); - } - if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { - std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, - (seg.text != nullptr) ? seg.text : ""); - } else { - // Speaker-attributed turns without timing (granite SAA). - std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); - } - } - } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_wrd = transcribe_n_words(ctx); - std::printf("words: %d\n", n_wrd); - for (int i = 0; i < n_wrd; ++i) { - struct transcribe_word wrd; - transcribe_word_init(&wrd); - (void) transcribe_get_word(ctx, i, &wrd); - std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, - (wrd.text != nullptr) ? wrd.text : ""); - } - } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_tok = transcribe_n_tokens(ctx); - std::printf("tokens: %d\n", n_tok); - for (int i = 0; i < n_tok; ++i) { - struct transcribe_token tok; - transcribe_token_init(&tok); - (void) transcribe_get_token(ctx, i, &tok); - std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, - (tok.text != nullptr) ? tok.text : ""); - } - } - } - - transcribe_print_timings(ctx); - - { - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_get_timings(ctx, &tm); - const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; - if (total_ms > 0.0 && duration_s > 0.0) { - std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, - duration_s); - } - } - - transcribe_session_free(ctx); - transcribe_model_free(model); - - if (run_st != TRANSCRIBE_OK || !output_ok) { - return EXIT_FAILURE; - } - } else { - std::printf("model: (none specified, skipping load)\n"); - } - - return EXIT_SUCCESS; + return transcribe_cli::run_asr_file(args, output); } From fdd1a191d68fc15af999d42d7c0d4237f0583660 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 20:54:20 +0800 Subject: [PATCH 04/51] host thread pools: join on every path, return worker exceptions Bug fix (H5). The host thread pools in the shared mel front end (transcribe-mel.cpp run_threaded), the gigaam mel, and parallel_for_all each hand-rolled spawn / run tid 0 / join: - a thread that failed to launch left already-launched threads joinable when the vector unwound -> std::terminate; - an exception from the calling thread's worker skipped the joins -> std::terminate; - an exception inside a pool thread escaped the thread function -> std::terminate. transcribe::run_on_threads (transcribe-batch-util.h) replaces all three: tid 0 runs on the caller, every launched thread is joined on every path, and the launch error or the lowest tid's worker exception is rethrown after the joins, so the public entry point's api_guard maps it to a status. The vDSP FFT setup in the mel front end is now released by a scope guard, since a worker exception can now unwind past it. Behavior change only on those failure paths (terminate -> error status). tests/run_on_threads_unit.cpp covers launch failure (injected through the Thread template seam), worker exceptions on the caller and pool threads, exception type preservation, lowest-tid ordering, and parallel_for_all propagation. Output on the success path is unchanged: transcribe-cli CPU parity vs main and the real-model ctest results are identical. --- src/arch/gigaam/mel.cpp | 10 +- src/transcribe-batch-util.cpp | 10 +- src/transcribe-batch-util.h | 51 +++++++++ src/transcribe-mel.cpp | 26 ++--- tests/CMakeLists.txt | 17 +++ tests/run_on_threads_unit.cpp | 195 ++++++++++++++++++++++++++++++++++ 6 files changed, 276 insertions(+), 33 deletions(-) create mode 100644 tests/run_on_threads_unit.cpp diff --git a/src/arch/gigaam/mel.cpp b/src/arch/gigaam/mel.cpp index bf17ce6f1..8670af0aa 100644 --- a/src/arch/gigaam/mel.cpp +++ b/src/arch/gigaam/mel.cpp @@ -167,15 +167,7 @@ transcribe_status GigaamMelFrontend::compute(const float * pcm, } } }; - std::vector workers; - workers.reserve(n_workers > 0 ? static_cast(n_workers - 1) : 0); - for (int worker_id = 1; worker_id < n_workers; ++worker_id) { - workers.emplace_back(worker, worker_id); - } - worker(0); - for (auto & thread : workers) { - thread.join(); - } + transcribe::run_on_threads(n_workers, worker); // mel[m, t] = sum_k filterbank[m, k] * power[t, k]. #if TRANSCRIBE_HAS_BLAS diff --git a/src/transcribe-batch-util.cpp b/src/transcribe-batch-util.cpp index 5fa509b63..e0d389bef 100644 --- a/src/transcribe-batch-util.cpp +++ b/src/transcribe-batch-util.cpp @@ -110,15 +110,7 @@ bool parallel_for_all(int n, int n_threads, const std::function & wor } }; - std::vector pool; - pool.reserve(static_cast(n_threads - 1)); - for (int w = 0; w < n_threads - 1; ++w) { - pool.emplace_back(worker); - } - worker(); // the calling thread participates - for (auto & th : pool) { - th.join(); - } + run_on_threads(n_threads, [&](int) { worker(); }); // the calling thread participates return all_ok.load(std::memory_order_relaxed); } diff --git a/src/transcribe-batch-util.h b/src/transcribe-batch-util.h index cb2586d6d..7aeef25b6 100644 --- a/src/transcribe-batch-util.h +++ b/src/transcribe-batch-util.h @@ -13,7 +13,9 @@ #include #include +#include #include +#include #include struct ggml_tensor; @@ -32,6 +34,55 @@ namespace transcribe { // falling back to the per-utterance path). bool parallel_for_all(int n, int n_threads, const std::function & work); +// Run `fn(tid)` once for every tid in [0, n_threads): tid 0 on the calling +// thread, the rest on their own threads. Every launched thread is joined on +// every path, and exceptions reach the caller instead of std::terminate: +// - a thread that fails to launch stops further launches, tid 0 is not run, +// and the launch error is rethrown once the launched threads have joined; +// - otherwise, if any fn(tid) threw, the lowest tid's exception is rethrown +// after all threads have joined. +// n_threads <= 1 runs fn(0) inline. `Thread` is a test seam (fault injection +// for thread launch); production code uses the default. +template void run_on_threads(int n_threads, Fn && fn) { + if (n_threads <= 1) { + fn(0); + return; + } + std::vector errors(static_cast(n_threads)); + auto guarded = [&fn, &errors](int tid) { + try { + fn(tid); + } catch (...) { + errors[static_cast(tid)] = std::current_exception(); + } + }; + std::vector pool; + pool.reserve(static_cast(n_threads - 1)); + std::exception_ptr launch_error; + for (int tid = 1; tid < n_threads; ++tid) { + try { + pool.emplace_back(guarded, tid); + } catch (...) { + launch_error = std::current_exception(); + break; + } + } + if (!launch_error) { + guarded(0); + } + for (auto & th : pool) { + th.join(); + } + if (launch_error) { + std::rethrow_exception(launch_error); + } + for (const auto & e : errors) { + if (e) { + std::rethrow_exception(e); + } + } +} + // Default CPU thread count for the ggml backends and the host parallel-for // when the caller passes n_threads <= 0. Counts the CPUs the process may // actually run on (the affinity mask via sched_getaffinity on Linux / diff --git a/src/transcribe-mel.cpp b/src/transcribe-mel.cpp index 085197e0a..0893486d6 100644 --- a/src/transcribe-mel.cpp +++ b/src/transcribe-mel.cpp @@ -490,21 +490,10 @@ transcribe_status MelFrontend::compute(const float * pcm, // Per-thread worker dispatcher. Strided frame assignment (stride = // stft_threads) keeps load balanced and per-thread memory access - // patterns mostly sequential. + // patterns mostly sequential. run_on_threads joins every thread and + // hands worker exceptions back to the caller. auto run_threaded = [&](auto && worker) { - if (stft_threads <= 1) { - worker(0); - return; - } - std::vector pool; - pool.reserve(static_cast(stft_threads - 1)); - for (int tid = 1; tid < stft_threads; ++tid) { - pool.emplace_back(worker, tid); - } - worker(0); - for (auto & th : pool) { - th.join(); - } + run_on_threads(stft_threads, worker); }; if (!n_fft_is_pow2) { @@ -579,6 +568,14 @@ transcribe_status MelFrontend::compute(const float * pcm, FFTSetupD fft_setup = vDSP_create_fftsetupD(log2n, FFT_RADIX2); const size_t half_n = static_cast(n_fft / 2); + // Destroyed on every exit, including a worker exception rethrown by + // run_threaded. + struct FftSetupGuard { + FFTSetupD setup; + + ~FftSetupGuard() { vDSP_destroy_fftsetupD(setup); } + } fft_setup_guard{ fft_setup }; + auto worker = [&](int tid) { std::vector fft_real(half_n); std::vector fft_imag(half_n); @@ -603,7 +600,6 @@ transcribe_status MelFrontend::compute(const float * pcm, } }; run_threaded(worker); - vDSP_destroy_fftsetupD(fft_setup); #else // Hand-rolled radix-2 Cooley-Tukey FFT fallback for non-Apple. auto worker = [&](int tid) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 93da81287..e6f049e14 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -335,6 +335,23 @@ transcribe_apply_warnings(transcribe_role_resolve_unit) add_test(NAME transcribe_role_resolve_unit COMMAND transcribe_role_resolve_unit) +# ----------------------------------------------------------------------------- +# Host thread pool: join on every path, exceptions reach the caller +# ----------------------------------------------------------------------------- + +add_executable(transcribe_run_on_threads_unit + run_on_threads_unit.cpp) + +target_link_libraries(transcribe_run_on_threads_unit PRIVATE transcribe) + +target_include_directories(transcribe_run_on_threads_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_run_on_threads_unit) + +add_test(NAME transcribe_run_on_threads_unit + COMMAND transcribe_run_on_threads_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- diff --git a/tests/run_on_threads_unit.cpp b/tests/run_on_threads_unit.cpp new file mode 100644 index 000000000..d7a1d7692 --- /dev/null +++ b/tests/run_on_threads_unit.cpp @@ -0,0 +1,195 @@ +// run_on_threads_unit.cpp - transcribe::run_on_threads joins every launched +// thread on every path and hands worker / launch exceptions to the caller +// instead of calling std::terminate (H5). + +#include "transcribe-batch-util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// std::thread wrapper whose construction fails on the k-th launch, like a +// system out of threads. Counts joins so the test can see every launched +// thread was joined before the error surfaced. +struct FlakyThread { + static int fail_at; // 0-based launch index that throws; -1 = never + static int launches; + static std::atomic joins; + + std::thread th; + + template explicit FlakyThread(F && f, A &&... a) { + if (launches++ == fail_at) { + throw std::system_error(std::make_error_code(std::errc::resource_unavailable_try_again)); + } + th = std::thread(std::forward(f), std::forward(a)...); + } + + void join() { + th.join(); + joins.fetch_add(1); + } +}; + +int FlakyThread::fail_at = -1; +int FlakyThread::launches = 0; +std::atomic FlakyThread::joins{ 0 }; + +void reset_flaky(int fail_at) { + FlakyThread::fail_at = fail_at; + FlakyThread::launches = 0; + FlakyThread::joins = 0; +} + +void test_every_tid_runs_once() { + for (int n : { 0, 1, 2, 7 }) { + std::vector> hits(static_cast(n > 0 ? n : 1)); + transcribe::run_on_threads(n, [&](int tid) { hits[static_cast(tid)].fetch_add(1); }); + for (auto & h : hits) { + CHECK(h.load() == 1); + } + } +} + +void test_worker_throw_on_caller_thread() { + std::atomic done{ 0 }; + bool caught = false; + try { + transcribe::run_on_threads(4, [&](int tid) { + if (tid == 0) { + throw std::runtime_error("tid0"); + } + done.fetch_add(1); + }); + } catch (const std::runtime_error &) { + caught = true; + } + CHECK(caught); + CHECK(done.load() == 3); // every pool thread ran to completion and was joined +} + +void test_worker_throw_on_pool_thread() { + std::atomic done{ 0 }; + bool caught = false; + try { + transcribe::run_on_threads(4, [&](int tid) { + if (tid == 2) { + throw std::bad_alloc(); + } + done.fetch_add(1); + }); + } catch (const std::bad_alloc &) { + caught = true; // exception type preserved across the thread boundary + } + CHECK(caught); + CHECK(done.load() == 3); +} + +void test_lowest_tid_exception_wins() { + int which = -1; + try { + transcribe::run_on_threads(4, [&](int tid) { + if (tid == 1 || tid == 3) { + throw std::runtime_error(tid == 1 ? "1" : "3"); + } + }); + } catch (const std::runtime_error & e) { + which = e.what()[0] - '0'; + } + CHECK(which == 1); +} + +void test_launch_failure_joins_launched_threads() { + // Launch index 0 is tid 1. Failing at index 2 (tid 3) means tids 1 and 2 + // were launched and must be joined; tid 0 must not run. + reset_flaky(2); + std::atomic ran{ 0 }; + bool tid0_ran = false; + bool caught = false; + try { + transcribe::run_on_threads(5, [&](int tid) { + if (tid == 0) { + tid0_ran = true; + } + ran.fetch_add(1); + }); + } catch (const std::system_error &) { + caught = true; + } + CHECK(caught); + CHECK(!tid0_ran); + CHECK(FlakyThread::joins.load() == 2); + CHECK(ran.load() == 2); +} + +void test_first_launch_failure() { + reset_flaky(0); + bool caught = false; + try { + transcribe::run_on_threads(3, [](int) {}); + } catch (const std::system_error &) { + caught = true; + } + CHECK(caught); + CHECK(FlakyThread::joins.load() == 0); +} + +void test_launch_ok_with_seam() { + reset_flaky(-1); + std::atomic ran{ 0 }; + transcribe::run_on_threads(4, [&](int) { ran.fetch_add(1); }); + CHECK(ran.load() == 4); + CHECK(FlakyThread::joins.load() == 3); +} + +void test_parallel_for_all_propagates() { + bool caught = false; + try { + (void) transcribe::parallel_for_all(16, 4, [](int i) -> bool { + if (i == 9) { + throw std::runtime_error("work"); + } + return true; + }); + } catch (const std::runtime_error &) { + caught = true; + } + CHECK(caught); +} + +} // namespace + +int main() { + test_every_tid_runs_once(); + test_worker_throw_on_caller_thread(); + test_worker_throw_on_pool_thread(); + test_lowest_tid_exception_wins(); + test_launch_failure_joins_launched_threads(); + test_first_launch_failure(); + test_launch_ok_with_seam(); + test_parallel_for_all_propagates(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("run_on_threads_unit: ok\n"); + return EXIT_SUCCESS; +} From 201384911b0e3d226c349e62bbd32a248e445534 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 20:58:59 +0800 Subject: [PATCH 05/51] reject non-finite PCM on every ASR entry point Behavior change (D16a, H3). transcribe_run, transcribe_run_batch and transcribe_stream_feed now return TRANSCRIBE_ERR_INVALID_ARG when any input sample is NaN or +-Inf. Before, a NaN went through the front end and could come out as a confident-looking transcript. Silence stays valid. - run: rejected with the other malformed-input checks, before the previous result (scratch slot and batch_results) is touched and before the family hook runs. - run_batch: a non-finite sample in any utterance rejects the whole batch before anything is cleared; the families' batched hooks never see it. (A NULL / empty utterance remains a per-utterance failure.) - stream_feed: rejected before the hook, so the stream stays ACTIVE with its revision unchanged and the caller can keep feeding. transcribe::pcm_is_finite lives in transcribe-abi.h so the upcoming role dispatchers share it. include/transcribe.h documents the rule on all three entry points (abihash unchanged). tests/nonfinite_input_unit.cpp covers NaN / +Inf / -Inf at the start, middle and end, result and stream-state preservation, and silence; it fails before this change. Valid-input behavior unchanged: transcribe-cli CPU parity vs main and the real-model ctest results are identical. --- include/transcribe.h | 16 ++- src/transcribe-abi.h | 15 +++ src/transcribe-asr.cpp | 23 ++++ tests/CMakeLists.txt | 17 +++ tests/nonfinite_input_unit.cpp | 203 +++++++++++++++++++++++++++++++++ 5 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 tests/nonfinite_input_unit.cpp diff --git a/include/transcribe.h b/include/transcribe.h index 1a9a17b28..d8bf29590 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -1562,7 +1562,10 @@ TRANSCRIBE_API const struct transcribe_model * transcribe_get_model(const struct /* * Run one batch transcription. * - * pcm: mono float32 PCM samples in [-1.0, 1.0] at 16 kHz. + * pcm: mono float32 PCM samples in [-1.0, 1.0] at 16 kHz. Every + * sample must be finite: NaN or +-Inf returns + * TRANSCRIBE_ERR_INVALID_ARG and leaves the previous result + * in place. Silence is valid. * n_samples: number of samples in pcm. Must be strictly positive; * a non-positive count returns TRANSCRIBE_ERR_INVALID_ARG * (same rule as transcribe_stream_feed). @@ -1641,7 +1644,10 @@ TRANSCRIBE_API transcribe_status transcribe_run(struct transcribe_session * * or n_samples[i] <= 0) fails only that * utterance and is reported there. * TRANSCRIBE_ERR_INVALID_ARG session / pcm / n_samples NULL, n <= 0, - * or the session is in an ACTIVE stream. + * any utterance holds a non-finite sample + * (NaN / +-Inf; the previous result is + * preserved), or the session is in an + * ACTIVE stream. * TRANSCRIBE_ERR_BAD_STRUCT_SIZE params->struct_size below the minimum. * TRANSCRIBE_ERR_NOT_IMPLEMENTED the model has no run path at all. * ... plus the same shared-param rejections as transcribe_run @@ -2181,8 +2187,10 @@ TRANSCRIBE_API transcribe_status transcribe_stream_begin(struct transcribe_sessi * Feed PCM into the active stream. 16 kHz mono float32, same as * transcribe_run. * - * pcm must be non-null and n_samples must be strictly greater than - * zero. Polling the stream without supplying audio is unsupported — + * pcm must be non-null, n_samples must be strictly greater than zero, + * and every sample must be finite (NaN / +-Inf returns + * TRANSCRIBE_ERR_INVALID_ARG and the stream stays ACTIVE, unchanged). + * Polling the stream without supplying audio is unsupported — * use the stream accessors (transcribe_stream_revision, * transcribe_stream_get_text, transcribe_stream_n_committed_*, * transcribe_stream_last_status, transcribe_stream_get_state) to inspect diff --git a/src/transcribe-abi.h b/src/transcribe-abi.h index 05d38cb39..7f5903c7b 100644 --- a/src/transcribe-abi.h +++ b/src/transcribe-abi.h @@ -9,6 +9,7 @@ #include "transcribe.h" +#include #include #include @@ -31,6 +32,20 @@ inline int enum_field_raw(const void * field) { static_assert(sizeof(transcribe_backend_request) == sizeof(int), "public enums must be int-sized for raw boundary reads"); +// Every PCM entry point rejects non-finite samples (NaN / +-Inf) with +// TRANSCRIBE_ERR_INVALID_ARG before it modifies any result or stream state: +// a NaN would otherwise flow through the front end and come out as a +// confident-looking result. Silence is valid input. O(n), negligible next to +// any encoder. +inline bool pcm_is_finite(const float * pcm, int n_samples) { + for (int i = 0; i < n_samples; ++i) { + if (!std::isfinite(pcm[i])) { + return false; + } + } + return true; +} + // Strict size check for every caller-owned struct (inputs AND outputs): // struct_size smaller than the prefix the library relies on is rejected. // `0` is invalid — defaults are reached by passing NULL where the entry diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp index f1ac4cc16..1ce89ba56 100644 --- a/src/transcribe-asr.cpp +++ b/src/transcribe-asr.cpp @@ -49,6 +49,7 @@ using transcribe::api_guard_status; using transcribe::api_guard_value; using transcribe::api_guard_void; using transcribe::enum_field_raw; +using transcribe::pcm_is_finite; namespace { @@ -1178,6 +1179,10 @@ static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { return TRANSCRIBE_ERR_INVALID_ARG; } + // Rejected before the hook, so the stream stays ACTIVE and usable. + if (!pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } // begin already confirmed model/arch/hook; we re-check defensively // so a malformed context (never happens in practice from a real // begin) does not deref a null function pointer. @@ -1387,6 +1392,9 @@ static transcribe_status run_one_inner(struct transcribe_session * sess if (n_samples <= 0) { return TRANSCRIBE_ERR_INVALID_ARG; } + if (!pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } // NULL params means "all defaults" (transcribe vs translate, no // timestamps, etc.). A well-formed default run is not a malformed // call, so it proceeds and replaces the previous result. @@ -1524,6 +1532,11 @@ static transcribe_status transcribe_run_impl(struct transcribe_session * // call must not touch the session at all — the API smoke test probes // this with a fake (session *)0x1 and a NULL pcm / non-positive // n_samples, expecting INVALID_ARG with no dereference. + // Non-finite PCM is malformed too: reject it here, before batch_results + // is touched (run_one_inner repeats the check for the batch fallback). + if (pcm != nullptr && n_samples > 0 && !pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } if (session != nullptr && pcm != nullptr && n_samples > 0) { session->batch_results.clear(); } @@ -1589,6 +1602,16 @@ static transcribe_status transcribe_run_batch_impl(struct transcribe_session * if (session == nullptr || pcm == nullptr || n_samples == nullptr || n <= 0) { return TRANSCRIBE_ERR_INVALID_ARG; } + // Non-finite samples in any utterance reject the whole batch here, before + // anything is cleared (unlike a NULL / empty utterance, which is a + // per-utterance failure): the families' batched hooks never see them. + for (int i = 0; i < n; ++i) { + if (pcm[i] != nullptr && n_samples[i] > 0 && !pcm_is_finite(pcm[i], n_samples[i])) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_run_batch: utterance %d has non-finite samples (NaN / Inf)", i); + return TRANSCRIBE_ERR_INVALID_ARG; + } + } struct transcribe_run_params params_defaults; transcribe_run_params_init(¶ms_defaults); if (params == nullptr) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e6f049e14..ed50d8a77 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -352,6 +352,23 @@ transcribe_apply_warnings(transcribe_run_on_threads_unit) add_test(NAME transcribe_run_on_threads_unit COMMAND transcribe_run_on_threads_unit) +# ----------------------------------------------------------------------------- +# Non-finite PCM rejection on every ASR entry point +# ----------------------------------------------------------------------------- + +add_executable(transcribe_nonfinite_input_unit + nonfinite_input_unit.cpp) + +target_link_libraries(transcribe_nonfinite_input_unit PRIVATE transcribe) + +target_include_directories(transcribe_nonfinite_input_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_nonfinite_input_unit) + +add_test(NAME transcribe_nonfinite_input_unit + COMMAND transcribe_nonfinite_input_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- diff --git a/tests/nonfinite_input_unit.cpp b/tests/nonfinite_input_unit.cpp new file mode 100644 index 000000000..38503873d --- /dev/null +++ b/tests/nonfinite_input_unit.cpp @@ -0,0 +1,203 @@ +// nonfinite_input_unit.cpp - every ASR PCM entry point rejects NaN / Inf +// samples with INVALID_ARG before any result or stream state is modified +// and before the family hook runs (D16a). Silence stays valid input. + +#include "transcribe-arch.h" +#include "transcribe-model.h" +#include "transcribe-session.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +int g_run_calls = 0; +int g_run_batch_calls = 0; +int g_feed_calls = 0; + +transcribe_status fake_run(transcribe_session * s, const float *, int, const transcribe_run_params *) { + ++g_run_calls; + s->full_text = "fresh"; + s->has_result = true; + return TRANSCRIBE_OK; +} + +transcribe_status fake_run_batch(transcribe_session * s, + const float * const *, + const int *, + int n, + const transcribe_run_params *) { + ++g_run_batch_calls; + for (int i = 0; i < n; ++i) { + s->batch_results.push_back(s->capture_result(TRANSCRIBE_OK)); + } + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_begin(transcribe_session *, + const transcribe_run_params *, + const transcribe_stream_params *) { + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_feed(transcribe_session *, const float *, int, transcribe_stream_update *) { + ++g_feed_calls; + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_finalize(transcribe_session *, transcribe_stream_update *) { + return TRANSCRIBE_OK; +} + +const transcribe::Arch k_arch = { + /* .name = */ "fake-nonfinite", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ fake_run, + /* .run_batch = */ fake_run_batch, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ fake_stream_begin, + /* .stream_feed = */ fake_stream_feed, + /* .stream_finalize = */ fake_stream_finalize, +}; + +const float k_nan = std::numeric_limits::quiet_NaN(); +const float k_inf = std::numeric_limits::infinity(); +const float k_bads[] = { k_nan, k_inf, -k_inf }; + +// A session holding a prior single result and a prior batch result. +void seed(transcribe_session & s, transcribe_model & m) { + s.model = &m; + s.full_text = "previous"; + s.has_result = true; + s.batch_results.clear(); + s.batch_results.push_back(s.capture_result(TRANSCRIBE_OK)); +} + +bool prior_intact(const transcribe_session & s) { + return s.full_text == "previous" && s.has_result && s.batch_results.size() == 1; +} + +void test_run_rejects_nonfinite() { + transcribe_model model; + model.arch = &k_arch; + for (float bad : k_bads) { + for (size_t pos : { size_t{ 0 }, size_t{ 799 }, size_t{ 1599 } }) { + transcribe_session s; + seed(s, model); + std::vector pcm(1600, 0.1f); + pcm[pos] = bad; + g_run_calls = 0; + CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_run_calls == 0); + CHECK(prior_intact(s)); + } + } +} + +void test_run_accepts_silence() { + transcribe_model model; + model.arch = &k_arch; + transcribe_session s; + seed(s, model); + std::vector pcm(1600, 0.0f); + g_run_calls = 0; + CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + CHECK(g_run_calls == 1); + CHECK(s.full_text == "fresh"); +} + +void test_batch_rejects_nonfinite_whole_batch() { + transcribe_model model; + model.arch = &k_arch; + for (float bad : k_bads) { + transcribe_session s; + seed(s, model); + std::vector a(800, 0.1f), b(800, 0.1f), c(800, 0.1f); + b[400] = bad; + const float * pcm[] = { a.data(), b.data(), c.data() }; + const int n_samples[] = { 800, 800, 800 }; + g_run_calls = 0; + g_run_batch_calls = 0; + CHECK(transcribe_run_batch(&s, pcm, n_samples, 3, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_run_calls == 0); + CHECK(g_run_batch_calls == 0); + CHECK(prior_intact(s)); + } +} + +void test_batch_accepts_silence() { + transcribe_model model; + model.arch = &k_arch; + transcribe_session s; + seed(s, model); + std::vector a(800, 0.0f), b(800, 0.0f); + const float * pcm[] = { a.data(), b.data() }; + const int n_samples[] = { 800, 800 }; + g_run_batch_calls = 0; + CHECK(transcribe_run_batch(&s, pcm, n_samples, 2, nullptr) == TRANSCRIBE_OK); + CHECK(g_run_batch_calls == 1); + CHECK(transcribe_batch_n_results(&s) == 2); +} + +void test_stream_feed_rejects_nonfinite() { + transcribe_model model; + model.arch = &k_arch; + model.caps.supports_streaming = true; + transcribe_session s; + s.model = &model; + CHECK(transcribe_stream_begin(&s, nullptr, nullptr) == TRANSCRIBE_OK); + const int revision = transcribe_stream_revision(&s); + + for (float bad : k_bads) { + std::vector pcm(1600, 0.1f); + pcm[1000] = bad; + g_feed_calls = 0; + transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + CHECK(transcribe_stream_feed(&s, pcm.data(), static_cast(pcm.size()), &upd) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_feed_calls == 0); + // The stream stays usable: rejected before the hook, so no FAILED. + CHECK(transcribe_stream_get_state(&s) == TRANSCRIBE_STREAM_ACTIVE); + CHECK(transcribe_stream_revision(&s) == revision); + } + + std::vector ok(1600, 0.0f); + g_feed_calls = 0; + CHECK(transcribe_stream_feed(&s, ok.data(), static_cast(ok.size()), nullptr) == TRANSCRIBE_OK); + CHECK(g_feed_calls == 1); + CHECK(transcribe_stream_finalize(&s, nullptr) == TRANSCRIBE_OK); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); + + test_run_rejects_nonfinite(); + test_run_accepts_silence(); + test_batch_rejects_nonfinite_whole_batch(); + test_batch_accepts_silence(); + test_stream_feed_rejects_nonfinite(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("nonfinite_input_unit: ok\n"); + return EXIT_SUCCESS; +} From bc2e05178316b3029fa4129d423204992ed13135 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:07:13 +0800 Subject: [PATCH 06/51] rust: characterize session compute rules Pin today's behavior of the native compute sites before they move onto a shared helper. New tests/compute_rules.rs (model-gated, skips cleanly without the canaries): - busy_errors_name_the_refused_call: run, run_batch and a second stream on another session of the model each get Error::Busy with their own exact message while a stream is active. - option_validation_precedes_busy_check: a NUL in run options is reported as Error::Nul, not Busy, by run/run_batch/stream. - busy_check_precedes_native_stream_validation: a wrong-family stream extension is refused as Busy while another stream is live, and as InvalidArgument once the lease is free. - failed_stream_begin_does_not_take_the_lease. - ended_stream_never_releases_another_sessions_lease: reset() or drop of a finalized stream leaves another session's lease alone. - run_and_run_batch_across_threads_serialize: runs and batches on sessions of one shared model queue and each get their own results. - cancel_token_is_retained_until_cleared: the session keeps the token's flag alive after the caller drops it, and clear_cancel_token stops it. Also a compile_fail(E0505) doctest on Stream showing a session cannot be dropped while its stream borrows it, so no native free can race a call. (Rustdoc on stable does not check the code; verified by hand that the snippet fails only with E0505.) All new tests pass on the unchanged code (whisper-tiny Q8_0 and moonshine-streaming-tiny Q8_0 canaries). --- bindings/rust/transcribe-cpp/src/session.rs | 14 + .../transcribe-cpp/tests/compute_rules.rs | 298 ++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 bindings/rust/transcribe-cpp/tests/compute_rules.rs diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index d81647d8b..9008a2094 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -542,6 +542,20 @@ fn attach_partial(err: Error, partial: Box) -> Error { /// [`Stream::text`], and end input with [`Stream::finalize`]. Dropping the /// `Stream` (without finalizing) abandons it and returns the session to idle. /// `Send` but not `Sync`, like the session it borrows. +/// +/// The borrow means a session can't be freed while its stream (or any other +/// call on it) is still in use, so no native free ever races a compute: +/// +/// ```compile_fail,E0505 +/// use transcribe_cpp::{Model, RunOptions, StreamOptions}; +/// let model = Model::load("model.gguf").unwrap(); +/// let mut session = model.session().unwrap(); +/// let mut stream = session +/// .stream(&RunOptions::default(), &StreamOptions::default()) +/// .unwrap(); +/// drop(session); // error[E0505]: cannot move out of `session` while borrowed +/// stream.feed(&[0.0; 160]).unwrap(); +/// ``` pub struct Stream<'a> { session: &'a mut Session, // True while this stream holds the model's compute lease. Set at begin, diff --git a/bindings/rust/transcribe-cpp/tests/compute_rules.rs b/bindings/rust/transcribe-cpp/tests/compute_rules.rs new file mode 100644 index 000000000..100551ca5 --- /dev/null +++ b/bindings/rust/transcribe-cpp/tests/compute_rules.rs @@ -0,0 +1,298 @@ +//! Characterization of the rules every native compute call follows: the +//! per-model compute lock and stream lease, the order of the checks in front +//! of the native call (and therefore which error a racing caller sees), lease +//! ownership across sessions, cancel-token install/clear, and serialization of +//! run/run_batch across threads. These pin today's behavior so the compute +//! sites can be moved onto one helper without changing it. Model-gated: skip +//! cleanly when the canary GGUFs / jfk.wav are absent. + +mod common; + +use std::path::PathBuf; +use std::sync::Arc; +use std::thread; + +use transcribe_cpp::{ + CancelToken, Error, Feature, Model, RunOptions, StreamExtension, StreamOptions, StreamState, +}; + +const RUN_BUSY: &str = "a stream is active on this model; finish or drop it before run()"; +const BATCH_BUSY: &str = "a stream is active on this model; finish or drop it before run_batch()"; +const STREAM_BUSY: &str = "a stream is already active on this model"; + +fn streaming_fixtures(test: &str) -> Option<(PathBuf, Vec)> { + match (common::smoke_streaming_model(), common::smoke_audio()) { + (Some(m), Some(a)) => Some((m, a)), + _ => { + eprintln!("skip {test}: streaming model/audio absent"); + None + } + } +} + +fn expect_busy(r: transcribe_cpp::Result, want: &str, what: &str) { + match r { + Err(Error::Busy(msg)) => assert_eq!(msg, want, "{what}: Busy message"), + other => panic!("{what}: expected Error::Busy({want:?}), got {other:?}"), + } +} + +fn wrong_family_stream() -> StreamOptions { + StreamOptions { + family: Some(StreamExtension::ParakeetStream(Default::default())), + ..Default::default() + } +} + +#[test] +fn busy_errors_name_the_refused_call() { + // Each refusing site carries its own message: run and run_batch name the + // call, a second stream says "already active". + let Some((model_path, pcm)) = streaming_fixtures("busy_errors_name_the_refused_call") else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + + expect_busy(s2.run(&pcm, &RunOptions::default()), RUN_BUSY, "run"); + expect_busy( + s2.run_batch(&[&pcm], &RunOptions::default()), + BATCH_BUSY, + "run_batch", + ); + expect_busy( + s2.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + "stream", + ); + // The refusals did not disturb the live stream. + stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + assert_eq!(stream1.state(), StreamState::Active); +} + +#[test] +fn option_validation_precedes_busy_check() { + // Run options are marshalled before the compute lock is taken, so a bad + // option is reported as itself even while another session streams. + let Some((model_path, pcm)) = streaming_fixtures("option_validation_precedes_busy_check") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + let _stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + + let bad = RunOptions { + language: Some("e\0n".into()), + ..Default::default() + }; + assert!( + matches!(s2.run(&pcm, &bad), Err(Error::Nul(_))), + "run: NUL option must win over Busy" + ); + assert!( + matches!(s2.run_batch(&[&pcm], &bad), Err(Error::Nul(_))), + "run_batch: NUL option must win over Busy" + ); + assert!( + matches!( + s2.stream(&bad, &StreamOptions::default()), + Err(Error::Nul(_)) + ), + "stream: NUL option must win over Busy" + ); +} + +#[test] +fn busy_check_precedes_native_stream_validation() { + // A begin that the native layer would reject (wrong-family extension) is + // refused as Busy first when another session's stream holds the lease. + let Some((model_path, pcm)) = + streaming_fixtures("busy_check_precedes_native_stream_validation") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + expect_busy( + s2.stream(&RunOptions::default(), &wrong_family_stream()), + STREAM_BUSY, + "wrong-family stream while busy", + ); + drop(stream1); + + // With the lease free, the same begin reaches native validation. + let err = s2 + .stream(&RunOptions::default(), &wrong_family_stream()) + .unwrap_err(); + assert!( + matches!(err, Error::InvalidArgument(_)), + "wrong-family stream when idle: {err:?}" + ); +} + +#[test] +fn failed_stream_begin_does_not_take_the_lease() { + let Some((model_path, _pcm)) = + streaming_fixtures("failed_stream_begin_does_not_take_the_lease") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + + assert!(s1 + .stream(&RunOptions::default(), &wrong_family_stream()) + .is_err()); + let stream2 = s2.stream(&RunOptions::default(), &StreamOptions::default()); + assert!( + stream2.is_ok(), + "a failed begin must leave the lease free: {stream2:?}" + ); +} + +#[test] +fn ended_stream_never_releases_another_sessions_lease() { + // After finalize() the lease belongs to whoever takes it next; a later + // reset() or drop of the ended stream must not free that other lease. + let Some((model_path, pcm)) = + streaming_fixtures("ended_stream_never_releases_another_sessions_lease") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + let mut s3 = model.session().unwrap(); + let chunk = &pcm[..pcm.len().min(1600)]; + + // finalize -> other session takes the lease -> ended stream reset()s. + { + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(chunk).unwrap(); + stream1.finalize().unwrap(); + let stream2 = s2 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.reset(); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + "after ended stream reset", + ); + drop(stream2); + } + + // finalize -> other session takes the lease -> ended stream is dropped. + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(chunk).unwrap(); + stream1.finalize().unwrap(); + let stream2 = s2 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + drop(stream1); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + "after ended stream drop", + ); + expect_busy(s3.run(&pcm, &RunOptions::default()), RUN_BUSY, "run"); + + // Releasing the real holder frees the model. + drop(stream2); + let stream3 = s3 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + assert_eq!(stream3.state(), StreamState::Active); +} + +#[test] +fn run_and_run_batch_across_threads_serialize() { + // One-shot runs and batches on different sessions of one model queue on + // the compute lock (never Busy), and each gets its own results back. + let Some((model_path, pcm)) = + common::smoke_fixtures("run_and_run_batch_across_threads_serialize") + else { + return; + }; + let model = Arc::new(Model::load(&model_path).unwrap()); + let pcm = Arc::new(pcm); + let handles: Vec<_> = (0..4) + .map(|i| { + let model = Arc::clone(&model); + let pcm = Arc::clone(&pcm); + thread::spawn(move || { + let mut s = model.session().unwrap(); + if i % 2 == 0 { + vec![s.run(&pcm, &RunOptions::default()).unwrap().text] + } else { + s.run_batch(&[&pcm, &pcm], &RunOptions::default()) + .unwrap() + .into_iter() + .map(|r| r.unwrap().text) + .collect() + } + }) + }) + .collect(); + for (i, h) in handles.into_iter().enumerate() { + let texts = h.join().unwrap(); + assert_eq!(texts.len(), if i % 2 == 0 { 1 } else { 2 }); + for t in texts { + assert!(t.to_lowercase().contains("country"), "thread {i}: {t}"); + } + } +} + +#[test] +fn cancel_token_is_retained_until_cleared() { + // The session keeps the installed token's flag alive (the caller may drop + // every handle) and keeps consulting it on later runs until cleared. + let Some((model_path, pcm)) = common::smoke_fixtures("cancel_token_is_retained_until_cleared") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + if !model.supports(Feature::Cancellation) { + eprintln!("skip: model does not support cancellation"); + return; + } + let mut session = model.session().unwrap(); + { + let token = CancelToken::new(); + token.cancel(); + session.set_cancel_token(&token); + } // every caller-side handle to the flag is gone + + for attempt in 0..2 { + match session.run(&pcm, &RunOptions::default()) { + Err(Error::Aborted { .. }) => assert!(session.was_aborted()), + other => panic!("attempt {attempt}: pre-cancelled token must abort, got {other:?}"), + } + } + + session.clear_cancel_token(); + let result = session.run(&pcm, &RunOptions::default()).unwrap(); + assert!(result.text.to_lowercase().contains("country")); + assert!(!session.was_aborted()); +} From 6e1bbf6eeb643cdc1ff03b792383455e2e400a5e Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:10:07 +0800 Subject: [PATCH 07/51] rust: route session compute through with_compute The per-model compute lock and stream-lease check were open-coded at seven sites (Session::run, run_batch, stream begin; Stream::feed, finalize, reset, drop). They now all go through one crate-private helper: Session::with_compute(&mut self, refuse_if_streaming: Option<&str>, f: impl FnOnce(*mut transcribe_session, &mut bool) -> R) -> Result It takes the model's compute lock (poison is recovered as before), then if refuse_if_streaming is Some(msg) and a stream holds the lease returns Error::Busy(msg), else calls f with the raw session handle and the lease flag under the lock. The closure gets the lease flag so stream begin can take it, and finalize/reset/drop can clear it only if this stream still holds it, atomically with their native call. Behavior-preserving. Each site keeps its exact order and error text: - run / run_batch: options marshalled and lengths checked before the lock; Busy "...before run()" / "...before run_batch()"; results are still materialized after the lock is released. - stream: Busy "a stream is already active on this model" before the native begin; the begin status is still checked under the lock and the lease only set on success. - feed / finalize / reset / drop: no busy check (the caller holds the lease); finalize/reset/drop clear the lease only when holds_lease. Verified (macOS arm64, metal, all model-tier canaries set incl. voxtral): cargo xtask bindgen --check, cargo fmt --all --check, cargo test -p transcribe-cpp-sys, cargo test -p transcribe-cpp, the serde test, clippy -D warnings, and the five examples. Per-test results identical to the base commit, plus the new compute_rules tests and doctest passing; no model-tier test skipped. --- bindings/rust/transcribe-cpp/src/model.rs | 3 +- bindings/rust/transcribe-cpp/src/session.rs | 169 ++++++++++---------- 2 files changed, 83 insertions(+), 89 deletions(-) diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index 1e32daabc..c971c6385 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -73,7 +73,8 @@ pub(crate) struct ModelInner { /// is `true` while a stream is ACTIVE (from `begin` until the earliest of /// `finalize` / `reset` / the `Stream` being dropped); a held lock plus a /// `true` flag is how `run`/`run_batch`/`stream` detect and refuse an - /// overlapping compute. See the module docs. + /// overlapping compute. Only taken through `Session::with_compute`. See + /// the module docs. pub(crate) compute_lock: Mutex, } diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index 9008a2094..2331a1a8a 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -175,19 +175,10 @@ impl Session { // The compute path is serialized per model; hold the lock for the native // call, then materialize this session's own result storage. Refuse a run // that would overlap an active stream on another session (the C contract). - let status = { - let guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is active on this model; finish or drop it before run()".into(), - )); - } - unsafe { sys::transcribe_run(self.ptr, pcm.as_ptr(), n, ¶ms) } - }; + let status = self.with_compute( + Some("a stream is active on this model; finish or drop it before run()"), + |ptr, _| unsafe { sys::transcribe_run(ptr, pcm.as_ptr(), n, ¶ms) }, + )?; match status { s if s == sys::transcribe_status::TRANSCRIBE_OK => Ok(self.materialize_run()), @@ -221,19 +212,12 @@ impl Session { .collect::>()?; let n = clamp_len(pcms.len())?; - let status = { - let guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is active on this model; finish or drop it before run_batch()".into(), - )); - } - unsafe { sys::transcribe_run_batch(self.ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) } - }; + let status = self.with_compute( + Some("a stream is active on this model; finish or drop it before run_batch()"), + |ptr, _| unsafe { + sys::transcribe_run_batch(ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) + }, + )?; // OK and ABORTED both populate one per-utterance slot per input; fold // the batch-level abort into the per-utterance view (the C contract). @@ -293,32 +277,56 @@ impl Session { pub fn stream(&mut self, run: &RunOptions, stream: &StreamOptions) -> Result> { let (run_params, _lang, _target, _family, _prompting) = build_run_params(run)?; let (stream_params, _stream_family) = build_stream_params(stream); - { - // Claim the model's compute lease for the whole stream lifetime: a - // stream spans begin..drop, so per-call locking alone would let a - // second stream (or a run) on another session race it. - let mut guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is already active on this model".into(), - )); - } - check( - unsafe { sys::transcribe_stream_begin(self.ptr, &run_params, &stream_params) }, - "stream begin", - )?; - *guard = true; // released at finalize/reset, or by Stream::drop - } + // Claim the model's compute lease for the whole stream lifetime: a + // stream spans begin..drop, so per-call locking alone would let a + // second stream (or a run) on another session race it. + self.with_compute( + Some("a stream is already active on this model"), + |ptr, lease| -> Result<()> { + check( + unsafe { sys::transcribe_stream_begin(ptr, &run_params, &stream_params) }, + "stream begin", + )?; + *lease = true; // released at finalize/reset, or by Stream::drop + Ok(()) + }, + )??; Ok(Stream { session: self, holds_lease: true, }) } + /// Make one native compute call on this session under the model's compute + /// lock. Every call that drives native compute (run, batch, stream + /// begin/feed/finalize/reset/drop) goes through here. + /// + /// Under the lock, in order: if `refuse_if_streaming` is `Some(msg)` and a + /// stream on this model holds the lease, return `Error::Busy(msg)` without + /// calling `f`; otherwise call `f` with this session's native handle and + /// the lease flag (so a stream begin/end can take or release the lease + /// atomically with its native call). Pass `None` from the stream that holds + /// the lease; the call then never fails. The lock is released when this + /// returns, so callers copy results out of the session's own result + /// storage afterwards. + pub(crate) fn with_compute( + &mut self, + refuse_if_streaming: Option<&str>, + f: impl FnOnce(*mut sys::transcribe_session, &mut bool) -> R, + ) -> Result { + let mut lease = self + .model + .compute_lock + .lock() + .unwrap_or_else(|e| e.into_inner()); + if let Some(msg) = refuse_if_streaming { + if *lease { + return Err(Error::Busy(msg.into())); + } + } + Ok(f(self.ptr, &mut lease)) + } + // --- result materialization (top-level / single accessors) --------------- fn materialize_run(&self) -> Transcript { @@ -580,16 +588,14 @@ impl Drop for Stream<'_> { // state) and release the lease IF this stream still holds it — under the // lock. After finalize/reset the lease is already gone (and may now be // another session's), so only clear it when holds_lease is still true. - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_reset(self.session.ptr) }; - if self.holds_lease { - *guard = false; - } + // (No busy check, so this cannot fail.) + let held = self.holds_lease; + let _ = self.session.with_compute(None, |ptr, lease| { + unsafe { sys::transcribe_stream_reset(ptr) }; + if held { + *lease = false; + } + }); } } @@ -599,17 +605,11 @@ impl Stream<'_> { let n = clamp_len(pcm.len())?; let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; - let status = { - // This stream already holds the model's compute lease (it is in - // flight); the lock here just serializes the native call. - let _guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_feed(self.session.ptr, pcm.as_ptr(), n, &mut update) } - }; + // This stream already holds the model's compute lease (it is in + // flight); the lock here just serializes the native call. + let status = self.session.with_compute(None, |ptr, _| unsafe { + sys::transcribe_stream_feed(ptr, pcm.as_ptr(), n, &mut update) + })?; check(status, "stream feed")?; Ok(StreamUpdate::from_raw(&update)) } @@ -619,22 +619,17 @@ impl Stream<'_> { pub fn finalize(&mut self) -> Result { let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; - let status = { - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - let st = unsafe { sys::transcribe_stream_finalize(self.session.ptr, &mut update) }; + let held = self.holds_lease; + let status = self.session.with_compute(None, |ptr, lease| { + let st = unsafe { sys::transcribe_stream_finalize(ptr, &mut update) }; // Finalize ends the active stream (Finished, or Failed on error); // either way it is no longer active, so free the model's compute // lease now rather than holding other sessions off until drop. - if self.holds_lease { - *guard = false; + if held { + *lease = false; } st - }; + })?; self.holds_lease = false; check(status, "stream finalize")?; Ok(StreamUpdate::from_raw(&update)) @@ -645,16 +640,14 @@ impl Stream<'_> { /// active), so other sessions of the model can proceed without waiting for /// this `Stream` to drop. pub fn reset(&mut self) { - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_reset(self.session.ptr) }; - if self.holds_lease { - *guard = false; - } + // No busy check, so this cannot fail. + let held = self.holds_lease; + let _ = self.session.with_compute(None, |ptr, lease| { + unsafe { sys::transcribe_stream_reset(ptr) }; + if held { + *lease = false; + } + }); self.holds_lease = false; } From d8ffa353f732f49576be619a8c8b80eafc5fc240 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:11:07 +0800 Subject: [PATCH 08/51] swift: characterize session compute rules Pin the rules every native compute call follows today, ahead of moving the model lock behind one helper: - run, runBatch, stream begin/feed/finalize/reset and a dropped Stream's deinit reset all wait on the model-wide runLock (tested by holding the lock and checking the call blocks, then completes once released); - run/runBatch/stream validate string options before taking the lock, and refuse an active stream with .busy only after taking it (exact busy messages pinned, including a second stream on the same session); - the async bridge removes its token after the call, including after an abort; the sync path leaves a caller token installed; - a Session keeps its Model alive, a Stream keeps its Session and Model alive, and an in-flight async run survives the caller dropping refs; - results (Transcript, batch results, stream snapshot/text) are owned copies that later compute does not disturb. Tests only, no library change. All 11 pass on the current code; a quick mutation check (dropping feed's lock, the stream busy check, or moving run's option check under the lock) makes them fail. --- .../ComputeRulesTests.swift | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift diff --git a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift new file mode 100644 index 000000000..9e980e844 --- /dev/null +++ b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift @@ -0,0 +1,364 @@ +import Foundation +import XCTest + +@testable import TranscribeCpp + +/// Characterization of the rules every native compute call follows today, so +/// moving them behind one helper can be checked against the old behavior: +/// +/// - every compute site (run, runBatch, stream begin/feed/finalize/reset, and a +/// dropped `Stream`'s reset) waits on the model-wide `runLock`, which all +/// sessions of one model share; +/// - run / runBatch / stream begin validate their string options BEFORE taking +/// the lock, and refuse an active stream with `.busy` only AFTER taking it; +/// - the async bridge installs its own cancel token only when none is set, and +/// removes it when the call returns (including after an abort); +/// - a `Session` keeps its `Model` alive and a `Stream` keeps its `Session` +/// alive, so dropping the caller's references mid-use is safe; +/// - results are owned Swift copies that later compute cannot disturb. +final class ComputeRulesTests: XCTestCase { + // MARK: - Helpers + + private final class Box { var value: T? } + + /// Hold `model.runLock` on this thread, run `op` on a background queue, + /// and require it to still be blocked after a short wait; then release the + /// lock and require it to finish. Bounded waits, so a regression fails + /// rather than hanging the suite. + private func assertWaitsOnModelLock( + _ model: Model, _ what: String, + file: StaticString = #filePath, line: UInt = #line, + _ op: @escaping () -> Void + ) { + let done = DispatchSemaphore(value: 0) + model.runLock.lock() + var held = true + defer { if held { model.runLock.unlock() } } + DispatchQueue.global().async { op(); done.signal() } + let early = done.wait(timeout: .now() + 0.3) + XCTAssertEqual( + early, .timedOut, "\(what) must wait on the model compute lock", + file: file, line: line) + model.runLock.unlock() + held = false + if early == .success { return } + XCTAssertEqual( + done.wait(timeout: .now() + 60), .success, + "\(what) must proceed once the model lock is free", file: file, line: line) + } + + /// Hold `model.runLock` on this thread and require `op` (on a background + /// queue) to finish anyway: it must not touch the lock. + private func assertSkipsModelLock( + _ model: Model, _ what: String, + file: StaticString = #filePath, line: UInt = #line, + _ op: @escaping () -> Void + ) { + let done = DispatchSemaphore(value: 0) + model.runLock.lock() + DispatchQueue.global().async { op(); done.signal() } + let result = done.wait(timeout: .now() + 10) + model.runLock.unlock() + XCTAssertEqual( + result, .success, "\(what) must not wait on the model compute lock", + file: file, line: line) + if result == .timedOut { _ = done.wait(timeout: .now() + 60) } + } + + private func busyMessage(_ error: Error?) -> String? { + guard case TranscribeError.busy(let message)? = error else { return nil } + return message + } + + private func isInvalidArgument(_ error: Error?) -> Bool { + guard case TranscribeError.invalidArgument? = error else { return false } + return true + } + + // MARK: - Model-wide compute exclusion + + func testRunAndRunBatchWaitOnTheModelLock() throws { + let (path, pcm) = try Fixtures.modelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + + let run = Box() + assertWaitsOnModelLock(model, "run") { run.value = try? s1.run(pcm) } + XCTAssertTrue(run.value?.text.lowercased().contains("country") == true) + + // A sibling session of the same model waits on the same lock. + let batch = Box<[Result]>() + assertWaitsOnModelLock(model, "runBatch") { batch.value = try? s2.runBatch([pcm]) } + XCTAssertEqual(batch.value?.count, 1) + XCTAssertTrue( + (try? batch.value?.first?.get().text.lowercased().contains("country")) == true) + } + + func testEveryStreamSiteWaitsOnTheModelLock() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let model = try Model(path: path) + let session = try model.session() + let chunk = Array(pcm.prefix(1600)) + let stream = Box() + + assertWaitsOnModelLock(model, "stream begin") { stream.value = try? session.stream() } + XCTAssertNotNil(stream.value) + XCTAssertTrue(model.streamActive) + + let fed = Box() + assertWaitsOnModelLock(model, "feed") { fed.value = try? stream.value?.feed(chunk) } + XCTAssertNotNil(fed.value) + + let finalized = Box() + assertWaitsOnModelLock(model, "finalize") { + finalized.value = try? stream.value?.finalize() + } + XCTAssertEqual(finalized.value?.isFinal, true) + XCTAssertFalse(model.streamActive) + stream.value = nil + + stream.value = try session.stream() + _ = try stream.value?.feed(chunk) + let state = Box() + assertWaitsOnModelLock(model, "reset") { state.value = stream.value?.reset() } + XCTAssertEqual(state.value, .idle) + XCTAssertFalse(model.streamActive) + stream.value = nil + + // A dropped ACTIVE stream resets under the lock in deinit. The last + // reference is released on the background thread so the test thread + // (which holds the lock) never runs the deinit itself. + stream.value = try session.stream() + _ = try stream.value?.feed(chunk) + XCTAssertTrue(model.streamActive) + assertWaitsOnModelLock(model, "Stream deinit") { stream.value = nil } + XCTAssertFalse(model.streamActive) + XCTAssertEqual(try session.stream().reset(), .idle) + } + + // MARK: - Active-stream rejection and its ordering + + /// The `.busy` check runs only once the lock is held: with a stream active + /// and the lock held elsewhere, run / runBatch / stream block first and + /// then refuse. Also pins the exact busy messages. + func testActiveStreamIsRefusedAfterTakingTheLock() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + let active = try s1.stream() + _ = try active.feed(Array(pcm.prefix(1600))) + + let runError = Box() + assertWaitsOnModelLock(model, "run while streaming") { + do { _ = try s2.run(pcm) } catch { runError.value = error } + } + XCTAssertEqual( + busyMessage(runError.value), + "a stream is active on this model; finish or drop it before run()") + + let batchError = Box() + assertWaitsOnModelLock(model, "runBatch while streaming") { + do { _ = try s2.runBatch([pcm]) } catch { batchError.value = error } + } + XCTAssertEqual( + busyMessage(batchError.value), + "a stream is active on this model; finish or drop it before runBatch()") + + let streamError = Box() + assertWaitsOnModelLock(model, "stream while streaming") { + do { _ = try s2.stream() } catch { streamError.value = error } + } + XCTAssertEqual(busyMessage(streamError.value), "a stream is already active on this model") + + // The same session's second stream gets the binding's `.busy` too, not + // a native error. + XCTAssertEqual( + busyMessage(Result { try s1.stream() }.failure), + "a stream is already active on this model") + + active.reset() + } + + /// String-option validation runs before the lock and before the busy + /// check: with a stream active AND the lock held, a NUL in an option is + /// reported as `.invalidArgument` without waiting. + func testOptionValidationPrecedesLockAndBusyCheck() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + let active = try s1.stream() + let bad = RunOptions(language: "e\0n") + + let runError = Box() + assertSkipsModelLock(model, "run option check") { + do { _ = try s2.run(pcm, options: bad) } catch { runError.value = error } + } + XCTAssertTrue(isInvalidArgument(runError.value), "\(String(describing: runError.value))") + + let batchError = Box() + assertSkipsModelLock(model, "runBatch option check") { + do { _ = try s2.runBatch([pcm], options: bad) } catch { batchError.value = error } + } + XCTAssertTrue(isInvalidArgument(batchError.value), "\(String(describing: batchError.value))") + + let streamError = Box() + assertSkipsModelLock(model, "stream option check") { + do { _ = try s2.stream(bad) } catch { streamError.value = error } + } + XCTAssertTrue( + isInvalidArgument(streamError.value), "\(String(describing: streamError.value))") + + active.reset() + } + + // MARK: - Cancel hook install / cleanup + + /// The async bridge installs a token only for the call and removes it + /// afterwards, on success and after an abort alike. + func testAsyncBridgeRemovesItsTokenAfterTheCall() async throws { + let (path, pcm) = try Fixtures.modelAndAudio() + let session = try Model(path: path).session() + XCTAssertNil(session.cancelToken) + + _ = try await session.run(pcm) + XCTAssertNil(session.cancelToken, "bridged token must be removed after run") + _ = try await session.runBatch([pcm]) + XCTAssertNil(session.cancelToken, "bridged token must be removed after runBatch") + + let long = Array(repeating: pcm, count: 6).flatMap { $0 } + let task = Task { try await session.run(long) } + task.cancel() + _ = try? await task.value + XCTAssertTrue(session.wasAborted) + XCTAssertNil(session.cancelToken, "bridged token must be removed after an aborted run") + + // The cancelled token is not left behind: the next call is not aborted. + _ = try await session.run(pcm) + XCTAssertFalse(session.wasAborted) + XCTAssertNil(session.cancelToken) + } + + /// The synchronous path never installs or clears a token: a caller token + /// stays installed (and stays cancelled) across runs until the caller + /// clears it. + func testSyncRunLeavesCallerTokenInstalled() throws { + let (path, pcm) = try Fixtures.modelAndAudio() + let session = try Model(path: path).session() + let token = CancellationToken() + token.cancel() + session.setCancellationToken(token) + for _ in 0..<2 { + XCTAssertThrowsError(try session.run(pcm)) { error in + guard case TranscribeError.aborted = error else { + return XCTFail("expected .aborted, got \(error)") + } + } + XCTAssertTrue(session.cancelToken === token) + } + session.clearCancellationToken() + XCTAssertNil(session.cancelToken) + _ = try session.run(pcm) + XCTAssertFalse(session.wasAborted) + } + + // MARK: - Keep-alive + + func testSessionKeepsModelAliveForRun() throws { + let (path, pcm) = try Fixtures.modelAndAudio() + weak var weakModel: Model? + var session: Session? = try { + let model = try Model(path: path) + weakModel = model + return try model.session() + }() + XCTAssertNotNil(weakModel, "a live Session must keep its Model alive") + XCTAssertTrue(try session!.run(pcm).text.lowercased().contains("country")) + session = nil + XCTAssertNil(weakModel, "the Model is freed once its last Session is") + } + + func testStreamKeepsSessionAndModelAlive() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + weak var weakModel: Model? + weak var weakSession: Session? + var stream: TranscribeCpp.Stream? = try { + let model = try Model(path: path) + let session = try model.session() + weakModel = model + weakSession = session + return try session.stream() + }() + XCTAssertNotNil(weakSession, "a live Stream must keep its Session alive") + XCTAssertNotNil(weakModel, "a live Stream must keep its Model alive") + try Fixtures.drive(stream!, pcm: pcm) + XCTAssertTrue(stream!.text.full.lowercased().contains("country")) + stream = nil + XCTAssertNil(weakSession) + XCTAssertNil(weakModel) + } + + /// The caller may drop every reference while an async call is in flight: + /// the call itself keeps the session (and so the model) alive until it + /// returns. + func testCallerDropsReferencesDuringAsyncRun() async throws { + let (path, pcm) = try Fixtures.modelAndAudio() + let long = Array(repeating: pcm, count: 3).flatMap { $0 } + let task: Task = try { + let session = try Model(path: path).session() + return Task { try await session.run(long) } + }() + let transcript = try await task.value + XCTAssertTrue(transcript.text.lowercased().contains("country")) + } + + // MARK: - Copy-out + + /// Results are owned copies: later compute on the same session or on a + /// sibling session leaves them untouched. + func testResultsSurviveLaterCompute() throws { + let (path, pcm) = try Fixtures.modelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + let first = try s1.run(pcm) + let batch = try s1.runBatch([pcm]) + let text = first.text + let segments = first.segments.count + let batchText = try batch[0].get().text + XCTAssertTrue(text.lowercased().contains("country")) + + let short = Array(pcm.prefix(16000)) + _ = try s1.run(short) + _ = try s1.runBatch([short]) + _ = try s2.run(short) + + XCTAssertEqual(first.text, text) + XCTAssertEqual(first.segments.count, segments) + XCTAssertEqual(try batch[0].get().text, batchText) + XCTAssertNotEqual(try s1.run(short).text, text) + } + + func testStreamSnapshotSurvivesReset() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let session = try Model(path: path).session() + let stream = try session.stream() + try Fixtures.drive(stream, pcm: pcm) + let snapshot = stream.snapshot + let text = stream.text + XCTAssertTrue(text.full.lowercased().contains("country")) + stream.reset() + XCTAssertEqual(stream.snapshot.text, "") + XCTAssertTrue(snapshot.text.lowercased().contains("country")) + XCTAssertTrue(text.full.lowercased().contains("country")) + } +} + +private extension Result { + var failure: Failure? { + if case .failure(let error) = self { return error } + return nil + } +} From 310ec76e6a7384230b129400695dbf76b7ee9a41 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:12:53 +0800 Subject: [PATCH 09/51] swift: route session compute through withCompute Add an internal Model.withCompute(_:) (lock runLock, run the body, unlock via defer; rethrows) and move every native compute site onto it: Session.run, Session.runBatch, Session.stream (begin), Stream.feed, Stream.finalize, Stream.reset and Stream.deinit. No other code takes runLock directly now. The lock stays on Model so every session of one model shares it. Behavior-preserving. Each site keeps its checks in the same order: option NUL checks stay before the lock; the streamActive .busy refusal (run/runBatch/stream only, same messages) is the first step inside the body; finalize still releases the lease before checking the status; reset still reads `state` after the lock is released; deinit still checks holdsLease before locking; results are still copied into Swift values inside the body. reset and deinit used a manual unlock instead of defer; their bodies cannot throw, so defer is equivalent. The async cancel-token bridge stays in Session, outside the lock, as before. Verified with the CI gate (abihash check, macOS xcframework build, swift test with every TRANSCRIBE_SMOKE_* model set, the five examples): 73/73 tests pass, 0 skipped, identical to the run before this change, and the example outputs match. --- .../swift/Sources/TranscribeCpp/Model.swift | 15 ++- .../swift/Sources/TranscribeCpp/Session.swift | 99 ++++++++++--------- .../Sources/TranscribeCpp/Streaming.swift | 72 +++++++------- 3 files changed, 101 insertions(+), 85 deletions(-) diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index 6045b17b0..d38030460 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -10,7 +10,7 @@ import Foundation public final class Model: @unchecked Sendable { let ptr: OpaquePointer /// Serializes the run/feed/finalize compute path across all sessions, and - /// guards `streamActive`. + /// guards `streamActive`. Taken only through `withCompute`. let runLock = NSLock() /// The compute lease: `true` while some session holds an ACTIVE stream. /// The C contract allows at most one in-flight run/stream across ALL @@ -21,6 +21,19 @@ public final class Model: @unchecked Sendable { /// `runLock`. var streamActive = false + /// Run `body` holding this model's compute lock. Every native call that + /// computes or mutates a session's results goes through here, so all + /// sessions of one model share the C "one in-flight compute per model" + /// exclusion. `body` also copies results into owned Swift values before + /// the lock is released, and may read or change `streamActive`. Argument + /// checks that must not wait on the lock stay before the call; the + /// `streamActive` refusal, where a site has one, is `body`'s first step. + func withCompute(_ body: () throws -> R) rethrows -> R { + runLock.lock() + defer { runLock.unlock() } + return try body() + } + /// Load a model from a GGUF file. Runs the pre-1.0 version gate first. public init(path: String, options: ModelOptions = .init()) throws { try Transcribe.ensureCompatible() diff --git a/bindings/swift/Sources/TranscribeCpp/Session.swift b/bindings/swift/Sources/TranscribeCpp/Session.swift index 9357899d3..17ffc8f98 100644 --- a/bindings/swift/Sources/TranscribeCpp/Session.swift +++ b/bindings/swift/Sources/TranscribeCpp/Session.swift @@ -51,17 +51,17 @@ public final class Session { /// Transcribe one utterance. `pcm` is mono float32 at 16 kHz in [-1, 1]. public func run(_ pcm: [Float], options: RunOptions = .init()) throws -> Transcript { try options.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before run()") - } - return try options.withCParams { params in - let status = pcm.withUnsafeBufferPointer { - transcribe_run(ptr, $0.baseAddress, Int32($0.count), params) + return try model.withCompute { + if model.streamActive { + throw TranscribeError.busy( + "a stream is active on this model; finish or drop it before run()") + } + return try options.withCParams { params in + let status = pcm.withUnsafeBufferPointer { + transcribe_run(ptr, $0.baseAddress, Int32($0.count), params) + } + return try makeTranscript(status, context: "run") } - return try makeTranscript(status, context: "run") } } @@ -72,46 +72,49 @@ public final class Session { _ inputs: [[Float]], options: RunOptions = .init() ) throws -> [Result] { try options.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before runBatch()") - } - return try options.withCParams { params in - let counts = inputs.map { Int32($0.count) } - let status = withPCMPointers(inputs[...], []) { pointers in - pointers.withUnsafeBufferPointer { pp in - counts.withUnsafeBufferPointer { cc in - transcribe_run_batch( - ptr, pp.baseAddress, cc.baseAddress, Int32(inputs.count), params) + return try model.withCompute { + if model.streamActive { + throw TranscribeError.busy( + "a stream is active on this model; finish or drop it before runBatch()") + } + return try options.withCParams { params in + let counts = inputs.map { Int32($0.count) } + let status = withPCMPointers(inputs[...], []) { pointers in + pointers.withUnsafeBufferPointer { pp in + counts.withUnsafeBufferPointer { cc in + transcribe_run_batch( + ptr, pp.baseAddress, cc.baseAddress, Int32(inputs.count), params) + } } } - } - if status != TRANSCRIBE_OK && status != TRANSCRIBE_ERR_ABORTED { - throw TranscribeError.make(status, context: "run_batch") - } - let n = Int(transcribe_batch_n_results(ptr)) - return (0.. StreamUpdate { - session.model.runLock.lock() - defer { session.model.runLock.unlock() } - var update = transcribe_stream_update() - transcribe_stream_update_init(&update) - let status = frame.withUnsafeBufferPointer { - transcribe_stream_feed(session.ptr, $0.baseAddress, Int32($0.count), &update) + try session.model.withCompute { + var update = transcribe_stream_update() + transcribe_stream_update_init(&update) + let status = frame.withUnsafeBufferPointer { + transcribe_stream_feed(session.ptr, $0.baseAddress, Int32($0.count), &update) + } + try TranscribeError.check(status, context: "stream_feed") + return StreamUpdate(update) } - try TranscribeError.check(status, context: "stream_feed") - return StreamUpdate(update) } /// Signal end of input; flushes buffered audio and emits remaining text. @@ -138,24 +138,24 @@ public final class Stream { /// proceed without waiting for this `Stream` to drop. @discardableResult public func finalize() throws -> StreamUpdate { - session.model.runLock.lock() - defer { session.model.runLock.unlock() } - var update = transcribe_stream_update() - transcribe_stream_update_init(&update) - let status = transcribe_stream_finalize(session.ptr, &update) - if holdsLease { session.model.streamActive = false; holdsLease = false } - try TranscribeError.check(status, context: "stream_finalize") - return StreamUpdate(update) + try session.model.withCompute { + var update = transcribe_stream_update() + transcribe_stream_update_init(&update) + let status = transcribe_stream_finalize(session.ptr, &update) + if holdsLease { session.model.streamActive = false; holdsLease = false } + try TranscribeError.check(status, context: "stream_finalize") + return StreamUpdate(update) + } } /// Abandon the stream and return the session to idle. Releases the model's /// compute lease (the stream is no longer active). @discardableResult public func reset() -> StreamState { - session.model.runLock.lock() - transcribe_stream_reset(session.ptr) - if holdsLease { session.model.streamActive = false; holdsLease = false } - session.model.runLock.unlock() + session.model.withCompute { + transcribe_stream_reset(session.ptr) + if holdsLease { session.model.streamActive = false; holdsLease = false } + } return state } @@ -192,18 +192,18 @@ extension Session { _ runOptions: RunOptions = .init(), _ streamOptions: StreamOptions = .init() ) throws -> Stream { try runOptions.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy("a stream is already active on this model") - } - let status = runOptions.withCParams { runParams in - streamOptions.withCParams { streamParams in - transcribe_stream_begin(ptr, runParams, streamParams) + return try model.withCompute { + if model.streamActive { + throw TranscribeError.busy("a stream is already active on this model") + } + let status = runOptions.withCParams { runParams in + streamOptions.withCParams { streamParams in + transcribe_stream_begin(ptr, runParams, streamParams) + } } + try TranscribeError.check(status, context: "stream_begin") + model.streamActive = true + return Stream(self) } - try TranscribeError.check(status, context: "stream_begin") - model.streamActive = true - return Stream(self) } } From a80060485d21c547334f0621e9b08b116bba7d4d Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:10:04 +0800 Subject: [PATCH 10/51] ts: characterize session compute rules Add test/compute-rules.test.mjs, pinning the rules every native compute call follows today so the next commit can route run / runBatch / stream begin / feed / finalize through one helper and show nothing moved: - model-wide exclusion: a sibling session's run queues FIFO behind the in-flight one, and only the lock holder is marked in flight; runBatch and finalize use their own in-flight labels - active-stream rejection: run, runBatch and stream begin are refused with Busy (exact "cannot :" prefix) on the stream's own session and on a sibling; disposal is rechecked inside the lock BEFORE the lease, so a queued call on a disposed session sees "disposed", not Busy - feed / finalize do not recheck disposal inside the lock: one queued before session dispose still runs to completion - a non-finite feed is rejected with InvalidArgument and the native stream stays active, but the binding releases the model lease on any non-OK feed status, so a sibling run is no longer refused. Pinned as current behavior, not endorsed - cancel: the abort listener exists only for the call, is removed after success and after abort, is never installed when the call is refused (Busy or disposed), and a later signal-less run is not aborted; a pre-aborted runBatch resolves with per-item Aborted + partials - session or model dispose while run / runBatch / feed is on a worker defers the native free behind it and the call still returns its full result (reads during the call report "in flight" before "disposed") - results are copied out before the lock is released: two back-to-back runs on one session each keep their own text All 13 pass against the current code (3 repeated runs), alongside the existing 52. --- .../typescript/test/compute-rules.test.mjs | 333 ++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 bindings/typescript/test/compute-rules.test.mjs diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs new file mode 100644 index 000000000..11c8fe493 --- /dev/null +++ b/bindings/typescript/test/compute-rules.test.mjs @@ -0,0 +1,333 @@ +// Characterization of the rules every native compute call obeys: model-wide +// exclusion, active-stream rejection, the disposed recheck and its order +// against Busy, cancel install/cleanup, the model staying alive for the call, +// deferred frees when dispose() races an in-flight worker call, and results +// copied out before the model lock is released. These pin today's behavior, +// including where call sites differ, so routing compute through one helper can +// be checked as behavior-preserving. + +import assert from "node:assert/strict"; +import { getEventListeners } from "node:events"; +import { modelTest, MODEL, STREAMING_MODEL, jfk } from "./common.mjs"; +import { TranscribeModel, Aborted, Busy, InvalidArgument } from "../dist/index.js"; + +const second = () => jfk().subarray(0, 16000); +const half = () => { + const pcm = jfk(); + return pcm.subarray(0, pcm.length / 2); +}; + +// ---- model-wide exclusion -------------------------------------------------- + +modelTest("compute is exclusive model-wide: a sibling run waits its turn", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const order = []; + const pa = a.run(jfk()).then((r) => (order.push("a"), r)); + const pb = b.run(jfk()).then((r) => (order.push("b"), r)); + await Promise.resolve(); // a's body reaches its worker; b's is queued + // Only the lock holder is marked in flight; the queued sibling is not. + assert.throws(() => a.limits, /run\(\).*in flight/); + assert.doesNotThrow(() => b.limits); + const [ra, rb] = await Promise.all([pa, pb]); + assert.deepEqual(order, ["a", "b"], "FIFO: the queued sibling runs after"); + assert.match(ra.text, /ask not/i); + assert.match(rb.text, /ask not/i); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("runBatch and finalize mark the session in flight with their own label", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const s = m.createSession(); + const pending = s.runBatch([second()]); + await Promise.resolve(); + assert.throws(() => s.limits, /runBatch\(\).*in flight/); + await pending; + assert.doesNotThrow(() => s.limits); + + const stream = await s.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + const fin = stream.finalize(); + await Promise.resolve(); + assert.throws(() => stream.text, /feed\(\)\/finalize\(\).*in flight/); + assert.throws(() => s.wasAborted, /feed\(\)\/finalize\(\).*in flight/); + await fin; + assert.doesNotThrow(() => stream.text); + stream.reset(); + s.dispose(); + } finally { + m.dispose(); + } +}); + +// ---- active-stream rejection ----------------------------------------------- + +modelTest("an active stream refuses every offline call and begin, same or sibling session", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + for (const s of [a, b]) { + await assert.rejects(() => s.run(second()), (e) => e instanceof Busy && /^cannot run:/.test(e.message)); + await assert.rejects( + () => s.runBatch([second()]), + (e) => e instanceof Busy && /^cannot runBatch:/.test(e.message), + ); + await assert.rejects( + () => s.stream(), + (e) => e instanceof Busy && /^cannot begin a stream:/.test(e.message), + ); + } + // The stream holding the lease keeps feeding through all of that. + await stream.feed(second()); + await stream.finalize(); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("disposal is checked before the stream lease inside the lock", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + // Each call is queued while a's stream holds the lease, then its session + // is disposed before the queued body runs: the caller sees "disposed", + // not Busy. + for (const call of [(s) => s.run(second()), (s) => s.runBatch([second()]), (s) => s.stream()]) { + const b = m.createSession(); + const p = call(b); + b.dispose(); + await assert.rejects(p, (e) => !(e instanceof Busy) && /disposed/i.test(e.message)); + } + stream.reset(); + a.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a feed or finalize queued before session dispose still runs", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + // feed/finalize check disposal only before queuing, not inside the lock: + // a call already queued when the session is disposed runs to completion + // (the native free is queued behind it). + const a = m.createSession(); + const sa = await a.stream({ commitPolicy: "stable_prefix" }); + const feed = sa.feed(second()); + a.dispose(); + const u = await feed; + assert.equal(typeof u.revision, "number"); + + const b = m.createSession(); + const sb = await b.stream({ commitPolicy: "stable_prefix" }); + await sb.feed(second()); + const fin = sb.finalize(); + b.dispose(); + const f = await fin; + assert.equal(f.isFinal, true); + + // Both leases are gone: a fresh session can stream. + const c = m.createSession(); + const sc = await c.stream({ commitPolicy: "stable_prefix" }); + sc.reset(); + c.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a rejected feed releases the stream lease (current behavior)", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + const bad = new Float32Array(1600); + bad[5] = Number.NaN; + await assert.rejects(() => stream.feed(bad), InvalidArgument); + // The native stream stays ACTIVE after a non-finite feed, but the binding + // releases the model-wide lease on any non-OK feed status, so a sibling + // run is no longer refused with Busy. Pinned as-is so that changing it is + // a deliberate, visible decision. The probe run is itself non-finite, so + // native rejects it up front and no compute overlaps the live stream. + assert.equal(stream.state, "active"); + await assert.rejects(() => b.run(bad), (e) => e instanceof InvalidArgument && !(e instanceof Busy)); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +// ---- cancel install / cleanup ---------------------------------------------- + +modelTest("the abort listener lives only for the call and is removed after", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const ac = new AbortController(); + const p = s.run(jfk(), { signal: ac.signal }); + await Promise.resolve(); + assert.equal(getEventListeners(ac.signal, "abort").length, 1, "installed for the run"); + const r = await p; + assert.equal(r.aborted, false); + assert.equal(getEventListeners(ac.signal, "abort").length, 0, "removed after the run"); + + const pb = s.runBatch([second()], { signal: ac.signal }); + await Promise.resolve(); + assert.equal(getEventListeners(ac.signal, "abort").length, 1, "installed for the batch"); + await pb; + assert.equal(getEventListeners(ac.signal, "abort").length, 0, "removed after the batch"); + s.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("an aborted call uninstalls its callback; the next call is not aborted", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const ac = new AbortController(); + ac.abort(); + const items = await s.runBatch([jfk(), second()], { signal: ac.signal }); + assert.equal(items.length, 2); + for (const it of items) { + assert.equal(it.ok, false); + assert.ok(it.error instanceof Aborted); + assert.ok(it.error.partialResult); + } + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + await assert.rejects(() => s.run(jfk(), { signal: ac.signal }), Aborted); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + + const r = await s.run(jfk()); + assert.equal(r.aborted, false); + assert.equal(s.wasAborted, false); + assert.match(r.text, /ask not/i); + s.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a refused call never installs its abort listener", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + const ac = new AbortController(); + await assert.rejects(() => b.run(second(), { signal: ac.signal }), Busy); + await assert.rejects(() => b.runBatch([second()], { signal: ac.signal }), Busy); + assert.equal(getEventListeners(ac.signal, "abort").length, 0, "Busy is decided before install"); + + const c = m.createSession(); + const p = c.run(second(), { signal: ac.signal }); + c.dispose(); + await assert.rejects(p, /disposed/i); + assert.equal(getEventListeners(ac.signal, "abort").length, 0, "disposed is decided before install"); + + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +// ---- dispose racing an in-flight call; keepalive; copy-out ----------------- + +modelTest("session dispose during an in-flight run defers the free; the result survives", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const p = s.run(jfk()); + await Promise.resolve(); // the run is on its worker + s.dispose(); + // The in-flight guard is checked before disposal on reads. + assert.throws(() => s.limits, /run\(\).*in flight/); + const r = await p; + assert.match(r.text, /ask not what your country/i); + assert.ok(r.segments.length >= 1); + assert.throws(() => s.limits, /disposed/); + + const s2 = m.createSession(); + const pb = s2.runBatch([jfk(), half()]); + await Promise.resolve(); + s2.dispose(); + const items = await pb; + assert.equal(items.length, 2); + assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); + } finally { + m.dispose(); + } +}); + +modelTest("model dispose during an in-flight run keeps the model alive for the call", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + const s = m.createSession(); + const p = s.run(jfk()); + await Promise.resolve(); + m.dispose(); // disposes s too; both native frees queue behind the run + assert.throws(() => m.capabilities, /disposed/); + const r = await p; + assert.match(r.text, /ask not what your country/i); + assert.throws(() => s.limits, /disposed/); + + const m2 = await TranscribeModel.load(MODEL); + const s2 = m2.createSession(); + const pb = s2.runBatch([jfk()]); + await Promise.resolve(); + m2.dispose(); + const items = await pb; + assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); +}); + +modelTest("model dispose during an in-flight feed keeps the model alive for the call", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + const s = m.createSession(); + const stream = await s.stream({ commitPolicy: "stable_prefix" }); + const p = stream.feed(second()); + await Promise.resolve(); + m.dispose(); + const u = await p; + assert.equal(typeof u.revision, "number"); + assert.throws(() => stream.text, /disposed/i); +}); + +modelTest("results are copied out before the next call on the session overwrites them", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + // Both run on the same session back to back; the second overwrites the + // session's native result storage as soon as it gets the lock. + const [full, part] = await Promise.all([s.run(jfk()), s.run(half())]); + assert.match(full.text, /ask what you can do for your country/i); + assert.doesNotMatch(part.text, /ask what you can do for your country/i); + + const [b1, b2] = await Promise.all([s.runBatch([jfk()]), s.runBatch([half()])]); + assert.match(b1[0].result.text, /ask what you can do for your country/i); + assert.doesNotMatch(b2[0].result.text, /ask what you can do for your country/i); + s.dispose(); + } finally { + m.dispose(); + } +}); From e50ded0baa94c293229569257b15b529a69eb125 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:12:31 +0800 Subject: [PATCH 11/51] ts: route session compute through #exclusive Every native compute call now goes through one private Session helper, #exclusive(gate, body), instead of five inline copies of the lock sequence. Pure refactor, no behavior change. What moved: - #exclusive queues on the model-wide FIFO lock (the Mutex still lives on TranscribeModel and is shared by all its sessions), then applies the gate in the old order: disposal recheck, then the stream-lease Busy check. Each site passes its old settings: run / runBatch / stream begin recheck disposal and refuse with Busy("run" / "runBatch" / "begin a stream"); feed / finalize (STREAM_LEASE_HOLDER) do neither, as before. - The in-flight window (install the abort callback, then set the in-flight label; on close clear the label, then uninstall) is a ComputeWindow the body opens around its worker await. The labels stay "run()", "runBatch()" and "feed()/finalize()". stream begin opens no window (synchronous native call), and result copy-out stays in the body, after the window closes and before the lock is released. - Stream reaches the helper through the existing module-private SESSION_CONTROL WeakMap (the STREAM_TEARDOWN pattern). Its enterCompute/leaveCompute entries are replaced by exclusive(); nothing new is exported. - #exclusive is not async. Refusals come back as rejected promises and the body's promise is returned unchanged, so promise settle timing matches the inline lock bodies. The one non-literal translation: run/runBatch used to clear the in-flight label unconditionally, while feed/finalize cleared it only if it still matched. The window now always uses the matching check. Only the lock holder sets the label, so at close it always equals the caller's kind, and the two forms behave the same. deferFree (stream reset, Session.dispose, Model.dispose) stays as is. Those are queued frees, not computes, and must not recheck disposal. Verified against a worktree-built libtranscribe (Metal) with every TRANSCRIBE_SMOKE_* model set: generate.py --check, version sync, tsc, npm test (65/65; the 52 baseline tests plus the 13 characterization tests, the same results as before), the 5 CI examples and the deno smoke. Mutation checks: swapping the disposed/Busy order, or making feed recheck disposal, each fails the expected characterization tests. --- bindings/typescript/src/index.ts | 123 ++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index c2e07394a..4a3275e4e 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -713,9 +713,40 @@ const STREAM_TEARDOWN = new WeakMap< { deactivate(): void; invalidate(): void; releaseLease(): void } >(); +/** + * How Session.#exclusive admits a compute call once it holds the model lock. + * The checks run in field order: disposal, then the stream lease. + */ +interface ComputeGate { + /** + * Recheck disposal inside the lock: dispose() may have run after the caller + * captured the handle but before its queued body. feed()/finalize() check + * only before queuing (their native free is queued behind them). + */ + recheckDisposed: boolean; + /** + * The op named in the Busy refusal while a stream holds the model lease, or + * null when the caller is that stream (feed()/finalize()). + */ + busyOp: string | null; +} + +/** + * Opens the in-flight window around one worker call: installs `signal`'s abort + * callback, then marks the session busy as `kind` so result reads fail fast. + * The returned close() clears the mark, then uninstalls the callback; call it + * in a `finally` right after the worker await. + */ +type ComputeWindow = (kind: string, signal?: AbortSignal) => () => void; + +/** The gate for calls made by the stream that holds the model lease. */ +const STREAM_LEASE_HOLDER: ComputeGate = { recheckDisposed: false, busyOp: null }; + interface SessionControl { - enterCompute(kind: string): void; - leaveCompute(kind: string): void; + exclusive( + gate: ComputeGate, + body: (open: ComputeWindow) => Promise, + ): Promise; currentCompute(): string | null; isCurrentStream(stream: Stream): boolean; replaceCurrentStream(stream: Stream): void; @@ -750,12 +781,7 @@ export class Session { this.#lock = lock; this.#untrack = untrack; SESSION_CONTROL.set(this, { - enterCompute: (kind) => { - this.#inFlight = kind; - }, - leaveCompute: (kind) => { - if (this.#inFlight === kind) this.#inFlight = null; - }, + exclusive: (gate, body) => this.#exclusive(gate, body), currentCompute: () => this.#inFlight, isCurrentStream: (stream) => this.#activeStream === stream, replaceCurrentStream: (stream) => { @@ -785,6 +811,43 @@ export class Session { } } + /** + * Run `body` as this session's one native compute: the single path every + * run / batch / stream begin / feed / finalize takes (Stream reaches it via + * SESSION_CONTROL). It queues on the model-wide FIFO lock, so no two + * computes on one model overlap and queued native frees (deferFree) run only + * after it drains. Once the lock is held, `gate` is applied — disposal, then + * the stream lease — before `body` runs. `body` opens the in-flight window + * (abort callback + in-flight mark) around its worker await, and copies its + * result out before returning, i.e. before the lock is released. + * + * Deliberately not `async`: refusals are returned as rejected promises and + * `body`'s promise is returned as-is, so settle timing matches the inline + * lock bodies this replaced. + */ + #exclusive( + gate: ComputeGate, + body: (open: ComputeWindow) => Promise, + ): Promise { + return this.#lock.run(() => { + if (gate.recheckDisposed && this.#disposed) + return Promise.reject(new TranscribeError("session has been disposed")); + if (gate.busyOp !== null && this.#lock.streamActive) + return Promise.reject(busyError(gate.busyOp)); + return body(this.#openComputeWindow); + }); + } + + /** See ComputeWindow. Only valid inside an #exclusive body. */ + #openComputeWindow: ComputeWindow = (kind, signal) => { + const cancel = this.#installAbort(signal); + this.#inFlight = kind; + return () => { + if (this.#inFlight === kind) this.#inFlight = null; + cancel?.(); + }; + }; + get limits(): SessionLimits { this.#assertNotComputing("limits"); const n = this.#n; @@ -814,18 +877,13 @@ export class Session { const p = this.#buildRunParams(opts); - return this.#lock.run(async () => { - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("run"); - const cancel = this.#installAbort(opts.signal); + return this.#exclusive({ recheckDisposed: true, busyOp: "run" }, async (open) => { + const close = open("run()", opts.signal); let status: number; - this.#inFlight = "run()"; try { status = await callAsync(F.run, h, samples, samples.length, p); } finally { - this.#inFlight = null; - cancel?.(); + close(); } if ( @@ -914,13 +972,9 @@ export class Session { const counts = Int32Array.from(arrays, (a) => a.length); const p = this.#buildRunParams(opts); - return this.#lock.run(async () => { - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("runBatch"); - const cancel = this.#installAbort(opts.signal); + return this.#exclusive({ recheckDisposed: true, busyOp: "runBatch" }, async (open) => { + const close = open("runBatch()", opts.signal); let status: number; - this.#inFlight = "runBatch()"; try { status = await callAsync( F.runBatch, @@ -931,8 +985,7 @@ export class Session { p, ); } finally { - this.#inFlight = null; - cancel?.(); + close(); } // A batch returns OK even with per-utterance failures; only a top-level // error (or a whole-batch abort) is fatal here. @@ -1009,12 +1062,10 @@ export class Session { if (opts.family) sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); - return this.#lock.run(async () => { - // Recheck inside the lock: dispose() may have run after we captured `h` - // but before this queued body — don't begin a stream on a dead session. - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("begin a stream"); + // The gate rechecks disposal inside the lock: dispose() may have run after + // we captured `h` but before this queued body — don't begin a stream on a + // dead session. Begin is a synchronous native call, so no in-flight window. + return this.#exclusive({ recheckDisposed: true, busyOp: "begin a stream" }, async () => { check(n, F.streamBegin(h, rp, sp), "transcribe_stream_begin"); this.#lock.streamActive = true; // claim the lease for the whole stream lifetime // The Stream holds the Session (not a raw handle) so its calls fail fast @@ -1164,7 +1215,7 @@ export class Stream { this.#assertCurrent("feed"); if (!this.#active) throw new TranscribeError("stream has been reset"); const samples = toFloat32(pcm); - return this.#lock.run(async () => { + return this.#sessionControl.exclusive(STREAM_LEASE_HOLDER, async (open) => { const u: any = {}; n.F.streamUpdateInit(u); // The native feed runs on a libuv worker. While it is in flight the @@ -1172,7 +1223,7 @@ export class Stream { // is single-threaded (transcribe.h), and stream_get_text hands back // pointers the feed may free/realloc. Flag the owning session so every // result getter fails fast instead of racing into a use-after-free. - this.#sessionControl.enterCompute("feed()/finalize()"); + const close = open("feed()/finalize()"); try { const status = await callAsync( n.F.streamFeed, @@ -1189,7 +1240,7 @@ export class Stream { } check(n, status, "transcribe_stream_feed"); } finally { - this.#sessionControl.leaveCompute("feed()/finalize()"); + close(); } return toStreamUpdate(u); }); @@ -1201,10 +1252,10 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("finalize stream"); if (!this.#active) throw new TranscribeError("stream has been reset"); - return this.#lock.run(async () => { + return this.#sessionControl.exclusive(STREAM_LEASE_HOLDER, async (open) => { const u: any = {}; n.F.streamUpdateInit(u); - this.#sessionControl.enterCompute("feed()/finalize()"); + const close = open("feed()/finalize()"); try { check( n, @@ -1212,7 +1263,7 @@ export class Stream { "transcribe_stream_finalize", ); } finally { - this.#sessionControl.leaveCompute("feed()/finalize()"); + close(); // Finalize ends the active stream (FINISHED on success, FAILED on // error), so the model is free again — release the lease either way. this.#releaseLease(); From 82740039af29b67d99a1382c390adf110f5a5ed1 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:17:54 +0800 Subject: [PATCH 12/51] ts: keep the stream lease when a feed is rejected feed() released the model-wide stream lease on any non-OK status. Since 20138491, native rejects non-finite samples before the family hook and leaves the stream ACTIVE. The binding still freed the lease, so a sibling run or stream could start alongside a live stream, and later feeds ran without holding the lease. The lease now follows the native stream lifecycle. On a failed feed, the binding reads transcribe_stream_get_state, still inside the same exclusive slot after the native feed returns, and releases the lease only if the stream is no longer ACTIVE. - Pre-hook rejections leave the stream ACTIVE and keep the lease. That covers non-finite samples and any other pre-hook INVALID_ARG or NOT_IMPLEMENTED. The stream stays usable and siblings stay Busy. - A hook failure (native FAILED) releases the lease exactly as before. - A feed on a stream that is no longer ACTIVE (e.g. FINISHED) releases as before. That release is a no-op, because finalize already released. - finalize is unchanged and always releases. The feed() comment now describes this, and the README gets one sentence. JS-side checks still stop empty and misaligned PCM, reset or stale wrappers, and disposed sessions before any native call. Tests: the "a rejected feed releases the stream lease (current behavior)" characterization is replaced by two tests. - A NaN feed throws InvalidArgument, and the stream stays active. Sibling run, sibling stream, and a same-session run are all Busy. A following valid feed and finalize succeed, and then a sibling run goes through. - A feed that fails inside the native hook still releases the lease. The test makes the hook fail with an always-firing abort callback, which moonshine-streaming's hook polls, returning ABORTED and leaving the stream FAILED. The callback is installed on the session handle through koffi, test-only. The stream reads "failed" with an Aborted lastStatus, and a sibling run then goes through. The first fails on the previous code. The second fails if the release is removed. Verified with a worktree-built libtranscribe (Metal) and every TRANSCRIBE_SMOKE_* model set: - generate.py --check, version sync, and tsc all pass - npm test passes 66/66 - the 5 CI examples and the deno smoke all pass --- bindings/typescript/README.md | 5 +- bindings/typescript/src/index.ts | 16 +++-- .../typescript/test/compute-rules.test.mjs | 66 ++++++++++++++++--- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 5121e7999..836c68a99 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -164,7 +164,10 @@ lease only after the native teardown runs on the shared queue, so the slot is never freed early. Issue the teardown before the next `stream()`/`run()` and it is correctly serialized — `stream.reset(); const next = await session.stream()` works without awaiting the (void) `reset()`. The reverse order — beginning before -the teardown — is refused with `Busy`, by design. +the teardown — is refused with `Busy`, by design. A rejected `feed()` follows the +stream's state: input refused up front (e.g. NaN/Inf samples) leaves the stream +`"active"` and the lease held, so you can keep feeding; a failure inside the +model moves it to `"failed"` and frees the lease. Because the compute is genuinely on another thread, **do not touch a session while a call against it is in flight** — it is single-threaded in the C library: diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 4a3275e4e..ff75f0516 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -1232,10 +1232,18 @@ export class Stream { samples.length, u, ); - if (status !== g.TRANSCRIBE_OK) { - // Native feed failures leave the stream in FAILED, which is no longer - // an active stream in the C API. Keep the wrapper readable for - // state/lastStatus, but free the model-wide compute slot. + // The lease follows the native stream lifecycle, so on failure ask + // the stream where it ended up (still inside this exclusive slot, the + // worker call has returned). A feed rejected before the family hook + // (non-finite samples, other pre-hook INVALID_ARG) leaves it ACTIVE: + // keep the lease, so the stream stays usable and siblings stay Busy. + // A hook failure moves it to FAILED, which is no longer an active + // stream in the C API: keep the wrapper readable for state/lastStatus, + // but free the model-wide compute slot. + if ( + status !== g.TRANSCRIBE_OK && + n.F.streamGetState(h) !== g.TRANSCRIBE_STREAM_ACTIVE + ) { this.#releaseLease(); } check(n, status, "transcribe_stream_feed"); diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs index 11c8fe493..59db3f6d9 100644 --- a/bindings/typescript/test/compute-rules.test.mjs +++ b/bindings/typescript/test/compute-rules.test.mjs @@ -8,8 +8,9 @@ import assert from "node:assert/strict"; import { getEventListeners } from "node:events"; +import koffi from "koffi"; import { modelTest, MODEL, STREAMING_MODEL, jfk } from "./common.mjs"; -import { TranscribeModel, Aborted, Busy, InvalidArgument } from "../dist/index.js"; +import { TranscribeModel, Aborted, Busy, InvalidArgument, libraryPath } from "../dist/index.js"; const second = () => jfk().subarray(0, 16000); const half = () => { @@ -151,7 +152,7 @@ modelTest("a feed or finalize queued before session dispose still runs", STREAMI } }); -modelTest("a rejected feed releases the stream lease (current behavior)", STREAMING_MODEL, async () => { +modelTest("a feed rejected before the native hook keeps the stream lease", STREAMING_MODEL, async () => { const m = await TranscribeModel.load(STREAMING_MODEL); try { const a = m.createSession(); @@ -160,14 +161,63 @@ modelTest("a rejected feed releases the stream lease (current behavior)", STREAM await stream.feed(second()); const bad = new Float32Array(1600); bad[5] = Number.NaN; + // Native rejects non-finite samples before the family hook and leaves the + // stream ACTIVE, so the binding keeps the model-wide lease. await assert.rejects(() => stream.feed(bad), InvalidArgument); - // The native stream stays ACTIVE after a non-finite feed, but the binding - // releases the model-wide lease on any non-OK feed status, so a sibling - // run is no longer refused with Busy. Pinned as-is so that changing it is - // a deliberate, visible decision. The probe run is itself non-finite, so - // native rejects it up front and no compute overlaps the live stream. assert.equal(stream.state, "active"); - await assert.rejects(() => b.run(bad), (e) => e instanceof InvalidArgument && !(e instanceof Busy)); + await assert.rejects(() => b.run(second()), Busy); + await assert.rejects(() => b.stream(), Busy); + await assert.rejects(() => a.run(second()), Busy); + + // The stream is still usable, and finalize releases the lease as usual. + const u = await stream.feed(jfk().subarray(16000, 32000)); + assert.equal(typeof u.revision, "number"); + const fin = await stream.finalize(); + assert.equal(fin.isFinal, true); + assert.equal(stream.state, "finished"); + const r = await b.run(second()); + assert.equal(typeof r.text, "string"); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a feed that fails inside the native hook releases the stream lease", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + + // Provoke a hook failure: an abort callback that always fires makes the + // streaming hook return ABORTED, which moves the native stream to FAILED. + // feed() has no signal option, so install it on the native session + // directly (test-only; the same library instance the binding loaded). + const lib = koffi.load(libraryPath()); + const proto = koffi.proto("bool ComputeRulesTestAbortCb(void *udata)"); + const setAbort = lib.func("transcribe_set_abort_callback", "void", [ + "void *", + koffi.pointer(proto), + "void *", + ]); + const cb = koffi.register(() => true, koffi.pointer(proto)); + setAbort(a.handle, cb, null); + try { + await assert.rejects(() => stream.feed(second()), Aborted); + } finally { + setAbort(a.handle, null, null); + koffi.unregister(cb); + } + assert.equal(stream.state, "failed"); + assert.ok(stream.lastStatus instanceof Aborted); + + // FAILED is no longer an active stream: the lease is free for a sibling. + const r = await b.run(second()); + assert.equal(typeof r.text, "string"); stream.reset(); a.dispose(); b.dispose(); From f95bb5fa6c2d2059c28da77ffec0f219cffff938 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:08:58 +0800 Subject: [PATCH 13/51] python: characterize session compute rules Pin the per-call rules the Python binding relies on today before the model-wide compute lock lands, so the next commit can show they are unchanged: - run() / run_batch() on a session with an ACTIVE stream are rejected (native InvalidArgument) and the stream stays usable afterwards (begin-while-active was already covered). - a cancel() requested before run() / run_batch() / stream() is cleared when the call starts and does not abort it. - a Stream keeps its Session (and Model) alive when every other reference is dropped. Verified on the unmodified binding against a build-shared libtranscribe with whisper-tiny Q8_0 and moonshine-streaming-tiny Q8_0: all new tests pass. --- bindings/python/tests/test_streaming.py | 48 ++++++++++++++++++++++++ bindings/python/tests/test_transcribe.py | 12 ++++++ 2 files changed, 60 insertions(+) diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index 6c1695092..1fbc70a96 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -158,3 +158,51 @@ def test_second_stream_while_active_rejected(streaming_model_path, audio_pcm): stream.feed(audio_pcm[:16000]) with pytest.raises(t.InvalidArgument): session.stream() + + +def test_run_while_stream_active_rejected(streaming_model_path, audio_pcm): + # A run cannot replace an active stream's results on the SAME session: + # the C dispatcher rejects it, the binding surfaces InvalidArgument, and + # the stream stays usable afterwards. + with t.Model(streaming_model_path) as model, model.session() as session: + with session.stream() as stream: + stream.feed(audio_pcm[:16000]) + with pytest.raises(t.InvalidArgument): + session.run(audio_pcm[:16000]) + with pytest.raises(t.InvalidArgument): + session.run_batch([audio_pcm[:16000]]) + for i in range(16000, len(audio_pcm), 16000): + stream.feed(audio_pcm[i : i + 16000]) + stream.finalize() + assert stream.state == "finished" + committed = stream.text().committed + assert "country" in committed.lower(), committed + + +def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): + # A cancel() requested before stream() is cleared by the begin: the new + # stream's feeds are not aborted by a stale flag. (A cancel() AFTER begin + # is honoured on the next feed — test_cancellation_aborts_pending_feed.) + with t.Model(streaming_model_path) as model, model.session() as session: + session.cancel() + with session.stream() as stream: + stream.feed(audio_pcm[:16000]) + assert not session.was_aborted + assert stream.state == "active" + + +def test_stream_keeps_session_and_model_alive(streaming_model_path, audio_pcm): + # The Stream holds a strong reference to its Session (which holds the + # Model): dropping every other reference must not free native handles + # under the stream. + import gc + + stream = t.Model(streaming_model_path).session().stream() + gc.collect() + for i in range(0, len(audio_pcm), 16000): + stream.feed(audio_pcm[i : i + 16000]) + stream.finalize() + assert "country" in stream.text().committed.lower() + stream.reset() + del stream + gc.collect() diff --git a/bindings/python/tests/test_transcribe.py b/bindings/python/tests/test_transcribe.py index e242bcaa4..bc8779b39 100644 --- a/bindings/python/tests/test_transcribe.py +++ b/bindings/python/tests/test_transcribe.py @@ -193,6 +193,18 @@ def test_was_aborted_false_after_clean_run(self, model_path, audio_pcm): session.run(audio_pcm) assert session.was_aborted is False + def test_run_clears_pending_cancel(self, model_path, audio_pcm): + # A cancel() requested before run()/run_batch() is cleared when the + # call starts: the stale flag must not abort the new call. + with t.Model(model_path) as model, model.session() as session: + session.cancel() + assert "country" in session.run(audio_pcm).text.lower() + assert session.was_aborted is False + session.cancel() + results = session.run_batch([audio_pcm]) + assert "country" in results[0].text.lower() + assert session.was_aborted is False + def test_cross_thread_cancel_in_flight_run(self, model_path, audio_pcm): # Long input (tiled jfk) so the run is mid-flight when the timer # fires from another thread. The abort must surface as Aborted with From 2c337e72b3162bde3f3572d1e4773c78437701dd Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:15:42 +0800 Subject: [PATCH 14/51] python: model-wide compute lock BEHAVIOR CHANGE (0.4.0 release notes): compute calls on one Model now serialize. Session.run(), run_batch(), stream() and Stream.feed() / finalize() / reset() take a new model-wide lock, so concurrent calls on sessions of the same model from different threads WAIT for each other instead of racing in native code. Up to 0.3 the binding did not lock: ctypes releases the GIL around every foreign call (CDLL), so such calls overlapped natively and could corrupt decodes or fail Metal command buffers. Callers that serialized runs themselves keep working (their lock simply nests outside this one). Contended calls never raise. Model._exclusive(kind) is the single bridge every compute site goes through. It holds the lock for the native call AND the copy-out of the results, checks the model is still open once the lock is held, and the caller captures its session handle inside the block. - Plain threading.Lock plus an owner thread id, not an RLock. A thread can only re-enter through a callback (the log handler) firing inside a native call. Letting that start a second compute on the same model mid-flight is the race this lock prevents, so a re-entrant call raises TranscribeError instead of deadlocking. The abort callback only reads a threading.Event, so cancel() stays lock-free. - Session.close(), Model.close() and __del__ never wait for the lock. They mark the object closed at once and queue the native free, which runs immediately if the lock is free. Otherwise the current holder runs it just before releasing, so an in-flight call finishes its copy-out on the handle it captured and the free follows it. FIFO order frees sessions before their model, and the deferred session free pins the abort trampoline. This also covers GC running a finalizer on the thread that holds the lock. - The cancel flag is still cleared before the lock wait, so a cancel() issued while a call is queued aborts it as soon as it starts. - Reads (text, snapshot, state, revision, last_status, limits, was_aborted) and model queries stay lock-free. - There is no stream lease: an active stream does not block runs on other sessions between feeds. The docstrings tell callers not to do that. The Model, Session, cancel(), close() and set_log_callback docstrings and the README now describe this instead of "serialize runs yourself". Tests: the new test_compute_lock.py needs no model. It patches the _lib.* native entry points (looked up at call time) with probes on fake handles and covers these cases: - no overlap across two sessions and a stream - a contended call waits rather than raises - the lock is per model - every compute site, including copy-out, runs under the lock - the lock is released after a native error - a cancel issued while queued is kept - session/model close during an in-flight call defers the frees (sessions before model) and does not deadlock - GC on the holder thread defers the free - re-entrant compute raises Ordering uses Events and a lock spy that signals a blocking acquire, not sleeps. With _exclusive swapped for a no-op, 9 of these 11 tests fail; 15 repeated runs of the file all passed. test_concurrent_sessions_on_shared_model (two threads, no caller lock, real whisper-tiny) was an xfail and is now a plain passing test. It was run 5x on Metal and 5x with TRANSCRIBE_BACKEND=cpu. test_rejected_feed_keeps_stream_active checks a real moonshine stream: a NaN feed is rejected before the family hook (InvalidArgument), the stream stays "active", the lock is released, and valid feeds plus finalize still work. The binding has no lease flag, so it has nothing to release on that path. Verified against a build-shared libtranscribe with every model tier enabled (whisper-tiny, moonshine-streaming-tiny, nemotron-3.5 prompted, canary-180m-flash, SenseVoiceSmall, nemotron-speech-streaming, parakeet-unified, Voxtral-Mini-4B-Realtime). Baseline was 176 passed, 1 skipped, 1 xfailed. After: 193 passed, 1 skipped. Every baseline result is unchanged except the xfail, which now passes. The one skip is the speed-dependent cross-thread cancel test, same as baseline. The FFI drift gate and version sync are clean. --- bindings/python/README.md | 10 +- .../python/src/transcribe_cpp/__init__.py | 351 ++++++++++---- bindings/python/tests/test_compute_lock.py | 443 ++++++++++++++++++ bindings/python/tests/test_lifetime.py | 44 +- bindings/python/tests/test_streaming.py | 22 + 5 files changed, 747 insertions(+), 123 deletions(-) create mode 100644 bindings/python/tests/test_compute_lock.py diff --git a/bindings/python/README.md b/bindings/python/README.md index c24be3453..be12b81d2 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -120,9 +120,13 @@ TRANSCRIBE_LIBRARY=../../build-shared/src/libtranscribe.dylib \ ## Notes -- One run/stream at a time per `Model` in 0.x: sessions share the model's - compute backend, so serialize runs across sessions (or load one model per - worker). See the `Model` docstring. +- One compute call at a time per `Model` in 0.x: sessions share the model's + compute backend, so the binding serializes `run` / `run_batch` / stream + calls across all sessions of a model with a model-wide lock. Calls from + other threads wait their turn (they no longer race); load one model per + worker for parallel transcription. An active stream still occupies the + model between feeds, so finish it before running other sessions of that + model. See the `Model` docstring. - Import package: `transcribe_cpp` - Distribution: `transcribe-cpp` - License: MIT diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index cf5814cd3..635a08435 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -16,14 +16,18 @@ Long-running native calls (model load, run) release the GIL — ctypes does this for every foreign call — so other Python threads make progress during inference. +Compute calls on one Model are serialized by a model-wide lock (see ``Model``): +concurrent calls from other threads wait their turn instead of racing. """ from __future__ import annotations +import collections import ctypes import os import threading import weakref +from contextlib import contextmanager from dataclasses import dataclass, field from typing import Literal, Optional, Sequence, Union @@ -391,7 +395,12 @@ def set_log_callback(handler) -> None: contract for transcribe_log_set). The handler may be invoked from ggml worker threads, so it must be thread-safe — route to the ``logging`` module or a queue rather than doing heavy work inline. Levels mirror - TRANSCRIBE_LOG_LEVEL_* (1=info, 2=warn, 3=error, 4=debug).""" + TRANSCRIBE_LOG_LEVEL_* (1=info, 2=warn, 3=error, 4=debug). + + The handler can fire while a compute call holds its model's lock, so it + must not call compute APIs (run, run_batch, stream, feed, finalize, + reset) on that model: from the computing thread such a call raises + TranscribeError, and from a ggml worker thread it would deadlock.""" global _log_handler, _log_trampoline _log_handler = handler if _log_trampoline is None: @@ -907,13 +916,30 @@ def _apply(self, ext) -> None: class Model: - """A loaded model. Sharing it across threads for queries and session - creation is safe; it must outlive its sessions. - - Known 0.x limitation: at most one run/stream may be IN FLIGHT across all - sessions of a model at a time — sessions share the model's compute - backend, so overlapping runs race. Run sessions serially (a pool behind - a lock is fine), or load one Model per worker for true parallelism. + """A loaded model. Sharing it across threads is safe: queries + (``arch``, ``capabilities``, ``supports`` ...) and session creation run + concurrently. It must outlive its sessions (``close()`` enforces that). + + Compute calls are serialized per model. The native library allows at + most one compute call in flight across ALL sessions of a model (they + share its compute backend), so ``Session.run()``, ``run_batch()``, + ``stream()`` and ``Stream.feed()`` / ``finalize()`` / ``reset()`` each + hold a model-wide lock for the native call and for copying its results + out. Calls on any session of the same model from other threads WAIT for + the lock — they never raise a "busy" error — and a waiting call can + still be cancelled with its session's ``cancel()``. Load one Model per + worker for parallel transcription. (Up to 0.3 the binding did not lock: + overlapping calls on one model raced and could corrupt decodes.) + + The lock covers individual calls, not a stream's lifetime: an active + stream still occupies the model between feeds (the C contract), so do + not run other sessions of this model until the stream is finalized or + reset. + + ``close()`` (and garbage collection) never waits for the lock: closing + the model or a session while a call is in flight marks it closed at + once, lets the in-flight call finish and copy out its results, and + frees the native handles right after it. ``backend="auto"`` (the default) picks the best available device. The ``TRANSCRIBE_BACKEND`` environment variable overrides that *default* — @@ -929,6 +955,7 @@ def __init__(self, path: str | os.PathLike, *, # derived session is gone (use-after-free otherwise). Created before # the load call so the failure path of __del__ finds it. self._sessions = weakref.WeakSet() + self._init_compute_state() backend_source = "backend" if backend == "auto" and os.environ.get("TRANSCRIBE_BACKEND"): backend = os.environ["TRANSCRIBE_BACKEND"] # type: ignore[assignment] @@ -956,6 +983,91 @@ def _h(self) -> ctypes.c_void_p: raise TranscribeError("model is closed") return self._handle + # --- compute exclusion ------------------------------------------------- + # + # One plain (non-reentrant) Lock per model serializes every native compute + # call and the copy-out of its results. Not an RLock: the only way a + # thread can re-enter is a callback (the log handler) firing inside a + # native call, and letting that start a second compute on the same model + # mid-flight is exactly the race the lock exists to prevent — so a + # re-entrant attempt is detected via _compute_owner and raises instead. + # The abort callback only reads a threading.Event and never touches the + # lock, so cancel() stays lock-free. + # + # Native frees never wait for the lock (they run from close() and from + # __del__, which GC may invoke on a thread that already holds it, e.g. + # mid copy-out). A free is queued on _deferred and executed by whichever + # thread holds the lock — immediately if it is free, else by the holder + # just before it releases. FIFO order keeps the C contract that sessions + # are freed before their model. + + def _init_compute_state(self) -> None: + self._compute_lock = threading.Lock() + self._compute_owner: Optional[int] = None # thread ident while held + self._deferred: collections.deque = collections.deque() + + @contextmanager + def _exclusive(self, kind: str): + """Hold the model-wide compute lock for one compute call of *kind* + (blocking until it is free) and check the model is still open. Run + the native call AND the copy-out of its results inside the block, + and capture the session handle inside it too.""" + me = threading.get_ident() + if self._compute_owner == me: + raise TranscribeError( + f"cannot start {kind}: this thread is already inside a " + "compute call on this model (a re-entrant call from a " + "callback?)") + self._compute_lock.acquire() + self._compute_owner = me + try: + if self._handle is None: + raise TranscribeError("model is closed") + yield + finally: + try: + self._drain_deferred_locked() + finally: + # Owner stays set while draining: a native free can log, and + # the re-entrancy check must still see this thread. + self._compute_owner = None + self._compute_lock.release() + self._try_drain() + + def _drain_deferred_locked(self) -> None: + """Run queued native frees. Caller holds the compute lock.""" + while True: + try: + free = self._deferred.popleft() + except IndexError: + return + try: + free() + except Exception: + pass # a native free cannot meaningfully fail + + def _try_drain(self) -> None: + """Run queued frees now if the lock is free; otherwise leave them to + the current holder, which drains before releasing. Never blocks. + Re-checks after releasing so a free queued in the gap between the + holder's drain and its release is never stranded.""" + while self._deferred: + if not self._compute_lock.acquire(blocking=False): + return + self._compute_owner = threading.get_ident() + try: + self._drain_deferred_locked() + finally: + self._compute_owner = None + self._compute_lock.release() + + def _free_or_defer(self, free) -> None: + """Run the native *free* behind any in-flight compute call without + waiting for it. *free* must not reference the Session/Model object + (it may run after that object is gone).""" + self._deferred.append(free) + self._try_drain() + @property def arch(self) -> str: return _decode(_lib.transcribe_model_arch_string(self._h)) @@ -1046,12 +1158,20 @@ def session(self, *, n_threads: int = 0, kv_type: KVType = "auto", def close(self) -> None: """Free the model. Any session still open on it is closed first — the C contract forbids freeing a model before its sessions, so this - keeps explicit close()/context-manager exit safe in any order.""" - if getattr(self, "_handle", None) is not None: - for session in list(getattr(self, "_sessions", ()) or ()): - session.close() - _lib.transcribe_model_free(self._handle) - self._handle = None + keeps explicit close()/context-manager exit safe in any order. + + Never waits for the compute lock: if a call is in flight on one of + the sessions, the model is marked closed now and the native frees + (sessions first, then the model) run as soon as that call ends.""" + handle = getattr(self, "_handle", None) + if handle is None: + return + # Closed from here on: a call queued on the lock sees it once it gets + # the lock; an in-flight call already captured its handles. + self._handle = None + for session in list(getattr(self, "_sessions", ()) or ()): + session.close() + self._free_or_defer(lambda: _lib.transcribe_model_free(handle)) def __enter__(self) -> "Model": return self @@ -1067,11 +1187,16 @@ def __del__(self): class Session: - """A single transcription context bound to one Model. Not thread-safe. + """A single transcription context bound to one Model. - Sessions of one model must also not RUN concurrently with each other in - 0.x (they share the model's compute backend — see the Model docstring); - interleave or serialize their runs, or use one Model per worker.""" + Compute calls (``run``, ``run_batch``, ``stream`` and the Stream's + ``feed`` / ``finalize`` / ``reset``) take the model-wide compute lock, + so calls on sessions of one model — from any threads — serialize: a + call made while another is in flight waits for it (see ``Model``). + Reads (``limits``, ``was_aborted``, ``Stream.text()`` / ``snapshot()`` + / ``state`` ...) do not take the lock; don't read a session from one + thread while another thread is computing on that same session. + ``cancel()`` is safe from any thread.""" def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0): @@ -1158,25 +1283,29 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` (including its ``OutputRepetition`` subclass) the partial transcript is preserved and attached to the exception as ``partial_result``.""" + # Cleared BEFORE waiting for the compute lock, so a cancel() issued + # while this call is queued behind another still aborts it. self._cancel.clear() array, n_samples = _pcm_to_carray(pcm) params = _build_run_params(task, language, target_language, timestamps, keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt, prefix) - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - try: - _check(_lib.transcribe_run(self._h, array, n_samples, _byref(params)), - "transcribe_run") - except (Aborted, OutputTruncated) as exc: - # The C API preserves the partial transcript on the session for - # these statuses (OutputRepetition included, as an OutputTruncated - # subclass); surface it rather than discard it. - exc.partial_result = self._materialize() - raise - return self._materialize() + with self._model._exclusive("run"): + ext = self._resolve_family(family, "run") if family is not None else None + if ext is not None: + params.family = ctypes.cast( + _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + h = self._h # captured under the lock; close() defers its free + try: + _check(_lib.transcribe_run(h, array, n_samples, _byref(params)), + "transcribe_run") + except (Aborted, OutputTruncated) as exc: + # The C API preserves the partial transcript on the session for + # these statuses (OutputRepetition included, as an OutputTruncated + # subclass); surface it rather than discard it. + exc.partial_result = self._materialize(h) + raise + return self._materialize(h) def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", language: str | None = None, @@ -1211,7 +1340,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", ``vocabulary`` / ``prompt`` apply to every utterance (see :meth:`run`); a transcript prefix is per-utterance and so is not accepted here.""" - self._cancel.clear() + self._cancel.clear() # before the lock wait, as in run() pcms = list(pcms) if not pcms: raise InvalidArgument("run_batch requires at least one PCM buffer") @@ -1228,44 +1357,47 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", params = _build_run_params(task, language, target_language, timestamps, keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt) - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - batch_abort = None - try: - _check( - _lib.transcribe_run_batch(self._h, ptrs, counts, len(pcms), _byref(params)), - "transcribe_run_batch", - ) - except Aborted as exc: - # Cancellation surfaces at the BATCH level, but the native side - # pads the per-utterance result set so clips completed before the - # abort survive. Fall through to the per-utterance loop, which - # restores utterance context instead of discarding that work. - batch_abort = exc - - out: list = [] - first_exc = None - for i in range(_lib.transcribe_batch_n_results(self._h)): - status = _lib.transcribe_batch_status(self._h, i) - if status == 0: - out.append(self._materialize(i)) - continue - exc = exception_for_status(status, _status_string(status), - f"utterance {i} in batch") - exc.utterance_index = i - if isinstance(exc, (Aborted, OutputTruncated)): - # Attach the partial transcript only when the native layer - # actually snapshotted one for this slot — some batch paths - # record failed slots as empty, and None is more honest than - # a confidently empty Result. - partial = self._materialize(i) - if partial.text or partial.tokens or partial.segments: - exc.partial_result = partial - out.append(exc) - if first_exc is None: - first_exc = exc + # The native call AND the per-utterance copy-out run under the lock. + with self._model._exclusive("run_batch"): + ext = self._resolve_family(family, "run") if family is not None else None + if ext is not None: + params.family = ctypes.cast( + _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + h = self._h # captured under the lock; close() defers its free + batch_abort = None + try: + _check( + _lib.transcribe_run_batch(h, ptrs, counts, len(pcms), _byref(params)), + "transcribe_run_batch", + ) + except Aborted as exc: + # Cancellation surfaces at the BATCH level, but the native side + # pads the per-utterance result set so clips completed before the + # abort survive. Fall through to the per-utterance loop, which + # restores utterance context instead of discarding that work. + batch_abort = exc + + out: list = [] + first_exc = None + for i in range(_lib.transcribe_batch_n_results(h)): + status = _lib.transcribe_batch_status(h, i) + if status == 0: + out.append(self._materialize(h, i)) + continue + exc = exception_for_status(status, _status_string(status), + f"utterance {i} in batch") + exc.utterance_index = i + if isinstance(exc, (Aborted, OutputTruncated)): + # Attach the partial transcript only when the native layer + # actually snapshotted one for this slot — some batch paths + # record failed slots as empty, and None is more honest than + # a confidently empty Result. + partial = self._materialize(h, i) + if partial.text or partial.tokens or partial.segments: + exc.partial_result = partial + out.append(exc) + if first_exc is None: + first_exc = exc if first_exc is None and batch_abort is not None: # Anomalous: an abort with no per-utterance trace. Re-raise loud # (even under return_exceptions) rather than swallow a cancel. @@ -1294,7 +1426,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, the Stream as a context manager so it is reset when you are done. ``vocabulary`` and ``prompt`` are as in :meth:`run`; ``task="instruct"`` raises.""" - self._cancel.clear() + self._cancel.clear() # before the lock wait, as in run() # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). run_params = _build_run_params(task, language, target_language, timestamps, @@ -1304,14 +1436,15 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, _lib.transcribe_stream_params_init(_byref(sp)) sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy") sp.stable_prefix_agreement_n = stable_prefix_agreement_n - ext = self._resolve_family(family, "stream") if family is not None else None - if ext is not None: - sp.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - _check( - _lib.transcribe_stream_begin(self._h, _byref(run_params), _byref(sp)), - "transcribe_stream_begin", - ) + with self._model._exclusive("stream_begin"): + ext = self._resolve_family(family, "stream") if family is not None else None + if ext is not None: + sp.family = ctypes.cast( + _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + _check( + _lib.transcribe_stream_begin(self._h, _byref(run_params), _byref(sp)), + "transcribe_stream_begin", + ) # The C contract says everything passed to begin may be freed once it # returns (strings are copied into session-owned storage). The Stream # still pins the params structs until reset() as defense in depth — @@ -1319,11 +1452,16 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, # or out-of-tree native library that predates that contract. return Stream(self, _keepalive=(run_params, sp, ext)) - def _materialize(self, utt: int | None = None) -> Result: - """Copy out one result. utt is None for the single-result accessors, or - an utterance index for the batch accessors (index 0 aliases the single - result after a plain run, so both paths share this code).""" - h = self._h + def _materialize(self, h: ctypes.c_void_p | None = None, + utt: int | None = None) -> Result: + """Copy out one result. A compute call's copy-out passes the session + handle *h* it captured under the compute lock (so a concurrent close() + cannot pull it away mid-copy); None reads this session's live handle. + utt is None for the single-result accessors, or an utterance index for + the batch accessors (index 0 aliases the single result after a plain + run, so both paths share this code).""" + if h is None: + h = self._h if utt is None: n_seg = lambda: _lib.transcribe_n_segments(h) get_seg = lambda j, out: _lib.transcribe_get_segment(h, j, out) @@ -1416,8 +1554,11 @@ def cancel(self) -> None: """Request cancellation of an in-flight run/stream from another thread. The active call aborts at the next chunk/decode boundary and raises ``Aborted`` (with ``partial_result`` attached); ``was_aborted`` then - reports True. The flag is cleared at the start of the next - run/stream.""" + reports True. The flag is cleared when the next run/run_batch/stream + call starts — before it waits for the model's compute lock, so a + cancel() issued while that call is queued behind another session's + call aborts it as soon as it begins. Lock-free; safe from any + thread.""" self._cancel.set() @property @@ -1426,9 +1567,22 @@ def was_aborted(self) -> bool: return bool(_lib.transcribe_was_aborted(self._h)) def close(self) -> None: - if getattr(self, "_handle", None) is not None: - _lib.transcribe_session_free(self._handle) - self._handle = None + """Free the session. Idempotent. Never waits for the compute lock: if + a call is in flight on this model, the session is marked closed now + and the native free runs as soon as that call ends.""" + handle = getattr(self, "_handle", None) + if handle is None: + return + self._handle = None + # The abort trampoline must outlive the native session that points at + # it, so the deferred free pins it. Nothing here references self: the + # free may run after this object is gone (close() from __del__). + trampoline = getattr(self, "_abort_trampoline", None) + + def free(handle=handle, _pin=trampoline): + _lib.transcribe_session_free(handle) + + self._model._free_or_defer(free) def __enter__(self) -> "Session": return self @@ -1469,17 +1623,19 @@ def feed(self, pcm: PCMLike) -> StreamUpdate: array, n_samples = _pcm_to_carray(pcm) update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) - _check(_lib.transcribe_stream_feed(self._h, array, n_samples, _byref(update)), - "transcribe_stream_feed") - return _stream_update_from(update) + with self._session._model._exclusive("stream_feed"): + _check(_lib.transcribe_stream_feed(self._h, array, n_samples, _byref(update)), + "transcribe_stream_feed") + return _stream_update_from(update) def finalize(self) -> StreamUpdate: """Signal end of audio and flush the final hypothesis.""" update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) - _check(_lib.transcribe_stream_finalize(self._h, _byref(update)), - "transcribe_stream_finalize") - return _stream_update_from(update) + with self._session._model._exclusive("stream_finalize"): + _check(_lib.transcribe_stream_finalize(self._h, _byref(update)), + "transcribe_stream_finalize") + return _stream_update_from(update) def text(self) -> StreamText: """Current committed / tentative / full text views (owned copies).""" @@ -1522,7 +1678,8 @@ def last_status(self) -> TranscribeError | None: def reset(self) -> None: """Return the session to idle, discarding stream state. Idempotent.""" if self._active: - _lib.transcribe_stream_reset(self._session._h) + with self._session._model._exclusive("stream_reset"): + _lib.transcribe_stream_reset(self._session._h) self._active = False self._keepalive = None diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py new file mode 100644 index 000000000..f84d281b7 --- /dev/null +++ b/bindings/python/tests/test_compute_lock.py @@ -0,0 +1,443 @@ +"""Model-wide compute lock: deterministic tests that need NO model. + +The native library allows one compute call in flight per model across all of +its sessions; the binding enforces that with ``Model._exclusive(kind)``. These +tests build Model/Session objects around fake handles and replace the native +entry points the high-level code calls (``transcribe_cpp._lib.``, looked +up at call time) with Python probes, so overlap, waiting, deferred frees and +re-entrancy are observed directly — no GGUF, no native compute. + +Every native call that could touch a fake handle is patched, and the fixture +closes every fake (and collects garbage) BEFORE monkeypatch restores the real +functions, so a fake handle never reaches the real library. +""" + +from __future__ import annotations + +import ctypes +import gc +import itertools +import threading +import time +import weakref +from types import SimpleNamespace + +import pytest + +import transcribe_cpp as t +from transcribe_cpp import _generated + +TIMEOUT = 10.0 # generous upper bound for "must not deadlock" joins + + +def _in_thread(fn, *args): + """Run fn in a daemon thread; returns (thread, box) where box collects the + return value or the exception.""" + box: dict = {} + + def target(): + try: + box["value"] = fn(*args) + except BaseException as exc: # surfaced by the test, never swallowed + box["error"] = exc + + th = threading.Thread(target=target, daemon=True) + th.start() + return th, box + + +def _join(th, what): + th.join(TIMEOUT) + assert not th.is_alive(), f"{what} did not finish (deadlock?)" + + +@pytest.fixture +def fake(monkeypatch): + ids = itertools.count(0x1000, 0x10) + frees: list = [] # ("session" | "model", handle) in native-free order + made: list = [] + + monkeypatch.setattr(t._lib, "transcribe_session_free", + lambda h: frees.append(("session", h.value))) + monkeypatch.setattr(t._lib, "transcribe_model_free", + lambda h: frees.append(("model", h.value))) + monkeypatch.setattr(t._lib, "transcribe_stream_reset", lambda h: None) + # The copy-out reads many native accessors; stub it to report which + # handle it was given (proves the in-flight call kept its handle). + monkeypatch.setattr(t.Session, "_materialize", + lambda self, h=None, utt=None: + ("result", (h or self._h).value, utt)) + + def model(): + m = t.Model.__new__(t.Model) + m._sessions = weakref.WeakSet() + m._init_compute_state() # the same init Model.__init__ runs + m._handle = ctypes.c_void_p(next(ids)) + made.append(m) + return m + + def session(m, track=True): + s = t.Session.__new__(t.Session) + s._model = m + s._handle = ctypes.c_void_p(next(ids)) + s._cancel = threading.Event() + s._abort_trampoline = None + m._sessions.add(s) + if track: + made.append(s) + return s + + yield SimpleNamespace(model=model, session=session, frees=frees) + + for obj in reversed(made): + obj.close() + made.clear() + gc.collect() + + +PCM = [0.0] * 160 + + +class _LockSpy: + """Wraps a model's compute lock and signals when a thread starts a + BLOCKING acquire (i.e. is parked waiting for it). The non-blocking + acquires of the deferred-free path do not count. Install it while the + real lock is held: release() is looked up on the attribute at call + time, so the holder releases the same underlying lock.""" + + def __init__(self, model): + self._lock = model._compute_lock + self.waiting = threading.Event() + model._compute_lock = self + + def acquire(self, blocking=True, timeout=-1): + if blocking: + self.waiting.set() + return self._lock.acquire(blocking, timeout) + + def release(self): + self._lock.release() + + def locked(self): + return self._lock.locked() + + +class _Gate: + """A native probe that parks inside the 'native call' until released.""" + + def __init__(self): + self.entered = threading.Event() + self.release = threading.Event() + + def __call__(self, *args): + self.entered.set() + assert self.release.wait(TIMEOUT), "gate never released" + return 0 + + +# --- serialization ------------------------------------------------------------ + + +def test_calls_on_two_sessions_never_overlap(fake, monkeypatch): + m = fake.model() + s1, s2, s3 = fake.session(m), fake.session(m), fake.session(m) + stream = t.Stream(s3) + + guard = threading.Lock() + state = {"active": 0, "max": 0, "calls": 0} + + def probe(*args): + with guard: + state["active"] += 1 + state["calls"] += 1 + state["max"] = max(state["max"], state["active"]) + time.sleep(0.02) # widen the window an unlocked overlap would hit + with guard: + state["active"] -= 1 + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + monkeypatch.setattr(t._lib, "transcribe_stream_feed", probe) + + start = threading.Barrier(3) + n = 5 + + def runs(session): + start.wait(TIMEOUT) + return [session.run(PCM) for _ in range(n)] + + def feeds(): + start.wait(TIMEOUT) + return [stream.feed(PCM) for _ in range(n)] + + workers = [_in_thread(runs, s1), _in_thread(runs, s2), _in_thread(feeds)] + for th, _ in workers: + _join(th, "worker") + for _, box in workers: + assert "error" not in box, box["error"] + + assert state["calls"] == 3 * n + assert state["max"] == 1, "compute calls on one model overlapped" + assert workers[0][1]["value"] == [("result", s1._handle.value, None)] * n + assert workers[1][1]["value"] == [("result", s2._handle.value, None)] * n + + +def test_contended_call_waits_instead_of_raising(fake, monkeypatch): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + gate = _Gate() + second_entered = threading.Event() + + def probe(h, *rest): + if h.value == s1._handle.value: + return gate(h) + second_entered.set() + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, a_box = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + b, b_box = _in_thread(s2.run, PCM) + # B must be parked on the lock: not in native, not failed, not done. + assert not second_entered.wait(0.3), "second call overlapped the first" + assert b.is_alive() and not b_box, f"contended call did not wait: {b_box}" + finally: + gate.release.set() + _join(a, "first run") + _join(b, "second run") + assert "error" not in a_box and "error" not in b_box, (a_box, b_box) + assert b_box["value"] == ("result", s2._handle.value, None) + + +def test_lock_is_per_model(fake, monkeypatch): + s1 = fake.session(fake.model()) + s2 = fake.session(fake.model()) + gate = _Gate() + + def probe(h, *rest): + return gate(h) if h.value == s1._handle.value else 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, _ = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + b, b_box = _in_thread(s2.run, PCM) + _join(b, "run on another model") # must not wait behind model 1 + assert "error" not in b_box, b_box + finally: + gate.release.set() + _join(a, "first run") + + +def test_every_compute_site_holds_the_lock(fake, monkeypatch): + m = fake.model() + s = fake.session(m) + seen: list = [] + + def holds(name, ret=0): + def probe(*args): + seen.append((name, m._compute_lock.locked() + and m._compute_owner == threading.get_ident())) + return ret + return probe + + for name in ("transcribe_run", "transcribe_run_batch", + "transcribe_stream_begin", "transcribe_stream_feed", + "transcribe_stream_finalize", "transcribe_stream_reset", + "transcribe_batch_status"): + monkeypatch.setattr(t._lib, name, holds(name)) + monkeypatch.setattr(t._lib, "transcribe_batch_n_results", + holds("transcribe_batch_n_results", ret=1)) + real_materialize = t.Session._materialize + + def materialize(self, h=None, utt=None): + seen.append(("copy-out", m._compute_lock.locked())) + return real_materialize(self, h, utt) + + monkeypatch.setattr(t.Session, "_materialize", materialize) + + s.run(PCM) + assert s.run_batch([PCM]) == [("result", s._handle.value, 0)] + stream = s.stream() + stream.feed(PCM) + stream.finalize() + stream.reset() + + names = [n for n, _ in seen] + for site in ("transcribe_run", "transcribe_run_batch", + "transcribe_batch_n_results", "transcribe_batch_status", + "transcribe_stream_begin", "transcribe_stream_feed", + "transcribe_stream_finalize", "transcribe_stream_reset", + "copy-out"): + assert site in names, f"{site} never reached" + assert all(ok for _, ok in seen), [n for n, ok in seen if not ok] + assert not m._compute_lock.locked() + + +def test_lock_released_after_native_error(fake, monkeypatch): + m = fake.model() + s = fake.session(m) + monkeypatch.setattr(t._lib, "transcribe_run", + lambda *a: _generated.TRANSCRIBE_ERR_INVALID_ARG) + with pytest.raises(t.InvalidArgument): + s.run(PCM) + assert not m._compute_lock.locked() + monkeypatch.setattr(t._lib, "transcribe_run", lambda *a: 0) + assert s.run(PCM) == ("result", s._handle.value, None) + + +# --- cancellation ------------------------------------------------------------- + + +def test_cancel_while_queued_is_kept(fake, monkeypatch): + # cancel() is lock-free and the flag is cleared BEFORE the lock wait, so + # a cancel issued while a call is queued is still set when it starts. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + gate = _Gate() + flag_at_start: list = [] + + def probe(h, *rest): + if h.value == s1._handle.value: + return gate(h) + flag_at_start.append(s2._cancel.is_set()) + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, _ = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + b, _ = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) # B cleared its flag and is parked + s2.cancel() # must not block although the lock is held + finally: + gate.release.set() + _join(a, "first run") + _join(b, "queued run") + assert flag_at_start == [True] + + +# --- close / GC during an in-flight call ------------------------------------------ + + +def test_session_close_during_in_flight_call_defers_free(fake, monkeypatch): + m = fake.model() + s = fake.session(m) + handle = s._handle.value + gate = _Gate() + monkeypatch.setattr(t._lib, "transcribe_run", gate) + try: + a, a_box = _in_thread(s.run, PCM) + assert gate.entered.wait(TIMEOUT) + closer, _ = _in_thread(s.close) + _join(closer, "session.close() during an in-flight call") + assert fake.frees == [], "session freed under an in-flight call" + finally: + gate.release.set() + _join(a, "in-flight run") + # The in-flight call finished its copy-out on the handle it captured... + assert a_box.get("value") == ("result", handle, None), a_box + # ...and the deferred free ran exactly once, after it. + assert fake.frees == [("session", handle)] + with pytest.raises(t.TranscribeError, match="closed"): + s.run(PCM) + + +def test_model_close_during_in_flight_call_defers_frees(fake, monkeypatch): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + h1, h2, hm = s1._handle.value, s2._handle.value, m._handle.value + gate = _Gate() + + def probe(h, *rest): + return gate(h) if h.value == h1 else 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, a_box = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + queued, q_box = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) # parked on the lock before close + closer, _ = _in_thread(m.close) + _join(closer, "model.close() during an in-flight call") + assert fake.frees == [], "freed under an in-flight call" + finally: + gate.release.set() + _join(a, "in-flight run") + _join(queued, "queued run") + assert a_box.get("value") == ("result", h1, None), a_box + # The queued call sees the closed model once it gets the lock. + assert isinstance(q_box.get("error"), t.TranscribeError), q_box + assert "closed" in str(q_box["error"]) + # Sessions first (either order), then the model; each exactly once. + assert sorted(fake.frees[:2]) == sorted([("session", h1), ("session", h2)]) + assert fake.frees[2:] == [("model", hm)] + + +def test_gc_on_holder_thread_defers_free(fake, monkeypatch): + # A finalizer can run on the thread that holds the lock (GC mid copy-out). + # It must neither deadlock nor free under the in-flight call. + m = fake.model() + s = fake.session(m) + doomed: list = [] + during: dict = {} + + def probe(*args): + victim = fake.session(m, track=False) + victim._cycle = victim # only the cyclic GC can reclaim it + doomed.append(victim._handle.value) + del victim + gc.collect() # runs Session.__del__ -> close() on THIS thread + during["frees"] = list(fake.frees) + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + th, box = _in_thread(s.run, PCM) + _join(th, "run with GC on the holder thread") + assert "error" not in box, box + assert doomed, "probe never ran" + victim = ("session", doomed[0]) + assert victim not in during["frees"], "freed while the lock was held" + assert fake.frees.count(victim) == 1 + + +def test_close_from_holder_thread_does_not_deadlock(fake, monkeypatch): + m = fake.model() + s, other = fake.session(m), fake.session(m) + hother = other._handle.value + + def probe(*args): + other.close() # same thread as the lock holder: must return at once + assert fake.frees == [] + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + th, box = _in_thread(s.run, PCM) + _join(th, "run closing another session from inside the call") + assert "error" not in box, box + assert fake.frees == [("session", hother)] + + +def test_reentrant_compute_raises_instead_of_deadlocking(fake, monkeypatch): + # e.g. a log handler that calls run() from inside a native call. + m = fake.model() + s, other = fake.session(m), fake.session(m) + inner: dict = {} + + def probe(h, *rest): + if h.value == s._handle.value: + try: + other.run(PCM) + except t.TranscribeError as exc: + inner["error"] = exc + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + th, box = _in_thread(s.run, PCM) + _join(th, "re-entrant run") + assert "error" not in box, box + assert "re-entrant" in str(inner.get("error")), inner + assert not m._compute_lock.locked() diff --git a/bindings/python/tests/test_lifetime.py b/bindings/python/tests/test_lifetime.py index e094b5f91..164aab4e1 100644 --- a/bindings/python/tests/test_lifetime.py +++ b/bindings/python/tests/test_lifetime.py @@ -76,18 +76,19 @@ def test_gc_order_session_keeps_model_alive(model_path, audio_pcm): # --- multi-session use of one model ------------------------------------------ # -# SUPPORTED in 0.x: many sessions on one model, run serially (from any -# threads). NOT yet supported: overlapping runs across sessions of one model -# — they share the model's compute backend and some family state, so -# concurrent runs race (observed: corrupted whisper decodes on CPU, -# command-buffer failures on Metal). The xfail below pins the limitation so -# the day a per-session backend architecture lands, the XPASS flags it for -# promotion to a hard test. +# Natively, at most one compute call may be in flight across the sessions of +# one model — they share its compute backend and some family state, so +# overlapping runs race (observed: corrupted whisper decodes on CPU, +# command-buffer failures on Metal). The binding enforces that itself with a +# model-wide compute lock: concurrent calls from several threads serialize +# (they wait, never raise), so callers no longer need their own lock. The +# deterministic, model-free lock tests live in test_compute_lock.py. def test_serial_sessions_across_threads(model_path, audio_pcm): - # Two threads, each with its own session, runs SERIALIZED by a lock: - # the supported session-pool pattern. + # Two threads, each with its own session, runs serialized by a caller + # lock: the session-pool pattern written before the binding locked + # internally. Must keep working (the two locks nest without deadlock). import threading lock = threading.Lock() @@ -114,12 +115,11 @@ def worker(tag: str, model: "t.Model") -> None: assert all("country" in text for text in results.values()), results -# Run INSIDE a subprocess: the known failure mode is NATIVE (corrupted -# decodes, Metal command-buffer failures, potentially a crash or a wedged -# thread). xfail can absorb a Python assert but not a segfault of the test -# runner, and a wedged native thread must never reach interpreter teardown -# in the CI process (freeing the session under an in-flight native call is -# a use-after-free). os._exit on the wedge path skips teardown on purpose. +# Run INSIDE a subprocess: a regression of the compute lock fails NATIVELY +# (corrupted decodes, Metal command-buffer failures, potentially a crash or a +# wedged thread). A segfault must fail this test, not kill the runner, and a +# wedged native thread must never reach interpreter teardown in the CI +# process. os._exit on the wedge path skips teardown on purpose. _CONCURRENT_RUNS_SNIPPET = """\ import array, os, sys, threading, wave @@ -136,7 +136,8 @@ def worker(tag: str, model: "t.Model") -> None: def worker(tag, model): try: with model.session() as session: - results[tag] = session.run(pcm).text.lower() + for k in range(2): # NO caller lock: the binding serializes + results[(tag, k)] = session.run(pcm).text.lower() except Exception as exc: errors.append((tag, repr(exc))) @@ -152,18 +153,15 @@ def worker(tag, model): os._exit(3) # never free handles under an in-flight native call assert not errors, errors -assert len(results) == 2 and all("country" in v for v in results.values()), results +assert len(results) == 4 and all("country" in v for v in results.values()), results print("CONCURRENT-OK") """ -@pytest.mark.xfail( - strict=False, - reason="known 0.x limitation: overlapping runs on sessions of one model " - "race on the shared backend/model state (see Model docstring and " - "notes/bindings-review-fixes.md); per-session backends planned", -) def test_concurrent_sessions_on_shared_model(model_path, audio_path): + # Was an xfail (overlapping runs raced and corrupted decodes) until the + # binding's model-wide compute lock: unsynchronized callers on two + # sessions of one model now wait for each other and both decode cleanly. import os import subprocess import sys diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index 1fbc70a96..aa5fceb6e 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -179,6 +179,28 @@ def test_run_while_stream_active_rejected(streaming_model_path, audio_pcm): assert "country" in committed.lower(), committed +def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): + # Non-finite PCM is rejected BEFORE the family hook: the feed raises + # InvalidArgument but the native stream stays ACTIVE (only failures + # inside the hook move it to FAILED). The binding must release the + # model's compute lock on that error and leave the stream usable. + with t.Model(streaming_model_path) as model, model.session() as session: + with session.stream() as stream: + stream.feed(audio_pcm[:16000]) + bad = list(audio_pcm[16000:32000]) + bad[100] = float("nan") + with pytest.raises(t.InvalidArgument): + stream.feed(bad) + assert stream.state == "active" + assert not model._compute_lock.locked() + for i in range(16000, len(audio_pcm), 16000): + stream.feed(audio_pcm[i : i + 16000]) + update = stream.finalize() + committed = stream.text().committed + assert update.is_final + assert "country" in committed.lower(), committed + + def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): # A cancel() requested before stream() is cleared by the begin: the new # stream's feeds are not aborted by a stale flag. (A cancel() AFTER begin From f87bcaacfa59b8f7de92b723215b063b2bfb32f5 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:40:56 +0800 Subject: [PATCH 15/51] stream: a throwing family hook ends the stream Bug fix. The streaming dispatcher marked a stream FAILED only when the family hook returned a non-OK status. If begin / feed / finalize threw, the exception unwound straight to the api_guard: the caller got OOM or BACKEND, but the stream stayed ACTIVE (and last_status unset), so bindings that follow the native state kept treating it as live. A throwing stream_reset skipped the dispatcher's wipe, so reset could leave the stream ACTIVE too. - call_stream_hook wraps the begin / feed / finalize hooks: on an exception it sets FAILED and last_status to the status the guard will return (OOM for bad_alloc, BACKEND otherwise), then rethrows. - reset runs the clear + IDLE + was_aborted wipe even when its hook throws, then rethrows for the guard to log. tests/stream_hook_throw_unit.cpp covers each hook with bad_alloc and a runtime_error (21 failures before this change). Valid-input behavior is unchanged: ctest green and transcribe-cli CPU parity vs main identical. --- src/transcribe-asr.cpp | 40 ++++++- tests/CMakeLists.txt | 17 +++ tests/stream_hook_throw_unit.cpp | 176 +++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 tests/stream_hook_throw_unit.cpp diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp index 1ce89ba56..3bb8928c7 100644 --- a/src/transcribe-asr.cpp +++ b/src/transcribe-asr.cpp @@ -965,6 +965,25 @@ extern "C" bool transcribe_was_truncated(const struct transcribe_session * sessi // last_status, and the public committed/tentative text view. clear_result() // wipes the snapshot but never touches stream_state. +// A family stream hook that throws must still end the stream: mark it FAILED +// with the status the api_guard will report for the exception (OOM for +// bad_alloc, BACKEND otherwise), then let the exception reach the guard. +// Without this a throwing hook left the stream ACTIVE while the caller got +// an error. +template static transcribe_status call_stream_hook(transcribe_session * session, Fn && hook) { + try { + return hook(); + } catch (const std::bad_alloc &) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = TRANSCRIBE_ERR_OOM; + throw; + } catch (...) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = TRANSCRIBE_ERR_BACKEND; + throw; + } +} + static transcribe_status transcribe_stream_begin_impl(struct transcribe_session * session, const struct transcribe_run_params * run_params, const struct transcribe_stream_params * stream_params) { @@ -1130,7 +1149,8 @@ static transcribe_status transcribe_stream_begin_impl(struct transcribe_session run_params_owned.prompt = run_params->prompt != nullptr ? session->stream_prompt_owned.c_str() : nullptr; run_params_owned.prefix = nullptr; - const transcribe_status st = session->model->arch->stream_begin(session, &run_params_owned, stream_params); + const transcribe_status st = call_stream_hook( + session, [&] { return session->model->arch->stream_begin(session, &run_params_owned, stream_params); }); if (st != TRANSCRIBE_OK) { // Family hook rejected the begin (config it does not understand, // memory allocation failure, etc.). Roll lifecycle back to @@ -1194,7 +1214,8 @@ static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * const std::string prev_full_text = session->full_text; const bool prev_has_result = session->has_result; - const transcribe_status st = session->model->arch->stream_feed(session, pcm, n_samples, update); + const transcribe_status st = + call_stream_hook(session, [&] { return session->model->arch->stream_feed(session, pcm, n_samples, update); }); if (st != TRANSCRIBE_OK) { session->stream_state = TRANSCRIBE_STREAM_FAILED; session->stream_last_status = st; @@ -1233,7 +1254,8 @@ static transcribe_status transcribe_stream_finalize_impl(struct transcribe_sessi const std::string prev_full_text = session->full_text; const bool prev_has_result = session->has_result; - const transcribe_status st = session->model->arch->stream_finalize(session, update); + const transcribe_status st = + call_stream_hook(session, [&] { return session->model->arch->stream_finalize(session, update); }); if (update != nullptr) { // Force is_final true regardless of family hook return; the // marker describes the call site, not the result. Set after @@ -1259,14 +1281,24 @@ static void transcribe_stream_reset_impl(struct transcribe_session * session) { // audio contents while keeping the allocations. A family without // streaming just has no hook installed; reset becomes a pure // dispatcher state wipe. + // The wipe below runs even if the hook throws, so reset always ends + // IDLE; the exception still reaches the api_guard afterwards. + std::exception_ptr hook_error; if (session->model != nullptr && session->model->arch != nullptr && session->model->arch->stream_reset != nullptr) { - session->model->arch->stream_reset(session); + try { + session->model->arch->stream_reset(session); + } catch (...) { + hook_error = std::current_exception(); + } } session->clear_result(); session->stream_state = TRANSCRIBE_STREAM_IDLE; // was_aborted is per-stream; reset re-arms it the same way begin // does so a caller that resets after an abort starts clean. session->was_aborted = false; + if (hook_error) { + std::rethrow_exception(hook_error); + } } extern "C" enum transcribe_stream_state transcribe_stream_get_state(const struct transcribe_session * session) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ed50d8a77..a0d5c52eb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -369,6 +369,23 @@ transcribe_apply_warnings(transcribe_nonfinite_input_unit) add_test(NAME transcribe_nonfinite_input_unit COMMAND transcribe_nonfinite_input_unit) +# ----------------------------------------------------------------------------- +# A throwing stream hook ends the stream (FAILED, or IDLE for reset) +# ----------------------------------------------------------------------------- + +add_executable(transcribe_stream_hook_throw_unit + stream_hook_throw_unit.cpp) + +target_link_libraries(transcribe_stream_hook_throw_unit PRIVATE transcribe) + +target_include_directories(transcribe_stream_hook_throw_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_stream_hook_throw_unit) + +add_test(NAME transcribe_stream_hook_throw_unit + COMMAND transcribe_stream_hook_throw_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- diff --git a/tests/stream_hook_throw_unit.cpp b/tests/stream_hook_throw_unit.cpp new file mode 100644 index 000000000..27494e2f0 --- /dev/null +++ b/tests/stream_hook_throw_unit.cpp @@ -0,0 +1,176 @@ +// stream_hook_throw_unit.cpp - a family stream hook that throws ends the +// stream: begin / feed / finalize leave it FAILED with last_status matching +// the status the call returned (OOM for bad_alloc, BACKEND otherwise), and +// reset still ends IDLE. Before, a throwing hook left the stream ACTIVE +// while the caller got an error. + +#include "transcribe-arch.h" +#include "transcribe-model.h" +#include "transcribe-session.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +enum class Throw { None, BadAlloc, Runtime }; + +Throw g_begin = Throw::None; +Throw g_feed = Throw::None; +Throw g_finalize = Throw::None; +Throw g_reset = Throw::None; + +void maybe_throw(Throw t) { + if (t == Throw::BadAlloc) { + throw std::bad_alloc(); + } + if (t == Throw::Runtime) { + throw std::runtime_error("hook failed"); + } +} + +transcribe_status fake_begin(transcribe_session *, const transcribe_run_params *, const transcribe_stream_params *) { + maybe_throw(g_begin); + return TRANSCRIBE_OK; +} + +transcribe_status fake_feed(transcribe_session *, const float *, int, transcribe_stream_update *) { + maybe_throw(g_feed); + return TRANSCRIBE_OK; +} + +transcribe_status fake_finalize(transcribe_session *, transcribe_stream_update *) { + maybe_throw(g_finalize); + return TRANSCRIBE_OK; +} + +void fake_reset(transcribe_session *) { + maybe_throw(g_reset); +} + +const transcribe::Arch k_arch = { + /* .name = */ "fake-stream-throw", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ nullptr, + /* .run_batch = */ nullptr, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ fake_begin, + /* .stream_feed = */ fake_feed, + /* .stream_finalize = */ fake_finalize, + /* .stream_reset = */ fake_reset, +}; + +void reset_globals() { + g_begin = g_feed = g_finalize = g_reset = Throw::None; +} + +struct Fixture { + transcribe_model model; + transcribe_session session; + std::vector pcm = std::vector(1600, 0.0f); + + Fixture() { + model.arch = &k_arch; + model.caps.supports_streaming = true; + session.model = &model; + } + + transcribe_status feed() { + return transcribe_stream_feed(&session, pcm.data(), static_cast(pcm.size()), nullptr); + } +}; + +void expect_failed(const transcribe_session & s, transcribe_status st, transcribe_status want) { + CHECK(st == want); + CHECK(transcribe_stream_get_state(&s) == TRANSCRIBE_STREAM_FAILED); + CHECK(transcribe_stream_last_status(&s) == want); +} + +void test_begin_throw() { + for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { + reset_globals(); + Fixture f; + g_begin = t; + const transcribe_status st = transcribe_stream_begin(&f.session, nullptr, nullptr); + expect_failed(f.session, st, t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + // A failed stream accepts a fresh begin. + g_begin = Throw::None; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + CHECK(transcribe_stream_get_state(&f.session) == TRANSCRIBE_STREAM_ACTIVE); + } +} + +void test_feed_throw() { + for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { + reset_globals(); + Fixture f; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + g_feed = t; + expect_failed(f.session, f.feed(), t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + // No longer ACTIVE, so further feeds are refused as on any failed stream. + g_feed = Throw::None; + CHECK(f.feed() == TRANSCRIBE_ERR_INVALID_ARG); + } +} + +void test_finalize_throw() { + for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { + reset_globals(); + Fixture f; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + CHECK(f.feed() == TRANSCRIBE_OK); + g_finalize = t; + const transcribe_status st = transcribe_stream_finalize(&f.session, nullptr); + expect_failed(f.session, st, t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + } +} + +void test_reset_throw_still_idles() { + reset_globals(); + Fixture f; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + f.session.full_text = "partial"; + f.session.has_result = true; + f.session.was_aborted = true; + g_reset = Throw::Runtime; + transcribe_stream_reset(&f.session); // void; the guard swallows the exception + CHECK(transcribe_stream_get_state(&f.session) == TRANSCRIBE_STREAM_IDLE); + CHECK(!f.session.has_result); + CHECK(f.session.full_text.empty()); + CHECK(!transcribe_was_aborted(&f.session)); + g_reset = Throw::None; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); // the guard logs each caught exception + + test_begin_throw(); + test_feed_throw(); + test_finalize_throw(); + test_reset_throw_still_idles(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("stream_hook_throw_unit: ok\n"); + return EXIT_SUCCESS; +} From 78d37829dc887fa7188cf87ac55b220cc44facdc Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:49:49 +0800 Subject: [PATCH 16/51] python: model-wide stream lease BEHAVIOR CHANGE (0.4.0 release notes): while a stream is active on a Model, Session.run(), run_batch() and stream() on ANY session of that model now raise the new transcribe_cpp.Busy at once. Before, a run on another session waited for the compute lock and then ran between the stream's feeds, which the C header forbids. A run(), run_batch() or second stream() on the stream's own session also raises Busy now, where it used to raise InvalidArgument from the native dispatcher. Runs that contend with other runs still wait, as before. This matches the stream lease the Rust, Swift and TypeScript bindings already have. Busy is a new TranscribeError subclass in errors.py, exported from the package. It is raised on the Python side only, so status is 0. No native status maps to it, the same as Rust's Error::Busy and TS's Busy. The messages reuse the Rust/Swift wording: - "a stream is active on this model; finish or drop it before run()" - the same with run_batch() - "a stream is already active on this model" (stream begin) How it works: - Model._exclusive(kind, busy=...) checks the lease after it acquires the lock and after the model-closed check. A call that was already queued when a stream began therefore sees that stream's lease. - The lease is _stream_lease.owner on the model: the _StreamLease token of the stream that holds it, or None. It is claimed under the lock only after transcribe_stream_begin succeeds. The Stream is built inside the same locked block. - Each Stream keeps its own token and releases the lease only while it is still the owner. A stream that already ended therefore never clears a lease a later stream took (the per-stream holds_lease of the other bindings). - The stream's own feed() and finalize() take the lock with no busy check. The lease is released by: - finalize(), always, on success or failure. - reset(), and so context-manager exit. - A failed feed(), but only if transcribe_stream_get_state no longer reads ACTIVE. That state is read under the same lock right after the feed. A feed rejected before the family hook (NaN/Inf -> InvalidArgument) leaves the stream ACTIVE and keeps the lease, as in TS commit 82740039. - Session.close(), inside the session's deferred free and right after transcribe_session_free, so no call can start while the stream's native session still exists. Model.close() closes its sessions, and its own deferred free clears the slot too. - Stream.__del__, for a stream that still holds the lease. Like the native frees, it never waits for the lock: GC can run it on the thread that holds the lock. It queues a closure through _free_or_defer that resets the native stream and releases the lease. The closure captures the slot and the token, never the Stream, Session or Model. Under the lock it re-checks that the token is still the owner. While the token is the owner, the session is still open, because session close clears the owner in the same step as its free. So the reset never touches a freed or reused session. Docstrings for Model, Session, Session.stream/run/run_batch/close, Stream, Model.close and the module, the README concurrency note and a test_lifetime comment now describe the lease. Before, they told callers not to run other sessions during a stream. Tests (test_compute_lock.py, model-free, using the fake-handle probes): - Sibling and same-session run / run_batch / stream raise Busy with the exact messages and status 0, and never reach native code. The lease holder keeps feeding and finalizing. - The lease is per model. A failed native begin takes no lease. - The lease is released by finalize, a failed finalize, reset, context-manager exit, GC of an active stream (reset exactly once), session close (a later GC of that stream does no native reset) and model close. - A rejected feed with the stream still ACTIVE keeps the lease, and a valid feed plus finalize then release it. A feed failure that leaves the stream FAILED releases it. - Stream A ends and stream B on another session begins. A failed feed on A, a finalize on A, A.reset(), GC of A and closing A's session then leave B's lease alone. - The busy check runs after the lock is acquired: a run queued behind a gated begin raises Busy if the begin succeeds, and runs if it fails. - An active stream GC'd on the lock-holder thread does not deadlock. It defers its reset and release until that call ends. - A session closed while another call holds the lock is freed before a queued run starts. test_streaming.py (real moonshine-streaming-tiny): - test_second_stream_while_active_rejected and test_run_while_stream_active_rejected now expect Busy on the same session and on a sibling. A sibling run works after finalize. - test_rejected_feed_keeps_stream_active now also checks that sibling run, sibling stream and same-session run are Busy after the NaN feed, and that a sibling run works after finalize. - New: a feed that fails in the hook (pending cancel -> Aborted, state "failed") releases the lease. - New: reset, GC or session close of an active stream frees the model. Mutation checks over test_compute_lock/test_streaming/test_lifetime: - busy check removed: 16 tests fail. - the feed never releases: 2 fail. - the feed always releases (pre-82740039 TS behavior): 2 fail. - the GC release removed: 3 fail. - the GC release run inline instead of deferred: 1 fails. - the session-close release removed: 3 fail. - release without the owner check: 1 fails. test_compute_lock.py passed 15 of 15 repeated runs. Verified against a build-shared libtranscribe built from this tree, with every TRANSCRIBE_SMOKE_* model set (whisper-tiny, moonshine-streaming- tiny, nemotron-3.5 prompted, canary-180m-flash, SenseVoiceSmall, nemotron-speech-streaming, parakeet-unified, Voxtral-Mini-4B-Realtime). Baseline at 2c337e72 was 193 passed, 1 skipped. After: 215 passed, 1 skipped. Every baseline test still passes. The 22 new results are the tests above. The one skip is the same speed-dependent cross-thread cancel test. The FFI drift gate and version sync are clean. --- bindings/python/README.md | 9 +- .../python/src/transcribe_cpp/__init__.py | 245 +++++++++--- bindings/python/src/transcribe_cpp/errors.py | 13 + bindings/python/tests/test_compute_lock.py | 351 +++++++++++++++++- bindings/python/tests/test_lifetime.py | 5 +- bindings/python/tests/test_streaming.py | 87 ++++- 6 files changed, 642 insertions(+), 68 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index be12b81d2..e42c5a143 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -124,9 +124,12 @@ TRANSCRIBE_LIBRARY=../../build-shared/src/libtranscribe.dylib \ compute backend, so the binding serializes `run` / `run_batch` / stream calls across all sessions of a model with a model-wide lock. Calls from other threads wait their turn (they no longer race); load one model per - worker for parallel transcription. An active stream still occupies the - model between feeds, so finish it before running other sessions of that - model. See the `Model` docstring. + worker for parallel transcription. An active stream occupies the model + between feeds, so while one is active, `run` / `run_batch` / `stream` on + any session of that model raise `transcribe_cpp.Busy` at once instead of + waiting. `finalize()` or `reset()` the stream (or close its session) to + free the model. A feed rejected for NaN/Inf samples keeps the stream + active. See the `Model` docstring. - Import package: `transcribe_cpp` - Distribution: `transcribe-cpp` - License: MIT diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 635a08435..db05dc4ad 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -17,7 +17,8 @@ Long-running native calls (model load, run) release the GIL — ctypes does this for every foreign call — so other Python threads make progress during inference. Compute calls on one Model are serialized by a model-wide lock (see ``Model``): -concurrent calls from other threads wait their turn instead of racing. +concurrent calls from other threads wait their turn instead of racing. While a +stream is active on a model, other runs and stream begins on it raise ``Busy``. """ from __future__ import annotations @@ -37,6 +38,7 @@ AbiError, Aborted, BackendError, + Busy, InputTooLong, InvalidArgument, ModelFileNotFound, @@ -112,6 +114,7 @@ "UnsupportedRequest", "AbiError", "Aborted", + "Busy", "InputTooLong", "OutputTruncated", "OutputRepetition", @@ -915,6 +918,28 @@ def _apply(self, ext) -> None: # --- high-level handles --------------------------------------------------- +class _StreamLeaseSlot: + """A model's stream lease: ``owner`` is the _StreamLease of the active + stream, or None. Guarded by the model's compute lock.""" + + __slots__ = ("owner",) + + def __init__(self): + self.owner: Optional[_StreamLease] = None + + +class _StreamLease: + """One stream's claim on its model's stream lease (identity token). + ``handle`` is the session handle the stream began on; while this lease + is the owner, that native session is alive (closing the session clears + the owner in the same deferred step that frees it).""" + + __slots__ = ("handle",) + + def __init__(self, handle: ctypes.c_void_p): + self.handle = handle + + class Model: """A loaded model. Sharing it across threads is safe: queries (``arch``, ``capabilities``, ``supports`` ...) and session creation run @@ -926,15 +951,23 @@ class Model: ``stream()`` and ``Stream.feed()`` / ``finalize()`` / ``reset()`` each hold a model-wide lock for the native call and for copying its results out. Calls on any session of the same model from other threads WAIT for - the lock — they never raise a "busy" error — and a waiting call can - still be cancelled with its session's ``cancel()``. Load one Model per - worker for parallel transcription. (Up to 0.3 the binding did not lock: - overlapping calls on one model raced and could corrupt decodes.) - - The lock covers individual calls, not a stream's lifetime: an active - stream still occupies the model between feeds (the C contract), so do - not run other sessions of this model until the stream is finalized or - reset. + the lock, and a waiting call can still be cancelled with its session's + ``cancel()``. Load one Model per worker for parallel transcription. (Up + to 0.3 the binding did not lock: overlapping calls on one model raced + and could corrupt decodes.) + + An active stream occupies the model for its whole lifetime, not just + during a feed (the C contract forbids a run between the feeds of a + stream on another session). A successful ``stream()`` therefore takes + a model-wide stream lease. Until that stream ends, ``run()``, + ``run_batch()`` and ``stream()`` on ANY session of this model (its own + included) raise :class:`Busy` at once instead of waiting; the stream's + own ``feed()`` / ``finalize()`` proceed. The lease is released by + ``finalize()`` (success or failure), ``reset()``, a ``feed()`` that + leaves the stream no longer active, closing the session or model, or + the Stream being garbage-collected while still active. A call that was + already waiting for the lock when a stream began also raises + :class:`Busy` once it gets the lock. ``close()`` (and garbage collection) never waits for the lock: closing the model or a session while a call is in flight marks it closed at @@ -1000,18 +1033,31 @@ def _h(self) -> ctypes.c_void_p: # thread holds the lock — immediately if it is free, else by the holder # just before it releases. FIFO order keeps the C contract that sessions # are freed before their model. + # + # The stream lease: _stream_lease.owner is the _StreamLease of the stream + # that is active on this model, or None. It is claimed under the lock + # only when a native stream begin succeeds, and read and cleared only + # under the lock (in an _exclusive body or a deferred closure). A stream + # releases it only if it is still the owner, so a stream that already + # ended can never clear a lease a later stream took. The deferred + # closures capture the slot, never the Model, like the native frees. def _init_compute_state(self) -> None: self._compute_lock = threading.Lock() self._compute_owner: Optional[int] = None # thread ident while held self._deferred: collections.deque = collections.deque() + self._stream_lease = _StreamLeaseSlot() @contextmanager - def _exclusive(self, kind: str): + def _exclusive(self, kind: str, busy: Optional[str] = None): """Hold the model-wide compute lock for one compute call of *kind* (blocking until it is free) and check the model is still open. Run the native call AND the copy-out of its results inside the block, - and capture the session handle inside it too.""" + and capture the session handle inside it too. + + With *busy* set (run, run_batch, stream begin), raise Busy(*busy*) + once the lock is held if a stream holds this model's lease. The + stream that holds the lease passes None for its own calls.""" me = threading.get_ident() if self._compute_owner == me: raise TranscribeError( @@ -1023,6 +1069,8 @@ def _exclusive(self, kind: str): try: if self._handle is None: raise TranscribeError("model is closed") + if busy is not None and self._stream_lease.owner is not None: + raise Busy(busy) yield finally: try: @@ -1162,7 +1210,8 @@ def close(self) -> None: Never waits for the compute lock: if a call is in flight on one of the sessions, the model is marked closed now and the native frees - (sessions first, then the model) run as soon as that call ends.""" + (sessions first, then the model) run as soon as that call ends. + Closing the session of an active stream releases the stream lease.""" handle = getattr(self, "_handle", None) if handle is None: return @@ -1171,7 +1220,13 @@ def close(self) -> None: self._handle = None for session in list(getattr(self, "_sessions", ()) or ()): session.close() - self._free_or_defer(lambda: _lib.transcribe_model_free(handle)) + slot = self._stream_lease + + def free(handle=handle, slot=slot): + _lib.transcribe_model_free(handle) + slot.owner = None # every session is gone; nothing can stream + + self._free_or_defer(free) def __enter__(self) -> "Model": return self @@ -1193,10 +1248,13 @@ class Session: ``feed`` / ``finalize`` / ``reset``) take the model-wide compute lock, so calls on sessions of one model — from any threads — serialize: a call made while another is in flight waits for it (see ``Model``). - Reads (``limits``, ``was_aborted``, ``Stream.text()`` / ``snapshot()`` - / ``state`` ...) do not take the lock; don't read a session from one - thread while another thread is computing on that same session. - ``cancel()`` is safe from any thread.""" + While a stream is active on ANY session of the model (this one + included), ``run``, ``run_batch`` and ``stream`` raise :class:`Busy` + instead; finalize or reset the stream first. Reads (``limits``, + ``was_aborted``, ``Stream.text()`` / ``snapshot()`` / ``state`` ...) do + not take the lock; don't read a session from one thread while another + thread is computing on that same session. ``cancel()`` is safe from any + thread.""" def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0): @@ -1282,7 +1340,9 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` (including its ``OutputRepetition`` subclass) the partial transcript is preserved - and attached to the exception as ``partial_result``.""" + and attached to the exception as ``partial_result``. + + Raises :class:`Busy` if a stream is active on this model.""" # Cleared BEFORE waiting for the compute lock, so a cancel() issued # while this call is queued behind another still aborts it. self._cancel.clear() @@ -1290,7 +1350,9 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", params = _build_run_params(task, language, target_language, timestamps, keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt, prefix) - with self._model._exclusive("run"): + with self._model._exclusive( + "run", busy="a stream is active on this model; " + "finish or drop it before run()"): ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1339,7 +1401,9 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", directly (the ``asyncio.gather`` convention). ``vocabulary`` / ``prompt`` apply to every utterance (see :meth:`run`); - a transcript prefix is per-utterance and so is not accepted here.""" + a transcript prefix is per-utterance and so is not accepted here. + + Raises :class:`Busy` if a stream is active on this model.""" self._cancel.clear() # before the lock wait, as in run() pcms = list(pcms) if not pcms: @@ -1358,7 +1422,9 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt) # The native call AND the per-utterance copy-out run under the lock. - with self._model._exclusive("run_batch"): + with self._model._exclusive( + "run_batch", busy="a stream is active on this model; " + "finish or drop it before run_batch()"): ext = self._resolve_family(family, "run") if family is not None else None if ext is not None: params.family = ctypes.cast( @@ -1422,10 +1488,15 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, Requires a model whose capabilities advertise ``supports_streaming``; otherwise raises NotImplementedByModel. ``family`` is an optional family-specific stream extension (e.g. MoonshineStreamingOptions). The - session is single-threaded and runs at most one stream at a time. Use - the Stream as a context manager so it is reset when you are done. - ``vocabulary`` and ``prompt`` are as in :meth:`run`; ``task="instruct"`` - raises.""" + session is single-threaded. Use the Stream as a context manager so it + is reset when you are done. ``vocabulary`` and ``prompt`` are as in + :meth:`run`; ``task="instruct"`` raises. + + A successful begin takes the model's stream lease: until this stream + is finalized, reset, fails, or is garbage-collected (or its session + or model is closed), ``run()``, ``run_batch()`` and ``stream()`` on + any session of the model raise :class:`Busy`. A begin while another + stream on the model is active raises :class:`Busy` too.""" self._cancel.clear() # before the lock wait, as in run() # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). @@ -1436,21 +1507,30 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, _lib.transcribe_stream_params_init(_byref(sp)) sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy") sp.stable_prefix_agreement_n = stable_prefix_agreement_n - with self._model._exclusive("stream_begin"): + with self._model._exclusive( + "stream_begin", busy="a stream is already active on this model"): ext = self._resolve_family(family, "stream") if family is not None else None if ext is not None: sp.family = ctypes.cast( _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + h = self._h _check( - _lib.transcribe_stream_begin(self._h, _byref(run_params), _byref(sp)), + _lib.transcribe_stream_begin(h, _byref(run_params), _byref(sp)), "transcribe_stream_begin", ) - # The C contract says everything passed to begin may be freed once it - # returns (strings are copied into session-owned storage). The Stream - # still pins the params structs until reset() as defense in depth — - # it costs nothing and keeps the binding safe even against an older - # or out-of-tree native library that predates that contract. - return Stream(self, _keepalive=(run_params, sp, ext)) + # Claim the lease for the whole stream lifetime, only now that the + # native begin succeeded. Released at finalize/reset, by a feed + # that ends the stream, by session/model close, or by Stream GC. + lease = _StreamLease(h) + self._model._stream_lease.owner = lease + # The C contract says everything passed to begin may be freed once + # it returns (strings are copied into session-owned storage). The + # Stream still pins the params structs until reset() as defense in + # depth — it costs nothing and keeps the binding safe even against + # an older or out-of-tree native library that predates that + # contract. Built under the lock, so the lease always has a Stream + # that can release it. + return Stream(self, _keepalive=(run_params, sp, ext), _lease=lease) def _materialize(self, h: ctypes.c_void_p | None = None, utt: int | None = None) -> Result: @@ -1569,7 +1649,9 @@ def was_aborted(self) -> bool: def close(self) -> None: """Free the session. Idempotent. Never waits for the compute lock: if a call is in flight on this model, the session is marked closed now - and the native free runs as soon as that call ends.""" + and the native free runs as soon as that call ends. If a stream on + this session holds the model's stream lease, the lease is released + right after the native free.""" handle = getattr(self, "_handle", None) if handle is None: return @@ -1578,9 +1660,16 @@ def close(self) -> None: # it, so the deferred free pins it. Nothing here references self: the # free may run after this object is gone (close() from __del__). trampoline = getattr(self, "_abort_trampoline", None) + slot = self._model._stream_lease - def free(handle=handle, _pin=trampoline): + def free(handle=handle, _pin=trampoline, slot=slot): _lib.transcribe_session_free(handle) + # Release the lease only after the free, in the same locked step: + # a run queued behind this must not start while the stream's + # native session still exists. Only this session's stream's lease. + owner = slot.owner + if owner is not None and owner.handle is handle: + slot.owner = None self._model._free_or_defer(free) @@ -1602,15 +1691,36 @@ class Stream: Feed audio in chunks with ``feed()``, read the committed/tentative text with ``text()``, and call ``finalize()`` when the audio ends. The session is - returned to idle on context-manager exit (or ``reset()``).""" - - def __init__(self, session: Session, *, _keepalive=None): + returned to idle on context-manager exit (or ``reset()``). + + While the stream is active it holds its model's stream lease: ``run()``, + ``run_batch()`` and ``stream()`` on every session of the model raise + :class:`Busy` (see ``Model``). ``finalize()`` and ``reset()`` release + it, and so does a ``feed()`` failure that leaves the stream no longer + active (``state`` is then ``"failed"``). A feed rejected before the + model sees it (e.g. NaN/Inf samples, ``InvalidArgument``) keeps the + stream ``"active"`` and keeps the lease. A Stream garbage-collected + while active is reset and releases the lease, behind any in-flight + call.""" + + def __init__(self, session: Session, *, _keepalive=None, _lease=None): self._session = session # keep the session (and its model) alive # Pins the ctypes params structs (and any family ext) passed to # transcribe_stream_begin until reset(). The native library copies # what it needs at begin; this is belt-and-braces for the FFI layer. self._keepalive = _keepalive self._active = True + # This stream's claim on the model's stream lease (None: holds none). + # It holds the lease while model._stream_lease.owner is this object. + self._lease: Optional[_StreamLease] = _lease + + def _release_lease_locked(self) -> None: + """Release the model's stream lease if THIS stream still holds it. + Caller holds the compute lock. A lease another stream has taken since + is left alone.""" + slot = self._session._model._stream_lease + if self._lease is not None and slot.owner is self._lease: + slot.owner = None @property def _h(self) -> ctypes.c_void_p: @@ -1623,9 +1733,20 @@ def feed(self, pcm: PCMLike) -> StreamUpdate: array, n_samples = _pcm_to_carray(pcm) update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) + # No busy check: this stream holds the lease (or has none to take). with self._session._model._exclusive("stream_feed"): - _check(_lib.transcribe_stream_feed(self._h, array, n_samples, _byref(update)), - "transcribe_stream_feed") + h = self._h + status = _lib.transcribe_stream_feed(h, array, n_samples, _byref(update)) + # The lease follows the native stream lifecycle. A feed rejected + # before the family hook (non-finite samples, other pre-hook + # INVALID_ARG) leaves the stream ACTIVE: keep the lease, so the + # stream stays usable and other calls stay Busy. A failure that + # leaves it no longer ACTIVE (FAILED) ends the stream: release. + if (status != _generated.TRANSCRIBE_OK + and _lib.transcribe_stream_get_state(h) + != _generated.TRANSCRIBE_STREAM_ACTIVE): + self._release_lease_locked() + _check(status, "transcribe_stream_feed") return _stream_update_from(update) def finalize(self) -> StreamUpdate: @@ -1633,8 +1754,14 @@ def finalize(self) -> StreamUpdate: update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) with self._session._model._exclusive("stream_finalize"): - _check(_lib.transcribe_stream_finalize(self._h, _byref(update)), - "transcribe_stream_finalize") + h = self._h + try: + status = _lib.transcribe_stream_finalize(h, _byref(update)) + finally: + # Finalize ends the active stream (FINISHED, or FAILED on + # error): release the lease either way. + self._release_lease_locked() + _check(status, "transcribe_stream_finalize") return _stream_update_from(update) def text(self) -> StreamText: @@ -1676,10 +1803,12 @@ def last_status(self) -> TranscribeError | None: return exception_for_status(status, _status_string(status), "stream") def reset(self) -> None: - """Return the session to idle, discarding stream state. Idempotent.""" + """Return the session to idle, discarding stream state, and release + the model's stream lease. Idempotent.""" if self._active: with self._session._model._exclusive("stream_reset"): _lib.transcribe_stream_reset(self._session._h) + self._release_lease_locked() self._active = False self._keepalive = None @@ -1689,6 +1818,34 @@ def __enter__(self) -> "Stream": def __exit__(self, *exc) -> None: self.reset() + def __del__(self): + # Abandoned while still holding the lease (never finalized or reset): + # reset the native stream and release the lease. Never waits for the + # lock, like the native frees: GC may run this on the thread that + # holds it. The closure captures the lease slot and token, not self + # or the Session/Model. Reading the owner here without the lock is + # safe: a stream's lease is only ever released, never re-taken, so + # "not the owner" is final; the closure re-checks under the lock. + try: + lease = self._lease + if lease is None: + return + model = self._session._model + slot = model._stream_lease + if slot.owner is not lease: + return + + def reset_and_release(slot=slot, lease=lease): + # Still the owner means the session is still open: closing it + # clears the owner in the same deferred step as its free. + if slot.owner is lease: + _lib.transcribe_stream_reset(lease.handle) + slot.owner = None + + model._free_or_defer(reset_and_release) + except Exception: + pass + def transcribe( model: Model | str | os.PathLike, diff --git a/bindings/python/src/transcribe_cpp/errors.py b/bindings/python/src/transcribe_cpp/errors.py index c92e3f16f..88828aa85 100644 --- a/bindings/python/src/transcribe_cpp/errors.py +++ b/bindings/python/src/transcribe_cpp/errors.py @@ -91,6 +91,19 @@ class InputTooLong(TranscribeError): pass +class Busy(TranscribeError): + """A stream is active on this model, so the call was refused instead of + started. + + The C library allows one run / batch / active stream per model across + ALL of its sessions, and an active stream occupies the model from begin + until it ends, not just during a feed. While one is active, + ``Session.run()``, ``run_batch()`` and ``stream()`` on any session of + that model raise this at once. Finalize or reset the stream first, or + give each concurrent worker its own Model. Raised purely on the Python + side, so ``status`` is 0.""" + + class Aborted(TranscribeError): """The run/stream was ended by ``Session.cancel()``. diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index f84d281b7..78dfa055b 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -1,7 +1,10 @@ -"""Model-wide compute lock: deterministic tests that need NO model. +"""Model-wide compute lock and stream lease: deterministic tests that need NO +model. The native library allows one compute call in flight per model across all of -its sessions; the binding enforces that with ``Model._exclusive(kind)``. These +its sessions, and an active stream occupies the model from begin until it +ends. The binding enforces that with ``Model._exclusive(kind)`` and the +model-wide stream lease (``Model._stream_lease``). These tests build Model/Session objects around fake handles and replace the native entry points the high-level code calls (``transcribe_cpp._lib.``, looked up at call time) with Python probes, so overlap, waiting, deferred frees and @@ -61,7 +64,12 @@ def fake(monkeypatch): lambda h: frees.append(("session", h.value))) monkeypatch.setattr(t._lib, "transcribe_model_free", lambda h: frees.append(("model", h.value))) - monkeypatch.setattr(t._lib, "transcribe_stream_reset", lambda h: None) + resets: list = [] # session handles transcribe_stream_reset was given + monkeypatch.setattr(t._lib, "transcribe_stream_reset", + lambda h: resets.append(h.value)) + state = {"value": _generated.TRANSCRIBE_STREAM_ACTIVE} + monkeypatch.setattr(t._lib, "transcribe_stream_get_state", + lambda h: state["value"]) # The copy-out reads many native accessors; stub it to report which # handle it was given (proves the in-flight call kept its handle). monkeypatch.setattr(t.Session, "_materialize", @@ -87,7 +95,8 @@ def session(m, track=True): made.append(s) return s - yield SimpleNamespace(model=model, session=session, frees=frees) + yield SimpleNamespace(model=model, session=session, frees=frees, + resets=resets, state=state) for obj in reversed(made): obj.close() @@ -441,3 +450,337 @@ def probe(h, *rest): assert "error" not in box, box assert "re-entrant" in str(inner.get("error")), inner assert not m._compute_lock.locked() + + +# --- stream lease --------------------------------------------------------------- +# +# From a successful stream begin until that stream ends, run / run_batch / +# stream on ANY session of the model raise Busy at once (the C contract +# forbids a run between the feeds of an active stream on another session). +# The stream itself keeps feeding. These mirror the Rust/Swift/TS bindings. + +RUN_BUSY = "a stream is active on this model; finish or drop it before run()" +BATCH_BUSY = ("a stream is active on this model; " + "finish or drop it before run_batch()") +BEGIN_BUSY = "a stream is already active on this model" + + +@pytest.fixture +def native(fake, monkeypatch): + """Probes for the native compute entry points. ``calls`` records + (name, session handle) for every native compute call that ran; + ``status`` sets what each one returns.""" + calls: list = [] + status = {"run": 0, "begin": 0, "feed": 0, "finalize": 0} + + def probe(name, key): + def fn(h, *rest): + calls.append((name, h.value)) + return status[key] + return fn + + monkeypatch.setattr(t._lib, "transcribe_run", probe("run", "run")) + monkeypatch.setattr(t._lib, "transcribe_run_batch", probe("run_batch", "run")) + monkeypatch.setattr(t._lib, "transcribe_batch_n_results", lambda h: 1) + monkeypatch.setattr(t._lib, "transcribe_batch_status", lambda h, i: 0) + monkeypatch.setattr(t._lib, "transcribe_stream_begin", probe("begin", "begin")) + monkeypatch.setattr(t._lib, "transcribe_stream_feed", probe("feed", "feed")) + monkeypatch.setattr(t._lib, "transcribe_stream_finalize", + probe("finalize", "finalize")) + return SimpleNamespace(calls=calls, status=status) + + +def _assert_busy(session, *, ran: list): + """Every lease-checked call on *session* raises Busy without reaching + native code.""" + before = list(ran) + with pytest.raises(t.Busy) as ei: + session.run(PCM) + assert str(ei.value) == RUN_BUSY + assert ei.value.status == 0 and isinstance(ei.value, t.TranscribeError) + with pytest.raises(t.Busy) as ei: + session.run_batch([PCM]) + assert str(ei.value) == BATCH_BUSY + with pytest.raises(t.Busy) as ei: + session.stream() + assert str(ei.value) == BEGIN_BUSY + assert ran == before, "a refused call reached native code" + assert not session._model._compute_lock.locked() + + +def _assert_free(session): + """The model is free again: a run on *session* goes through.""" + assert session.run(PCM) == ("result", session._handle.value, None) + + +def test_active_stream_refuses_sibling_and_same_session_calls(fake, native): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + stream = s1.stream() + _assert_busy(s2, ran=native.calls) # sibling session + _assert_busy(s1, ran=native.calls) # the stream's own session + # The lease holder itself feeds and finalizes without a busy check. + stream.feed(PCM) + stream.feed(PCM) + stream.finalize() + assert [n for n, _ in native.calls] == ["begin", "feed", "feed", "finalize"] + _assert_free(s2) + _assert_free(s1) + + +def test_busy_is_exported(): + assert "Busy" in t.__all__ + assert t.Busy is t.errors.Busy + assert issubclass(t.Busy, t.TranscribeError) + assert not issubclass(t.Busy, t.InvalidArgument) + + +def test_lease_is_per_model(fake, native): + s1 = fake.session(fake.model()) + other = fake.session(fake.model()) + s1.stream() + _assert_free(other) + + +def test_failed_begin_takes_no_lease(fake, native): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + native.status["begin"] = _generated.TRANSCRIBE_ERR_NOT_IMPLEMENTED + with pytest.raises(t.NotImplementedByModel): + s1.stream() + assert m._stream_lease.owner is None + _assert_free(s2) + + +@pytest.mark.parametrize("ending", [ + "finalize", "finalize_error", "reset", "context_exit", "gc", + "session_close", +]) +def test_lease_released_when_stream_ends(fake, native, ending): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + h1 = s1._handle.value + stream = s1.stream() + stream.feed(PCM) + _assert_busy(s2, ran=native.calls) + + if ending == "finalize": + stream.finalize() + elif ending == "finalize_error": + native.status["finalize"] = _generated.TRANSCRIBE_ERR_BACKEND + with pytest.raises(t.BackendError): + stream.finalize() + elif ending == "reset": + stream.reset() + assert fake.resets == [h1] + elif ending == "context_exit": + with stream: + pass + assert fake.resets == [h1] + elif ending == "gc": + stream._cycle = stream # only the cyclic GC can reclaim it + del stream + gc.collect() + assert fake.resets == [h1], "GC'd active stream was not reset once" + elif ending == "session_close": + s1.close() + assert fake.frees == [("session", h1)] + + assert m._stream_lease.owner is None + _assert_free(s2) + if ending == "session_close": + # The stream outlived its session: dropping it now must not reset + # a freed native session. + del stream + gc.collect() + assert fake.resets == [] + + +def test_model_close_releases_lease(fake, native): + m = fake.model() + s1 = fake.session(m) + stream = s1.stream() + m.close() + assert m._stream_lease.owner is None + with pytest.raises(t.TranscribeError, match="closed"): + stream.feed(PCM) + del stream + gc.collect() + assert fake.resets == [] + + +def test_rejected_feed_keeps_lease_while_stream_active(fake, native): + # A feed rejected before the family hook (NaN/Inf -> InvalidArgument) + # leaves the native stream ACTIVE: the lease stays with it. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + stream = s1.stream() + native.status["feed"] = _generated.TRANSCRIBE_ERR_INVALID_ARG + fake.state["value"] = _generated.TRANSCRIBE_STREAM_ACTIVE + with pytest.raises(t.InvalidArgument): + stream.feed(PCM) + assert not m._compute_lock.locked() + _assert_busy(s2, ran=native.calls) + _assert_busy(s1, ran=native.calls) + # A later valid feed and finalize then release it. + native.status["feed"] = 0 + stream.feed(PCM) + _assert_busy(s2, ran=native.calls) + stream.finalize() + _assert_free(s2) + + +def test_feed_failure_that_ends_stream_releases_lease(fake, native): + # A failure inside the family hook leaves the stream FAILED, which is no + # longer an active stream: the lease is released with the error. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + stream = s1.stream() + native.status["feed"] = _generated.TRANSCRIBE_ERR_ABORTED + fake.state["value"] = _generated.TRANSCRIBE_STREAM_FAILED + with pytest.raises(t.Aborted): + stream.feed(PCM) + assert m._stream_lease.owner is None + _assert_free(s2) + + +def test_ended_stream_never_clears_a_later_lease(fake, native): + m = fake.model() + sa, sb, sc = fake.session(m), fake.session(m), fake.session(m) + a = sa.stream() + a.finalize() + b = sb.stream() # B now holds the lease + _assert_busy(sc, ran=native.calls) + + # A, already ended, fails a feed (not ACTIVE), finalizes again, resets + # and is garbage-collected: none of that may touch B's lease. + native.status["feed"] = _generated.TRANSCRIBE_ERR_INVALID_ARG + fake.state["value"] = _generated.TRANSCRIBE_STREAM_FINISHED + with pytest.raises(t.InvalidArgument): + a.feed(PCM) + _assert_busy(sc, ran=native.calls) + a.finalize() + _assert_busy(sc, ran=native.calls) + a.reset() + _assert_busy(sc, ran=native.calls) + a._cycle = a + del a + gc.collect() + _assert_busy(sc, ran=native.calls) + sa.close() # A's session closing leaves B's lease alone too + _assert_busy(sc, ran=native.calls) + + native.status["feed"] = 0 + fake.state["value"] = _generated.TRANSCRIBE_STREAM_ACTIVE + b.feed(PCM) + b.finalize() + _assert_free(sc) + + +@pytest.mark.parametrize("begin_ok", [True, False]) +def test_busy_check_happens_after_acquiring_the_lock(fake, native, monkeypatch, + begin_ok): + # A run that is already waiting for the lock when a stream begins must + # see that stream's lease once it gets the lock; if the begin fails, + # no lease was taken and the queued run proceeds. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + gate = _Gate() + begin_status = 0 if begin_ok else _generated.TRANSCRIBE_ERR_INVALID_ARG + + def begin(h, *rest): + gate(h) + return begin_status + + monkeypatch.setattr(t._lib, "transcribe_stream_begin", begin) + try: + a, a_box = _in_thread(s1.stream) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + b, b_box = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) # parked on the lock, before the lease + finally: + gate.release.set() + _join(a, "stream begin") + _join(b, "queued run") + if begin_ok: + assert isinstance(a_box.get("value"), t.Stream), a_box + assert isinstance(b_box.get("error"), t.Busy), b_box + assert native.calls == [] # the queued run never reached native + a_box["value"].reset() + else: + assert isinstance(a_box.get("error"), t.InvalidArgument), a_box + assert b_box.get("value") == ("result", s2._handle.value, None), b_box + assert not m._compute_lock.locked() + + +def test_stream_gc_on_holder_thread_defers_reset(fake, native, monkeypatch): + # GC can finalize an abandoned stream on the thread that holds the lock. + # Its reset + lease release must neither deadlock nor run under the + # in-flight call; it runs right after, and frees the model. + m = fake.model() + s, victim_session = fake.session(m), fake.session(m) + hv = victim_session._handle.value + during: dict = {} + + def run(h, *rest): + # Simulate a stream on victim_session that holds the lease and is + # dropped mid-call (a real begin here would be a re-entrant call). + lease = t._StreamLease(victim_session._handle) + m._stream_lease.owner = lease + victim = t.Stream(victim_session, _lease=lease) + victim._cycle = victim + del victim + gc.collect() # runs Stream.__del__ on THIS thread, lock held + during["resets"] = list(fake.resets) + during["owner"] = m._stream_lease.owner + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", run) + th, box = _in_thread(s.run, PCM) + _join(th, "run with a stream GC'd on the holder thread") + assert "error" not in box, box + assert during["resets"] == [], "stream reset under the in-flight call" + assert during["owner"] is not None, "lease released under the lock early" + assert fake.resets == [hv] + assert m._stream_lease.owner is None + monkeypatch.setattr(t._lib, "transcribe_run", lambda *a: 0) + _assert_free(s) + + +def test_session_close_releases_lease_after_free(fake, native, monkeypatch): + # Closing the stream's session while another call holds the lock defers + # the free; the lease is released only after it, so the next call never + # starts while the stream's native session still exists. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + h1 = s1._handle.value + stream = s1.stream() + gate = _Gate() + seen: dict = {} + + def run(h, *rest): + seen["frees"] = list(fake.frees) + return 0 + + monkeypatch.setattr(t._lib, "transcribe_stream_feed", gate) + monkeypatch.setattr(t._lib, "transcribe_run", run) + try: + a, a_box = _in_thread(stream.feed, PCM) + assert gate.entered.wait(TIMEOUT) + s1.close() # never waits; the free is queued behind the feed + assert fake.frees == [] and m._stream_lease.owner is not None + spy = _LockSpy(m) + b, b_box = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) + finally: + gate.release.set() + _join(a, "in-flight feed") + _join(b, "queued run") + assert "error" not in a_box, a_box + assert b_box.get("value") == ("result", s2._handle.value, None), b_box + assert seen["frees"] == [("session", h1)] + assert m._stream_lease.owner is None + del stream + gc.collect() + assert fake.resets == [] diff --git a/bindings/python/tests/test_lifetime.py b/bindings/python/tests/test_lifetime.py index 164aab4e1..c6aecc487 100644 --- a/bindings/python/tests/test_lifetime.py +++ b/bindings/python/tests/test_lifetime.py @@ -81,8 +81,9 @@ def test_gc_order_session_keeps_model_alive(model_path, audio_pcm): # overlapping runs race (observed: corrupted whisper decodes on CPU, # command-buffer failures on Metal). The binding enforces that itself with a # model-wide compute lock: concurrent calls from several threads serialize -# (they wait, never raise), so callers no longer need their own lock. The -# deterministic, model-free lock tests live in test_compute_lock.py. +# (they wait; only an active stream makes them raise Busy), so callers no +# longer need their own lock. The deterministic, model-free lock and stream +# lease tests live in test_compute_lock.py. def test_serial_sessions_across_threads(model_path, audio_pcm): diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index aa5fceb6e..be3ba50c0 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -151,41 +151,49 @@ def test_stream_reset_idempotent_and_session_reusable( def test_second_stream_while_active_rejected(streaming_model_path, audio_pcm): - # One stream at a time per session: the C dispatcher rejects a begin - # while ACTIVE, and the binding surfaces it as InvalidArgument. + # One active stream per MODEL: the stream lease refuses a second begin + # with Busy, on the same session and on a sibling, before native code. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: stream.feed(audio_pcm[:16000]) - with pytest.raises(t.InvalidArgument): + with pytest.raises(t.Busy, match="already active"): session.stream() + with pytest.raises(t.Busy, match="already active"): + sibling.stream() + assert stream.state == "active" def test_run_while_stream_active_rejected(streaming_model_path, audio_pcm): - # A run cannot replace an active stream's results on the SAME session: - # the C dispatcher rejects it, the binding surfaces InvalidArgument, and - # the stream stays usable afterwards. + # While a stream is active, the stream lease refuses run/run_batch on + # ANY session of the model with Busy (the C contract forbids a run + # between another session's feeds), the stream stays usable, and once + # it is finalized a sibling run goes through. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: stream.feed(audio_pcm[:16000]) - with pytest.raises(t.InvalidArgument): - session.run(audio_pcm[:16000]) - with pytest.raises(t.InvalidArgument): - session.run_batch([audio_pcm[:16000]]) + for target in (session, sibling): + with pytest.raises(t.Busy, match=r"before run\(\)"): + target.run(audio_pcm[:16000]) + with pytest.raises(t.Busy, match=r"before run_batch\(\)"): + target.run_batch([audio_pcm[:16000]]) for i in range(16000, len(audio_pcm), 16000): stream.feed(audio_pcm[i : i + 16000]) stream.finalize() assert stream.state == "finished" committed = stream.text().committed + ran = sibling.run(audio_pcm).text assert "country" in committed.lower(), committed + assert "country" in ran.lower(), ran def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): # Non-finite PCM is rejected BEFORE the family hook: the feed raises # InvalidArgument but the native stream stays ACTIVE (only failures - # inside the hook move it to FAILED). The binding must release the - # model's compute lock on that error and leave the stream usable. + # inside the hook move it to FAILED). The binding releases the compute + # lock on that error but KEEPS the stream lease: the stream stays + # usable and sibling calls stay Busy until it is finalized. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: stream.feed(audio_pcm[:16000]) bad = list(audio_pcm[16000:32000]) bad[100] = float("nan") @@ -193,12 +201,61 @@ def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): stream.feed(bad) assert stream.state == "active" assert not model._compute_lock.locked() + with pytest.raises(t.Busy): + sibling.run(audio_pcm[:16000]) + with pytest.raises(t.Busy): + sibling.stream() + with pytest.raises(t.Busy): + session.run(audio_pcm[:16000]) for i in range(16000, len(audio_pcm), 16000): stream.feed(audio_pcm[i : i + 16000]) update = stream.finalize() committed = stream.text().committed + ran = sibling.run(audio_pcm).text # finalize released the lease assert update.is_final assert "country" in committed.lower(), committed + assert "country" in ran.lower(), ran + + +def test_hook_failure_feed_releases_lease(streaming_model_path, audio_pcm): + # A feed that fails INSIDE the family hook (here: a pending cancel the + # hook polls -> Aborted) leaves the stream FAILED, which is no longer + # active: the lease goes with it and a sibling run proceeds at once. + with t.Model(streaming_model_path) as model, model.session() as session: + with model.session() as sibling, session.stream() as stream: + stream.feed(audio_pcm[:16000]) + session.cancel() + with pytest.raises(t.Aborted): + stream.feed(audio_pcm[16000:32000]) + assert stream.state == "failed" + assert isinstance(stream.last_status, t.Aborted) + ran = sibling.run(audio_pcm).text + assert "country" in ran.lower(), ran + + +@pytest.mark.parametrize("ending", ["reset", "gc", "session_close"]) +def test_stream_lease_released_without_finalize(streaming_model_path, audio_pcm, + ending): + # Abandoning an active stream (reset, dropping it, or closing its + # session) frees the model for other sessions. + import gc + + with t.Model(streaming_model_path) as model, model.session() as sibling: + session = model.session() + stream = session.stream() + stream.feed(audio_pcm[:16000]) + with pytest.raises(t.Busy): + sibling.run(audio_pcm[:16000]) + if ending == "reset": + stream.reset() + elif ending == "gc": + del stream + gc.collect() + else: + session.close() + ran = sibling.run(audio_pcm).text + session.close() + assert "country" in ran.lower(), ran def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): From e2752c2817a611b0dc0954ebdce355200546576e Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:54:36 +0800 Subject: [PATCH 17/51] roles: public transcribe_role, transcribe_model_roles, UNSUPPORTED_ROLE Additive (D2, D5). - transcribe_role (TRANSCRIBE_ROLE_ASR, TRANSCRIBE_ROLE_DIARIZE) and transcribe_model_roles(model): the bitmask a loaded model serves, fixed at load, 0 for NULL. The internal k_role_* bits now take the public values (static_assert'd). - TRANSCRIBE_ERR_UNSUPPORTED_ROLE (20, appended) with a status string. - transcribe_session_init (and so transcribe_open) and transcribe_model_get_capabilities return UNSUPPORTED_ROLE for a model without the ASR role. Capabilities leave the caller's struct untouched rather than returning zeros that read as "supports nothing". Every model that loads today serves ASR, so no current caller sees the new code. Low-level bindings regenerated from the header (Python / TypeScript via generate.py, Rust via cargo xtask bindgen); transcribe.abihash updated. The Swift pinned hash and per-binding status mapping follow in their own commits. tests: role_resolve_unit covers the role checks (session_init leaves *out NULL, capabilities untouched) and the enum/bit agreement; api_smoke covers the new status string and transcribe_model_roles(NULL). --- .../python/src/transcribe_cpp/_generated.py | 7 +++- bindings/rust/sys/src/transcribe_sys.rs | 15 ++++++- bindings/typescript/src/_generated.ts | 6 ++- include/transcribe.abihash | 2 +- include/transcribe.h | 41 +++++++++++++++++-- src/transcribe-asr.cpp | 8 ++++ src/transcribe-model.h | 10 ++--- src/transcribe.cpp | 6 +++ tests/api_smoke.c | 2 + tests/role_resolve_unit.cpp | 39 +++++++++++++++++- 10 files changed, 121 insertions(+), 15 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index d1122d2ae..518a57770 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "59b9a92b47074666" +PUBLIC_HEADER_HASH = "1537ea60b442dd53" # === enum constants === TRANSCRIBE_OK = 0 @@ -36,6 +36,7 @@ TRANSCRIBE_ERR_INPUT_TOO_LONG = 17 TRANSCRIBE_ERR_OUTPUT_TRUNCATED = 18 TRANSCRIBE_ERR_OUTPUT_REPETITION = 19 +TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20 TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0 TRANSCRIBE_ABI_SESSION_PARAMS = 1 TRANSCRIBE_ABI_RUN_PARAMS = 2 @@ -90,6 +91,8 @@ TRANSCRIBE_DEVICE_TYPE_GPU = 1 TRANSCRIBE_DEVICE_TYPE_IGPU = 2 TRANSCRIBE_DEVICE_TYPE_ACCEL = 3 +TRANSCRIBE_ROLE_ASR = 1 +TRANSCRIBE_ROLE_DIARIZE = 2 TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0 TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1 TRANSCRIBE_FEATURE_LONG_FORM = 2 @@ -335,6 +338,8 @@ def configure(lib): lib.transcribe_model_load_params_init.argtypes = [_c.POINTER(transcribe_model_load_params)] lib.transcribe_model_meta_val_str.restype = _c.c_char_p lib.transcribe_model_meta_val_str.argtypes = [_c.c_void_p, _c.c_char_p] + lib.transcribe_model_roles.restype = _c.c_uint32 + lib.transcribe_model_roles.argtypes = [_c.c_void_p] lib.transcribe_model_supports.restype = _c.c_bool lib.transcribe_model_supports.argtypes = [_c.c_void_p, _c.c_int] lib.transcribe_model_variant_string.restype = _c.c_char_p diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index cbccfab04..7d009856f 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 59b9a92b47074666 +// Pinned to include/transcribe.abihash = 1537ea60b442dd53 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "59b9a92b47074666"; +pub const PUBLIC_HEADER_HASH: &str = "1537ea60b442dd53"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -36,6 +36,7 @@ impl transcribe_status { pub const TRANSCRIBE_ERR_INPUT_TOO_LONG: transcribe_status = transcribe_status(17); pub const TRANSCRIBE_ERR_OUTPUT_TRUNCATED: transcribe_status = transcribe_status(18); pub const TRANSCRIBE_ERR_OUTPUT_REPETITION: transcribe_status = transcribe_status(19); + pub const TRANSCRIBE_ERR_UNSUPPORTED_ROLE: transcribe_status = transcribe_status(20); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -441,6 +442,16 @@ const _: () = { unsafe extern "C" { pub fn transcribe_capabilities_init(out: *mut transcribe_capabilities); } +impl transcribe_role { + pub const TRANSCRIBE_ROLE_ASR: transcribe_role = transcribe_role(1); + pub const TRANSCRIBE_ROLE_DIARIZE: transcribe_role = transcribe_role(2); +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct transcribe_role(pub ::std::os::raw::c_uint); +unsafe extern "C" { + pub fn transcribe_model_roles(model: *const transcribe_model) -> u32; +} unsafe extern "C" { pub fn transcribe_model_get_capabilities( model: *const transcribe_model, diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index 7c4db3ce9..423891c35 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "59b9a92b47074666"; +export const PUBLIC_HEADER_HASH = "1537ea60b442dd53"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -34,6 +34,7 @@ export const TRANSCRIBE_ERR_UNSUPPORTED_ITN = 16; export const TRANSCRIBE_ERR_INPUT_TOO_LONG = 17; export const TRANSCRIBE_ERR_OUTPUT_TRUNCATED = 18; export const TRANSCRIBE_ERR_OUTPUT_REPETITION = 19; +export const TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20; export const TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0; export const TRANSCRIBE_ABI_SESSION_PARAMS = 1; export const TRANSCRIBE_ABI_RUN_PARAMS = 2; @@ -88,6 +89,8 @@ export const TRANSCRIBE_DEVICE_TYPE_CPU = 0; export const TRANSCRIBE_DEVICE_TYPE_GPU = 1; export const TRANSCRIBE_DEVICE_TYPE_IGPU = 2; export const TRANSCRIBE_DEVICE_TYPE_ACCEL = 3; +export const TRANSCRIBE_ROLE_ASR = 1; +export const TRANSCRIBE_ROLE_DIARIZE = 2; export const TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0; export const TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1; export const TRANSCRIBE_FEATURE_LONG_FORM = 2; @@ -242,6 +245,7 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_model_load_file': { ret: 'transcribe_status', args: ['const char *', 'const struct transcribe_model_load_params *', 'struct transcribe_model **'] }, 'transcribe_model_load_params_init': { ret: 'void', args: ['struct transcribe_model_load_params *'] }, 'transcribe_model_meta_val_str': { ret: 'const char *', args: ['const struct transcribe_model *', 'const char *'] }, + 'transcribe_model_roles': { ret: 'uint32_t', args: ['const struct transcribe_model *'] }, 'transcribe_model_supports': { ret: '_Bool', args: ['const struct transcribe_model *', 'transcribe_feature'] }, 'transcribe_model_variant_string': { ret: 'const char *', args: ['const struct transcribe_model *'] }, 'transcribe_moonshine_streaming_stream_ext_init': { ret: 'void', args: ['struct transcribe_moonshine_streaming_stream_ext *'] }, diff --git a/include/transcribe.abihash b/include/transcribe.abihash index ae00915b6..1449a2276 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -59b9a92b47074666 +1537ea60b442dd53 diff --git a/include/transcribe.h b/include/transcribe.h index d8bf29590..e8fd68896 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -313,6 +313,15 @@ typedef enum { * the check. See docs/input-limits.md. */ TRANSCRIBE_ERR_OUTPUT_REPETITION = 19, + /* + * The model does not serve the role the call needs. Returned by + * transcribe_session_init / transcribe_open and + * transcribe_model_get_capabilities (all ASR) for a model whose + * transcribe_model_roles() lacks TRANSCRIBE_ROLE_ASR, and by each + * role's session init for a model lacking that role. Check the mask + * with transcribe_model_roles() first. + */ + TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20, } transcribe_status; /* @@ -1303,6 +1312,25 @@ struct transcribe_capabilities { TRANSCRIBE_API void transcribe_capabilities_init(struct transcribe_capabilities * out); +/* + * Roles: the kinds of work a loaded model can do. ASR is this header's + * transcribe_session_* / transcribe_run* API. The other roles each have + * their own header under include/transcribe/ and their own session type. + * A model may serve more than one role. Values are bits; append-only. + */ +typedef enum { + TRANSCRIBE_ROLE_ASR = 1u << 0, + TRANSCRIBE_ROLE_DIARIZE = 1u << 1, +} transcribe_role; + +/* + * Bitmask of transcribe_role values the loaded model serves. Fixed at load + * and never changes for the model's lifetime. Returns 0 for NULL. Entry + * points that need a role the model lacks return + * TRANSCRIBE_ERR_UNSUPPORTED_ROLE. + */ +TRANSCRIBE_API uint32_t transcribe_model_roles(const struct transcribe_model * model); + /* * Read model capabilities into caller-owned storage. The caller * initializes *out_caps via transcribe_capabilities_init() (zero-fill); @@ -1310,9 +1338,12 @@ TRANSCRIBE_API void transcribe_capabilities_init(struct transcribe_capabilities * beyond the caller's struct_size untouched. * * Returns: - * TRANSCRIBE_ERR_INVALID_ARG model or out_caps is NULL. - * TRANSCRIBE_ERR_BAD_STRUCT_SIZE out_caps->struct_size is 0 or - * smaller than the library's minimum. + * TRANSCRIBE_ERR_INVALID_ARG model or out_caps is NULL. + * TRANSCRIBE_ERR_BAD_STRUCT_SIZE out_caps->struct_size is 0 or + * smaller than the library's minimum. + * TRANSCRIBE_ERR_UNSUPPORTED_ROLE the model does not serve + * TRANSCRIBE_ROLE_ASR. These are ASR + * capabilities; *out_caps is untouched. * * Pointer fields written by the library (e.g. `languages`) point at * model-owned storage and remain valid until transcribe_model_free(). @@ -1495,6 +1526,10 @@ TRANSCRIBE_API void transcribe_model_free(struct transcribe_model * model); * * params may be NULL for library defaults, or initialize a struct with * transcribe_session_params_init(). See transcribe_model_load_file. + * + * Returns TRANSCRIBE_ERR_UNSUPPORTED_ROLE when the model does not serve + * TRANSCRIBE_ROLE_ASR (see transcribe_model_roles); transcribe_open + * returns the same and frees the model it loaded. */ TRANSCRIBE_API transcribe_status transcribe_session_init(struct transcribe_model * model, const struct transcribe_session_params * params, diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp index 3bb8928c7..d541a4277 100644 --- a/src/transcribe-asr.cpp +++ b/src/transcribe-asr.cpp @@ -816,6 +816,9 @@ static transcribe_status transcribe_session_init_impl(struct transcribe_model * if (model == nullptr) { return TRANSCRIBE_ERR_INVALID_ARG; } + if ((model->roles & transcribe::k_role_asr) == 0) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } // NULL params means "all defaults" (see transcribe_model_load_file). struct transcribe_session_params params_defaults; transcribe_session_params_init(¶ms_defaults); @@ -1770,6 +1773,11 @@ extern "C" transcribe_status transcribe_model_get_capabilities(const struct tran if (const auto st = check_struct_size(out_caps->struct_size, k_min_capabilities_size); st != TRANSCRIBE_OK) { return st; } + // These are ASR capabilities: a model without the ASR role gets an + // error, not a zeroed struct that reads as "supports nothing". + if ((model->roles & transcribe::k_role_asr) == 0) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } // Preserve the caller-declared size, then write only the prefix // that fits in both the caller's buffer and this library's view; // any tail bytes stay as the caller initialized them (zero, by diff --git a/src/transcribe-model.h b/src/transcribe-model.h index 5dcc93cb7..72849b48c 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -23,11 +23,9 @@ namespace transcribe { struct Arch; class Tokenizer; -// Role bits for transcribe_model::roles: the kinds of work a loaded model -// can do. Internal until the public transcribe_role enum lands; the values -// match it. -constexpr uint32_t k_role_asr = 1u << 0; -constexpr uint32_t k_role_diarize = 1u << 1; +// Role bits for transcribe_model::roles: the public transcribe_role values. +constexpr uint32_t k_role_asr = TRANSCRIBE_ROLE_ASR; +constexpr uint32_t k_role_diarize = TRANSCRIBE_ROLE_DIARIZE; // Transparent comparator keeps transcribe_model_meta_val_str allocation-free. // Loader and model must use the same map type for assignment. @@ -47,7 +45,7 @@ struct transcribe_model { // for per-call dispatch (init_context, run). const transcribe::Arch * arch = nullptr; - // Bitmask of transcribe::k_role_* this loaded model serves. Set by the + // Bitmask of transcribe_role this loaded model serves. Set by the // family's load() when it serves anything other than ASR alone; left 0, // it defaults to ASR for an arch with ASR hooks. Validated against the // arch at load (transcribe::resolve_roles). Immutable after load. diff --git a/src/transcribe.cpp b/src/transcribe.cpp index f89c22be7..9f2a1b195 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -112,6 +112,8 @@ extern "C" const char * transcribe_status_string(int status) { return "output truncated: decode hit the context/generation cap before end-of-stream"; case TRANSCRIBE_ERR_OUTPUT_REPETITION: return "output repetition: decode stopped when the output began repeating itself"; + case TRANSCRIBE_ERR_UNSUPPORTED_ROLE: + return "model does not serve the requested role"; default: return "unknown status"; } @@ -937,6 +939,10 @@ extern "C" const char * transcribe_model_meta_val_str(const struct transcribe_mo return it != model->meta.end() ? it->second.c_str() : ""; } +extern "C" uint32_t transcribe_model_roles(const struct transcribe_model * model) { + return model != nullptr ? model->roles : 0; +} + extern "C" const char * transcribe_model_backend(const struct transcribe_model * model) { // Empty string means "no runtime backend bound" — see the public header // for the full semantic. diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 0fb680a08..c39081364 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -77,6 +77,7 @@ static void test_status_string(void) { TRANSCRIBE_ERR_INPUT_TOO_LONG, TRANSCRIBE_ERR_OUTPUT_TRUNCATED, TRANSCRIBE_ERR_OUTPUT_REPETITION, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE, }; for (size_t i = 0; i < sizeof(all) / sizeof(all[0]); ++i) { const char * s = transcribe_status_string(all[i]); @@ -526,6 +527,7 @@ static void test_model_introspection_null(void) { /* The feature probe is also NULL-safe and returns false for every * known feature value plus any out-of-range enum. */ + CHECK(transcribe_model_roles(NULL) == 0); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_INITIAL_PROMPT) == false); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK) == false); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_LONG_FORM) == false); diff --git a/tests/role_resolve_unit.cpp b/tests/role_resolve_unit.cpp index cb49bda21..c7e39fc40 100644 --- a/tests/role_resolve_unit.cpp +++ b/tests/role_resolve_unit.cpp @@ -1,4 +1,4 @@ -// role_resolve_unit.cpp - load-time role validation (transcribe::resolve_roles). +// role_resolve_unit.cpp - load-time role validation (transcribe::resolve_roles) and the public role API (transcribe_model_roles, UNSUPPORTED_ROLE). #include "transcribe-arch.h" #include "transcribe-model.h" @@ -6,6 +6,7 @@ #include #include +#include namespace { @@ -85,6 +86,40 @@ void test_unknown_bit_rejected() { CHECK(resolve(k_asr_arch, transcribe::k_role_asr | (1u << 31), &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); } +// D5: ASR entry points refuse a model that does not serve ASR, before +// touching params or the arch; capabilities leave the caller's struct alone. +void test_asr_entry_points_check_role() { + transcribe_model model; + model.arch = &k_asr_arch; + model.roles = transcribe::k_role_diarize; // not ASR + + transcribe_session * s = reinterpret_cast(0x1); + CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(s == nullptr); + + transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + caps.max_audio_ms = 1234; // sentinel: must survive the rejection + CHECK(transcribe_model_get_capabilities(&model, &caps) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(caps.max_audio_ms == 1234); + + CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_DIARIZE); + + // Same model with the ASR bit: capabilities succeed; session_init gets + // past the role check to the fake arch's init_context. + model.roles = transcribe::k_role_asr; + CHECK(transcribe_model_get_capabilities(&model, &caps) == TRANSCRIBE_OK); + CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(s == nullptr); + CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_ASR); +} + +void test_role_bits_match_public_enum() { + static_assert(transcribe::k_role_asr == TRANSCRIBE_ROLE_ASR, "internal and public role bits must match"); + static_assert(transcribe::k_role_diarize == TRANSCRIBE_ROLE_DIARIZE, "internal and public role bits must match"); + CHECK(std::strcmp(transcribe_status_string(TRANSCRIBE_ERR_UNSUPPORTED_ROLE), "unknown status") != 0); +} + void test_null_model() { CHECK(transcribe::resolve_roles(nullptr) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); transcribe_model model; @@ -102,6 +137,8 @@ int main() { test_unbacked_role_rejected(); test_unknown_bit_rejected(); test_null_model(); + test_asr_entry_points_check_role(); + test_role_bits_match_public_enum(); if (g_failures != 0) { std::fprintf(stderr, "%d failure(s)\n", g_failures); From b560905420ed070b53e33c8f96fc5b33859fab5a Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:58:09 +0800 Subject: [PATCH 18/51] ts: map TRANSCRIBE_ERR_UNSUPPORTED_ROLE Status 20 now raises a new UnsupportedRole (direct TranscribeError subclass: the model loaded fine but lacks the role the call needs, so it is neither a request option nor a load failure). Session init and capabilities already route through check() -> exceptionForStatus, so they pick it up unchanged. test/errors.test.mjs (no-model tier) pins the full status -> class table against every TRANSCRIBE_ERR_* in _generated.ts, mirroring the Python test_errors.py, plus OK, unknown-status, and OutputRepetition cases. --- bindings/typescript/src/errors.ts | 10 +++ bindings/typescript/test/errors.test.mjs | 99 ++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 bindings/typescript/test/errors.test.mjs diff --git a/bindings/typescript/src/errors.ts b/bindings/typescript/src/errors.ts index be517cf8a..d8baef070 100644 --- a/bindings/typescript/src/errors.ts +++ b/bindings/typescript/src/errors.ts @@ -37,6 +37,15 @@ export class AbiError extends TranscribeError {} export class InputTooLong extends TranscribeError {} export class VersionMismatch extends TranscribeError {} +/** + * Raised when the model does not serve the role the call needs + * (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`): e.g. opening an ASR session on, or + * reading ASR capabilities of, a model whose roles lack ASR. The model loaded + * fine; it is the wrong kind of model for this call. Not an UnsupportedRequest, + * since no request option would make it succeed. + */ +export class UnsupportedRole extends TranscribeError {} + /** Raised when a run is cancelled; carries any partial transcript in `partialResult`. */ export class Aborted extends TranscribeError {} @@ -70,6 +79,7 @@ const STATUS_TO_EXC: Record TranscribeErr [g.TRANSCRIBE_ERR_INPUT_TOO_LONG]: InputTooLong, [g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED]: OutputTruncated, [g.TRANSCRIBE_ERR_OUTPUT_REPETITION]: OutputRepetition, + [g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE]: UnsupportedRole, }; /** Build (do not throw) the mapped exception for a status. */ diff --git a/bindings/typescript/test/errors.test.mjs b/bindings/typescript/test/errors.test.mjs new file mode 100644 index 000000000..1ed485c8b --- /dev/null +++ b/bindings/typescript/test/errors.test.mjs @@ -0,0 +1,99 @@ +// No-model tier: status -> error-class mapping, pure (no native call, no GGUF). +// Mirrors bindings/python/tests/test_errors.py. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as g from "../dist/_generated.js"; +import { + exceptionForStatus, + raiseForStatus, + TranscribeError, + InvalidArgument, + NotImplementedByModel, + ModelFileNotFound, + ModelLoadError, + OutOfMemory, + BackendError, + UnsupportedRequest, + UnsupportedRole, + Aborted, + AbiError, + InputTooLong, + OutputTruncated, + OutputRepetition, +} from "../dist/index.js"; + +const EXPECTED = { + TRANSCRIBE_ERR_INVALID_ARG: InvalidArgument, + TRANSCRIBE_ERR_NOT_IMPLEMENTED: NotImplementedByModel, + TRANSCRIBE_ERR_FILE_NOT_FOUND: ModelFileNotFound, + TRANSCRIBE_ERR_GGUF: ModelLoadError, + TRANSCRIBE_ERR_UNSUPPORTED_ARCH: ModelLoadError, + TRANSCRIBE_ERR_UNSUPPORTED_VARIANT: ModelLoadError, + TRANSCRIBE_ERR_OOM: OutOfMemory, + TRANSCRIBE_ERR_BACKEND: BackendError, + TRANSCRIBE_ERR_SAMPLE_RATE: InvalidArgument, + TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_TASK: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS: UnsupportedRequest, + TRANSCRIBE_ERR_ABORTED: Aborted, + TRANSCRIBE_ERR_BAD_STRUCT_SIZE: AbiError, + TRANSCRIBE_ERR_UNSUPPORTED_PNC: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_ITN: UnsupportedRequest, + TRANSCRIBE_ERR_INPUT_TOO_LONG: InputTooLong, + TRANSCRIBE_ERR_OUTPUT_TRUNCATED: OutputTruncated, + TRANSCRIBE_ERR_OUTPUT_REPETITION: OutputRepetition, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE: UnsupportedRole, +}; + +test("every transcribe_status maps to its documented error class", () => { + // The expectation covers every non-OK status the header defines, and nothing + // else: a new C status must be mapped (and listed here) deliberately. + const generated = Object.keys(g) + .filter((k) => k.startsWith("TRANSCRIBE_ERR_")) + .sort(); + assert.deepEqual(generated, Object.keys(EXPECTED).sort()); + + for (const [name, Cls] of Object.entries(EXPECTED)) { + const status = g[name]; + assert.throws( + () => raiseForStatus(status, "synthetic", "ctx"), + (e) => { + assert.equal(e.constructor, Cls, name); + assert.ok(e instanceof TranscribeError); + assert.equal(e.status, status); + assert.equal(e.name, Cls.name); + assert.match(e.message, /ctx: synthetic/); + return true; + }, + ); + } +}); + +test("UNSUPPORTED_ROLE maps to UnsupportedRole, not a request or load error", () => { + const e = exceptionForStatus(g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "wrong role", "opening session"); + assert.equal(g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE, 20); + assert.ok(e instanceof UnsupportedRole); + assert.ok(e instanceof TranscribeError); + assert.equal(e.status, 20); + assert.equal(e.name, "UnsupportedRole"); + assert.equal(e instanceof UnsupportedRequest, false); + assert.equal(e instanceof ModelLoadError, false); + assert.equal(e instanceof NotImplementedByModel, false); +}); + +test("OK does not throw", () => { + assert.equal(raiseForStatus(g.TRANSCRIBE_OK, "ok"), undefined); +}); + +test("unknown status degrades to the base class", () => { + const e = exceptionForStatus(999, "mystery"); + assert.equal(e.constructor, TranscribeError); + assert.equal(e.status, 999); +}); + +test("OutputRepetition is an OutputTruncated", () => { + const e = exceptionForStatus(g.TRANSCRIBE_ERR_OUTPUT_REPETITION, "looped", "run"); + assert.ok(e instanceof OutputRepetition); + assert.ok(e instanceof OutputTruncated); + assert.equal(e.partialResult, undefined); +}); From abbf9fc78853da5993a10ada4ad9d941609783d7 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:59:00 +0800 Subject: [PATCH 19/51] swift: map TRANSCRIBE_ERR_UNSUPPORTED_ROLE, pin header hash --- .../swift/Sources/TranscribeCpp/ABIHash.swift | 2 +- .../TranscribeCpp/TranscribeError.swift | 7 +++ .../TranscribeCppTests/NoModelTests.swift | 63 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index f55cc4c77..3749df4cd 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "59b9a92b47074666" + public static let pinnedHeaderHash = "1537ea60b442dd53" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift index ce2c8b032..4217c4e7c 100644 --- a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift +++ b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift @@ -28,6 +28,11 @@ public enum TranscribeError: Error { /// The decode was stopped because the output began repeating itself; the /// repeats are dropped from `partial`, which is incomplete. case outputRepetition(message: String, partial: Transcript?) + /// The model does not serve the role the call needs + /// (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`); e.g. creating an ASR session on a + /// model whose roles lack ASR. A model-level refusal, distinct from + /// `.unsupported` (a per-run option the model can't honor). + case unsupportedRole(String) case versionMismatch(String) case busy(String) case other(status: Int32, message: String) @@ -69,6 +74,8 @@ public enum TranscribeError: Error { return .outputTruncated(message: message, partial: nil) case TRANSCRIBE_ERR_OUTPUT_REPETITION: return .outputRepetition(message: message, partial: nil) + case TRANSCRIBE_ERR_UNSUPPORTED_ROLE: + return .unsupportedRole(message) default: return .other(status: raw, message: message) } diff --git a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift index 61fc38e15..b9370dfeb 100644 --- a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift @@ -1,5 +1,6 @@ import Foundation import XCTest +import CTranscribe @testable import TranscribeCpp @@ -48,6 +49,68 @@ final class NoModelTests: XCTestCase { XCTAssertNil(TranscribeError.inputTooLong("").partial) } + // No loadable model lacks the ASR role yet, so the mapping is checked + // directly rather than through a native call. + func testUnsupportedRoleMapsToItsOwnCase() { + XCTAssertNotEqual(Transcribe.statusString(20), "unknown status") + let error = TranscribeError.make(TRANSCRIBE_ERR_UNSUPPORTED_ROLE, context: "creating session") + guard case .unsupportedRole(let message) = error else { + return XCTFail("expected .unsupportedRole, got \(error)") + } + XCTAssertEqual(message, "creating session: \(Transcribe.statusString(20))") + XCTAssertNil(error.partial) + XCTAssertFalse(error.isTruncated) + XCTAssertThrowsError(try TranscribeError.check(TRANSCRIBE_ERR_UNSUPPORTED_ROLE)) { thrown in + guard case TranscribeError.unsupportedRole = thrown else { + return XCTFail("expected .unsupportedRole, got \(thrown)") + } + } + } + + // Every native status maps to its case. `label` has no `default`, so a new + // TranscribeError case does not compile until this table is updated; the + // "unknown status" check below flags a newly appended C status. + func testEveryStatusMapsToItsCase() { + let expected: [Int32: String] = [ + 1: "invalidArgument", 2: "notImplemented", 3: "modelFileNotFound", + 4: "modelLoad", 5: "modelLoad", 6: "modelLoad", 7: "outOfMemory", + 8: "backend", + 9: "other", // SAMPLE_RATE: reserved, never returned; not mapped + 10: "unsupported", 11: "unsupported", 12: "unsupported", + 13: "aborted", 14: "badStructSize", 15: "unsupported", + 16: "unsupported", 17: "inputTooLong", 18: "outputTruncated", + 19: "outputRepetition", 20: "unsupportedRole", + ] + for raw in 1...20 { + let status = transcribe_status(rawValue: UInt32(raw)) + XCTAssertNotEqual(Transcribe.statusString(Int32(raw)), "unknown status", "status \(raw)") + XCTAssertEqual(label(TranscribeError.make(status)), expected[Int32(raw)], "status \(raw)") + } + XCTAssertEqual(Transcribe.statusString(21), "unknown status", + "a new status was appended; map it in TranscribeError.make") + } + + private func label(_ error: TranscribeError) -> String { + switch error { + case .invalidArgument: return "invalidArgument" + case .notImplemented: return "notImplemented" + case .modelFileNotFound: return "modelFileNotFound" + case .modelLoad: return "modelLoad" + case .outOfMemory: return "outOfMemory" + case .backend: return "backend" + case .unsupported: return "unsupported" + case .badStructSize: return "badStructSize" + case .inputTooLong: return "inputTooLong" + case .aborted: return "aborted" + case .outputTruncated: return "outputTruncated" + case .outputRepetition: return "outputRepetition" + case .unsupportedRole: return "unsupportedRole" + case .versionMismatch: return "versionMismatch" + case .busy: return "busy" + case .other: return "other" + } + } + func testAtLeastOneDevice() { XCTAssertGreaterThanOrEqual(Transcribe.devices().count, 1) } From 3cbdb7f44db8226cd4f90c32c3710143ddba4722 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:59:26 +0800 Subject: [PATCH 20/51] python: map TRANSCRIBE_ERR_UNSUPPORTED_ROLE --- .../python/src/transcribe_cpp/__init__.py | 2 + bindings/python/src/transcribe_cpp/errors.py | 9 +++ bindings/python/tests/test_abi.py | 4 +- bindings/python/tests/test_errors.py | 61 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index db05dc4ad..dcf66a32c 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -49,6 +49,7 @@ OutputTruncated, TranscribeError, UnsupportedRequest, + UnsupportedRole, exception_for_status, raise_for_status, ) @@ -112,6 +113,7 @@ "OutOfMemory", "BackendError", "UnsupportedRequest", + "UnsupportedRole", "AbiError", "Aborted", "Busy", diff --git a/bindings/python/src/transcribe_cpp/errors.py b/bindings/python/src/transcribe_cpp/errors.py index 88828aa85..3dfcac54d 100644 --- a/bindings/python/src/transcribe_cpp/errors.py +++ b/bindings/python/src/transcribe_cpp/errors.py @@ -29,6 +29,7 @@ TRANSCRIBE_ERR_UNSUPPORTED_ITN as ERR_UNSUPPORTED_ITN, TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE as ERR_UNSUPPORTED_LANGUAGE, TRANSCRIBE_ERR_UNSUPPORTED_PNC as ERR_UNSUPPORTED_PNC, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE as ERR_UNSUPPORTED_ROLE, TRANSCRIBE_ERR_UNSUPPORTED_TASK as ERR_UNSUPPORTED_TASK, TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS as ERR_UNSUPPORTED_TIMESTAMPS, TRANSCRIBE_ERR_UNSUPPORTED_VARIANT as ERR_UNSUPPORTED_VARIANT, @@ -83,6 +84,13 @@ class UnsupportedRequest(TranscribeError): """Task / language / timestamp granularity the model does not support.""" +class UnsupportedRole(TranscribeError): + """The model does not serve the role the call needs, e.g. opening a + transcription session on, or reading the (ASR) capabilities of, a model + without the ASR role. A property of the loaded model, not of the request + options, so retrying with different run options cannot succeed.""" + + class AbiError(TranscribeError): """Caller-owned struct layout did not match the library (struct_size).""" @@ -158,6 +166,7 @@ class OutputRepetition(OutputTruncated): ERR_INPUT_TOO_LONG: InputTooLong, ERR_OUTPUT_TRUNCATED: OutputTruncated, ERR_OUTPUT_REPETITION: OutputRepetition, + ERR_UNSUPPORTED_ROLE: UnsupportedRole, } diff --git a/bindings/python/tests/test_abi.py b/bindings/python/tests/test_abi.py index 3ec807c16..894dddd7e 100644 --- a/bindings/python/tests/test_abi.py +++ b/bindings/python/tests/test_abi.py @@ -66,9 +66,9 @@ def test_base_version_strips_suffixes(version, base): def test_status_constants_discovered(): - # transcribe_status has OK + 18 error codes as of this writing; never let + # transcribe_status has OK + 20 error codes as of this writing; never let # this collapse to a near-empty set that would make the checks vacuous. - assert len(_STATUS_NAMES) >= 19 + assert len(_STATUS_NAMES) >= 21 def test_every_generated_status_is_aliased_and_mapped(): diff --git a/bindings/python/tests/test_errors.py b/bindings/python/tests/test_errors.py index 3aaee5d2b..04e63b3c4 100644 --- a/bindings/python/tests/test_errors.py +++ b/bindings/python/tests/test_errors.py @@ -42,6 +42,7 @@ def test_every_status_maps_to_documented_subclass(): errors.ERR_INPUT_TOO_LONG: t.InputTooLong, errors.ERR_OUTPUT_TRUNCATED: t.OutputTruncated, errors.ERR_OUTPUT_REPETITION: t.OutputRepetition, + errors.ERR_UNSUPPORTED_ROLE: t.UnsupportedRole, } # The mapping table covers every non-OK status the header defines, and # nothing else (a new C status must be mapped deliberately, not by @@ -73,6 +74,17 @@ def test_output_repetition_is_an_output_truncated(): assert exc.partial_result is None +def test_unsupported_role_is_its_own_type(): + # A role mismatch is a property of the loaded model, not of a run option: + # it must not be caught by an UnsupportedRequest handler. + exc = errors.exception_for_status(errors.ERR_UNSUPPORTED_ROLE, "no role") + assert type(exc) is t.UnsupportedRole + assert not isinstance(exc, t.UnsupportedRequest) + assert exc.status == errors.ERR_UNSUPPORTED_ROLE == 20 + assert "UnsupportedRole" in t.__all__ + assert t.UnsupportedRole is errors.UnsupportedRole + + def test_exception_for_status_builds_without_raising(): exc = errors.exception_for_status(errors.ERR_ABORTED, "aborted", "run") assert isinstance(exc, t.Aborted) @@ -88,6 +100,7 @@ def test_status_constants_come_from_generated_layer(): assert errors.ERR_ABORTED == _generated.TRANSCRIBE_ERR_ABORTED assert errors.ERR_OUTPUT_TRUNCATED == _generated.TRANSCRIBE_ERR_OUTPUT_TRUNCATED assert errors.OK == _generated.TRANSCRIBE_OK + assert errors.ERR_UNSUPPORTED_ROLE == _generated.TRANSCRIBE_ERR_UNSUPPORTED_ROLE # --- integration: real native calls raise the mapped classes ---------------- @@ -106,6 +119,54 @@ def test_junk_model_file_raises_model_load_error(tmp_path): t.Model(junk) +def _fake_model(monkeypatch): + """A Model around a fake handle (no GGUF). transcribe_model_free is + patched to a no-op; the caller must close() it while that patch is still + active so the fake handle never reaches the real library.""" + import ctypes + import weakref + + monkeypatch.setattr(t._lib, "transcribe_model_free", lambda h: None) + m = t.Model.__new__(t.Model) + m._sessions = weakref.WeakSet() + m._init_compute_state() + m._handle = ctypes.c_void_p(0x1000) + return m + + +def test_capabilities_on_model_without_asr_role_raises_unsupported_role( + monkeypatch, +): + # transcribe_model_get_capabilities returns UNSUPPORTED_ROLE for a model + # without the ASR role; Model.capabilities surfaces it as the typed error. + monkeypatch.setattr(t._lib, "transcribe_model_get_capabilities", + lambda h, caps: errors.ERR_UNSUPPORTED_ROLE) + m = _fake_model(monkeypatch) + try: + with pytest.raises(t.UnsupportedRole) as ei: + m.capabilities + assert ei.value.status == errors.ERR_UNSUPPORTED_ROLE + assert "reading capabilities" in str(ei.value) + finally: + m.close() + + +def test_session_on_model_without_asr_role_raises_unsupported_role(monkeypatch): + # transcribe_session_init returns UNSUPPORTED_ROLE (no handle) for a model + # without the ASR role; opening a session surfaces the typed error. + monkeypatch.setattr(t._lib, "transcribe_session_init", + lambda m, params, out: errors.ERR_UNSUPPORTED_ROLE) + m = _fake_model(monkeypatch) + try: + with pytest.raises(t.UnsupportedRole) as ei: + m.session() + assert ei.value.status == errors.ERR_UNSUPPORTED_ROLE + assert "opening session" in str(ei.value) + assert not list(m._sessions) + finally: + m.close() + + def test_invalid_spec_k_drafts_rejected_before_native_call(): from transcribe_cpp import _build_run_params From d81e04ce16e4a3fc3006fad73b22f48ce5dedff0 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 21:58:26 +0800 Subject: [PATCH 21/51] rust: map TRANSCRIBE_ERR_UNSUPPORTED_ROLE Status 20 now maps to its own Error::UnsupportedRole variant ("unsupported role: ...") instead of falling through to Error::Other, and raw_status() reports it back as 20. Kept separate from Error::Unsupported, which is the per-request shape group (task / language / timestamps / PNC / ITN). tests: error.rs unit tests walk every status the linked library names (until transcribe_status_string returns its unknown text) and check that each one maps to a typed variant whose raw_status maps back to the same variant, plus a test that pins the UnsupportedRole message and code. --- bindings/rust/transcribe-cpp/src/error.rs | 76 +++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/bindings/rust/transcribe-cpp/src/error.rs b/bindings/rust/transcribe-cpp/src/error.rs index f874dbb1a..10f0a838b 100644 --- a/bindings/rust/transcribe-cpp/src/error.rs +++ b/bindings/rust/transcribe-cpp/src/error.rs @@ -75,6 +75,14 @@ pub enum Error { /// The (incomplete) transcript produced before the loop, one copy kept. partial: Option>, }, + /// `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` — the model does not serve the role + /// the call needs. Sessions are ASR sessions, so [`Model::session`] / + /// [`Model::session_with`] on a model without the ASR role fail with it. + /// + /// [`Model::session`]: crate::Model::session + /// [`Model::session_with`]: crate::Model::session_with + #[error("unsupported role: {0}")] + UnsupportedRole(String), /// The loaded library's base version disagrees with the headers this crate /// was generated against (the pre-1.0 version lock). Raised on first use. #[error("native library version mismatch: {0}")] @@ -113,6 +121,7 @@ impl Error { Error::Aborted { .. } => S::TRANSCRIBE_ERR_ABORTED, Error::OutputTruncated { .. } => S::TRANSCRIBE_ERR_OUTPUT_TRUNCATED, Error::OutputRepetition { .. } => S::TRANSCRIBE_ERR_OUTPUT_REPETITION, + Error::UnsupportedRole(_) => S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, _ => S::TRANSCRIBE_OK, }; s.0 as i32 @@ -186,6 +195,7 @@ pub(crate) fn error_for_status(status: sys::transcribe_status, context: &str) -> message: msg, partial: None, }, + S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE => Error::UnsupportedRole(msg), _ => Error::Other(msg), } } @@ -198,3 +208,69 @@ pub(crate) fn check(status: sys::transcribe_status, context: &str) -> Result<()> Err(error_for_status(status, context)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::discriminant; + use sys::transcribe_status as S; + + /// Every non-OK status the linked library knows, found by walking codes + /// upward until `transcribe_status_string` falls back to its "unknown" + /// text. A status appended to the header without a Rust mapping shows up + /// here and fails the test below. + fn known_error_statuses() -> Vec { + let unknown = status_string(-1); + let codes: Vec = (1..256u32) + .take_while(|&c| status_string(c as i32) != unknown) + .map(S) + .collect(); + assert!( + codes.len() >= S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE.0 as usize, + "status walk stopped early at {} codes", + codes.len() + ); + codes + } + + #[test] + fn every_status_maps_to_a_typed_variant() { + for status in known_error_statuses() { + let code = status.0; + let err = error_for_status(status, "ctx"); + assert!( + !matches!(err, Error::Other(_)), + "status {code} falls through to Error::Other: {err:?}" + ); + // raw_status must name a status that maps back to the same + // variant (grouped codes like SAMPLE_RATE report their group's + // primary status, never 0). + let raw = err.raw_status(); + assert_ne!(raw, 0, "status {code} reports raw_status 0: {err:?}"); + let back = error_for_status(S(raw as u32), "ctx"); + assert_eq!( + discriminant(&back), + discriminant(&err), + "status {code} -> {err:?} -> raw {raw} -> {back:?}" + ); + } + } + + #[test] + fn unsupported_role_is_its_own_variant() { + let err = error_for_status(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "session_init"); + let Error::UnsupportedRole(msg) = &err else { + panic!("expected UnsupportedRole, got {err:?}"); + }; + let c_text = status_string(20); + assert_eq!(*msg, format!("session_init: {c_text} (status 20)")); + assert_eq!(err.to_string(), format!("unsupported role: {msg}")); + assert_eq!(err.raw_status(), 20); + assert!(err.partial().is_none()); + let checked = check(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "open"); + assert!( + matches!(checked, Err(Error::UnsupportedRole(_))), + "got {checked:?}" + ); + } +} From 0ad04b8503ce5e1815f3482e21ad8436e3f6ea4f Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:09:08 +0800 Subject: [PATCH 22/51] roles: DIARIZE role (run only); Sortformer serves it alongside ASR Additive. include/transcribe/diarize.h: transcribe_diarize_session with get_info / session_init / session_free / set_abort_callback / run / n_segments / get_segment / get_timings, plus info / session_params / params structs, their inits and ABI ids. Streaming is deferred to #175, the first streaming diarizer (plan Q16). TRANSCRIBE_EXT_SLOT_DIARIZE_RUN is appended; diarize.h joins the extensions.h umbrella. src/transcribe-diarize.{h,cpp}: the dispatcher follows the ASR rules (api_guard on compute / ownership entry points, *out NULL on failure, UNSUPPORTED_ROLE for a model without DIARIZE, malformed input / bad extension / family run_validate rejected before the previous result is cleared, non-finite PCM rejected, scratch released after every run). Families only compute: DiarizeOps::run returns frame probabilities and the dispatcher turns them into segments with transcribe::probs_to_segments, the threshold-0.5 per-speaker run builder that used to be sortformer::probs_to_speaker_segments (#175's builder is the same algorithm). The multitalker parakeet bundle now calls the shared one. Sortformer: roles = ASR | DIARIZE. Both session types share one compute path (diarize_pcm over SortformerWork); the DIARIZE run takes the new SFDR kind (transcribe_sortformer_diarize_ext) on the DIARIZE_RUN slot. The ASR path stays until the break commit. tests: diarize_dispatch_unit (fake arch: roles, info, pre-clear rejections, segments, accessors); sortformer_diarize_unit (real model: diarize_run segments byte-identical to the ASR-path transcribe_run at DEFAULT and LOW_LATENCY, bad preset preserves the result, and with a whisper GGUF an ASR session and a diarize session on two threads match their sequential results). ctest green; transcribe-cli CPU parity vs main identical (multitalker --diarize included). Low-level bindings are regenerated with the binding wrappers in a later commit. --- docs/extension-kinds.md | 3 + include/transcribe.h | 40 +++-- include/transcribe/diarize.h | 99 +++++++++++ include/transcribe/extensions.h | 1 + include/transcribe/sortformer.h | 13 ++ src/CMakeLists.txt | 1 + src/arch/parakeet/multitalker.cpp | 2 +- src/arch/sortformer/model.cpp | 195 +++++++++++++-------- src/arch/sortformer/sortformer.h | 26 ++- src/transcribe-diarize.cpp | 278 ++++++++++++++++++++++++++++++ src/transcribe-diarize.h | 53 ++++++ src/transcribe.cpp | 13 ++ tests/CMakeLists.txt | 32 ++++ tests/diarize_dispatch_unit.cpp | 226 ++++++++++++++++++++++++ tests/sortformer_diarize_unit.cpp | 200 +++++++++++++++++++++ 15 files changed, 1077 insertions(+), 105 deletions(-) create mode 100644 include/transcribe/diarize.h create mode 100644 src/transcribe-diarize.cpp create mode 100644 src/transcribe-diarize.h create mode 100644 tests/diarize_dispatch_unit.cpp create mode 100644 tests/sortformer_diarize_unit.cpp diff --git a/docs/extension-kinds.md b/docs/extension-kinds.md index cb8dd94da..416f5e27e 100644 --- a/docs/extension-kinds.md +++ b/docs/extension-kinds.md @@ -32,6 +32,8 @@ entry-point family whose `family` field the typed extension is pointed at: by `transcribe_run`. - `TRANSCRIBE_EXT_SLOT_STREAM` — `transcribe_stream_params::family`, validated by `transcribe_stream_begin`. +- `TRANSCRIBE_EXT_SLOT_DIARIZE_RUN` — `transcribe_diarize_params::family`, + validated by `transcribe_diarize_run`. Mismatched-slot extensions are rejected with `TRANSCRIBE_ERR_INVALID_ARG` at the entry point. A future slot (e.g. `_SESSION`, `_MODEL_LOAD`, @@ -48,6 +50,7 @@ parameter; existing kinds keep their slot for life. | `0x4E524857` | `WHRN` | `RUN` | `TRANSCRIBE_EXT_KIND_WHISPER_RUN` | whisper | `include/transcribe/whisper.h` | | `0x54535256` | `VRST` | `STREAM` | `TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM` | voxtral_realtime | `include/transcribe/voxtral_realtime.h` | | `0x54534653` | `SFST` | `RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM` | sortformer | `include/transcribe/sortformer.h` | +| `0x52444653` | `SFDR` | `DIARIZE_RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE` | sortformer | `include/transcribe/sortformer.h` | Empty rows reserved for future allocations: diff --git a/include/transcribe.h b/include/transcribe.h index e8fd68896..0988db64b 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -378,21 +378,25 @@ TRANSCRIBE_API const char * transcribe_version_commit(void); * is append-only; do not renumber existing values. */ typedef enum { - TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0, - TRANSCRIBE_ABI_SESSION_PARAMS = 1, - TRANSCRIBE_ABI_RUN_PARAMS = 2, - TRANSCRIBE_ABI_STREAM_PARAMS = 3, - TRANSCRIBE_ABI_CAPABILITIES = 4, - TRANSCRIBE_ABI_TIMINGS = 5, - TRANSCRIBE_ABI_SEGMENT = 6, - TRANSCRIBE_ABI_WORD = 7, - TRANSCRIBE_ABI_TOKEN = 8, - TRANSCRIBE_ABI_STREAM_UPDATE = 9, - TRANSCRIBE_ABI_STREAM_TEXT = 10, - TRANSCRIBE_ABI_SESSION_LIMITS = 11, - TRANSCRIBE_ABI_EXT = 12, - TRANSCRIBE_ABI_DEVICE_INFO = 13, - TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14, + TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0, + TRANSCRIBE_ABI_SESSION_PARAMS = 1, + TRANSCRIBE_ABI_RUN_PARAMS = 2, + TRANSCRIBE_ABI_STREAM_PARAMS = 3, + TRANSCRIBE_ABI_CAPABILITIES = 4, + TRANSCRIBE_ABI_TIMINGS = 5, + TRANSCRIBE_ABI_SEGMENT = 6, + TRANSCRIBE_ABI_WORD = 7, + TRANSCRIBE_ABI_TOKEN = 8, + TRANSCRIBE_ABI_STREAM_UPDATE = 9, + TRANSCRIBE_ABI_STREAM_TEXT = 10, + TRANSCRIBE_ABI_SESSION_LIMITS = 11, + TRANSCRIBE_ABI_EXT = 12, + TRANSCRIBE_ABI_DEVICE_INFO = 13, + TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14, + /* include/transcribe/diarize.h */ + TRANSCRIBE_ABI_DIARIZE_INFO = 15, + TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 16, + TRANSCRIBE_ABI_DIARIZE_PARAMS = 17, } transcribe_abi_struct; /* sizeof / alignof of the selected public struct, or 0 for an unknown id. @@ -676,9 +680,11 @@ TRANSCRIBE_API transcribe_status transcribe_ext_check(const struct transcribe_ex */ typedef enum { /* transcribe_run_params::family */ - TRANSCRIBE_EXT_SLOT_RUN = 0, + TRANSCRIBE_EXT_SLOT_RUN = 0, /* transcribe_stream_params::family */ - TRANSCRIBE_EXT_SLOT_STREAM = 1, + TRANSCRIBE_EXT_SLOT_STREAM = 1, + /* transcribe_diarize_params::family (include/transcribe/diarize.h) */ + TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2, } transcribe_ext_slot; /* diff --git a/include/transcribe/diarize.h b/include/transcribe/diarize.h new file mode 100644 index 000000000..e00a864a4 --- /dev/null +++ b/include/transcribe/diarize.h @@ -0,0 +1,99 @@ +/* + * include/transcribe/diarize.h - DIARIZE role: who spoke when. + * + * Includes transcribe.h; safe to include in C or C++ TUs. For models whose + * product is speaker turns rather than text (transcribe_model_roles() has + * TRANSCRIBE_ROLE_DIARIZE). ASR models that attribute speakers inside a + * transcript keep using transcribe_run_params::diarize instead. + * + * Shape: load a model with transcribe_model_load_file, open a + * transcribe_diarize_session on it, run, then read the speaker segments. + * Sessions follow the transcribe_session threading contract in + * transcribe.h: one thread at a time per session, the model outlives its + * sessions, and at most one compute in flight per model across every + * session of every role. + */ + +#ifndef TRANSCRIBE_DIARIZE_H +#define TRANSCRIBE_DIARIZE_H + +#include "transcribe.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct transcribe_diarize_session; + +/* Static facts about a diarization model. */ +struct transcribe_diarize_info { + uint64_t struct_size; + int32_t sample_rate; /* input PCM rate (16000) */ + int32_t max_speakers; /* speaker_id is in [1, max_speakers] */ +}; + +struct transcribe_diarize_session_params { + uint64_t struct_size; + int32_t n_threads; /* 0 = library default */ +}; + +struct transcribe_diarize_params { + uint64_t struct_size; + /* Typed family extension on TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, or NULL. */ + const struct transcribe_ext * family; +}; + +TRANSCRIBE_API void transcribe_diarize_info_init(struct transcribe_diarize_info * out); +TRANSCRIBE_API void transcribe_diarize_session_params_init(struct transcribe_diarize_session_params * params); +TRANSCRIBE_API void transcribe_diarize_params_init(struct transcribe_diarize_params * params); + +/* UNSUPPORTED_ROLE when the model does not serve TRANSCRIBE_ROLE_DIARIZE. */ +TRANSCRIBE_API transcribe_status transcribe_diarize_get_info(const struct transcribe_model * model, + struct transcribe_diarize_info * out); + +/* params may be NULL for defaults. UNSUPPORTED_ROLE when the model does not + * serve TRANSCRIBE_ROLE_DIARIZE. On failure *out_session is NULL. */ +TRANSCRIBE_API transcribe_status +transcribe_diarize_session_init(struct transcribe_model * model, + const struct transcribe_diarize_session_params * params, + struct transcribe_diarize_session ** out_session); + +/* NULL is a no-op. */ +TRANSCRIBE_API void transcribe_diarize_session_free(struct transcribe_diarize_session * session); + +/* Polled during a run; returning true stops it with TRANSCRIBE_ERR_ABORTED. */ +TRANSCRIBE_API void transcribe_diarize_set_abort_callback(struct transcribe_diarize_session * session, + transcribe_abort_callback cb, + void * user_data); + +/* + * Diarize one recording: 16 kHz mono float32 PCM, every sample finite. + * params may be NULL for defaults. Replaces the previous result. Malformed + * input (NULL pointers, n_samples <= 0, NaN / Inf, a bad extension) returns + * an error before the previous result is touched. + */ +TRANSCRIBE_API transcribe_status transcribe_diarize_run(struct transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const struct transcribe_diarize_params * params); + +/* + * Speaker segments of the last successful run, grouped by speaker and + * time-ordered within a speaker; segments of different speakers may + * overlap. 0 before any run. An out-of-range index returns OK with a + * zeroed row. + */ +TRANSCRIBE_API int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session); +TRANSCRIBE_API transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session, + int i, + struct transcribe_speaker_segment * out); + +/* load_ms plus the last run's mel / encode time; decode_ms is 0. */ +TRANSCRIBE_API transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, + struct transcribe_timings * out); + +#ifdef __cplusplus +} +#endif + +#endif /* TRANSCRIBE_DIARIZE_H */ diff --git a/include/transcribe/extensions.h b/include/transcribe/extensions.h index b93bf1460..16c5684ad 100644 --- a/include/transcribe/extensions.h +++ b/include/transcribe/extensions.h @@ -14,6 +14,7 @@ #define TRANSCRIBE_EXTENSIONS_H #include "transcribe.h" +#include "transcribe/diarize.h" #include "transcribe/moonshine_streaming.h" #include "transcribe/parakeet.h" #include "transcribe/sortformer.h" diff --git a/include/transcribe/sortformer.h b/include/transcribe/sortformer.h index 751b11c88..c1208cbda 100644 --- a/include/transcribe/sortformer.h +++ b/include/transcribe/sortformer.h @@ -74,6 +74,19 @@ struct transcribe_sortformer_stream_ext { /* Fills ext.size/kind and preset = DEFAULT (GGUF-shipped cfg). */ TRANSCRIBE_API void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * ext); +/* 'SFDR' little-endian = 0x52444653 (DIARIZE_RUN slot) */ +#define TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE 0x52444653u + +/* transcribe_diarize_params::family: the operating point for + * transcribe_diarize_run. */ +struct transcribe_sortformer_diarize_ext { + struct transcribe_ext ext; + transcribe_sortformer_preset preset; +}; + +/* Fills ext.size/kind and preset = DEFAULT. */ +TRANSCRIBE_API void transcribe_sortformer_diarize_ext_init(struct transcribe_sortformer_diarize_ext * ext); + #ifdef __cplusplus } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 532ad8441..5758f3dd6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,7 @@ add_library(transcribe transcribe.cpp transcribe-asr.cpp + transcribe-diarize.cpp transcribe-loader.cpp transcribe-arch.cpp transcribe-meta.cpp diff --git a/src/arch/parakeet/multitalker.cpp b/src/arch/parakeet/multitalker.cpp index 5b9a29d12..0c5b18122 100644 --- a/src/arch/parakeet/multitalker.cpp +++ b/src/arch/parakeet/multitalker.cpp @@ -651,7 +651,7 @@ transcribe_status run_multitalker(ParakeetSession * pc, // Transcript-independent "who spoke when" rows from the diarizer preds // (same emission as the standalone sortformer family). - sf::probs_to_speaker_segments(pc, probs, T_diar, n_spk, diar_ms_per_frame, /*threshold=*/0.5f); + transcribe::probs_to_segments(probs.data(), T_diar, n_spk, diar_ms_per_frame, pc->speaker_segments); // ---- Elide to the caller's requested granularity (same convention // as decode_and_populate). ---- diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index d1c229a9d..3cf8f8fa6 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -624,6 +624,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } + m->roles = k_role_asr | k_role_diarize; m->t_load_us = ggml_time_us() - t_load_start; *out_model = m.release(); return TRANSCRIBE_OK; @@ -670,39 +671,9 @@ transcribe_status init_context(transcribe_model * model, return TRANSCRIBE_OK; } -// Threshold-based probs -> speaker segments (simple offline segmentation; -// the DER-grade dihard3 post-processing is a later, streaming-checkpoint -// concern). probs is row-major [T, n_spk]. Emits one contiguous run per -// speaker as a speaker_segment row. Non-static: the parakeet multitalker -// bundle path emits its transcript-independent view through this too. -void probs_to_speaker_segments(transcribe_session * pc, - const std::vector & probs, - int T, - int n_spk, - double ms_per_frame, - float threshold) { - for (int s = 0; s < n_spk; ++s) { - int run_start = -1; - for (int t = 0; t <= T; ++t) { - const bool active = (t < T) && (probs[static_cast(t) * n_spk + s] > threshold); - if (active && run_start < 0) { - run_start = t; - } else if (!active && run_start >= 0) { - transcribe_session::SpeakerSegmentEntry row; - row.t0_ms = static_cast(std::llround(run_start * ms_per_frame)); - row.t1_ms = static_cast(std::llround(t * ms_per_frame)); - row.speaker_id = s + 1; // 1-based - row.p = std::numeric_limits::quiet_NaN(); - pc->speaker_segments.push_back(row); - run_start = -1; - } - } - } -} - // Lazily create the persistent multi-backend scheduler shared by the offline // and streaming graphs. -static transcribe_status ensure_sched(SortformerSession * pc, SortformerModel * pm) { +template static transcribe_status ensure_sched(Session * pc, SortformerModel * pm) { if (pc->sched == nullptr) { pc->sched = ggml_backend_sched_new(pm->plan.scheduler_list.data(), nullptr, static_cast(pm->plan.scheduler_list.size()), @@ -718,7 +689,8 @@ static transcribe_status ensure_sched(SortformerSession * pc, SortformerModel * // head over the entire mel and dumps the Stage-4 parity tensors (enc.* + // diar.preds_offline). Invoked only when tensor dumping is active (the // streaming path is the product); keeps the offline tensor gate green. -static transcribe_status run_offline_forward(SortformerSession * pc, SortformerModel * pm, int mel_n_frames) { +template +static transcribe_status run_offline_forward(Session * pc, SortformerModel * pm, int mel_n_frames) { if (pc->compute_ctx != nullptr) { ggml_free(pc->compute_ctx); pc->compute_ctx = nullptr; @@ -811,7 +783,7 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & sc, const float * mel_buf, int mel_n_mels, int mel_n_frames, - transcribe_session * abort_session) { + transcribe::SessionCore * abort_session) { const int ed = chp.enc_d_model; const int n_spk = hp.max_speakers; const int sub = hp.enc_subsampling_factor; @@ -974,13 +946,15 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & sc, } // Family wrapper: resolve the operating point, run the core, trim, dump, -// and emit speaker segments on the session. -static transcribe_status run_streaming(SortformerSession * pc, +// and hand back the frame probabilities. +template +static transcribe_status run_streaming(Session * pc, SortformerModel * pm, int mel_n_mels, int mel_n_frames, double ms_per_frame, - transcribe_sortformer_preset preset) { + transcribe_sortformer_preset preset, + DiarizeProbs & out) { const SortformerStreamParams P = resolve_stream_params(pm->hparams, preset); if (const transcribe_status st = ensure_sched(pc, pm); st != TRANSCRIBE_OK) { return st; @@ -1007,33 +981,23 @@ static transcribe_status run_streaming(SortformerSession * pc, static_cast(used_n) * n_spk, shape, 2, "diarize"); } - probs_to_speaker_segments(pc, pc->scratch.stream.total_preds, used_n, n_spk, ms_per_frame, /*threshold=*/0.5f); + out.probs.assign(pc->scratch.stream.total_preds.begin(), + pc->scratch.stream.total_preds.begin() + static_cast(used_n) * n_spk); + out.n_frames = used_n; + out.n_speakers = n_spk; + out.frame_ms = ms_per_frame; return TRANSCRIBE_OK; } -transcribe_status run(transcribe_session * session, - const float * pcm, - int n_samples, - const transcribe_run_params * params) { - auto * pc = static_cast(session); - auto * pm = static_cast(session->model); - - if (pc->poll_abort()) { - return TRANSCRIBE_ERR_ABORTED; - } - - // Streaming operating-point run ext (kind/size/range already validated - // by the dispatcher + run_validate pre-clear; re-check is belt-and-braces). - transcribe_sortformer_preset preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; - if (params != nullptr && params->family != nullptr) { - if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - sizeof(struct transcribe_sortformer_stream_ext)); - st != TRANSCRIBE_OK) { - return st; - } - preset = reinterpret_cast(params->family)->preset; - } - pc->clear_result(); +// Mel -> (optional offline parity dump) -> streaming forward, shared by the +// ASR run and the DIARIZE run. +template +static transcribe_status diarize_pcm(Session * pc, + SortformerModel * pm, + const float * pcm, + int n_samples, + transcribe_sortformer_preset preset, + DiarizeProbs & out) { transcribe::debug::init(); if (!pm->mel.has_value()) { @@ -1061,10 +1025,38 @@ transcribe_status run(transcribe_session * session, } // Streaming AOSC/FIFO forward drives the product output + diar.probs. - if (const transcribe_status st = run_streaming(pc, pm, mel_n_mels, mel_n_frames, ms_per_frame, preset); - st != TRANSCRIBE_OK) { + return run_streaming(pc, pm, mel_n_mels, mel_n_frames, ms_per_frame, preset, out); +} + +transcribe_status run(transcribe_session * session, + const float * pcm, + int n_samples, + const transcribe_run_params * params) { + auto * pc = static_cast(session); + auto * pm = static_cast(session->model); + + if (pc->poll_abort()) { + return TRANSCRIBE_ERR_ABORTED; + } + + // Streaming operating-point run ext (kind/size/range already validated + // by the dispatcher + run_validate pre-clear; re-check is belt-and-braces). + transcribe_sortformer_preset preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; + if (params != nullptr && params->family != nullptr) { + if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, + sizeof(struct transcribe_sortformer_stream_ext)); + st != TRANSCRIBE_OK) { + return st; + } + preset = reinterpret_cast(params->family)->preset; + } + pc->clear_result(); + + DiarizeProbs probs; + if (const transcribe_status st = diarize_pcm(pc, pm, pcm, n_samples, preset, probs); st != TRANSCRIBE_OK) { return st; } + probs_to_segments(probs.probs.data(), probs.n_frames, probs.n_speakers, probs.frame_ms, pc->speaker_segments); pc->result_kind = TRANSCRIBE_TIMESTAMPS_NONE; pc->has_result = true; @@ -1078,10 +1070,19 @@ static bool accepts_ext_kind(const transcribe_model * model, transcribe_ext_slot if (model == nullptr) { return false; } - if (slot != TRANSCRIBE_EXT_SLOT_RUN) { - return false; + return (slot == TRANSCRIBE_EXT_SLOT_RUN && kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM) || + (slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && kind == TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE); +} + +static transcribe_status check_preset(transcribe_sortformer_preset preset) { + switch (preset) { + case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: + case TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY: + case TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY: + case TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY: + return TRANSCRIBE_OK; } - return kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM; + return TRANSCRIBE_ERR_INVALID_ARG; } // Pre-clear validation for the _RUN slot (see Arch::run_validate): reject a @@ -1096,17 +1097,54 @@ static transcribe_status run_validate(const transcribe_session * /*ctx*/, const st != TRANSCRIBE_OK) { return st; } - const auto * ext = reinterpret_cast(params->family); - switch (ext->preset) { - case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: - case TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY: - case TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY: - case TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY: - return TRANSCRIBE_OK; + return check_preset(reinterpret_cast(params->family)->preset); +} + +// ---- DIARIZE role ---- + +static int diarize_max_speakers(const transcribe_model * model) { + return static_cast(model)->hparams.max_speakers; +} + +static transcribe_status diarize_init_session(transcribe_model *, + const transcribe_diarize_session_params *, + transcribe_diarize_session ** out) { + *out = new SortformerDiarizeSession(); + return TRANSCRIBE_OK; +} + +static transcribe_status diarize_run_validate(const transcribe_diarize_params * params) { + if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, + sizeof(struct transcribe_sortformer_diarize_ext)); + st != TRANSCRIBE_OK) { + return st; } - return TRANSCRIBE_ERR_INVALID_ARG; + return check_preset(reinterpret_cast(params->family)->preset); } +static transcribe_status diarize_run(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params, + DiarizeProbs & out) { + auto * pc = static_cast(session); + if (pc->poll_abort()) { + return TRANSCRIBE_ERR_ABORTED; + } + const transcribe_sortformer_preset preset = + params->family != nullptr ? + reinterpret_cast(params->family)->preset : + TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; + return diarize_pcm(pc, static_cast(session->model), pcm, n_samples, preset, out); +} + +static const DiarizeOps k_diarize_ops = { + /* .max_speakers = */ diarize_max_speakers, + /* .init_session = */ diarize_init_session, + /* .run_validate = */ diarize_run_validate, + /* .run = */ diarize_run, +}; + extern const Arch arch = { /* .name = */ "sortformer", /* .load = */ load, @@ -1120,6 +1158,7 @@ extern const Arch arch = { /* .stream_reset = */ nullptr, /* .accepts_ext_kind = */ accepts_ext_kind, /* .run_validate = */ run_validate, + /* .diarize = */ &k_diarize_ops, }; } // namespace transcribe::sortformer @@ -1130,6 +1169,16 @@ extern const Arch arch = { // transcribe_ext header (size + kind) and the preset default. // --------------------------------------------------------------------------- +extern "C" void transcribe_sortformer_diarize_ext_init(struct transcribe_sortformer_diarize_ext * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->ext.size = sizeof(*p); + p->ext.kind = TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; + p->preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; +} + extern "C" void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * p) { if (p == nullptr) { return; diff --git a/src/arch/sortformer/sortformer.h b/src/arch/sortformer/sortformer.h index bb31eeddc..8910c1198 100644 --- a/src/arch/sortformer/sortformer.h +++ b/src/arch/sortformer/sortformer.h @@ -14,6 +14,7 @@ #include "../parakeet/encoder.h" // build_encoder_graph #include "../parakeet/weights.h" // ParakeetHParams / ParakeetWeights (conformer) #include "transcribe-backend.h" +#include "transcribe-diarize.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -186,21 +187,27 @@ struct DiarStreamScratch { ~DiarStreamScratch(); }; -// Concrete context. The per-call compute context and multi-backend -// scheduler are owned by the transcribe_session base (sched / compute_ctx). -struct SortformerSession final : public transcribe_session { - // Per-context scratch reused across runs. +// Per-session scratch reused across runs, shared by the ASR and DIARIZE +// session types. The per-call compute context and multi-backend scheduler +// are owned by the SessionCore base (sched / compute_ctx). +struct SortformerWork { std::vector mel_buf; std::vector probs_host; // [n_spk * T], read back from diar.preds // Streaming scratch (AOSC/FIFO path + rel-pos tables, shared with the // offline forward's pos_emb fill). DiarStreamScratch scratch; +}; +struct SortformerSession final : public transcribe_session, + SortformerWork { SortformerSession() = default; ~SortformerSession() override; }; +struct SortformerDiarizeSession final : public transcribe_diarize_session, + SortformerWork {}; + // ---- Embedded-diarizer surface (multitalker bundle) -------------------- // // // A multitalker Parakeet bundle GGUF carries this diarizer's tensors under @@ -252,15 +259,6 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & const float * mel_buf, int mel_n_mels, int mel_n_frames, - transcribe_session * abort_session); - -// Threshold-based probs -> speaker_segment rows (probs row-major [T, n_spk], -// speaker_id 1-based, p = NaN). Shared with the multitalker bundle path. -void probs_to_speaker_segments(transcribe_session * session, - const std::vector & probs, - int T, - int n_spk, - double ms_per_frame, - float threshold); + transcribe::SessionCore * abort_session); } // namespace transcribe::sortformer diff --git a/src/transcribe-diarize.cpp b/src/transcribe-diarize.cpp new file mode 100644 index 000000000..cedd4eea4 --- /dev/null +++ b/src/transcribe-diarize.cpp @@ -0,0 +1,278 @@ +// transcribe-diarize.cpp - DIARIZE role C ABI (include/transcribe/diarize.h) +// and the shared probs -> speaker segments step. + +#include "transcribe-diarize.h" + +#include "transcribe-abi.h" +#include "transcribe-api-guard.h" +#include "transcribe-arch.h" +#include "transcribe-model.h" + +#include +#include +#include +#include + +using transcribe::api_guard_status; +using transcribe::api_guard_void; +using transcribe::check_input_struct_size; +using transcribe::check_struct_size; +using transcribe::copy_out_prefix; + +namespace { + +#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) +constexpr size_t k_min_info_size = TRANSCRIBE_FIELD_END(transcribe_diarize_info, max_speakers); +constexpr size_t k_min_session_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_session_params, n_threads); +constexpr size_t k_min_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_params, family); +constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); +constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); +#undef TRANSCRIBE_FIELD_END + +const transcribe::DiarizeOps * diarize_ops(const transcribe_model * model) { + if (model == nullptr || (model->roles & transcribe::k_role_diarize) == 0) { + return nullptr; + } + return model->arch->diarize; // non-null: resolve_roles checked it at load +} + +} // namespace + +void transcribe::probs_to_segments(const float * probs, + int n_frames, + int n_speakers, + double frame_ms, + std::vector & out) { + for (int s = 0; s < n_speakers; ++s) { + int run_start = -1; + for (int t = 0; t <= n_frames; ++t) { + const bool active = t < n_frames && probs[static_cast(t) * n_speakers + s] > 0.5f; + if (active && run_start < 0) { + run_start = t; + } else if (!active && run_start >= 0) { + SpeakerSegmentEntry row; + row.t0_ms = static_cast(std::llround(run_start * frame_ms)); + row.t1_ms = static_cast(std::llround(t * frame_ms)); + row.speaker_id = s + 1; + row.p = std::numeric_limits::quiet_NaN(); + out.push_back(row); + run_start = -1; + } + } + } +} + +extern "C" void transcribe_diarize_info_init(struct transcribe_diarize_info * p) { + if (p != nullptr) { + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + } +} + +extern "C" void transcribe_diarize_session_params_init(struct transcribe_diarize_session_params * p) { + if (p != nullptr) { + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + } +} + +extern "C" void transcribe_diarize_params_init(struct transcribe_diarize_params * p) { + if (p != nullptr) { + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + } +} + +static transcribe_status diarize_get_info_impl(const transcribe_model * model, transcribe_diarize_info * out) { + if (model == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_info_size); st != TRANSCRIBE_OK) { + return st; + } + const transcribe::DiarizeOps * ops = diarize_ops(model); + if (ops == nullptr) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + transcribe_diarize_info staged{}; + staged.struct_size = out->struct_size; + staged.sample_rate = 16000; + staged.max_speakers = ops->max_speakers(model); + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static transcribe_status diarize_session_init_impl(transcribe_model * model, + const transcribe_diarize_session_params * params, + transcribe_diarize_session ** out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out = nullptr; + if (model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const transcribe::DiarizeOps * ops = diarize_ops(model); + if (ops == nullptr) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + transcribe_diarize_session_params defaults; + transcribe_diarize_session_params_init(&defaults); + if (params == nullptr) { + params = &defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_session_params_size); st != TRANSCRIBE_OK) { + return st; + } + if (params->n_threads < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const transcribe_status st = ops->init_session(model, params, out); + if (st == TRANSCRIBE_OK) { + (*out)->model = model; + (*out)->n_threads = params->n_threads; + } + return st; +} + +static transcribe_status diarize_run_impl(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params) { + // Everything up to the commit point leaves the previous result intact. + if (session == nullptr || pcm == nullptr || n_samples <= 0 || !transcribe::pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + transcribe_diarize_params defaults; + transcribe_diarize_params_init(&defaults); + if (params == nullptr) { + params = &defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_params_size); st != TRANSCRIBE_OK) { + return st; + } + const transcribe::DiarizeOps * ops = diarize_ops(session->model); + if (params->family != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (ops->run_validate != nullptr) { + if (const auto st = ops->run_validate(params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + session->segments.clear(); + session->has_result = false; + session->was_aborted = false; + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + + // Release the compute scratch however the run ends (Handy #2000). + struct ScratchRelease { + transcribe_diarize_session * s; + + ~ScratchRelease() { s->release_scratch(); } + } release{ session }; + + transcribe::DiarizeProbs out; + if (const auto st = ops->run(session, pcm, n_samples, params, out); st != TRANSCRIBE_OK) { + return st; + } + transcribe::probs_to_segments(out.probs.data(), out.n_frames, out.n_speakers, out.frame_ms, session->segments); + session->has_result = true; + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_diarize_set_abort_callback(struct transcribe_diarize_session * session, + transcribe_abort_callback cb, + void * user_data) { + if (session != nullptr) { + session->abort_cb = cb; + session->abort_userdata = user_data; + } +} + +extern "C" int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->segments.size()); +} + +extern "C" transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session, + int i, + struct transcribe_speaker_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { + return st; + } + transcribe_speaker_segment staged{}; + staged.struct_size = out->struct_size; + if (i >= 0 && i < transcribe_diarize_n_segments(session)) { + const auto & s = session->segments[static_cast(i)]; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.speaker_id = s.speaker_id; + staged.p = s.p; + } + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, + struct transcribe_timings * out) { + if (session == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + transcribe_timings staged{}; + staged.struct_size = out->struct_size; + staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; + staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; + staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; + staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +// C ABI forwarders for the entry points that allocate, compute, or transfer +// ownership; the ones above are nothrow by construction. + +extern "C" transcribe_status transcribe_diarize_get_info(const struct transcribe_model * model, + struct transcribe_diarize_info * out) { + return api_guard_status("transcribe_diarize_get_info", [&] { return diarize_get_info_impl(model, out); }); +} + +extern "C" transcribe_status transcribe_diarize_session_init(struct transcribe_model * model, + const struct transcribe_diarize_session_params * params, + struct transcribe_diarize_session ** out) { + const transcribe_status st = api_guard_status("transcribe_diarize_session_init", + [&] { return diarize_session_init_impl(model, params, out); }); + if (st != TRANSCRIBE_OK && out != nullptr && *out != nullptr) { + transcribe_diarize_session_free(*out); + *out = nullptr; + } + return st; +} + +extern "C" void transcribe_diarize_session_free(struct transcribe_diarize_session * session) { + api_guard_void("transcribe_diarize_session_free", [&] { delete session; }); +} + +extern "C" transcribe_status transcribe_diarize_run(struct transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const struct transcribe_diarize_params * params) { + return api_guard_status("transcribe_diarize_run", + [&] { return diarize_run_impl(session, pcm, n_samples, params); }); +} diff --git a/src/transcribe-diarize.h b/src/transcribe-diarize.h new file mode 100644 index 000000000..4ae6ea836 --- /dev/null +++ b/src/transcribe-diarize.h @@ -0,0 +1,53 @@ +// transcribe-diarize.h - internal DIARIZE role surface: the session base, +// the per-arch ops table, and the shared probs -> segments step. +// +// Families compute frame probabilities; the role dispatcher +// (transcribe-diarize.cpp) turns them into speaker segments. + +#pragma once + +#include "transcribe-session-core.h" +#include "transcribe/diarize.h" + +#include + +struct transcribe_diarize_session : transcribe::SessionCore { + std::vector segments; + bool has_result = false; +}; + +namespace transcribe { + +// One run's family output: probs is row-major [n_frames, n_speakers]. +struct DiarizeProbs { + std::vector probs; + int n_frames = 0; + int n_speakers = 0; + double frame_ms = 0.0; +}; + +struct DiarizeOps { + int (*max_speakers)(const transcribe_model * model); + // params is never NULL (the dispatcher substitutes defaults). + transcribe_status (*init_session)(transcribe_model * model, + const transcribe_diarize_session_params * params, + transcribe_diarize_session ** out); + // Pure pre-clear check of params->family (already kind/size-checked); + // NULL = nothing to check. + transcribe_status (*run_validate)(const transcribe_diarize_params * params); + transcribe_status (*run)(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params, + DiarizeProbs & out); +}; + +// Append one segment per contiguous run of probs > 0.5 per speaker, +// speaker-major and time-ordered (speaker_id 1-based, p = NaN). +void probs_to_segments(const float * probs, + int n_frames, + int n_speakers, + double frame_ms, + std::vector & out); + +} // namespace transcribe diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 9f2a1b195..f6c54668b 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -28,6 +28,7 @@ #include "transcribe-log.h" #include "transcribe-model.h" #include "transcribe-path.h" +#include "transcribe/diarize.h" #if defined(TRANSCRIBE_GGML_BACKEND_DL) && defined(_WIN32) # ifndef WIN32_LEAN_AND_MEAN @@ -172,6 +173,12 @@ extern "C" size_t transcribe_abi_struct_size(transcribe_abi_struct which) { return sizeof(struct transcribe_device_info); case TRANSCRIBE_ABI_SPEAKER_SEGMENT: return sizeof(struct transcribe_speaker_segment); + case TRANSCRIBE_ABI_DIARIZE_INFO: + return sizeof(struct transcribe_diarize_info); + case TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: + return sizeof(struct transcribe_diarize_session_params); + case TRANSCRIBE_ABI_DIARIZE_PARAMS: + return sizeof(struct transcribe_diarize_params); } return 0; // unknown id: "cannot verify", never a real size } @@ -208,6 +215,12 @@ extern "C" size_t transcribe_abi_struct_align(transcribe_abi_struct which) { return alignof(struct transcribe_device_info); case TRANSCRIBE_ABI_SPEAKER_SEGMENT: return alignof(struct transcribe_speaker_segment); + case TRANSCRIBE_ABI_DIARIZE_INFO: + return alignof(struct transcribe_diarize_info); + case TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: + return alignof(struct transcribe_diarize_session_params); + case TRANSCRIBE_ABI_DIARIZE_PARAMS: + return alignof(struct transcribe_diarize_params); } return 0; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a0d5c52eb..d97948b1c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -386,6 +386,23 @@ transcribe_apply_warnings(transcribe_stream_hook_throw_unit) add_test(NAME transcribe_stream_hook_throw_unit COMMAND transcribe_stream_hook_throw_unit) +# ----------------------------------------------------------------------------- +# DIARIZE role dispatcher (fake arch) +# ----------------------------------------------------------------------------- + +add_executable(transcribe_diarize_dispatch_unit + diarize_dispatch_unit.cpp) + +target_link_libraries(transcribe_diarize_dispatch_unit PRIVATE transcribe) + +target_include_directories(transcribe_diarize_dispatch_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_diarize_dispatch_unit) + +add_test(NAME transcribe_diarize_dispatch_unit + COMMAND transcribe_diarize_dispatch_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- @@ -1397,6 +1414,21 @@ if(TRANSCRIBE_BUILD_REAL_MODEL_TESTS) COMMAND transcribe_sortformer_stream_ext_unit) set_tests_properties(transcribe_sortformer_stream_ext_unit PROPERTIES SKIP_RETURN_CODE 77) + + # Sortformer on the DIARIZE role: old-vs-new segment parity, pre-clear + # rejection, and (with TRANSCRIBE_WHISPER_GGUF) cross-role concurrency. + # Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). + add_executable(transcribe_sortformer_diarize_unit + sortformer_diarize_unit.cpp) + target_link_libraries(transcribe_sortformer_diarize_unit + PRIVATE transcribe transcribe-common-example) + target_compile_definitions(transcribe_sortformer_diarize_unit PRIVATE + "TRANSCRIBE_TEST_SAMPLES_DIR=\"${CMAKE_SOURCE_DIR}/samples\"") + transcribe_apply_warnings(transcribe_sortformer_diarize_unit) + add_test(NAME transcribe_sortformer_diarize_unit + COMMAND transcribe_sortformer_diarize_unit) + set_tests_properties(transcribe_sortformer_diarize_unit PROPERTIES + SKIP_RETURN_CODE 77) endif() # ----------------------------------------------------------------------------- diff --git a/tests/diarize_dispatch_unit.cpp b/tests/diarize_dispatch_unit.cpp new file mode 100644 index 000000000..f37249a39 --- /dev/null +++ b/tests/diarize_dispatch_unit.cpp @@ -0,0 +1,226 @@ +// diarize_dispatch_unit.cpp - DIARIZE role dispatcher (transcribe-diarize.cpp) +// against a fake arch: role checks, pre-clear validation, probs -> segments, +// accessors. + +#include "transcribe-arch.h" +#include "transcribe-diarize.h" +#include "transcribe-model.h" +#include "transcribe.h" +#include "transcribe/diarize.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +constexpr uint32_t k_fake_kind = 0x4B414644; // 'DFAK' + +int g_run_calls = 0; +transcribe_status g_run_status = TRANSCRIBE_OK; +transcribe_status g_validate_st = TRANSCRIBE_OK; + +int fake_max_speakers(const transcribe_model *) { + return 3; +} + +transcribe_status fake_init_session(transcribe_model *, + const transcribe_diarize_session_params *, + transcribe_diarize_session ** out) { + *out = new transcribe_diarize_session(); + return TRANSCRIBE_OK; +} + +transcribe_status fake_run_validate(const transcribe_diarize_params *) { + return g_validate_st; +} + +// Frames at 80 ms: speaker 1 active in frames 1-2, speaker 2 in frames 2-3. +transcribe_status fake_run(transcribe_diarize_session *, + const float *, + int, + const transcribe_diarize_params *, + transcribe::DiarizeProbs & out) { + ++g_run_calls; + if (g_run_status != TRANSCRIBE_OK) { + return g_run_status; + } + out.probs = { 0.1f, 0.0f, 0.9f, 0.0f, 0.8f, 0.7f, 0.2f, 0.6f }; + out.n_frames = 4; + out.n_speakers = 2; + out.frame_ms = 80.0; + return TRANSCRIBE_OK; +} + +bool fake_accepts(const transcribe_model *, transcribe_ext_slot slot, uint32_t kind) { + return slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && kind == k_fake_kind; +} + +const transcribe::DiarizeOps k_ops = { fake_max_speakers, fake_init_session, fake_run_validate, fake_run }; + +const transcribe::Arch k_arch = { + /* .name = */ "fake-diarize", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ nullptr, + /* .run_batch = */ nullptr, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ nullptr, + /* .stream_feed = */ nullptr, + /* .stream_finalize = */ nullptr, + /* .stream_reset = */ nullptr, + /* .accepts_ext_kind = */ fake_accepts, + /* .run_validate = */ nullptr, + /* .diarize = */ &k_ops, +}; + +struct Fixture { + transcribe_model model; + transcribe_diarize_session * session = nullptr; + std::vector pcm = std::vector(1600, 0.0f); + + Fixture() { + model.arch = &k_arch; + model.roles = transcribe::k_role_diarize; + CHECK(transcribe_diarize_session_init(&model, nullptr, &session) == TRANSCRIBE_OK); + } + + ~Fixture() { transcribe_diarize_session_free(session); } + + transcribe_status run(const transcribe_diarize_params * p = nullptr) { + return transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), p); + } +}; + +void test_probs_to_segments() { + // [frames=5, speakers=2]; speaker 2 still active at the end. + const float probs[] = { 0.6f, 0.0f, 0.6f, 0.0f, 0.4f, 0.0f, 0.9f, 0.51f, 0.0f, 0.7f }; + std::vector out; + transcribe::probs_to_segments(probs, 5, 2, 80.0, out); + CHECK(out.size() == 3); + CHECK(out[0].speaker_id == 1 && out[0].t0_ms == 0 && out[0].t1_ms == 160); + CHECK(out[1].speaker_id == 1 && out[1].t0_ms == 240 && out[1].t1_ms == 320); + CHECK(out[2].speaker_id == 2 && out[2].t0_ms == 240 && out[2].t1_ms == 400); + CHECK(std::isnan(out[0].p)); +} + +void test_role_checks() { + transcribe_model model; + model.arch = &k_arch; + model.roles = transcribe::k_role_asr; // no DIARIZE bit + + transcribe_diarize_session * s = reinterpret_cast(0x1); + CHECK(transcribe_diarize_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(s == nullptr); + + transcribe_diarize_info info; + transcribe_diarize_info_init(&info); + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + + model.roles = transcribe::k_role_diarize; + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_OK); + CHECK(info.sample_rate == 16000 && info.max_speakers == 3); + CHECK(transcribe_diarize_get_info(nullptr, &info) == TRANSCRIBE_ERR_INVALID_ARG); + info.struct_size = 0; + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + + transcribe_diarize_session_params sp; + transcribe_diarize_session_params_init(&sp); + sp.n_threads = -1; + CHECK(transcribe_diarize_session_init(&model, &sp, &s) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(s == nullptr); + sp.n_threads = 2; + CHECK(transcribe_diarize_session_init(&model, &sp, &s) == TRANSCRIBE_OK); + CHECK(s != nullptr && s->model == &model && s->n_threads == 2); + transcribe_diarize_session_free(s); + transcribe_diarize_session_free(nullptr); +} + +void test_run_and_accessors() { + Fixture f; + CHECK(transcribe_diarize_n_segments(f.session) == 0); + CHECK(f.run() == TRANSCRIBE_OK); + CHECK(transcribe_diarize_n_segments(f.session) == 2); + + transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + CHECK(transcribe_diarize_get_segment(f.session, 1, &row) == TRANSCRIBE_OK); + CHECK(row.speaker_id == 2 && row.t0_ms == 160 && row.t1_ms == 320); + CHECK(transcribe_diarize_get_segment(f.session, 2, &row) == TRANSCRIBE_OK); // out of range: zeroed + CHECK(row.speaker_id == 0 && row.t0_ms == 0 && row.t1_ms == 0); + CHECK(transcribe_diarize_get_segment(f.session, 0, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + + transcribe_timings tm; + transcribe_timings_init(&tm); + CHECK(transcribe_diarize_get_timings(f.session, &tm) == TRANSCRIBE_OK); +} + +// Malformed input and family-rejected params leave the previous result alone +// and never reach the family run. +void test_pre_clear_rejections_preserve_result() { + g_validate_st = TRANSCRIBE_OK; + g_run_status = TRANSCRIBE_OK; + Fixture f; + CHECK(f.run() == TRANSCRIBE_OK); + g_run_calls = 0; + + std::vector bad(1600, 0.0f); + bad[7] = std::numeric_limits::quiet_NaN(); + CHECK(transcribe_diarize_run(f.session, bad.data(), 1600, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_run(f.session, nullptr, 1600, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_run(f.session, f.pcm.data(), 0, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + + transcribe_ext ext{ sizeof(transcribe_ext), 0x12345678u }; + transcribe_diarize_params p; + transcribe_diarize_params_init(&p); + p.family = &ext; + CHECK(f.run(&p) == TRANSCRIBE_ERR_INVALID_ARG); // kind not accepted on DIARIZE_RUN + ext.size = 4; + CHECK(f.run(&p) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + ext = { sizeof(transcribe_ext), k_fake_kind }; + g_validate_st = TRANSCRIBE_ERR_INVALID_ARG; + CHECK(f.run(&p) == TRANSCRIBE_ERR_INVALID_ARG); + p.struct_size = 0; + CHECK(f.run(&p) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + + CHECK(g_run_calls == 0); + CHECK(transcribe_diarize_n_segments(f.session) == 2); + + // A family failure after the commit point clears the result. + g_validate_st = TRANSCRIBE_OK; + g_run_status = TRANSCRIBE_ERR_ABORTED; + p.struct_size = sizeof(p); + CHECK(f.run(&p) == TRANSCRIBE_ERR_ABORTED); + CHECK(transcribe_diarize_n_segments(f.session) == 0); + g_run_status = TRANSCRIBE_OK; +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); + + test_probs_to_segments(); + test_role_checks(); + test_run_and_accessors(); + test_pre_clear_rejections_preserve_result(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("diarize_dispatch_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp new file mode 100644 index 000000000..ab194bb8a --- /dev/null +++ b/tests/sortformer_diarize_unit.cpp @@ -0,0 +1,200 @@ +// sortformer_diarize_unit.cpp - Sortformer on the DIARIZE role, against a +// real GGUF (TRANSCRIBE_SORTFORMER_GGUF; RC 77 skip when unset). +// +// 1. roles, info, and the SFDR kind on the DIARIZE_RUN slot. +// 2. transcribe_diarize_run gives byte-identical segments to the ASR-path +// transcribe_run at the same preset (DEFAULT and LOW_LATENCY). +// 3. An out-of-range preset is rejected before the previous result is +// cleared. +// 4. Cross-role concurrency: when TRANSCRIBE_WHISPER_GGUF is also set, an +// ASR session and a diarize session on two models run on two threads +// at once and match their sequential results. + +#include "transcribe.h" +#include "transcribe/diarize.h" +#include "transcribe/sortformer.h" +#include "wav.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +struct Row { + int64_t t0, t1; + int32_t spk; + + bool operator==(const Row & o) const { return t0 == o.t0 && t1 == o.t1 && spk == o.spk; } +}; + +std::vector asr_rows(const transcribe_session * s) { + std::vector rows; + for (int i = 0; i < transcribe_n_speaker_segments(s); ++i) { + transcribe_speaker_segment r; + transcribe_speaker_segment_init(&r); + transcribe_get_speaker_segment(s, i, &r); + rows.push_back({ r.t0_ms, r.t1_ms, r.speaker_id }); + } + return rows; +} + +std::vector diarize_rows(const transcribe_diarize_session * s) { + std::vector rows; + for (int i = 0; i < transcribe_diarize_n_segments(s); ++i) { + transcribe_speaker_segment r; + transcribe_speaker_segment_init(&r); + transcribe_diarize_get_segment(s, i, &r); + rows.push_back({ r.t0_ms, r.t1_ms, r.speaker_id }); + } + return rows; +} + +transcribe_model * load_cpu(const char * path) { + transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = TRANSCRIBE_BACKEND_CPU; + transcribe_model * model = nullptr; + return transcribe_model_load_file(path, &mp, &model) == TRANSCRIBE_OK ? model : nullptr; +} + +std::vector diarize(transcribe_diarize_session * s, + const std::vector & pcm, + transcribe_sortformer_preset preset) { + transcribe_sortformer_diarize_ext ext; + transcribe_sortformer_diarize_ext_init(&ext); + ext.preset = preset; + transcribe_diarize_params dp; + transcribe_diarize_params_init(&dp); + dp.family = &ext.ext; + CHECK(transcribe_diarize_run(s, pcm.data(), static_cast(pcm.size()), &dp) == TRANSCRIBE_OK); + return diarize_rows(s); +} + +void test_cross_role_concurrency(transcribe_model * sf, const std::vector & pcm) { + const char * whisper_path = std::getenv("TRANSCRIBE_WHISPER_GGUF"); + if (whisper_path == nullptr || whisper_path[0] == '\0') { + std::fprintf(stderr, "sortformer_diarize_unit: TRANSCRIBE_WHISPER_GGUF not set; skipping concurrency leg\n"); + return; + } + transcribe_model * asr_model = load_cpu(whisper_path); + CHECK(asr_model != nullptr); + if (asr_model == nullptr) { + return; + } + transcribe_session * asr = nullptr; + transcribe_diarize_session * dia = nullptr; + CHECK(transcribe_session_init(asr_model, nullptr, &asr) == TRANSCRIBE_OK); + CHECK(transcribe_diarize_session_init(sf, nullptr, &dia) == TRANSCRIBE_OK); + + CHECK(transcribe_run(asr, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + const std::string text_seq = transcribe_full_text(asr); + const std::vector rows_seq = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); + + transcribe_status asr_st = TRANSCRIBE_ERR_BACKEND; + std::thread t([&] { asr_st = transcribe_run(asr, pcm.data(), static_cast(pcm.size()), nullptr); }); + const std::vector rows_par = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); + t.join(); + + CHECK(asr_st == TRANSCRIBE_OK); + CHECK(text_seq == transcribe_full_text(asr)); + CHECK(rows_par == rows_seq); + + transcribe_session_free(asr); + transcribe_diarize_session_free(dia); + transcribe_model_free(asr_model); +} + +} // namespace + +int main() { + const char * path = std::getenv("TRANSCRIBE_SORTFORMER_GGUF"); + if (path == nullptr || path[0] == '\0') { + std::fprintf(stderr, "sortformer_diarize_unit: TRANSCRIBE_SORTFORMER_GGUF not set; skipping.\n"); + return 77; + } + std::vector pcm; + std::string err; + if (!transcribe_cli::load_wav_mono_16k(std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/sortformer-2spk-mix.wav", pcm, + err)) { + std::fprintf(stderr, "sortformer_diarize_unit: wav: %s\n", err.c_str()); + return 77; + } + ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); + + transcribe_model * model = load_cpu(path); + if (model == nullptr) { + std::fprintf(stderr, "FAIL: model load\n"); + return EXIT_FAILURE; + } + + // 1. Roles, info, extension probe. + CHECK(transcribe_model_roles(model) == (TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE)); + transcribe_diarize_info info; + transcribe_diarize_info_init(&info); + CHECK(transcribe_diarize_get_info(model, &info) == TRANSCRIBE_OK); + CHECK(info.sample_rate == 16000 && info.max_speakers == 4); + CHECK(transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, + TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); + CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, + TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); + CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); + + transcribe_session * asr = nullptr; + transcribe_diarize_session * dia = nullptr; + CHECK(transcribe_session_init(model, nullptr, &asr) == TRANSCRIBE_OK); + CHECK(transcribe_diarize_session_init(model, nullptr, &dia) == TRANSCRIBE_OK); + + // 2. Old (ASR path) vs new (DIARIZE role) parity. + for (transcribe_sortformer_preset preset : + { TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY }) { + transcribe_sortformer_stream_ext ext; + transcribe_sortformer_stream_ext_init(&ext); + ext.preset = preset; + transcribe_run_params rp; + transcribe_run_params_init(&rp); + rp.family = &ext.ext; + CHECK(transcribe_run(asr, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); + const std::vector old_rows = asr_rows(asr); + const std::vector new_rows = diarize(dia, pcm, preset); + CHECK(!new_rows.empty()); + CHECK(new_rows == old_rows); + } + + // 3. Pre-clear rejection of a bad preset keeps the previous result. + const int n_before = transcribe_diarize_n_segments(dia); + transcribe_sortformer_diarize_ext bad; + transcribe_sortformer_diarize_ext_init(&bad); + bad.preset = static_cast(99); + transcribe_diarize_params dp; + transcribe_diarize_params_init(&dp); + dp.family = &bad.ext; + CHECK(transcribe_diarize_run(dia, pcm.data(), static_cast(pcm.size()), &dp) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_n_segments(dia) == n_before); + + transcribe_session_free(asr); + transcribe_diarize_session_free(dia); + + // 4. Cross-role concurrency on two models. + test_cross_role_concurrency(model, pcm); + + transcribe_model_free(model); + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("sortformer_diarize_unit: ok\n"); + return EXIT_SUCCESS; +} From 1fd3c5a1aa67f0cec6385c47de0ddaede9fe21f3 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:11:43 +0800 Subject: [PATCH 23/51] cli: diarize path transcribe-cli runs the DIARIZE role on a model without ASR, or on any model with --role diarize (--role asr|diarize picks the role on a model serving both; the default stays ASR when available). The shared prelude (WAV load, model load and its metadata lines) is unchanged; right after it, run_asr_file hands a diarize model to run_diarize_file (diarize.cpp), which prints the speaker segments and the realtime line. validate.py and scripts/diar/run_cpp_sortformer.py call the CLI on the Sortformer GGUF and keep working once Sortformer is DIARIZE-only. tests: cli_diarize_smoke (gated on TRANSCRIBE_SORTFORMER_GGUF, reported as skipped otherwise). transcribe-cli CPU parity vs main identical apart from the two new --role usage lines. --- examples/cli/CMakeLists.txt | 1 + examples/cli/asr.cpp | 5 ++++ examples/cli/cli.h | 24 ++++++++++++----- examples/cli/diarize.cpp | 51 +++++++++++++++++++++++++++++++++++ examples/cli/main.cpp | 12 +++++++++ tests/CMakeLists.txt | 8 ++++++ tests/cli_diarize_smoke.cmake | 19 +++++++++++++ 7 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 examples/cli/diarize.cpp create mode 100644 tests/cli_diarize_smoke.cmake diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 441a0ba27..07a681522 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -7,6 +7,7 @@ add_executable(transcribe-cli main.cpp asr.cpp + diarize.cpp ) target_link_libraries(transcribe-cli diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index 78e37541e..74984eb2a 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -651,6 +651,11 @@ int run_asr_file(const cli_args & args, std::ofstream * output) { std::printf(" license: %s\n", lic); } + const uint32_t roles = transcribe_model_roles(model); + if (args.role == "diarize" || (args.role.empty() && (roles & TRANSCRIBE_ROLE_ASR) == 0)) { + return transcribe_cli::run_diarize_file(args, model, pcm, duration_s); + } + struct transcribe_session_params cp; transcribe_session_params_init(&cp); cp.n_threads = args.n_threads; diff --git a/examples/cli/cli.h b/examples/cli/cli.h index dbb477e12..87f55d408 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -75,33 +75,36 @@ struct cli_args { // ms-aligned chunks instead of one transcribe_run call. Requires // the loaded model to advertise supports_streaming. Set by // --stream-chunk-ms N. - int stream_chunk_ms = 0; + int stream_chunk_ms = 0; // Parakeet streaming: pick a right-context (lookahead) setting // from the model's training menu. -1 = model default (max // accuracy / max latency); 0/1/6/13 select the published // nemotron-speech-streaming-en-0.6b settings. Set by // --stream-att-right N. Ignored when stream_chunk_ms == 0. - int stream_att_right = -1; + int stream_att_right = -1; // Parakeet buffered streaming (parakeet-unified-en-0.6b): override // the (L, C, R) attention context tuple in milliseconds. -1 = use // the model's default (highest-accuracy row of the training menu). // Frame-aligned: the lib rounds each value down to the nearest // post-subsample frame (80ms at 4x subsampling). Ignored when // stream_chunk_ms == 0 or when the model is not buffered-streaming. - int stream_buf_left_ms = -1; - int stream_buf_chunk_ms = -1; - int stream_buf_right_ms = -1; + int stream_buf_left_ms = -1; + int stream_buf_chunk_ms = -1; + int stream_buf_right_ms = -1; // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when // the model is not voxtral_realtime. - int stream_voxtral_delay = -1; + int stream_voxtral_delay = -1; // Speculative-decode draft length passed through to // transcribe_run_params::spec_k_drafts on the offline path. -1 = family // default (each family picks its tuned K). 0 = explicitly off. >0 = // explicit K. Silently ignored by families without // supports_spec_decode. Set by --spec-k-drafts N. - int spec_k_drafts = -1; + int spec_k_drafts = -1; + // --role asr|diarize: which role to run on a multi-role model. Empty = + // ASR when the model serves it, else diarize. + std::string role; }; // Minimal JSON string escape (main.cpp). @@ -116,4 +119,11 @@ bool write_output_file(std::ofstream * output, const std::string & path, const c int run_asr_batch(const cli_args & args, std::ofstream * output); int run_asr_file(const cli_args & args, std::ofstream * output); +// DIARIZE driver (diarize.cpp): runs on a model run_asr_file already loaded +// and printed. Takes ownership of the model. +int run_diarize_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s); + } // namespace transcribe_cli diff --git a/examples/cli/diarize.cpp b/examples/cli/diarize.cpp new file mode 100644 index 000000000..f811edf99 --- /dev/null +++ b/examples/cli/diarize.cpp @@ -0,0 +1,51 @@ +// diarize.cpp - transcribe-cli DIARIZE driver: speaker segments for one file +// through include/transcribe/diarize.h. + +#include "transcribe/diarize.h" + +#include "cli.h" + +#include +#include +#include + +int transcribe_cli::run_diarize_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s) { + transcribe_diarize_session_params sp; + transcribe_diarize_session_params_init(&sp); + sp.n_threads = args.n_threads; + transcribe_diarize_session * session = nullptr; + transcribe_status st = transcribe_diarize_session_init(model, &sp, &session); + if (st != TRANSCRIBE_OK) { + std::fprintf(stderr, "diarize session init: %s\n", transcribe_status_string(st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + st = transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), nullptr); + std::printf("run: %s\n", transcribe_status_string(st)); + if (st == TRANSCRIBE_OK) { + const int n = transcribe_diarize_n_segments(session); + std::printf("speaker segments: %d\n", n); + for (int i = 0; i < n; ++i) { + transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + transcribe_diarize_get_segment(session, i, &row); + std::printf(" [%7.2f -> %7.2f] S%d\n", row.t0_ms / 1000.0, row.t1_ms / 1000.0, row.speaker_id); + } + } + + transcribe_timings tm; + transcribe_timings_init(&tm); + transcribe_diarize_get_timings(session, &tm); + const double total_ms = tm.mel_ms + tm.encode_ms; + if (total_ms > 0.0 && duration_s > 0.0) { + std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", duration_s * 1000.0 / total_ms, total_ms, duration_s); + } + + transcribe_diarize_session_free(session); + transcribe_model_free(model); + return st == TRANSCRIBE_OK ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index b740ff2f2..5dcea047b 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -114,6 +114,8 @@ void print_usage(const char * argv0) { " supports_spec_decode. -1 = family default,\n" " 0 = off, > 0 = explicit K. Silently ignored\n" " by families without spec support.\n" + " --role ROLE asr or diarize, for a model serving both\n" + " (default: asr when available)\n" " --list-devices list registered compute devices (with memory)\n" " and exit; ignores all other options\n" " -h, --help show this help\n", @@ -487,6 +489,16 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --spec-k-drafts must be -1 (family default), 0 (off), or > 0\n"); return false; } + } else if (a == "--role") { + const char * v = take_value(a.c_str()); + if (!v) { + return false; + } + out.role = v; + if (out.role != "asr" && out.role != "diarize") { + std::fprintf(stderr, "error: --role must be asr or diarize\n"); + return false; + } } else if (!a.empty() && a[0] == '-') { std::fprintf(stderr, "error: unknown option '%s'\n", a.c_str()); return false; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d97948b1c..38c0ff1df 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1608,4 +1608,12 @@ if(TRANSCRIBE_BUILD_EXAMPLES) -DCLI=$ -DWAV=${CMAKE_SOURCE_DIR}/samples/jfk.wav -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_device_arg_smoke.cmake) + + add_test(NAME transcribe_cli_diarize_smoke + COMMAND ${CMAKE_COMMAND} + -DCLI=$ + -DWAV=${CMAKE_SOURCE_DIR}/samples/sortformer-2spk-mix.wav + -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_diarize_smoke.cmake) + set_tests_properties(transcribe_cli_diarize_smoke PROPERTIES + SKIP_REGULAR_EXPRESSION "SKIP:") endif() diff --git a/tests/cli_diarize_smoke.cmake b/tests/cli_diarize_smoke.cmake new file mode 100644 index 000000000..eb35f1840 --- /dev/null +++ b/tests/cli_diarize_smoke.cmake @@ -0,0 +1,19 @@ +# transcribe-cli --role diarize prints speaker segments for the 2-speaker +# oracle mix. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). + +if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") + message("SKIP: TRANSCRIBE_SORTFORMER_GGUF not set") + return() +endif() + +execute_process( + COMMAND "${CLI}" -q --backend cpu --role diarize -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(NOT rc EQUAL 0) + message(FATAL_ERROR "transcribe-cli exited ${rc}\n${out}\n${err}") +endif() +if(NOT out MATCHES "run: ok" OR NOT out MATCHES "speaker segments: [1-9]" OR NOT out MATCHES "S2\n") + message(FATAL_ERROR "unexpected output:\n${out}") +endif() From c6bb5ab9d32165d7e48cd5b4f15c76c990291da2 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:12:08 +0800 Subject: [PATCH 24/51] bindings: regenerate low-level layers for diarize.h Generated only: Python / TypeScript via _generate/generate.py and Rust via cargo xtask bindgen pick up include/transcribe/diarize.h, the DIARIZE_RUN slot, the diarize ABI ids and the Sortformer SFDR extension. transcribe.abihash updated; the Swift pin and the hand-written wrappers follow per binding. --- .../python/src/transcribe_cpp/_generated.py | 50 +++++- bindings/rust/sys/src/transcribe_sys.rs | 146 +++++++++++++++++- bindings/typescript/src/_generated.ts | 30 +++- include/transcribe.abihash | 2 +- 4 files changed, 223 insertions(+), 5 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 518a57770..25aaa98c6 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "1537ea60b442dd53" +PUBLIC_HEADER_HASH = "c2979c7e891982ee" # === enum constants === TRANSCRIBE_OK = 0 @@ -52,6 +52,9 @@ TRANSCRIBE_ABI_EXT = 12 TRANSCRIBE_ABI_DEVICE_INFO = 13 TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14 +TRANSCRIBE_ABI_DIARIZE_INFO = 15 +TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 16 +TRANSCRIBE_ABI_DIARIZE_PARAMS = 17 TRANSCRIBE_LOG_LEVEL_NONE = 0 TRANSCRIBE_LOG_LEVEL_INFO = 1 TRANSCRIBE_LOG_LEVEL_WARN = 2 @@ -80,6 +83,7 @@ TRANSCRIBE_DIARIZE_MODE_ON = 2 TRANSCRIBE_EXT_SLOT_RUN = 0 TRANSCRIBE_EXT_SLOT_STREAM = 1 +TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2 TRANSCRIBE_BACKEND_AUTO = 0 TRANSCRIBE_BACKEND_CPU = 1 TRANSCRIBE_BACKEND_METAL = 2 @@ -122,6 +126,7 @@ TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM = 1414746957 TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584 TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912 +TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163 TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635 TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710 TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319 @@ -157,6 +162,12 @@ class transcribe_token(_c.Structure): pass class transcribe_speaker_segment(_c.Structure): pass +class transcribe_diarize_info(_c.Structure): + pass +class transcribe_diarize_session_params(_c.Structure): + pass +class transcribe_diarize_params(_c.Structure): + pass class transcribe_moonshine_streaming_stream_ext(_c.Structure): pass class transcribe_parakeet_stream_ext(_c.Structure): @@ -165,6 +176,8 @@ class transcribe_parakeet_buffered_stream_ext(_c.Structure): pass class transcribe_sortformer_stream_ext(_c.Structure): pass +class transcribe_sortformer_diarize_ext(_c.Structure): + pass class transcribe_voxtral_realtime_stream_ext(_c.Structure): pass class transcribe_whisper_run_ext(_c.Structure): @@ -187,10 +200,14 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_word._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("seg_index", _c.c_int), ("first_token", _c.c_int), ("n_tokens", _c.c_int), ("text", _c.c_char_p)] transcribe_token._fields_ = [("struct_size", _c.c_uint64), ("id", _c.c_int), ("p", _c.c_float), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("seg_index", _c.c_int), ("word_index", _c.c_int), ("text", _c.c_char_p)] transcribe_speaker_segment._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("speaker_id", _c.c_int32), ("p", _c.c_float)] +transcribe_diarize_info._fields_ = [("struct_size", _c.c_uint64), ("sample_rate", _c.c_int32), ("max_speakers", _c.c_int32)] +transcribe_diarize_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int32)] +transcribe_diarize_params._fields_ = [("struct_size", _c.c_uint64), ("family", _c.POINTER(transcribe_ext))] transcribe_moonshine_streaming_stream_ext._fields_ = [("ext", transcribe_ext), ("min_decode_interval_ms", _c.c_int32)] transcribe_parakeet_stream_ext._fields_ = [("ext", transcribe_ext), ("att_context_right", _c.c_int32)] transcribe_parakeet_buffered_stream_ext._fields_ = [("ext", transcribe_ext), ("left_ms", _c.c_int32), ("chunk_ms", _c.c_int32), ("right_ms", _c.c_int32)] transcribe_sortformer_stream_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] +transcribe_sortformer_diarize_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] transcribe_voxtral_realtime_stream_ext._fields_ = [("ext", transcribe_ext), ("num_delay_tokens", _c.c_int32), ("min_decode_interval_ms", _c.c_int32)] transcribe_whisper_run_ext._fields_ = [("ext", transcribe_ext), ("initial_prompt", _c.c_char_p), ("prompt_tokens", _c.POINTER(_c.c_int32)), ("n_prompt_tokens", _c.c_size_t), ("prompt_condition", _c.c_int), ("condition_on_prev_tokens", _c.c_bool), ("max_prev_context_tokens", _c.c_int32), ("temperature", _c.c_float), ("temperature_inc", _c.c_float), ("compression_ratio_thold", _c.c_float), ("logprob_thold", _c.c_float), ("no_speech_thold", _c.c_float), ("seed", _c.c_uint32), ("max_initial_timestamp", _c.c_float)] transcribe_whisper_chunk_trace._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("temperature_used", _c.c_float), ("compression_ratio", _c.c_float), ("avg_logprob", _c.c_float), ("no_speech_prob", _c.c_float), ("no_speech_triggered", _c.c_bool), ("n_fallbacks", _c.c_int32)] @@ -213,6 +230,9 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_word': 7, 'transcribe_token': 8, 'transcribe_speaker_segment': 14, + 'transcribe_diarize_info': 15, + 'transcribe_diarize_session_params': 16, + 'transcribe_diarize_params': 17, } # C-compiler layout captured at generation (for offset self-check). @@ -232,10 +252,14 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_word': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'seg_index': 24, 'first_token': 28, 'n_tokens': 32, 'text': 40}}, 'transcribe_token': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 'id': 8, 'p': 12, 't0_ms': 16, 't1_ms': 24, 'seg_index': 32, 'word_index': 36, 'text': 40}}, 'transcribe_speaker_segment': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'speaker_id': 24, 'p': 28}}, + 'transcribe_diarize_info': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'sample_rate': 8, 'max_speakers': 12}}, + 'transcribe_diarize_session_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8}}, + 'transcribe_diarize_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'family': 8}}, 'transcribe_moonshine_streaming_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'min_decode_interval_ms': 16}}, 'transcribe_parakeet_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'att_context_right': 16}}, 'transcribe_parakeet_buffered_stream_ext': {'size': 32, 'align': 8, 'offsets': {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24}}, 'transcribe_sortformer_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, + 'transcribe_sortformer_diarize_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, 'transcribe_voxtral_realtime_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20}}, 'transcribe_whisper_run_ext': {'size': 80, 'align': 8, 'offsets': {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76}}, 'transcribe_whisper_chunk_trace': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'temperature_used': 24, 'compression_ratio': 28, 'avg_logprob': 32, 'no_speech_prob': 36, 'no_speech_triggered': 40, 'n_fallbacks': 44}}, @@ -294,6 +318,28 @@ def configure(lib): lib.transcribe_device_get_info.argtypes = [_c.c_void_p, _c.POINTER(transcribe_device_info)] lib.transcribe_device_info_init.restype = None lib.transcribe_device_info_init.argtypes = [_c.POINTER(transcribe_device_info)] + lib.transcribe_diarize_get_info.restype = _c.c_int + lib.transcribe_diarize_get_info.argtypes = [_c.c_void_p, _c.POINTER(transcribe_diarize_info)] + lib.transcribe_diarize_get_segment.restype = _c.c_int + lib.transcribe_diarize_get_segment.argtypes = [_c.c_void_p, _c.c_int, _c.POINTER(transcribe_speaker_segment)] + lib.transcribe_diarize_get_timings.restype = _c.c_int + lib.transcribe_diarize_get_timings.argtypes = [_c.c_void_p, _c.POINTER(transcribe_timings)] + lib.transcribe_diarize_info_init.restype = None + lib.transcribe_diarize_info_init.argtypes = [_c.POINTER(transcribe_diarize_info)] + lib.transcribe_diarize_n_segments.restype = _c.c_int + lib.transcribe_diarize_n_segments.argtypes = [_c.c_void_p] + lib.transcribe_diarize_params_init.restype = None + lib.transcribe_diarize_params_init.argtypes = [_c.POINTER(transcribe_diarize_params)] + lib.transcribe_diarize_run.restype = _c.c_int + lib.transcribe_diarize_run.argtypes = [_c.c_void_p, _c.POINTER(_c.c_float), _c.c_int, _c.POINTER(transcribe_diarize_params)] + lib.transcribe_diarize_session_free.restype = None + lib.transcribe_diarize_session_free.argtypes = [_c.c_void_p] + lib.transcribe_diarize_session_init.restype = _c.c_int + lib.transcribe_diarize_session_init.argtypes = [_c.c_void_p, _c.POINTER(transcribe_diarize_session_params), _c.POINTER(_c.c_void_p)] + lib.transcribe_diarize_session_params_init.restype = None + lib.transcribe_diarize_session_params_init.argtypes = [_c.POINTER(transcribe_diarize_session_params)] + lib.transcribe_diarize_set_abort_callback.restype = None + lib.transcribe_diarize_set_abort_callback.argtypes = [_c.c_void_p, _c.CFUNCTYPE(_c.c_bool, _c.c_void_p), _c.c_void_p] lib.transcribe_ext_check.restype = _c.c_int lib.transcribe_ext_check.argtypes = [_c.POINTER(transcribe_ext), _c.c_uint32, _c.c_uint64] lib.transcribe_full_text.restype = _c.c_char_p @@ -388,6 +434,8 @@ def configure(lib): lib.transcribe_session_params_init.argtypes = [_c.POINTER(transcribe_session_params)] lib.transcribe_set_abort_callback.restype = None lib.transcribe_set_abort_callback.argtypes = [_c.c_void_p, _c.CFUNCTYPE(_c.c_bool, _c.c_void_p), _c.c_void_p] + lib.transcribe_sortformer_diarize_ext_init.restype = None + lib.transcribe_sortformer_diarize_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_diarize_ext)] lib.transcribe_sortformer_stream_ext_init.restype = None lib.transcribe_sortformer_stream_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_stream_ext)] lib.transcribe_speaker_segment_init.restype = None diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index 7d009856f..522cf1871 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 1537ea60b442dd53 +// Pinned to include/transcribe.abihash = c2979c7e891982ee /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "1537ea60b442dd53"; +pub const PUBLIC_HEADER_HASH: &str = "c2979c7e891982ee"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -13,6 +13,7 @@ pub const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM: u32 = 1414746957; pub const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM: u32 = 1414744912; pub const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM: u32 = 1396853584; pub const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM: u32 = 1414743635; +pub const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE: u32 = 1380206163; pub const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM: u32 = 1414746710; pub const TRANSCRIBE_EXT_KIND_WHISPER_RUN: u32 = 1314015319; impl transcribe_status { @@ -67,6 +68,10 @@ impl transcribe_abi_struct { pub const TRANSCRIBE_ABI_EXT: transcribe_abi_struct = transcribe_abi_struct(12); pub const TRANSCRIBE_ABI_DEVICE_INFO: transcribe_abi_struct = transcribe_abi_struct(13); pub const TRANSCRIBE_ABI_SPEAKER_SEGMENT: transcribe_abi_struct = transcribe_abi_struct(14); + pub const TRANSCRIBE_ABI_DIARIZE_INFO: transcribe_abi_struct = transcribe_abi_struct(15); + pub const TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: transcribe_abi_struct = + transcribe_abi_struct(16); + pub const TRANSCRIBE_ABI_DIARIZE_PARAMS: transcribe_abi_struct = transcribe_abi_struct(17); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -184,6 +189,7 @@ unsafe extern "C" { impl transcribe_ext_slot { pub const TRANSCRIBE_EXT_SLOT_RUN: transcribe_ext_slot = transcribe_ext_slot(0); pub const TRANSCRIBE_EXT_SLOT_STREAM: transcribe_ext_slot = transcribe_ext_slot(1); + pub const TRANSCRIBE_EXT_SLOT_DIARIZE_RUN: transcribe_ext_slot = transcribe_ext_slot(2); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -1121,6 +1127,122 @@ unsafe extern "C" { } #[repr(C)] #[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_session { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_info { + pub struct_size: u64, + pub sample_rate: i32, + pub max_speakers: i32, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_info"][::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_info"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_info::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_info, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_info::sample_rate"] + [::std::mem::offset_of!(transcribe_diarize_info, sample_rate) - 8usize]; + ["Offset of field: transcribe_diarize_info::max_speakers"] + [::std::mem::offset_of!(transcribe_diarize_info, max_speakers) - 12usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_session_params { + pub struct_size: u64, + pub n_threads: i32, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_session_params"] + [::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_session_params"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_session_params::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_session_params, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_session_params::n_threads"] + [::std::mem::offset_of!(transcribe_diarize_session_params, n_threads) - 8usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_params { + pub struct_size: u64, + pub family: *const transcribe_ext, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_params"] + [::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_params"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_params::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_params, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_params::family"] + [::std::mem::offset_of!(transcribe_diarize_params, family) - 8usize]; +}; +unsafe extern "C" { + pub fn transcribe_diarize_info_init(out: *mut transcribe_diarize_info); +} +unsafe extern "C" { + pub fn transcribe_diarize_session_params_init(params: *mut transcribe_diarize_session_params); +} +unsafe extern "C" { + pub fn transcribe_diarize_params_init(params: *mut transcribe_diarize_params); +} +unsafe extern "C" { + pub fn transcribe_diarize_get_info( + model: *const transcribe_model, + out: *mut transcribe_diarize_info, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_session_init( + model: *mut transcribe_model, + params: *const transcribe_diarize_session_params, + out_session: *mut *mut transcribe_diarize_session, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_session_free(session: *mut transcribe_diarize_session); +} +unsafe extern "C" { + pub fn transcribe_diarize_set_abort_callback( + session: *mut transcribe_diarize_session, + cb: transcribe_abort_callback, + user_data: *mut ::std::os::raw::c_void, + ); +} +unsafe extern "C" { + pub fn transcribe_diarize_run( + session: *mut transcribe_diarize_session, + pcm: *const f32, + n_samples: ::std::os::raw::c_int, + params: *const transcribe_diarize_params, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_n_segments( + session: *const transcribe_diarize_session, + ) -> ::std::os::raw::c_int; +} +unsafe extern "C" { + pub fn transcribe_diarize_get_segment( + session: *const transcribe_diarize_session, + i: ::std::os::raw::c_int, + out: *mut transcribe_speaker_segment, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_get_timings( + session: *const transcribe_diarize_session, + out: *mut transcribe_timings, + ) -> transcribe_status; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] pub struct transcribe_moonshine_streaming_stream_ext { pub ext: transcribe_ext, pub min_decode_interval_ms: i32, @@ -1227,6 +1349,26 @@ unsafe extern "C" { } #[repr(C)] #[derive(Debug, Copy, Clone)] +pub struct transcribe_sortformer_diarize_ext { + pub ext: transcribe_ext, + pub preset: transcribe_sortformer_preset, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_sortformer_diarize_ext"] + [::std::mem::size_of::() - 24usize]; + ["Alignment of transcribe_sortformer_diarize_ext"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_sortformer_diarize_ext::ext"] + [::std::mem::offset_of!(transcribe_sortformer_diarize_ext, ext) - 0usize]; + ["Offset of field: transcribe_sortformer_diarize_ext::preset"] + [::std::mem::offset_of!(transcribe_sortformer_diarize_ext, preset) - 16usize]; +}; +unsafe extern "C" { + pub fn transcribe_sortformer_diarize_ext_init(ext: *mut transcribe_sortformer_diarize_ext); +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] pub struct transcribe_voxtral_realtime_stream_ext { pub ext: transcribe_ext, pub num_delay_tokens: i32, diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index 423891c35..32d5f5c13 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "1537ea60b442dd53"; +export const PUBLIC_HEADER_HASH = "c2979c7e891982ee"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -50,6 +50,9 @@ export const TRANSCRIBE_ABI_SESSION_LIMITS = 11; export const TRANSCRIBE_ABI_EXT = 12; export const TRANSCRIBE_ABI_DEVICE_INFO = 13; export const TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14; +export const TRANSCRIBE_ABI_DIARIZE_INFO = 15; +export const TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 16; +export const TRANSCRIBE_ABI_DIARIZE_PARAMS = 17; export const TRANSCRIBE_LOG_LEVEL_NONE = 0; export const TRANSCRIBE_LOG_LEVEL_INFO = 1; export const TRANSCRIBE_LOG_LEVEL_WARN = 2; @@ -78,6 +81,7 @@ export const TRANSCRIBE_DIARIZE_MODE_OFF = 1; export const TRANSCRIBE_DIARIZE_MODE_ON = 2; export const TRANSCRIBE_EXT_SLOT_RUN = 0; export const TRANSCRIBE_EXT_SLOT_STREAM = 1; +export const TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2; export const TRANSCRIBE_BACKEND_AUTO = 0; export const TRANSCRIBE_BACKEND_CPU = 1; export const TRANSCRIBE_BACKEND_METAL = 2; @@ -120,6 +124,7 @@ export const TRANSCRIBE_WHISPER_PROMPT_ALL_SEGMENTS = 1; export const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM = 1414746957; export const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584; export const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912; +export const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163; export const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635; export const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710; export const TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319; @@ -141,10 +146,14 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_word': { size: 48, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'seg_index': 24, 'first_token': 28, 'n_tokens': 32, 'text': 40} }, 'transcribe_token': { size: 48, align: 8, offsets: {'struct_size': 0, 'id': 8, 'p': 12, 't0_ms': 16, 't1_ms': 24, 'seg_index': 32, 'word_index': 36, 'text': 40} }, 'transcribe_speaker_segment': { size: 32, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'speaker_id': 24, 'p': 28} }, + 'transcribe_diarize_info': { size: 16, align: 8, offsets: {'struct_size': 0, 'sample_rate': 8, 'max_speakers': 12} }, + 'transcribe_diarize_session_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'n_threads': 8} }, + 'transcribe_diarize_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'family': 8} }, 'transcribe_moonshine_streaming_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'min_decode_interval_ms': 16} }, 'transcribe_parakeet_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'att_context_right': 16} }, 'transcribe_parakeet_buffered_stream_ext': { size: 32, align: 8, offsets: {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24} }, 'transcribe_sortformer_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, + 'transcribe_sortformer_diarize_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, 'transcribe_voxtral_realtime_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20} }, 'transcribe_whisper_run_ext': { size: 80, align: 8, offsets: {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76} }, 'transcribe_whisper_chunk_trace': { size: 48, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'temperature_used': 24, 'compression_ratio': 28, 'avg_logprob': 32, 'no_speech_prob': 36, 'no_speech_triggered': 40, 'n_fallbacks': 44} }, @@ -166,6 +175,9 @@ export const ABI_STRUCT_IDS: Record = { 'transcribe_word': 7, 'transcribe_token': 8, 'transcribe_speaker_segment': 14, + 'transcribe_diarize_info': 15, + 'transcribe_diarize_session_params': 16, + 'transcribe_diarize_params': 17, }; // Build koffi struct types; returns a name -> koffi.IKoffiCType map. @@ -186,10 +198,14 @@ export function defineTypes(koffi: any): Record { T['transcribe_word'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', seg_index: 'int', first_token: 'int', n_tokens: 'int', text: 'char *' }); T['transcribe_token'] = koffi.struct({ struct_size: 'uint64_t', id: 'int', p: 'float', t0_ms: 'int64_t', t1_ms: 'int64_t', seg_index: 'int', word_index: 'int', text: 'char *' }); T['transcribe_speaker_segment'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', speaker_id: 'int32_t', p: 'float' }); + T['transcribe_diarize_info'] = koffi.struct({ struct_size: 'uint64_t', sample_rate: 'int32_t', max_speakers: 'int32_t' }); + T['transcribe_diarize_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int32_t' }); + T['transcribe_diarize_params'] = koffi.struct({ struct_size: 'uint64_t', family: 'void *' }); T['transcribe_moonshine_streaming_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], min_decode_interval_ms: 'int32_t' }); T['transcribe_parakeet_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], att_context_right: 'int32_t' }); T['transcribe_parakeet_buffered_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], left_ms: 'int32_t', chunk_ms: 'int32_t', right_ms: 'int32_t' }); T['transcribe_sortformer_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); + T['transcribe_sortformer_diarize_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); T['transcribe_voxtral_realtime_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], num_delay_tokens: 'int32_t', min_decode_interval_ms: 'int32_t' }); T['transcribe_whisper_run_ext'] = koffi.struct({ ext: T['transcribe_ext'], initial_prompt: 'char *', prompt_tokens: 'void *', n_prompt_tokens: 'size_t', prompt_condition: 'int', condition_on_prev_tokens: 'bool', max_prev_context_tokens: 'int32_t', temperature: 'float', temperature_inc: 'float', compression_ratio_thold: 'float', logprob_thold: 'float', no_speech_thold: 'float', seed: 'uint32_t', max_initial_timestamp: 'float' }); T['transcribe_whisper_chunk_trace'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', temperature_used: 'float', compression_ratio: 'float', avg_logprob: 'float', no_speech_prob: 'float', no_speech_triggered: 'bool', n_fallbacks: 'int32_t' }); @@ -223,6 +239,17 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_device_get': { ret: 'transcribe_device_t', args: ['int'] }, 'transcribe_device_get_info': { ret: 'transcribe_status', args: ['transcribe_device_t', 'struct transcribe_device_info *'] }, 'transcribe_device_info_init': { ret: 'void', args: ['struct transcribe_device_info *'] }, + 'transcribe_diarize_get_info': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_diarize_info *'] }, + 'transcribe_diarize_get_segment': { ret: 'transcribe_status', args: ['const struct transcribe_diarize_session *', 'int', 'struct transcribe_speaker_segment *'] }, + 'transcribe_diarize_get_timings': { ret: 'transcribe_status', args: ['const struct transcribe_diarize_session *', 'struct transcribe_timings *'] }, + 'transcribe_diarize_info_init': { ret: 'void', args: ['struct transcribe_diarize_info *'] }, + 'transcribe_diarize_n_segments': { ret: 'int', args: ['const struct transcribe_diarize_session *'] }, + 'transcribe_diarize_params_init': { ret: 'void', args: ['struct transcribe_diarize_params *'] }, + 'transcribe_diarize_run': { ret: 'transcribe_status', args: ['struct transcribe_diarize_session *', 'const float *', 'int', 'const struct transcribe_diarize_params *'] }, + 'transcribe_diarize_session_free': { ret: 'void', args: ['struct transcribe_diarize_session *'] }, + 'transcribe_diarize_session_init': { ret: 'transcribe_status', args: ['struct transcribe_model *', 'const struct transcribe_diarize_session_params *', 'struct transcribe_diarize_session **'] }, + 'transcribe_diarize_session_params_init': { ret: 'void', args: ['struct transcribe_diarize_session_params *'] }, + 'transcribe_diarize_set_abort_callback': { ret: 'void', args: ['struct transcribe_diarize_session *', 'transcribe_abort_callback', 'void *'] }, 'transcribe_ext_check': { ret: 'transcribe_status', args: ['const struct transcribe_ext *', 'uint32_t', 'uint64_t'] }, 'transcribe_full_text': { ret: 'const char *', args: ['const struct transcribe_session *'] }, 'transcribe_get_model': { ret: 'const struct transcribe_model *', args: ['const struct transcribe_session *'] }, @@ -270,6 +297,7 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_session_limits_init': { ret: 'void', args: ['struct transcribe_session_limits *'] }, 'transcribe_session_params_init': { ret: 'void', args: ['struct transcribe_session_params *'] }, 'transcribe_set_abort_callback': { ret: 'void', args: ['struct transcribe_session *', 'transcribe_abort_callback', 'void *'] }, + 'transcribe_sortformer_diarize_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_diarize_ext *'] }, 'transcribe_sortformer_stream_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_stream_ext *'] }, 'transcribe_speaker_segment_init': { ret: 'void', args: ['struct transcribe_speaker_segment *'] }, 'transcribe_status_string': { ret: 'const char *', args: ['int'] }, diff --git a/include/transcribe.abihash b/include/transcribe.abihash index 1449a2276..f3f29e54d 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -1537ea60b442dd53 +c2979c7e891982ee From 1f3e7bc8e56d9f5527fdeeccbeea39b9ee21cf95 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:16:52 +0800 Subject: [PATCH 25/51] roles: catalog role field, validate.py on the diarize API, docs - catalog: optional "role" (asr | diarize, absent = asr); Sortformer is "diarize". render.py's family-index groups README tables by role (role=diarize replaces transcribe=false). format / check / render / db-mapping checks pass. - validate.py: Sortformer's C++ dump runs with --role diarize, and a diarizer run (speaker segments, no text line) no longer needs a transcript. The diarize-API dumps are byte-identical to main's ASR-path dumps on the oracle mix. (The sortformer compare is 1/6 within tolerance against a freshly generated reference on both main and this branch: reference drift, not this change.) - docs/roles.md: what a role is and the rules every role follows. - porting skills: intake records the role; acceptance scores DER for a DIARIZE model. --- .claude/skills/porting-1-intake/SKILL.md | 6 ++++ .claude/skills/porting-7-wer/SKILL.md | 2 +- README.md | 2 +- catalog/_schema.json | 4 +++ .../diar_streaming_sortformer_4spk-v2.1.json | 1 + docs/roles.md | 33 +++++++++++++++++++ scripts/catalog/render.py | 8 ++--- scripts/validate.py | 4 +++ 8 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 docs/roles.md diff --git a/.claude/skills/porting-1-intake/SKILL.md b/.claude/skills/porting-1-intake/SKILL.md index 0b7a5f987..0da7b7a89 100644 --- a/.claude/skills/porting-1-intake/SKILL.md +++ b/.claude/skills/porting-1-intake/SKILL.md @@ -55,6 +55,12 @@ field semantics. 2. **Architecture pattern.** One of `encoder-transducer`, `encoder-decoder`, `audio-llm`, `encoder-ctc`. The script's `config.architecture_candidates` is a heuristic starting point. If it doesn't fit, propose a new pattern and have the user accept it based on your research of the architecture. + **Role.** ASR (the product is a transcript) or DIARIZE (the product is + who spoke when; see `docs/roles.md`). A DIARIZE port implements + `DiarizeOps` (`src/transcribe-diarize.h`) instead of the ASR hooks, sets + `"role": "diarize"` in its catalog record, and is accepted on DER + (`scripts/diar/`), not WER. + 3. **Acceptance dataset.** Default: LibriSpeech test-clean. Capture any publisher-reported score in `upstream_benchmarks` when available for context, but downstream gates use the measured Oracle reference diff --git a/.claude/skills/porting-7-wer/SKILL.md b/.claude/skills/porting-7-wer/SKILL.md index 3b27c3111..4005e5e55 100644 --- a/.claude/skills/porting-7-wer/SKILL.md +++ b/.claude/skills/porting-7-wer/SKILL.md @@ -42,7 +42,7 @@ WER progress: ### Step 1: Acceptance manifest (execute or ask-point) -Read the acceptance dataset and metric from `upstream_benchmarks[0].{dataset, metric}`. Slugify: lowercase, spaces → hyphens. `"LibriSpeech test-clean"` → `librispeech-test-clean`. Resolve to `$MANIFEST`; every later step uses `$MANIFEST`, never a reconstructed path. Confirm `metric` is `wer` or `cer`; anything else is out of scope. Do not use any publisher-reported score for pass/fail; the measured Oracle reference score is the gate target. +Read the acceptance dataset and metric from `upstream_benchmarks[0].{dataset, metric}`. Slugify: lowercase, spaces → hyphens. `"LibriSpeech test-clean"` → `librispeech-test-clean`. Resolve to `$MANIFEST`; every later step uses `$MANIFEST`, never a reconstructed path. Confirm `metric` is `wer` or `cer` (`der` for a DIARIZE-role model: score with `scripts/diar/` instead; anything else is out of scope). Do not use any publisher-reported score for pass/fail; the measured Oracle reference score is the gate target. If the intake's dataset is not covered by `scripts/wer/ingest.py`, extend that script before running this step. diff --git a/README.md b/README.md index 19c924c49..1e9309777 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ C/C++ speech-to-text inference library. Runs diverse STT model families via [GGU **Speaker diarization models** (no transcription; verified by DER/JER rather than WER): - + | Family | Variants | Available capabilities | Docs | | --- | --- | --- | --- | | Streaming Sortformer Diarizer 4spk v2.1 | `diar_streaming_sortformer_4spk-v2.1` | diarize, streaming | [docs/models/diar_streaming_sortformer_4spk-v2.1.md](docs/models/diar_streaming_sortformer_4spk-v2.1.md) | diff --git a/catalog/_schema.json b/catalog/_schema.json index 697287654..219e539b4 100644 --- a/catalog/_schema.json +++ b/catalog/_schema.json @@ -24,6 +24,10 @@ "type": "string", "description": "Loader family (the GGUF's general.architecture)" }, + "role": { + "enum": ["asr", "diarize"], + "description": "The role the model is published for (transcribe_model_roles). Absent means asr." + }, "display_name": { "type": "string" }, diff --git a/catalog/diar_streaming_sortformer_4spk-v2.1.json b/catalog/diar_streaming_sortformer_4spk-v2.1.json index 3e243a663..845a037e0 100644 --- a/catalog/diar_streaming_sortformer_4spk-v2.1.json +++ b/catalog/diar_streaming_sortformer_4spk-v2.1.json @@ -2,6 +2,7 @@ "schema": "transcribe-catalog-v1", "variant": "diar_streaming_sortformer_4spk-v2.1", "family": "sortformer", + "role": "diarize", "display_name": "diar_streaming_sortformer_4spk-v2.1", "params": 117711368, "license": { diff --git a/docs/roles.md b/docs/roles.md new file mode 100644 index 000000000..4358b61f8 --- /dev/null +++ b/docs/roles.md @@ -0,0 +1,33 @@ +# Roles + +A loaded model serves one or more **roles**, the kinds of work it can do. +`transcribe_model_roles(model)` returns them as a bitmask of +`transcribe_role`; an entry point that needs a role the model lacks returns +`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`. + +| Role | API | Session | Product | +|---|---|---|---| +| `TRANSCRIBE_ROLE_ASR` | `include/transcribe.h` | `transcribe_session` | text, timestamps, speaker-attributed segments | +| `TRANSCRIBE_ROLE_DIARIZE` | `include/transcribe/diarize.h` | `transcribe_diarize_session` | who spoke when | + +Every role follows the same rules: + +- **One model, any role.** Load with `transcribe_model_load_file` and pick + the backend per model with `transcribe_model_load_params`. Running a + diarizer on CPU and ASR on the GPU is two loads. +- **Sessions.** Each role has its own session type created from a model. + A session is used by one thread at a time; the model outlives its + sessions; at most one compute is in flight per model across all sessions + of all roles. Different models compute in parallel. +- **Composition is by handle**, never by path in params. +- **Input.** 16 kHz mono float32 PCM; NaN / Inf is `INVALID_ARG`. +- **Results** are copied out and replaced by the next run; malformed input + is rejected before the previous result is touched. + +ASR models that attribute speakers inside a transcript (granite, moss, +multitalker parakeet) keep `transcribe_run_params::diarize` and stay ASR. +The DIARIZE role is for models whose product is speaker turns +(Sortformer). + +Family extensions name the entry point they apply to through their slot; +see `docs/extension-kinds.md`. diff --git a/scripts/catalog/render.py b/scripts/catalog/render.py index 44d3947f6..3a38d4f47 100755 --- a/scripts/catalog/render.py +++ b/scripts/catalog/render.py @@ -369,12 +369,12 @@ def doc_for(record: dict) -> pathlib.Path | None: def block_family_index(records: dict[str, dict], attrs: dict[str, str]) -> list[str]: """The root README's supported-models table: one row per documentation - page, listing the variants it covers. `transcribe=false` selects the - models that only diarize.""" - want = as_bool(attrs.get("transcribe"), True) + page, listing the variants it covers, for one role (`role=`, default + asr).""" + want = attrs.get("role", "asr") groups: dict[str, dict] = {} for variant, record in records.items(): - if bool(record.get("capabilities", {}).get("transcribe", {}).get("supported")) != want: + if record.get("role", "asr") != want: continue doc = doc_for(record) if doc is not None: diff --git a/scripts/validate.py b/scripts/validate.py index 43af1e3fb..85d1a4756 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -501,6 +501,8 @@ def cmd_cpp(args: argparse.Namespace) -> int: cmd += ["--language", language] if args.family == "whisper": cmd += ["--timestamps", "none"] + if args.family == "sortformer": + cmd += ["--role", "diarize"] if args.family in ("sensevoice", "parakeet"): # The reference dumper emits the raw token stream including # control / language tags (sensevoice: language / event / @@ -548,6 +550,8 @@ def cmd_cpp(args: argparse.Namespace) -> int: f"with exit code {result.returncode}" ) transcript = parse_cli_transcript(result.stdout or "") + if transcript is None and "speaker segments:" in (result.stdout or ""): + continue # a diarizer has no transcript if transcript is None: raise SystemExit( f"error: cpp dump [{args.family}/{case_name}] did not emit a transcript line" From 4a2c82bb227ecaba5fc79fbd002f5a273d0af29f Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:19:01 +0800 Subject: [PATCH 26/51] rust: DIARIZE role wrappers Model::roles() -> Roles (contains(Role), bits()). Model::diarize_info(), Model::diarize_session()/_with(DiarizeSessionOptions), and DiarizeSession with run(pcm, &DiarizeOptions) -> Vec, timings(), and set/clear_cancel_token. DiarizeOptions::family takes DiarizeExtension::Sortformer(SortformerDiarizeOptions { preset }), which materializes the SFDR ext on the new ExtSlot::DiarizeRun. The compute lock helper moves to ModelInner::with_compute; Session's with_compute delegates to it with its handle, and DiarizeSession::run uses it directly, so diarize runs serialize with every session of the model and are refused with Busy while a stream holds the lease. Breaking (0.4.0): Model::capabilities() returns Result and reports the C status (UnsupportedRole on a model without ASR) instead of zeroed defaults. ExtSlot gains DiarizeRun. tests: Busy before native (model-free unit test); Sortformer roles, info, ext acceptance, diarize run parity with the ASR-path speaker turns at DEFAULT and LOW_LATENCY, bad input, cancel, model kept alive; whisper refuses diarize with UnsupportedRole. --- bindings/rust/transcribe-cpp/README.md | 23 +++ .../rust/transcribe-cpp/examples/streaming.rs | 2 +- .../examples/transcribe-file.rs | 2 +- bindings/rust/transcribe-cpp/src/diarize.rs | 193 ++++++++++++++++++ bindings/rust/transcribe-cpp/src/family.rs | 53 ++++- bindings/rust/transcribe-cpp/src/lib.rs | 15 +- bindings/rust/transcribe-cpp/src/model.rs | 101 +++++++-- bindings/rust/transcribe-cpp/src/session.rs | 29 +-- bindings/rust/transcribe-cpp/src/types.rs | 35 ++++ .../rust/transcribe-cpp/tests/common/mod.rs | 17 ++ bindings/rust/transcribe-cpp/tests/diarize.rs | 134 ++++++++++++ .../rust/transcribe-cpp/tests/no_model.rs | 3 +- bindings/rust/transcribe-cpp/tests/serde.rs | 20 +- .../rust/transcribe-cpp/tests/streaming.rs | 2 +- .../rust/transcribe-cpp/tests/transcribe.rs | 6 +- 15 files changed, 572 insertions(+), 63 deletions(-) create mode 100644 bindings/rust/transcribe-cpp/src/diarize.rs create mode 100644 bindings/rust/transcribe-cpp/tests/diarize.rs diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index c8f037a63..180a3288f 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -76,6 +76,29 @@ let transcript = stream.snapshot(); // language, segments, words, tokens, timing # Ok::<(), transcribe_cpp::Error>(()) ``` +### Diarization (who spoke when) + +`model.roles()` reports what a model can do. A model with `Role::Diarize` +(e.g. Sortformer) opens a `DiarizeSession`, whose `run` returns speaker turns +(`SpeakerSegment`: `t0_ms`, `t1_ms`, 1-based `speaker_id`). ASR-only calls such +as `capabilities()` return `Error::UnsupportedRole` on a model without +`Role::Asr`, and the diarize calls do the same without `Role::Diarize`. + +```rust +use transcribe_cpp::{DiarizeOptions, Model, Role}; +let model = Model::load("diarizer.gguf")?; +assert!(model.roles().contains(Role::Diarize)); +let mut diarize = model.diarize_session()?; +for turn in diarize.run(&pcm, &DiarizeOptions::default())? { + println!("speaker {}: {}..{} ms", turn.speaker_id, turn.t0_ms, turn.t1_ms); +} +# Ok::<(), transcribe_cpp::Error>(()) +``` + +`DiarizeOptions::family` takes `DiarizeExtension::Sortformer` with a +`SortformerPreset`. Diarize runs share the model-wide compute lock with ASR +sessions and accept a `CancelToken`. + Runnable examples: ```sh diff --git a/bindings/rust/transcribe-cpp/examples/streaming.rs b/bindings/rust/transcribe-cpp/examples/streaming.rs index 2afb3f7cc..4c2bbc0e6 100644 --- a/bindings/rust/transcribe-cpp/examples/streaming.rs +++ b/bindings/rust/transcribe-cpp/examples/streaming.rs @@ -27,7 +27,7 @@ fn main() -> Result<(), Box> { let pcm = common::load_wav(&audio_path); let model = Model::load(&model_path)?; - if !model.capabilities().supports_streaming { + if !model.capabilities()?.supports_streaming { // Not an error — the wrong-model case, stated plainly (a whisper model // would land here). Exit 0 so CI's no-canary path stays green. eprintln!("{} does not support streaming", model.arch()); diff --git a/bindings/rust/transcribe-cpp/examples/transcribe-file.rs b/bindings/rust/transcribe-cpp/examples/transcribe-file.rs index 0def25b0f..f5974ea4e 100644 --- a/bindings/rust/transcribe-cpp/examples/transcribe-file.rs +++ b/bindings/rust/transcribe-cpp/examples/transcribe-file.rs @@ -36,7 +36,7 @@ fn main() -> Result<(), Box> { let pcm = common::load_wav(&audio_path); let model = Model::load(&model_path)?; - let caps = model.capabilities(); + let caps = model.capabilities()?; println!( "model: {} on {} | max_ts={:?}", model.arch(), diff --git a/bindings/rust/transcribe-cpp/src/diarize.rs b/bindings/rust/transcribe-cpp/src/diarize.rs new file mode 100644 index 000000000..c79a3b8a1 --- /dev/null +++ b/bindings/rust/transcribe-cpp/src/diarize.rs @@ -0,0 +1,193 @@ +//! [`DiarizeSession`] — the DIARIZE role: who spoke when. +//! +//! For models whose product is speaker turns rather than text +//! ([`Model::roles`] contains [`Role::Diarize`](crate::Role)). Open one with +//! [`Model::diarize_session`]. Same threading and lifetime rules as +//! [`Session`](crate::Session): `Send` but not `Sync`, keeps its model alive, +//! and its run shares the model-wide compute lock with every other session of +//! the model. + +use std::os::raw::c_void; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +use transcribe_cpp_sys as sys; + +use crate::cancel::{abort_trampoline, CancelToken}; +use crate::error::{check, Result}; +use crate::family::DiarizeExtension; +use crate::model::{Model, ModelInner}; +use crate::result::{SpeakerSegment, Timings}; +use crate::session::clamp_len; + +/// Static facts about a diarization model ([`Model::diarize_info`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct DiarizeInfo { + /// Input PCM rate (16000). + pub sample_rate: i32, + /// `speaker_id` is in `[1, max_speakers]`. + pub max_speakers: i32, +} + +/// Options for creating a diarization session. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default) +)] +pub struct DiarizeSessionOptions { + /// CPU threads for ops that run on CPU; 0 = library default. + pub n_threads: i32, +} + +/// Per-run diarization parameters. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default) +)] +pub struct DiarizeOptions { + /// Optional family-specific extension (e.g. the Sortformer preset). + pub family: Option, +} + +/// A diarization session. +pub struct DiarizeSession { + ptr: *mut sys::transcribe_diarize_session, + // Keeps the native model alive and carries the per-model compute lock. + model: Arc, + // Keeps the abort callback's userdata alive while installed. + cancel: Option>, +} + +impl std::fmt::Debug for DiarizeSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DiarizeSession").finish_non_exhaustive() + } +} + +// SAFETY: as for `Session` — `&mut self` on the mutating calls keeps use to +// one thread at a time; deliberately NOT Sync. +unsafe impl Send for DiarizeSession {} + +impl Drop for DiarizeSession { + fn drop(&mut self) { + unsafe { sys::transcribe_diarize_session_free(self.ptr) }; + } +} + +impl DiarizeSession { + pub(crate) fn new(model: &Model, options: &DiarizeSessionOptions) -> Result { + let mut params: sys::transcribe_diarize_session_params = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_session_params_init(&mut params) }; + params.n_threads = options.n_threads; + + let mut out: *mut sys::transcribe_diarize_session = std::ptr::null_mut(); + let status = + unsafe { sys::transcribe_diarize_session_init(model.inner.ptr, ¶ms, &mut out) }; + check(status, "diarize session init")?; + debug_assert!(!out.is_null()); + + Ok(DiarizeSession { + ptr: out, + model: Arc::clone(&model.inner), + cancel: None, + }) + } + + /// Install a [`CancelToken`] so an in-flight run can be aborted from + /// another thread (the run then returns [`Error::Aborted`](crate::Error)). + /// Replaces any previously installed token. + pub fn set_cancel_token(&mut self, token: &CancelToken) { + let flag = Arc::clone(&token.flag); + let userdata = Arc::as_ptr(&flag) as *mut c_void; + unsafe { + sys::transcribe_diarize_set_abort_callback(self.ptr, Some(abort_trampoline), userdata) + }; + self.cancel = Some(flag); + } + + /// Remove any installed cancel token. + pub fn clear_cancel_token(&mut self) { + unsafe { sys::transcribe_diarize_set_abort_callback(self.ptr, None, std::ptr::null_mut()) }; + self.cancel = None; + } + + /// Diarize one recording of 16 kHz mono float32 PCM. Returns the speaker + /// segments, grouped by speaker and time-ordered within a speaker; + /// segments of different speakers may overlap. + pub fn run(&mut self, pcm: &[f32], options: &DiarizeOptions) -> Result> { + let n = clamp_len(pcm.len())?; + let family = options.family.as_ref().map(DiarizeExtension::materialize); + let mut params: sys::transcribe_diarize_params = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_params_init(&mut params) }; + params.family = family.as_ref().map_or(std::ptr::null(), |f| f.ext_ptr()); + + let ptr = self.ptr; + let status = self.model.with_compute( + Some("a stream is active on this model; finish or drop it before diarize run()"), + |_| unsafe { sys::transcribe_diarize_run(ptr, pcm.as_ptr(), n, ¶ms) }, + )?; + check(status, "diarize run")?; + + let count = unsafe { sys::transcribe_diarize_n_segments(self.ptr) }; + Ok((0..count) + .map(|i| { + let mut raw: sys::transcribe_speaker_segment = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_speaker_segment_init(&mut raw) }; + let _ = unsafe { sys::transcribe_diarize_get_segment(self.ptr, i, &mut raw) }; + SpeakerSegment::from_raw(&raw) + }) + .collect()) + } + + /// Model load time plus the last run's stage timings (`decode_ms` is 0). + pub fn timings(&self) -> Timings { + let mut raw: sys::transcribe_timings = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_timings_init(&mut raw) }; + let _ = unsafe { sys::transcribe_diarize_get_timings(self.ptr, &mut raw) }; + Timings::from_raw(&raw) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Error; + use std::sync::Mutex; + + /// A session over null native handles (both frees are NULL no-ops): the + /// Busy check runs under the lock before any native call. + fn null_session(stream_holds_lease: bool) -> DiarizeSession { + DiarizeSession { + ptr: std::ptr::null_mut(), + model: Arc::new(ModelInner { + ptr: std::ptr::null_mut(), + compute_lock: Mutex::new(stream_holds_lease), + }), + cancel: None, + } + } + + #[test] + fn run_is_busy_while_a_stream_holds_the_lease() { + let err = null_session(true) + .run(&[0.0; 160], &DiarizeOptions::default()) + .unwrap_err(); + let Error::Busy(msg) = err else { + panic!("expected Busy, got {err:?}"); + }; + assert_eq!( + msg, + "a stream is active on this model; finish or drop it before diarize run()" + ); + // With the lease free the same call reaches native (NULL session). + let err = null_session(false) + .run(&[0.0; 160], &DiarizeOptions::default()) + .unwrap_err(); + assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); + } +} diff --git a/bindings/rust/transcribe-cpp/src/family.rs b/bindings/rust/transcribe-cpp/src/family.rs index 33907ac05..9348a04e8 100644 --- a/bindings/rust/transcribe-cpp/src/family.rs +++ b/bindings/rust/transcribe-cpp/src/family.rs @@ -5,7 +5,8 @@ //! every field is an `Option`, and only the fields you set override the //! defaults the C `*_init()` stamps. A [`RunExtension`] attaches to //! [`RunOptions`](crate::RunOptions); a [`StreamExtension`] attaches to -//! [`StreamOptions`](crate::StreamOptions). +//! [`StreamOptions`](crate::StreamOptions); a [`DiarizeExtension`] attaches to +//! [`DiarizeOptions`](crate::DiarizeOptions). //! //! Probe [`Model::accepts_ext`](crate::Model::accepts_ext) to learn whether a //! loaded model accepts a given kind on a slot; an unaccepted extension is @@ -135,6 +136,56 @@ pub struct SortformerStreamOptions { pub preset: Option, } +/// Sortformer diarize-extension knobs (diarize-run slot). `None` keeps the +/// family default (the GGUF-shipped cfg). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default) +)] +pub struct SortformerDiarizeOptions { + pub preset: Option, +} + +/// A family extension for the diarize-run slot +/// ([`DiarizeSession::run`](crate::DiarizeSession::run)). +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum DiarizeExtension { + Sortformer(SortformerDiarizeOptions), +} + +/// Owns a materialized diarize-slot C extension struct. +pub(crate) enum DiarizeExtRaw { + Sortformer(Box), +} + +impl DiarizeExtRaw { + pub(crate) fn ext_ptr(&self) -> *const sys::transcribe_ext { + match self { + DiarizeExtRaw::Sortformer(e) => { + (&**e) as *const sys::transcribe_sortformer_diarize_ext + as *const sys::transcribe_ext + } + } + } +} + +impl DiarizeExtension { + pub(crate) fn materialize(&self) -> DiarizeExtRaw { + match self { + DiarizeExtension::Sortformer(o) => { + let mut e: sys::transcribe_sortformer_diarize_ext = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_sortformer_diarize_ext_init(&mut e) }; + set(&mut e.preset, o.preset.map(SortformerPreset::to_sys)); + DiarizeExtRaw::Sortformer(Box::new(e)) + } + } + } +} + /// A family extension for the run slot (offline `run`/`run_batch`). #[derive(Debug, Clone, PartialEq)] #[non_exhaustive] diff --git a/bindings/rust/transcribe-cpp/src/lib.rs b/bindings/rust/transcribe-cpp/src/lib.rs index d2dc48c76..5f82cd576 100644 --- a/bindings/rust/transcribe-cpp/src/lib.rs +++ b/bindings/rust/transcribe-cpp/src/lib.rs @@ -27,7 +27,8 @@ //! - [`Session`] is `Send` but not `Sync`; its mutating calls take `&mut self`, //! so the type system enforces one-call-at-a-time. //! - **0.x concurrency:** the C library permits at most one in-flight -//! `run`/stream across all sessions of a model. This crate enforces that with +//! `run`/stream across all sessions of a model, [`DiarizeSession`]s +//! included. This crate enforces that with //! a per-model mutex (held only for the native compute call), so concurrent //! calls from many sessions queue rather than race. For real parallelism, use //! one [`Model`] per worker today. @@ -47,6 +48,7 @@ pub use transcribe_cpp_sys as sys; mod backend; mod cancel; +mod diarize; mod error; mod family; mod logging; @@ -62,11 +64,12 @@ pub use backend::{ DeviceType, }; pub use cancel::CancelToken; +pub use diarize::{DiarizeInfo, DiarizeOptions, DiarizeSession, DiarizeSessionOptions}; pub use error::{Error, Result}; pub use family::{ - MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, RunExtension, - SortformerPreset, SortformerStreamOptions, StreamExtension, VoxtralRealtimeStreamOptions, - WhisperRunOptions, + DiarizeExtension, MoonshineStreamingOptions, ParakeetBufferedStreamOptions, + ParakeetStreamOptions, RunExtension, SortformerDiarizeOptions, SortformerPreset, + SortformerStreamOptions, StreamExtension, VoxtralRealtimeStreamOptions, WhisperRunOptions, }; pub use logging::{disable_logging, init_logging}; pub use model::{Capabilities, Model, ModelOptions, SessionLimits, SessionOptions}; @@ -74,8 +77,8 @@ pub use result::{Segment, SpeakerSegment, Timings, Token, Transcript, Word}; pub use session::{RunOptions, Session, Stream}; pub use streaming::{StreamOptions, StreamText, StreamUpdate}; pub use types::{ - AbiStruct, Backend, CommitPolicy, Diarize, ExtSlot, Feature, Itn, KvType, Pnc, StreamState, - Task, TimestampKind, + AbiStruct, Backend, CommitPolicy, Diarize, ExtSlot, Feature, Itn, KvType, Pnc, Role, Roles, + StreamState, Task, TimestampKind, }; pub use version::{ abi_struct_align, abi_struct_size, compiled_version, header_hash, version, version_commit, diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index c971c6385..968d8985b 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -2,17 +2,19 @@ //! //! A `Model` is `Arc`-backed and `Send + Sync`: cloning it is cheap and hands //! out another handle to the same native model. The native model is freed when -//! the last handle AND every [`Session`](crate::Session) derived from it have -//! been dropped — Rust ownership gives the C "model must outlive its sessions" -//! contract (and close-ordering safety) for free, in any drop order. +//! the last handle AND every [`Session`](crate::Session) / +//! [`DiarizeSession`](crate::DiarizeSession) derived from it have been dropped +//! — Rust ownership gives the C "model must outlive its sessions" contract (and +//! close-ordering safety) for free, in any drop order. //! //! The `Arc` also carries the per-model compute lock that enforces the C //! library's 0.x concurrency limitation: at most one `run` / `run_batch` / -//! active stream may be in flight across ALL sessions of a model. One-shot -//! runs/batches queue on the lock (serialized); an *active stream* spans many -//! native calls (`begin`→`feed`*→`finalize`), so the lock also carries a flag -//! marking a stream in flight — any run/batch/stream that would overlap it is -//! refused with [`Error::Busy`](crate::Error) rather than allowed to race. +//! diarize run / active stream may be in flight across ALL sessions of a model. +//! One-shot runs/batches queue on the lock (serialized); an *active stream* +//! spans many native calls (`begin`→`feed`*→`finalize`), so the lock also +//! carries a flag marking a stream in flight — any run/batch/stream that would +//! overlap it is refused with [`Error::Busy`](crate::Error) rather than allowed +//! to race. use std::ffi::CString; use std::path::Path; @@ -21,10 +23,11 @@ use std::sync::{Arc, Mutex}; use transcribe_cpp_sys as sys; use crate::backend::Device; -use crate::error::{check, Result}; +use crate::diarize::{DiarizeInfo, DiarizeSession, DiarizeSessionOptions}; +use crate::error::{check, Error, Result}; use crate::result::owned_str; use crate::session::Session; -use crate::types::{Backend, ExtSlot, Feature, TimestampKind}; +use crate::types::{Backend, ExtSlot, Feature, Roles, TimestampKind}; use crate::version; /// Options for loading a model. @@ -73,11 +76,38 @@ pub(crate) struct ModelInner { /// is `true` while a stream is ACTIVE (from `begin` until the earliest of /// `finalize` / `reset` / the `Stream` being dropped); a held lock plus a /// `true` flag is how `run`/`run_batch`/`stream` detect and refuse an - /// overlapping compute. Only taken through `Session::with_compute`. See - /// the module docs. + /// overlapping compute. Only taken through [`ModelInner::with_compute`]. + /// See the module docs. pub(crate) compute_lock: Mutex, } +impl ModelInner { + /// Run `f` under the model's compute lock. Every call that drives native + /// compute, on any session type of this model (run, batch, diarize run, + /// stream begin/feed/finalize/reset/drop), goes through here. + /// + /// Under the lock, in order: if `refuse_if_streaming` is `Some(msg)` and a + /// stream on this model holds the lease, return `Error::Busy(msg)` without + /// calling `f`; otherwise call `f` with the lease flag (so a stream + /// begin/end can take or release the lease atomically with its native + /// call). Pass `None` from the stream that holds the lease; the call then + /// never fails. The lock is released when this returns, so callers copy + /// results out of their session's own result storage afterwards. + pub(crate) fn with_compute( + &self, + refuse_if_streaming: Option<&str>, + f: impl FnOnce(&mut bool) -> R, + ) -> Result { + let mut lease = self.compute_lock.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(msg) = refuse_if_streaming { + if *lease { + return Err(Error::Busy(msg.into())); + } + } + Ok(f(&mut lease)) + } +} + // SAFETY: the native model is documented thread-safe for query + session // creation; the compute path is serialized by `compute_lock`. The raw pointer // is only ever used behind `&ModelInner`, never mutated through aliasing. @@ -152,13 +182,46 @@ impl Model { Session::new(self, options) } - /// The model's immutable capabilities. - pub fn capabilities(&self) -> Capabilities { + /// Open a diarization session with default options. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Diarize`](crate::Role). + pub fn diarize_session(&self) -> Result { + self.diarize_session_with(&DiarizeSessionOptions::default()) + } + + /// Open a diarization session with explicit options. + pub fn diarize_session_with(&self, options: &DiarizeSessionOptions) -> Result { + DiarizeSession::new(self, options) + } + + /// Static facts about a diarization model. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Diarize`](crate::Role). + pub fn diarize_info(&self) -> Result { + let mut raw: sys::transcribe_diarize_info = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_info_init(&mut raw) }; + check( + unsafe { sys::transcribe_diarize_get_info(self.inner.ptr, &mut raw) }, + "diarize info", + )?; + Ok(DiarizeInfo { + sample_rate: raw.sample_rate, + max_speakers: raw.max_speakers, + }) + } + + /// The roles (kinds of work) this model serves. + pub fn roles(&self) -> Roles { + Roles(unsafe { sys::transcribe_model_roles(self.inner.ptr) }) + } + + /// The model's immutable ASR capabilities. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Asr`](crate::Role). + pub fn capabilities(&self) -> Result { let mut caps: sys::transcribe_capabilities = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_capabilities_init(&mut caps) }; - // A NULL/struct-size fault cannot happen here (we own a valid model - // and an init'd struct), so a non-OK status leaves zeroed defaults. - let _ = unsafe { sys::transcribe_model_get_capabilities(self.inner.ptr, &mut caps) }; + check( + unsafe { sys::transcribe_model_get_capabilities(self.inner.ptr, &mut caps) }, + "capabilities", + )?; let mut languages = Vec::new(); if !caps.languages.is_null() && caps.n_languages > 0 { @@ -181,7 +244,7 @@ impl Model { } } - Capabilities { + Ok(Capabilities { native_sample_rate: caps.native_sample_rate, languages, translate_target_languages, @@ -191,7 +254,7 @@ impl Model { supports_streaming: caps.supports_streaming, supports_spec_decode: caps.supports_spec_decode, max_audio_ms: caps.max_audio_ms, - } + }) } /// Probe a yes/no feature. diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index 2331a1a8a..a44e5d2ab 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -297,34 +297,15 @@ impl Session { }) } - /// Make one native compute call on this session under the model's compute - /// lock. Every call that drives native compute (run, batch, stream - /// begin/feed/finalize/reset/drop) goes through here. - /// - /// Under the lock, in order: if `refuse_if_streaming` is `Some(msg)` and a - /// stream on this model holds the lease, return `Error::Busy(msg)` without - /// calling `f`; otherwise call `f` with this session's native handle and - /// the lease flag (so a stream begin/end can take or release the lease - /// atomically with its native call). Pass `None` from the stream that holds - /// the lease; the call then never fails. The lock is released when this - /// returns, so callers copy results out of the session's own result - /// storage afterwards. + /// [`ModelInner::with_compute`], handing `f` this session's native handle. pub(crate) fn with_compute( &mut self, refuse_if_streaming: Option<&str>, f: impl FnOnce(*mut sys::transcribe_session, &mut bool) -> R, ) -> Result { - let mut lease = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if let Some(msg) = refuse_if_streaming { - if *lease { - return Err(Error::Busy(msg.into())); - } - } - Ok(f(self.ptr, &mut lease)) + let ptr = self.ptr; + self.model + .with_compute(refuse_if_streaming, |lease| f(ptr, lease)) } // --- result materialization (top-level / single accessors) --------------- @@ -520,7 +501,7 @@ fn build_run_params(o: &RunOptions) -> Result { } /// PCM/utterance lengths cross the ABI as `int`; reject anything that overflows. -fn clamp_len(len: usize) -> Result { +pub(crate) fn clamp_len(len: usize) -> Result { i32::try_from(len).map_err(|_| Error::InvalidArgument(format!("length {len} exceeds i32::MAX"))) } diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index af9d95000..867c0a67f 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -352,6 +352,8 @@ pub enum ExtSlot { Run, /// `transcribe_stream_params::family`. Stream, + /// `transcribe_diarize_params::family`. + DiarizeRun, } impl ExtSlot { @@ -360,6 +362,39 @@ impl ExtSlot { match self { ExtSlot::Run => E::TRANSCRIBE_EXT_SLOT_RUN, ExtSlot::Stream => E::TRANSCRIBE_EXT_SLOT_STREAM, + ExtSlot::DiarizeRun => E::TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, } } } + +/// A kind of work a model can do (`transcribe_role`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Role { + /// Transcription: [`Session`](crate::Session). + Asr, + /// Speaker diarization: [`DiarizeSession`](crate::DiarizeSession). + Diarize, +} + +/// The set of [`Role`]s a model serves ([`Model::roles`](crate::Model::roles)). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Roles(pub(crate) u32); + +impl Roles { + /// Whether the model serves `role`. + pub fn contains(self, role: Role) -> bool { + let bit = match role { + Role::Asr => sys::transcribe_role::TRANSCRIBE_ROLE_ASR, + Role::Diarize => sys::transcribe_role::TRANSCRIBE_ROLE_DIARIZE, + }; + self.0 & bit.0 != 0 + } + + /// The raw `transcribe_role` bitmask. + pub fn bits(self) -> u32 { + self.0 + } +} diff --git a/bindings/rust/transcribe-cpp/tests/common/mod.rs b/bindings/rust/transcribe-cpp/tests/common/mod.rs index 3c5ebfa09..326612778 100644 --- a/bindings/rust/transcribe-cpp/tests/common/mod.rs +++ b/bindings/rust/transcribe-cpp/tests/common/mod.rs @@ -111,6 +111,23 @@ pub fn smoke_itn_model() -> Option { ) } +/// Sortformer diarizer (ASR + DIARIZE roles) plus its two-speaker sample, or +/// `None` (with a skip note) when either is absent. +pub fn smoke_sortformer_fixtures(test: &str) -> Option<(PathBuf, Vec)> { + let model = family_model( + "TRANSCRIBE_SMOKE_SORTFORMER_MODEL", + "models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-F32.gguf", + ); + let audio = repo_root().join("samples/sortformer-2spk-mix.wav"); + match model { + Some(m) if audio.is_file() => Some((m, load_wav(&audio))), + _ => { + eprintln!("skip {test}: sortformer model/audio absent (set TRANSCRIBE_SMOKE_SORTFORMER_MODEL)"); + None + } + } +} + /// Both fixtures together; prints a skip note and returns `None` if either is /// missing (so the caller can `return` early — the Rust equivalent of skip). pub fn smoke_fixtures(test: &str) -> Option<(PathBuf, Vec)> { diff --git a/bindings/rust/transcribe-cpp/tests/diarize.rs b/bindings/rust/transcribe-cpp/tests/diarize.rs new file mode 100644 index 000000000..95f4e9c67 --- /dev/null +++ b/bindings/rust/transcribe-cpp/tests/diarize.rs @@ -0,0 +1,134 @@ +//! DIARIZE role: roles, diarize_info, DiarizeSession runs, and refusal on an +//! ASR-only model. Model-gated: Sortformer on TRANSCRIBE_SMOKE_SORTFORMER_MODEL +//! (+ samples/sortformer-2spk-mix.wav), whisper on TRANSCRIBE_SMOKE_MODEL. +//! The Busy path is covered model-free in `src/diarize.rs` (no model serves +//! both DIARIZE and streaming today). + +mod common; + +use transcribe_cpp::sys::TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE as SFDR; +use transcribe_cpp::{ + CancelToken, DiarizeExtension, DiarizeOptions, Error, ExtSlot, Model, Role, RunExtension, + RunOptions, SortformerDiarizeOptions, SortformerPreset, SortformerStreamOptions, + SpeakerSegment, +}; + +fn diarize_opts(preset: Option) -> DiarizeOptions { + DiarizeOptions { + family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { + preset, + })), + } +} + +fn rows(segs: &[SpeakerSegment]) -> Vec<(i64, i64, i32, u32)> { + segs.iter() + .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id, s.p.to_bits())) + .collect() +} + +#[test] +fn sortformer_serves_asr_and_diarize() { + let Some((model_path, _)) = + common::smoke_sortformer_fixtures("sortformer_serves_asr_and_diarize") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let roles = model.roles(); + assert!(roles.contains(Role::Asr) && roles.contains(Role::Diarize)); + assert_eq!(roles.bits(), 0b11, "{roles:?}"); + let info = model.diarize_info().unwrap(); + assert_eq!((info.sample_rate, info.max_speakers), (16000, 4)); + assert!(model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); + assert!(!model.accepts_ext(ExtSlot::Run, SFDR)); + assert!(model.capabilities().is_ok()); +} + +#[test] +fn diarize_run_matches_asr_path_speaker_turns() { + let Some((model_path, pcm)) = + common::smoke_sortformer_fixtures("diarize_run_matches_asr_path_speaker_turns") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut session = model.session().unwrap(); + let mut diarize = model.diarize_session().unwrap(); + for preset in [None, Some(SortformerPreset::LowLatency)] { + let turns = diarize.run(&pcm, &diarize_opts(preset)).unwrap(); + assert!(!turns.is_empty(), "{preset:?}: no speaker turns"); + assert!(turns.iter().all(|s| (1..=4).contains(&s.speaker_id))); + assert!(diarize.timings().encode_ms > 0.0); + + // PARITY with the ASR path (Session::run + SFST preset). The ASR path + // for Sortformer is removed in a later commit; drop this block then. + let asr = RunOptions { + family: Some(RunExtension::Sortformer(SortformerStreamOptions { preset })), + ..Default::default() + }; + let transcript = session.run(&pcm, &asr).unwrap(); + assert_eq!( + rows(&turns), + rows(&transcript.speaker_segments), + "{preset:?}" + ); + } +} + +#[test] +fn diarize_session_keeps_model_alive_and_rejects_bad_input() { + let Some((model_path, pcm)) = common::smoke_sortformer_fixtures( + "diarize_session_keeps_model_alive_and_rejects_bad_input", + ) else { + return; + }; + let mut diarize = Model::load(&model_path).unwrap().diarize_session().unwrap(); + let opts = DiarizeOptions::default(); + for bad in [&[][..], &[f32::NAN; 16][..]] { + let err = diarize.run(bad, &opts).unwrap_err(); + assert!(matches!(err, Error::InvalidArgument(_)), "{err:?}"); + } + assert!(!diarize.run(&pcm, &opts).unwrap().is_empty()); +} + +#[test] +fn cancelled_diarize_run_is_aborted() { + let Some((model_path, pcm)) = + common::smoke_sortformer_fixtures("cancelled_diarize_run_is_aborted") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut diarize = model.diarize_session().unwrap(); + let token = CancelToken::new(); + diarize.set_cancel_token(&token); + token.cancel(); + let err = diarize.run(&pcm, &DiarizeOptions::default()).unwrap_err(); + assert!(matches!(err, Error::Aborted { .. }), "{err:?}"); + diarize.clear_cancel_token(); + assert!(!diarize + .run(&pcm, &DiarizeOptions::default()) + .unwrap() + .is_empty()); +} + +#[test] +fn asr_only_model_refuses_diarize() { + let Some(model_path) = common::smoke_model() else { + eprintln!("skip asr_only_model_refuses_diarize: smoke model absent"); + return; + }; + let model = Model::load(&model_path).unwrap(); + let roles = model.roles(); + assert!(roles.contains(Role::Asr) && !roles.contains(Role::Diarize)); + assert!(matches!( + model.diarize_info(), + Err(Error::UnsupportedRole(_)) + )); + assert!(matches!( + model.diarize_session(), + Err(Error::UnsupportedRole(_)) + )); + assert!(!model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); +} diff --git a/bindings/rust/transcribe-cpp/tests/no_model.rs b/bindings/rust/transcribe-cpp/tests/no_model.rs index 89a750be5..307795aa8 100644 --- a/bindings/rust/transcribe-cpp/tests/no_model.rs +++ b/bindings/rust/transcribe-cpp/tests/no_model.rs @@ -137,5 +137,6 @@ fn handles_are_send_sync() { fn assert_send() {} assert_send_sync::(); assert_send::(); - // Session is intentionally NOT Sync (single-threaded use). + assert_send::(); + // Sessions are intentionally NOT Sync (single-threaded use). } diff --git a/bindings/rust/transcribe-cpp/tests/serde.rs b/bindings/rust/transcribe-cpp/tests/serde.rs index d6fc94f30..94687aff3 100644 --- a/bindings/rust/transcribe-cpp/tests/serde.rs +++ b/bindings/rust/transcribe-cpp/tests/serde.rs @@ -4,12 +4,13 @@ #![cfg(feature = "serde")] use transcribe_cpp::{ - Backend, Capabilities, CommitPolicy, DeviceType, Diarize, ExtSlot, Feature, Itn, KvType, - MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, Pnc, - RunExtension, RunOptions, Segment, SessionLimits, SessionOptions, SortformerPreset, - SortformerStreamOptions, SpeakerSegment, StreamExtension, StreamOptions, StreamState, - StreamText, StreamUpdate, Task, TimestampKind, Timings, Token, Transcript, - VoxtralRealtimeStreamOptions, WhisperRunOptions, Word, + Backend, Capabilities, CommitPolicy, DeviceType, Diarize, DiarizeExtension, DiarizeInfo, + DiarizeOptions, DiarizeSessionOptions, ExtSlot, Feature, Itn, KvType, + MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, Pnc, Role, + Roles, RunExtension, RunOptions, Segment, SessionLimits, SessionOptions, + SortformerDiarizeOptions, SortformerPreset, SortformerStreamOptions, SpeakerSegment, + StreamExtension, StreamOptions, StreamState, StreamText, StreamUpdate, Task, TimestampKind, + Timings, Token, Transcript, VoxtralRealtimeStreamOptions, WhisperRunOptions, Word, }; fn assert_serde() {} @@ -30,7 +31,12 @@ fn plain_data_types_are_serializable() { assert_serde::(); assert_serde::(); assert_serde::(); + assert_serde::(); + assert_serde::(); + assert_serde::(); + assert_serde::(); // Results. + assert_serde::(); assert_serde::(); assert_serde::(); assert_serde::(); @@ -54,6 +60,8 @@ fn plain_data_types_are_serializable() { assert_serde::(); assert_serde::(); assert_serde::(); + assert_serde::(); + assert_serde::(); } fn nan_transcript() -> Transcript { diff --git a/bindings/rust/transcribe-cpp/tests/streaming.rs b/bindings/rust/transcribe-cpp/tests/streaming.rs index c5b8d12e0..85bf4fbb6 100644 --- a/bindings/rust/transcribe-cpp/tests/streaming.rs +++ b/bindings/rust/transcribe-cpp/tests/streaming.rs @@ -33,7 +33,7 @@ fn streams_jfk_committed_text() { }; let model = Model::load(&model_path).unwrap(); assert!( - model.capabilities().supports_streaming, + model.capabilities().unwrap().supports_streaming, "model does not advertise streaming" ); diff --git a/bindings/rust/transcribe-cpp/tests/transcribe.rs b/bindings/rust/transcribe-cpp/tests/transcribe.rs index 2a10642aa..32ce2f6be 100644 --- a/bindings/rust/transcribe-cpp/tests/transcribe.rs +++ b/bindings/rust/transcribe-cpp/tests/transcribe.rs @@ -68,7 +68,7 @@ fn capabilities_and_identity() { let model = Model::load(&model_path).unwrap(); assert_eq!(model.arch(), "whisper", "arch: {}", model.arch()); assert!(model.backend().to_lowercase().contains("cpu") || !model.backend().is_empty()); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); assert!(caps.native_sample_rate > 0, "{caps:?}"); } @@ -196,7 +196,7 @@ fn requested_timestamps_populate_rows() { return; }; let model = Model::load(&model_path).unwrap(); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); // Request the finest granularity the model actually supports (a request // finer than max_timestamp_kind correctly returns Unsupported, status 12). if caps.max_timestamp_kind == TimestampKind::None { @@ -228,7 +228,7 @@ fn timestamps_finer_than_supported_is_unsupported() { return; }; let model = Model::load(&model_path).unwrap(); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); if caps.max_timestamp_kind == TimestampKind::Token { return; // already at the finest; nothing finer to ask for } From 9eabeebaefe4d3a2fa8230fe2b21180dd9ec95ff Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:19:28 +0800 Subject: [PATCH 27/51] swift: DIARIZE role wrappers Model.roles (Roles option set), Model.diarizeInfo, Model.diarizeSession and DiarizeSession.run returning [SpeakerSegment], with the Sortformer preset on the DIARIZE_RUN slot (DiarizeExtension.sortformer), cancellation tokens and timings. Diarize runs go through Model.withCompute, so they share the model lock and stream lease with sessions. Model.capabilities now throws instead of discarding the native status (.unsupportedRole on a model without ASR); breaking change for 0.4.0. Pin header hash c2979c7e891982ee after auditing diarize.h and the sortformer.h / transcribe.h additions. --- bindings/swift/README.md | 17 ++++ .../swift/Sources/TranscribeCpp/ABIHash.swift | 2 +- .../Sources/TranscribeCpp/Cancellation.swift | 15 +++ .../swift/Sources/TranscribeCpp/Diarize.swift | 90 +++++++++++++++++ .../swift/Sources/TranscribeCpp/Family.swift | 35 +++++++ .../swift/Sources/TranscribeCpp/Model.swift | 26 ++++- .../ComputeRulesTests.swift | 34 ++++++- .../TranscribeCppTests/DiarizeTests.swift | 97 +++++++++++++++++++ .../TranscribeCppTests/TestSupport.swift | 10 ++ .../TranscribeCppTests/TranscribeTests.swift | 4 +- 10 files changed, 320 insertions(+), 10 deletions(-) create mode 100644 bindings/swift/Sources/TranscribeCpp/Diarize.swift create mode 100644 bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift diff --git a/bindings/swift/README.md b/bindings/swift/README.md index e647712a4..178da89f6 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -98,6 +98,23 @@ let transcript = stream.snapshot // language, segments, words, tokens, timings Runnable examples live in `Sources/{transcribe-file,streaming,batch,backend-select,error-handling}`. +## Diarization + +A model's `roles` say what it serves (`.asr`, `.diarize`). Diarization models +such as Sortformer return speaker turns from a `DiarizeSession`: + +```swift +let model = try Model(path: "diar_streaming_sortformer_4spk-v2.1-F32.gguf") +let info = try model.diarizeInfo // sampleRate, maxSpeakers +let turns = try model.diarizeSession().run( + pcm, options: DiarizeOptions(family: .sortformer(.init(preset: .veryHighLatency)))) +for t in turns { print(t.speakerId, t.t0Ms, t.t1Ms) } +``` + +Diarize runs share the model's compute lock and stream lease with sessions, and +take a `CancellationToken` the same way. `model.capabilities` describes ASR and +throws `TranscribeError.unsupportedRole` on a model without `.asr`. + ## Backends Backends are compiled into the xcframework per Apple slice: diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 3749df4cd..aa156f23d 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "1537ea60b442dd53" + public static let pinnedHeaderHash = "c2979c7e891982ee" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift index de1fae26b..4b47e2e11 100644 --- a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift +++ b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift @@ -38,3 +38,18 @@ extension Session { cancelToken = nil } } + +extension DiarizeSession { + /// Install a cancellation token; a cancelled run throws `.aborted`. + public func setCancellationToken(_ token: CancellationToken) { + cancelToken = token + let context = Unmanaged.passUnretained(token).toOpaque() + transcribe_diarize_set_abort_callback(ptr, abortTrampoline, context) + } + + /// Remove any installed cancellation token. + public func clearCancellationToken() { + transcribe_diarize_set_abort_callback(ptr, nil, nil) + cancelToken = nil + } +} diff --git a/bindings/swift/Sources/TranscribeCpp/Diarize.swift b/bindings/swift/Sources/TranscribeCpp/Diarize.swift new file mode 100644 index 000000000..f0d00908d --- /dev/null +++ b/bindings/swift/Sources/TranscribeCpp/Diarize.swift @@ -0,0 +1,90 @@ +import CTranscribe + +/// Static facts about a diarization model. +public struct DiarizeInfo: Sendable, Equatable { + /// Input PCM rate (16000). + public let sampleRate: Int32 + /// `SpeakerSegment.speakerId` is in `1...maxSpeakers`. + public let maxSpeakers: Int32 +} + +public struct DiarizeOptions: Sendable { + public var family: DiarizeExtension? + public init(family: DiarizeExtension? = nil) { self.family = family } +} + +extension Model { + /// Throws `.unsupportedRole` when `roles` lacks `.diarize`. + public var diarizeInfo: DiarizeInfo { + get throws { + var info = transcribe_diarize_info() + transcribe_diarize_info_init(&info) + try TranscribeError.check(transcribe_diarize_get_info(ptr, &info), context: "diarize_get_info") + return DiarizeInfo(sampleRate: info.sample_rate, maxSpeakers: info.max_speakers) + } + } + + /// Create a DIARIZE session (`threads` 0 = library default). Throws + /// `.unsupportedRole` when `roles` lacks `.diarize`. + public func diarizeSession(threads: Int32 = 0) throws -> DiarizeSession { + var params = transcribe_diarize_session_params() + transcribe_diarize_session_params_init(¶ms) + params.n_threads = threads + var out: OpaquePointer? + try TranscribeError.check( + transcribe_diarize_session_init(ptr, ¶ms, &out), context: "creating diarize session") + guard let out else { + throw TranscribeError.other(status: 0, message: "null diarize session handle") + } + return DiarizeSession(model: self, ptr: out) + } +} + +/// A DIARIZE-role session: who spoke when. Same threading and lifetime +/// contract as `Session` (single-threaded; holds its `Model` alive), and its +/// runs share the model's compute lock and stream lease with every `Session`. +public final class DiarizeSession { + let model: Model + let ptr: OpaquePointer + /// Strong ref to the installed cancellation token (see `Session`). + var cancelToken: CancellationToken? + + init(model: Model, ptr: OpaquePointer) { + self.model = model + self.ptr = ptr + } + + deinit { transcribe_diarize_session_free(ptr) } + + /// Diarize one recording (16 kHz mono float32). Rows are grouped by + /// speaker, time-ordered within a speaker; speakers may overlap. + public func run(_ pcm: [Float], options: DiarizeOptions = .init()) throws -> [SpeakerSegment] { + try model.withCompute { + if model.streamActive { + throw TranscribeError.busy( + "a stream is active on this model; finish or drop it before diarize run()") + } + let status = withDiarizeExtension(options.family) { ext in + var params = transcribe_diarize_params() + transcribe_diarize_params_init(¶ms) + params.family = ext + return pcm.withUnsafeBufferPointer { + transcribe_diarize_run(ptr, $0.baseAddress, Int32($0.count), ¶ms) + } + } + try TranscribeError.check(status, context: "diarize_run") + return (0..( } } +// MARK: - DIARIZE_RUN-slot extensions + +/// Sortformer options for `DiarizeSession.run`. +public struct SortformerDiarizeOptions: Sendable { + public var preset: SortformerPreset? + public init(preset: SortformerPreset? = nil) { self.preset = preset } +} + +public enum DiarizeExtension: Sendable { + case sortformer(SortformerDiarizeOptions) + + var kind: UInt32 { + switch self { + case .sortformer: return TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE + } + } +} + +func withDiarizeExtension( + _ ext: DiarizeExtension?, _ body: (UnsafePointer?) throws -> R +) rethrows -> R { + guard let ext else { return try body(nil) } + switch ext { + case .sortformer(let o): + var c = transcribe_sortformer_diarize_ext() + transcribe_sortformer_diarize_ext_init(&c) + if let v = o.preset { c.preset = v.cValue } + return try withUnsafePointer(to: &c.ext) { try body($0) } + } +} + // MARK: - Acceptance probe extension Model { @@ -204,4 +235,8 @@ extension Model { public func accepts(_ family: StreamExtension) -> Bool { transcribe_model_accepts_ext_kind(ptr, TRANSCRIBE_EXT_SLOT_STREAM, family.kind) } + /// Whether this model accepts the given extension on the DIARIZE_RUN slot. + public func accepts(_ family: DiarizeExtension) -> Bool { + transcribe_model_accepts_ext_kind(ptr, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, family.kind) + } } diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index d38030460..81322ccb0 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -7,6 +7,15 @@ import Foundation /// contract — the same per-model mutex + stream lease the Rust binding uses). A /// `Model` outlives every `Session` derived from it; the session holds a strong /// reference, so close ordering is automatic under ARC. +/// The kinds of work a model serves (`transcribe_model_roles`). ASR is +/// `Session`; DIARIZE is `DiarizeSession`. +public struct Roles: OptionSet, Sendable { + public let rawValue: UInt32 + public init(rawValue: UInt32) { self.rawValue = rawValue } + public static let asr = Roles(rawValue: TRANSCRIBE_ROLE_ASR.rawValue) + public static let diarize = Roles(rawValue: TRANSCRIBE_ROLE_DIARIZE.rawValue) +} + public final class Model: @unchecked Sendable { let ptr: OpaquePointer /// Serializes the run/feed/finalize compute path across all sessions, and @@ -15,7 +24,7 @@ public final class Model: @unchecked Sendable { /// The compute lease: `true` while some session holds an ACTIVE stream. /// The C contract allows at most one in-flight run/stream across ALL /// sessions of a model, and an active stream spans begin..finalize/reset/ - /// drop — so `run`/`runBatch`/another `stream` are refused with `.busy` + /// drop — so `run`/`runBatch`/another `stream`/a diarize `run` are refused with `.busy` /// while it is held, rather than racing into the documented UB (corrupted /// decodes on CPU, command-buffer failures on Metal). Always accessed under /// `runLock`. @@ -68,11 +77,18 @@ public final class Model: @unchecked Sendable { return Session(model: self, ptr: out) } + /// The roles this model serves, fixed at load. + public var roles: Roles { Roles(rawValue: transcribe_model_roles(ptr)) } + + /// ASR capabilities. Throws `.unsupportedRole` when `roles` lacks `.asr`. public var capabilities: Capabilities { - var caps = transcribe_capabilities() - transcribe_capabilities_init(&caps) - _ = transcribe_model_get_capabilities(ptr, &caps) - return Capabilities(caps) + get throws { + var caps = transcribe_capabilities() + transcribe_capabilities_init(&caps) + try TranscribeError.check( + transcribe_model_get_capabilities(ptr, &caps), context: "capabilities") + return Capabilities(caps) + } } public func supports(_ feature: Feature) -> Bool { diff --git a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift index 9e980e844..46c3bd621 100644 --- a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift @@ -6,8 +6,8 @@ import XCTest /// Characterization of the rules every native compute call follows today, so /// moving them behind one helper can be checked against the old behavior: /// -/// - every compute site (run, runBatch, stream begin/feed/finalize/reset, and a -/// dropped `Stream`'s reset) waits on the model-wide `runLock`, which all +/// - every compute site (run, runBatch, stream begin/feed/finalize/reset, a +/// dropped `Stream`'s reset, and a diarize run) waits on the model-wide `runLock`, which all /// sessions of one model share; /// - run / runBatch / stream begin validate their string options BEFORE taking /// the lock, and refuse an active stream with `.busy` only AFTER taking it; @@ -137,6 +137,36 @@ final class ComputeRulesTests: XCTestCase { XCTAssertEqual(try session.stream().reset(), .idle) } + /// A diarize run follows the Session rules: it keeps its model alive, + /// waits on the model lock, and refuses a held stream lease with `.busy` + /// only once the lock is taken. + func testDiarizeRunSharesTheModelLockAndLease() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + weak var weakModel: Model? + let session: DiarizeSession = try { + let model = try Model(path: path) + weakModel = model + return try model.diarizeSession() + }() + XCTAssertNotNil(weakModel, "a live DiarizeSession must keep its Model alive") + let model = session.model + + let rows = Box<[SpeakerSegment]>() + assertWaitsOnModelLock(model, "diarize run") { rows.value = try? session.run(pcm) } + XCTAssertEqual(rows.value?.isEmpty, false) + + // Sortformer cannot stream, so the lease a stream would hold is set directly. + model.withCompute { model.streamActive = true } + defer { model.withCompute { model.streamActive = false } } + let error = Box() + assertWaitsOnModelLock(model, "diarize run while streaming") { + do { _ = try session.run(pcm) } catch let e { error.value = e } + } + XCTAssertEqual( + busyMessage(error.value), + "a stream is active on this model; finish or drop it before diarize run()") + } + // MARK: - Active-stream rejection and its ordering /// The `.busy` check runs only once the lock is held: with a stream active diff --git a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift new file mode 100644 index 000000000..c749bf0f0 --- /dev/null +++ b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift @@ -0,0 +1,97 @@ +import CTranscribe +import XCTest + +@testable import TranscribeCpp + +/// DIARIZE role (`DiarizeSession`). Lock / lease / keep-alive rules are in +/// ComputeRulesTests. +final class DiarizeTests: XCTestCase { + /// `p` is NaN on diarize rows, so compare the turns, not `SpeakerSegment`s. + private func turns(_ rows: [SpeakerSegment]) -> [[Int64]] { + rows.map { [$0.t0Ms, $0.t1Ms, Int64($0.speakerId)] } + } + + func testRoleBitsMatchTheHeader() { + XCTAssertEqual(Roles.asr.rawValue, 1) + XCTAssertEqual(Roles.diarize.rawValue, 2) + } + + func testAsrOnlyModelRefusesDiarize() throws { + guard let path = Fixtures.modelPath() else { throw XCTSkip("no canary model") } + let model = try Model(path: path) + XCTAssertEqual(model.roles, .asr) + XCTAssertFalse(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) + for (what, call) in [ + ("diarizeInfo", { _ = try model.diarizeInfo }), + ("diarizeSession", { _ = try model.diarizeSession() }), + ] as [(String, () throws -> Void)] { + XCTAssertThrowsError(try call(), what) { error in + guard case TranscribeError.unsupportedRole = error else { + return XCTFail("\(what): expected .unsupportedRole, got \(error)") + } + } + } + } + + func testSortformerDiarizes() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + let model = try Model(path: path) + XCTAssertEqual(model.roles, [.asr, .diarize]) + XCTAssertEqual(try model.diarizeInfo, DiarizeInfo(sampleRate: 16000, maxSpeakers: 4)) + XCTAssertTrue(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) + XCTAssertGreaterThan(try model.capabilities.nativeSampleRate, 0) + + let session = try model.diarizeSession() + // On this clip only .lowLatency moves the turns, so it shows the preset + // reaches the native run. + let preset = SortformerPreset.lowLatency + let rows = try session.run( + pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: preset)))) + XCTAssertFalse(rows.isEmpty) + XCTAssertTrue(rows.allSatisfy { (1...4).contains($0.speakerId) && $0.t0Ms < $0.t1Ms }) + XCTAssertGreaterThan(session.timings.encodeMs, 0) + XCTAssertNotEqual(turns(rows), turns(try session.run(pcm))) + + // TEMPORARY: the Sortformer ASR path (SFST on the RUN slot) is removed + // in a later commit; until then both roles must yield the same turns. + let asr = try model.session().run( + pcm, options: RunOptions(family: .sortformer(SortformerStreamOptions(preset: preset)))) + XCTAssertEqual(turns(rows), turns(asr.speakerSegments)) + } + + func testBadPresetAndCancellation() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + let session = try Model(path: path).diarizeSession() + let first = try session.run(pcm) + XCTAssertFalse(first.isEmpty) + + // `SortformerPreset` cannot express an out-of-range preset, so the + // native rejection is driven through the raw C call. + var ext = transcribe_sortformer_diarize_ext() + transcribe_sortformer_diarize_ext_init(&ext) + ext.preset = transcribe_sortformer_preset(rawValue: 99) + let status = withUnsafePointer(to: &ext.ext) { family in + var params = transcribe_diarize_params() + transcribe_diarize_params_init(¶ms) + params.family = family + return pcm.withUnsafeBufferPointer { + transcribe_diarize_run(session.ptr, $0.baseAddress, Int32($0.count), ¶ms) + } + } + guard case .invalidArgument = TranscribeError.make(status) else { + return XCTFail("expected .invalidArgument, got \(status)") + } + XCTAssertEqual(Int(transcribe_diarize_n_segments(session.ptr)), first.count) + + let token = CancellationToken() + token.cancel() + session.setCancellationToken(token) + XCTAssertThrowsError(try session.run(pcm)) { error in + guard case TranscribeError.aborted = error else { + return XCTFail("expected .aborted, got \(error)") + } + } + session.clearCancellationToken() + XCTAssertEqual(turns(try session.run(pcm)), turns(first)) + } +} diff --git a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift index 43eff078c..754f13e9c 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift @@ -86,6 +86,16 @@ enum Fixtures { "models/SenseVoiceSmall/SenseVoiceSmall-Q8_0.gguf") } + /// Sortformer (ASR + DIARIZE roles) + its in-repo two-speaker sample. + static func sortformerModelAndAudio() throws -> (model: String, pcm: [Float]) { + guard let model = familyModel( + "TRANSCRIBE_SMOKE_SORTFORMER_MODEL", + "models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-F32.gguf") + else { throw XCTSkip("no Sortformer model (set TRANSCRIBE_SMOKE_SORTFORMER_MODEL)") } + let audio = repoRoot().appendingPathComponent("samples/sortformer-2spk-mix.wav").path + return (model, try loadWav(audio)) + } + /// The model path + decoded PCM, or `XCTSkip` when either is absent. static func modelAndAudio() throws -> (model: String, pcm: [Float]) { guard let model = modelPath() else { diff --git a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift index 2d8cbed54..6b7f7bac0 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift @@ -24,7 +24,7 @@ final class TranscribeTests: XCTestCase { func testFinerThanSupportedTimestampsIsUnsupported() throws { let (path, pcm) = try Fixtures.modelAndAudio() let model = try Model(path: path) - guard let finer = finerThanSupported(model.capabilities.maxTimestampKind) else { + guard let finer = finerThanSupported(try model.capabilities.maxTimestampKind) else { throw XCTSkip("model already supports the finest timestamps") } let session = try model.session() @@ -61,7 +61,7 @@ final class TranscribeTests: XCTestCase { let model = try Model(path: path) XCTAssertFalse(model.arch.isEmpty) XCTAssertFalse(model.backend.isEmpty) - XCTAssertGreaterThan(model.capabilities.nativeSampleRate, 0) + XCTAssertGreaterThan(try model.capabilities.nativeSampleRate, 0) } func testPncChangesCanaryPrompt() throws { From 8599661a1ad928923d37cda0b08052588cca5ced Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:20:18 +0800 Subject: [PATCH 28/51] python: DIARIZE role wrappers --- bindings/python/README.md | 18 ++ .../python/src/transcribe_cpp/__init__.py | 153 +++++++++++- bindings/python/tests/conftest.py | 11 + bindings/python/tests/test_compute_lock.py | 217 +++++++++++++++++- bindings/python/tests/test_diarize.py | 87 +++++++ bindings/python/tests/test_family_ext.py | 11 +- 6 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 bindings/python/tests/test_diarize.py diff --git a/bindings/python/README.md b/bindings/python/README.md index e42c5a143..02ab2caff 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -70,6 +70,24 @@ Long transcriptions can be cancelled from another thread with `session.cancel()` — the run raises `Aborted` with the partial transcript on `exc.partial_result` (same for `OutputTruncated`). +### Diarization + +Models whose `model.roles` include `Role.DIARIZE` (Sortformer) answer "who +spoke when" through a diarize session. `run()` returns `SpeakerSegment` +rows (`speaker_id` in `1..model.diarize_info.max_speakers`), grouped by +speaker. It shares the model-wide compute lock with transcription runs and +raises `Busy` while a stream is active. `cancel()` and `close()` work as on +`Session`. A model without the role raises `UnsupportedRole`, as +`model.session()` does on a model without `Role.ASR`. + +```python +with model.diarize_session() as diarizer: + turns = diarizer.run(pcm, family=transcribe_cpp.SortformerDiarizeOptions( + preset="very_high_latency")) + for turn in turns: + print(turn.speaker_id, turn.t0_ms, turn.t1_ms) +``` + ## Backends `Model(backend=...)` applies a backend policy (`"auto"` uses the best diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index dcf66a32c..bec5a335f 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -25,6 +25,7 @@ import collections import ctypes +import enum import os import threading import weakref @@ -77,6 +78,9 @@ "transcribe", "Model", "Session", + "DiarizeSession", + "DiarizeInfo", + "Role", "Result", "Segment", "SpeakerSegment", @@ -94,6 +98,7 @@ "ParakeetStreamOptions", "ParakeetBufferedStreamOptions", "SortformerStreamOptions", + "SortformerDiarizeOptions", "VoxtralRealtimeStreamOptions", "Backend", "SortformerPreset", @@ -252,6 +257,7 @@ _EXT_SLOTS = { "run": _generated.TRANSCRIBE_EXT_SLOT_RUN, "stream": _generated.TRANSCRIBE_EXT_SLOT_STREAM, + "diarize_run": _generated.TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, } _FEATURES = { "initial_prompt": _generated.TRANSCRIBE_FEATURE_INITIAL_PROMPT, @@ -556,6 +562,20 @@ class Capabilities: translate_target_languages: tuple[str, ...] +class Role(enum.Enum): + """What a model serves (``Model.roles``): ASR is ``Model.session()``, + DIARIZE is ``Model.diarize_session()``.""" + + ASR = _generated.TRANSCRIBE_ROLE_ASR + DIARIZE = _generated.TRANSCRIBE_ROLE_DIARIZE + + +@dataclass(frozen=True) +class DiarizeInfo: + sample_rate: int + max_speakers: int # speaker_id is in [1, max_speakers] + + @dataclass(frozen=True) class SessionLimits: """Effective per-session limits (model bound narrowed by session params). @@ -917,6 +937,20 @@ def _apply(self, ext) -> None: ext.preset = self._presets[self.preset] +class SortformerDiarizeOptions(FamilyExtension): + """Sortformer operating-point options for ``DiarizeSession.run()`` + (diarize-run slot); same presets as :class:`SortformerStreamOptions`.""" + + _slot = "diarize_run" + _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE + _struct = _generated.transcribe_sortformer_diarize_ext + _init = "transcribe_sortformer_diarize_ext_init" + + _presets = SortformerStreamOptions._presets + __init__ = SortformerStreamOptions.__init__ + _apply = SortformerStreamOptions._apply + + # --- high-level handles --------------------------------------------------- @@ -950,7 +984,8 @@ class Model: Compute calls are serialized per model. The native library allows at most one compute call in flight across ALL sessions of a model (they share its compute backend), so ``Session.run()``, ``run_batch()``, - ``stream()`` and ``Stream.feed()`` / ``finalize()`` / ``reset()`` each + ``stream()``, ``Stream.feed()`` / ``finalize()`` / ``reset()`` and + ``DiarizeSession.run()`` each hold a model-wide lock for the native call and for copying its results out. Calls on any session of the same model from other threads WAIT for the lock, and a waiting call can still be cancelled with its session's @@ -962,7 +997,8 @@ class Model: during a feed (the C contract forbids a run between the feeds of a stream on another session). A successful ``stream()`` therefore takes a model-wide stream lease. Until that stream ends, ``run()``, - ``run_batch()`` and ``stream()`` on ANY session of this model (its own + ``run_batch()``, ``stream()`` and ``DiarizeSession.run()`` on ANY + session of this model (its own included) raise :class:`Busy` at once instead of waiting; the stream's own ``feed()`` / ``finalize()`` proceed. The lease is released by ``finalize()`` (success or failure), ``reset()``, a ``feed()`` that @@ -1205,6 +1241,25 @@ def session(self, *, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0) -> "Session": return Session(self, n_threads=n_threads, kv_type=kv_type, n_ctx=n_ctx) + @property + def roles(self) -> frozenset[Role]: + mask = _lib.transcribe_model_roles(self._h) + return frozenset(r for r in Role if mask & r.value) + + @property + def diarize_info(self) -> DiarizeInfo: + """Raises :class:`UnsupportedRole` without the DIARIZE role.""" + info = _generated.transcribe_diarize_info() + _lib.transcribe_diarize_info_init(_byref(info)) + _check(_lib.transcribe_diarize_get_info(self._h, _byref(info)), + "reading diarize info") + return DiarizeInfo(sample_rate=info.sample_rate, + max_speakers=info.max_speakers) + + def diarize_session(self, *, n_threads: int = 0) -> "DiarizeSession": + """Raises :class:`UnsupportedRole` without the DIARIZE role.""" + return DiarizeSession(self, n_threads=n_threads) + def close(self) -> None: """Free the model. Any session still open on it is closed first — the C contract forbids freeing a model before its sessions, so this @@ -1849,6 +1904,100 @@ def reset_and_release(slot=slot, lease=lease): pass +class DiarizeSession: + """A diarization (who spoke when) context on a model with the DIARIZE + role. ``run()`` follows the Session rules (see ``Model``): it takes the + model-wide compute lock, raises :class:`Busy` while a stream is active + on the model, and ``cancel()`` / ``close()`` never wait for the lock.""" + + def __init__(self, model: Model, *, n_threads: int = 0): + self._model = model # keep the model alive for the session's lifetime + params = _generated.transcribe_diarize_session_params() + _lib.transcribe_diarize_session_params_init(_byref(params)) + params.n_threads = n_threads + + handle = ctypes.c_void_p() + _check(_lib.transcribe_diarize_session_init(model._h, _byref(params), _byref(handle)), + "opening diarize session") + if not handle.value: + raise TranscribeError("diarize session init returned a null handle") + self._handle = handle + + self._cancel = threading.Event() # as in Session: bind the Event, not self + _event = self._cancel + self._abort_trampoline = _ABORT_CFUNC(lambda _ud: _event.is_set()) + _lib.transcribe_diarize_set_abort_callback(self._handle, self._abort_trampoline, None) + model._sessions.add(self) + + _h = Session._h + _resolve_family = Session._resolve_family + + def cancel(self) -> None: + """Abort an in-flight (or queued) ``run()``, which raises + :class:`Aborted`. Lock-free; safe from any thread.""" + self._cancel.set() + + def run(self, pcm: PCMLike, *, + family: FamilyExtension | None = None) -> list[SpeakerSegment]: + """Diarize one recording (16 kHz mono float32 PCM) and return its + speaker turns: grouped by speaker, time-ordered within a speaker, + possibly overlapping across speakers. ``family`` is an optional + diarize-run extension (e.g. SortformerDiarizeOptions). + + Raises :class:`Aborted` after :meth:`cancel`, and :class:`Busy` if a + stream is active on this model.""" + self._cancel.clear() # before the lock wait, as in Session.run() + array, n_samples = _pcm_to_carray(pcm) + params = _generated.transcribe_diarize_params() + _lib.transcribe_diarize_params_init(_byref(params)) + with self._model._exclusive( + "diarize_run", busy="a stream is active on this model; " + "finish or drop it before diarize run()"): + ext = self._resolve_family(family, "diarize_run") if family is not None else None + if ext is not None: + params.family = ctypes.cast( + _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + h = self._h # captured under the lock; close() defers its free + _check(_lib.transcribe_diarize_run(h, array, n_samples, _byref(params)), + "transcribe_diarize_run") + rows = [] + for i in range(_lib.transcribe_diarize_n_segments(h)): + s = _SpeakerSegment() + _lib.transcribe_speaker_segment_init(_byref(s)) + _check(_lib.transcribe_diarize_get_segment(h, i, _byref(s)), + "transcribe_diarize_get_segment") + rows.append(_speaker_segment_from(s)) + return rows + + @property + def timings(self) -> Timings: + """Load time plus the last run's mel / encode time. Not locked, like + ``Session.limits``.""" + tm = _Timings() + _lib.transcribe_timings_init(_byref(tm)) + _check(_lib.transcribe_diarize_get_timings(self._h, _byref(tm)), + "transcribe_diarize_get_timings") + return _timings_from(tm) + + def close(self) -> None: + """Free the session. Idempotent; never waits for the compute lock + (the free runs right after an in-flight call, as in Session).""" + handle = getattr(self, "_handle", None) + if handle is None: + return + self._handle = None + trampoline = getattr(self, "_abort_trampoline", None) + + def free(handle=handle, _pin=trampoline): + _lib.transcribe_diarize_session_free(handle) + + self._model._free_or_defer(free) + + __enter__ = Session.__enter__ + __exit__ = Session.__exit__ + __del__ = Session.__del__ + + def transcribe( model: Model | str | os.PathLike, pcm: PCMLike, diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index aeb0036df..5c82e9a13 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -59,6 +59,11 @@ REPO / "models/Voxtral-Mini-4B-Realtime-2602/Voxtral-Mini-4B-Realtime-2602-Q4_K_M.gguf" ) +SORTFORMER_MODEL = ( + REPO + / "models/diar_streaming_sortformer_4spk-v2.1" + / "diar_streaming_sortformer_4spk-v2.1-F32.gguf" +) PNC_MODEL = REPO / "models/canary-180m-flash/canary-180m-flash-Q8_0.gguf" ITN_MODEL = REPO / "models/SenseVoiceSmall/SenseVoiceSmall-Q8_0.gguf" @@ -161,6 +166,12 @@ def voxtral_model_path() -> Path: return _family_model("TRANSCRIBE_SMOKE_VOXTRAL_MODEL", VOXTRAL_MODEL) +@pytest.fixture(scope="session") +def sortformer_model_path() -> Path: + """Sortformer diarizer (serves the ASR and DIARIZE roles).""" + return _family_model("TRANSCRIBE_SMOKE_SORTFORMER_MODEL", SORTFORMER_MODEL) + + @pytest.fixture(scope="session") def pnc_model_path() -> Path: """Canary model whose generic PNC run parameter changes the prompt.""" diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index 78dfa055b..0855f439e 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -64,6 +64,8 @@ def fake(monkeypatch): lambda h: frees.append(("session", h.value))) monkeypatch.setattr(t._lib, "transcribe_model_free", lambda h: frees.append(("model", h.value))) + monkeypatch.setattr(t._lib, "transcribe_diarize_session_free", + lambda h: frees.append(("diarize", h.value))) resets: list = [] # session handles transcribe_stream_reset was given monkeypatch.setattr(t._lib, "transcribe_stream_reset", lambda h: resets.append(h.value)) @@ -95,8 +97,18 @@ def session(m, track=True): made.append(s) return s - yield SimpleNamespace(model=model, session=session, frees=frees, - resets=resets, state=state) + def diarize(m): + d = t.DiarizeSession.__new__(t.DiarizeSession) + d._model = m + d._handle = ctypes.c_void_p(next(ids)) + d._cancel = threading.Event() + d._abort_trampoline = None + m._sessions.add(d) + made.append(d) + return d + + yield SimpleNamespace(model=model, session=session, diarize=diarize, + frees=frees, resets=resets, state=state) for obj in reversed(made): obj.close() @@ -784,3 +796,204 @@ def run(h, *rest): del stream gc.collect() assert fake.resets == [] + + +# --- DiarizeSession.run ----------------------------------------------------------- +# +# The DIARIZE role's run goes through the same Model._exclusive bridge as +# Session.run: model-wide lock, Busy under a stream lease, copy-out under the +# lock on the captured handle, deferred frees, lock-free cancel. + +DIARIZE_BUSY = ("a stream is active on this model; " + "finish or drop it before diarize run()") + + +@pytest.fixture +def diarize_native(fake, monkeypatch): + """Probes for the diarize entry points. ``calls`` records (name, handle, + whether this thread held ``owner["model"]``'s lock); two rows per run.""" + calls: list = [] + owner: dict = {} + + def held(): + m = owner["model"] + return m._compute_lock.locked() and m._compute_owner == threading.get_ident() + + def run(h, pcm, n, params): + calls.append(("run", h.value, held())) + return 0 + + def n_segments(h): + calls.append(("n_segments", h.value, held())) + return 2 + + def get_segment(h, i, out): + calls.append(("get_segment", h.value, held())) + row = out._obj + row.t0_ms, row.t1_ms, row.speaker_id, row.p = 100 * i, 100 * i + 50, i + 1, 0.5 + return 0 + + monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) + monkeypatch.setattr(t._lib, "transcribe_diarize_n_segments", n_segments) + monkeypatch.setattr(t._lib, "transcribe_diarize_get_segment", get_segment) + return SimpleNamespace(calls=calls, owner=owner, run=run) + + +ROWS = [t.SpeakerSegment(t0_ms=0, t1_ms=50, speaker_id=1, p=0.5), + t.SpeakerSegment(t0_ms=100, t1_ms=150, speaker_id=2, p=0.5)] + + +def test_diarize_run_and_copy_out_hold_the_lock(fake, diarize_native): + m = fake.model() + d = fake.diarize(m) + diarize_native.owner["model"] = m + assert d.run(PCM) == ROWS + names = [c[0] for c in diarize_native.calls] + assert names == ["run", "n_segments", "get_segment", "get_segment"] + assert all(c[1] == d._handle.value and c[2] for c in diarize_native.calls) + assert not m._compute_lock.locked() + + +def test_diarize_session_init_wires_cancel_and_registers(fake, monkeypatch): + m = fake.model() + hm = m._handle.value + cbs: list = [] + + def init(model_h, params, out): + out._obj.value = 0x9000 + return 0 + + monkeypatch.setattr(t._lib, "transcribe_diarize_session_init", init) + monkeypatch.setattr(t._lib, "transcribe_diarize_set_abort_callback", + lambda h, cb, ud: cbs.append((h.value, cb))) + d = m.diarize_session(n_threads=2) + assert d in set(m._sessions) + [(h, cb)] = cbs + assert h == 0x9000 and cb(None) is False + d.cancel() # lock-free; flips what the native abort callback polls + assert cb(None) is True + m.close() # closes the diarize session before freeing the model + assert fake.frees == [("diarize", 0x9000), ("model", hm)] + + +def test_diarize_session_init_failure_raises_unsupported_role(fake, monkeypatch): + m = fake.model() + monkeypatch.setattr(t._lib, "transcribe_diarize_session_init", + lambda mh, p, out: _generated.TRANSCRIBE_ERR_UNSUPPORTED_ROLE) + with pytest.raises(t.UnsupportedRole): + m.diarize_session() + assert not list(m._sessions) + + +def test_diarize_run_waits_for_session_run(fake, diarize_native, monkeypatch): + m = fake.model() + s, d = fake.session(m), fake.diarize(m) + diarize_native.owner["model"] = m + gate = _Gate() + monkeypatch.setattr(t._lib, "transcribe_run", gate) + try: + a, a_box = _in_thread(s.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + b, b_box = _in_thread(d.run, PCM) + assert spy.waiting.wait(TIMEOUT) # parked on the lock, not failed + assert b.is_alive() and not b_box and diarize_native.calls == [] + finally: + gate.release.set() + _join(a, "session run") + _join(b, "queued diarize run") + assert "error" not in a_box and b_box.get("value") == ROWS, (a_box, b_box) + + +def test_diarize_run_busy_while_stream_active(fake, native, diarize_native): + m = fake.model() + s, d = fake.session(m), fake.diarize(m) + diarize_native.owner["model"] = m + stream = s.stream() + with pytest.raises(t.Busy) as ei: + d.run(PCM) + assert str(ei.value) == DIARIZE_BUSY + assert diarize_native.calls == [] and not m._compute_lock.locked() + stream.finalize() + assert d.run(PCM) == ROWS + + +def _gated_diarize_run(diarize_native, monkeypatch): + gate = _Gate() + + def run(*args): + gate() + return diarize_native.run(*args) + + monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) + return gate + + +def test_diarize_close_during_in_flight_run_defers_free(fake, diarize_native, + monkeypatch): + m = fake.model() + d = fake.diarize(m) + diarize_native.owner["model"] = m + hd = d._handle.value + gate = _gated_diarize_run(diarize_native, monkeypatch) + try: + a, a_box = _in_thread(d.run, PCM) + assert gate.entered.wait(TIMEOUT) + closer, _ = _in_thread(d.close) + _join(closer, "diarize close() during an in-flight run") + assert fake.frees == [], "diarize session freed under an in-flight run" + finally: + gate.release.set() + _join(a, "in-flight diarize run") + assert a_box.get("value") == ROWS, a_box + assert all(c[1] == hd for c in diarize_native.calls) # the captured handle + assert fake.frees == [("diarize", hd)] + with pytest.raises(t.TranscribeError, match="closed"): + d.run(PCM) + + +def test_model_close_during_in_flight_diarize_run_frees_session_first( + fake, diarize_native, monkeypatch): + m = fake.model() + d = fake.diarize(m) + diarize_native.owner["model"] = m + hd, hm = d._handle.value, m._handle.value + gate = _gated_diarize_run(diarize_native, monkeypatch) + try: + a, a_box = _in_thread(d.run, PCM) + assert gate.entered.wait(TIMEOUT) + closer, _ = _in_thread(m.close) + _join(closer, "model.close() during an in-flight diarize run") + assert fake.frees == [] + finally: + gate.release.set() + _join(a, "in-flight diarize run") + assert a_box.get("value") == ROWS, a_box + assert fake.frees == [("diarize", hd), ("model", hm)] + + +def test_diarize_cancel_while_queued_is_kept(fake, diarize_native, monkeypatch): + m = fake.model() + s, d = fake.session(m), fake.diarize(m) + diarize_native.owner["model"] = m + gate = _Gate() + flag_at_start: list = [] + + def run(*args): + flag_at_start.append(d._cancel.is_set()) + return diarize_native.run(*args) + + monkeypatch.setattr(t._lib, "transcribe_run", gate) + monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) + try: + a, _ = _in_thread(s.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + b, _ = _in_thread(d.run, PCM) + assert spy.waiting.wait(TIMEOUT) # cleared its flag and is parked + d.cancel() # must not block although the lock is held + finally: + gate.release.set() + _join(a, "session run") + _join(b, "queued diarize run") + assert flag_at_start == [True] diff --git a/bindings/python/tests/test_diarize.py b/bindings/python/tests/test_diarize.py new file mode 100644 index 000000000..b586429ed --- /dev/null +++ b/bindings/python/tests/test_diarize.py @@ -0,0 +1,87 @@ +"""DIARIZE role against real models: roles, diarize info, DiarizeSession.run +and its Sortformer preset extension. The locking / Busy / close / cancel +rules are pinned model-free in test_compute_lock.py.""" + +from __future__ import annotations + +import gc +import math + +import pytest + +import transcribe_cpp as t +from conftest import SAMPLES, load_wav + + +@pytest.fixture(scope="module") +def mix_pcm(): + return load_wav(SAMPLES / "sortformer-2spk-mix.wav") + + +def _turns(rows): + return [(r.t0_ms, r.t1_ms, r.speaker_id) for r in rows] + + +def test_sortformer_roles_and_info(sortformer_model_path): + with t.Model(sortformer_model_path) as model: + assert model.roles == {t.Role.ASR, t.Role.DIARIZE} + assert model.diarize_info == t.DiarizeInfo(sample_rate=16000, max_speakers=4) + + +def test_asr_only_model_rejects_diarize(model_path): + with t.Model(model_path) as model: + assert model.roles == {t.Role.ASR} + with pytest.raises(t.UnsupportedRole): + model.diarize_info + with pytest.raises(t.UnsupportedRole): + model.diarize_session() + + +# TEMPORARY: Sortformer still serves the ASR role, whose Session.run returns +# the same speaker turns on Result.speaker_segments. This cross-check goes +# away when that ASR path is removed in a later commit. +@pytest.mark.parametrize("preset", ["default", "low_latency"]) +def test_diarize_run_matches_asr_path(sortformer_model_path, mix_pcm, preset): + with t.Model(sortformer_model_path) as model: + with model.diarize_session() as d: + rows = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset=preset)) + timings = d.timings + with model.session() as s: + asr = s.run(mix_pcm, family=t.SortformerStreamOptions(preset=preset)) + with pytest.raises(t.InvalidArgument, match="slot"): + s.run(mix_pcm, family=t.SortformerDiarizeOptions()) + assert rows and isinstance(rows, list) + assert all(isinstance(r, t.SpeakerSegment) for r in rows) + assert {r.speaker_id for r in rows} <= {1, 2, 3, 4} + assert all(math.isnan(r.p) for r in rows) # Sortformer has no per-turn p + assert _turns(rows) == _turns(asr.speaker_segments) + assert timings.encode_ms > 0 + + +def test_diarize_run_without_extension(sortformer_model_path, mix_pcm): + with t.Model(sortformer_model_path) as model, model.diarize_session() as d: + assert _turns(d.run(mix_pcm)) == _turns( + d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset="default"))) + + +def test_bad_preset_rejected(sortformer_model_path, mix_pcm): + class OutOfRange(t.SortformerDiarizeOptions): + _presets = {**t.SortformerDiarizeOptions._presets, "bogus": 99} + + with pytest.raises(ValueError, match="preset"): + t.SortformerDiarizeOptions(preset="bogus") # type: ignore[arg-type] + with t.Model(sortformer_model_path) as model, model.diarize_session() as d: + with pytest.raises(t.InvalidArgument): + d.run(mix_pcm, family=OutOfRange(preset="bogus")) # type: ignore[arg-type] + with pytest.raises(t.InvalidArgument, match="slot"): + d.run(mix_pcm, family=t.SortformerStreamOptions()) + assert d.run(mix_pcm) # still usable + + +def test_diarize_session_keeps_model_alive(sortformer_model_path, mix_pcm): + d = t.Model(sortformer_model_path).diarize_session() + gc.collect() + assert d.run(mix_pcm) + d.close() + with pytest.raises(t.TranscribeError, match="closed"): + d.run(mix_pcm) diff --git a/bindings/python/tests/test_family_ext.py b/bindings/python/tests/test_family_ext.py index 3b0a433ee..2391634e8 100644 --- a/bindings/python/tests/test_family_ext.py +++ b/bindings/python/tests/test_family_ext.py @@ -23,6 +23,7 @@ t.ParakeetStreamOptions, t.ParakeetBufferedStreamOptions, t.SortformerStreamOptions, + t.SortformerDiarizeOptions, t.VoxtralRealtimeStreamOptions, ] @@ -35,7 +36,7 @@ def test_build_stamps_kind_and_size(cls): ext = cls()._build() assert ext.ext.kind == cls._kind assert ext.ext.size == ctypes.sizeof(cls._struct) - assert cls._slot in ("run", "stream") + assert cls._slot in t._EXT_SLOTS @pytest.mark.parametrize("cls", ALL_OPTION_TYPES) @@ -87,6 +88,14 @@ def test_sortformer_unknown_preset_rejected(): t.SortformerStreamOptions(preset="ultra_low_latency") # type: ignore[arg-type] +def test_sortformer_diarize_preset_maps_to_enum_value(): + built = t.SortformerDiarizeOptions(preset="low_latency")._build() + assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY + default = t.SortformerDiarizeOptions()._build() + assert default.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT + assert not isinstance(t.SortformerDiarizeOptions(), t.SortformerStreamOptions) + + # --- model-gated: resolve_family validation + a real extension run ---------- From e832e332f877bd54eadbb05598c287f18115b2b8 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:20:08 +0800 Subject: [PATCH 29/51] ts: DIARIZE role wrappers model.roles (readonly array of "asr" | "diarize"), model.diarizeInfo (sampleRate, maxSpeakers), model.createDiarizeSession({ nThreads }), and DiarizeSession: run(pcm, { family, signal }) -> SpeakerSegment[] (copied out), timings, dispose() / Symbol.dispose. Both throw UnsupportedRole on a model without the role; ASR calls already map status 20 the same way. The Sortformer diarize preset is modeled like the ASR one: a { kind: "sortformer_diarize", preset } family extension on the new "diarize_run" slot (SFDR). Both Sortformer kinds now share one preset mapper that rejects an unknown preset string instead of silently sending DEFAULT. Execution bridge: Session's handle, disposed flag, in-flight mark, compute admission (#exclusive: model lock, disposed recheck, then stream-lease Busy), abort install, and deferred free move into a module-private SessionCore, held in a private field by both Session and DiarizeSession and parameterized by the role's set_abort_callback and free. Stream still reaches it through SESSION_CONTROL. The speaker-segment and timings readers are shared with materialize(). tests: diarize.test.mjs. Model-free: DiarizeSession's public surface. Whisper: roles ["asr"], diarizeInfo / createDiarizeSession throw UnsupportedRole, run() rejects the diarize_run extension. Sortformer (TRANSCRIBE_SMOKE_SORTFORMER_MODEL, local-only): roles, maxSpeakers 4, diarize rows equal the ASR path's speakerSegments at default and low_latency, bad preset and wrong-slot extensions rejected, diarize and ASR runs serialize FIFO on one model (these two use the Sortformer ASR path and are marked to drop with it), abort listener lifetime and pre-aborted Aborted, disposed recheck inside the lock, deferred free on session and model dispose during an in-flight run. Busy is not covered for diarize: Sortformer cannot begin a stream, so no model holds a stream lease and serves DIARIZE. --- bindings/typescript/README.md | 28 +- bindings/typescript/src/ffi.ts | 46 +++ bindings/typescript/src/index.ts | 387 +++++++++++++++------- bindings/typescript/src/types.ts | 33 +- bindings/typescript/test/common.mjs | 3 + bindings/typescript/test/diarize.test.mjs | 175 ++++++++++ 6 files changed, 555 insertions(+), 117 deletions(-) create mode 100644 bindings/typescript/test/diarize.test.mjs diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 836c68a99..b004c5d93 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -109,10 +109,29 @@ const stream = await session.stream({ family: { kind: "moonshine" } }); model.accepts({ kind: "whisper" }); // does this model take that extension? ``` +### Diarization (DIARIZE role) + +`model.roles` lists what a model serves (`"asr"`, `"diarize"`). A diarization +model (e.g. Sortformer) answers who spoke when; ASR calls on a model without +`"asr"`, and diarize calls on one without `"diarize"`, throw `UnsupportedRole`. + +```ts +const { sampleRate, maxSpeakers } = model.diarizeInfo; +using diarizer = model.createDiarizeSession(); +const turns = await diarizer.run(pcm, { + family: { kind: "sortformer_diarize", preset: "very_high_latency" }, +}); +for (const t of turns) console.log(t.speakerId, t.t0Ms, t.t1Ms); +``` + +`run` takes a `signal` like `Session.run`, and `diarizer.timings` reports the +last run. A diarize run is a compute like any other: it shares the model's +one-at-a-time rule and `Busy` refusal below. + ### Resource management -`TranscribeModel`, `Session`, and `Stream` all implement `Symbol.dispose`, so -`using` works (TypeScript 5.2+ / Node 22+): +`TranscribeModel`, `Session`, `DiarizeSession`, and `Stream` all implement +`Symbol.dispose`, so `using` works (TypeScript 5.2+ / Node 22+): ```ts using model = await TranscribeModel.load("model.gguf"); @@ -150,8 +169,9 @@ native compute to a **libuv worker thread** (via koffi's async calls), so the event loop stays responsive while inference runs. The C library allows **one compute in flight per model** — a `run`, a `runBatch`, -or an *active stream* — across all of its sessions. The binding enforces this: -every compute call serializes through an internal model-wide mutex, and an active +a diarize `run`, or an *active stream* — across all of its sessions. The binding +enforces this: every compute call serializes through an internal model-wide +mutex, and an active stream holds a model-wide lease for its whole lifetime. While a stream is active (after `stream()`, before `finalize()`/`reset()`), a `run`/`runBatch`/`stream` on any session of that model is refused with a `Busy` error rather than allowed to diff --git a/bindings/typescript/src/ffi.ts b/bindings/typescript/src/ffi.ts index a02226db9..75050a907 100644 --- a/bindings/typescript/src/ffi.ts +++ b/bindings/typescript/src/ffi.ts @@ -77,6 +77,7 @@ export function bindLibrary(libraryPath: string): Bound { modelBackend: lib.func("transcribe_model_backend", "str", ["void *"]), modelDevice: lib.func("transcribe_model_device", "void *", ["void *"]), modelSupports: lib.func("transcribe_model_supports", "bool", ["void *", "int"]), + modelRoles: lib.func("transcribe_model_roles", "uint32", ["void *"]), tokenize: lib.func("transcribe_tokenize", "int", ["void *", "str", "int32_t *", "size_t"]), capabilitiesInit: lib.func("transcribe_capabilities_init", "void", [ outp(T.transcribe_capabilities), @@ -176,6 +177,51 @@ export function bindLibrary(libraryPath: string): Bound { sortformerStreamExtInit: lib.func("transcribe_sortformer_stream_ext_init", "void", [ outp(T.transcribe_sortformer_stream_ext), ]), + sortformerDiarizeExtInit: lib.func("transcribe_sortformer_diarize_ext_init", "void", [ + outp(T.transcribe_sortformer_diarize_ext), + ]), + + // diarize role + diarizeInfoInit: lib.func("transcribe_diarize_info_init", "void", [ + outp(T.transcribe_diarize_info), + ]), + diarizeGetInfo: lib.func("transcribe_diarize_get_info", "int", [ + "void *", + iop(T.transcribe_diarize_info), + ]), + diarizeSessionParamsInit: lib.func("transcribe_diarize_session_params_init", "void", [ + outp(T.transcribe_diarize_session_params), + ]), + diarizeSessionInit: lib.func("transcribe_diarize_session_init", "int", [ + "void *", + inp(T.transcribe_diarize_session_params), + handleOut, + ]), + diarizeSessionFree: lib.func("transcribe_diarize_session_free", "void", ["void *"]), + diarizeSetAbortCallback: lib.func("transcribe_diarize_set_abort_callback", "void", [ + "void *", + "void *", + "void *", + ]), + diarizeParamsInit: lib.func("transcribe_diarize_params_init", "void", [ + outp(T.transcribe_diarize_params), + ]), + diarizeRun: lib.func("transcribe_diarize_run", "int", [ + "void *", + inp("float"), + "int", + inp(T.transcribe_diarize_params), + ]), + diarizeNSegments: lib.func("transcribe_diarize_n_segments", "int", ["void *"]), + diarizeGetSegment: lib.func("transcribe_diarize_get_segment", "int", [ + "void *", + "int", + iop(T.transcribe_speaker_segment), + ]), + diarizeGetTimings: lib.func("transcribe_diarize_get_timings", "int", [ + "void *", + iop(T.transcribe_timings), + ]), // batch (offline) runBatch: lib.func("transcribe_run_batch", "int", [ diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index ff75f0516..7d4ef259e 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -34,6 +34,9 @@ import type { CommitPolicy, DeviceType, Diarize, + DiarizeInfo, + DiarizeOptions, + DiarizeSessionOptions, ExtSlot, FamilyExtension, Feature, @@ -42,6 +45,7 @@ import type { ModelOptions, PcmLike, Pnc, + Role, Segment, SpeakerSegment, SessionLimits, @@ -122,6 +126,10 @@ const FEATURES: Record = { instruct: g.TRANSCRIBE_FEATURE_INSTRUCT, transcript_prefix: g.TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX, }; +const ROLES: Record = { + asr: g.TRANSCRIBE_ROLE_ASR, + diarize: g.TRANSCRIBE_ROLE_DIARIZE, +}; // ---- helpers --------------------------------------------------------------- @@ -499,40 +507,50 @@ function materialize(n: Native, acc: Accessors): Transcript { }); } - const speakerSegments: SpeakerSegment[] = []; - for (let i = 0, c = acc.nSpeakerSegments(); i < c; i++) { + return { + text: acc.fullText() ?? "", + rawText: acc.rawText() ?? "", + language: acc.detectedLanguage() ?? "", + timestampKind: TIMESTAMP_NAMES[acc.returnedTimestampKind()] ?? "none", + segments, + speakerSegments: readSpeakerSegments(n, acc.nSpeakerSegments(), acc.getSpeakerSegment), + words, + tokens, + timings: readTimings(n, acc.getTimings), + }; +} + +// Shared by transcripts and the DIARIZE role's results. +function readSpeakerSegments( + n: Native, + count: number, + get: (i: number, out: any) => number, +): SpeakerSegment[] { + const out: SpeakerSegment[] = []; + for (let i = 0; i < count; i++) { const s: any = {}; - F.speakerSegmentInit(s); - check(n, acc.getSpeakerSegment(i, s), `reading speaker segment ${i}`); - speakerSegments.push({ + n.F.speakerSegmentInit(s); + check(n, get(i, s), `reading speaker segment ${i}`); + out.push({ t0Ms: num(s.t0_ms), t1Ms: num(s.t1_ms), speakerId: s.speaker_id, p: s.p, }); } + return out; +} +function readTimings(n: Native, get: (out: any) => number): Timings { const tm: any = {}; - F.timingsInit(tm); - check(n, acc.getTimings(tm), "reading timings"); - const timings: Timings = { + n.F.timingsInit(tm); + check(n, get(tm), "reading timings"); + return { loadMs: tm.load_ms, melMs: tm.mel_ms, encodeMs: tm.encode_ms, decodeMs: tm.decode_ms, }; - - return { - text: acc.fullText() ?? "", - rawText: acc.rawText() ?? "", - language: acc.detectedLanguage() ?? "", - timestampKind: TIMESTAMP_NAMES[acc.returnedTimestampKind()] ?? "none", - segments, - speakerSegments, - words, - tokens, - timings, - }; } // ---- family extensions ----------------------------------------------------- @@ -551,6 +569,7 @@ const STREAM_STATES: Record = { const SLOT: Record = { run: g.TRANSCRIBE_EXT_SLOT_RUN, stream: g.TRANSCRIBE_EXT_SLOT_STREAM, + diarize_run: g.TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, }; interface FamilyReg { @@ -621,10 +640,21 @@ const FAMILY: Record = { kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, type: "transcribe_sortformer_stream_ext", init: "sortformerStreamExtInit", - map: (o) => ({ preset: SORTFORMER_PRESET[o.preset as string] }), + map: (o) => sortformerPreset(o), + }, + sortformer_diarize: { + slot: "diarize_run", + kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, + type: "transcribe_sortformer_diarize_ext", + init: "sortformerDiarizeExtInit", + map: (o) => sortformerPreset(o), }, }; +const sortformerPreset = (o: any) => ({ + preset: o.preset === undefined ? undefined : lookup(SORTFORMER_PRESET, o.preset, "sortformer preset"), +}); + const SORTFORMER_PRESET: Record = { default: g.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, very_high_latency: g.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY, @@ -755,55 +785,41 @@ interface SessionControl { const SESSION_CONTROL = new WeakMap(); -// ---- Session --------------------------------------------------------------- - -export class Session { +/** + * One native session handle under the model's compute rules, shared by Session + * and DiarizeSession. Each holds it in a private field, so none of this is + * reachable from user code. `setAbort` is the role's set_abort_callback. + */ +class SessionCore { #n: Native; #h: any; - #model: TranscribeModel; // keep the model alive while this session lives #lock: Mutex; // shared with the model; serializes compute model-wide - #untrack: (self: Session) => void; // drop self from the model's session set + #setAbort: any; #inFlight: string | null = null; // set while a native call runs on a worker - #activeStream: Stream | null = null; // current wrapper for the session's native stream slot #disposed = false; - /** @internal */ - constructor( - n: Native, - model: TranscribeModel, - handle: any, - lock: Mutex, - untrack: (self: Session) => void, - ) { + constructor(n: Native, handle: any, lock: Mutex, setAbort: any) { this.#n = n; - this.#model = model; this.#h = handle; this.#lock = lock; - this.#untrack = untrack; - SESSION_CONTROL.set(this, { - exclusive: (gate, body) => this.#exclusive(gate, body), - currentCompute: () => this.#inFlight, - isCurrentStream: (stream) => this.#activeStream === stream, - replaceCurrentStream: (stream) => { - if (this.#activeStream && this.#activeStream !== stream) { - STREAM_TEARDOWN.get(this.#activeStream)?.invalidate(); - } - this.#activeStream = stream; - }, - clearCurrentStream: (stream) => { - if (this.#activeStream === stream) this.#activeStream = null; - }, - }); + this.#setAbort = setAbort; } - /** @internal */ get handle(): any { if (this.#disposed) throw new TranscribeError("session has been disposed"); return this.#h; } + get disposed(): boolean { + return this.#disposed; + } + + get inFlight(): string | null { + return this.#inFlight; + } + /** Reads touch the session; forbidden while a worker call is in flight. */ - #assertNotComputing(what: string): void { + assertNotComputing(what: string): void { if (this.#inFlight) { throw new TranscribeError( `cannot read session ${what} while ${this.#inFlight} is in flight; await it first`, @@ -813,19 +829,20 @@ export class Session { /** * Run `body` as this session's one native compute: the single path every - * run / batch / stream begin / feed / finalize takes (Stream reaches it via - * SESSION_CONTROL). It queues on the model-wide FIFO lock, so no two - * computes on one model overlap and queued native frees (deferFree) run only - * after it drains. Once the lock is held, `gate` is applied — disposal, then - * the stream lease — before `body` runs. `body` opens the in-flight window - * (abort callback + in-flight mark) around its worker await, and copies its - * result out before returning, i.e. before the lock is released. + * run / batch / stream begin / feed / finalize / diarize run takes (Stream + * reaches it via SESSION_CONTROL). It queues on the model-wide FIFO lock, so + * no two computes on one model overlap and queued native frees (deferFree) + * run only after it drains. Once the lock is held, `gate` is applied — + * disposal, then the stream lease — before `body` runs. `body` opens the + * in-flight window (abort callback + in-flight mark) around its worker + * await, and copies its result out before returning, i.e. before the lock + * is released. * * Deliberately not `async`: refusals are returned as rejected promises and * `body`'s promise is returned as-is, so settle timing matches the inline * lock bodies this replaced. */ - #exclusive( + exclusive( gate: ComputeGate, body: (open: ComputeWindow) => Promise, ): Promise { @@ -838,7 +855,7 @@ export class Session { }); } - /** See ComputeWindow. Only valid inside an #exclusive body. */ + /** See ComputeWindow. Only valid inside an exclusive() body. */ #openComputeWindow: ComputeWindow = (kind, signal) => { const cancel = this.#installAbort(signal); this.#inFlight = kind; @@ -848,8 +865,94 @@ export class Session { }; }; + /** + * Wire an AbortSignal to a native abort callback for one run. The callback is + * installed on *this session's* handle, but install/run/uninstall is only safe + * because every caller holds the model-wide #lock for the whole run — the lock, + * not the per-session handle, is what guarantees no run overlaps the window + * between setAbort(cb) and setAbort(null). A future change that relaxes the + * lock must keep this install/uninstall paired within one run. + */ + #installAbort(signal?: AbortSignal): (() => void) | null { + if (!signal) return null; + const n = this.#n; + const flag = { aborted: signal.aborted }; + const onAbort = () => { + flag.aborted = true; + }; + signal.addEventListener("abort", onAbort, { once: true }); + const cbPtr = n.koffi.register( + () => flag.aborted, + n.koffi.pointer(n.abortProto), + ); + this.#setAbort(this.handle, cbPtr, null); + return () => { + this.#setAbort(this.handle, null, null); + n.koffi.unregister(cbPtr); + signal.removeEventListener("abort", onAbort); + }; + } + + /** + * Mark disposed now (use-after-dispose throws immediately) and free the + * native handle behind the model lock: a worker may still hold it, and a + * free mid-call is a use-after-free. Queuing on the FIFO lock runs the free + * (then `after`) once any in-flight and queued compute drains. + */ + dispose(free: (h: any) => void, after?: () => void): void { + this.#disposed = true; + const h = this.#h; + this.#h = null; + deferFree(this.#lock, () => free(h), after); + } +} + +// ---- Session --------------------------------------------------------------- + +export class Session { + #n: Native; + #core: SessionCore; + #model: TranscribeModel; // keep the model alive while this session lives + #lock: Mutex; // shared with the model; serializes compute model-wide + #untrack: (self: Session) => void; // drop self from the model's session set + #activeStream: Stream | null = null; // current wrapper for the session's native stream slot + + /** @internal */ + constructor( + n: Native, + model: TranscribeModel, + handle: any, + lock: Mutex, + untrack: (self: Session) => void, + ) { + this.#n = n; + this.#model = model; + this.#core = new SessionCore(n, handle, lock, n.F.setAbortCallback); + this.#lock = lock; + this.#untrack = untrack; + SESSION_CONTROL.set(this, { + exclusive: (gate, body) => this.#core.exclusive(gate, body), + currentCompute: () => this.#core.inFlight, + isCurrentStream: (stream) => this.#activeStream === stream, + replaceCurrentStream: (stream) => { + if (this.#activeStream && this.#activeStream !== stream) { + STREAM_TEARDOWN.get(this.#activeStream)?.invalidate(); + } + this.#activeStream = stream; + }, + clearCurrentStream: (stream) => { + if (this.#activeStream === stream) this.#activeStream = null; + }, + }); + } + + /** @internal */ + get handle(): any { + return this.#core.handle; + } + get limits(): SessionLimits { - this.#assertNotComputing("limits"); + this.#core.assertNotComputing("limits"); const n = this.#n; const l: any = {}; n.F.sessionLimitsInit(l); @@ -877,7 +980,7 @@ export class Session { const p = this.#buildRunParams(opts); - return this.#exclusive({ recheckDisposed: true, busyOp: "run" }, async (open) => { + return this.#core.exclusive({ recheckDisposed: true, busyOp: "run" }, async (open) => { const close = open("run()", opts.signal); let status: number; try { @@ -972,7 +1075,7 @@ export class Session { const counts = Int32Array.from(arrays, (a) => a.length); const p = this.#buildRunParams(opts); - return this.#exclusive({ recheckDisposed: true, busyOp: "runBatch" }, async (open) => { + return this.#core.exclusive({ recheckDisposed: true, busyOp: "runBatch" }, async (open) => { const close = open("runBatch()", opts.signal); let status: number; try { @@ -1065,7 +1168,7 @@ export class Session { // The gate rechecks disposal inside the lock: dispose() may have run after // we captured `h` but before this queued body — don't begin a stream on a // dead session. Begin is a synchronous native call, so no in-flight window. - return this.#exclusive({ recheckDisposed: true, busyOp: "begin a stream" }, async () => { + return this.#core.exclusive({ recheckDisposed: true, busyOp: "begin a stream" }, async () => { check(n, F.streamBegin(h, rp, sp), "transcribe_stream_begin"); this.#lock.streamActive = true; // claim the lease for the whole stream lifetime // The Stream holds the Session (not a raw handle) so its calls fail fast @@ -1078,42 +1181,13 @@ export class Session { }).finally(() => freeRunParams(n, rp)); // begin copied the prompting strings } - /** - * Wire an AbortSignal to a native abort callback for one run. The callback is - * installed on *this session's* handle, but install/run/uninstall is only safe - * because every caller holds the model-wide #lock for the whole run — the lock, - * not the per-session handle, is what guarantees no run overlaps the window - * between setAbortCallback(cb) and setAbortCallback(null). A future change that - * relaxes the lock must keep this install/uninstall paired within one run. - */ - #installAbort(signal?: AbortSignal): (() => void) | null { - if (!signal) return null; - const n = this.#n; - const flag = { aborted: signal.aborted }; - const onAbort = () => { - flag.aborted = true; - }; - signal.addEventListener("abort", onAbort, { once: true }); - const cbPtr = n.koffi.register( - () => flag.aborted, - n.koffi.pointer(n.abortProto), - ); - n.F.setAbortCallback(this.handle, cbPtr, null); - return () => { - n.F.setAbortCallback(this.handle, null, null); - n.koffi.unregister(cbPtr); - signal.removeEventListener("abort", onAbort); - }; - } - get wasAborted(): boolean { - this.#assertNotComputing("wasAborted"); + this.#core.assertNotComputing("wasAborted"); return this.#n.F.wasAborted(this.handle); } dispose(): void { - if (this.#disposed) return; - this.#disposed = true; + if (this.#core.disposed) return; this.#untrack(this); // stop the model from holding a dead session // Deactivate any live stream NOW (its reset() no-ops, reads throw via the // disposed handle), but release its model lease only inside the deferred @@ -1123,19 +1197,7 @@ export class Session { this.#activeStream = null; const teardown = stream ? STREAM_TEARDOWN.get(stream) : undefined; teardown?.deactivate(); - // Free behind the model lock: a run/feed worker may still hold this handle, - // and sessionFree mid-call is a use-after-free. Queuing on the FIFO lock - // runs the free after any in-flight (and queued) compute drains. The JS-side - // guard is already synchronous (#disposed/handle), so use-after-dispose - // still throws immediately; only the native free + lease release are deferred. - const n = this.#n; - const h = this.#h; - this.#h = null; - deferFree( - this.#lock, - () => n.F.sessionFree(h), - () => teardown?.releaseLease(), - ); + this.#core.dispose(this.#n.F.sessionFree, () => teardown?.releaseLease()); } [Symbol.dispose](): void { @@ -1380,13 +1442,84 @@ export class Stream { } } +// ---- DiarizeSession -------------------------------------------------------- + +/** A DIARIZE-role session: who spoke when. Same compute rules as Session. */ +export class DiarizeSession { + #n: Native; + #core: SessionCore; + #model: TranscribeModel; // keep the model alive while this session lives + #untrack: (self: DiarizeSession) => void; + + /** @internal */ + constructor( + n: Native, + model: TranscribeModel, + handle: any, + lock: Mutex, + untrack: (self: DiarizeSession) => void, + ) { + this.#n = n; + this.#model = model; + this.#core = new SessionCore(n, handle, lock, n.F.diarizeSetAbortCallback); + this.#untrack = untrack; + } + + /** + * Diarize one recording; returns its speaker turns, grouped by speaker and + * time-ordered within a speaker (turns of different speakers may overlap). + * The input PCM is borrowed, not copied (see Session.run). + */ + async run(pcm: PcmLike, opts: DiarizeOptions = {}): Promise { + const n = this.#n; + const F = n.F; + const h = this.#core.handle; + const samples = toFloat32(pcm); + const p: any = {}; + F.diarizeParamsInit(p); + if (opts.family) + p.family = buildFamily(n, this.#model.handle, opts.family, "diarize_run"); + + return this.#core.exclusive({ recheckDisposed: true, busyOp: "diarize" }, async (open) => { + const close = open("run()", opts.signal); + let status: number; + try { + status = await callAsync(F.diarizeRun, h, samples, samples.length, p); + } finally { + close(); + } + check(n, status, "transcribe_diarize_run"); + return readSpeakerSegments(n, F.diarizeNSegments(h), (i, o) => + F.diarizeGetSegment(h, i, o), + ); + }); + } + + /** load_ms plus the last run's mel / encode time. */ + get timings(): Timings { + this.#core.assertNotComputing("timings"); + const h = this.#core.handle; + return readTimings(this.#n, (o) => this.#n.F.diarizeGetTimings(h, o)); + } + + dispose(): void { + if (this.#core.disposed) return; + this.#untrack(this); + this.#core.dispose(this.#n.F.diarizeSessionFree); + } + + [Symbol.dispose](): void { + this.dispose(); + } +} + // ---- Model ----------------------------------------------------------------- export class TranscribeModel { #n: Native; #h: any; #disposed = false; - #sessions = new Set(); + #sessions = new Set(); #lock = new Mutex(); // serializes compute across all sessions of this model private constructor(n: Native, handle: any) { @@ -1468,6 +1601,38 @@ export class TranscribeModel { } } + /** The roles this model serves; ASR calls on a model without "asr" throw UnsupportedRole. */ + get roles(): readonly Role[] { + const mask = this.#n.F.modelRoles(this.handle); + return (Object.keys(ROLES) as Role[]).filter((r) => mask & ROLES[r]); + } + + /** Static facts of a "diarize" model; UnsupportedRole otherwise. */ + get diarizeInfo(): DiarizeInfo { + const n = this.#n; + const info: any = {}; + n.F.diarizeInfoInit(info); + check(n, n.F.diarizeGetInfo(this.handle, info), "reading diarize info"); + return { sampleRate: info.sample_rate, maxSpeakers: info.max_speakers }; + } + + /** Open a DIARIZE-role session; UnsupportedRole on a model without "diarize". */ + createDiarizeSession(opts: DiarizeSessionOptions = {}): DiarizeSession { + const n = this.#n; + const p: any = {}; + n.F.diarizeSessionParamsInit(p); + if (opts.nThreads !== undefined) p.n_threads = opts.nThreads; + const out: any[] = [null]; + check(n, n.F.diarizeSessionInit(this.handle, p, out), "opening diarize session"); + if (!out[0]) + throw new TranscribeError("diarize session init returned a null handle"); + const session = new DiarizeSession(n, this, out[0], this.#lock, (s) => + this.#sessions.delete(s), + ); + this.#sessions.add(session); + return session; + } + get capabilities(): Capabilities { const n = this.#n; const c: any = {}; diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index d986a5c8f..99ec3418c 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -197,7 +197,7 @@ export type BatchItem = export type CommitPolicy = "auto" | "on_finalize" | "stable_prefix"; export type StreamState = "idle" | "active" | "finished" | "failed"; -export type ExtSlot = "run" | "stream"; +export type ExtSlot = "run" | "stream" | "diarize_run"; export interface StreamUpdate { resultChanged: boolean; @@ -281,6 +281,10 @@ export type SortformerPreset = export interface SortformerStreamOptions { preset?: SortformerPreset; } +/** Sortformer options for {@link DiarizeOptions.family} (diarize_run slot). */ +export interface SortformerDiarizeOptions { + preset?: SortformerPreset; +} export type FamilyExtension = | ({ kind: "whisper" } & WhisperRunOptions) @@ -288,4 +292,29 @@ export type FamilyExtension = | ({ kind: "parakeet" } & ParakeetStreamOptions) | ({ kind: "parakeet_buffered" } & ParakeetBufferedStreamOptions) | ({ kind: "voxtral" } & VoxtralRealtimeStreamOptions) - | ({ kind: "sortformer" } & SortformerStreamOptions); + | ({ kind: "sortformer" } & SortformerStreamOptions) + | ({ kind: "sortformer_diarize" } & SortformerDiarizeOptions); + +// ---- roles ----------------------------------------------------------------- + +/** What a model serves: "asr" (transcription) and/or "diarize" (speaker turns). */ +export type Role = "asr" | "diarize"; + +export interface DiarizeInfo { + /** Input PCM rate. */ + sampleRate: number; + /** Speaker ids are in [1, maxSpeakers]. */ + maxSpeakers: number; +} + +export interface DiarizeSessionOptions { + /** CPU threads for CPU-side ops; 0 = library default. */ + nThreads?: number; +} + +export interface DiarizeOptions { + /** Cancel the run cooperatively. */ + signal?: AbortSignal; + /** A diarize_run-slot family extension (e.g. sortformer_diarize). */ + family?: FamilyExtension; +} diff --git a/bindings/typescript/test/common.mjs b/bindings/typescript/test/common.mjs index e26d09cae..4d70ffee4 100644 --- a/bindings/typescript/test/common.mjs +++ b/bindings/typescript/test/common.mjs @@ -22,6 +22,9 @@ export const ITN_MODEL = // Voxtral realtime is local-only (~2.5 GB+, too heavy for the CI canary set): // its env var is NOT exported by fetch-canary, so this skips cleanly in CI. export const VOXTRAL_MODEL = process.env.TRANSCRIBE_SMOKE_VOXTRAL_MODEL || ""; +// Sortformer (ASR + DIARIZE roles) is local-only too. +export const SORTFORMER_MODEL = process.env.TRANSCRIBE_SMOKE_SORTFORMER_MODEL || ""; +export const SORTFORMER_AUDIO = path.resolve(HERE, "../../../samples/sortformer-2spk-mix.wav"); // jfk.wav ships in-repo; fetch-canary exports only the model paths. export const AUDIO = diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs new file mode 100644 index 000000000..a352465a3 --- /dev/null +++ b/bindings/typescript/test/diarize.test.mjs @@ -0,0 +1,175 @@ +// DIARIZE role: model.roles, diarizeInfo, DiarizeSession, and the compute +// rules a diarize run shares with Session (lock, in-flight mark, abort, +// disposed recheck, deferred free). + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { getEventListeners } from "node:events"; +import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav, jfk } from "./common.mjs"; +import { + TranscribeModel, + DiarizeSession, + Aborted, + Busy, + InvalidArgument, + TranscribeError, + UnsupportedRole, +} from "../dist/index.js"; + +const mix = () => readWav(SORTFORMER_AUDIO); + +test("DiarizeSession exposes only its public surface", () => { + assert.deepEqual(Object.getOwnPropertyNames(DiarizeSession.prototype).sort(), [ + "constructor", + "dispose", + "run", + "timings", + ]); +}); + +modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + assert.deepEqual(m.roles, ["asr"]); + assert.throws(() => m.diarizeInfo, UnsupportedRole); + assert.throws(() => m.createDiarizeSession(), UnsupportedRole); + assert.equal(m.accepts({ kind: "sortformer_diarize" }), false); + await assert.rejects( + () => m.createSession().run(jfk(), { family: { kind: "sortformer_diarize" } }), + InvalidArgument, + ); + } finally { + m.dispose(); + } +}); + +modelTest("sortformer serves asr and diarize; diarizeInfo", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + assert.deepEqual(m.roles, ["asr", "diarize"]); + assert.deepEqual(m.diarizeInfo, { sampleRate: 16000, maxSpeakers: 4 }); + assert.equal(m.accepts({ kind: "sortformer_diarize" }), true); + } finally { + m.dispose(); + } +}); + +// Compares against the ASR path (Session.run with the SFST preset), which a +// later commit removes; drop the comparison then. +modelTest("diarize run returns the ASR path's speaker turns for the same preset", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const pcm = mix(); + const d = m.createDiarizeSession({ nThreads: 4 }); + const s = m.createSession(); + for (const preset of ["default", "low_latency"]) { + const rows = await d.run(pcm, { family: { kind: "sortformer_diarize", preset } }); + assert.ok(rows.length > 0, preset); + for (const r of rows) assert.ok(r.speakerId >= 1 && r.speakerId <= 4 && r.t1Ms > r.t0Ms); + const asr = await s.run(pcm, { family: { kind: "sortformer", preset } }); + assert.deepEqual(rows, asr.speakerSegments, preset); + } + assert.ok(d.timings.encodeMs > 0); + d.dispose(); + s.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const d = m.createDiarizeSession(); + const pcm = mix(); + await assert.rejects( + () => d.run(pcm, { family: { kind: "sortformer_diarize", preset: "ultra_low_latency" } }), + (e) => e instanceof TranscribeError && /invalid sortformer preset/.test(e.message), + ); + await assert.rejects(() => d.run(pcm, { family: { kind: "sortformer" } }), InvalidArgument); + assert.ok((await d.run(pcm)).length > 0); + d.dispose(); + } finally { + m.dispose(); + } +}); + +// ---- compute rules shared with Session ------------------------------------- +// Busy is not covered here: Sortformer cannot begin a stream, so no model +// serves both a stream lease and the diarize role. The refusal is the same +// SessionCore.exclusive gate compute-rules.test.mjs covers for Session. + +// Also uses the Sortformer ASR path (Session.run); drop it with that path. +modelTest("a diarize run and an ASR run on one model are exclusive", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const d = m.createDiarizeSession(); + const s = m.createSession(); + const order = []; + const pd = d.run(mix()).then((r) => (order.push("d"), r)); + const ps = s.run(mix()).then((r) => (order.push("s"), r)); + await Promise.resolve(); + assert.throws(() => d.timings, /run\(\).*in flight/); + assert.doesNotThrow(() => s.limits); + const [rows, r] = await Promise.all([pd, ps]); + assert.deepEqual(order, ["d", "s"]); + assert.deepEqual(rows, r.speakerSegments); + } finally { + m.dispose(); + } +}); + +modelTest("the abort listener lives only for the call; a pre-aborted run raises Aborted", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const d = m.createDiarizeSession(); + const ac = new AbortController(); + const p = d.run(mix(), { signal: ac.signal }); + await Promise.resolve(); + assert.equal(getEventListeners(ac.signal, "abort").length, 1); + assert.ok((await p).length > 0); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + + ac.abort(); + await assert.rejects(() => d.run(mix(), { signal: ac.signal }), Aborted); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + assert.ok((await d.run(mix())).length > 0, "the next run is not aborted"); + } finally { + m.dispose(); + } +}); + +modelTest("a diarize run queued before its dispose is rejected as disposed", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const running = m.createDiarizeSession().run(mix()); + const d = m.createDiarizeSession(); + const ac = new AbortController(); + const queued = d.run(mix(), { signal: ac.signal }); + d.dispose(); + await assert.rejects(queued, (e) => !(e instanceof Busy) && /disposed/i.test(e.message)); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + await running; + await assert.rejects(() => d.run(mix()), /disposed/); + } finally { + m.dispose(); + } +}); + +modelTest("dispose during an in-flight diarize run defers the free; the rows survive", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + const d = m.createDiarizeSession(); + const p = d.run(mix()); + await Promise.resolve(); + d.dispose(); + assert.throws(() => d.timings, /run\(\).*in flight/); + assert.ok((await p).length > 0); + assert.throws(() => d.timings, /disposed/); + + const d2 = m.createDiarizeSession(); + const p2 = d2.run(mix()); + await Promise.resolve(); + m.dispose(); // disposes d2 too; both native frees queue behind the run + assert.throws(() => m.diarizeInfo, /disposed/); + assert.ok((await p2).length > 0); +}); From f562481d3c65935d8f2e5d7adfb6b24698a1cf8d Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:22:46 +0800 Subject: [PATCH 30/51] docs: bindings contract for roles and the DIARIZE session --- docs/bindings.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/bindings.md b/docs/bindings.md index 51bd6d5c8..b801c55ee 100644 --- a/docs/bindings.md +++ b/docs/bindings.md @@ -96,6 +96,28 @@ explicit unsupported combinations return `UNSUPPORTED_TIMESTAMPS`; `AUTO` chooses the richest granularity compatible with the selected task. Result objects own copies of every row and remain valid after the next run. +That contract is for ASR models that attribute speakers inside a transcript. + +## Roles and the DIARIZE session + +Each binding exposes the role mask as `Model.roles` (Python `frozenset[Role]`, +TypeScript `readonly ('asr' | 'diarize')[]`, Rust `Roles`, Swift `Roles` +option set). Status 20 surfaces as `UnsupportedRole` (`.unsupportedRole` in +Swift), including from capabilities on a model without ASR. + +A DIARIZE model (`docs/roles.md`) gets its own session type: `DiarizeSession` +from `model.diarize_session()` (Python, Rust), `model.diarizeSession()` +(Swift) or `model.createDiarizeSession()` (TypeScript), plus `diarize_info` / +`diarizeInfo` (sample rate, max speakers). `run(pcm, …)` returns copied-out +speaker-turn rows (the same row type as above) and takes the family's +diarize extension (Sortformer: the preset on the `diarize_run` slot). + +A diarize run follows the same execution rules as an ASR run: it holds the +model-wide compute lock, waits behind other compute on the model, raises +`Busy` while a stream on any session of the model holds the stream lease, +keeps its model alive, honours cancellation, and defers native frees that +race an in-flight call. + ## Raw text Every first-class binding exposes `raw_text` / `rawText` on the materialized From 4535564050b6d2d149b95fecaced4cee01c03b18 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:23:16 +0800 Subject: [PATCH 31/51] roles: break: Sortformer is DIARIZE-only; retire SFST Breaking (0.4.0). Sortformer drops the ASR role: transcribe_session_init / transcribe_open / transcribe_model_get_capabilities return UNSUPPORTED_ROLE for it, and its ASR path (init_context, run, run_validate, the dual-session plumbing) is gone; one SortformerSession serves transcribe_diarize_run. The SFST RUN-slot kind and transcribe_sortformer_stream_ext are removed and the kind is retired in docs/extension-kinds.md (SFDR replaces it). Sortformer no longer sets capabilities or feature bits (capabilities.cpp removed), so TRANSCRIBE_FEATURE_DIARIZATION keeps its ASR meaning only. transcribe-cli routes the DIARIZE-only model to the diarize path by itself, so validate.py (its --role flag dropped again) and scripts/diar/run_cpp_sortformer.py work unchanged; the C++ dumps on the oracle mix are byte-identical to main's. Batch mode stays ASR-only. tests: sortformer_stream_ext_unit removed (its env-preset parity check moves into sortformer_diarize_unit); sortformer_diarize_unit now checks the DIARIZE-only roles, ASR refusal, golden segments at DEFAULT and LOW_LATENCY, ext == TRANSCRIBE_SORTFORMER_STREAM_PRESET env, pre-clear rejection, and cross-role concurrency. Low-level bindings regenerated; the binding-side removals follow per binding. docs: docs/migrating-to-0.4.md; Sortformer model page, HF card and family doc moved to the diarize API; env-var table. --- .../python/src/transcribe_cpp/_generated.py | 9 +- bindings/rust/sys/src/transcribe_sys.rs | 25 +-- bindings/typescript/src/_generated.ts | 6 +- docs/environment-variables.md | 2 +- docs/extension-kinds.md | 2 +- docs/migrating-to-0.4.md | 66 ++++++ .../diar_streaming_sortformer_4spk-v2.1.md | 23 ++- docs/porting/families/sortformer.md | 22 +- include/transcribe.abihash | 2 +- include/transcribe/sortformer.h | 38 +--- .../diar_streaming_sortformer_4spk-v2.1.yaml | 21 +- scripts/validate.py | 2 - src/CMakeLists.txt | 1 - src/arch/sortformer/capabilities.cpp | 35 ---- src/arch/sortformer/model.cpp | 123 ++---------- src/arch/sortformer/sortformer.h | 25 +-- tests/CMakeLists.txt | 22 +- tests/sortformer_diarize_unit.cpp | 69 +++---- tests/sortformer_stream_ext_unit.cpp | 188 ------------------ 19 files changed, 166 insertions(+), 515 deletions(-) create mode 100644 docs/migrating-to-0.4.md delete mode 100644 src/arch/sortformer/capabilities.cpp delete mode 100644 tests/sortformer_stream_ext_unit.cpp diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 25aaa98c6..511b401fa 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "c2979c7e891982ee" +PUBLIC_HEADER_HASH = "30ec88d3b51fc344" # === enum constants === TRANSCRIBE_OK = 0 @@ -127,7 +127,6 @@ TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584 TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912 TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163 -TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635 TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710 TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319 @@ -174,8 +173,6 @@ class transcribe_parakeet_stream_ext(_c.Structure): pass class transcribe_parakeet_buffered_stream_ext(_c.Structure): pass -class transcribe_sortformer_stream_ext(_c.Structure): - pass class transcribe_sortformer_diarize_ext(_c.Structure): pass class transcribe_voxtral_realtime_stream_ext(_c.Structure): @@ -206,7 +203,6 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_moonshine_streaming_stream_ext._fields_ = [("ext", transcribe_ext), ("min_decode_interval_ms", _c.c_int32)] transcribe_parakeet_stream_ext._fields_ = [("ext", transcribe_ext), ("att_context_right", _c.c_int32)] transcribe_parakeet_buffered_stream_ext._fields_ = [("ext", transcribe_ext), ("left_ms", _c.c_int32), ("chunk_ms", _c.c_int32), ("right_ms", _c.c_int32)] -transcribe_sortformer_stream_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] transcribe_sortformer_diarize_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] transcribe_voxtral_realtime_stream_ext._fields_ = [("ext", transcribe_ext), ("num_delay_tokens", _c.c_int32), ("min_decode_interval_ms", _c.c_int32)] transcribe_whisper_run_ext._fields_ = [("ext", transcribe_ext), ("initial_prompt", _c.c_char_p), ("prompt_tokens", _c.POINTER(_c.c_int32)), ("n_prompt_tokens", _c.c_size_t), ("prompt_condition", _c.c_int), ("condition_on_prev_tokens", _c.c_bool), ("max_prev_context_tokens", _c.c_int32), ("temperature", _c.c_float), ("temperature_inc", _c.c_float), ("compression_ratio_thold", _c.c_float), ("logprob_thold", _c.c_float), ("no_speech_thold", _c.c_float), ("seed", _c.c_uint32), ("max_initial_timestamp", _c.c_float)] @@ -258,7 +254,6 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_moonshine_streaming_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'min_decode_interval_ms': 16}}, 'transcribe_parakeet_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'att_context_right': 16}}, 'transcribe_parakeet_buffered_stream_ext': {'size': 32, 'align': 8, 'offsets': {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24}}, - 'transcribe_sortformer_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, 'transcribe_sortformer_diarize_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, 'transcribe_voxtral_realtime_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20}}, 'transcribe_whisper_run_ext': {'size': 80, 'align': 8, 'offsets': {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76}}, @@ -436,8 +431,6 @@ def configure(lib): lib.transcribe_set_abort_callback.argtypes = [_c.c_void_p, _c.CFUNCTYPE(_c.c_bool, _c.c_void_p), _c.c_void_p] lib.transcribe_sortformer_diarize_ext_init.restype = None lib.transcribe_sortformer_diarize_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_diarize_ext)] - lib.transcribe_sortformer_stream_ext_init.restype = None - lib.transcribe_sortformer_stream_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_stream_ext)] lib.transcribe_speaker_segment_init.restype = None lib.transcribe_speaker_segment_init.argtypes = [_c.POINTER(transcribe_speaker_segment)] lib.transcribe_status_string.restype = _c.c_char_p diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index 522cf1871..c4daa925a 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,18 +1,17 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = c2979c7e891982ee +// Pinned to include/transcribe.abihash = 30ec88d3b51fc344 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "c2979c7e891982ee"; +pub const PUBLIC_HEADER_HASH: &str = "30ec88d3b51fc344"; /* automatically generated by rust-bindgen 0.72.1 */ pub const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM: u32 = 1414746957; pub const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM: u32 = 1414744912; pub const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM: u32 = 1396853584; -pub const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM: u32 = 1414743635; pub const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE: u32 = 1380206163; pub const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM: u32 = 1414746710; pub const TRANSCRIBE_EXT_KIND_WHISPER_RUN: u32 = 1314015319; @@ -1329,26 +1328,6 @@ impl transcribe_sortformer_preset { pub struct transcribe_sortformer_preset(pub ::std::os::raw::c_uint); #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct transcribe_sortformer_stream_ext { - pub ext: transcribe_ext, - pub preset: transcribe_sortformer_preset, -} -#[allow(clippy::unnecessary_operation, clippy::identity_op)] -const _: () = { - ["Size of transcribe_sortformer_stream_ext"] - [::std::mem::size_of::() - 24usize]; - ["Alignment of transcribe_sortformer_stream_ext"] - [::std::mem::align_of::() - 8usize]; - ["Offset of field: transcribe_sortformer_stream_ext::ext"] - [::std::mem::offset_of!(transcribe_sortformer_stream_ext, ext) - 0usize]; - ["Offset of field: transcribe_sortformer_stream_ext::preset"] - [::std::mem::offset_of!(transcribe_sortformer_stream_ext, preset) - 16usize]; -}; -unsafe extern "C" { - pub fn transcribe_sortformer_stream_ext_init(ext: *mut transcribe_sortformer_stream_ext); -} -#[repr(C)] -#[derive(Debug, Copy, Clone)] pub struct transcribe_sortformer_diarize_ext { pub ext: transcribe_ext, pub preset: transcribe_sortformer_preset, diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index 32d5f5c13..1f98bdb6f 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "c2979c7e891982ee"; +export const PUBLIC_HEADER_HASH = "30ec88d3b51fc344"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -125,7 +125,6 @@ export const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM = 1414746957; export const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584; export const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912; export const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163; -export const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635; export const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710; export const TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319; @@ -152,7 +151,6 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_moonshine_streaming_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'min_decode_interval_ms': 16} }, 'transcribe_parakeet_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'att_context_right': 16} }, 'transcribe_parakeet_buffered_stream_ext': { size: 32, align: 8, offsets: {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24} }, - 'transcribe_sortformer_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, 'transcribe_sortformer_diarize_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, 'transcribe_voxtral_realtime_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20} }, 'transcribe_whisper_run_ext': { size: 80, align: 8, offsets: {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76} }, @@ -204,7 +202,6 @@ export function defineTypes(koffi: any): Record { T['transcribe_moonshine_streaming_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], min_decode_interval_ms: 'int32_t' }); T['transcribe_parakeet_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], att_context_right: 'int32_t' }); T['transcribe_parakeet_buffered_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], left_ms: 'int32_t', chunk_ms: 'int32_t', right_ms: 'int32_t' }); - T['transcribe_sortformer_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); T['transcribe_sortformer_diarize_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); T['transcribe_voxtral_realtime_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], num_delay_tokens: 'int32_t', min_decode_interval_ms: 'int32_t' }); T['transcribe_whisper_run_ext'] = koffi.struct({ ext: T['transcribe_ext'], initial_prompt: 'char *', prompt_tokens: 'void *', n_prompt_tokens: 'size_t', prompt_condition: 'int', condition_on_prev_tokens: 'bool', max_prev_context_tokens: 'int32_t', temperature: 'float', temperature_inc: 'float', compression_ratio_thold: 'float', logprob_thold: 'float', no_speech_thold: 'float', seed: 'uint32_t', max_initial_timestamp: 'float' }); @@ -298,7 +295,6 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_session_params_init': { ret: 'void', args: ['struct transcribe_session_params *'] }, 'transcribe_set_abort_callback': { ret: 'void', args: ['struct transcribe_session *', 'transcribe_abort_callback', 'void *'] }, 'transcribe_sortformer_diarize_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_diarize_ext *'] }, - 'transcribe_sortformer_stream_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_stream_ext *'] }, 'transcribe_speaker_segment_init': { ret: 'void', args: ['struct transcribe_speaker_segment *'] }, 'transcribe_status_string': { ret: 'const char *', args: ['int'] }, 'transcribe_stream_begin': { ret: 'transcribe_status', args: ['struct transcribe_session *', 'const struct transcribe_run_params *', 'const struct transcribe_stream_params *'] }, diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 848a6257a..74a1691b0 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -85,7 +85,7 @@ its var is unset. Convention: `TRANSCRIBE__GGUF`. | `TRANSCRIBE_PARAKEET_UNIFIED_GGUF` | `parakeet_buffered_stream_eos_smoke`, `stream_offline_interleave_smoke` | | `TRANSCRIBE_GIGAAM_GGUF` | `gigaam_workspace_release_smoke` | | `TRANSCRIBE_MULTITALKER_BUNDLE_GGUF` | `parakeet_multitalker_e2e_smoke` | -| `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_stream_ext_unit` | +| `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_diarize_unit`, `cli_diarize_smoke` | | `TRANSCRIBE_COHERE_GGUF` | `cohere_real_smoke`, `cohere_e2e_smoke` | | `TRANSCRIBE_GRANITE5_CTC_GGUF` | `granite5_ctc_real_smoke`, `granite5_ctc_e2e_smoke` | | `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity` | diff --git a/docs/extension-kinds.md b/docs/extension-kinds.md index 416f5e27e..c2df730cf 100644 --- a/docs/extension-kinds.md +++ b/docs/extension-kinds.md @@ -49,7 +49,7 @@ parameter; existing kinds keep their slot for life. | `0x5453534D` | `MSST` | `STREAM` | `TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM` | moonshine_streaming | `include/transcribe/moonshine_streaming.h` | | `0x4E524857` | `WHRN` | `RUN` | `TRANSCRIBE_EXT_KIND_WHISPER_RUN` | whisper | `include/transcribe/whisper.h` | | `0x54535256` | `VRST` | `STREAM` | `TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM` | voxtral_realtime | `include/transcribe/voxtral_realtime.h` | -| `0x54534653` | `SFST` | `RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM` | sortformer | `include/transcribe/sortformer.h` | +| `0x54534653` | `SFST` | `RUN` | retired in 0.4 (was `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM`); never reuse | sortformer | - | | `0x52444653` | `SFDR` | `DIARIZE_RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE` | sortformer | `include/transcribe/sortformer.h` | Empty rows reserved for future allocations: diff --git a/docs/migrating-to-0.4.md b/docs/migrating-to-0.4.md new file mode 100644 index 000000000..c74dc0e09 --- /dev/null +++ b/docs/migrating-to-0.4.md @@ -0,0 +1,66 @@ +# Migrating to transcribe.cpp 0.4 + +Version 0.4 introduces **roles** (`docs/roles.md`): a loaded model serves +one or more kinds of work, each with its own API and session type. Rebuild +native consumers against the 0.4 headers and upgrade each language package +and native provider together. + +## Diarizers moved to the DIARIZE role + +Models whose product is speaker turns (Sortformer) no longer run through the +ASR API. `transcribe_model_roles()` reports `TRANSCRIBE_ROLE_DIARIZE` for +them, and `transcribe_session_init` / `transcribe_open` return +`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`. + +| 0.3 | 0.4 | +| --- | --- | +| `transcribe_session_init(model, ...)` | `transcribe_diarize_session_init(model, ...)` | +| `transcribe_run(session, pcm, n, &run_params)` | `transcribe_diarize_run(session, pcm, n, &diarize_params)` | +| `transcribe_n_speaker_segments` / `transcribe_get_speaker_segment` | `transcribe_diarize_n_segments` / `transcribe_diarize_get_segment` (same `transcribe_speaker_segment` rows) | +| `transcribe_sortformer_stream_ext` (`SFST`, RUN slot) | `transcribe_sortformer_diarize_ext` (`SFDR`, DIARIZE_RUN slot); same preset enum | +| `transcribe_model_supports(sortformer, TRANSCRIBE_FEATURE_DIARIZATION)` was true | false; check `transcribe_model_roles()` instead | +| `transcribe-cli --batch ... --batch-jsonl` "speakers" for Sortformer | `transcribe-cli -m sortformer.gguf audio.wav` prints the segments | + +Segments are byte-identical to 0.3 for the same audio, preset and backend. +ASR models that attribute speakers inside a transcript (granite, moss, +multitalker parakeet) are unchanged: they keep `run_params.diarize`, +`TRANSCRIBE_FEATURE_DIARIZATION` and the ASR speaker accessors. + +## Capabilities are ASR-only + +`transcribe_model_get_capabilities` returns `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` +for a model without the ASR role, instead of a zeroed struct. Role facts come +from the role's own query (`transcribe_diarize_get_info`). + +## Behavior changes on every model + +- **Non-finite audio is rejected.** `transcribe_run`, `transcribe_run_batch` + and `transcribe_stream_feed` return `TRANSCRIBE_ERR_INVALID_ARG` when any + sample is NaN or +-Inf, before the previous result is touched. In a batch, + one such utterance rejects the whole batch. A rejected feed leaves the + stream ACTIVE. Silence is still valid. +- **A stream whose family hook throws now ends.** begin / feed / finalize + mark it FAILED with `last_status` equal to the returned status (OOM or + BACKEND); `transcribe_stream_reset` always ends IDLE. +- **Host thread-pool failures return an error** instead of terminating the + process. + +## Language bindings + +- New error type for status 20: `UnsupportedRole` (Python, TypeScript, + Rust), `.unsupportedRole` (Swift). +- Each binding gains `Model.roles` and a `DiarizeSession` (see + `docs/bindings.md`). The Sortformer run extension for the ASR path + (`SortformerStreamOptions` and its equivalents) is replaced by the diarize + one. +- **Rust:** `Model::capabilities()` returns `Result` + (`Err(Error::UnsupportedRole)` on a model without ASR), and `ExtSlot` + gains `DiarizeRun`. +- **Swift:** `Model.capabilities` is `get throws`. +- **Python:** calls on one model now serialize (they used to race), and a + run / run_batch / second stream on any session of a model with an active + stream raises `transcribe_cpp.Busy`, matching the other bindings. On the + stream's own session these raised `InvalidArgument` in 0.3. +- **TypeScript:** a feed rejected before the native hook (e.g. NaN) keeps the + model's stream lease; the stream stays usable. An unknown Sortformer preset + string is rejected instead of silently using the default. diff --git a/docs/models/diar_streaming_sortformer_4spk-v2.1.md b/docs/models/diar_streaming_sortformer_4spk-v2.1.md index 93952f7e0..b167e53e8 100644 --- a/docs/models/diar_streaming_sortformer_4spk-v2.1.md +++ b/docs/models/diar_streaming_sortformer_4spk-v2.1.md @@ -63,12 +63,12 @@ Q5_K_M on one AMI meeting; details in cmake -B build cmake --build build -# Speaker segments as JSON (one line per file): -echo audio.wav > files.txt build/bin/transcribe-cli \ -m models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf \ - --batch files.txt --batch-jsonl -# {"file":"audio.wav","text":"","speakers":[{"t0_ms":320,"t1_ms":2400,"speaker_id":1},...]} + audio.wav +# speaker segments: 5 +# [ 0.32 -> 2.40] S1 +# ... ``` If your audio is not already 16 kHz mono WAV, convert it first: @@ -77,16 +77,17 @@ If your audio is not already 16 kHz mono WAV, convert it first: ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav ``` -From the C API, read results via `transcribe_n_speaker_segments` / -`transcribe_get_speaker_segment` (the transcript accessors return empty -text). The streaming operating point (latency / accuracy trade-off) is -selected with the run extension in `include/transcribe/sortformer.h`: +From the C API, use the DIARIZE role (`include/transcribe/diarize.h`): +`transcribe_diarize_session_init`, `transcribe_diarize_run`, then +`transcribe_diarize_n_segments` / `transcribe_diarize_get_segment`. The +streaming operating point (latency / accuracy trade-off) is selected with +the extension in `include/transcribe/sortformer.h`: ```c -transcribe_sortformer_stream_ext ext; -transcribe_sortformer_stream_ext_init(&ext); /* DEFAULT = model cfg */ +transcribe_sortformer_diarize_ext ext; +transcribe_sortformer_diarize_ext_init(&ext); /* DEFAULT = model cfg */ ext.preset = TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY; -run_params.family = &ext.ext; +diarize_params.family = &ext.ext; ``` `VERY_HIGH_LATENCY` (~30 s lookahead) is the offline-file operating diff --git a/docs/porting/families/sortformer.md b/docs/porting/families/sortformer.md index 118c20146..8ecd93a91 100644 --- a/docs/porting/families/sortformer.md +++ b/docs/porting/families/sortformer.md @@ -17,7 +17,7 @@ deferral; single-session `transcribe_run` is the shipped path; revisit with multitalker interop). Streaming posture: **natively streaming — PASS**; the chunk/lookahead contract is the preset menu in "Public API (run extension)" (default = GGUF-shipped checkpoint cfg; very_high_latency ~30.4 s lookahead -... low_latency ~1.04 s), exposed via `transcribe_sortformer_stream_ext`. +... low_latency ~1.04 s), exposed via `transcribe_sortformer_diarize_ext`. Push-audio `transcribe_stream_*` entry point is future work (STREAM-slot kind reserved by design). @@ -60,18 +60,18 @@ exposed because it falls out of the multitalker dependency for free, via the pre-existing transcript-independent `transcribe_speaker_segment` ABI — no diarizer-specific output surface was added. -## Public API (run extension) +## Public API (diarize extension) -`include/transcribe/sortformer.h` — `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM` -(`SFST`, RUN slot; registered in `docs/extension-kinds.md`). A run produces -no text; results are read via `transcribe_n_speaker_segments` / -`transcribe_get_speaker_segment` (`TRANSCRIBE_FEATURE_DIARIZATION`). +Sortformer serves the DIARIZE role only (`include/transcribe/diarize.h`). +`include/transcribe/sortformer.h` — `TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE` +(`SFDR`, DIARIZE_RUN slot; registered in `docs/extension-kinds.md`). The +0.3 RUN-slot kind `SFST` is retired. ```c -transcribe_sortformer_stream_ext ext; -transcribe_sortformer_stream_ext_init(&ext); /* preset = DEFAULT (GGUF cfg) */ +transcribe_sortformer_diarize_ext ext; +transcribe_sortformer_diarize_ext_init(&ext); /* preset = DEFAULT (GGUF cfg) */ ext.preset = TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY; -run_params.family = &ext.ext; /* probe accepts_ext_kind first */ +diarize_params.family = &ext.ext; /* probe accepts_ext_kind first */ ``` Preset menu (frames are 80 ms; chunk/rc/fifo/update/spkcache geometry per @@ -90,8 +90,8 @@ these bundles are accuracy-validated. Precedence: GGUF cfg < ext preset < `TRANSCRIBE_SORTFORMER_STREAM_PRESET` env < per-field env (env layers are validation hooks; `small` is a diagnostic-only env preset, deliberately not in the public enum). An out-of-range preset or wrong-kind ext is -rejected pre-clear (`run_validate`), preserving the previous result. -Unit test: `tests/sortformer_stream_ext_unit.cpp` +rejected pre-clear (`DiarizeOps::run_validate`), preserving the previous +result. Unit test: `tests/sortformer_diarize_unit.cpp` (`TRANSCRIBE_SORTFORMER_GGUF`-gated). ## Quant policy (Stage 7) diff --git a/include/transcribe.abihash b/include/transcribe.abihash index f3f29e54d..9d6354be1 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -c2979c7e891982ee +30ec88d3b51fc344 diff --git a/include/transcribe/sortformer.h b/include/transcribe/sortformer.h index c1208cbda..a488671b0 100644 --- a/include/transcribe/sortformer.h +++ b/include/transcribe/sortformer.h @@ -1,23 +1,12 @@ /* * include/transcribe/sortformer.h - Sortformer-family public extension. * - * Includes transcribe.h; safe to include in C or C++ TUs. Holds the - * streaming-operating-point run extension, its kind constant, and its - * init function. - * - * Sortformer (diar_streaming_sortformer_4spk-v2.1) is a diarization-only model: a run - * produces no text; the product is the who-spoke-when rows read back via - * transcribe_n_speaker_segments / transcribe_get_speaker_segment - * (TRANSCRIBE_FEATURE_DIARIZATION). The compute core is streaming - * (AOSC speaker cache + FIFO) but the shipped entry point is the batch - * transcribe_run over a whole recording, so this extension lives on the - * RUN slot. A future push-audio entry point (transcribe_stream_begin/ - * feed) would register a separate STREAM-slot kind taking the same - * preset enum. - * - * Probe via transcribe_model_accepts_ext_kind(model, - * TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM) - * before pointing transcribe_run_params::family at the struct. + * Includes transcribe.h; safe to include in C or C++ TUs. Sortformer + * (diar_streaming_sortformer_4spk-v2.1) serves the DIARIZE role + * (include/transcribe/diarize.h). Its one extension picks the streaming + * operating point for transcribe_diarize_run; probe with + * transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, + * TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE). * * FourCC kinds are reserved in docs/extension-kinds.md. */ @@ -31,9 +20,6 @@ extern "C" { #endif -/* 'SFST' little-endian = 0x54534653 */ -#define TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM 0x54534653u - /* * Streaming operating point (latency / accuracy trade-off). * @@ -56,8 +42,8 @@ extern "C" { * (many small windows); see the family doc for * measured throughput. * - * Values outside the enum range are rejected by transcribe_run with - * TRANSCRIBE_ERR_INVALID_ARG before the previous result is cleared. + * Values outside the enum range are rejected by transcribe_diarize_run + * with TRANSCRIBE_ERR_INVALID_ARG before the previous result is cleared. */ typedef enum { TRANSCRIBE_SORTFORMER_PRESET_DEFAULT = 0, @@ -66,14 +52,6 @@ typedef enum { TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY = 3, } transcribe_sortformer_preset; -struct transcribe_sortformer_stream_ext { - struct transcribe_ext ext; - transcribe_sortformer_preset preset; -}; - -/* Fills ext.size/kind and preset = DEFAULT (GGUF-shipped cfg). */ -TRANSCRIBE_API void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * ext); - /* 'SFDR' little-endian = 0x52444653 (DIARIZE_RUN slot) */ #define TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE 0x52444653u diff --git a/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml b/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml index d9d6365bc..dc4a5be46 100644 --- a/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml +++ b/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml @@ -40,22 +40,19 @@ usage: | cmake -B build && cmake --build build ``` - Run on a 16 kHz mono WAV. This is a diarizer, not a transcription model — - use the JSONL output to get the speaker segments (the plain output's - `text:` line is empty by design): + Run on a 16 kHz mono WAV. This is a diarizer, not a transcription model: + the CLI prints speaker segments. ```bash - echo input.wav > files.txt - build/bin/transcribe-cli \ - -m diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf \ - --batch files.txt --batch-jsonl - # {"file":"input.wav","text":"","speakers":[{"t0_ms":320,"t1_ms":2400,"speaker_id":1},...]} + build/bin/transcribe-cli -m diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf input.wav + # speaker segments: 5 + # [ 0.32 -> 2.40] S1 + # ... ``` - From the C API, read segments via `transcribe_n_speaker_segments` / - `transcribe_get_speaker_segment`, and select the streaming operating - point (latency / accuracy trade-off) with the run extension in - `include/transcribe/sortformer.h`. + From the C API, use the DIARIZE role (`include/transcribe/diarize.h`), + and select the streaming operating point (latency / accuracy trade-off) + with the extension in `include/transcribe/sortformer.h`. If your audio isn't already 16 kHz mono WAV, convert it first: diff --git a/scripts/validate.py b/scripts/validate.py index 85d1a4756..cbd9fc717 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -501,8 +501,6 @@ def cmd_cpp(args: argparse.Namespace) -> int: cmd += ["--language", language] if args.family == "whisper": cmd += ["--timestamps", "none"] - if args.family == "sortformer": - cmd += ["--role", "diarize"] if args.family in ("sensevoice", "parakeet"): # The reference dumper emits the raw token stream including # control / language tags (sensevoice: language / event / diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5758f3dd6..081473822 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -65,7 +65,6 @@ add_library(transcribe arch/moss/decoder.cpp arch/sortformer/model.cpp arch/sortformer/stream.cpp - arch/sortformer/capabilities.cpp arch/sortformer/weights.cpp arch/voxtral/model.cpp arch/voxtral/capabilities.cpp diff --git a/src/arch/sortformer/capabilities.cpp b/src/arch/sortformer/capabilities.cpp deleted file mode 100644 index 7c929ef27..000000000 --- a/src/arch/sortformer/capabilities.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// arch/sortformer/capabilities.cpp - Sortformer capability defaults. -// -// Applied before transcribe::read_capability_kv (KV present overrides, -// KV absent keeps the default). Sortformer is a pure frame-level diarizer: -// no transcript, no translation, no transcript-timestamp capability. Its -// output is speaker segments via the transcript-independent speaker_segment -// surface, so DIARIZATION is a family invariant. - -#include "sortformer.h" - -namespace transcribe::sortformer { - -void apply_family_invariants(transcribe_model & model) { - transcribe_capabilities & caps = model.caps; - - // Fixed 16 kHz mel bank (NeMo AudioToMelSpectrogramPreprocessor). - caps.native_sample_rate = 16000; - - // Not a transcription model. - caps.supports_translate = false; - - // Diarization segment times are intrinsic; there is no transcript, so - // there is no transcript-timestamp capability to advertise. - caps.max_timestamp_kind = TRANSCRIBE_TIMESTAMPS_NONE; - - // The model emits speaker-attributed output (T x 4 activity -> "who - // spoke when" segments). run() populates the transcript-independent - // speaker_segment surface directly from the probs. - transcribe::set_feature(&model, TRANSCRIBE_FEATURE_DIARIZATION, true); - - // Abort callback honored at the top of each run. - transcribe::set_feature(&model, TRANSCRIBE_FEATURE_CANCELLATION, true); -} - -} // namespace transcribe::sortformer diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index 3cf8f8fa6..66cbf583e 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -70,8 +70,6 @@ DiarStreamScratch::~DiarStreamScratch() { } } -SortformerSession::~SortformerSession() = default; - namespace { // Map the Sortformer hparams onto the parakeet encoder hparams that @@ -540,16 +538,6 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->variant = loader.variant().empty() ? k_default_variant : loader.variant(); m->backend.clear(); - apply_family_invariants(*m); - m->caps.n_languages = 0; - m->caps.languages = nullptr; - - if (const transcribe_status st = read_capability_kv(loader.gguf(), m->caps); st != TRANSCRIBE_OK) { - return st; - } - if (const transcribe_status st = read_languages_kv(loader.gguf(), *m); st != TRANSCRIBE_OK) { - return st; - } if (const transcribe_status st = read_sortformer_hparams(loader.gguf(), m->hparams); st != TRANSCRIBE_OK) { return st; } @@ -624,7 +612,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } - m->roles = k_role_asr | k_role_diarize; + m->roles = k_role_diarize; m->t_load_us = ggml_time_us() - t_load_start; *out_model = m.release(); return TRANSCRIBE_OK; @@ -657,23 +645,9 @@ transcribe_status fuse_embedded_diar_bn(SortformerEmbedded & e, return fuse_conformer_bn_core(e.conformer.blocks, e.conformer_hp.enc_d_model, backend, out_ctx, out_buffer); } -transcribe_status init_context(transcribe_model * model, - const transcribe_session_params * params, - transcribe_session ** out_ctx) { - if (model->arch != &arch) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - auto pc = std::make_unique(); - pc->model = model; - pc->n_threads = params->n_threads; - pc->kv_type = params->kv_type; - *out_ctx = pc.release(); - return TRANSCRIBE_OK; -} - // Lazily create the persistent multi-backend scheduler shared by the offline // and streaming graphs. -template static transcribe_status ensure_sched(Session * pc, SortformerModel * pm) { +static transcribe_status ensure_sched(SortformerSession * pc, SortformerModel * pm) { if (pc->sched == nullptr) { pc->sched = ggml_backend_sched_new(pm->plan.scheduler_list.data(), nullptr, static_cast(pm->plan.scheduler_list.size()), @@ -689,8 +663,7 @@ template static transcribe_status ensure_sched(Session * pc, // head over the entire mel and dumps the Stage-4 parity tensors (enc.* + // diar.preds_offline). Invoked only when tensor dumping is active (the // streaming path is the product); keeps the offline tensor gate green. -template -static transcribe_status run_offline_forward(Session * pc, SortformerModel * pm, int mel_n_frames) { +static transcribe_status run_offline_forward(SortformerSession * pc, SortformerModel * pm, int mel_n_frames) { if (pc->compute_ctx != nullptr) { ggml_free(pc->compute_ctx); pc->compute_ctx = nullptr; @@ -947,8 +920,7 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & sc, // Family wrapper: resolve the operating point, run the core, trim, dump, // and hand back the frame probabilities. -template -static transcribe_status run_streaming(Session * pc, +static transcribe_status run_streaming(SortformerSession * pc, SortformerModel * pm, int mel_n_mels, int mel_n_frames, @@ -989,10 +961,8 @@ static transcribe_status run_streaming(Session * pc, return TRANSCRIBE_OK; } -// Mel -> (optional offline parity dump) -> streaming forward, shared by the -// ASR run and the DIARIZE run. -template -static transcribe_status diarize_pcm(Session * pc, +// Mel -> (optional offline parity dump) -> streaming forward. +static transcribe_status diarize_pcm(SortformerSession * pc, SortformerModel * pm, const float * pcm, int n_samples, @@ -1028,50 +998,10 @@ static transcribe_status diarize_pcm(Session * pc, return run_streaming(pc, pm, mel_n_mels, mel_n_frames, ms_per_frame, preset, out); } -transcribe_status run(transcribe_session * session, - const float * pcm, - int n_samples, - const transcribe_run_params * params) { - auto * pc = static_cast(session); - auto * pm = static_cast(session->model); - - if (pc->poll_abort()) { - return TRANSCRIBE_ERR_ABORTED; - } - - // Streaming operating-point run ext (kind/size/range already validated - // by the dispatcher + run_validate pre-clear; re-check is belt-and-braces). - transcribe_sortformer_preset preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; - if (params != nullptr && params->family != nullptr) { - if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - sizeof(struct transcribe_sortformer_stream_ext)); - st != TRANSCRIBE_OK) { - return st; - } - preset = reinterpret_cast(params->family)->preset; - } - pc->clear_result(); - - DiarizeProbs probs; - if (const transcribe_status st = diarize_pcm(pc, pm, pcm, n_samples, preset, probs); st != TRANSCRIBE_OK) { - return st; - } - probs_to_segments(probs.probs.data(), probs.n_frames, probs.n_speakers, probs.frame_ms, pc->speaker_segments); - - pc->result_kind = TRANSCRIBE_TIMESTAMPS_NONE; - pc->has_result = true; - return TRANSCRIBE_OK; -} - -// Kind+slot probe. Sortformer ships one RUN-slot extension (the streaming -// operating-point preset); there is no STREAM-slot surface (no push-audio -// entry point yet — a future one registers a separate kind). +// Kind+slot probe: the operating-point preset on the DIARIZE_RUN slot. static bool accepts_ext_kind(const transcribe_model * model, transcribe_ext_slot slot, uint32_t kind) { - if (model == nullptr) { - return false; - } - return (slot == TRANSCRIBE_EXT_SLOT_RUN && kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM) || - (slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && kind == TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE); + return model != nullptr && slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && + kind == TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; } static transcribe_status check_preset(transcribe_sortformer_preset preset) { @@ -1085,21 +1015,6 @@ static transcribe_status check_preset(transcribe_sortformer_preset preset) { return TRANSCRIBE_ERR_INVALID_ARG; } -// Pre-clear validation for the _RUN slot (see Arch::run_validate): reject a -// malformed ext or an out-of-range preset before the previous result -// snapshot is destroyed. -static transcribe_status run_validate(const transcribe_session * /*ctx*/, const transcribe_run_params * params) { - if (params == nullptr || params->family == nullptr) { - return TRANSCRIBE_OK; // NULL ext -> family defaults - } - if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - sizeof(struct transcribe_sortformer_stream_ext)); - st != TRANSCRIBE_OK) { - return st; - } - return check_preset(reinterpret_cast(params->family)->preset); -} - // ---- DIARIZE role ---- static int diarize_max_speakers(const transcribe_model * model) { @@ -1109,7 +1024,7 @@ static int diarize_max_speakers(const transcribe_model * model) { static transcribe_status diarize_init_session(transcribe_model *, const transcribe_diarize_session_params *, transcribe_diarize_session ** out) { - *out = new SortformerDiarizeSession(); + *out = new SortformerSession(); return TRANSCRIBE_OK; } @@ -1127,7 +1042,7 @@ static transcribe_status diarize_run(transcribe_diarize_session * session, int n_samples, const transcribe_diarize_params * params, DiarizeProbs & out) { - auto * pc = static_cast(session); + auto * pc = static_cast(session); if (pc->poll_abort()) { return TRANSCRIBE_ERR_ABORTED; } @@ -1148,8 +1063,8 @@ static const DiarizeOps k_diarize_ops = { extern const Arch arch = { /* .name = */ "sortformer", /* .load = */ load, - /* .init_context = */ init_context, - /* .run = */ run, + /* .init_context = */ nullptr, + /* .run = */ nullptr, /* .run_batch = */ nullptr, /* .stream_validate = */ nullptr, /* .stream_begin = */ nullptr, @@ -1157,7 +1072,7 @@ extern const Arch arch = { /* .stream_finalize = */ nullptr, /* .stream_reset = */ nullptr, /* .accepts_ext_kind = */ accepts_ext_kind, - /* .run_validate = */ run_validate, + /* .run_validate = */ nullptr, /* .diarize = */ &k_diarize_ops, }; @@ -1178,13 +1093,3 @@ extern "C" void transcribe_sortformer_diarize_ext_init(struct transcribe_sortfor p->ext.kind = TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; p->preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; } - -extern "C" void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->ext.size = sizeof(*p); - p->ext.kind = TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM; - p->preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; -} diff --git a/src/arch/sortformer/sortformer.h b/src/arch/sortformer/sortformer.h index 8910c1198..dd1bebeba 100644 --- a/src/arch/sortformer/sortformer.h +++ b/src/arch/sortformer/sortformer.h @@ -17,8 +17,7 @@ #include "transcribe-diarize.h" #include "transcribe-mel.h" #include "transcribe-model.h" -#include "transcribe-session.h" -#include "transcribe/sortformer.h" // public preset enum + run ext +#include "transcribe/sortformer.h" // public preset enum + diarize ext #include "weights.h" #include @@ -38,11 +37,6 @@ typedef struct ggml_backend_sched * ggml_backend_sched_t; namespace transcribe::sortformer { -// Family defaults, applied before transcribe::read_capability_kv (KV -// present overrides, KV absent keeps the default). Defined in -// capabilities.cpp. -void apply_family_invariants(transcribe_model & model); - // Concrete model. Owns ctx_meta (every weight tensor's data buffer); // the destructor frees it, invalidating every borrowed ggml_tensor* in // `conformer` / `weights`. @@ -187,10 +181,10 @@ struct DiarStreamScratch { ~DiarStreamScratch(); }; -// Per-session scratch reused across runs, shared by the ASR and DIARIZE -// session types. The per-call compute context and multi-backend scheduler -// are owned by the SessionCore base (sched / compute_ctx). -struct SortformerWork { +// Concrete DIARIZE session. The per-call compute context and multi-backend +// scheduler are owned by the SessionCore base (sched / compute_ctx). +struct SortformerSession final : public transcribe_diarize_session { + // Per-session scratch reused across runs. std::vector mel_buf; std::vector probs_host; // [n_spk * T], read back from diar.preds @@ -199,15 +193,6 @@ struct SortformerWork { DiarStreamScratch scratch; }; -struct SortformerSession final : public transcribe_session, - SortformerWork { - SortformerSession() = default; - ~SortformerSession() override; -}; - -struct SortformerDiarizeSession final : public transcribe_diarize_session, - SortformerWork {}; - // ---- Embedded-diarizer surface (multitalker bundle) -------------------- // // // A multitalker Parakeet bundle GGUF carries this diarizer's tensors under diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 38c0ff1df..6db3621ab 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1395,27 +1395,7 @@ if(TRANSCRIBE_BUILD_REAL_MODEL_TESTS) set_tests_properties(transcribe_qwen3_asr_e2e_smoke PROPERTIES SKIP_RETURN_CODE 77) - # Sortformer streaming operating-point run ext (SFST, RUN slot). - # Kind/slot probe, init stamping, pre-clear rejection with snapshot - # preservation, and ext-vs-env preset parity on the committed 2-spk - # oracle mix. Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). - add_executable(transcribe_sortformer_stream_ext_unit - sortformer_stream_ext_unit.cpp) - - target_link_libraries(transcribe_sortformer_stream_ext_unit - PRIVATE transcribe transcribe-common-example) - - target_compile_definitions(transcribe_sortformer_stream_ext_unit PRIVATE - "TRANSCRIBE_TEST_SAMPLES_DIR=\"${CMAKE_SOURCE_DIR}/samples\"") - - transcribe_apply_warnings(transcribe_sortformer_stream_ext_unit) - - add_test(NAME transcribe_sortformer_stream_ext_unit - COMMAND transcribe_sortformer_stream_ext_unit) - set_tests_properties(transcribe_sortformer_stream_ext_unit PROPERTIES - SKIP_RETURN_CODE 77) - - # Sortformer on the DIARIZE role: old-vs-new segment parity, pre-clear + # Sortformer on the DIARIZE role: golden segments per preset, pre-clear # rejection, and (with TRANSCRIBE_WHISPER_GGUF) cross-role concurrency. # Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). add_executable(transcribe_sortformer_diarize_unit diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp index ab194bb8a..cfd008738 100644 --- a/tests/sortformer_diarize_unit.cpp +++ b/tests/sortformer_diarize_unit.cpp @@ -1,9 +1,10 @@ // sortformer_diarize_unit.cpp - Sortformer on the DIARIZE role, against a // real GGUF (TRANSCRIBE_SORTFORMER_GGUF; RC 77 skip when unset). // -// 1. roles, info, and the SFDR kind on the DIARIZE_RUN slot. -// 2. transcribe_diarize_run gives byte-identical segments to the ASR-path -// transcribe_run at the same preset (DEFAULT and LOW_LATENCY). +// 1. roles (DIARIZE only), info, and the SFDR kind on the DIARIZE_RUN slot. +// 2. Golden segments on the 2-speaker oracle mix per preset (CPU), and the +// ext preset matching the TRANSCRIBE_SORTFORMER_STREAM_PRESET env path +// the DER tooling uses. // 3. An out-of-range preset is rejected before the previous result is // cleared. // 4. Cross-role concurrency: when TRANSCRIBE_WHISPER_GGUF is also set, an @@ -40,17 +41,6 @@ struct Row { bool operator==(const Row & o) const { return t0 == o.t0 && t1 == o.t1 && spk == o.spk; } }; -std::vector asr_rows(const transcribe_session * s) { - std::vector rows; - for (int i = 0; i < transcribe_n_speaker_segments(s); ++i) { - transcribe_speaker_segment r; - transcribe_speaker_segment_init(&r); - transcribe_get_speaker_segment(s, i, &r); - rows.push_back({ r.t0_ms, r.t1_ms, r.speaker_id }); - } - return rows; -} - std::vector diarize_rows(const transcribe_diarize_session * s) { std::vector rows; for (int i = 0; i < transcribe_diarize_n_segments(s); ++i) { @@ -140,38 +130,46 @@ int main() { return EXIT_FAILURE; } - // 1. Roles, info, extension probe. - CHECK(transcribe_model_roles(model) == (TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE)); + // 1. Roles, info, extension probe. ASR entry points refuse the model. + CHECK(transcribe_model_roles(model) == TRANSCRIBE_ROLE_DIARIZE); + transcribe_session * asr = nullptr; + CHECK(transcribe_session_init(model, nullptr, &asr) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + CHECK(transcribe_model_get_capabilities(model, &caps) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(!transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION)); transcribe_diarize_info info; transcribe_diarize_info_init(&info); CHECK(transcribe_diarize_get_info(model, &info) == TRANSCRIBE_OK); CHECK(info.sample_rate == 16000 && info.max_speakers == 4); CHECK(transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); - CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, - TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); - transcribe_session * asr = nullptr; transcribe_diarize_session * dia = nullptr; - CHECK(transcribe_session_init(model, nullptr, &asr) == TRANSCRIBE_OK); CHECK(transcribe_diarize_session_init(model, nullptr, &dia) == TRANSCRIBE_OK); - // 2. Old (ASR path) vs new (DIARIZE role) parity. - for (transcribe_sortformer_preset preset : - { TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY }) { - transcribe_sortformer_stream_ext ext; - transcribe_sortformer_stream_ext_init(&ext); - ext.preset = preset; - transcribe_run_params rp; - transcribe_run_params_init(&rp); - rp.family = &ext.ext; - CHECK(transcribe_run(asr, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - const std::vector old_rows = asr_rows(asr); - const std::vector new_rows = diarize(dia, pcm, preset); - CHECK(!new_rows.empty()); - CHECK(new_rows == old_rows); - } + // 2. Golden segments (ms, speaker) recorded on CPU, and ext == env preset. + const std::vector golden_default = { + { 320, 2400, 1 }, + { 7360, 9360, 1 }, + { 10240, 10640, 1 }, + { 4240, 6640, 2 }, + { 9760, 12000, 2 }, + }; + const std::vector golden_low_latency = { + { 320, 2480, 1 }, + { 7360, 9360, 1 }, + { 10240, 10640, 1 }, + { 4160, 6640, 2 }, + { 9760, 12000, 2 }, + }; + CHECK(diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT) == golden_default); + CHECK(diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY) == golden_low_latency); + ::setenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET", "low_latency", 1); + CHECK(transcribe_diarize_run(dia, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + CHECK(diarize_rows(dia) == golden_low_latency); + ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); // 3. Pre-clear rejection of a bad preset keeps the previous result. const int n_before = transcribe_diarize_n_segments(dia); @@ -184,7 +182,6 @@ int main() { CHECK(transcribe_diarize_run(dia, pcm.data(), static_cast(pcm.size()), &dp) == TRANSCRIBE_ERR_INVALID_ARG); CHECK(transcribe_diarize_n_segments(dia) == n_before); - transcribe_session_free(asr); transcribe_diarize_session_free(dia); // 4. Cross-role concurrency on two models. diff --git a/tests/sortformer_stream_ext_unit.cpp b/tests/sortformer_stream_ext_unit.cpp deleted file mode 100644 index 31bd1001c..000000000 --- a/tests/sortformer_stream_ext_unit.cpp +++ /dev/null @@ -1,188 +0,0 @@ -// sortformer_stream_ext_unit.cpp - Sortformer streaming operating-point run -// extension (TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, RUN slot). -// -// Covers, against a real GGUF (env-gated, RC 77 skip): -// -// 1. transcribe_model_accepts_ext_kind: SFST accepted on _RUN only; -// foreign kinds rejected. -// 2. transcribe_sortformer_stream_ext_init stamps size/kind/preset. -// 3. Pre-clear rejection: a wrong-kind ext and an out-of-range preset -// both fail with INVALID_ARG and PRESERVE the previous result -// (run_validate fires before clear_result). -// 4. Ext preset == env preset parity: running the committed 2-speaker -// oracle mix with ext=VERY_HIGH_LATENCY produces exactly the same -// speaker-segment rows as TRANSCRIBE_SORTFORMER_STREAM_PRESET= -// very_high_latency (the DER-validated env path), and a different -// geometry (LOW_LATENCY) is accepted and produces rows. -// -// Gated by TRANSCRIBE_SORTFORMER_GGUF (same pattern as the other -// real-model tests) because the preset resolution runs inside run() and -// needs real weights to produce comparable segments. - -#include "transcribe.h" -#include "transcribe/sortformer.h" -#include "wav.h" - -#include - -#include -#include -#include -#include - -namespace { - -int g_failures = 0; - -#define CHECK(cond) \ - do { \ - if (!(cond)) { \ - std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - ++g_failures; \ - } \ - } while (0) - -bool file_exists(const std::string & path) { - struct stat st{}; - return ::stat(path.c_str(), &st) == 0; -} - -std::vector read_segments(const transcribe_session * session) { - std::vector rows; - const int n = transcribe_n_speaker_segments(session); - for (int i = 0; i < n; ++i) { - transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - if (transcribe_get_speaker_segment(session, i, &row) == TRANSCRIBE_OK) { - rows.push_back(row); - } - } - return rows; -} - -bool same_segments(const std::vector & a, - const std::vector & b) { - if (a.size() != b.size()) { - return false; - } - for (size_t i = 0; i < a.size(); ++i) { - if (a[i].t0_ms != b[i].t0_ms || a[i].t1_ms != b[i].t1_ms || a[i].speaker_id != b[i].speaker_id) { - return false; - } - } - return true; -} - -} // namespace - -int main() { - const char * env = std::getenv("TRANSCRIBE_SORTFORMER_GGUF"); - if (env == nullptr || env[0] == '\0') { - std::fprintf(stderr, - "sortformer_stream_ext_unit: TRANSCRIBE_SORTFORMER_GGUF not set; skipping.\n" - "Re-run with TRANSCRIBE_SORTFORMER_GGUF=models/diar_streaming_sortformer_4spk-v2.1/" - "diar_streaming_sortformer_4spk-v2.1-F32.gguf\n"); - return 77; - } - const std::string gguf = env; - if (!file_exists(gguf)) { - std::fprintf(stderr, "sortformer_stream_ext_unit: file not found: %s\n", gguf.c_str()); - return 77; - } - const std::string wav_path = std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/sortformer-2spk-mix.wav"; - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav_path, pcm, wav_err)) { - std::fprintf(stderr, "sortformer_stream_ext_unit: wav load: %s\n", wav_err.c_str()); - return 77; - } - - // The parity leg compares the ext path against the env path, so the - // environment must start clean of the validation overrides. - ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); - - transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = TRANSCRIBE_BACKEND_CPU; // deterministic float order for the parity leg - struct transcribe_model * model = nullptr; - if (transcribe_model_load_file(gguf.c_str(), &mp, &model) != TRANSCRIBE_OK || model == nullptr) { - std::fprintf(stderr, "FAIL: model load\n"); - return EXIT_FAILURE; - } - - // 1. Kind+slot probe. - CHECK(transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); - CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); - CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, 0x4E524857u /* WHRN */)); - - // 2. Init function stamps the header + default. - transcribe_sortformer_stream_ext ext; - transcribe_sortformer_stream_ext_init(&ext); - CHECK(ext.ext.size == sizeof(ext)); - CHECK(ext.ext.kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM); - CHECK(ext.preset == TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); - - struct transcribe_session * session = nullptr; - if (transcribe_session_init(model, nullptr, &session) != TRANSCRIBE_OK || session == nullptr) { - std::fprintf(stderr, "FAIL: session create\n"); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - transcribe_run_params rp; - transcribe_run_params_init(&rp); - - // Baseline run (env preset path, the DER-validated harness route). - ::setenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET", "very_high_latency", 1); - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - const std::vector env_rows = read_segments(session); - ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); - CHECK(!env_rows.empty()); // the oracle mix has two speakers - - // 3. Pre-clear rejection preserves the previous result. - { - transcribe_sortformer_stream_ext bad; - transcribe_sortformer_stream_ext_init(&bad); - bad.ext.kind = 0x4E524857u; // WHRN: wrong family kind - rp.family = &bad.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(same_segments(read_segments(session), env_rows)); // snapshot intact - - transcribe_sortformer_stream_ext oor; - transcribe_sortformer_stream_ext_init(&oor); - oor.preset = static_cast(99); - rp.family = &oor.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(same_segments(read_segments(session), env_rows)); - } - - // 4a. Ext preset == env preset parity (same operating point, same rows). - { - transcribe_sortformer_stream_ext vh; - transcribe_sortformer_stream_ext_init(&vh); - vh.preset = TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY; - rp.family = &vh.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - CHECK(same_segments(read_segments(session), env_rows)); - } - - // 4b. A different geometry is accepted and produces speaker rows. - { - transcribe_sortformer_stream_ext lo; - transcribe_sortformer_stream_ext_init(&lo); - lo.preset = TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY; - rp.family = &lo.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - CHECK(!read_segments(session).empty()); - } - - transcribe_session_free(session); - transcribe_model_free(model); - - if (g_failures != 0) { - std::fprintf(stderr, "sortformer_stream_ext_unit: %d failure(s)\n", g_failures); - return EXIT_FAILURE; - } - std::printf("sortformer_stream_ext_unit: OK\n"); - return 0; -} From 3387785dde8d9adac7f577264f9972f67e4557f5 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:26:19 +0800 Subject: [PATCH 32/51] swift: Sortformer is DIARIZE-only Follows the C break: Sortformer serves only the DIARIZE role and the SFST RUN-slot extension is gone. Remove RunExtension.sortformer and SortformerStreamOptions; SortformerPreset stays for the diarize extension. Pin the reviewed header hash to 30ec88d3b51fc344. tests: drop the temporary Session.run/SFST parity check; testSortformerDiarizes now loads on CPU and checks golden turns on samples/sortformer-2spk-mix.wav at DEFAULT and LOW_LATENCY, roles == [.diarize], and that capabilities and session() throw .unsupportedRole. --- bindings/swift/README.md | 7 ++- .../swift/Sources/TranscribeCpp/ABIHash.swift | 2 +- .../swift/Sources/TranscribeCpp/Family.swift | 16 +---- .../TranscribeCppTests/DiarizeTests.swift | 59 +++++++++++-------- .../TranscribeCppTests/TestSupport.swift | 2 +- 5 files changed, 41 insertions(+), 45 deletions(-) diff --git a/bindings/swift/README.md b/bindings/swift/README.md index 178da89f6..b216d16e4 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -101,7 +101,7 @@ Runnable examples live in ## Diarization A model's `roles` say what it serves (`.asr`, `.diarize`). Diarization models -such as Sortformer return speaker turns from a `DiarizeSession`: +such as Sortformer (DIARIZE only) return speaker turns from a `DiarizeSession`: ```swift let model = try Model(path: "diar_streaming_sortformer_4spk-v2.1-F32.gguf") @@ -112,8 +112,9 @@ for t in turns { print(t.speakerId, t.t0Ms, t.t1Ms) } ``` Diarize runs share the model's compute lock and stream lease with sessions, and -take a `CancellationToken` the same way. `model.capabilities` describes ASR and -throws `TranscribeError.unsupportedRole` on a model without `.asr`. +take a `CancellationToken` the same way. `model.capabilities` and +`model.session()` are ASR-only and throw `TranscribeError.unsupportedRole` on a +model without `.asr`, such as Sortformer. ## Backends diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index aa156f23d..3f7911ee1 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "c2979c7e891982ee" + public static let pinnedHeaderHash = "30ec88d3b51fc344" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Family.swift b/bindings/swift/Sources/TranscribeCpp/Family.swift index 29dff29e5..67566d142 100644 --- a/bindings/swift/Sources/TranscribeCpp/Family.swift +++ b/bindings/swift/Sources/TranscribeCpp/Family.swift @@ -7,7 +7,7 @@ import CTranscribe // `transcribe_ext` pointer is handed to the run/begin call; the library copies // what it needs before returning. -// MARK: - Run-slot extensions (whisper, sortformer) +// MARK: - Run-slot extensions (whisper) public struct WhisperRunOptions: Sendable { public var initialPrompt: String? @@ -67,21 +67,12 @@ public enum SortformerPreset: Sendable { } } -/// Sortformer diarizer options (run slot). Sortformer produces speaker -/// segments, no text; read results via the speaker-segment accessors. -public struct SortformerStreamOptions: Sendable { - public var preset: SortformerPreset? - public init(preset: SortformerPreset? = nil) { self.preset = preset } -} - public enum RunExtension: Sendable { case whisper(WhisperRunOptions) - case sortformer(SortformerStreamOptions) var kind: UInt32 { switch self { case .whisper: return TRANSCRIBE_EXT_KIND_WHISPER_RUN - case .sortformer: return TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM } } } @@ -109,11 +100,6 @@ func withRunExtension( c.initial_prompt = prompt return try withUnsafePointer(to: &c.ext) { try body($0) } } - case .sortformer(let o): - var c = transcribe_sortformer_stream_ext() - transcribe_sortformer_stream_ext_init(&c) - if let v = o.preset { c.preset = v.cValue } - return try withUnsafePointer(to: &c.ext) { try body($0) } } } diff --git a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift index c749bf0f0..0cdc8c06b 100644 --- a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift @@ -11,6 +11,16 @@ final class DiarizeTests: XCTestCase { rows.map { [$0.t0Ms, $0.t1Ms, Int64($0.speakerId)] } } + private func assertUnsupportedRole(_ calls: [(String, () throws -> Void)]) { + for (what, call) in calls { + XCTAssertThrowsError(try call(), what) { error in + guard case TranscribeError.unsupportedRole = error else { + return XCTFail("\(what): expected .unsupportedRole, got \(error)") + } + } + } + } + func testRoleBitsMatchTheHeader() { XCTAssertEqual(Roles.asr.rawValue, 1) XCTAssertEqual(Roles.diarize.rawValue, 2) @@ -21,42 +31,41 @@ final class DiarizeTests: XCTestCase { let model = try Model(path: path) XCTAssertEqual(model.roles, .asr) XCTAssertFalse(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) - for (what, call) in [ + assertUnsupportedRole([ ("diarizeInfo", { _ = try model.diarizeInfo }), ("diarizeSession", { _ = try model.diarizeSession() }), - ] as [(String, () throws -> Void)] { - XCTAssertThrowsError(try call(), what) { error in - guard case TranscribeError.unsupportedRole = error else { - return XCTFail("\(what): expected .unsupportedRole, got \(error)") - } - } - } + ]) } func testSortformerDiarizes() throws { let (path, pcm) = try Fixtures.sortformerModelAndAudio() - let model = try Model(path: path) - XCTAssertEqual(model.roles, [.asr, .diarize]) + // The golden turns are CPU numerics. + let model = try Model(path: path, options: ModelOptions(backend: .cpu)) + XCTAssertEqual(model.roles, [.diarize]) XCTAssertEqual(try model.diarizeInfo, DiarizeInfo(sampleRate: 16000, maxSpeakers: 4)) XCTAssertTrue(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) - XCTAssertGreaterThan(try model.capabilities.nativeSampleRate, 0) + assertUnsupportedRole([ + ("capabilities", { _ = try model.capabilities }), + ("session", { _ = try model.session() }), + ]) + // (t0Ms, t1Ms, speakerId) on samples/sortformer-2spk-mix.wav. Only + // .lowLatency moves the turns here, so it shows the preset reaches the + // native run. + let golden: [(SortformerPreset, [[Int64]])] = [ + (.default, [[320, 2400, 1], [7360, 9360, 1], [10240, 10640, 1], + [4240, 6640, 2], [9760, 12000, 2]]), + (.lowLatency, [[320, 2480, 1], [7360, 9360, 1], [10240, 10640, 1], + [4160, 6640, 2], [9760, 12000, 2]]), + ] let session = try model.diarizeSession() - // On this clip only .lowLatency moves the turns, so it shows the preset - // reaches the native run. - let preset = SortformerPreset.lowLatency - let rows = try session.run( - pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: preset)))) - XCTAssertFalse(rows.isEmpty) - XCTAssertTrue(rows.allSatisfy { (1...4).contains($0.speakerId) && $0.t0Ms < $0.t1Ms }) + for (preset, want) in golden { + let rows = try session.run( + pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: preset)))) + XCTAssertEqual(turns(rows), want, "\(preset)") + } XCTAssertGreaterThan(session.timings.encodeMs, 0) - XCTAssertNotEqual(turns(rows), turns(try session.run(pcm))) - - // TEMPORARY: the Sortformer ASR path (SFST on the RUN slot) is removed - // in a later commit; until then both roles must yield the same turns. - let asr = try model.session().run( - pcm, options: RunOptions(family: .sortformer(SortformerStreamOptions(preset: preset)))) - XCTAssertEqual(turns(rows), turns(asr.speakerSegments)) + XCTAssertEqual(turns(try session.run(pcm)), golden[0].1, "no extension == .default") } func testBadPresetAndCancellation() throws { diff --git a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift index 754f13e9c..5d495ecaa 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift @@ -86,7 +86,7 @@ enum Fixtures { "models/SenseVoiceSmall/SenseVoiceSmall-Q8_0.gguf") } - /// Sortformer (ASR + DIARIZE roles) + its in-repo two-speaker sample. + /// Sortformer (DIARIZE only) + its in-repo two-speaker sample. static func sortformerModelAndAudio() throws -> (model: String, pcm: [Float]) { guard let model = familyModel( "TRANSCRIBE_SMOKE_SORTFORMER_MODEL", From afa6f33af252bc9a5097dd38cec1f6646162a75c Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:26:32 +0800 Subject: [PATCH 33/51] rust: Sortformer is DIARIZE-only Follows the core break: the SFST run-slot extension is gone, so the safe crate drops RunExtension::Sortformer and SortformerStreamOptions. SortformerPreset stays for DiarizeExtension::Sortformer. tests: the ASR-path parity block in tests/diarize.rs is replaced by CPU golden segments on samples/sortformer-2spk-mix.wav (DEFAULT and LOW_LATENCY); role expectations are DIARIZE only, and capabilities() and session() on the Sortformer model return UnsupportedRole. serde coverage drops the removed type. --- bindings/rust/transcribe-cpp/README.md | 9 +-- bindings/rust/transcribe-cpp/src/family.rs | 24 ------- bindings/rust/transcribe-cpp/src/lib.rs | 2 +- .../rust/transcribe-cpp/tests/common/mod.rs | 2 +- bindings/rust/transcribe-cpp/tests/diarize.rs | 72 +++++++++++-------- bindings/rust/transcribe-cpp/tests/serde.rs | 7 +- 6 files changed, 51 insertions(+), 65 deletions(-) diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index 180a3288f..3e6bb16e5 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -79,10 +79,11 @@ let transcript = stream.snapshot(); // language, segments, words, tokens, timing ### Diarization (who spoke when) `model.roles()` reports what a model can do. A model with `Role::Diarize` -(e.g. Sortformer) opens a `DiarizeSession`, whose `run` returns speaker turns -(`SpeakerSegment`: `t0_ms`, `t1_ms`, 1-based `speaker_id`). ASR-only calls such -as `capabilities()` return `Error::UnsupportedRole` on a model without -`Role::Asr`, and the diarize calls do the same without `Role::Diarize`. +(e.g. Sortformer, which is DIARIZE-only) opens a `DiarizeSession`, whose `run` +returns speaker turns (`SpeakerSegment`: `t0_ms`, `t1_ms`, 1-based +`speaker_id`). ASR-only calls such as `capabilities()` and `session()` return +`Error::UnsupportedRole` on a model without `Role::Asr`, and the diarize calls +do the same without `Role::Diarize`. ```rust use transcribe_cpp::{DiarizeOptions, Model, Role}; diff --git a/bindings/rust/transcribe-cpp/src/family.rs b/bindings/rust/transcribe-cpp/src/family.rs index 9348a04e8..4de0b0506 100644 --- a/bindings/rust/transcribe-cpp/src/family.rs +++ b/bindings/rust/transcribe-cpp/src/family.rs @@ -123,19 +123,6 @@ impl SortformerPreset { } } -/// Sortformer diarizer run-extension knobs (run slot). Sortformer produces -/// speaker segments, no text; read results via the speaker-segment -/// accessors. `None` keeps the family default (the GGUF-shipped cfg). -#[derive(Debug, Clone, Default, PartialEq, Eq)] -#[cfg_attr( - feature = "serde", - derive(serde::Serialize, serde::Deserialize), - serde(default) -)] -pub struct SortformerStreamOptions { - pub preset: Option, -} - /// Sortformer diarize-extension knobs (diarize-run slot). `None` keeps the /// family default (the GGUF-shipped cfg). #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -192,7 +179,6 @@ impl DiarizeExtension { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum RunExtension { Whisper(WhisperRunOptions), - Sortformer(SortformerStreamOptions), } /// A family extension for the stream slot. @@ -215,7 +201,6 @@ pub(crate) enum RunExtRaw { ext: Box, _prompt: Option, }, - Sortformer(Box), } impl RunExtRaw { @@ -225,9 +210,6 @@ impl RunExtRaw { RunExtRaw::Whisper { ext, .. } => { (&**ext) as *const sys::transcribe_whisper_run_ext as *const sys::transcribe_ext } - RunExtRaw::Sortformer(e) => { - (&**e) as *const sys::transcribe_sortformer_stream_ext as *const sys::transcribe_ext - } } } } @@ -261,12 +243,6 @@ impl RunExtension { _prompt: prompt, }) } - RunExtension::Sortformer(o) => { - let mut ext: sys::transcribe_sortformer_stream_ext = unsafe { std::mem::zeroed() }; - unsafe { sys::transcribe_sortformer_stream_ext_init(&mut ext) }; - set(&mut ext.preset, o.preset.map(SortformerPreset::to_sys)); - Ok(RunExtRaw::Sortformer(Box::new(ext))) - } } } } diff --git a/bindings/rust/transcribe-cpp/src/lib.rs b/bindings/rust/transcribe-cpp/src/lib.rs index 5f82cd576..07c3edeea 100644 --- a/bindings/rust/transcribe-cpp/src/lib.rs +++ b/bindings/rust/transcribe-cpp/src/lib.rs @@ -69,7 +69,7 @@ pub use error::{Error, Result}; pub use family::{ DiarizeExtension, MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, RunExtension, SortformerDiarizeOptions, SortformerPreset, - SortformerStreamOptions, StreamExtension, VoxtralRealtimeStreamOptions, WhisperRunOptions, + StreamExtension, VoxtralRealtimeStreamOptions, WhisperRunOptions, }; pub use logging::{disable_logging, init_logging}; pub use model::{Capabilities, Model, ModelOptions, SessionLimits, SessionOptions}; diff --git a/bindings/rust/transcribe-cpp/tests/common/mod.rs b/bindings/rust/transcribe-cpp/tests/common/mod.rs index 326612778..6e66a8a12 100644 --- a/bindings/rust/transcribe-cpp/tests/common/mod.rs +++ b/bindings/rust/transcribe-cpp/tests/common/mod.rs @@ -111,7 +111,7 @@ pub fn smoke_itn_model() -> Option { ) } -/// Sortformer diarizer (ASR + DIARIZE roles) plus its two-speaker sample, or +/// Sortformer diarizer (DIARIZE role only) plus its two-speaker sample, or /// `None` (with a skip note) when either is absent. pub fn smoke_sortformer_fixtures(test: &str) -> Option<(PathBuf, Vec)> { let model = family_model( diff --git a/bindings/rust/transcribe-cpp/tests/diarize.rs b/bindings/rust/transcribe-cpp/tests/diarize.rs index 95f4e9c67..672b5f202 100644 --- a/bindings/rust/transcribe-cpp/tests/diarize.rs +++ b/bindings/rust/transcribe-cpp/tests/diarize.rs @@ -8,9 +8,8 @@ mod common; use transcribe_cpp::sys::TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE as SFDR; use transcribe_cpp::{ - CancelToken, DiarizeExtension, DiarizeOptions, Error, ExtSlot, Model, Role, RunExtension, - RunOptions, SortformerDiarizeOptions, SortformerPreset, SortformerStreamOptions, - SpeakerSegment, + Backend, CancelToken, DiarizeExtension, DiarizeOptions, Error, ExtSlot, Model, ModelOptions, + Role, SortformerDiarizeOptions, SortformerPreset, SpeakerSegment, }; fn diarize_opts(preset: Option) -> DiarizeOptions { @@ -21,58 +20,69 @@ fn diarize_opts(preset: Option) -> DiarizeOptions { } } -fn rows(segs: &[SpeakerSegment]) -> Vec<(i64, i64, i32, u32)> { +fn rows(segs: &[SpeakerSegment]) -> Vec<(i64, i64, i32)> { segs.iter() - .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id, s.p.to_bits())) + .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id)) .collect() } #[test] -fn sortformer_serves_asr_and_diarize() { - let Some((model_path, _)) = - common::smoke_sortformer_fixtures("sortformer_serves_asr_and_diarize") +fn sortformer_is_diarize_only() { + let Some((model_path, _)) = common::smoke_sortformer_fixtures("sortformer_is_diarize_only") else { return; }; let model = Model::load(&model_path).unwrap(); let roles = model.roles(); - assert!(roles.contains(Role::Asr) && roles.contains(Role::Diarize)); - assert_eq!(roles.bits(), 0b11, "{roles:?}"); + assert!(!roles.contains(Role::Asr) && roles.contains(Role::Diarize)); + assert_eq!(roles.bits(), 0b10, "{roles:?}"); let info = model.diarize_info().unwrap(); assert_eq!((info.sample_rate, info.max_speakers), (16000, 4)); assert!(model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); assert!(!model.accepts_ext(ExtSlot::Run, SFDR)); - assert!(model.capabilities().is_ok()); + assert!(matches!( + model.capabilities(), + Err(Error::UnsupportedRole(_)) + )); + assert!(matches!(model.session(), Err(Error::UnsupportedRole(_)))); } #[test] -fn diarize_run_matches_asr_path_speaker_turns() { +fn diarize_run_matches_cpu_golden_segments() { let Some((model_path, pcm)) = - common::smoke_sortformer_fixtures("diarize_run_matches_asr_path_speaker_turns") + common::smoke_sortformer_fixtures("diarize_run_matches_cpu_golden_segments") else { return; }; - let model = Model::load(&model_path).unwrap(); - let mut session = model.session().unwrap(); + // Goldens are recorded on CPU (tests/sortformer_diarize_unit.cpp). + let cpu = ModelOptions { + backend: Backend::Cpu, + ..Default::default() + }; + let model = Model::load_with(&model_path, &cpu).unwrap(); let mut diarize = model.diarize_session().unwrap(); - for preset in [None, Some(SortformerPreset::LowLatency)] { + let golden_default = [ + (320, 2400, 1), + (7360, 9360, 1), + (10240, 10640, 1), + (4240, 6640, 2), + (9760, 12000, 2), + ]; + let golden_low_latency = [ + (320, 2480, 1), + (7360, 9360, 1), + (10240, 10640, 1), + (4160, 6640, 2), + (9760, 12000, 2), + ]; + for (preset, golden) in [ + (None, &golden_default), + (Some(SortformerPreset::Default), &golden_default), + (Some(SortformerPreset::LowLatency), &golden_low_latency), + ] { let turns = diarize.run(&pcm, &diarize_opts(preset)).unwrap(); - assert!(!turns.is_empty(), "{preset:?}: no speaker turns"); - assert!(turns.iter().all(|s| (1..=4).contains(&s.speaker_id))); + assert_eq!(rows(&turns), golden.to_vec(), "{preset:?}"); assert!(diarize.timings().encode_ms > 0.0); - - // PARITY with the ASR path (Session::run + SFST preset). The ASR path - // for Sortformer is removed in a later commit; drop this block then. - let asr = RunOptions { - family: Some(RunExtension::Sortformer(SortformerStreamOptions { preset })), - ..Default::default() - }; - let transcript = session.run(&pcm, &asr).unwrap(); - assert_eq!( - rows(&turns), - rows(&transcript.speaker_segments), - "{preset:?}" - ); } } diff --git a/bindings/rust/transcribe-cpp/tests/serde.rs b/bindings/rust/transcribe-cpp/tests/serde.rs index 94687aff3..1088f78b3 100644 --- a/bindings/rust/transcribe-cpp/tests/serde.rs +++ b/bindings/rust/transcribe-cpp/tests/serde.rs @@ -8,9 +8,9 @@ use transcribe_cpp::{ DiarizeOptions, DiarizeSessionOptions, ExtSlot, Feature, Itn, KvType, MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, Pnc, Role, Roles, RunExtension, RunOptions, Segment, SessionLimits, SessionOptions, - SortformerDiarizeOptions, SortformerPreset, SortformerStreamOptions, SpeakerSegment, - StreamExtension, StreamOptions, StreamState, StreamText, StreamUpdate, Task, TimestampKind, - Timings, Token, Transcript, VoxtralRealtimeStreamOptions, WhisperRunOptions, Word, + SortformerDiarizeOptions, SortformerPreset, SpeakerSegment, StreamExtension, StreamOptions, + StreamState, StreamText, StreamUpdate, Task, TimestampKind, Timings, Token, Transcript, + VoxtralRealtimeStreamOptions, WhisperRunOptions, Word, }; fn assert_serde() {} @@ -29,7 +29,6 @@ fn plain_data_types_are_serializable() { assert_serde::(); assert_serde::(); assert_serde::(); - assert_serde::(); assert_serde::(); assert_serde::(); assert_serde::(); From 6ce4903f87db9f51c8c006bb94f0f5d6a5fe1c0c Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:26:04 +0800 Subject: [PATCH 34/51] ts: Sortformer is DIARIZE-only Follows the core break: Sortformer serves only the DIARIZE role and the RUN-slot SFST extension is gone from the C ABI. Drop the "sortformer" family kind (SortformerStreamOptions, its FAMILY entry, and the transcribe_sortformer_stream_ext_init binding); "sortformer_diarize" keeps the shared preset mapping. README notes the DIARIZE-only model. tests: the ASR-path equality test is replaced by golden segments on samples/sortformer-2spk-mix.wav at DEFAULT and LOW_LATENCY (CPU backend, the goldens are CPU); roles are ["diarize"], and capabilities / createSession throw UnsupportedRole on the Sortformer model. The diarize-vs-ASR exclusivity test is dropped: with two models there is no shared compute lock to order. The retired "sortformer" kind is rejected as an unknown kind, and a run-slot kind (whisper) on diarize_run as InvalidArgument. --- bindings/typescript/README.md | 6 ++- bindings/typescript/src/ffi.ts | 3 -- bindings/typescript/src/index.ts | 7 --- bindings/typescript/src/types.ts | 6 --- bindings/typescript/test/common.mjs | 2 +- bindings/typescript/test/diarize.test.mjs | 54 +++++++++-------------- 6 files changed, 25 insertions(+), 53 deletions(-) diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index b004c5d93..4ff23d972 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -112,8 +112,10 @@ model.accepts({ kind: "whisper" }); // does this model take that extension? ### Diarization (DIARIZE role) `model.roles` lists what a model serves (`"asr"`, `"diarize"`). A diarization -model (e.g. Sortformer) answers who spoke when; ASR calls on a model without -`"asr"`, and diarize calls on one without `"diarize"`, throw `UnsupportedRole`. +model (e.g. Sortformer, which serves only `"diarize"`) answers who spoke when; +ASR calls (`capabilities`, `createSession`, `transcribe`, ...) on a model +without `"asr"`, and diarize calls on one without `"diarize"`, throw +`UnsupportedRole`. ```ts const { sampleRate, maxSpeakers } = model.diarizeInfo; diff --git a/bindings/typescript/src/ffi.ts b/bindings/typescript/src/ffi.ts index 75050a907..c6d3c4bf6 100644 --- a/bindings/typescript/src/ffi.ts +++ b/bindings/typescript/src/ffi.ts @@ -174,9 +174,6 @@ export function bindLibrary(libraryPath: string): Bound { voxtralRealtimeStreamExtInit: lib.func("transcribe_voxtral_realtime_stream_ext_init", "void", [ outp(T.transcribe_voxtral_realtime_stream_ext), ]), - sortformerStreamExtInit: lib.func("transcribe_sortformer_stream_ext_init", "void", [ - outp(T.transcribe_sortformer_stream_ext), - ]), sortformerDiarizeExtInit: lib.func("transcribe_sortformer_diarize_ext_init", "void", [ outp(T.transcribe_sortformer_diarize_ext), ]), diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 7d4ef259e..b27772e0e 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -635,13 +635,6 @@ const FAMILY: Record = { min_decode_interval_ms: o.minDecodeIntervalMs, }), }, - sortformer: { - slot: "run", - kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - type: "transcribe_sortformer_stream_ext", - init: "sortformerStreamExtInit", - map: (o) => sortformerPreset(o), - }, sortformer_diarize: { slot: "diarize_run", kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 99ec3418c..1ddd6bfb1 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -276,11 +276,6 @@ export type SortformerPreset = | "very_high_latency" | "high_latency" | "low_latency"; -/** Sortformer diarizer options (run slot). A run produces speaker - * segments, no text. */ -export interface SortformerStreamOptions { - preset?: SortformerPreset; -} /** Sortformer options for {@link DiarizeOptions.family} (diarize_run slot). */ export interface SortformerDiarizeOptions { preset?: SortformerPreset; @@ -292,7 +287,6 @@ export type FamilyExtension = | ({ kind: "parakeet" } & ParakeetStreamOptions) | ({ kind: "parakeet_buffered" } & ParakeetBufferedStreamOptions) | ({ kind: "voxtral" } & VoxtralRealtimeStreamOptions) - | ({ kind: "sortformer" } & SortformerStreamOptions) | ({ kind: "sortformer_diarize" } & SortformerDiarizeOptions); // ---- roles ----------------------------------------------------------------- diff --git a/bindings/typescript/test/common.mjs b/bindings/typescript/test/common.mjs index 4d70ffee4..7dcaa9d13 100644 --- a/bindings/typescript/test/common.mjs +++ b/bindings/typescript/test/common.mjs @@ -22,7 +22,7 @@ export const ITN_MODEL = // Voxtral realtime is local-only (~2.5 GB+, too heavy for the CI canary set): // its env var is NOT exported by fetch-canary, so this skips cleanly in CI. export const VOXTRAL_MODEL = process.env.TRANSCRIBE_SMOKE_VOXTRAL_MODEL || ""; -// Sortformer (ASR + DIARIZE roles) is local-only too. +// Sortformer (DIARIZE role only) is local-only too. export const SORTFORMER_MODEL = process.env.TRANSCRIBE_SMOKE_SORTFORMER_MODEL || ""; export const SORTFORMER_AUDIO = path.resolve(HERE, "../../../samples/sortformer-2spk-mix.wav"); diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs index a352465a3..830101bae 100644 --- a/bindings/typescript/test/diarize.test.mjs +++ b/bindings/typescript/test/diarize.test.mjs @@ -43,35 +43,37 @@ modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MOD } }); -modelTest("sortformer serves asr and diarize; diarizeInfo", SORTFORMER_MODEL, async () => { +modelTest("sortformer serves only diarize; diarizeInfo; ASR calls are refused", SORTFORMER_MODEL, async () => { const m = await TranscribeModel.load(SORTFORMER_MODEL); try { - assert.deepEqual(m.roles, ["asr", "diarize"]); + assert.deepEqual(m.roles, ["diarize"]); assert.deepEqual(m.diarizeInfo, { sampleRate: 16000, maxSpeakers: 4 }); assert.equal(m.accepts({ kind: "sortformer_diarize" }), true); + assert.throws(() => m.capabilities, UnsupportedRole); + assert.throws(() => m.createSession(), UnsupportedRole); } finally { m.dispose(); } }); -// Compares against the ASR path (Session.run with the SFST preset), which a -// later commit removes; drop the comparison then. -modelTest("diarize run returns the ASR path's speaker turns for the same preset", SORTFORMER_MODEL, async () => { - const m = await TranscribeModel.load(SORTFORMER_MODEL); +// Goldens are CPU (as sortformer_diarize_unit): (t0Ms, t1Ms, speakerId), +// grouped by speaker and time-ordered within one. +const GOLDEN = { + default: [[320, 2400, 1], [7360, 9360, 1], [10240, 10640, 1], [4240, 6640, 2], [9760, 12000, 2]], + low_latency: [[320, 2480, 1], [7360, 9360, 1], [10240, 10640, 1], [4160, 6640, 2], [9760, 12000, 2]], +}; + +modelTest("diarize run returns the golden speaker turns per preset", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL, { backend: "cpu" }); try { const pcm = mix(); const d = m.createDiarizeSession({ nThreads: 4 }); - const s = m.createSession(); - for (const preset of ["default", "low_latency"]) { + for (const [preset, want] of Object.entries(GOLDEN)) { const rows = await d.run(pcm, { family: { kind: "sortformer_diarize", preset } }); - assert.ok(rows.length > 0, preset); - for (const r of rows) assert.ok(r.speakerId >= 1 && r.speakerId <= 4 && r.t1Ms > r.t0Ms); - const asr = await s.run(pcm, { family: { kind: "sortformer", preset } }); - assert.deepEqual(rows, asr.speakerSegments, preset); + assert.deepEqual(rows.map((r) => [r.t0Ms, r.t1Ms, r.speakerId]), want, preset); } assert.ok(d.timings.encodeMs > 0); d.dispose(); - s.dispose(); } finally { m.dispose(); } @@ -86,7 +88,11 @@ modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, () => d.run(pcm, { family: { kind: "sortformer_diarize", preset: "ultra_low_latency" } }), (e) => e instanceof TranscribeError && /invalid sortformer preset/.test(e.message), ); - await assert.rejects(() => d.run(pcm, { family: { kind: "sortformer" } }), InvalidArgument); + await assert.rejects(() => d.run(pcm, { family: { kind: "whisper" } }), InvalidArgument); + await assert.rejects( + () => d.run(pcm, { family: { kind: "sortformer" } }), + (e) => e instanceof InvalidArgument && /unknown family extension kind/.test(e.message), + ); assert.ok((await d.run(pcm)).length > 0); d.dispose(); } finally { @@ -99,26 +105,6 @@ modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, // serves both a stream lease and the diarize role. The refusal is the same // SessionCore.exclusive gate compute-rules.test.mjs covers for Session. -// Also uses the Sortformer ASR path (Session.run); drop it with that path. -modelTest("a diarize run and an ASR run on one model are exclusive", SORTFORMER_MODEL, async () => { - const m = await TranscribeModel.load(SORTFORMER_MODEL); - try { - const d = m.createDiarizeSession(); - const s = m.createSession(); - const order = []; - const pd = d.run(mix()).then((r) => (order.push("d"), r)); - const ps = s.run(mix()).then((r) => (order.push("s"), r)); - await Promise.resolve(); - assert.throws(() => d.timings, /run\(\).*in flight/); - assert.doesNotThrow(() => s.limits); - const [rows, r] = await Promise.all([pd, ps]); - assert.deepEqual(order, ["d", "s"]); - assert.deepEqual(rows, r.speakerSegments); - } finally { - m.dispose(); - } -}); - modelTest("the abort listener lives only for the call; a pre-aborted run raises Aborted", SORTFORMER_MODEL, async () => { const m = await TranscribeModel.load(SORTFORMER_MODEL); try { From 37233eeb3cf7bd274ddc55b97302de62e0453634 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:26:02 +0800 Subject: [PATCH 35/51] python: Sortformer is DIARIZE-only Remove SortformerStreamOptions (the SFST run-slot extension retired in the C API); SortformerDiarizeOptions now owns the preset table, __init__ and _apply it borrowed. README notes Sortformer serves only Role.DIARIZE. tests: the temporary DIARIZE-vs-ASR cross-check becomes CPU golden segments on sortformer-2spk-mix.wav at DEFAULT and LOW_LATENCY; roles are {Role.DIARIZE}, and Model.capabilities / model.session() raise UnsupportedRole on the real Sortformer model. test_family_ext drops the removed class. --- bindings/python/README.md | 3 +- .../python/src/transcribe_cpp/__init__.py | 29 ++++--------- bindings/python/tests/conftest.py | 2 +- bindings/python/tests/test_diarize.py | 41 ++++++++++++------- bindings/python/tests/test_family_ext.py | 15 ++----- 5 files changed, 40 insertions(+), 50 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index 02ab2caff..f49215bd2 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -78,7 +78,8 @@ rows (`speaker_id` in `1..model.diarize_info.max_speakers`), grouped by speaker. It shares the model-wide compute lock with transcription runs and raises `Busy` while a stream is active. `cancel()` and `close()` work as on `Session`. A model without the role raises `UnsupportedRole`, as -`model.session()` does on a model without `Role.ASR`. +`model.session()` and `model.capabilities` do on a model without `Role.ASR` +(Sortformer serves only `Role.DIARIZE`). ```python with model.diarize_session() as diarizer: diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index bec5a335f..3bf8d20a0 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -97,7 +97,6 @@ "MoonshineStreamingOptions", "ParakeetStreamOptions", "ParakeetBufferedStreamOptions", - "SortformerStreamOptions", "SortformerDiarizeOptions", "VoxtralRealtimeStreamOptions", "Backend", @@ -903,10 +902,10 @@ def _apply(self, ext) -> None: ext.min_decode_interval_ms = self.min_decode_interval_ms -class SortformerStreamOptions(FamilyExtension): - """Sortformer streaming operating-point options (run slot). +class SortformerDiarizeOptions(FamilyExtension): + """Sortformer operating-point options for ``DiarizeSession.run()`` + (diarize-run slot). - Sortformer is a diarizer: a run produces speaker segments, no text. ``preset`` selects the latency / accuracy trade-off from the model's published menu; ``"default"`` keeps the GGUF-shipped checkpoint configuration. ``"very_high_latency"`` (~30 s lookahead) is the @@ -914,10 +913,10 @@ class SortformerStreamOptions(FamilyExtension): real-time point and costs substantially more compute per audio second.""" - _slot = "run" - _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM - _struct = _generated.transcribe_sortformer_stream_ext - _init = "transcribe_sortformer_stream_ext_init" + _slot = "diarize_run" + _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE + _struct = _generated.transcribe_sortformer_diarize_ext + _init = "transcribe_sortformer_diarize_ext_init" _presets = { "default": _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, @@ -937,20 +936,6 @@ def _apply(self, ext) -> None: ext.preset = self._presets[self.preset] -class SortformerDiarizeOptions(FamilyExtension): - """Sortformer operating-point options for ``DiarizeSession.run()`` - (diarize-run slot); same presets as :class:`SortformerStreamOptions`.""" - - _slot = "diarize_run" - _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE - _struct = _generated.transcribe_sortformer_diarize_ext - _init = "transcribe_sortformer_diarize_ext_init" - - _presets = SortformerStreamOptions._presets - __init__ = SortformerStreamOptions.__init__ - _apply = SortformerStreamOptions._apply - - # --- high-level handles --------------------------------------------------- diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 5c82e9a13..392ef2935 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -168,7 +168,7 @@ def voxtral_model_path() -> Path: @pytest.fixture(scope="session") def sortformer_model_path() -> Path: - """Sortformer diarizer (serves the ASR and DIARIZE roles).""" + """Sortformer diarizer (serves only the DIARIZE role).""" return _family_model("TRANSCRIBE_SMOKE_SORTFORMER_MODEL", SORTFORMER_MODEL) diff --git a/bindings/python/tests/test_diarize.py b/bindings/python/tests/test_diarize.py index b586429ed..fa6e2e000 100644 --- a/bindings/python/tests/test_diarize.py +++ b/bindings/python/tests/test_diarize.py @@ -24,10 +24,20 @@ def _turns(rows): def test_sortformer_roles_and_info(sortformer_model_path): with t.Model(sortformer_model_path) as model: - assert model.roles == {t.Role.ASR, t.Role.DIARIZE} + assert model.roles == {t.Role.DIARIZE} assert model.diarize_info == t.DiarizeInfo(sample_rate=16000, max_speakers=4) +def test_sortformer_rejects_asr(sortformer_model_path): + with t.Model(sortformer_model_path) as model: + with pytest.raises(t.UnsupportedRole): + model.capabilities + with pytest.raises(t.UnsupportedRole): + model.session() + with model.diarize_session(): # the DIARIZE role still opens + pass + + def test_asr_only_model_rejects_diarize(model_path): with t.Model(model_path) as model: assert model.roles == {t.Role.ASR} @@ -37,24 +47,25 @@ def test_asr_only_model_rejects_diarize(model_path): model.diarize_session() -# TEMPORARY: Sortformer still serves the ASR role, whose Session.run returns -# the same speaker turns on Result.speaker_segments. This cross-check goes -# away when that ASR path is removed in a later commit. -@pytest.mark.parametrize("preset", ["default", "low_latency"]) -def test_diarize_run_matches_asr_path(sortformer_model_path, mix_pcm, preset): - with t.Model(sortformer_model_path) as model: +# CPU goldens on the oracle mix, (t0_ms, t1_ms, speaker_id) grouped by speaker. +GOLDEN = { + "default": [(320, 2400, 1), (7360, 9360, 1), (10240, 10640, 1), + (4240, 6640, 2), (9760, 12000, 2)], + "low_latency": [(320, 2480, 1), (7360, 9360, 1), (10240, 10640, 1), + (4160, 6640, 2), (9760, 12000, 2)], +} + + +@pytest.mark.parametrize("preset", sorted(GOLDEN)) +def test_diarize_run_golden_segments(sortformer_model_path, mix_pcm, preset): + with t.Model(sortformer_model_path, backend="cpu") as model: with model.diarize_session() as d: rows = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset=preset)) timings = d.timings - with model.session() as s: - asr = s.run(mix_pcm, family=t.SortformerStreamOptions(preset=preset)) - with pytest.raises(t.InvalidArgument, match="slot"): - s.run(mix_pcm, family=t.SortformerDiarizeOptions()) - assert rows and isinstance(rows, list) + assert isinstance(rows, list) assert all(isinstance(r, t.SpeakerSegment) for r in rows) - assert {r.speaker_id for r in rows} <= {1, 2, 3, 4} assert all(math.isnan(r.p) for r in rows) # Sortformer has no per-turn p - assert _turns(rows) == _turns(asr.speaker_segments) + assert _turns(rows) == GOLDEN[preset] assert timings.encode_ms > 0 @@ -74,7 +85,7 @@ class OutOfRange(t.SortformerDiarizeOptions): with pytest.raises(t.InvalidArgument): d.run(mix_pcm, family=OutOfRange(preset="bogus")) # type: ignore[arg-type] with pytest.raises(t.InvalidArgument, match="slot"): - d.run(mix_pcm, family=t.SortformerStreamOptions()) + d.run(mix_pcm, family=t.WhisperRunOptions()) assert d.run(mix_pcm) # still usable diff --git a/bindings/python/tests/test_family_ext.py b/bindings/python/tests/test_family_ext.py index 2391634e8..2f8d13619 100644 --- a/bindings/python/tests/test_family_ext.py +++ b/bindings/python/tests/test_family_ext.py @@ -22,7 +22,6 @@ t.MoonshineStreamingOptions, t.ParakeetStreamOptions, t.ParakeetBufferedStreamOptions, - t.SortformerStreamOptions, t.SortformerDiarizeOptions, t.VoxtralRealtimeStreamOptions, ] @@ -75,25 +74,19 @@ def test_parakeet_buffered_partial_overrides(): assert built.right_ms == fresh.right_ms -def test_sortformer_preset_maps_to_enum_value(): - built = t.SortformerStreamOptions(preset="very_high_latency")._build() - assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY - # None keeps the init default (DEFAULT = GGUF-shipped cfg). - default = t.SortformerStreamOptions()._build() - assert default.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT - - def test_sortformer_unknown_preset_rejected(): with pytest.raises(ValueError, match="preset"): - t.SortformerStreamOptions(preset="ultra_low_latency") # type: ignore[arg-type] + t.SortformerDiarizeOptions(preset="ultra_low_latency") # type: ignore[arg-type] def test_sortformer_diarize_preset_maps_to_enum_value(): built = t.SortformerDiarizeOptions(preset="low_latency")._build() assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY + built = t.SortformerDiarizeOptions(preset="very_high_latency")._build() + assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY + # None keeps the init default (DEFAULT = GGUF-shipped cfg). default = t.SortformerDiarizeOptions()._build() assert default.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT - assert not isinstance(t.SortformerDiarizeOptions(), t.SortformerStreamOptions) # --- model-gated: resolve_family validation + a real extension run ---------- From f79d562129064ff96602c29db443b703feb55f01 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:44:17 +0800 Subject: [PATCH 36/51] c++: trim code and comments from review No behavior change; exported symbols unchanged. - transcribe-abi.h: TRANSCRIBE_FIELD_END defined once; init_sized (every *_init body) and copy_out_timings; transcribe-session-core.h: copy_out_speaker_segment and ScratchReleaseGuard, shared by the ASR and DIARIZE dispatchers instead of three copies each. - transcribe_model_load_file calls resolve_roles once, in the forwarder. - run_one_inner no longer re-scans PCM for NaN / Inf (both callers did). - DiarizeOps::new_session replaces init_session (the dispatcher sets model and n_threads); the diarize session drops has_result (segments are built into a local and swapped in). - k_role_* aliases removed in favour of TRANSCRIBE_ROLE_*; transcribe_session drops constructor / copy boilerplate SessionCore already provides; the mel run_threaded pass-through is gone. - Unused includes from the split and the Sortformer ASR removal dropped; comments trimmed (history, ASR-only wording on the shared base, the UNSUPPORTED_ROLE and diarize timings docs, stale Sortformer notes). --- include/transcribe.h | 9 +- include/transcribe/diarize.h | 2 +- src/arch/sortformer/model.cpp | 17 +-- src/arch/sortformer/stream.cpp | 4 +- src/transcribe-abi.h | 35 +++++- src/transcribe-api-guard.h | 14 +-- src/transcribe-arch.cpp | 8 +- src/transcribe-arch.h | 3 +- src/transcribe-asr.cpp | 204 ++++++-------------------------- src/transcribe-diarize.cpp | 93 +++------------ src/transcribe-diarize.h | 8 +- src/transcribe-model.h | 10 +- src/transcribe-session-core.h | 57 ++++++--- src/transcribe.cpp | 66 ++--------- tests/diarize_dispatch_unit.cpp | 15 +-- tests/role_resolve_unit.cpp | 38 ++---- 16 files changed, 177 insertions(+), 406 deletions(-) diff --git a/include/transcribe.h b/include/transcribe.h index 0988db64b..aa07f21f3 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -313,14 +313,7 @@ typedef enum { * the check. See docs/input-limits.md. */ TRANSCRIBE_ERR_OUTPUT_REPETITION = 19, - /* - * The model does not serve the role the call needs. Returned by - * transcribe_session_init / transcribe_open and - * transcribe_model_get_capabilities (all ASR) for a model whose - * transcribe_model_roles() lacks TRANSCRIBE_ROLE_ASR, and by each - * role's session init for a model lacking that role. Check the mask - * with transcribe_model_roles() first. - */ + /* The model does not serve the role this call needs; see transcribe_model_roles(). */ TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20, } transcribe_status; diff --git a/include/transcribe/diarize.h b/include/transcribe/diarize.h index e00a864a4..ea00355b7 100644 --- a/include/transcribe/diarize.h +++ b/include/transcribe/diarize.h @@ -88,7 +88,7 @@ TRANSCRIBE_API transcribe_status transcribe_diarize_get_segment(const struct tra int i, struct transcribe_speaker_segment * out); -/* load_ms plus the last run's mel / encode time; decode_ms is 0. */ +/* load_ms plus the last run's stage times. */ TRANSCRIBE_API transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, struct transcribe_timings * out); diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index 66cbf583e..403d878fb 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -21,7 +21,6 @@ #include "transcribe-loader.h" #include "transcribe-log.h" #include "transcribe-mel.h" -#include "transcribe-meta.h" #include #include @@ -29,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -398,8 +396,8 @@ conf::BlockView sf_block_view(const pk::ParakeetBlock & b) { return v; } -// Host sinusoidal rel-pos table [pos_len, d_model] (same as the offline -// builder in run(); factored for per-chunk reuse in the streaming path). +// Host sinusoidal rel-pos table [pos_len, d_model], shared by the offline +// and streaming paths. void fill_rel_pos_emb(std::vector & buf, std::vector & div_term, int pos_len, int d_model) { const int zero_index = (pos_len - 1) / 2; const float ln_10000 = std::log(10000.0f); @@ -612,7 +610,7 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } - m->roles = k_role_diarize; + m->roles = TRANSCRIBE_ROLE_DIARIZE; m->t_load_us = ggml_time_us() - t_load_start; *out_model = m.release(); return TRANSCRIBE_OK; @@ -1021,11 +1019,8 @@ static int diarize_max_speakers(const transcribe_model * model) { return static_cast(model)->hparams.max_speakers; } -static transcribe_status diarize_init_session(transcribe_model *, - const transcribe_diarize_session_params *, - transcribe_diarize_session ** out) { - *out = new SortformerSession(); - return TRANSCRIBE_OK; +static transcribe_diarize_session * diarize_new_session() { + return new SortformerSession(); } static transcribe_status diarize_run_validate(const transcribe_diarize_params * params) { @@ -1055,7 +1050,7 @@ static transcribe_status diarize_run(transcribe_diarize_session * session, static const DiarizeOps k_diarize_ops = { /* .max_speakers = */ diarize_max_speakers, - /* .init_session = */ diarize_init_session, + /* .new_session = */ diarize_new_session, /* .run_validate = */ diarize_run_validate, /* .run = */ diarize_run, }; diff --git a/src/arch/sortformer/stream.cpp b/src/arch/sortformer/stream.cpp index 643795b90..f6bb19ea0 100644 --- a/src/arch/sortformer/stream.cpp +++ b/src/arch/sortformer/stream.cpp @@ -83,8 +83,8 @@ SortformerStreamParams resolve_stream_params(const SortformerHParams & hp, trans p.spkcache_update_period = hp.stream_spkcache_update_period > 0 ? hp.stream_spkcache_update_period : p.spkcache_update_period; - // Public run-ext preset (DEFAULT keeps the GGUF cfg). Range-checked by - // run_validate before the dispatcher clears the previous result. + // Public diarize-ext preset (DEFAULT keeps the GGUF cfg). Range-checked by + // diarize_run_validate before the dispatcher clears the previous result. const char * ext_name = nullptr; switch (preset) { case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: diff --git a/src/transcribe-abi.h b/src/transcribe-abi.h index 7f5903c7b..dfe9fe163 100644 --- a/src/transcribe-abi.h +++ b/src/transcribe-abi.h @@ -13,6 +13,10 @@ #include #include +// End offset of `field`: the minimum struct_size accepted for a caller-owned +// struct is the prefix through the last field the library reads or writes. +#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) + namespace transcribe { // Raw enum reads at the public ABI boundary @@ -32,11 +36,8 @@ inline int enum_field_raw(const void * field) { static_assert(sizeof(transcribe_backend_request) == sizeof(int), "public enums must be int-sized for raw boundary reads"); -// Every PCM entry point rejects non-finite samples (NaN / +-Inf) with -// TRANSCRIBE_ERR_INVALID_ARG before it modifies any result or stream state: -// a NaN would otherwise flow through the front end and come out as a -// confident-looking result. Silence is valid input. O(n), negligible next to -// any encoder. +// Rejects NaN / +-Inf; every PCM entry point checks this before touching +// result or stream state. inline bool pcm_is_finite(const float * pcm, int n_samples) { for (int i = 0; i < n_samples; ++i) { if (!std::isfinite(pcm[i])) { @@ -76,4 +77,28 @@ inline void copy_out_prefix(void * dst, const void * src, uint64_t caller_size, std::memcpy(dst, src, static_cast(n)); } +// The body of every public *_init(): zero-fill, then stamp struct_size. +template inline void init_sized(T * p) { + if (p != nullptr) { + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + } +} + +// Copy stage times (microseconds) out as transcribe_timings (milliseconds); +// out->struct_size is already checked. +inline void copy_out_timings(int64_t load_us, + int64_t mel_us, + int64_t encode_us, + int64_t decode_us, + struct transcribe_timings * out) { + transcribe_timings staged{}; + staged.struct_size = out->struct_size; + staged.load_ms = static_cast(load_us) / 1000.0f; + staged.mel_ms = static_cast(mel_us) / 1000.0f; + staged.encode_ms = static_cast(encode_us) / 1000.0f; + staged.decode_ms = static_cast(decode_us) / 1000.0f; + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); +} + } // namespace transcribe diff --git a/src/transcribe-api-guard.h b/src/transcribe-api-guard.h index 27cd2e4d1..5c1254351 100644 --- a/src/transcribe-api-guard.h +++ b/src/transcribe-api-guard.h @@ -1,8 +1,7 @@ -// transcribe-api-guard.h - exception guards for public C ABI entry points. -// -// Shared by every public dispatcher (transcribe.cpp, transcribe-asr.cpp) so -// the mapping from C++ exceptions to status codes lives in one place. Not -// part of the public API. +// transcribe-api-guard.h - exception guards for the public C ABI entry +// points (bad_alloc -> OOM, other exceptions -> BACKEND, teardown never +// throws). MSVC builds use /EHs so exceptions thrown by ggml C-ABI frames +// remain catchable here. Not part of the public API. #pragma once @@ -12,11 +11,6 @@ #include #include -// Public C ABI entry points must not leak C++ exceptions. The guarded -// paths below map bad_alloc to OOM, other exceptions to BACKEND, and make -// teardown best-effort/no-throw. MSVC builds use /EHs so exceptions thrown -// by ggml C-ABI frames remain catchable here. - namespace transcribe { template transcribe_status api_guard_status(const char * fn_name, Fn && body) { diff --git a/src/transcribe-arch.cpp b/src/transcribe-arch.cpp index 166b2c638..0840408b4 100644 --- a/src/transcribe-arch.cpp +++ b/src/transcribe-arch.cpp @@ -125,18 +125,18 @@ transcribe_status resolve_roles(transcribe_model * model) { const bool has_diarize = arch.diarize != nullptr; if (model->roles == 0 && has_asr) { - model->roles = k_role_asr; + model->roles = TRANSCRIBE_ROLE_ASR; } - const uint32_t known = k_role_asr | k_role_diarize; + const uint32_t known = TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE; const char * why = nullptr; if (model->roles == 0) { why = "serves no role"; } else if ((model->roles & ~known) != 0) { why = "sets an unknown role bit"; - } else if ((model->roles & k_role_asr) != 0 && !has_asr) { + } else if ((model->roles & TRANSCRIBE_ROLE_ASR) != 0 && !has_asr) { why = "sets the ASR role without init_context / run hooks"; - } else if ((model->roles & k_role_diarize) != 0 && !has_diarize) { + } else if ((model->roles & TRANSCRIBE_ROLE_DIARIZE) != 0 && !has_diarize) { why = "sets the DIARIZE role without a diarize ops table"; } if (why != nullptr) { diff --git a/src/transcribe-arch.h b/src/transcribe-arch.h index d2a4f28d4..6a6e0e0a0 100644 --- a/src/transcribe-arch.h +++ b/src/transcribe-arch.h @@ -131,8 +131,7 @@ struct Arch { // checks remain in force. transcribe_status (*run_validate)(const struct transcribe_session * ctx, const transcribe_run_params * params); - // Per-role ops tables for the non-ASR roles. nullptr = the arch does not - // implement the role. The ASR hooks above stay flat on the trait. + // Ops tables for the non-ASR roles; nullptr = role not implemented. const DiarizeOps * diarize; }; diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp index d541a4277..ffbfa0c2a 100644 --- a/src/transcribe-asr.cpp +++ b/src/transcribe-asr.cpp @@ -27,21 +27,13 @@ #include "transcribe.h" #include -#include #include -#include -#include #include #include #include #include #include -#include -#include -#include -#include #include -#include #include #include @@ -308,11 +300,7 @@ transcribe_status prepare_prompting(const transcribe_model * model, transcribe_r // See transcribe.cpp for the zero-fill + struct_size convention. extern "C" void transcribe_session_params_init(struct transcribe_session_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { @@ -328,69 +316,37 @@ extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { } extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } // Output struct init functions extern "C" void transcribe_capabilities_init(struct transcribe_capabilities * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_session_limits_init(struct transcribe_session_limits * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_stream_update_init(struct transcribe_stream_update * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_stream_text_init(struct transcribe_stream_text * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_segment_init(struct transcribe_segment * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_word_init(struct transcribe_word * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_token_init(struct transcribe_token * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } // Whisper telemetry + run-extension init and chunk-trace accessors live in @@ -401,13 +357,6 @@ extern "C" void transcribe_token_init(struct transcribe_token * p) { namespace { -// Minimum struct_size accepted on each caller-owned input/output struct. -// Sized to the prefix the library currently relies on: any field the -// library writes on a given call path must lie inside this prefix. New -// fields appended at the end of the public struct without growing the -// library-side prefix do NOT raise this value. -#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) - constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); // run_params is the one 0.2.0 exception to the append-only rule: `diarize` // was inserted mid-struct, shifting every field from `language` on by 8 @@ -426,23 +375,16 @@ constexpr size_t k_stream_update_committed_changed_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, committed_changed); constexpr size_t k_stream_update_tentative_changed_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, tentative_changed); -constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); -constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); -constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); -constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); -constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); -constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); -constexpr size_t k_min_speaker_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); -constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); +constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); +constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); +constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); +constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); +constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); +constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); +constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); // k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with // the chunk-trace accessor that uses it. -#undef TRANSCRIBE_FIELD_END - -// Size-aware ABI helpers (check_struct_size / check_input_struct_size / -// copy_out_prefix) live in transcribe-abi.h so per-family public -// accessors (arch/whisper/public.cpp) share one definition. Pull them -// into this TU's unqualified scope so existing call sites are unchanged. using transcribe::check_input_struct_size; using transcribe::check_struct_size; using transcribe::copy_out_prefix; @@ -816,7 +758,7 @@ static transcribe_status transcribe_session_init_impl(struct transcribe_model * if (model == nullptr) { return TRANSCRIBE_ERR_INVALID_ARG; } - if ((model->roles & transcribe::k_role_asr) == 0) { + if ((model->roles & TRANSCRIBE_ROLE_ASR) == 0) { return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; } // NULL params means "all defaults" (see transcribe_model_load_file). @@ -971,8 +913,6 @@ extern "C" bool transcribe_was_truncated(const struct transcribe_session * sessi // A family stream hook that throws must still end the stream: mark it FAILED // with the status the api_guard will report for the exception (OOM for // bad_alloc, BACKEND otherwise), then let the exception reach the guard. -// Without this a throwing hook left the stream ACTIVE while the caller got -// an error. template static transcribe_status call_stream_hook(transcribe_session * session, Fn && hook) { try { return hook(); @@ -1375,26 +1315,6 @@ extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_ return TRANSCRIBE_OK; } -// Scope guard that calls transcribe_session::release_scratch on exit once -// armed. Both offline entry points use it so release also happens when a -// family hook throws and the api_guard unwinds the stack, including the path -// where memory pressure matters most. release_scratch is noexcept, so running -// it during unwinding is safe. -namespace { - -struct scratch_release_guard { - transcribe_session * session = nullptr; - bool armed = false; - - ~scratch_release_guard() { - if (armed && session != nullptr) { - session->release_scratch(); - } - } -}; - -} // namespace - // Shared one-utterance run body. Does NOT touch session->batch_results, so // the batch dispatcher can call it once per utterance inside a loop without // erasing already-accumulated entries; the public transcribe_run wrapper @@ -1427,9 +1347,6 @@ static transcribe_status run_one_inner(struct transcribe_session * sess if (n_samples <= 0) { return TRANSCRIBE_ERR_INVALID_ARG; } - if (!pcm_is_finite(pcm, n_samples)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } // NULL params means "all defaults" (transcribe vs translate, no // timestamps, etc.). A well-formed default run is not a malformed // call, so it proceeds and replaces the previous result. @@ -1567,8 +1484,7 @@ static transcribe_status transcribe_run_impl(struct transcribe_session * // call must not touch the session at all — the API smoke test probes // this with a fake (session *)0x1 and a NULL pcm / non-positive // n_samples, expecting INVALID_ARG with no dereference. - // Non-finite PCM is malformed too: reject it here, before batch_results - // is touched (run_one_inner repeats the check for the batch fallback). + // Non-finite PCM is malformed: reject before batch_results is touched. if (pcm != nullptr && n_samples > 0 && !pcm_is_finite(pcm, n_samples)) { return TRANSCRIBE_ERR_INVALID_ARG; } @@ -1578,7 +1494,7 @@ static transcribe_status transcribe_run_impl(struct transcribe_session * // run_one_inner arms the guard at its commit point, so a pre-clear // rejection never releases and everything after (family error, abort, // throw) always does. - scratch_release_guard scratch_release; + transcribe::ScratchReleaseGuard scratch_release; scratch_release.session = session; return run_one_inner(session, pcm, n_samples, params, &scratch_release.armed); } @@ -1718,7 +1634,7 @@ static transcribe_status transcribe_run_batch_impl(struct transcribe_session * // Release the compute scratch once the batch has run, whichever path it // took (see transcribe_session::release_scratch). Once per call, not per // utterance, so the serial fallback keeps its workspace across the loop. - scratch_release_guard scratch_release; + transcribe::ScratchReleaseGuard scratch_release; scratch_release.session = session; scratch_release.armed = true; @@ -1775,7 +1691,7 @@ extern "C" transcribe_status transcribe_model_get_capabilities(const struct tran } // These are ASR capabilities: a model without the ASR role gets an // error, not a zeroed struct that reads as "supports nothing". - if ((model->roles & transcribe::k_role_asr) == 0) { + if ((model->roles & TRANSCRIBE_ROLE_ASR) == 0) { return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; } // Preserve the caller-declared size, then write only the prefix @@ -1908,16 +1824,8 @@ extern "C" transcribe_status transcribe_get_timings(const struct transcribe_sess if (const auto st = check_struct_size(out_timings->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { return st; } - const uint64_t caller_size = out_timings->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; - staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; - staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; - copy_out_prefix(out_timings, &staged, caller_size, sizeof(staged)); + transcribe::copy_out_timings(session->model != nullptr ? session->model->t_load_us : 0, session->t_mel_us, + session->t_encode_us, session->t_decode_us, out_timings); return TRANSCRIBE_OK; } @@ -2120,29 +2028,10 @@ extern "C" transcribe_status transcribe_get_token(const struct transcribe_sessio extern "C" transcribe_status transcribe_get_speaker_segment(const struct transcribe_session * session, int i, struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || - static_cast(i) >= session->speaker_segments.size()) { - return TRANSCRIBE_OK; - } - const auto & s = session->speaker_segments[static_cast(i)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; + const bool in_range = session != nullptr && session->has_result && i >= 0 && + static_cast(i) < session->speaker_segments.size(); + return transcribe::copy_out_speaker_segment(in_range ? &session->speaker_segments[static_cast(i)] : nullptr, + out); } // Batch result accessors @@ -2379,29 +2268,10 @@ extern "C" transcribe_status transcribe_batch_get_speaker_segment(const struct t int i, int j, struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.speaker_segments->size()) { - return TRANSCRIBE_OK; - } - const auto & s = (*v.speaker_segments)[static_cast(j)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; + const BatchResultView v = batch_result_view(session, i); + const bool in_range = v.valid && j >= 0 && static_cast(j) < v.speaker_segments->size(); + return transcribe::copy_out_speaker_segment(in_range ? &(*v.speaker_segments)[static_cast(j)] : nullptr, + out); } extern "C" transcribe_status transcribe_batch_get_timings(const struct transcribe_session * session, @@ -2429,16 +2299,8 @@ extern "C" transcribe_status transcribe_batch_get_timings(const struct transcrib } else { return TRANSCRIBE_ERR_INVALID_ARG; } - const uint64_t caller_size = out->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(mel_us) / 1000.0f; - staged.encode_ms = static_cast(enc_us) / 1000.0f; - staged.decode_ms = static_cast(dec_us) / 1000.0f; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + transcribe::copy_out_timings(session->model != nullptr ? session->model->t_load_us : 0, mel_us, enc_us, dec_us, + out); return TRANSCRIBE_OK; } diff --git a/src/transcribe-diarize.cpp b/src/transcribe-diarize.cpp index cedd4eea4..370e38fa7 100644 --- a/src/transcribe-diarize.cpp +++ b/src/transcribe-diarize.cpp @@ -10,8 +10,6 @@ #include #include -#include -#include using transcribe::api_guard_status; using transcribe::api_guard_void; @@ -21,19 +19,13 @@ using transcribe::copy_out_prefix; namespace { -#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) constexpr size_t k_min_info_size = TRANSCRIBE_FIELD_END(transcribe_diarize_info, max_speakers); constexpr size_t k_min_session_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_session_params, n_threads); constexpr size_t k_min_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_params, family); -constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); -#undef TRANSCRIBE_FIELD_END const transcribe::DiarizeOps * diarize_ops(const transcribe_model * model) { - if (model == nullptr || (model->roles & transcribe::k_role_diarize) == 0) { - return nullptr; - } - return model->arch->diarize; // non-null: resolve_roles checked it at load + return (model->roles & TRANSCRIBE_ROLE_DIARIZE) != 0 ? model->arch->diarize : nullptr; } } // namespace @@ -54,7 +46,6 @@ void transcribe::probs_to_segments(const float * probs, row.t0_ms = static_cast(std::llround(run_start * frame_ms)); row.t1_ms = static_cast(std::llround(t * frame_ms)); row.speaker_id = s + 1; - row.p = std::numeric_limits::quiet_NaN(); out.push_back(row); run_start = -1; } @@ -63,24 +54,15 @@ void transcribe::probs_to_segments(const float * probs, } extern "C" void transcribe_diarize_info_init(struct transcribe_diarize_info * p) { - if (p != nullptr) { - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - } + transcribe::init_sized(p); } extern "C" void transcribe_diarize_session_params_init(struct transcribe_diarize_session_params * p) { - if (p != nullptr) { - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - } + transcribe::init_sized(p); } extern "C" void transcribe_diarize_params_init(struct transcribe_diarize_params * p) { - if (p != nullptr) { - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - } + transcribe::init_sized(p); } static transcribe_status diarize_get_info_impl(const transcribe_model * model, transcribe_diarize_info * out) { @@ -127,12 +109,10 @@ static transcribe_status diarize_session_init_impl(transcribe_model * if (params->n_threads < 0) { return TRANSCRIBE_ERR_INVALID_ARG; } - const transcribe_status st = ops->init_session(model, params, out); - if (st == TRANSCRIBE_OK) { - (*out)->model = model; - (*out)->n_threads = params->n_threads; - } - return st; + *out = ops->new_session(); + (*out)->model = model; + (*out)->n_threads = params->n_threads; + return TRANSCRIBE_OK; } static transcribe_status diarize_run_impl(transcribe_diarize_session * session, @@ -167,25 +147,18 @@ static transcribe_status diarize_run_impl(transcribe_diarize_session * sess } session->segments.clear(); - session->has_result = false; - session->was_aborted = false; session->t_mel_us = 0; session->t_encode_us = 0; session->t_decode_us = 0; - - // Release the compute scratch however the run ends (Handy #2000). - struct ScratchRelease { - transcribe_diarize_session * s; - - ~ScratchRelease() { s->release_scratch(); } - } release{ session }; + transcribe::ScratchReleaseGuard release{ session, true }; transcribe::DiarizeProbs out; if (const auto st = ops->run(session, pcm, n_samples, params, out); st != TRANSCRIBE_OK) { return st; } - transcribe::probs_to_segments(out.probs.data(), out.n_frames, out.n_speakers, out.frame_ms, session->segments); - session->has_result = true; + std::vector segments; + transcribe::probs_to_segments(out.probs.data(), out.n_frames, out.n_speakers, out.frame_ms, segments); + session->segments.swap(segments); return TRANSCRIBE_OK; } @@ -199,32 +172,14 @@ extern "C" void transcribe_diarize_set_abort_callback(struct transcribe_diarize_ } extern "C" int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->segments.size()); + return session != nullptr ? static_cast(session->segments.size()) : 0; } extern "C" transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session, int i, struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { - return st; - } - transcribe_speaker_segment staged{}; - staged.struct_size = out->struct_size; - if (i >= 0 && i < transcribe_diarize_n_segments(session)) { - const auto & s = session->segments[static_cast(i)]; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - } - copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); - return TRANSCRIBE_OK; + const bool in_range = i >= 0 && i < transcribe_diarize_n_segments(session); + return transcribe::copy_out_speaker_segment(in_range ? &session->segments[static_cast(i)] : nullptr, out); } extern "C" transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, @@ -235,13 +190,8 @@ extern "C" transcribe_status transcribe_diarize_get_timings(const struct transcr if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { return st; } - transcribe_timings staged{}; - staged.struct_size = out->struct_size; - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; - staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; - staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; - copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + transcribe::copy_out_timings(session->model->t_load_us, session->t_mel_us, session->t_encode_us, + session->t_decode_us, out); return TRANSCRIBE_OK; } @@ -256,13 +206,8 @@ extern "C" transcribe_status transcribe_diarize_get_info(const struct transcribe extern "C" transcribe_status transcribe_diarize_session_init(struct transcribe_model * model, const struct transcribe_diarize_session_params * params, struct transcribe_diarize_session ** out) { - const transcribe_status st = api_guard_status("transcribe_diarize_session_init", - [&] { return diarize_session_init_impl(model, params, out); }); - if (st != TRANSCRIBE_OK && out != nullptr && *out != nullptr) { - transcribe_diarize_session_free(*out); - *out = nullptr; - } - return st; + return api_guard_status("transcribe_diarize_session_init", + [&] { return diarize_session_init_impl(model, params, out); }); } extern "C" void transcribe_diarize_session_free(struct transcribe_diarize_session * session) { diff --git a/src/transcribe-diarize.h b/src/transcribe-diarize.h index 4ae6ea836..8e315e664 100644 --- a/src/transcribe-diarize.h +++ b/src/transcribe-diarize.h @@ -12,8 +12,7 @@ #include struct transcribe_diarize_session : transcribe::SessionCore { - std::vector segments; - bool has_result = false; + std::vector segments; // last successful run's rows }; namespace transcribe { @@ -28,10 +27,7 @@ struct DiarizeProbs { struct DiarizeOps { int (*max_speakers)(const transcribe_model * model); - // params is never NULL (the dispatcher substitutes defaults). - transcribe_status (*init_session)(transcribe_model * model, - const transcribe_diarize_session_params * params, - transcribe_diarize_session ** out); + transcribe_diarize_session * (*new_session)(); // Pure pre-clear check of params->family (already kind/size-checked); // NULL = nothing to check. transcribe_status (*run_validate)(const transcribe_diarize_params * params); diff --git a/src/transcribe-model.h b/src/transcribe-model.h index 72849b48c..e37c439ae 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -23,10 +23,6 @@ namespace transcribe { struct Arch; class Tokenizer; -// Role bits for transcribe_model::roles: the public transcribe_role values. -constexpr uint32_t k_role_asr = TRANSCRIBE_ROLE_ASR; -constexpr uint32_t k_role_diarize = TRANSCRIBE_ROLE_DIARIZE; - // Transparent comparator keeps transcribe_model_meta_val_str allocation-free. // Loader and model must use the same map type for assignment. using MetaMap = std::map>; @@ -45,10 +41,8 @@ struct transcribe_model { // for per-call dispatch (init_context, run). const transcribe::Arch * arch = nullptr; - // Bitmask of transcribe_role this loaded model serves. Set by the - // family's load() when it serves anything other than ASR alone; left 0, - // it defaults to ASR for an arch with ASR hooks. Validated against the - // arch at load (transcribe::resolve_roles). Immutable after load. + // transcribe_role bits; set by load() (0 = ASR when the arch has ASR + // hooks) and validated by resolve_roles. uint32_t roles = 0; // Identification, both surfaced via the public string accessors. diff --git a/src/transcribe-session-core.h b/src/transcribe-session-core.h index b23a22241..6fd7f7c32 100644 --- a/src/transcribe-session-core.h +++ b/src/transcribe-session-core.h @@ -4,11 +4,11 @@ // the thread count, the per-run ggml compute scratch, the abort callback and // the stage timings. transcribe_session (ASR) derives from it, and so does // each role's session type, so scratch release (Handy #2000), cancellation -// and timings behave the same for every role. Families keep reaching these -// members through their derived context (ctx->sched, ctx->poll_abort(), ...). +// and timings behave the same for every role. #pragma once +#include "transcribe-abi.h" #include "transcribe.h" #include @@ -26,10 +26,8 @@ typedef struct ggml_backend_sched * ggml_backend_sched_t; namespace transcribe { -// "Who spoke when" rows (diarization). Populated only when a run -// resolves diarize ON for a supporting family; may overlap in time. -// t0_ms == t1_ms == 0 means the family attributes text but has no -// timing information for the turn. +// "Who spoke when" rows; may overlap in time. t0_ms == t1_ms == 0 means the +// family attributes text but has no timing for the turn. struct SpeakerSegmentEntry { int64_t t0_ms = 0; int64_t t1_ms = 0; @@ -47,19 +45,13 @@ struct SessionCore { // "library picks a sensible default" (matches the factory). int n_threads = 0; - // Per-call timings, populated by the most recent transcribe_run. - // Surfaced via the public transcribe_get_timings accessor; reset - // by transcribe_reset_timings. + // Stage timings of the last run (microseconds). int64_t t_mel_us = 0; int64_t t_encode_us = 0; int64_t t_decode_us = 0; - // Abort / cancellation (set via transcribe_set_abort_callback). - // run() drivers call poll_abort() at chunk / decode-step boundaries; - // a callback returning true sets was_aborted and the run returns - // TRANSCRIBE_ERR_ABORTED with partial segments preserved. was_aborted - // is cleared at the top of every transcribe_run, NOT by clear_result - // (the partial result may be deliberately retained). + // Abort callback; run drivers call poll_abort() at chunk / step + // boundaries and return TRANSCRIBE_ERR_ABORTED when it fires. transcribe_abort_callback abort_cb = nullptr; void * abort_userdata = nullptr; bool was_aborted = false; @@ -128,4 +120,39 @@ struct SessionCore { virtual void on_scratch_released() noexcept {} }; +// Calls release_scratch on scope exit once armed, so release also happens +// when a family hook throws and the api_guard unwinds. +struct ScratchReleaseGuard { + SessionCore * session = nullptr; + bool armed = false; + + ~ScratchReleaseGuard() { + if (armed && session != nullptr) { + session->release_scratch(); + } + } +}; + +// Copy a speaker-segment row out (a zeroed row when s is NULL), after the +// struct_size check every speaker-segment accessor shares. +inline transcribe_status copy_out_speaker_segment(const SpeakerSegmentEntry * s, transcribe_speaker_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p)); + st != TRANSCRIBE_OK) { + return st; + } + transcribe_speaker_segment staged{}; + staged.struct_size = out->struct_size; + if (s != nullptr) { + staged.t0_ms = s->t0_ms; + staged.t1_ms = s->t1_ms; + staged.speaker_id = s->speaker_id; + staged.p = s->p; + } + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + } // namespace transcribe diff --git a/src/transcribe.cpp b/src/transcribe.cpp index f6c54668b..4c3d60133 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -42,24 +42,18 @@ # include #endif -#include #include -#include -#include #include #include #include #include #include -#include #include #include #include #include -#include #include #include -#include using transcribe::api_guard_status; using transcribe::api_guard_value; @@ -385,11 +379,7 @@ void log_msg(transcribe_log_level level, const char * fmt, ...) { // memory could hit the zero case); callers reach defaults via NULL only. extern "C" void transcribe_model_load_params_init(struct transcribe_model_load_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } // Output struct init functions @@ -401,27 +391,15 @@ extern "C" void transcribe_model_load_params_init(struct transcribe_model_load_p // the accessors as a "you forgot to init the buffer" error. extern "C" void transcribe_timings_init(struct transcribe_timings * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_device_info_init(struct transcribe_device_info * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_speaker_segment_init(struct transcribe_speaker_segment * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } // Extension helpers @@ -465,22 +443,9 @@ extern "C" bool transcribe_model_supports(const struct transcribe_model * model, namespace { -// Minimum struct_size accepted on each caller-owned input/output struct. -// Sized to the prefix the library currently relies on: any field the -// library writes on a given call path must lie inside this prefix. New -// fields appended at the end of the public struct without growing the -// library-side prefix do NOT raise this value. -#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) - constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); -#undef TRANSCRIBE_FIELD_END - -// Size-aware ABI helpers (check_struct_size / check_input_struct_size / -// copy_out_prefix) live in transcribe-abi.h so per-family public -// accessors (arch/whisper/public.cpp) share one definition. Pull them -// into this TU's unqualified scope so existing call sites are unchanged. using transcribe::check_input_struct_size; using transcribe::check_struct_size; using transcribe::copy_out_prefix; @@ -861,11 +826,7 @@ static transcribe_status transcribe_model_load_file_impl(const char * // to the whisper .bin adapter, which validates the hparams as // whisper-shaped (rejecting unrelated ggml-magic files like Silero VAD). if (magic == 0x67676d6cu) { - const transcribe_status st = transcribe::whisper::load_from_bin(path, params, out_model); - if (st != TRANSCRIBE_OK || *out_model == nullptr) { - return st; - } - return transcribe::resolve_roles(*out_model); + return transcribe::whisper::load_from_bin(path, params, out_model); } // Header-only GGUF inspection. The Loader is stack-allocated; if @@ -900,15 +861,10 @@ static transcribe_status transcribe_model_load_file_impl(const char * // here instead of in every per-family handler. `variant` is owned by the // family (it may default it when stt.variant was absent) and stays on its // own accessor; this map is the generic general.* / display surface. - if (st != TRANSCRIBE_OK || *out_model == nullptr) { - return st; + if (st == TRANSCRIBE_OK && *out_model != nullptr) { + (*out_model)->meta = loader.meta(); } - (*out_model)->meta = loader.meta(); - - // Role bits are validated against the arch here, for every family, so a - // model never reaches the caller with a role it cannot serve. On failure - // the forwarder frees the model. - return transcribe::resolve_roles(*out_model); + return st; } static void transcribe_model_free_impl(struct transcribe_model * model) { @@ -1015,8 +971,10 @@ extern "C" bool transcribe_backend_available(transcribe_backend_request kind) { extern "C" transcribe_status transcribe_model_load_file(const char * path, const struct transcribe_model_load_params * params, struct transcribe_model ** out_model) { - const transcribe_status st = api_guard_status( - "transcribe_model_load_file", [&] { return transcribe_model_load_file_impl(path, params, out_model); }); + const transcribe_status st = api_guard_status("transcribe_model_load_file", [&] { + const transcribe_status lst = transcribe_model_load_file_impl(path, params, out_model); + return lst == TRANSCRIBE_OK && *out_model != nullptr ? transcribe::resolve_roles(*out_model) : lst; + }); // Boundary-owned postcondition: failure => *out_model == NULL. if (st != TRANSCRIBE_OK && out_model != nullptr && *out_model != nullptr) { transcribe_model_free(*out_model); diff --git a/tests/diarize_dispatch_unit.cpp b/tests/diarize_dispatch_unit.cpp index f37249a39..ebadba14f 100644 --- a/tests/diarize_dispatch_unit.cpp +++ b/tests/diarize_dispatch_unit.cpp @@ -36,11 +36,8 @@ int fake_max_speakers(const transcribe_model *) { return 3; } -transcribe_status fake_init_session(transcribe_model *, - const transcribe_diarize_session_params *, - transcribe_diarize_session ** out) { - *out = new transcribe_diarize_session(); - return TRANSCRIBE_OK; +transcribe_diarize_session * fake_new_session() { + return new transcribe_diarize_session(); } transcribe_status fake_run_validate(const transcribe_diarize_params *) { @@ -68,7 +65,7 @@ bool fake_accepts(const transcribe_model *, transcribe_ext_slot slot, uint32_t k return slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && kind == k_fake_kind; } -const transcribe::DiarizeOps k_ops = { fake_max_speakers, fake_init_session, fake_run_validate, fake_run }; +const transcribe::DiarizeOps k_ops = { fake_max_speakers, fake_new_session, fake_run_validate, fake_run }; const transcribe::Arch k_arch = { /* .name = */ "fake-diarize", @@ -93,7 +90,7 @@ struct Fixture { Fixture() { model.arch = &k_arch; - model.roles = transcribe::k_role_diarize; + model.roles = TRANSCRIBE_ROLE_DIARIZE; CHECK(transcribe_diarize_session_init(&model, nullptr, &session) == TRANSCRIBE_OK); } @@ -119,7 +116,7 @@ void test_probs_to_segments() { void test_role_checks() { transcribe_model model; model.arch = &k_arch; - model.roles = transcribe::k_role_asr; // no DIARIZE bit + model.roles = TRANSCRIBE_ROLE_ASR; // no DIARIZE bit transcribe_diarize_session * s = reinterpret_cast(0x1); CHECK(transcribe_diarize_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); @@ -129,7 +126,7 @@ void test_role_checks() { transcribe_diarize_info_init(&info); CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); - model.roles = transcribe::k_role_diarize; + model.roles = TRANSCRIBE_ROLE_DIARIZE; CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_OK); CHECK(info.sample_rate == 16000 && info.max_speakers == 3); CHECK(transcribe_diarize_get_info(nullptr, &info) == TRANSCRIBE_ERR_INVALID_ARG); diff --git a/tests/role_resolve_unit.cpp b/tests/role_resolve_unit.cpp index c7e39fc40..e8c766fce 100644 --- a/tests/role_resolve_unit.cpp +++ b/tests/role_resolve_unit.cpp @@ -6,7 +6,6 @@ #include #include -#include namespace { @@ -55,16 +54,12 @@ transcribe_status resolve(const transcribe::Arch & arch, uint32_t roles, uint32_ return st; } -void test_asr_default() { +void test_asr_default_and_explicit() { uint32_t roles = 0; CHECK(resolve(k_asr_arch, 0, &roles) == TRANSCRIBE_OK); - CHECK(roles == transcribe::k_role_asr); -} - -void test_asr_explicit() { - uint32_t roles = 0; - CHECK(resolve(k_asr_arch, transcribe::k_role_asr, &roles) == TRANSCRIBE_OK); - CHECK(roles == transcribe::k_role_asr); + CHECK(roles == TRANSCRIBE_ROLE_ASR); + CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR, &roles) == TRANSCRIBE_OK); + CHECK(roles == TRANSCRIBE_ROLE_ASR); } void test_no_role_rejected() { @@ -75,23 +70,22 @@ void test_no_role_rejected() { void test_unbacked_role_rejected() { uint32_t roles = 0; - CHECK(resolve(k_half_asr_arch, transcribe::k_role_asr, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - CHECK(resolve(k_asr_arch, transcribe::k_role_diarize, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - CHECK(resolve(k_asr_arch, transcribe::k_role_asr | transcribe::k_role_diarize, &roles) == - TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_half_asr_arch, TRANSCRIBE_ROLE_ASR, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_DIARIZE, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); } void test_unknown_bit_rejected() { uint32_t roles = 0; - CHECK(resolve(k_asr_arch, transcribe::k_role_asr | (1u << 31), &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR | (1u << 31), &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); } -// D5: ASR entry points refuse a model that does not serve ASR, before +// ASR entry points refuse a model that does not serve ASR, before // touching params or the arch; capabilities leave the caller's struct alone. void test_asr_entry_points_check_role() { transcribe_model model; model.arch = &k_asr_arch; - model.roles = transcribe::k_role_diarize; // not ASR + model.roles = TRANSCRIBE_ROLE_DIARIZE; // not ASR transcribe_session * s = reinterpret_cast(0x1); CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); @@ -107,19 +101,13 @@ void test_asr_entry_points_check_role() { // Same model with the ASR bit: capabilities succeed; session_init gets // past the role check to the fake arch's init_context. - model.roles = transcribe::k_role_asr; + model.roles = TRANSCRIBE_ROLE_ASR; CHECK(transcribe_model_get_capabilities(&model, &caps) == TRANSCRIBE_OK); CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); CHECK(s == nullptr); CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_ASR); } -void test_role_bits_match_public_enum() { - static_assert(transcribe::k_role_asr == TRANSCRIBE_ROLE_ASR, "internal and public role bits must match"); - static_assert(transcribe::k_role_diarize == TRANSCRIBE_ROLE_DIARIZE, "internal and public role bits must match"); - CHECK(std::strcmp(transcribe_status_string(TRANSCRIBE_ERR_UNSUPPORTED_ROLE), "unknown status") != 0); -} - void test_null_model() { CHECK(transcribe::resolve_roles(nullptr) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); transcribe_model model; @@ -131,14 +119,12 @@ void test_null_model() { int main() { transcribe_log_set(nullptr, nullptr); // the rejections log at ERROR - test_asr_default(); - test_asr_explicit(); + test_asr_default_and_explicit(); test_no_role_rejected(); test_unbacked_role_rejected(); test_unknown_bit_rejected(); test_null_model(); test_asr_entry_points_check_role(); - test_role_bits_match_public_enum(); if (g_failures != 0) { std::fprintf(stderr, "%d failure(s)\n", g_failures); From a0efe19de76cb734b886c1abe4b7461eece4aa82 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:40:14 +0800 Subject: [PATCH 37/51] swift: trim code and tests from review --- bindings/swift/README.md | 6 +- .../swift/Sources/TranscribeCpp/Diarize.swift | 8 +- .../swift/Sources/TranscribeCpp/Family.swift | 42 ++--- .../swift/Sources/TranscribeCpp/Model.swift | 36 +++-- .../swift/Sources/TranscribeCpp/Session.swift | 20 +-- .../Sources/TranscribeCpp/Streaming.swift | 5 +- .../TranscribeCpp/TranscribeError.swift | 5 +- .../TranscribeCppTests/CancelTests.swift | 2 + .../ComputeRulesTests.swift | 146 +----------------- .../TranscribeCppTests/DiarizeTests.swift | 26 +--- .../TranscribeCppTests/NoModelTests.swift | 18 --- .../TranscribeCppTests/StreamingTests.swift | 42 ----- .../TranscribeCppTests/TranscribeTests.swift | 10 +- 13 files changed, 65 insertions(+), 301 deletions(-) diff --git a/bindings/swift/README.md b/bindings/swift/README.md index b216d16e4..c8e1ce376 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -105,16 +105,12 @@ such as Sortformer (DIARIZE only) return speaker turns from a `DiarizeSession`: ```swift let model = try Model(path: "diar_streaming_sortformer_4spk-v2.1-F32.gguf") -let info = try model.diarizeInfo // sampleRate, maxSpeakers let turns = try model.diarizeSession().run( pcm, options: DiarizeOptions(family: .sortformer(.init(preset: .veryHighLatency)))) for t in turns { print(t.speakerId, t.t0Ms, t.t1Ms) } ``` -Diarize runs share the model's compute lock and stream lease with sessions, and -take a `CancellationToken` the same way. `model.capabilities` and -`model.session()` are ASR-only and throw `TranscribeError.unsupportedRole` on a -model without `.asr`, such as Sortformer. +`capabilities` and `session()` throw `.unsupportedRole` on a model without `.asr`. ## Backends diff --git a/bindings/swift/Sources/TranscribeCpp/Diarize.swift b/bindings/swift/Sources/TranscribeCpp/Diarize.swift index f0d00908d..82ae05325 100644 --- a/bindings/swift/Sources/TranscribeCpp/Diarize.swift +++ b/bindings/swift/Sources/TranscribeCpp/Diarize.swift @@ -59,11 +59,9 @@ public final class DiarizeSession { /// Diarize one recording (16 kHz mono float32). Rows are grouped by /// speaker, time-ordered within a speaker; speakers may overlap. public func run(_ pcm: [Float], options: DiarizeOptions = .init()) throws -> [SpeakerSegment] { - try model.withCompute { - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before diarize run()") - } + try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before diarize run()" + ) { let status = withDiarizeExtension(options.family) { ext in var params = transcribe_diarize_params() transcribe_diarize_params_init(¶ms) diff --git a/bindings/swift/Sources/TranscribeCpp/Family.swift b/bindings/swift/Sources/TranscribeCpp/Family.swift index 67566d142..7f4165c91 100644 --- a/bindings/swift/Sources/TranscribeCpp/Family.swift +++ b/bindings/swift/Sources/TranscribeCpp/Family.swift @@ -46,27 +46,6 @@ public struct WhisperRunOptions: Sendable { } } -/// Sortformer streaming operating point (latency / accuracy trade-off). -/// The menu is discrete (jointly-tuned bundles), not a latency dial; -/// `.default` keeps the GGUF-shipped checkpoint configuration. -/// `.veryHighLatency` (~30 s lookahead) is the offline-file operating -/// point; `.lowLatency` (~1 s) is the real-time point. -public enum SortformerPreset: Sendable { - case `default` - case veryHighLatency - case highLatency - case lowLatency - - var cValue: transcribe_sortformer_preset { - switch self { - case .default: return TRANSCRIBE_SORTFORMER_PRESET_DEFAULT - case .veryHighLatency: return TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY - case .highLatency: return TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY - case .lowLatency: return TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY - } - } -} - public enum RunExtension: Sendable { case whisper(WhisperRunOptions) @@ -181,6 +160,27 @@ func withStreamExtension( // MARK: - DIARIZE_RUN-slot extensions +/// Sortformer streaming operating point (latency / accuracy trade-off). +/// The menu is discrete (jointly-tuned bundles), not a latency dial; +/// `.default` keeps the GGUF-shipped checkpoint configuration. +/// `.veryHighLatency` (~30 s lookahead) is the offline-file operating +/// point; `.lowLatency` (~1 s) is the real-time point. +public enum SortformerPreset: Sendable { + case `default` + case veryHighLatency + case highLatency + case lowLatency + + var cValue: transcribe_sortformer_preset { + switch self { + case .default: return TRANSCRIBE_SORTFORMER_PRESET_DEFAULT + case .veryHighLatency: return TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY + case .highLatency: return TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY + case .lowLatency: return TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY + } + } +} + /// Sortformer options for `DiarizeSession.run`. public struct SortformerDiarizeOptions: Sendable { public var preset: SortformerPreset? diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index 81322ccb0..34ef8dc7e 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -1,12 +1,6 @@ import CTranscribe import Foundation -/// A loaded model. Safe to share across threads (`@unchecked Sendable`): the C -/// API allows concurrent queries and session creation, and the compute path is -/// serialized by an internal lock (the C "one in-flight run per model" -/// contract — the same per-model mutex + stream lease the Rust binding uses). A -/// `Model` outlives every `Session` derived from it; the session holds a strong -/// reference, so close ordering is automatic under ARC. /// The kinds of work a model serves (`transcribe_model_roles`). ASR is /// `Session`; DIARIZE is `DiarizeSession`. public struct Roles: OptionSet, Sendable { @@ -16,6 +10,12 @@ public struct Roles: OptionSet, Sendable { public static let diarize = Roles(rawValue: TRANSCRIBE_ROLE_DIARIZE.rawValue) } +/// A loaded model. Safe to share across threads (`@unchecked Sendable`): the C +/// API allows concurrent queries and session creation, and the compute path is +/// serialized by an internal lock (the C "one in-flight run per model" +/// contract — the same per-model mutex + stream lease the Rust binding uses). A +/// `Model` outlives every `Session` derived from it; the session holds a strong +/// reference, so close ordering is automatic under ARC. public final class Model: @unchecked Sendable { let ptr: OpaquePointer /// Serializes the run/feed/finalize compute path across all sessions, and @@ -24,25 +24,27 @@ public final class Model: @unchecked Sendable { /// The compute lease: `true` while some session holds an ACTIVE stream. /// The C contract allows at most one in-flight run/stream across ALL /// sessions of a model, and an active stream spans begin..finalize/reset/ - /// drop — so `run`/`runBatch`/another `stream`/a diarize `run` are refused with `.busy` - /// while it is held, rather than racing into the documented UB (corrupted - /// decodes on CPU, command-buffer failures on Metal). Always accessed under - /// `runLock`. + /// drop — so `run`/`runBatch`/another `stream`/a diarize `run` are refused + /// with `.busy` while it is held, rather than racing into the documented UB + /// (corrupted decodes on CPU, command-buffer failures on Metal). Always + /// accessed under `runLock`. var streamActive = false - /// Run `body` holding this model's compute lock. Every native call that - /// computes or mutates a session's results goes through here, so all - /// sessions of one model share the C "one in-flight compute per model" - /// exclusion. `body` also copies results into owned Swift values before - /// the lock is released, and may read or change `streamActive`. Argument - /// checks that must not wait on the lock stay before the call; the - /// `streamActive` refusal, where a site has one, is `body`'s first step. + /// Run `body` (a native compute call and its copy-out) under the model-wide + /// compute lock; `busyIfStreaming` then refuses an active stream with `.busy`. func withCompute(_ body: () throws -> R) rethrows -> R { runLock.lock() defer { runLock.unlock() } return try body() } + func withCompute(busyIfStreaming message: String, _ body: () throws -> R) throws -> R { + try withCompute { + if streamActive { throw TranscribeError.busy(message) } + return try body() + } + } + /// Load a model from a GGUF file. Runs the pre-1.0 version gate first. public init(path: String, options: ModelOptions = .init()) throws { try Transcribe.ensureCompatible() diff --git a/bindings/swift/Sources/TranscribeCpp/Session.swift b/bindings/swift/Sources/TranscribeCpp/Session.swift index 17ffc8f98..e8393b102 100644 --- a/bindings/swift/Sources/TranscribeCpp/Session.swift +++ b/bindings/swift/Sources/TranscribeCpp/Session.swift @@ -51,12 +51,10 @@ public final class Session { /// Transcribe one utterance. `pcm` is mono float32 at 16 kHz in [-1, 1]. public func run(_ pcm: [Float], options: RunOptions = .init()) throws -> Transcript { try options.checkCStrings() - return try model.withCompute { - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before run()") - } - return try options.withCParams { params in + return try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before run()" + ) { + try options.withCParams { params in let status = pcm.withUnsafeBufferPointer { transcribe_run(ptr, $0.baseAddress, Int32($0.count), params) } @@ -72,12 +70,10 @@ public final class Session { _ inputs: [[Float]], options: RunOptions = .init() ) throws -> [Result] { try options.checkCStrings() - return try model.withCompute { - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before runBatch()") - } - return try options.withCParams { params in + return try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before runBatch()" + ) { + try options.withCParams { params in let counts = inputs.map { Int32($0.count) } let status = withPCMPointers(inputs[...], []) { pointers in pointers.withUnsafeBufferPointer { pp in diff --git a/bindings/swift/Sources/TranscribeCpp/Streaming.swift b/bindings/swift/Sources/TranscribeCpp/Streaming.swift index 8967ad36f..2097759bd 100644 --- a/bindings/swift/Sources/TranscribeCpp/Streaming.swift +++ b/bindings/swift/Sources/TranscribeCpp/Streaming.swift @@ -192,10 +192,7 @@ extension Session { _ runOptions: RunOptions = .init(), _ streamOptions: StreamOptions = .init() ) throws -> Stream { try runOptions.checkCStrings() - return try model.withCompute { - if model.streamActive { - throw TranscribeError.busy("a stream is already active on this model") - } + return try model.withCompute(busyIfStreaming: "a stream is already active on this model") { let status = runOptions.withCParams { runParams in streamOptions.withCParams { streamParams in transcribe_stream_begin(ptr, runParams, streamParams) diff --git a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift index 4217c4e7c..4e0db09c9 100644 --- a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift +++ b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift @@ -28,10 +28,7 @@ public enum TranscribeError: Error { /// The decode was stopped because the output began repeating itself; the /// repeats are dropped from `partial`, which is incomplete. case outputRepetition(message: String, partial: Transcript?) - /// The model does not serve the role the call needs - /// (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`); e.g. creating an ASR session on a - /// model whose roles lack ASR. A model-level refusal, distinct from - /// `.unsupported` (a per-run option the model can't honor). + /// The model's `roles` lack the one the call needs (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`). case unsupportedRole(String) case versionMismatch(String) case busy(String) diff --git a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift index e133008d2..024b217e3 100644 --- a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift @@ -30,6 +30,7 @@ final class CancelTests: XCTestCase { } } XCTAssertTrue(session.wasAborted) + XCTAssertTrue(session.cancelToken === token, "a sync run leaves the caller's token installed") } func testCrossThreadCancelOfInFlightRun() throws { @@ -77,6 +78,7 @@ final class CancelTests: XCTestCase { } } XCTAssertTrue(session.wasAborted) + XCTAssertNil(session.cancelToken, "the bridged token is removed after the call") } /// The bridge must never clobber a caller-installed token: there is a single diff --git a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift index 46c3bd621..99eb88422 100644 --- a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift @@ -3,19 +3,7 @@ import XCTest @testable import TranscribeCpp -/// Characterization of the rules every native compute call follows today, so -/// moving them behind one helper can be checked against the old behavior: -/// -/// - every compute site (run, runBatch, stream begin/feed/finalize/reset, a -/// dropped `Stream`'s reset, and a diarize run) waits on the model-wide `runLock`, which all -/// sessions of one model share; -/// - run / runBatch / stream begin validate their string options BEFORE taking -/// the lock, and refuse an active stream with `.busy` only AFTER taking it; -/// - the async bridge installs its own cancel token only when none is set, and -/// removes it when the call returns (including after an abort); -/// - a `Session` keeps its `Model` alive and a `Stream` keeps its `Session` -/// alive, so dropping the caller's references mid-use is safe; -/// - results are owned Swift copies that later compute cannot disturb. +/// Every compute site waits on the model-wide lock; option checks run before it, the busy check after. final class ComputeRulesTests: XCTestCase { // MARK: - Helpers @@ -202,12 +190,6 @@ final class ComputeRulesTests: XCTestCase { } XCTAssertEqual(busyMessage(streamError.value), "a stream is already active on this model") - // The same session's second stream gets the binding's `.busy` too, not - // a native error. - XCTAssertEqual( - busyMessage(Result { try s1.stream() }.failure), - "a stream is already active on this model") - active.reset() } @@ -244,72 +226,8 @@ final class ComputeRulesTests: XCTestCase { active.reset() } - // MARK: - Cancel hook install / cleanup - - /// The async bridge installs a token only for the call and removes it - /// afterwards, on success and after an abort alike. - func testAsyncBridgeRemovesItsTokenAfterTheCall() async throws { - let (path, pcm) = try Fixtures.modelAndAudio() - let session = try Model(path: path).session() - XCTAssertNil(session.cancelToken) - - _ = try await session.run(pcm) - XCTAssertNil(session.cancelToken, "bridged token must be removed after run") - _ = try await session.runBatch([pcm]) - XCTAssertNil(session.cancelToken, "bridged token must be removed after runBatch") - - let long = Array(repeating: pcm, count: 6).flatMap { $0 } - let task = Task { try await session.run(long) } - task.cancel() - _ = try? await task.value - XCTAssertTrue(session.wasAborted) - XCTAssertNil(session.cancelToken, "bridged token must be removed after an aborted run") - - // The cancelled token is not left behind: the next call is not aborted. - _ = try await session.run(pcm) - XCTAssertFalse(session.wasAborted) - XCTAssertNil(session.cancelToken) - } - - /// The synchronous path never installs or clears a token: a caller token - /// stays installed (and stays cancelled) across runs until the caller - /// clears it. - func testSyncRunLeavesCallerTokenInstalled() throws { - let (path, pcm) = try Fixtures.modelAndAudio() - let session = try Model(path: path).session() - let token = CancellationToken() - token.cancel() - session.setCancellationToken(token) - for _ in 0..<2 { - XCTAssertThrowsError(try session.run(pcm)) { error in - guard case TranscribeError.aborted = error else { - return XCTFail("expected .aborted, got \(error)") - } - } - XCTAssertTrue(session.cancelToken === token) - } - session.clearCancellationToken() - XCTAssertNil(session.cancelToken) - _ = try session.run(pcm) - XCTAssertFalse(session.wasAborted) - } - // MARK: - Keep-alive - func testSessionKeepsModelAliveForRun() throws { - let (path, pcm) = try Fixtures.modelAndAudio() - weak var weakModel: Model? - var session: Session? = try { - let model = try Model(path: path) - weakModel = model - return try model.session() - }() - XCTAssertNotNil(weakModel, "a live Session must keep its Model alive") - XCTAssertTrue(try session!.run(pcm).text.lowercased().contains("country")) - session = nil - XCTAssertNil(weakModel, "the Model is freed once its last Session is") - } - func testStreamKeepsSessionAndModelAlive() throws { let (path, pcm) = try Fixtures.streamingModelAndAudio() weak var weakModel: Model? @@ -329,66 +247,4 @@ final class ComputeRulesTests: XCTestCase { XCTAssertNil(weakSession) XCTAssertNil(weakModel) } - - /// The caller may drop every reference while an async call is in flight: - /// the call itself keeps the session (and so the model) alive until it - /// returns. - func testCallerDropsReferencesDuringAsyncRun() async throws { - let (path, pcm) = try Fixtures.modelAndAudio() - let long = Array(repeating: pcm, count: 3).flatMap { $0 } - let task: Task = try { - let session = try Model(path: path).session() - return Task { try await session.run(long) } - }() - let transcript = try await task.value - XCTAssertTrue(transcript.text.lowercased().contains("country")) - } - - // MARK: - Copy-out - - /// Results are owned copies: later compute on the same session or on a - /// sibling session leaves them untouched. - func testResultsSurviveLaterCompute() throws { - let (path, pcm) = try Fixtures.modelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - let first = try s1.run(pcm) - let batch = try s1.runBatch([pcm]) - let text = first.text - let segments = first.segments.count - let batchText = try batch[0].get().text - XCTAssertTrue(text.lowercased().contains("country")) - - let short = Array(pcm.prefix(16000)) - _ = try s1.run(short) - _ = try s1.runBatch([short]) - _ = try s2.run(short) - - XCTAssertEqual(first.text, text) - XCTAssertEqual(first.segments.count, segments) - XCTAssertEqual(try batch[0].get().text, batchText) - XCTAssertNotEqual(try s1.run(short).text, text) - } - - func testStreamSnapshotSurvivesReset() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let session = try Model(path: path).session() - let stream = try session.stream() - try Fixtures.drive(stream, pcm: pcm) - let snapshot = stream.snapshot - let text = stream.text - XCTAssertTrue(text.full.lowercased().contains("country")) - stream.reset() - XCTAssertEqual(stream.snapshot.text, "") - XCTAssertTrue(snapshot.text.lowercased().contains("country")) - XCTAssertTrue(text.full.lowercased().contains("country")) - } -} - -private extension Result { - var failure: Failure? { - if case .failure(let error) = self { return error } - return nil - } } diff --git a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift index 0cdc8c06b..6b07754da 100644 --- a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift @@ -1,4 +1,3 @@ -import CTranscribe import XCTest @testable import TranscribeCpp @@ -21,11 +20,6 @@ final class DiarizeTests: XCTestCase { } } - func testRoleBitsMatchTheHeader() { - XCTAssertEqual(Roles.asr.rawValue, 1) - XCTAssertEqual(Roles.diarize.rawValue, 2) - } - func testAsrOnlyModelRefusesDiarize() throws { guard let path = Fixtures.modelPath() else { throw XCTSkip("no canary model") } let model = try Model(path: path) @@ -68,30 +62,12 @@ final class DiarizeTests: XCTestCase { XCTAssertEqual(turns(try session.run(pcm)), golden[0].1, "no extension == .default") } - func testBadPresetAndCancellation() throws { + func testCancelledRunAbortsAndRecovers() throws { let (path, pcm) = try Fixtures.sortformerModelAndAudio() let session = try Model(path: path).diarizeSession() let first = try session.run(pcm) XCTAssertFalse(first.isEmpty) - // `SortformerPreset` cannot express an out-of-range preset, so the - // native rejection is driven through the raw C call. - var ext = transcribe_sortformer_diarize_ext() - transcribe_sortformer_diarize_ext_init(&ext) - ext.preset = transcribe_sortformer_preset(rawValue: 99) - let status = withUnsafePointer(to: &ext.ext) { family in - var params = transcribe_diarize_params() - transcribe_diarize_params_init(¶ms) - params.family = family - return pcm.withUnsafeBufferPointer { - transcribe_diarize_run(session.ptr, $0.baseAddress, Int32($0.count), ¶ms) - } - } - guard case .invalidArgument = TranscribeError.make(status) else { - return XCTFail("expected .invalidArgument, got \(status)") - } - XCTAssertEqual(Int(transcribe_diarize_n_segments(session.ptr)), first.count) - let token = CancellationToken() token.cancel() session.setCancellationToken(token) diff --git a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift index b9370dfeb..56e21e173 100644 --- a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift @@ -49,24 +49,6 @@ final class NoModelTests: XCTestCase { XCTAssertNil(TranscribeError.inputTooLong("").partial) } - // No loadable model lacks the ASR role yet, so the mapping is checked - // directly rather than through a native call. - func testUnsupportedRoleMapsToItsOwnCase() { - XCTAssertNotEqual(Transcribe.statusString(20), "unknown status") - let error = TranscribeError.make(TRANSCRIBE_ERR_UNSUPPORTED_ROLE, context: "creating session") - guard case .unsupportedRole(let message) = error else { - return XCTFail("expected .unsupportedRole, got \(error)") - } - XCTAssertEqual(message, "creating session: \(Transcribe.statusString(20))") - XCTAssertNil(error.partial) - XCTAssertFalse(error.isTruncated) - XCTAssertThrowsError(try TranscribeError.check(TRANSCRIBE_ERR_UNSUPPORTED_ROLE)) { thrown in - guard case TranscribeError.unsupportedRole = thrown else { - return XCTFail("expected .unsupportedRole, got \(thrown)") - } - } - } - // Every native status maps to its case. `label` has no `default`, so a new // TranscribeError case does not compile until this table is updated; the // "unknown status" check below flags a newly appended C status. diff --git a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift index d18743972..1d72d5f8c 100644 --- a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift @@ -79,48 +79,6 @@ final class StreamingTests: XCTestCase { // MARK: - Compute lease (C contract: one in-flight run/stream per model) - /// Regression for the streaming compute lease. The C library allows at most - /// one in-flight run/stream across ALL sessions of a model, and an active - /// stream spans begin..finalize/reset/drop. Before the lease, a second - /// stream — or an offline run — on another session of the same model began - /// concurrently and raced into the documented UB (corrupted decodes / Metal - /// command-buffer failures); now it is refused with `.busy`. Mirrors Rust's - /// `concurrent_compute_on_one_model_is_refused`. - func testConcurrentComputeOnOneModelIsRefused() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - - let stream1 = try s1.stream() - _ = try stream1.feed(Array(pcm.prefix(1600))) // s1's stream is ACTIVE - - // A second stream on the same model while the first is live -> .busy. - XCTAssertThrowsError(try s2.stream()) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a second stream, got \(error)") - } - } - // An offline run on the same model while a stream is live -> .busy too. - XCTAssertThrowsError(try s2.run(pcm)) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a run mid-stream, got \(error)") - } - } - // runBatch is gated on the same lease. - XCTAssertThrowsError(try s2.runBatch([pcm])) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a runBatch mid-stream, got \(error)") - } - } - - // Releasing the first stream frees the lease; s2 can now stream. - stream1.reset() - XCTAssertFalse(model.streamActive) - let stream2 = try s2.stream() - stream2.reset() - } - /// 2b: a `Stream` dropped without `finalize()`/`reset()` must reset the /// session and release the model's compute lease in `deinit` — otherwise /// the session is wedged ACTIVE forever and the model stays `.busy`. diff --git a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift index 6b7f7bac0..74cb4c1ce 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift @@ -125,12 +125,16 @@ final class TranscribeTests: XCTestCase { let (path, pcm) = try Fixtures.modelAndAudio() // Drop the local Model reference; the Session's strong ref must keep the // native model alive (close-ordering safety under ARC). - let session: Session = try { + weak var weakModel: Model? + var session: Session? = try { let model = try Model(path: path) + weakModel = model return try model.session() }() - let transcript = try session.run(pcm) - XCTAssertTrue(transcript.text.lowercased().contains("country")) + XCTAssertNotNil(weakModel, "a live Session must keep its Model alive") + XCTAssertTrue(try session!.run(pcm).text.lowercased().contains("country")) + session = nil + XCTAssertNil(weakModel, "the Model is freed once its last Session is") } func testSharedModelAcrossThreadsSerializes() throws { From e7cd28c8eec924aa71139dbfce05c0553fbea937 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:44:02 +0800 Subject: [PATCH 38/51] rust: trim code and tests from review --- bindings/rust/transcribe-cpp/README.md | 13 +- bindings/rust/transcribe-cpp/src/diarize.rs | 37 +-- bindings/rust/transcribe-cpp/src/error.rs | 44 +-- bindings/rust/transcribe-cpp/src/model.rs | 14 +- bindings/rust/transcribe-cpp/src/session.rs | 77 ++--- bindings/rust/transcribe-cpp/src/types.rs | 5 - bindings/rust/transcribe-cpp/tests/cancel.rs | 39 ++- .../transcribe-cpp/tests/compute_rules.rs | 298 ------------------ bindings/rust/transcribe-cpp/tests/diarize.rs | 65 ++-- .../rust/transcribe-cpp/tests/streaming.rs | 132 +++++++- .../rust/transcribe-cpp/tests/transcribe.rs | 13 +- 11 files changed, 226 insertions(+), 511 deletions(-) delete mode 100644 bindings/rust/transcribe-cpp/tests/compute_rules.rs diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index 3e6bb16e5..086ecbcd6 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -78,12 +78,9 @@ let transcript = stream.snapshot(); // language, segments, words, tokens, timing ### Diarization (who spoke when) -`model.roles()` reports what a model can do. A model with `Role::Diarize` -(e.g. Sortformer, which is DIARIZE-only) opens a `DiarizeSession`, whose `run` -returns speaker turns (`SpeakerSegment`: `t0_ms`, `t1_ms`, 1-based -`speaker_id`). ASR-only calls such as `capabilities()` and `session()` return -`Error::UnsupportedRole` on a model without `Role::Asr`, and the diarize calls -do the same without `Role::Diarize`. +A model whose `roles()` contain `Role::Diarize` (e.g. Sortformer) opens a +`DiarizeSession` that returns speaker turns; calls for a role the model lacks +return `Error::UnsupportedRole`. ```rust use transcribe_cpp::{DiarizeOptions, Model, Role}; @@ -96,10 +93,6 @@ for turn in diarize.run(&pcm, &DiarizeOptions::default())? { # Ok::<(), transcribe_cpp::Error>(()) ``` -`DiarizeOptions::family` takes `DiarizeExtension::Sortformer` with a -`SortformerPreset`. Diarize runs share the model-wide compute lock with ASR -sessions and accept a `CancelToken`. - Runnable examples: ```sh diff --git a/bindings/rust/transcribe-cpp/src/diarize.rs b/bindings/rust/transcribe-cpp/src/diarize.rs index c79a3b8a1..8ae20f588 100644 --- a/bindings/rust/transcribe-cpp/src/diarize.rs +++ b/bindings/rust/transcribe-cpp/src/diarize.rs @@ -1,11 +1,5 @@ -//! [`DiarizeSession`] — the DIARIZE role: who spoke when. -//! -//! For models whose product is speaker turns rather than text -//! ([`Model::roles`] contains [`Role::Diarize`](crate::Role)). Open one with -//! [`Model::diarize_session`]. Same threading and lifetime rules as -//! [`Session`](crate::Session): `Send` but not `Sync`, keeps its model alive, -//! and its run shares the model-wide compute lock with every other session of -//! the model. +//! [`DiarizeSession`] — the DIARIZE role (who spoke when), from [`Model::diarize_session`]. +//! Threading, lifetime, and compute-lock rules are those of [`Session`](crate::Session). use std::os::raw::c_void; use std::sync::atomic::AtomicBool; @@ -126,10 +120,9 @@ impl DiarizeSession { unsafe { sys::transcribe_diarize_params_init(&mut params) }; params.family = family.as_ref().map_or(std::ptr::null(), |f| f.ext_ptr()); - let ptr = self.ptr; let status = self.model.with_compute( Some("a stream is active on this model; finish or drop it before diarize run()"), - |_| unsafe { sys::transcribe_diarize_run(ptr, pcm.as_ptr(), n, ¶ms) }, + |_| unsafe { sys::transcribe_diarize_run(self.ptr, pcm.as_ptr(), n, ¶ms) }, )?; check(status, "diarize run")?; @@ -159,22 +152,19 @@ mod tests { use crate::Error; use std::sync::Mutex; - /// A session over null native handles (both frees are NULL no-ops): the - /// Busy check runs under the lock before any native call. - fn null_session(stream_holds_lease: bool) -> DiarizeSession { - DiarizeSession { + #[test] + fn run_is_busy_while_a_stream_holds_the_lease() { + // Null native handles (both frees are NULL no-ops): the Busy check runs + // under the lock before any native call. + let mut session = DiarizeSession { ptr: std::ptr::null_mut(), model: Arc::new(ModelInner { ptr: std::ptr::null_mut(), - compute_lock: Mutex::new(stream_holds_lease), + compute_lock: Mutex::new(true), }), cancel: None, - } - } - - #[test] - fn run_is_busy_while_a_stream_holds_the_lease() { - let err = null_session(true) + }; + let err = session .run(&[0.0; 160], &DiarizeOptions::default()) .unwrap_err(); let Error::Busy(msg) = err else { @@ -184,10 +174,5 @@ mod tests { msg, "a stream is active on this model; finish or drop it before diarize run()" ); - // With the lease free the same call reaches native (NULL session). - let err = null_session(false) - .run(&[0.0; 160], &DiarizeOptions::default()) - .unwrap_err(); - assert!(matches!(err, Error::InvalidArgument(_)), "got {err:?}"); } } diff --git a/bindings/rust/transcribe-cpp/src/error.rs b/bindings/rust/transcribe-cpp/src/error.rs index 10f0a838b..eaa274add 100644 --- a/bindings/rust/transcribe-cpp/src/error.rs +++ b/bindings/rust/transcribe-cpp/src/error.rs @@ -75,12 +75,7 @@ pub enum Error { /// The (incomplete) transcript produced before the loop, one copy kept. partial: Option>, }, - /// `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` — the model does not serve the role - /// the call needs. Sessions are ASR sessions, so [`Model::session`] / - /// [`Model::session_with`] on a model without the ASR role fail with it. - /// - /// [`Model::session`]: crate::Model::session - /// [`Model::session_with`]: crate::Model::session_with + /// `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` — see [`Model::roles`](crate::Model::roles). #[error("unsupported role: {0}")] UnsupportedRole(String), /// The loaded library's base version disagrees with the headers this crate @@ -215,11 +210,11 @@ mod tests { use std::mem::discriminant; use sys::transcribe_status as S; - /// Every non-OK status the linked library knows, found by walking codes - /// upward until `transcribe_status_string` falls back to its "unknown" - /// text. A status appended to the header without a Rust mapping shows up - /// here and fails the test below. - fn known_error_statuses() -> Vec { + #[test] + fn every_status_maps_to_a_typed_variant() { + // Every non-OK status the linked library knows, found by walking codes + // upward until `transcribe_status_string` falls back to its "unknown" + // text, so a status appended to the header without a mapping fails here. let unknown = status_string(-1); let codes: Vec = (1..256u32) .take_while(|&c| status_string(c as i32) != unknown) @@ -230,12 +225,7 @@ mod tests { "status walk stopped early at {} codes", codes.len() ); - codes - } - - #[test] - fn every_status_maps_to_a_typed_variant() { - for status in known_error_statuses() { + for status in codes { let code = status.0; let err = error_for_status(status, "ctx"); assert!( @@ -254,23 +244,7 @@ mod tests { "status {code} -> {err:?} -> raw {raw} -> {back:?}" ); } - } - - #[test] - fn unsupported_role_is_its_own_variant() { - let err = error_for_status(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "session_init"); - let Error::UnsupportedRole(msg) = &err else { - panic!("expected UnsupportedRole, got {err:?}"); - }; - let c_text = status_string(20); - assert_eq!(*msg, format!("session_init: {c_text} (status 20)")); - assert_eq!(err.to_string(), format!("unsupported role: {msg}")); - assert_eq!(err.raw_status(), 20); - assert!(err.partial().is_none()); - let checked = check(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "open"); - assert!( - matches!(checked, Err(Error::UnsupportedRole(_))), - "got {checked:?}" - ); + let role = error_for_status(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "ctx"); + assert!(matches!(role, Error::UnsupportedRole(_)), "{role:?}"); } } diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index 968d8985b..615f28506 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -82,17 +82,9 @@ pub(crate) struct ModelInner { } impl ModelInner { - /// Run `f` under the model's compute lock. Every call that drives native - /// compute, on any session type of this model (run, batch, diarize run, - /// stream begin/feed/finalize/reset/drop), goes through here. - /// - /// Under the lock, in order: if `refuse_if_streaming` is `Some(msg)` and a - /// stream on this model holds the lease, return `Error::Busy(msg)` without - /// calling `f`; otherwise call `f` with the lease flag (so a stream - /// begin/end can take or release the lease atomically with its native - /// call). Pass `None` from the stream that holds the lease; the call then - /// never fails. The lock is released when this returns, so callers copy - /// results out of their session's own result storage afterwards. + /// Run `f` under the model's compute lock; every native compute call goes + /// through here. With `Some(msg)`, returns `Error::Busy(msg)` instead if a + /// stream holds the lease. `f` gets the lease flag to take or release it. pub(crate) fn with_compute( &self, refuse_if_streaming: Option<&str>, diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index a44e5d2ab..bded8aa2a 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -172,12 +172,9 @@ impl Session { let (params, _lang, _target, _family, _prompting) = build_run_params(options)?; let n = clamp_len(pcm.len())?; - // The compute path is serialized per model; hold the lock for the native - // call, then materialize this session's own result storage. Refuse a run - // that would overlap an active stream on another session (the C contract). - let status = self.with_compute( + let status = self.model.with_compute( Some("a stream is active on this model; finish or drop it before run()"), - |ptr, _| unsafe { sys::transcribe_run(ptr, pcm.as_ptr(), n, ¶ms) }, + |_| unsafe { sys::transcribe_run(self.ptr, pcm.as_ptr(), n, ¶ms) }, )?; match status { @@ -212,10 +209,10 @@ impl Session { .collect::>()?; let n = clamp_len(pcms.len())?; - let status = self.with_compute( + let status = self.model.with_compute( Some("a stream is active on this model; finish or drop it before run_batch()"), - |ptr, _| unsafe { - sys::transcribe_run_batch(ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) + |_| unsafe { + sys::transcribe_run_batch(self.ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) }, )?; @@ -280,11 +277,11 @@ impl Session { // Claim the model's compute lease for the whole stream lifetime: a // stream spans begin..drop, so per-call locking alone would let a // second stream (or a run) on another session race it. - self.with_compute( + self.model.with_compute( Some("a stream is already active on this model"), - |ptr, lease| -> Result<()> { + |lease| -> Result<()> { check( - unsafe { sys::transcribe_stream_begin(ptr, &run_params, &stream_params) }, + unsafe { sys::transcribe_stream_begin(self.ptr, &run_params, &stream_params) }, "stream begin", )?; *lease = true; // released at finalize/reset, or by Stream::drop @@ -297,17 +294,6 @@ impl Session { }) } - /// [`ModelInner::with_compute`], handing `f` this session's native handle. - pub(crate) fn with_compute( - &mut self, - refuse_if_streaming: Option<&str>, - f: impl FnOnce(*mut sys::transcribe_session, &mut bool) -> R, - ) -> Result { - let ptr = self.ptr; - self.model - .with_compute(refuse_if_streaming, |lease| f(ptr, lease)) - } - // --- result materialization (top-level / single accessors) --------------- fn materialize_run(&self) -> Transcript { @@ -531,20 +517,6 @@ fn attach_partial(err: Error, partial: Box) -> Error { /// [`Stream::text`], and end input with [`Stream::finalize`]. Dropping the /// `Stream` (without finalizing) abandons it and returns the session to idle. /// `Send` but not `Sync`, like the session it borrows. -/// -/// The borrow means a session can't be freed while its stream (or any other -/// call on it) is still in use, so no native free ever races a compute: -/// -/// ```compile_fail,E0505 -/// use transcribe_cpp::{Model, RunOptions, StreamOptions}; -/// let model = Model::load("model.gguf").unwrap(); -/// let mut session = model.session().unwrap(); -/// let mut stream = session -/// .stream(&RunOptions::default(), &StreamOptions::default()) -/// .unwrap(); -/// drop(session); // error[E0505]: cannot move out of `session` while borrowed -/// stream.feed(&[0.0; 160]).unwrap(); -/// ``` pub struct Stream<'a> { session: &'a mut Session, // True while this stream holds the model's compute lease. Set at begin, @@ -565,18 +537,7 @@ impl std::fmt::Debug for Stream<'_> { impl Drop for Stream<'_> { fn drop(&mut self) { - // Abandon any unfinalized stream (reset is idempotent and safe from any - // state) and release the lease IF this stream still holds it — under the - // lock. After finalize/reset the lease is already gone (and may now be - // another session's), so only clear it when holds_lease is still true. - // (No busy check, so this cannot fail.) - let held = self.holds_lease; - let _ = self.session.with_compute(None, |ptr, lease| { - unsafe { sys::transcribe_stream_reset(ptr) }; - if held { - *lease = false; - } - }); + self.reset() } } @@ -588,8 +549,8 @@ impl Stream<'_> { unsafe { sys::transcribe_stream_update_init(&mut update) }; // This stream already holds the model's compute lease (it is in // flight); the lock here just serializes the native call. - let status = self.session.with_compute(None, |ptr, _| unsafe { - sys::transcribe_stream_feed(ptr, pcm.as_ptr(), n, &mut update) + let status = self.session.model.with_compute(None, |_| unsafe { + sys::transcribe_stream_feed(self.session.ptr, pcm.as_ptr(), n, &mut update) })?; check(status, "stream feed")?; Ok(StreamUpdate::from_raw(&update)) @@ -600,13 +561,12 @@ impl Stream<'_> { pub fn finalize(&mut self) -> Result { let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; - let held = self.holds_lease; - let status = self.session.with_compute(None, |ptr, lease| { - let st = unsafe { sys::transcribe_stream_finalize(ptr, &mut update) }; + let status = self.session.model.with_compute(None, |lease| { + let st = unsafe { sys::transcribe_stream_finalize(self.session.ptr, &mut update) }; // Finalize ends the active stream (Finished, or Failed on error); // either way it is no longer active, so free the model's compute // lease now rather than holding other sessions off until drop. - if held { + if self.holds_lease { *lease = false; } st @@ -621,11 +581,10 @@ impl Stream<'_> { /// active), so other sessions of the model can proceed without waiting for /// this `Stream` to drop. pub fn reset(&mut self) { - // No busy check, so this cannot fail. - let held = self.holds_lease; - let _ = self.session.with_compute(None, |ptr, lease| { - unsafe { sys::transcribe_stream_reset(ptr) }; - if held { + // Releases the lease only if this stream holds it (it may be another session's). + let _ = self.session.model.with_compute(None, |lease| { + unsafe { sys::transcribe_stream_reset(self.session.ptr) }; + if self.holds_lease { *lease = false; } }); diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index 867c0a67f..f9088bb76 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -392,9 +392,4 @@ impl Roles { }; self.0 & bit.0 != 0 } - - /// The raw `transcribe_role` bitmask. - pub fn bits(self) -> u32 { - self.0 - } } diff --git a/bindings/rust/transcribe-cpp/tests/cancel.rs b/bindings/rust/transcribe-cpp/tests/cancel.rs index 9b842f09a..c4645a8ed 100644 --- a/bindings/rust/transcribe-cpp/tests/cancel.rs +++ b/bindings/rust/transcribe-cpp/tests/cancel.rs @@ -1,6 +1,6 @@ -//! Cancellation: a clean baseline run, and a cross-thread timer cancel of an -//! in-flight run on long (tiled) audio. Ports the Python C5 case. Skips the -//! abort assertion if the machine wins the race (run finishes before cancel). +//! Cancellation: a clean baseline run, a cross-thread timer cancel of an +//! in-flight run on long (tiled) audio (ports the Python C5 case; skips the +//! abort assertion if the run wins the race), and token retention. mod common; @@ -72,3 +72,36 @@ fn cross_thread_cancel_of_in_flight_run() { Err(other) => panic!("unexpected error: {other:?}"), } } + +#[test] +fn cancel_token_is_retained_until_cleared() { + // The session keeps the installed token's flag alive (the caller may drop + // every handle) and keeps consulting it on later runs until cleared. + let Some((model_path, pcm)) = common::smoke_fixtures("cancel_token_is_retained_until_cleared") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + if !model.supports(Feature::Cancellation) { + eprintln!("skip: model does not support cancellation"); + return; + } + let mut session = model.session().unwrap(); + { + let token = CancelToken::new(); + token.cancel(); + session.set_cancel_token(&token); + } // every caller-side handle to the flag is gone + + for attempt in 0..2 { + match session.run(&pcm, &RunOptions::default()) { + Err(Error::Aborted { .. }) => assert!(session.was_aborted()), + other => panic!("attempt {attempt}: pre-cancelled token must abort, got {other:?}"), + } + } + + session.clear_cancel_token(); + let result = session.run(&pcm, &RunOptions::default()).unwrap(); + assert!(result.text.to_lowercase().contains("country")); + assert!(!session.was_aborted()); +} diff --git a/bindings/rust/transcribe-cpp/tests/compute_rules.rs b/bindings/rust/transcribe-cpp/tests/compute_rules.rs deleted file mode 100644 index 100551ca5..000000000 --- a/bindings/rust/transcribe-cpp/tests/compute_rules.rs +++ /dev/null @@ -1,298 +0,0 @@ -//! Characterization of the rules every native compute call follows: the -//! per-model compute lock and stream lease, the order of the checks in front -//! of the native call (and therefore which error a racing caller sees), lease -//! ownership across sessions, cancel-token install/clear, and serialization of -//! run/run_batch across threads. These pin today's behavior so the compute -//! sites can be moved onto one helper without changing it. Model-gated: skip -//! cleanly when the canary GGUFs / jfk.wav are absent. - -mod common; - -use std::path::PathBuf; -use std::sync::Arc; -use std::thread; - -use transcribe_cpp::{ - CancelToken, Error, Feature, Model, RunOptions, StreamExtension, StreamOptions, StreamState, -}; - -const RUN_BUSY: &str = "a stream is active on this model; finish or drop it before run()"; -const BATCH_BUSY: &str = "a stream is active on this model; finish or drop it before run_batch()"; -const STREAM_BUSY: &str = "a stream is already active on this model"; - -fn streaming_fixtures(test: &str) -> Option<(PathBuf, Vec)> { - match (common::smoke_streaming_model(), common::smoke_audio()) { - (Some(m), Some(a)) => Some((m, a)), - _ => { - eprintln!("skip {test}: streaming model/audio absent"); - None - } - } -} - -fn expect_busy(r: transcribe_cpp::Result, want: &str, what: &str) { - match r { - Err(Error::Busy(msg)) => assert_eq!(msg, want, "{what}: Busy message"), - other => panic!("{what}: expected Error::Busy({want:?}), got {other:?}"), - } -} - -fn wrong_family_stream() -> StreamOptions { - StreamOptions { - family: Some(StreamExtension::ParakeetStream(Default::default())), - ..Default::default() - } -} - -#[test] -fn busy_errors_name_the_refused_call() { - // Each refusing site carries its own message: run and run_batch name the - // call, a second stream says "already active". - let Some((model_path, pcm)) = streaming_fixtures("busy_errors_name_the_refused_call") else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - - let mut stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); - - expect_busy(s2.run(&pcm, &RunOptions::default()), RUN_BUSY, "run"); - expect_busy( - s2.run_batch(&[&pcm], &RunOptions::default()), - BATCH_BUSY, - "run_batch", - ); - expect_busy( - s2.stream(&RunOptions::default(), &StreamOptions::default()), - STREAM_BUSY, - "stream", - ); - // The refusals did not disturb the live stream. - stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); - assert_eq!(stream1.state(), StreamState::Active); -} - -#[test] -fn option_validation_precedes_busy_check() { - // Run options are marshalled before the compute lock is taken, so a bad - // option is reported as itself even while another session streams. - let Some((model_path, pcm)) = streaming_fixtures("option_validation_precedes_busy_check") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - let _stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - - let bad = RunOptions { - language: Some("e\0n".into()), - ..Default::default() - }; - assert!( - matches!(s2.run(&pcm, &bad), Err(Error::Nul(_))), - "run: NUL option must win over Busy" - ); - assert!( - matches!(s2.run_batch(&[&pcm], &bad), Err(Error::Nul(_))), - "run_batch: NUL option must win over Busy" - ); - assert!( - matches!( - s2.stream(&bad, &StreamOptions::default()), - Err(Error::Nul(_)) - ), - "stream: NUL option must win over Busy" - ); -} - -#[test] -fn busy_check_precedes_native_stream_validation() { - // A begin that the native layer would reject (wrong-family extension) is - // refused as Busy first when another session's stream holds the lease. - let Some((model_path, pcm)) = - streaming_fixtures("busy_check_precedes_native_stream_validation") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - - let mut stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); - expect_busy( - s2.stream(&RunOptions::default(), &wrong_family_stream()), - STREAM_BUSY, - "wrong-family stream while busy", - ); - drop(stream1); - - // With the lease free, the same begin reaches native validation. - let err = s2 - .stream(&RunOptions::default(), &wrong_family_stream()) - .unwrap_err(); - assert!( - matches!(err, Error::InvalidArgument(_)), - "wrong-family stream when idle: {err:?}" - ); -} - -#[test] -fn failed_stream_begin_does_not_take_the_lease() { - let Some((model_path, _pcm)) = - streaming_fixtures("failed_stream_begin_does_not_take_the_lease") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - - assert!(s1 - .stream(&RunOptions::default(), &wrong_family_stream()) - .is_err()); - let stream2 = s2.stream(&RunOptions::default(), &StreamOptions::default()); - assert!( - stream2.is_ok(), - "a failed begin must leave the lease free: {stream2:?}" - ); -} - -#[test] -fn ended_stream_never_releases_another_sessions_lease() { - // After finalize() the lease belongs to whoever takes it next; a later - // reset() or drop of the ended stream must not free that other lease. - let Some((model_path, pcm)) = - streaming_fixtures("ended_stream_never_releases_another_sessions_lease") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - let mut s3 = model.session().unwrap(); - let chunk = &pcm[..pcm.len().min(1600)]; - - // finalize -> other session takes the lease -> ended stream reset()s. - { - let mut stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - stream1.feed(chunk).unwrap(); - stream1.finalize().unwrap(); - let stream2 = s2 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - stream1.reset(); - expect_busy( - s3.stream(&RunOptions::default(), &StreamOptions::default()), - STREAM_BUSY, - "after ended stream reset", - ); - drop(stream2); - } - - // finalize -> other session takes the lease -> ended stream is dropped. - let mut stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - stream1.feed(chunk).unwrap(); - stream1.finalize().unwrap(); - let stream2 = s2 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - drop(stream1); - expect_busy( - s3.stream(&RunOptions::default(), &StreamOptions::default()), - STREAM_BUSY, - "after ended stream drop", - ); - expect_busy(s3.run(&pcm, &RunOptions::default()), RUN_BUSY, "run"); - - // Releasing the real holder frees the model. - drop(stream2); - let stream3 = s3 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - assert_eq!(stream3.state(), StreamState::Active); -} - -#[test] -fn run_and_run_batch_across_threads_serialize() { - // One-shot runs and batches on different sessions of one model queue on - // the compute lock (never Busy), and each gets its own results back. - let Some((model_path, pcm)) = - common::smoke_fixtures("run_and_run_batch_across_threads_serialize") - else { - return; - }; - let model = Arc::new(Model::load(&model_path).unwrap()); - let pcm = Arc::new(pcm); - let handles: Vec<_> = (0..4) - .map(|i| { - let model = Arc::clone(&model); - let pcm = Arc::clone(&pcm); - thread::spawn(move || { - let mut s = model.session().unwrap(); - if i % 2 == 0 { - vec![s.run(&pcm, &RunOptions::default()).unwrap().text] - } else { - s.run_batch(&[&pcm, &pcm], &RunOptions::default()) - .unwrap() - .into_iter() - .map(|r| r.unwrap().text) - .collect() - } - }) - }) - .collect(); - for (i, h) in handles.into_iter().enumerate() { - let texts = h.join().unwrap(); - assert_eq!(texts.len(), if i % 2 == 0 { 1 } else { 2 }); - for t in texts { - assert!(t.to_lowercase().contains("country"), "thread {i}: {t}"); - } - } -} - -#[test] -fn cancel_token_is_retained_until_cleared() { - // The session keeps the installed token's flag alive (the caller may drop - // every handle) and keeps consulting it on later runs until cleared. - let Some((model_path, pcm)) = common::smoke_fixtures("cancel_token_is_retained_until_cleared") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - if !model.supports(Feature::Cancellation) { - eprintln!("skip: model does not support cancellation"); - return; - } - let mut session = model.session().unwrap(); - { - let token = CancelToken::new(); - token.cancel(); - session.set_cancel_token(&token); - } // every caller-side handle to the flag is gone - - for attempt in 0..2 { - match session.run(&pcm, &RunOptions::default()) { - Err(Error::Aborted { .. }) => assert!(session.was_aborted()), - other => panic!("attempt {attempt}: pre-cancelled token must abort, got {other:?}"), - } - } - - session.clear_cancel_token(); - let result = session.run(&pcm, &RunOptions::default()).unwrap(); - assert!(result.text.to_lowercase().contains("country")); - assert!(!session.was_aborted()); -} diff --git a/bindings/rust/transcribe-cpp/tests/diarize.rs b/bindings/rust/transcribe-cpp/tests/diarize.rs index 672b5f202..dfc5b4f41 100644 --- a/bindings/rust/transcribe-cpp/tests/diarize.rs +++ b/bindings/rust/transcribe-cpp/tests/diarize.rs @@ -1,31 +1,14 @@ -//! DIARIZE role: roles, diarize_info, DiarizeSession runs, and refusal on an -//! ASR-only model. Model-gated: Sortformer on TRANSCRIBE_SMOKE_SORTFORMER_MODEL -//! (+ samples/sortformer-2spk-mix.wav), whisper on TRANSCRIBE_SMOKE_MODEL. -//! The Busy path is covered model-free in `src/diarize.rs` (no model serves -//! both DIARIZE and streaming today). +//! DIARIZE role on Sortformer (TRANSCRIBE_SMOKE_SORTFORMER_MODEL) and whisper +//! (TRANSCRIBE_SMOKE_MODEL); the Busy path is covered in `src/diarize.rs`. mod common; use transcribe_cpp::sys::TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE as SFDR; use transcribe_cpp::{ Backend, CancelToken, DiarizeExtension, DiarizeOptions, Error, ExtSlot, Model, ModelOptions, - Role, SortformerDiarizeOptions, SortformerPreset, SpeakerSegment, + Role, SortformerDiarizeOptions, SortformerPreset, }; -fn diarize_opts(preset: Option) -> DiarizeOptions { - DiarizeOptions { - family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { - preset, - })), - } -} - -fn rows(segs: &[SpeakerSegment]) -> Vec<(i64, i64, i32)> { - segs.iter() - .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id)) - .collect() -} - #[test] fn sortformer_is_diarize_only() { let Some((model_path, _)) = common::smoke_sortformer_fixtures("sortformer_is_diarize_only") @@ -35,7 +18,6 @@ fn sortformer_is_diarize_only() { let model = Model::load(&model_path).unwrap(); let roles = model.roles(); assert!(!roles.contains(Role::Asr) && roles.contains(Role::Diarize)); - assert_eq!(roles.bits(), 0b10, "{roles:?}"); let info = model.diarize_info().unwrap(); assert_eq!((info.sample_rate, info.max_speakers), (16000, 4)); assert!(model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); @@ -80,47 +62,42 @@ fn diarize_run_matches_cpu_golden_segments() { (Some(SortformerPreset::Default), &golden_default), (Some(SortformerPreset::LowLatency), &golden_low_latency), ] { - let turns = diarize.run(&pcm, &diarize_opts(preset)).unwrap(); - assert_eq!(rows(&turns), golden.to_vec(), "{preset:?}"); + let opts = DiarizeOptions { + family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { + preset, + })), + }; + let turns = diarize.run(&pcm, &opts).unwrap(); + let rows: Vec<_> = turns + .iter() + .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id)) + .collect(); + assert_eq!(rows, golden.to_vec(), "{preset:?}"); assert!(diarize.timings().encode_ms > 0.0); } } #[test] -fn diarize_session_keeps_model_alive_and_rejects_bad_input() { - let Some((model_path, pcm)) = common::smoke_sortformer_fixtures( - "diarize_session_keeps_model_alive_and_rejects_bad_input", - ) else { +fn diarize_session_rejects_bad_input_and_cancels() { + let Some((model_path, pcm)) = + common::smoke_sortformer_fixtures("diarize_session_rejects_bad_input_and_cancels") + else { return; }; + // The session keeps its model alive after the last Model handle drops. let mut diarize = Model::load(&model_path).unwrap().diarize_session().unwrap(); let opts = DiarizeOptions::default(); for bad in [&[][..], &[f32::NAN; 16][..]] { let err = diarize.run(bad, &opts).unwrap_err(); assert!(matches!(err, Error::InvalidArgument(_)), "{err:?}"); } - assert!(!diarize.run(&pcm, &opts).unwrap().is_empty()); -} - -#[test] -fn cancelled_diarize_run_is_aborted() { - let Some((model_path, pcm)) = - common::smoke_sortformer_fixtures("cancelled_diarize_run_is_aborted") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut diarize = model.diarize_session().unwrap(); let token = CancelToken::new(); diarize.set_cancel_token(&token); token.cancel(); - let err = diarize.run(&pcm, &DiarizeOptions::default()).unwrap_err(); + let err = diarize.run(&pcm, &opts).unwrap_err(); assert!(matches!(err, Error::Aborted { .. }), "{err:?}"); diarize.clear_cancel_token(); - assert!(!diarize - .run(&pcm, &DiarizeOptions::default()) - .unwrap() - .is_empty()); + assert!(!diarize.run(&pcm, &opts).unwrap().is_empty()); } #[test] diff --git a/bindings/rust/transcribe-cpp/tests/streaming.rs b/bindings/rust/transcribe-cpp/tests/streaming.rs index 85bf4fbb6..53494a6c1 100644 --- a/bindings/rust/transcribe-cpp/tests/streaming.rs +++ b/bindings/rust/transcribe-cpp/tests/streaming.rs @@ -3,9 +3,13 @@ mod common; -use transcribe_cpp::{Model, RunOptions, StreamExtension, StreamOptions, StreamState}; +use transcribe_cpp::{Error, Model, RunOptions, StreamExtension, StreamOptions, StreamState}; use transcribe_cpp::{MoonshineStreamingOptions, Stream}; +const RUN_BUSY: &str = "a stream is active on this model; finish or drop it before run()"; +const BATCH_BUSY: &str = "a stream is active on this model; finish or drop it before run_batch()"; +const STREAM_BUSY: &str = "a stream is already active on this model"; + /// Partial-decode cadence for the mechanics tests that only assert the /// post-finalize text. Aligned to the model, not an arbitrary number: the /// streaming encoder emits one frame per 20 ms (frame_len 80 × 4 subsampling), @@ -24,6 +28,20 @@ fn feed_in_chunks(stream: &mut Stream<'_>, pcm: &[f32], chunk: usize) { } } +fn expect_busy(r: transcribe_cpp::Result, want: &str) { + match r { + Err(Error::Busy(msg)) => assert_eq!(msg, want), + other => panic!("expected Error::Busy({want:?}), got {other:?}"), + } +} + +fn wrong_family_stream() -> StreamOptions { + StreamOptions { + family: Some(StreamExtension::ParakeetStream(Default::default())), + ..Default::default() + } +} + #[test] fn streams_jfk_committed_text() { let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) @@ -161,22 +179,32 @@ fn concurrent_compute_on_one_model_is_refused() { let model = Model::load(&model_path).unwrap(); let mut s1 = model.session().unwrap(); let mut s2 = model.session().unwrap(); + let chunk = &pcm[..pcm.len().min(1600)]; + // A failed begin does not take the lease. + assert!(s1 + .stream(&RunOptions::default(), &wrong_family_stream()) + .is_err()); let mut stream1 = s1 .stream(&RunOptions::default(), &StreamOptions::default()) .unwrap(); - stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + stream1.feed(chunk).unwrap(); - // Second stream on the same model while the first is live -> Busy. - match s2.stream(&RunOptions::default(), &StreamOptions::default()) { - Err(transcribe_cpp::Error::Busy(_)) => {} - other => panic!("expected Error::Busy for a second stream, got {other:?}"), - } - // An offline run on the same model while a stream is live -> Busy too. - match s2.run(&pcm, &RunOptions::default()) { - Err(transcribe_cpp::Error::Busy(_)) => {} - other => panic!("expected Error::Busy for a run mid-stream, got {other:?}"), - } + // Each refusal names the refused call; the busy check precedes native + // validation of a (wrong-family) stream begin. + expect_busy(s2.run(&pcm, &RunOptions::default()), RUN_BUSY); + expect_busy(s2.run_batch(&[&pcm], &RunOptions::default()), BATCH_BUSY); + expect_busy( + s2.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + expect_busy( + s2.stream(&RunOptions::default(), &wrong_family_stream()), + STREAM_BUSY, + ); + // The refusals did not disturb the live stream. + stream1.feed(chunk).unwrap(); + assert_eq!(stream1.state(), StreamState::Active); // Dropping the first stream releases the model's compute lease; now s2 can // begin its own stream. @@ -252,10 +280,82 @@ fn stream_family_extension_accepted_or_rejected() { drop(ok); // Wrong-family extension is rejected. - let wrong = StreamOptions { - family: Some(StreamExtension::ParakeetStream(Default::default())), + let err = session.stream(&RunOptions::default(), &wrong_family_stream()); + assert!(matches!(err, Err(Error::InvalidArgument(_))), "{err:?}"); +} + +#[test] +fn option_validation_precedes_busy_check() { + // Run options are marshalled before the compute lock is taken, so a bad + // option is reported as itself even while another session streams. + let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + let _stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + + let bad = RunOptions { + language: Some("e\0n".into()), ..Default::default() }; - let err = session.stream(&RunOptions::default(), &wrong); - assert!(err.is_err(), "wrong-family ext should be rejected"); + assert!( + matches!(s2.run(&pcm, &bad), Err(Error::Nul(_))), + "run: NUL option must win over Busy" + ); + assert!( + matches!(s2.run_batch(&[&pcm], &bad), Err(Error::Nul(_))), + "run_batch: NUL option must win over Busy" + ); + assert!( + matches!( + s2.stream(&bad, &StreamOptions::default()), + Err(Error::Nul(_)) + ), + "stream: NUL option must win over Busy" + ); +} + +#[test] +fn ended_stream_never_releases_another_sessions_lease() { + // After finalize() the lease belongs to whoever takes it next; a later + // reset() or drop of the ended stream must not free that other lease. + let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + let mut s3 = model.session().unwrap(); + + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + stream1.finalize().unwrap(); + let stream2 = s2 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.reset(); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + drop(stream1); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + + // Releasing the real holder frees the model. + drop(stream2); + let stream3 = s3 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + assert_eq!(stream3.state(), StreamState::Active); } diff --git a/bindings/rust/transcribe-cpp/tests/transcribe.rs b/bindings/rust/transcribe-cpp/tests/transcribe.rs index 32ce2f6be..f81f4fa5d 100644 --- a/bindings/rust/transcribe-cpp/tests/transcribe.rs +++ b/bindings/rust/transcribe-cpp/tests/transcribe.rs @@ -266,8 +266,8 @@ fn close_ordering_drop_model_before_session() { #[test] fn shared_model_across_threads_serializes() { // Model is Send+Sync; the per-model mutex serializes the compute path. Two - // threads each run on their own session of one shared model — they queue - // rather than race, and both succeed. + // threads each run (one as a batch) on their own session of one shared + // model — they queue rather than race (never Busy), and both succeed. let Some((model_path, pcm)) = common::smoke_fixtures("shared_model_across_threads_serializes") else { return; @@ -275,12 +275,17 @@ fn shared_model_across_threads_serializes() { let model = Arc::new(Model::load(&model_path).unwrap()); let pcm = Arc::new(pcm); let handles: Vec<_> = (0..2) - .map(|_| { + .map(|i| { let model = Arc::clone(&model); let pcm = Arc::clone(&pcm); thread::spawn(move || { let mut s = model.session().unwrap(); - s.run(&pcm, &RunOptions::default()).unwrap().text + let opts = RunOptions::default(); + if i == 0 { + s.run(&pcm, &opts).unwrap().text + } else { + s.run_batch(&[&pcm], &opts).unwrap().remove(0).unwrap().text + } }) }) .collect(); From a0172eb2a45e29b9fbde8ecfc5822a10eacfe232 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:47:37 +0800 Subject: [PATCH 39/51] tests, cli, docs: trim from review - transcribe-cli: drop --role (no model serves two roles; a DIARIZE-only model is routed by its role mask, which cli_diarize_smoke now exercises); json_escape / write_output_file move into asr.cpp, their only user; unused includes and stale comments removed. - tests: the five fake-arch units register through one CMake loop; redundant cases dropped (silence, tautological role-bit asserts, dispatcher refusal checks repeated in sortformer_diarize_unit); plan-ID tags and history notes removed from comments. - docs: stale Sortformer ASR-path references fixed; duplicated role / binding notes in migrating-to-0.4.md, roles.md and diarize.h shortened to pointers. --- docs/bindings.md | 2 +- docs/environment-variables.md | 2 +- docs/migrating-to-0.4.md | 15 ++-- .../diar_streaming_sortformer_4spk-v2.1.md | 4 +- docs/porting/families/sortformer.md | 7 +- docs/roles.md | 5 +- examples/cli/asr.cpp | 53 +++++++++-- examples/cli/cli.h | 26 ++---- examples/cli/main.cpp | 73 --------------- include/transcribe/diarize.h | 8 +- tests/CMakeLists.txt | 90 ++----------------- tests/api_smoke.c | 3 +- tests/cli_diarize_smoke.cmake | 8 +- tests/nonfinite_input_unit.cpp | 30 +------ tests/run_on_threads_unit.cpp | 2 +- tests/sortformer_diarize_unit.cpp | 17 +--- tests/stream_hook_throw_unit.cpp | 12 +-- 17 files changed, 94 insertions(+), 263 deletions(-) diff --git a/docs/bindings.md b/docs/bindings.md index b801c55ee..e92fa9019 100644 --- a/docs/bindings.md +++ b/docs/bindings.md @@ -110,7 +110,7 @@ from `model.diarize_session()` (Python, Rust), `model.diarizeSession()` (Swift) or `model.createDiarizeSession()` (TypeScript), plus `diarize_info` / `diarizeInfo` (sample rate, max speakers). `run(pcm, …)` returns copied-out speaker-turn rows (the same row type as above) and takes the family's -diarize extension (Sortformer: the preset on the `diarize_run` slot). +diarize extension (Sortformer: the preset on the DIARIZE_RUN slot). A diarize run follows the same execution rules as an ASR run: it holds the model-wide compute lock, waits behind other compute on the model, raises diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 74a1691b0..1e92e7d5a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -88,7 +88,7 @@ its var is unset. Convention: `TRANSCRIBE__GGUF`. | `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_diarize_unit`, `cli_diarize_smoke` | | `TRANSCRIBE_COHERE_GGUF` | `cohere_real_smoke`, `cohere_e2e_smoke` | | `TRANSCRIBE_GRANITE5_CTC_GGUF` | `granite5_ctc_real_smoke`, `granite5_ctc_e2e_smoke` | -| `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity` | +| `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity`, `sortformer_diarize_unit` (concurrency leg) | | `TRANSCRIBE_QWEN3_ASR_GGUF` (+ `_0_6B_GGUF` / `_1_7B_GGUF`) | qwen3_asr smokes / parity | | `TRANSCRIBE_MOONSHINE_STREAMING_TINY_GGUF` | moonshine_streaming smokes, `stream_offline_interleave_smoke` | | `TRANSCRIBE_VOXTRAL_REALTIME_GGUF` | `voxtral_realtime_real_smoke`, `stream_offline_interleave_smoke` | diff --git a/docs/migrating-to-0.4.md b/docs/migrating-to-0.4.md index c74dc0e09..a17a5ed94 100644 --- a/docs/migrating-to-0.4.md +++ b/docs/migrating-to-0.4.md @@ -22,9 +22,8 @@ them, and `transcribe_session_init` / `transcribe_open` return | `transcribe-cli --batch ... --batch-jsonl` "speakers" for Sortformer | `transcribe-cli -m sortformer.gguf audio.wav` prints the segments | Segments are byte-identical to 0.3 for the same audio, preset and backend. -ASR models that attribute speakers inside a transcript (granite, moss, -multitalker parakeet) are unchanged: they keep `run_params.diarize`, -`TRANSCRIBE_FEATURE_DIARIZATION` and the ASR speaker accessors. +Speaker-attributing ASR models (granite, moss, multitalker parakeet) are +unchanged; see `docs/roles.md`. ## Capabilities are ASR-only @@ -47,12 +46,10 @@ from the role's own query (`transcribe_diarize_get_info`). ## Language bindings -- New error type for status 20: `UnsupportedRole` (Python, TypeScript, - Rust), `.unsupportedRole` (Swift). -- Each binding gains `Model.roles` and a `DiarizeSession` (see - `docs/bindings.md`). The Sortformer run extension for the ASR path - (`SortformerStreamOptions` and its equivalents) is replaced by the diarize - one. +- New `Model.roles`, `DiarizeSession` and an `UnsupportedRole` error for + status 20 (`.unsupportedRole` in Swift); see `docs/bindings.md`. The + Sortformer ASR-path extension (`SortformerStreamOptions` and equivalents) + is removed. - **Rust:** `Model::capabilities()` returns `Result` (`Err(Error::UnsupportedRole)` on a model without ASR), and `ExtSlot` gains `DiarizeRun`. diff --git a/docs/models/diar_streaming_sortformer_4spk-v2.1.md b/docs/models/diar_streaming_sortformer_4spk-v2.1.md index b167e53e8..95a2a02e2 100644 --- a/docs/models/diar_streaming_sortformer_4spk-v2.1.md +++ b/docs/models/diar_streaming_sortformer_4spk-v2.1.md @@ -151,8 +151,8 @@ full 39-minute AMI meeting (87 compression calls). - **`LOW_LATENCY` is compute-heavy on CPU** (~1.2x realtime on Apple M4; the 0.5 s chunks rebuild the compute graph often). Use Metal or a higher-latency preset for offline files. -- **No batch fast path** (`run_batch`); multi-file CLI batches run - serially. +- **No batch mode**: run one file per call (`transcribe-cli --batch` is + ASR-only). ## Reproduction diff --git a/docs/porting/families/sortformer.md b/docs/porting/families/sortformer.md index 8ecd93a91..09032cab1 100644 --- a/docs/porting/families/sortformer.md +++ b/docs/porting/families/sortformer.md @@ -13,9 +13,9 @@ Shipped matrix: **F32 + F16 + Q8_0** (k tiers withdrawn; see "Quant policy `handy-computer/diar_streaming_sortformer_4spk-v2.1-gguf`. Batch posture: **ACCEPTED GAP — no `run_batch()`** (user-approved Stage 4 -deferral; single-session `transcribe_run` is the shipped path; revisit with +deferral; single-session `transcribe_diarize_run` is the shipped path; revisit with multitalker interop). Streaming posture: **natively streaming — PASS**; the -chunk/lookahead contract is the preset menu in "Public API (run extension)" +chunk/lookahead contract is the preset menu in "Public API (diarize extension)" (default = GGUF-shipped checkpoint cfg; very_high_latency ~30.4 s lookahead ... low_latency ~1.04 s), exposed via `transcribe_sortformer_diarize_ext`. Push-audio `transcribe_stream_*` entry point is future work (STREAM-slot @@ -57,8 +57,7 @@ verbatim). A diarizer meeting neither clause (e.g. a pyannote segmentation+clustering port: new architecture, new dependency surface, no in-repo ASR consumer) is out of scope. Standalone diarization output is exposed because it falls out of the multitalker dependency for free, via -the pre-existing transcript-independent `transcribe_speaker_segment` ABI — -no diarizer-specific output surface was added. +the DIARIZE role (`include/transcribe/diarize.h`). ## Public API (diarize extension) diff --git a/docs/roles.md b/docs/roles.md index 4358b61f8..7330de12a 100644 --- a/docs/roles.md +++ b/docs/roles.md @@ -12,14 +12,11 @@ A loaded model serves one or more **roles**, the kinds of work it can do. Every role follows the same rules: -- **One model, any role.** Load with `transcribe_model_load_file` and pick - the backend per model with `transcribe_model_load_params`. Running a - diarizer on CPU and ASR on the GPU is two loads. +- **One load per model**, with its own backend (`transcribe_model_load_params`). - **Sessions.** Each role has its own session type created from a model. A session is used by one thread at a time; the model outlives its sessions; at most one compute is in flight per model across all sessions of all roles. Different models compute in parallel. -- **Composition is by handle**, never by path in params. - **Input.** 16 kHz mono float32 PCM; NaN / Inf is `INVALID_ARG`. - **Results** are copied out and replaced by the next run; malformed input is rejected before the previous result is touched. diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index 74984eb2a..0ef2cb778 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -10,24 +10,64 @@ #include "wav.h" #include -#include -#include #include #include #include #include #include #include -#include #include #include using transcribe_cli::cli_args; -using transcribe_cli::json_escape; -using transcribe_cli::write_output_file; namespace { +// Minimal JSON string escape: covers the characters MUST be escaped by +// the JSON spec (quote, backslash, control chars). Transcribed text is +// short UTF-8 in practice; we don't need unicode escaping. +std::string json_escape(const char * s) { + std::string out; + for (const char * p = s ? s : ""; *p; ++p) { + const unsigned char c = static_cast(*p); + if (c == '"') { + out += "\\\""; + } else if (c == '\\') { + out += "\\\\"; + } else if (c == '\n') { + out += "\\n"; + } else if (c == '\r') { + out += "\\r"; + } else if (c == '\t') { + out += "\\t"; + } else if (c < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + return out; +} + +bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { + if (output == nullptr) { + return true; + } + const char * value = text != nullptr ? text : ""; + *output << value; + if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { + *output << '\n'; + } + output->flush(); + if (!*output) { + std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); + return false; + } + return true; +} + // Shared row formatter for segments_json / batch_segments_json below. std::string segment_row_json(const struct transcribe_segment & seg) { std::string out; @@ -614,7 +654,6 @@ int run_asr_batch(const cli_args & args, std::ofstream * output) { int run_asr_file(const cli_args & args, std::ofstream * output) { bool output_ok = true; - // Single-file mode. std::vector pcm; std::string load_err; if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { @@ -652,7 +691,7 @@ int run_asr_file(const cli_args & args, std::ofstream * output) { } const uint32_t roles = transcribe_model_roles(model); - if (args.role == "diarize" || (args.role.empty() && (roles & TRANSCRIBE_ROLE_ASR) == 0)) { + if ((roles & TRANSCRIBE_ROLE_ASR) == 0) { return transcribe_cli::run_diarize_file(args, model, pcm, duration_s); } diff --git a/examples/cli/cli.h b/examples/cli/cli.h index 87f55d408..b13f84f10 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -1,7 +1,7 @@ // cli.h - shared declarations for the transcribe-cli example. // // main.cpp parses arguments and owns process setup (log sink, output file); -// the per-role drivers (asr.cpp) do the work. Shared helpers live in +// the per-role drivers (asr.cpp, diarize.cpp) do the work. Shared helpers live in // namespace transcribe_cli next to the WAV loader in examples/common. #pragma once @@ -75,45 +75,35 @@ struct cli_args { // ms-aligned chunks instead of one transcribe_run call. Requires // the loaded model to advertise supports_streaming. Set by // --stream-chunk-ms N. - int stream_chunk_ms = 0; + int stream_chunk_ms = 0; // Parakeet streaming: pick a right-context (lookahead) setting // from the model's training menu. -1 = model default (max // accuracy / max latency); 0/1/6/13 select the published // nemotron-speech-streaming-en-0.6b settings. Set by // --stream-att-right N. Ignored when stream_chunk_ms == 0. - int stream_att_right = -1; + int stream_att_right = -1; // Parakeet buffered streaming (parakeet-unified-en-0.6b): override // the (L, C, R) attention context tuple in milliseconds. -1 = use // the model's default (highest-accuracy row of the training menu). // Frame-aligned: the lib rounds each value down to the nearest // post-subsample frame (80ms at 4x subsampling). Ignored when // stream_chunk_ms == 0 or when the model is not buffered-streaming. - int stream_buf_left_ms = -1; - int stream_buf_chunk_ms = -1; - int stream_buf_right_ms = -1; + int stream_buf_left_ms = -1; + int stream_buf_chunk_ms = -1; + int stream_buf_right_ms = -1; // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when // the model is not voxtral_realtime. - int stream_voxtral_delay = -1; + int stream_voxtral_delay = -1; // Speculative-decode draft length passed through to // transcribe_run_params::spec_k_drafts on the offline path. -1 = family // default (each family picks its tuned K). 0 = explicitly off. >0 = // explicit K. Silently ignored by families without // supports_spec_decode. Set by --spec-k-drafts N. - int spec_k_drafts = -1; - // --role asr|diarize: which role to run on a multi-role model. Empty = - // ASR when the model serves it, else diarize. - std::string role; + int spec_k_drafts = -1; }; -// Minimal JSON string escape (main.cpp). -std::string json_escape(const char * s); - -// Append `text` plus a trailing newline to the -o/--output file, when one is -// open. Returns false on a write error (main.cpp). -bool write_output_file(std::ofstream * output, const std::string & path, const char * text); - // ASR drivers (asr.cpp). Each loads the model, runs, prints, and returns the // process exit code. int run_asr_batch(const cli_args & args, std::ofstream * output); diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 5dcea047b..1a44392bb 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -6,17 +6,9 @@ // Run with --help for the full option list. #include "cli.h" -#include "transcribe.h" -#include "transcribe/parakeet.h" -#include "transcribe/voxtral_realtime.h" -#include "transcribe/whisper.h" -#include "wav.h" -#include #include #include -#include -#include #include #include #include @@ -114,8 +106,6 @@ void print_usage(const char * argv0) { " supports_spec_decode. -1 = family default,\n" " 0 = off, > 0 = explicit K. Silently ignored\n" " by families without spec support.\n" - " --role ROLE asr or diarize, for a model serving both\n" - " (default: asr when available)\n" " --list-devices list registered compute devices (with memory)\n" " and exit; ignores all other options\n" " -h, --help show this help\n", @@ -489,16 +479,6 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --spec-k-drafts must be -1 (family default), 0 (off), or > 0\n"); return false; } - } else if (a == "--role") { - const char * v = take_value(a.c_str()); - if (!v) { - return false; - } - out.role = v; - if (out.role != "asr" && out.role != "diarize") { - std::fprintf(stderr, "error: --role must be asr or diarize\n"); - return false; - } } else if (!a.empty() && a[0] == '-') { std::fprintf(stderr, "error: unknown option '%s'\n", a.c_str()); return false; @@ -561,55 +541,6 @@ void log_cb(transcribe_log_level level, const char * msg, void * userdata) { } // namespace -namespace transcribe_cli { - -// Minimal JSON string escape: covers the characters MUST be escaped by -// the JSON spec (quote, backslash, control chars). Transcribed text is -// short UTF-8 in practice; we don't need unicode escaping. -std::string json_escape(const char * s) { - std::string out; - for (const char * p = s ? s : ""; *p; ++p) { - const unsigned char c = static_cast(*p); - if (c == '"') { - out += "\\\""; - } else if (c == '\\') { - out += "\\\\"; - } else if (c == '\n') { - out += "\\n"; - } else if (c == '\r') { - out += "\\r"; - } else if (c == '\t') { - out += "\\t"; - } else if (c < 0x20) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", c); - out += buf; - } else { - out += static_cast(c); - } - } - return out; -} - -bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { - if (output == nullptr) { - return true; - } - const char * value = text != nullptr ? text : ""; - *output << value; - if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { - *output << '\n'; - } - output->flush(); - if (!*output) { - std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); - return false; - } - return true; -} - -} // namespace transcribe_cli - int main(int argc, char ** argv) { cli_args args; if (!parse_args(argc, argv, args)) { @@ -644,10 +575,6 @@ int main(int argc, char ** argv) { output = &output_file; } - // Batch mode: --batch reads a file list, one wav path per line. Loads - // the model ONCE and reuses the context across all files. Outputs one - // JSONL line per file to stdout when --batch-jsonl is set, otherwise - // the same human-readable format as single-file mode. if (!args.batch_file.empty()) { return transcribe_cli::run_asr_batch(args, output); } diff --git a/include/transcribe/diarize.h b/include/transcribe/diarize.h index ea00355b7..0168614d7 100644 --- a/include/transcribe/diarize.h +++ b/include/transcribe/diarize.h @@ -6,12 +6,8 @@ * TRANSCRIBE_ROLE_DIARIZE). ASR models that attribute speakers inside a * transcript keep using transcribe_run_params::diarize instead. * - * Shape: load a model with transcribe_model_load_file, open a - * transcribe_diarize_session on it, run, then read the speaker segments. - * Sessions follow the transcribe_session threading contract in - * transcribe.h: one thread at a time per session, the model outlives its - * sessions, and at most one compute in flight per model across every - * session of every role. + * Open a transcribe_diarize_session on a loaded model, run, then read the + * speaker segments. Threading and lifetime rules: docs/roles.md. */ #ifndef TRANSCRIBE_DIARIZE_H diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6db3621ab..2ea1319c5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -319,89 +319,17 @@ add_test(NAME transcribe_run_dispatch_unit COMMAND transcribe_run_dispatch_unit) # ----------------------------------------------------------------------------- -# Load-time role validation +# Dispatcher units against fake arches: roles, host thread pool, non-finite +# input, throwing stream hooks, DIARIZE role # ----------------------------------------------------------------------------- -add_executable(transcribe_role_resolve_unit - role_resolve_unit.cpp) - -target_link_libraries(transcribe_role_resolve_unit PRIVATE transcribe) - -target_include_directories(transcribe_role_resolve_unit PRIVATE - ${CMAKE_SOURCE_DIR}/src) - -transcribe_apply_warnings(transcribe_role_resolve_unit) - -add_test(NAME transcribe_role_resolve_unit - COMMAND transcribe_role_resolve_unit) - -# ----------------------------------------------------------------------------- -# Host thread pool: join on every path, exceptions reach the caller -# ----------------------------------------------------------------------------- - -add_executable(transcribe_run_on_threads_unit - run_on_threads_unit.cpp) - -target_link_libraries(transcribe_run_on_threads_unit PRIVATE transcribe) - -target_include_directories(transcribe_run_on_threads_unit PRIVATE - ${CMAKE_SOURCE_DIR}/src) - -transcribe_apply_warnings(transcribe_run_on_threads_unit) - -add_test(NAME transcribe_run_on_threads_unit - COMMAND transcribe_run_on_threads_unit) - -# ----------------------------------------------------------------------------- -# Non-finite PCM rejection on every ASR entry point -# ----------------------------------------------------------------------------- - -add_executable(transcribe_nonfinite_input_unit - nonfinite_input_unit.cpp) - -target_link_libraries(transcribe_nonfinite_input_unit PRIVATE transcribe) - -target_include_directories(transcribe_nonfinite_input_unit PRIVATE - ${CMAKE_SOURCE_DIR}/src) - -transcribe_apply_warnings(transcribe_nonfinite_input_unit) - -add_test(NAME transcribe_nonfinite_input_unit - COMMAND transcribe_nonfinite_input_unit) - -# ----------------------------------------------------------------------------- -# A throwing stream hook ends the stream (FAILED, or IDLE for reset) -# ----------------------------------------------------------------------------- - -add_executable(transcribe_stream_hook_throw_unit - stream_hook_throw_unit.cpp) - -target_link_libraries(transcribe_stream_hook_throw_unit PRIVATE transcribe) - -target_include_directories(transcribe_stream_hook_throw_unit PRIVATE - ${CMAKE_SOURCE_DIR}/src) - -transcribe_apply_warnings(transcribe_stream_hook_throw_unit) - -add_test(NAME transcribe_stream_hook_throw_unit - COMMAND transcribe_stream_hook_throw_unit) - -# ----------------------------------------------------------------------------- -# DIARIZE role dispatcher (fake arch) -# ----------------------------------------------------------------------------- - -add_executable(transcribe_diarize_dispatch_unit - diarize_dispatch_unit.cpp) - -target_link_libraries(transcribe_diarize_dispatch_unit PRIVATE transcribe) - -target_include_directories(transcribe_diarize_dispatch_unit PRIVATE - ${CMAKE_SOURCE_DIR}/src) - -transcribe_apply_warnings(transcribe_diarize_dispatch_unit) - -add_test(NAME transcribe_diarize_dispatch_unit - COMMAND transcribe_diarize_dispatch_unit) +foreach(_t role_resolve run_on_threads nonfinite_input stream_hook_throw diarize_dispatch) + add_executable(transcribe_${_t}_unit ${_t}_unit.cpp) + target_link_libraries(transcribe_${_t}_unit PRIVATE transcribe) + target_include_directories(transcribe_${_t}_unit PRIVATE ${CMAKE_SOURCE_DIR}/src) + transcribe_apply_warnings(transcribe_${_t}_unit) + add_test(NAME transcribe_${_t}_unit COMMAND transcribe_${_t}_unit) +endforeach() # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior diff --git a/tests/api_smoke.c b/tests/api_smoke.c index c39081364..dbdb6dca0 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -525,9 +525,10 @@ static void test_model_introspection_null(void) { false); CHECK(transcribe_model_accepts_ext_kind(NULL, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN) == false); + CHECK(transcribe_model_roles(NULL) == 0); + /* The feature probe is also NULL-safe and returns false for every * known feature value plus any out-of-range enum. */ - CHECK(transcribe_model_roles(NULL) == 0); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_INITIAL_PROMPT) == false); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK) == false); CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_LONG_FORM) == false); diff --git a/tests/cli_diarize_smoke.cmake b/tests/cli_diarize_smoke.cmake index eb35f1840..a60fd78c1 100644 --- a/tests/cli_diarize_smoke.cmake +++ b/tests/cli_diarize_smoke.cmake @@ -1,5 +1,5 @@ -# transcribe-cli --role diarize prints speaker segments for the 2-speaker -# oracle mix. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). +# transcribe-cli routes a DIARIZE model to the diarize path and prints speaker +# segments for the 2-speaker oracle mix. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") message("SKIP: TRANSCRIBE_SORTFORMER_GGUF not set") @@ -7,13 +7,13 @@ if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") endif() execute_process( - COMMAND "${CLI}" -q --backend cpu --role diarize -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" "${WAV}" + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" "${WAV}" RESULT_VARIABLE rc OUTPUT_VARIABLE out ERROR_VARIABLE err) if(NOT rc EQUAL 0) message(FATAL_ERROR "transcribe-cli exited ${rc}\n${out}\n${err}") endif() -if(NOT out MATCHES "run: ok" OR NOT out MATCHES "speaker segments: [1-9]" OR NOT out MATCHES "S2\n") +if(NOT out MATCHES "speaker segments: [1-9].*S2") message(FATAL_ERROR "unexpected output:\n${out}") endif() diff --git a/tests/nonfinite_input_unit.cpp b/tests/nonfinite_input_unit.cpp index 38503873d..5eb9688af 100644 --- a/tests/nonfinite_input_unit.cpp +++ b/tests/nonfinite_input_unit.cpp @@ -1,6 +1,6 @@ // nonfinite_input_unit.cpp - every ASR PCM entry point rejects NaN / Inf // samples with INVALID_ARG before any result or stream state is modified -// and before the family hook runs (D16a). Silence stays valid input. +// and before the family hook runs. #include "transcribe-arch.h" #include "transcribe-model.h" @@ -109,18 +109,6 @@ void test_run_rejects_nonfinite() { } } -void test_run_accepts_silence() { - transcribe_model model; - model.arch = &k_arch; - transcribe_session s; - seed(s, model); - std::vector pcm(1600, 0.0f); - g_run_calls = 0; - CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); - CHECK(g_run_calls == 1); - CHECK(s.full_text == "fresh"); -} - void test_batch_rejects_nonfinite_whole_batch() { transcribe_model model; model.arch = &k_arch; @@ -140,20 +128,6 @@ void test_batch_rejects_nonfinite_whole_batch() { } } -void test_batch_accepts_silence() { - transcribe_model model; - model.arch = &k_arch; - transcribe_session s; - seed(s, model); - std::vector a(800, 0.0f), b(800, 0.0f); - const float * pcm[] = { a.data(), b.data() }; - const int n_samples[] = { 800, 800 }; - g_run_batch_calls = 0; - CHECK(transcribe_run_batch(&s, pcm, n_samples, 2, nullptr) == TRANSCRIBE_OK); - CHECK(g_run_batch_calls == 1); - CHECK(transcribe_batch_n_results(&s) == 2); -} - void test_stream_feed_rejects_nonfinite() { transcribe_model model; model.arch = &k_arch; @@ -189,9 +163,7 @@ int main() { transcribe_log_set(nullptr, nullptr); test_run_rejects_nonfinite(); - test_run_accepts_silence(); test_batch_rejects_nonfinite_whole_batch(); - test_batch_accepts_silence(); test_stream_feed_rejects_nonfinite(); if (g_failures != 0) { diff --git a/tests/run_on_threads_unit.cpp b/tests/run_on_threads_unit.cpp index d7a1d7692..244b632ec 100644 --- a/tests/run_on_threads_unit.cpp +++ b/tests/run_on_threads_unit.cpp @@ -1,6 +1,6 @@ // run_on_threads_unit.cpp - transcribe::run_on_threads joins every launched // thread on every path and hands worker / launch exceptions to the caller -// instead of calling std::terminate (H5). +// instead of calling std::terminate. #include "transcribe-batch-util.h" diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp index cfd008738..3ef858e0f 100644 --- a/tests/sortformer_diarize_unit.cpp +++ b/tests/sortformer_diarize_unit.cpp @@ -1,15 +1,5 @@ // sortformer_diarize_unit.cpp - Sortformer on the DIARIZE role, against a // real GGUF (TRANSCRIBE_SORTFORMER_GGUF; RC 77 skip when unset). -// -// 1. roles (DIARIZE only), info, and the SFDR kind on the DIARIZE_RUN slot. -// 2. Golden segments on the 2-speaker oracle mix per preset (CPU), and the -// ext preset matching the TRANSCRIBE_SORTFORMER_STREAM_PRESET env path -// the DER tooling uses. -// 3. An out-of-range preset is rejected before the previous result is -// cleared. -// 4. Cross-role concurrency: when TRANSCRIBE_WHISPER_GGUF is also set, an -// ASR session and a diarize session on two models run on two threads -// at once and match their sequential results. #include "transcribe.h" #include "transcribe/diarize.h" @@ -130,13 +120,8 @@ int main() { return EXIT_FAILURE; } - // 1. Roles, info, extension probe. ASR entry points refuse the model. + // 1. Roles, info, extension probe. CHECK(transcribe_model_roles(model) == TRANSCRIBE_ROLE_DIARIZE); - transcribe_session * asr = nullptr; - CHECK(transcribe_session_init(model, nullptr, &asr) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); - transcribe_capabilities caps; - transcribe_capabilities_init(&caps); - CHECK(transcribe_model_get_capabilities(model, &caps) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); CHECK(!transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION)); transcribe_diarize_info info; transcribe_diarize_info_init(&info); diff --git a/tests/stream_hook_throw_unit.cpp b/tests/stream_hook_throw_unit.cpp index 27494e2f0..9b0fc53ab 100644 --- a/tests/stream_hook_throw_unit.cpp +++ b/tests/stream_hook_throw_unit.cpp @@ -1,8 +1,7 @@ // stream_hook_throw_unit.cpp - a family stream hook that throws ends the // stream: begin / feed / finalize leave it FAILED with last_status matching // the status the call returned (OOM for bad_alloc, BACKEND otherwise), and -// reset still ends IDLE. Before, a throwing hook left the stream ACTIVE -// while the caller got an error. +// reset still ends IDLE. #include "transcribe-arch.h" #include "transcribe-model.h" @@ -95,7 +94,8 @@ struct Fixture { } }; -void expect_failed(const transcribe_session & s, transcribe_status st, transcribe_status want) { +void expect_failed(const transcribe_session & s, transcribe_status st, Throw t) { + const transcribe_status want = t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND; CHECK(st == want); CHECK(transcribe_stream_get_state(&s) == TRANSCRIBE_STREAM_FAILED); CHECK(transcribe_stream_last_status(&s) == want); @@ -107,7 +107,7 @@ void test_begin_throw() { Fixture f; g_begin = t; const transcribe_status st = transcribe_stream_begin(&f.session, nullptr, nullptr); - expect_failed(f.session, st, t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + expect_failed(f.session, st, t); // A failed stream accepts a fresh begin. g_begin = Throw::None; CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); @@ -121,7 +121,7 @@ void test_feed_throw() { Fixture f; CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); g_feed = t; - expect_failed(f.session, f.feed(), t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + expect_failed(f.session, f.feed(), t); // No longer ACTIVE, so further feeds are refused as on any failed stream. g_feed = Throw::None; CHECK(f.feed() == TRANSCRIBE_ERR_INVALID_ARG); @@ -136,7 +136,7 @@ void test_finalize_throw() { CHECK(f.feed() == TRANSCRIBE_OK); g_finalize = t; const transcribe_status st = transcribe_stream_finalize(&f.session, nullptr); - expect_failed(f.session, st, t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND); + expect_failed(f.session, st, t); } } From df25d3d91fc8d4f0e0c665cb0d1874a98c0d6e0d Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:38:08 +0800 Subject: [PATCH 40/51] ts: abort-hook cleanup survives dispose during a run SessionCore.#installAbort's cleanup re-read the handle through the disposed-checking getter, so disposing the session (or model) during a run with a signal turned the finished result into a "disposed" error and leaked the koffi callback and signal listener. Capture the native handle at install; the native free is queued behind the lock, so it is still valid at cleanup. --- bindings/typescript/src/index.ts | 5 +++-- bindings/typescript/test/compute-rules.test.mjs | 15 +++++++++++++++ bindings/typescript/test/diarize.test.mjs | 4 +++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index b27772e0e..9db2af105 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -878,9 +878,10 @@ class SessionCore { () => flag.aborted, n.koffi.pointer(n.abortProto), ); - this.#setAbort(this.handle, cbPtr, null); + const h = this.#h; // still valid at cleanup: a mid-run dispose frees behind the lock + this.#setAbort(h, cbPtr, null); return () => { - this.#setAbort(this.handle, null, null); + this.#setAbort(h, null, null); n.koffi.unregister(cbPtr); signal.removeEventListener("abort", onAbort); }; diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs index 59db3f6d9..2c216b67c 100644 --- a/bindings/typescript/test/compute-rules.test.mjs +++ b/bindings/typescript/test/compute-rules.test.mjs @@ -331,6 +331,21 @@ modelTest("session dispose during an in-flight run defers the free; the result s } }); +modelTest("dispose during a run with a signal returns the result and removes the listener", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const ac = new AbortController(); + const p = s.run(jfk(), { signal: ac.signal }); + await Promise.resolve(); + s.dispose(); + assert.match((await p).text, /ask not what your country/i); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + } finally { + m.dispose(); + } +}); + modelTest("model dispose during an in-flight run keeps the model alive for the call", MODEL, async () => { const m = await TranscribeModel.load(MODEL); const s = m.createSession(); diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs index 830101bae..4b76d8636 100644 --- a/bindings/typescript/test/diarize.test.mjs +++ b/bindings/typescript/test/diarize.test.mjs @@ -145,11 +145,13 @@ modelTest("a diarize run queued before its dispose is rejected as disposed", SOR modelTest("dispose during an in-flight diarize run defers the free; the rows survive", SORTFORMER_MODEL, async () => { const m = await TranscribeModel.load(SORTFORMER_MODEL); const d = m.createDiarizeSession(); - const p = d.run(mix()); + const ac = new AbortController(); + const p = d.run(mix(), { signal: ac.signal }); await Promise.resolve(); d.dispose(); assert.throws(() => d.timings, /run\(\).*in flight/); assert.ok((await p).length > 0); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); assert.throws(() => d.timings, /disposed/); const d2 = m.createDiarizeSession(); From 457392ab82f492d0318184a322d4b3126a46fe86 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:44:10 +0800 Subject: [PATCH 41/51] ts: trim code and tests from review - SessionCore.exclusive takes busyOp (null = the lease-holding stream) instead of a ComputeGate; drop STREAM_LEASE_HOLDER. - One call(kind, signal, fn, ...args) helper replaces the open()/close() pairs at the five worker-call sites. - Stream reaches SessionCore via SESSION_CONTROL; one assertNotComputing (same messages); drop SessionCore.inFlight. - Inline sortformerPreset; shorten comments and UnsupportedRole's doc. - Tests: drop duplicates of streaming/errors coverage and dead guards. - README: fix a stranded reflow; tighten the diarize and feed paragraphs. --- bindings/typescript/README.md | 33 ++- bindings/typescript/src/errors.ts | 8 +- bindings/typescript/src/index.ts | 225 +++++------------- .../typescript/test/compute-rules.test.mjs | 49 +--- bindings/typescript/test/diarize.test.mjs | 33 +-- bindings/typescript/test/errors.test.mjs | 12 +- 6 files changed, 86 insertions(+), 274 deletions(-) diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 4ff23d972..6eecb8d1d 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -111,11 +111,8 @@ model.accepts({ kind: "whisper" }); // does this model take that extension? ### Diarization (DIARIZE role) -`model.roles` lists what a model serves (`"asr"`, `"diarize"`). A diarization -model (e.g. Sortformer, which serves only `"diarize"`) answers who spoke when; -ASR calls (`capabilities`, `createSession`, `transcribe`, ...) on a model -without `"asr"`, and diarize calls on one without `"diarize"`, throw -`UnsupportedRole`. +`model.roles` lists what a model serves (`"asr"`, `"diarize"`; Sortformer is +diarize-only). Calls for a role the model lacks throw `UnsupportedRole`. ```ts const { sampleRate, maxSpeakers } = model.diarizeInfo; @@ -126,9 +123,8 @@ const turns = await diarizer.run(pcm, { for (const t of turns) console.log(t.speakerId, t.t0Ms, t.t1Ms); ``` -`run` takes a `signal` like `Session.run`, and `diarizer.timings` reports the -last run. A diarize run is a compute like any other: it shares the model's -one-at-a-time rule and `Busy` refusal below. +`run` takes a `signal` like `Session.run`; `diarizer.timings` reports the last +run. Diarize runs share the model's one-at-a-time rule and `Busy` refusal below. ### Resource management @@ -173,23 +169,22 @@ event loop stays responsive while inference runs. The C library allows **one compute in flight per model** — a `run`, a `runBatch`, a diarize `run`, or an *active stream* — across all of its sessions. The binding enforces this: every compute call serializes through an internal model-wide -mutex, and an active -stream holds a model-wide lease for its whole lifetime. While a stream is active -(after `stream()`, before `finalize()`/`reset()`), a `run`/`runBatch`/`stream` on -any session of that model is refused with a `Busy` error rather than allowed to -race. So to parallelize, load one model per worker; to share a model, finalize or -reset the stream first. A single `Session` is single-use-at-a-time — don't call -`run`/`feed` on the same session concurrently. +mutex, and an active stream holds a model-wide lease for its whole lifetime. +While a stream is active (after `stream()`, before `finalize()`/`reset()`), a +`run`/`runBatch`/`stream` on any session of that model is refused with a `Busy` +error rather than allowed to race. So to parallelize, load one model per worker; +to share a model, finalize or reset the stream first. A single `Session` is +single-use-at-a-time — don't call `run`/`feed` on the same session concurrently. Hand-offs are ordered, not racy: `finalize()`/`reset()`/`dispose()` release the lease only after the native teardown runs on the shared queue, so the slot is never freed early. Issue the teardown before the next `stream()`/`run()` and it is correctly serialized — `stream.reset(); const next = await session.stream()` works without awaiting the (void) `reset()`. The reverse order — beginning before -the teardown — is refused with `Busy`, by design. A rejected `feed()` follows the -stream's state: input refused up front (e.g. NaN/Inf samples) leaves the stream -`"active"` and the lease held, so you can keep feeding; a failure inside the -model moves it to `"failed"` and frees the lease. +the teardown — is refused with `Busy`, by design. + +A `feed()` rejected up front (e.g. NaN samples) leaves the stream `"active"` and +the lease held; a failure inside the model makes it `"failed"` and frees the lease. Because the compute is genuinely on another thread, **do not touch a session while a call against it is in flight** — it is single-threaded in the C library: diff --git a/bindings/typescript/src/errors.ts b/bindings/typescript/src/errors.ts index d8baef070..f0495ea83 100644 --- a/bindings/typescript/src/errors.ts +++ b/bindings/typescript/src/errors.ts @@ -37,13 +37,7 @@ export class AbiError extends TranscribeError {} export class InputTooLong extends TranscribeError {} export class VersionMismatch extends TranscribeError {} -/** - * Raised when the model does not serve the role the call needs - * (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`): e.g. opening an ASR session on, or - * reading ASR capabilities of, a model whose roles lack ASR. The model loaded - * fine; it is the wrong kind of model for this call. Not an UnsupportedRequest, - * since no request option would make it succeed. - */ +/** Raised when the model does not serve the role (ASR, diarize) the call needs. */ export class UnsupportedRole extends TranscribeError {} /** Raised when a run is cancelled; carries any partial transcript in `partialResult`. */ diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 9db2af105..a03ac9bb2 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -640,14 +640,13 @@ const FAMILY: Record = { kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, type: "transcribe_sortformer_diarize_ext", init: "sortformerDiarizeExtInit", - map: (o) => sortformerPreset(o), + map: (o) => ({ + preset: + o.preset === undefined ? undefined : lookup(SORTFORMER_PRESET, o.preset, "sortformer preset"), + }), }, }; -const sortformerPreset = (o: any) => ({ - preset: o.preset === undefined ? undefined : lookup(SORTFORMER_PRESET, o.preset, "sortformer preset"), -}); - const SORTFORMER_PRESET: Record = { default: g.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, very_high_latency: g.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY, @@ -737,40 +736,19 @@ const STREAM_TEARDOWN = new WeakMap< >(); /** - * How Session.#exclusive admits a compute call once it holds the model lock. - * The checks run in field order: disposal, then the stream lease. - */ -interface ComputeGate { - /** - * Recheck disposal inside the lock: dispose() may have run after the caller - * captured the handle but before its queued body. feed()/finalize() check - * only before queuing (their native free is queued behind them). - */ - recheckDisposed: boolean; - /** - * The op named in the Busy refusal while a stream holds the model lease, or - * null when the caller is that stream (feed()/finalize()). - */ - busyOp: string | null; -} - -/** - * Opens the in-flight window around one worker call: installs `signal`'s abort - * callback, then marks the session busy as `kind` so result reads fail fast. - * The returned close() clears the mark, then uninstalls the callback; call it - * in a `finally` right after the worker await. + * Makes one worker call `fn(...args)` with `signal`'s abort callback installed + * and the session marked in flight as `kind` (result reads fail fast) until it + * settles. Only valid inside an exclusive() body. */ -type ComputeWindow = (kind: string, signal?: AbortSignal) => () => void; - -/** The gate for calls made by the stream that holds the model lease. */ -const STREAM_LEASE_HOLDER: ComputeGate = { recheckDisposed: false, busyOp: null }; +type ComputeCall = ( + kind: string, + signal: AbortSignal | undefined, + fn: any, + ...args: any[] +) => Promise; interface SessionControl { - exclusive( - gate: ComputeGate, - body: (open: ComputeWindow) => Promise, - ): Promise; - currentCompute(): string | null; + core: SessionCore; isCurrentStream(stream: Stream): boolean; replaceCurrentStream(stream: Stream): void; clearCurrentStream(stream: Stream): void; @@ -807,55 +785,41 @@ class SessionCore { return this.#disposed; } - get inFlight(): string | null { - return this.#inFlight; - } - /** Reads touch the session; forbidden while a worker call is in flight. */ assertNotComputing(what: string): void { if (this.#inFlight) { throw new TranscribeError( - `cannot read session ${what} while ${this.#inFlight} is in flight; await it first`, + `cannot read ${what} while ${this.#inFlight} is in flight; await it first`, ); } } /** - * Run `body` as this session's one native compute: the single path every - * run / batch / stream begin / feed / finalize / diarize run takes (Stream - * reaches it via SESSION_CONTROL). It queues on the model-wide FIFO lock, so - * no two computes on one model overlap and queued native frees (deferFree) - * run only after it drains. Once the lock is held, `gate` is applied — - * disposal, then the stream lease — before `body` runs. `body` opens the - * in-flight window (abort callback + in-flight mark) around its worker - * await, and copies its result out before returning, i.e. before the lock - * is released. - * - * Deliberately not `async`: refusals are returned as rejected promises and - * `body`'s promise is returned as-is, so settle timing matches the inline - * lock bodies this replaced. + * Run `body` as this session's one native compute on the model-wide FIFO + * lock (it copies results out before release). Refuses a disposed session, + * then an active stream (Busy naming `busyOp`); null `busyOp` = that stream. + * Not `async`: refusals and `body`'s promise are returned as-is (no extra ticks). */ exclusive( - gate: ComputeGate, - body: (open: ComputeWindow) => Promise, + busyOp: string | null, + body: (call: ComputeCall) => Promise, ): Promise { return this.#lock.run(() => { - if (gate.recheckDisposed && this.#disposed) + if (busyOp !== null && this.#disposed) return Promise.reject(new TranscribeError("session has been disposed")); - if (gate.busyOp !== null && this.#lock.streamActive) - return Promise.reject(busyError(gate.busyOp)); - return body(this.#openComputeWindow); + if (busyOp !== null && this.#lock.streamActive) + return Promise.reject(busyError(busyOp)); + return body(this.#call); }); } - /** See ComputeWindow. Only valid inside an exclusive() body. */ - #openComputeWindow: ComputeWindow = (kind, signal) => { + #call: ComputeCall = (kind, signal, fn, ...args) => { const cancel = this.#installAbort(signal); this.#inFlight = kind; - return () => { + return callAsync(fn, ...args).finally(() => { if (this.#inFlight === kind) this.#inFlight = null; cancel?.(); - }; + }); }; /** @@ -925,8 +889,7 @@ export class Session { this.#lock = lock; this.#untrack = untrack; SESSION_CONTROL.set(this, { - exclusive: (gate, body) => this.#core.exclusive(gate, body), - currentCompute: () => this.#core.inFlight, + core: this.#core, isCurrentStream: (stream) => this.#activeStream === stream, replaceCurrentStream: (stream) => { if (this.#activeStream && this.#activeStream !== stream) { @@ -946,7 +909,7 @@ export class Session { } get limits(): SessionLimits { - this.#core.assertNotComputing("limits"); + this.#core.assertNotComputing("session limits"); const n = this.#n; const l: any = {}; n.F.sessionLimitsInit(l); @@ -974,14 +937,8 @@ export class Session { const p = this.#buildRunParams(opts); - return this.#core.exclusive({ recheckDisposed: true, busyOp: "run" }, async (open) => { - const close = open("run()", opts.signal); - let status: number; - try { - status = await callAsync(F.run, h, samples, samples.length, p); - } finally { - close(); - } + return this.#core.exclusive("run", async (call) => { + const status = await call("run()", opts.signal, F.run, h, samples, samples.length, p); if ( status === g.TRANSCRIBE_ERR_ABORTED || @@ -1069,21 +1026,10 @@ export class Session { const counts = Int32Array.from(arrays, (a) => a.length); const p = this.#buildRunParams(opts); - return this.#core.exclusive({ recheckDisposed: true, busyOp: "runBatch" }, async (open) => { - const close = open("runBatch()", opts.signal); - let status: number; - try { - status = await callAsync( - F.runBatch, - h, - arrays, - counts, - arrays.length, - p, - ); - } finally { - close(); - } + return this.#core.exclusive("runBatch", async (call) => { + const status = await call( + "runBatch()", opts.signal, F.runBatch, h, arrays, counts, arrays.length, p, + ); // A batch returns OK even with per-utterance failures; only a top-level // error (or a whole-batch abort) is fatal here. if (status !== g.TRANSCRIBE_OK && status !== g.TRANSCRIBE_ERR_ABORTED) { @@ -1159,10 +1105,8 @@ export class Session { if (opts.family) sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); - // The gate rechecks disposal inside the lock: dispose() may have run after - // we captured `h` but before this queued body — don't begin a stream on a - // dead session. Begin is a synchronous native call, so no in-flight window. - return this.#core.exclusive({ recheckDisposed: true, busyOp: "begin a stream" }, async () => { + // Begin is a synchronous native call, so no in-flight window (no call()). + return this.#core.exclusive("begin a stream", async () => { check(n, F.streamBegin(h, rp, sp), "transcribe_stream_begin"); this.#lock.streamActive = true; // claim the lease for the whole stream lifetime // The Stream holds the Session (not a raw handle) so its calls fail fast @@ -1176,7 +1120,7 @@ export class Session { } get wasAborted(): boolean { - this.#core.assertNotComputing("wasAborted"); + this.#core.assertNotComputing("session wasAborted"); return this.#n.F.wasAborted(this.handle); } @@ -1271,41 +1215,19 @@ export class Stream { this.#assertCurrent("feed"); if (!this.#active) throw new TranscribeError("stream has been reset"); const samples = toFloat32(pcm); - return this.#sessionControl.exclusive(STREAM_LEASE_HOLDER, async (open) => { + return this.#sessionControl.core.exclusive(null, async (call) => { const u: any = {}; n.F.streamUpdateInit(u); - // The native feed runs on a libuv worker. While it is in flight the - // session must not be touched from the main thread — the C session API - // is single-threaded (transcribe.h), and stream_get_text hands back - // pointers the feed may free/realloc. Flag the owning session so every - // result getter fails fast instead of racing into a use-after-free. - const close = open("feed()/finalize()"); - try { - const status = await callAsync( - n.F.streamFeed, - h, - samples, - samples.length, - u, - ); - // The lease follows the native stream lifecycle, so on failure ask - // the stream where it ended up (still inside this exclusive slot, the - // worker call has returned). A feed rejected before the family hook - // (non-finite samples, other pre-hook INVALID_ARG) leaves it ACTIVE: - // keep the lease, so the stream stays usable and siblings stay Busy. - // A hook failure moves it to FAILED, which is no longer an active - // stream in the C API: keep the wrapper readable for state/lastStatus, - // but free the model-wide compute slot. - if ( - status !== g.TRANSCRIBE_OK && - n.F.streamGetState(h) !== g.TRANSCRIBE_STREAM_ACTIVE - ) { - this.#releaseLease(); - } - check(n, status, "transcribe_stream_feed"); - } finally { - close(); - } + const status = await call( + "feed()/finalize()", undefined, n.F.streamFeed, h, samples, samples.length, u, + ); + // The lease follows the native stream: a feed refused before the family + // hook (e.g. non-finite samples) leaves it ACTIVE, so keep the lease + // (siblings stay Busy); a hook failure moves it to FAILED, no longer + // active, so free the model-wide slot (state/lastStatus stay readable). + if (status !== g.TRANSCRIBE_OK && n.F.streamGetState(h) !== g.TRANSCRIBE_STREAM_ACTIVE) + this.#releaseLease(); + check(n, status, "transcribe_stream_feed"); return toStreamUpdate(u); }); } @@ -1316,18 +1238,13 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("finalize stream"); if (!this.#active) throw new TranscribeError("stream has been reset"); - return this.#sessionControl.exclusive(STREAM_LEASE_HOLDER, async (open) => { + return this.#sessionControl.core.exclusive(null, async (call) => { const u: any = {}; n.F.streamUpdateInit(u); - const close = open("feed()/finalize()"); try { - check( - n, - await callAsync(n.F.streamFinalize, h, u), - "transcribe_stream_finalize", - ); + const status = await call("feed()/finalize()", undefined, n.F.streamFinalize, h, u); + check(n, status, "transcribe_stream_finalize"); } finally { - close(); // Finalize ends the active stream (FINISHED on success, FAILED on // error), so the model is free again — release the lease either way. this.#releaseLease(); @@ -1336,25 +1253,11 @@ export class Stream { }); } - /** - * Reads borrow session-owned snapshot memory, so they are forbidden while - * any worker call is computing on this session (concurrent use is undefined - * per transcribe.h). The natural await-then-read pattern is unaffected. - */ - #assertNotComputing(what: string): void { - const compute = this.#sessionControl.currentCompute(); - if (compute) { - throw new TranscribeError( - `cannot read stream ${what} while ${compute} is in flight; await it first`, - ); - } - } - /** Current text snapshot (copied at the boundary). */ get text(): StreamText { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream text"); - this.#assertNotComputing("text"); + this.#sessionControl.core.assertNotComputing("stream text"); const n = this.#n; const t: any = {}; n.F.streamTextInit(t); @@ -1371,7 +1274,7 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream snapshot"); if (!this.#active) throw new TranscribeError("stream has been reset"); - this.#assertNotComputing("snapshot"); + this.#sessionControl.core.assertNotComputing("stream snapshot"); return materialize(this.#n, singleAccessors(this.#n, h)); } @@ -1379,14 +1282,14 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream state"); if (!this.#active) return "idle"; // reset() returns to idle; native reset may still be queued - this.#assertNotComputing("state"); + this.#sessionControl.core.assertNotComputing("stream state"); return STREAM_STATES[this.#n.F.streamGetState(h)] ?? "idle"; } get revision(): number { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream revision"); - this.#assertNotComputing("revision"); + this.#sessionControl.core.assertNotComputing("stream revision"); return this.#n.F.streamRevision(h); } @@ -1398,7 +1301,7 @@ export class Stream { get lastStatus(): TranscribeError | null { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream lastStatus"); - this.#assertNotComputing("lastStatus"); + this.#sessionControl.core.assertNotComputing("stream lastStatus"); const n = this.#n; const status = n.F.streamLastStatus(h); if (status === g.TRANSCRIBE_OK) return null; @@ -1474,14 +1377,8 @@ export class DiarizeSession { if (opts.family) p.family = buildFamily(n, this.#model.handle, opts.family, "diarize_run"); - return this.#core.exclusive({ recheckDisposed: true, busyOp: "diarize" }, async (open) => { - const close = open("run()", opts.signal); - let status: number; - try { - status = await callAsync(F.diarizeRun, h, samples, samples.length, p); - } finally { - close(); - } + return this.#core.exclusive("diarize", async (call) => { + const status = await call("run()", opts.signal, F.diarizeRun, h, samples, samples.length, p); check(n, status, "transcribe_diarize_run"); return readSpeakerSegments(n, F.diarizeNSegments(h), (i, o) => F.diarizeGetSegment(h, i, o), @@ -1491,7 +1388,7 @@ export class DiarizeSession { /** load_ms plus the last run's mel / encode time. */ get timings(): Timings { - this.#core.assertNotComputing("timings"); + this.#core.assertNotComputing("session timings"); const h = this.#core.handle; return readTimings(this.#n, (o) => this.#n.F.diarizeGetTimings(h, o)); } diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs index 2c216b67c..bcb4cd219 100644 --- a/bindings/typescript/test/compute-rules.test.mjs +++ b/bindings/typescript/test/compute-rules.test.mjs @@ -1,10 +1,5 @@ -// Characterization of the rules every native compute call obeys: model-wide -// exclusion, active-stream rejection, the disposed recheck and its order -// against Busy, cancel install/cleanup, the model staying alive for the call, -// deferred frees when dispose() races an in-flight worker call, and results -// copied out before the model lock is released. These pin today's behavior, -// including where call sites differ, so routing compute through one helper can -// be checked as behavior-preserving. +// The rules every native compute call obeys: model-wide exclusion, Busy and +// disposed refusals, abort install/cleanup, deferred frees, and copy-out. import assert from "node:assert/strict"; import { getEventListeners } from "node:events"; @@ -43,7 +38,7 @@ modelTest("compute is exclusive model-wide: a sibling run waits its turn", MODEL } }); -modelTest("runBatch and finalize mark the session in flight with their own label", STREAMING_MODEL, async () => { +modelTest("runBatch marks the session in flight with its own label", STREAMING_MODEL, async () => { const m = await TranscribeModel.load(STREAMING_MODEL); try { const s = m.createSession(); @@ -52,16 +47,6 @@ modelTest("runBatch and finalize mark the session in flight with their own label assert.throws(() => s.limits, /runBatch\(\).*in flight/); await pending; assert.doesNotThrow(() => s.limits); - - const stream = await s.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - const fin = stream.finalize(); - await Promise.resolve(); - assert.throws(() => stream.text, /feed\(\)\/finalize\(\).*in flight/); - assert.throws(() => s.wasAborted, /feed\(\)\/finalize\(\).*in flight/); - await fin; - assert.doesNotThrow(() => stream.text); - stream.reset(); s.dispose(); } finally { m.dispose(); @@ -166,15 +151,11 @@ modelTest("a feed rejected before the native hook keeps the stream lease", STREA await assert.rejects(() => stream.feed(bad), InvalidArgument); assert.equal(stream.state, "active"); await assert.rejects(() => b.run(second()), Busy); - await assert.rejects(() => b.stream(), Busy); - await assert.rejects(() => a.run(second()), Busy); // The stream is still usable, and finalize releases the lease as usual. const u = await stream.feed(jfk().subarray(16000, 32000)); assert.equal(typeof u.revision, "number"); - const fin = await stream.finalize(); - assert.equal(fin.isFinal, true); - assert.equal(stream.state, "finished"); + await stream.finalize(); const r = await b.run(second()); assert.equal(typeof r.text, "string"); stream.reset(); @@ -259,11 +240,7 @@ modelTest("an aborted call uninstalls its callback; the next call is not aborted ac.abort(); const items = await s.runBatch([jfk(), second()], { signal: ac.signal }); assert.equal(items.length, 2); - for (const it of items) { - assert.equal(it.ok, false); - assert.ok(it.error instanceof Aborted); - assert.ok(it.error.partialResult); - } + assert.ok(items.every((it) => it.error instanceof Aborted)); assert.equal(getEventListeners(ac.signal, "abort").length, 0); await assert.rejects(() => s.run(jfk(), { signal: ac.signal }), Aborted); assert.equal(getEventListeners(ac.signal, "abort").length, 0); @@ -318,14 +295,6 @@ modelTest("session dispose during an in-flight run defers the free; the result s assert.match(r.text, /ask not what your country/i); assert.ok(r.segments.length >= 1); assert.throws(() => s.limits, /disposed/); - - const s2 = m.createSession(); - const pb = s2.runBatch([jfk(), half()]); - await Promise.resolve(); - s2.dispose(); - const items = await pb; - assert.equal(items.length, 2); - assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); } finally { m.dispose(); } @@ -356,14 +325,6 @@ modelTest("model dispose during an in-flight run keeps the model alive for the c const r = await p; assert.match(r.text, /ask not what your country/i); assert.throws(() => s.limits, /disposed/); - - const m2 = await TranscribeModel.load(MODEL); - const s2 = m2.createSession(); - const pb = s2.runBatch([jfk()]); - await Promise.resolve(); - m2.dispose(); - const items = await pb; - assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); }); modelTest("model dispose during an in-flight feed keeps the model alive for the call", STREAMING_MODEL, async () => { diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs index 4b76d8636..eade7f13f 100644 --- a/bindings/typescript/test/diarize.test.mjs +++ b/bindings/typescript/test/diarize.test.mjs @@ -2,31 +2,13 @@ // rules a diarize run shares with Session (lock, in-flight mark, abort, // disposed recheck, deferred free). -import { test } from "node:test"; import assert from "node:assert/strict"; import { getEventListeners } from "node:events"; -import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav, jfk } from "./common.mjs"; -import { - TranscribeModel, - DiarizeSession, - Aborted, - Busy, - InvalidArgument, - TranscribeError, - UnsupportedRole, -} from "../dist/index.js"; +import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav } from "./common.mjs"; +import { TranscribeModel, Aborted, InvalidArgument, TranscribeError, UnsupportedRole } from "../dist/index.js"; const mix = () => readWav(SORTFORMER_AUDIO); -test("DiarizeSession exposes only its public surface", () => { - assert.deepEqual(Object.getOwnPropertyNames(DiarizeSession.prototype).sort(), [ - "constructor", - "dispose", - "run", - "timings", - ]); -}); - modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MODEL, async () => { const m = await TranscribeModel.load(MODEL); try { @@ -34,10 +16,6 @@ modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MOD assert.throws(() => m.diarizeInfo, UnsupportedRole); assert.throws(() => m.createDiarizeSession(), UnsupportedRole); assert.equal(m.accepts({ kind: "sortformer_diarize" }), false); - await assert.rejects( - () => m.createSession().run(jfk(), { family: { kind: "sortformer_diarize" } }), - InvalidArgument, - ); } finally { m.dispose(); } @@ -101,9 +79,7 @@ modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, }); // ---- compute rules shared with Session ------------------------------------- -// Busy is not covered here: Sortformer cannot begin a stream, so no model -// serves both a stream lease and the diarize role. The refusal is the same -// SessionCore.exclusive gate compute-rules.test.mjs covers for Session. +// No Busy test: no model serves both a stream lease and the diarize role. modelTest("the abort listener lives only for the call; a pre-aborted run raises Aborted", SORTFORMER_MODEL, async () => { const m = await TranscribeModel.load(SORTFORMER_MODEL); @@ -133,10 +109,9 @@ modelTest("a diarize run queued before its dispose is rejected as disposed", SOR const ac = new AbortController(); const queued = d.run(mix(), { signal: ac.signal }); d.dispose(); - await assert.rejects(queued, (e) => !(e instanceof Busy) && /disposed/i.test(e.message)); + await assert.rejects(queued, /disposed/i); assert.equal(getEventListeners(ac.signal, "abort").length, 0); await running; - await assert.rejects(() => d.run(mix()), /disposed/); } finally { m.dispose(); } diff --git a/bindings/typescript/test/errors.test.mjs b/bindings/typescript/test/errors.test.mjs index 1ed485c8b..eafec6dc3 100644 --- a/bindings/typescript/test/errors.test.mjs +++ b/bindings/typescript/test/errors.test.mjs @@ -67,18 +67,8 @@ test("every transcribe_status maps to its documented error class", () => { }, ); } -}); - -test("UNSUPPORTED_ROLE maps to UnsupportedRole, not a request or load error", () => { - const e = exceptionForStatus(g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "wrong role", "opening session"); + assert.equal(Object.getPrototypeOf(UnsupportedRole), TranscribeError); assert.equal(g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE, 20); - assert.ok(e instanceof UnsupportedRole); - assert.ok(e instanceof TranscribeError); - assert.equal(e.status, 20); - assert.equal(e.name, "UnsupportedRole"); - assert.equal(e instanceof UnsupportedRequest, false); - assert.equal(e instanceof ModelLoadError, false); - assert.equal(e instanceof NotImplementedByModel, false); }); test("OK does not throw", () => { From 4156b7f5220940f7075d2f6bcb79b2c2783489a1 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sat, 3 Oct 2026 22:48:13 +0800 Subject: [PATCH 42/51] python: trim code and tests from review --- bindings/python/README.md | 23 +- .../python/src/transcribe_cpp/__init__.py | 493 ++++++------------ bindings/python/src/transcribe_cpp/errors.py | 11 +- bindings/python/tests/test_compute_lock.py | 273 ++-------- bindings/python/tests/test_diarize.py | 4 - bindings/python/tests/test_errors.py | 56 +- bindings/python/tests/test_family_ext.py | 2 - bindings/python/tests/test_lifetime.py | 19 +- bindings/python/tests/test_prompting.py | 2 +- bindings/python/tests/test_streaming.py | 45 +- 10 files changed, 234 insertions(+), 694 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index f49215bd2..9422adf99 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -75,11 +75,10 @@ Long transcriptions can be cancelled from another thread with Models whose `model.roles` include `Role.DIARIZE` (Sortformer) answer "who spoke when" through a diarize session. `run()` returns `SpeakerSegment` rows (`speaker_id` in `1..model.diarize_info.max_speakers`), grouped by -speaker. It shares the model-wide compute lock with transcription runs and -raises `Busy` while a stream is active. `cancel()` and `close()` work as on -`Session`. A model without the role raises `UnsupportedRole`, as -`model.session()` and `model.capabilities` do on a model without `Role.ASR` -(Sortformer serves only `Role.DIARIZE`). +speaker. Locking, `Busy`, `cancel()` and `close()` work as on `Session`. A +model without the role raises `UnsupportedRole`, as `model.session()` and +`model.capabilities` do on a model without `Role.ASR` (Sortformer serves only +`Role.DIARIZE`). ```python with model.diarize_session() as diarizer: @@ -139,16 +138,10 @@ TRANSCRIBE_LIBRARY=../../build-shared/src/libtranscribe.dylib \ ## Notes -- One compute call at a time per `Model` in 0.x: sessions share the model's - compute backend, so the binding serializes `run` / `run_batch` / stream - calls across all sessions of a model with a model-wide lock. Calls from - other threads wait their turn (they no longer race); load one model per - worker for parallel transcription. An active stream occupies the model - between feeds, so while one is active, `run` / `run_batch` / `stream` on - any session of that model raise `transcribe_cpp.Busy` at once instead of - waiting. `finalize()` or `reset()` the stream (or close its session) to - free the model. A feed rejected for NaN/Inf samples keeps the stream - active. See the `Model` docstring. +- One compute call at a time per `Model`: the binding serializes calls across + all sessions of a model with a model-wide lock (load one model per worker + for parallelism). While a stream is active, other runs and stream begins on + that model raise `transcribe_cpp.Busy`. See the `Model` docstring. - Import package: `transcribe_cpp` - Distribution: `transcribe-cpp` - License: MIT diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 3bf8d20a0..c8c23c770 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -16,9 +16,7 @@ Long-running native calls (model load, run) release the GIL — ctypes does this for every foreign call — so other Python threads make progress during inference. -Compute calls on one Model are serialized by a model-wide lock (see ``Model``): -concurrent calls from other threads wait their turn instead of racing. While a -stream is active on a model, other runs and stream begins on it raise ``Busy``. +Compute locking and ``Busy`` rules: see ``Model``. """ from __future__ import annotations @@ -31,7 +29,7 @@ import weakref from contextlib import contextmanager from dataclasses import dataclass, field -from typing import Literal, Optional, Sequence, Union +from typing import Literal, Optional, Sequence, TypeVar, Union from . import _abi, _generated from ._library import _base_version, artifact_dir, load_library, selected_provider @@ -407,15 +405,12 @@ def set_log_callback(handler) -> None: queue rather than doing heavy work inline. Levels mirror TRANSCRIBE_LOG_LEVEL_* (1=info, 2=warn, 3=error, 4=debug). - The handler can fire while a compute call holds its model's lock, so it - must not call compute APIs (run, run_batch, stream, feed, finalize, - reset) on that model: from the computing thread such a call raises - TranscribeError, and from a ggml worker thread it would deadlock.""" + The handler may fire while its model's compute lock is held, so it must + not call compute APIs on that model (raises, or deadlocks off-thread).""" global _log_handler, _log_trampoline _log_handler = handler if _log_trampoline is None: - # Install the trampoline once; later calls just swap the Python handler - # behind it, so transcribe_log_set is never re-invoked after threads run. + # Installed once; later calls only swap the Python handler behind it. _log_trampoline = _LOG_CFUNC(_log_thunk) _lib.transcribe_log_set(_log_trampoline, None) @@ -939,21 +934,9 @@ def _apply(self, ext) -> None: # --- high-level handles --------------------------------------------------- -class _StreamLeaseSlot: - """A model's stream lease: ``owner`` is the _StreamLease of the active - stream, or None. Guarded by the model's compute lock.""" - - __slots__ = ("owner",) - - def __init__(self): - self.owner: Optional[_StreamLease] = None - - class _StreamLease: - """One stream's claim on its model's stream lease (identity token). - ``handle`` is the session handle the stream began on; while this lease - is the owner, that native session is alive (closing the session clears - the owner in the same deferred step that frees it).""" + """One stream's claim on its model's stream lease (identity token for + ``Model._stream_owner``). ``handle`` is the session handle it began on.""" __slots__ = ("handle",) @@ -966,36 +949,18 @@ class Model: (``arch``, ``capabilities``, ``supports`` ...) and session creation run concurrently. It must outlive its sessions (``close()`` enforces that). - Compute calls are serialized per model. The native library allows at - most one compute call in flight across ALL sessions of a model (they - share its compute backend), so ``Session.run()``, ``run_batch()``, - ``stream()``, ``Stream.feed()`` / ``finalize()`` / ``reset()`` and - ``DiarizeSession.run()`` each - hold a model-wide lock for the native call and for copying its results - out. Calls on any session of the same model from other threads WAIT for - the lock, and a waiting call can still be cancelled with its session's - ``cancel()``. Load one Model per worker for parallel transcription. (Up - to 0.3 the binding did not lock: overlapping calls on one model raced - and could corrupt decodes.) - - An active stream occupies the model for its whole lifetime, not just - during a feed (the C contract forbids a run between the feeds of a - stream on another session). A successful ``stream()`` therefore takes - a model-wide stream lease. Until that stream ends, ``run()``, - ``run_batch()``, ``stream()`` and ``DiarizeSession.run()`` on ANY - session of this model (its own - included) raise :class:`Busy` at once instead of waiting; the stream's - own ``feed()`` / ``finalize()`` proceed. The lease is released by - ``finalize()`` (success or failure), ``reset()``, a ``feed()`` that - leaves the stream no longer active, closing the session or model, or - the Stream being garbage-collected while still active. A call that was - already waiting for the lock when a stream began also raises - :class:`Busy` once it gets the lock. - - ``close()`` (and garbage collection) never waits for the lock: closing - the model or a session while a call is in flight marks it closed at - once, lets the in-flight call finish and copy out its results, and - frees the native handles right after it. + The native library allows one compute call in flight across ALL sessions + of a model, so ``Session.run()`` / ``run_batch()`` / ``stream()``, + ``Stream.feed()`` / ``finalize()`` / ``reset()`` and ``DiarizeSession.run()`` + hold a model-wide lock for the native call and its copy-out. Calls from + other threads wait (still cancellable via ``cancel()``); load one Model + per worker for parallelism. An active stream holds the model's stream + lease from ``stream()`` until it is finalized, reset, fails, is + garbage-collected or its session/model is closed. Meanwhile ``run()``, + ``run_batch()``, ``stream()`` and diarize ``run()`` on any session raise + :class:`Busy` once they hold the lock; the stream's own calls proceed. + ``close()`` and GC never wait for the lock: the handle is closed at once + and the native free runs right after any in-flight call. ``backend="auto"`` (the default) picks the best available device. The ``TRANSCRIBE_BACKEND`` environment variable overrides that *default* — @@ -1040,47 +1005,24 @@ def _h(self) -> ctypes.c_void_p: return self._handle # --- compute exclusion ------------------------------------------------- - # - # One plain (non-reentrant) Lock per model serializes every native compute - # call and the copy-out of its results. Not an RLock: the only way a - # thread can re-enter is a callback (the log handler) firing inside a - # native call, and letting that start a second compute on the same model - # mid-flight is exactly the race the lock exists to prevent — so a - # re-entrant attempt is detected via _compute_owner and raises instead. - # The abort callback only reads a threading.Event and never touches the - # lock, so cancel() stays lock-free. - # - # Native frees never wait for the lock (they run from close() and from - # __del__, which GC may invoke on a thread that already holds it, e.g. - # mid copy-out). A free is queued on _deferred and executed by whichever - # thread holds the lock — immediately if it is free, else by the holder - # just before it releases. FIFO order keeps the C contract that sessions - # are freed before their model. - # - # The stream lease: _stream_lease.owner is the _StreamLease of the stream - # that is active on this model, or None. It is claimed under the lock - # only when a native stream begin succeeds, and read and cleared only - # under the lock (in an _exclusive body or a deferred closure). A stream - # releases it only if it is still the owner, so a stream that already - # ended can never clear a lease a later stream took. The deferred - # closures capture the slot, never the Model, like the native frees. + # One plain Lock per model serializes native compute calls and copy-out. + # Not reentrant: the log callback can fire inside a native call, so a + # nested compute is detected via _compute_owner and raises instead. + # Native frees never wait: they queue FIFO on _deferred (sessions before + # their model) and the lock holder runs them before releasing. + # _stream_owner (the active stream's _StreamLease) is written only under it. def _init_compute_state(self) -> None: self._compute_lock = threading.Lock() self._compute_owner: Optional[int] = None # thread ident while held self._deferred: collections.deque = collections.deque() - self._stream_lease = _StreamLeaseSlot() + self._stream_owner: Optional[_StreamLease] = None @contextmanager def _exclusive(self, kind: str, busy: Optional[str] = None): - """Hold the model-wide compute lock for one compute call of *kind* - (blocking until it is free) and check the model is still open. Run - the native call AND the copy-out of its results inside the block, - and capture the session handle inside it too. - - With *busy* set (run, run_batch, stream begin), raise Busy(*busy*) - once the lock is held if a stream holds this model's lease. The - stream that holds the lease passes None for its own calls.""" + """Hold the compute lock for one call of *kind* and check the model + is open; run the native call and its copy-out inside the block. With + *busy* set, raise Busy(*busy*) if a stream holds the lease.""" me = threading.get_ident() if self._compute_owner == me: raise TranscribeError( @@ -1090,9 +1032,8 @@ def _exclusive(self, kind: str, busy: Optional[str] = None): self._compute_lock.acquire() self._compute_owner = me try: - if self._handle is None: - raise TranscribeError("model is closed") - if busy is not None and self._stream_lease.owner is not None: + self._h # raises if the model is closed + if busy is not None and self._stream_owner is not None: raise Busy(busy) yield finally: @@ -1107,21 +1048,15 @@ def _exclusive(self, kind: str, busy: Optional[str] = None): def _drain_deferred_locked(self) -> None: """Run queued native frees. Caller holds the compute lock.""" - while True: - try: - free = self._deferred.popleft() - except IndexError: - return + while self._deferred: try: - free() + self._deferred.popleft()() except Exception: pass # a native free cannot meaningfully fail def _try_drain(self) -> None: - """Run queued frees now if the lock is free; otherwise leave them to - the current holder, which drains before releasing. Never blocks. - Re-checks after releasing so a free queued in the gap between the - holder's drain and its release is never stranded.""" + """Run queued frees now if the lock is free, else leave them to the + holder. Never blocks; loops so a free queued during release is run.""" while self._deferred: if not self._compute_lock.acquire(blocking=False): return @@ -1133,9 +1068,8 @@ def _try_drain(self) -> None: self._compute_lock.release() def _free_or_defer(self, free) -> None: - """Run the native *free* behind any in-flight compute call without - waiting for it. *free* must not reference the Session/Model object - (it may run after that object is gone).""" + """Run *free* behind any in-flight compute call without waiting for + it. *free* must not reference the Session (it may already be gone).""" self._deferred.append(free) self._try_drain() @@ -1249,11 +1183,7 @@ def close(self) -> None: """Free the model. Any session still open on it is closed first — the C contract forbids freeing a model before its sessions, so this keeps explicit close()/context-manager exit safe in any order. - - Never waits for the compute lock: if a call is in flight on one of - the sessions, the model is marked closed now and the native frees - (sessions first, then the model) run as soon as that call ends. - Closing the session of an active stream releases the stream lease.""" + Never waits for the compute lock (see above).""" handle = getattr(self, "_handle", None) if handle is None: return @@ -1262,13 +1192,7 @@ def close(self) -> None: self._handle = None for session in list(getattr(self, "_sessions", ()) or ()): session.close() - slot = self._stream_lease - - def free(handle=handle, slot=slot): - _lib.transcribe_model_free(handle) - slot.owner = None # every session is gone; nothing can stream - - self._free_or_defer(free) + self._free_or_defer(lambda: _lib.transcribe_model_free(handle)) def __enter__(self) -> "Model": return self @@ -1283,49 +1207,26 @@ def __del__(self): pass -class Session: - """A single transcription context bound to one Model. +_S = TypeVar("_S", bound="_SessionBase") - Compute calls (``run``, ``run_batch``, ``stream`` and the Stream's - ``feed`` / ``finalize`` / ``reset``) take the model-wide compute lock, - so calls on sessions of one model — from any threads — serialize: a - call made while another is in flight waits for it (see ``Model``). - While a stream is active on ANY session of the model (this one - included), ``run``, ``run_batch`` and ``stream`` raise :class:`Busy` - instead; finalize or reset the stream first. Reads (``limits``, - ``was_aborted``, ``Stream.text()`` / ``snapshot()`` / ``state`` ...) do - not take the lock; don't read a session from one thread while another - thread is computing on that same session. ``cancel()`` is safe from any - thread.""" - def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", - n_ctx: int = 0): - self._model = model # keep the model alive for the session's lifetime - params = _SessionParams() - _lib.transcribe_session_params_init(_byref(params)) - params.n_threads = n_threads - params.kv_type = _enum(_KV_TYPES, kv_type, "kv_type") - params.n_ctx = n_ctx +class _SessionBase: + """Handle, cancellation and close plumbing shared by Session and + DiarizeSession. Subclasses name their native free in ``_free_fn``.""" - handle = ctypes.c_void_p() - status = _lib.transcribe_session_init(model._h, _byref(params), _byref(handle)) - _check(status, "opening session") - if not handle.value: - raise TranscribeError("session init returned a null handle") - self._handle = handle + _free_fn = "" - # Cancellation: an abort callback polled at chunk/decode boundaries - # returns True once cancel() is requested, so another thread can abort an - # in-flight run/stream. Bind the Event (not self) into the trampoline to - # avoid a reference cycle that would defer __del__. + def _arm_abort(self, set_cb) -> None: + """Install the abort callback (polled at chunk/decode boundaries; + True once cancel() is requested) and register with the model. Binds + the Event, not self, into the trampoline to avoid a reference cycle.""" self._cancel = threading.Event() - _event = self._cancel - self._abort_trampoline = _ABORT_CFUNC(lambda _ud: _event.is_set()) - _lib.transcribe_set_abort_callback(self._handle, self._abort_trampoline, None) - - # Registered last, once the session is fully constructed: Model.close() - # closes any session still tracked here before freeing the model. - model._sessions.add(self) + event = self._cancel + self._abort_trampoline = _ABORT_CFUNC(lambda _ud: event.is_set()) + set_cb(self._handle, self._abort_trampoline, None) + # Registered last, once fully constructed: Model.close() closes any + # session still tracked here before freeing the model. + self._model._sessions.add(self) @property def _h(self) -> ctypes.c_void_p: @@ -1333,9 +1234,17 @@ def _h(self) -> ctypes.c_void_p: raise TranscribeError("session is closed") return self._handle + def _attach_family(self, family: "FamilyExtension | None", slot: str, params) -> None: + """Resolve *family* for *slot* and point ``params.family`` at it (the + params struct keeps the built extension alive).""" + if family is not None: + ext = self._resolve_family(family, slot) + params.family = ctypes.cast(_byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + params._family_keepalive = ext + def _resolve_family(self, family: "FamilyExtension", slot: str): """Validate + probe a family extension and return its built ctypes - struct (which the caller keeps alive and points params.family at).""" + struct.""" if not isinstance(family, FamilyExtension): raise InvalidArgument("family must be a FamilyExtension instance") if family._slot != slot: @@ -1349,6 +1258,73 @@ def _resolve_family(self, family: "FamilyExtension", slot: str): f"{slot!r} slot") return family._build() + def cancel(self) -> None: + """Request cancellation of an in-flight call from another thread: it + aborts at the next chunk/decode boundary and raises ``Aborted`` (with + ``partial_result`` where available). The flag is cleared when the next + run / run_batch / stream / diarize run starts, before it waits for the + lock, so a cancel() while queued still aborts it. Lock-free.""" + self._cancel.set() + + def close(self) -> None: + """Free the session. Idempotent; never waits for the compute lock + (see ``Model``).""" + handle = getattr(self, "_handle", None) + if handle is None: + return + self._handle = None + # The deferred free pins the abort trampoline the native session + # points at, and never references self (close() may run from __del__). + trampoline = getattr(self, "_abort_trampoline", None) + free_fn, model = self._free_fn, self._model + + def free(_pin=trampoline): + getattr(_lib, free_fn)(handle) + # Release this session's stream lease only after the free, in the + # same locked step, so a queued run cannot start before it. + if getattr(model._stream_owner, "handle", None) is handle: + model._stream_owner = None + + model._free_or_defer(free) + + def __enter__(self: _S) -> _S: + return self + + def __exit__(self, *exc) -> None: + self.close() + + def __del__(self): + try: + self.close() + except Exception: + pass + + +class Session(_SessionBase): + """A single transcription context bound to one Model. Compute locking and + ``Busy`` rules: see ``Model``. Reads (``limits``, ``was_aborted``, Stream + ``text()`` ...) are unlocked; don't read a session while another thread + computes on it.""" + + _free_fn = "transcribe_session_free" + + def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", + n_ctx: int = 0): + self._model = model # keep the model alive for the session's lifetime + params = _SessionParams() + _lib.transcribe_session_params_init(_byref(params)) + params.n_threads = n_threads + params.kv_type = _enum(_KV_TYPES, kv_type, "kv_type") + params.n_ctx = n_ctx + + handle = ctypes.c_void_p() + status = _lib.transcribe_session_init(model._h, _byref(params), _byref(handle)) + _check(status, "opening session") + if not handle.value: + raise TranscribeError("session init returned a null handle") + self._handle = handle + self._arm_abort(_lib.transcribe_set_abort_callback) + def run(self, pcm: PCMLike, *, task: Task = "transcribe", language: str | None = None, target_language: str | None = None, @@ -1395,10 +1371,7 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", with self._model._exclusive( "run", busy="a stream is active on this model; " "finish or drop it before run()"): - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + self._attach_family(family, "run", params) h = self._h # captured under the lock; close() defers its free try: _check(_lib.transcribe_run(h, array, n_samples, _byref(params)), @@ -1467,10 +1440,7 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", with self._model._exclusive( "run_batch", busy="a stream is active on this model; " "finish or drop it before run_batch()"): - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + self._attach_family(family, "run", params) h = self._h # captured under the lock; close() defers its free batch_abort = None try: @@ -1534,11 +1504,8 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, is reset when you are done. ``vocabulary`` and ``prompt`` are as in :meth:`run`; ``task="instruct"`` raises. - A successful begin takes the model's stream lease: until this stream - is finalized, reset, fails, or is garbage-collected (or its session - or model is closed), ``run()``, ``run_batch()`` and ``stream()`` on - any session of the model raise :class:`Busy`. A begin while another - stream on the model is active raises :class:`Busy` too.""" + Takes the model's stream lease (see ``Model``); raises :class:`Busy` + if another stream is active on this model.""" self._cancel.clear() # before the lock wait, as in run() # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). @@ -1551,39 +1518,29 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, sp.stable_prefix_agreement_n = stable_prefix_agreement_n with self._model._exclusive( "stream_begin", busy="a stream is already active on this model"): - ext = self._resolve_family(family, "stream") if family is not None else None - if ext is not None: - sp.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + self._attach_family(family, "stream", sp) h = self._h _check( _lib.transcribe_stream_begin(h, _byref(run_params), _byref(sp)), "transcribe_stream_begin", ) - # Claim the lease for the whole stream lifetime, only now that the - # native begin succeeded. Released at finalize/reset, by a feed - # that ends the stream, by session/model close, or by Stream GC. + # Claim the lease only now that the native begin succeeded. lease = _StreamLease(h) - self._model._stream_lease.owner = lease + self._model._stream_owner = lease # The C contract says everything passed to begin may be freed once # it returns (strings are copied into session-owned storage). The # Stream still pins the params structs until reset() as defense in # depth — it costs nothing and keeps the binding safe even against # an older or out-of-tree native library that predates that - # contract. Built under the lock, so the lease always has a Stream - # that can release it. - return Stream(self, _keepalive=(run_params, sp, ext), _lease=lease) - - def _materialize(self, h: ctypes.c_void_p | None = None, - utt: int | None = None) -> Result: - """Copy out one result. A compute call's copy-out passes the session - handle *h* it captured under the compute lock (so a concurrent close() - cannot pull it away mid-copy); None reads this session's live handle. + # contract. Built under the lock, so the lease always has a Stream. + return Stream(self, _keepalive=(run_params, sp), _lease=lease) + + def _materialize(self, h: ctypes.c_void_p, utt: int | None = None) -> Result: + """Copy out one result from session handle *h* (captured under the + compute lock, so a concurrent close() cannot pull it away mid-copy). utt is None for the single-result accessors, or an utterance index for the batch accessors (index 0 aliases the single result after a plain run, so both paths share this code).""" - if h is None: - h = self._h if utt is None: n_seg = lambda: _lib.transcribe_n_segments(h) get_seg = lambda j, out: _lib.transcribe_get_segment(h, j, out) @@ -1672,61 +1629,11 @@ def limits(self) -> SessionLimits: max_kv_bytes=lm.max_kv_bytes, ) - def cancel(self) -> None: - """Request cancellation of an in-flight run/stream from another thread. - The active call aborts at the next chunk/decode boundary and raises - ``Aborted`` (with ``partial_result`` attached); ``was_aborted`` then - reports True. The flag is cleared when the next run/run_batch/stream - call starts — before it waits for the model's compute lock, so a - cancel() issued while that call is queued behind another session's - call aborts it as soon as it begins. Lock-free; safe from any - thread.""" - self._cancel.set() - @property def was_aborted(self) -> bool: """True if the most recent call was ended by cancel().""" return bool(_lib.transcribe_was_aborted(self._h)) - def close(self) -> None: - """Free the session. Idempotent. Never waits for the compute lock: if - a call is in flight on this model, the session is marked closed now - and the native free runs as soon as that call ends. If a stream on - this session holds the model's stream lease, the lease is released - right after the native free.""" - handle = getattr(self, "_handle", None) - if handle is None: - return - self._handle = None - # The abort trampoline must outlive the native session that points at - # it, so the deferred free pins it. Nothing here references self: the - # free may run after this object is gone (close() from __del__). - trampoline = getattr(self, "_abort_trampoline", None) - slot = self._model._stream_lease - - def free(handle=handle, _pin=trampoline, slot=slot): - _lib.transcribe_session_free(handle) - # Release the lease only after the free, in the same locked step: - # a run queued behind this must not start while the stream's - # native session still exists. Only this session's stream's lease. - owner = slot.owner - if owner is not None and owner.handle is handle: - slot.owner = None - - self._model._free_or_defer(free) - - def __enter__(self) -> "Session": - return self - - def __exit__(self, *exc) -> None: - self.close() - - def __del__(self): - try: - self.close() - except Exception: - pass - class Stream: """An active streaming transcription on a Session. @@ -1735,34 +1642,24 @@ class Stream: ``text()``, and call ``finalize()`` when the audio ends. The session is returned to idle on context-manager exit (or ``reset()``). - While the stream is active it holds its model's stream lease: ``run()``, - ``run_batch()`` and ``stream()`` on every session of the model raise - :class:`Busy` (see ``Model``). ``finalize()`` and ``reset()`` release - it, and so does a ``feed()`` failure that leaves the stream no longer - active (``state`` is then ``"failed"``). A feed rejected before the - model sees it (e.g. NaN/Inf samples, ``InvalidArgument``) keeps the - stream ``"active"`` and keeps the lease. A Stream garbage-collected - while active is reset and releases the lease, behind any in-flight - call.""" - - def __init__(self, session: Session, *, _keepalive=None, _lease=None): + While active it holds the model's stream lease (see ``Model``); a feed + rejected before the model sees it (e.g. NaN/Inf samples) keeps it.""" + + def __init__(self, session: Session, *, _lease: _StreamLease, _keepalive=None): self._session = session # keep the session (and its model) alive # Pins the ctypes params structs (and any family ext) passed to # transcribe_stream_begin until reset(). The native library copies # what it needs at begin; this is belt-and-braces for the FFI layer. self._keepalive = _keepalive self._active = True - # This stream's claim on the model's stream lease (None: holds none). - # It holds the lease while model._stream_lease.owner is this object. - self._lease: Optional[_StreamLease] = _lease + self._lease = _lease # holds the lease while model._stream_owner is this def _release_lease_locked(self) -> None: - """Release the model's stream lease if THIS stream still holds it. - Caller holds the compute lock. A lease another stream has taken since - is left alone.""" - slot = self._session._model._stream_lease - if self._lease is not None and slot.owner is self._lease: - slot.owner = None + """Release the model's stream lease if THIS stream still holds it + (never a later stream's). Caller holds the compute lock.""" + model = self._session._model + if model._stream_owner is self._lease: + model._stream_owner = None @property def _h(self) -> ctypes.c_void_p: @@ -1775,15 +1672,11 @@ def feed(self, pcm: PCMLike) -> StreamUpdate: array, n_samples = _pcm_to_carray(pcm) update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) - # No busy check: this stream holds the lease (or has none to take). - with self._session._model._exclusive("stream_feed"): + with self._session._model._exclusive("stream_feed"): # no busy check h = self._h status = _lib.transcribe_stream_feed(h, array, n_samples, _byref(update)) - # The lease follows the native stream lifecycle. A feed rejected - # before the family hook (non-finite samples, other pre-hook - # INVALID_ARG) leaves the stream ACTIVE: keep the lease, so the - # stream stays usable and other calls stay Busy. A failure that - # leaves it no longer ACTIVE (FAILED) ends the stream: release. + # A failure that leaves the stream no longer ACTIVE ends it: release + # the lease. A pre-hook rejection (e.g. NaN) keeps it ACTIVE. if (status != _generated.TRANSCRIBE_OK and _lib.transcribe_stream_get_state(h) != _generated.TRANSCRIBE_STREAM_ACTIVE): @@ -1820,8 +1713,7 @@ def text(self) -> StreamText: def snapshot(self) -> Result: """Full structured snapshot of the current hypothesis (owned copies).""" - _ = self._h # validate that this stream has not been reset - return self._session._materialize() + return self._session._materialize(self._h) @property def state(self) -> str: @@ -1861,39 +1753,28 @@ def __exit__(self, *exc) -> None: self.reset() def __del__(self): - # Abandoned while still holding the lease (never finalized or reset): - # reset the native stream and release the lease. Never waits for the - # lock, like the native frees: GC may run this on the thread that - # holds it. The closure captures the lease slot and token, not self - # or the Session/Model. Reading the owner here without the lock is - # safe: a stream's lease is only ever released, never re-taken, so - # "not the owner" is final; the closure re-checks under the lock. + # Abandoned while holding the lease: reset and release it behind any + # in-flight call (never waits). "Not the owner" is final; re-checked locked. try: - lease = self._lease - if lease is None: - return - model = self._session._model - slot = model._stream_lease - if slot.owner is not lease: + lease, model = self._lease, self._session._model + if model._stream_owner is not lease: return - def reset_and_release(slot=slot, lease=lease): - # Still the owner means the session is still open: closing it - # clears the owner in the same deferred step as its free. - if slot.owner is lease: + def reset_and_release(): + if model._stream_owner is lease: # so the session is still open _lib.transcribe_stream_reset(lease.handle) - slot.owner = None + model._stream_owner = None model._free_or_defer(reset_and_release) except Exception: pass -class DiarizeSession: +class DiarizeSession(_SessionBase): """A diarization (who spoke when) context on a model with the DIARIZE - role. ``run()`` follows the Session rules (see ``Model``): it takes the - model-wide compute lock, raises :class:`Busy` while a stream is active - on the model, and ``cancel()`` / ``close()`` never wait for the lock.""" + role. Compute locking and ``Busy`` rules: see ``Model``.""" + + _free_fn = "transcribe_diarize_session_free" def __init__(self, model: Model, *, n_threads: int = 0): self._model = model # keep the model alive for the session's lifetime @@ -1907,20 +1788,7 @@ def __init__(self, model: Model, *, n_threads: int = 0): if not handle.value: raise TranscribeError("diarize session init returned a null handle") self._handle = handle - - self._cancel = threading.Event() # as in Session: bind the Event, not self - _event = self._cancel - self._abort_trampoline = _ABORT_CFUNC(lambda _ud: _event.is_set()) - _lib.transcribe_diarize_set_abort_callback(self._handle, self._abort_trampoline, None) - model._sessions.add(self) - - _h = Session._h - _resolve_family = Session._resolve_family - - def cancel(self) -> None: - """Abort an in-flight (or queued) ``run()``, which raises - :class:`Aborted`. Lock-free; safe from any thread.""" - self._cancel.set() + self._arm_abort(_lib.transcribe_diarize_set_abort_callback) def run(self, pcm: PCMLike, *, family: FamilyExtension | None = None) -> list[SpeakerSegment]: @@ -1938,10 +1806,7 @@ def run(self, pcm: PCMLike, *, with self._model._exclusive( "diarize_run", busy="a stream is active on this model; " "finish or drop it before diarize run()"): - ext = self._resolve_family(family, "diarize_run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + self._attach_family(family, "diarize_run", params) h = self._h # captured under the lock; close() defers its free _check(_lib.transcribe_diarize_run(h, array, n_samples, _byref(params)), "transcribe_diarize_run") @@ -1964,24 +1829,6 @@ def timings(self) -> Timings: "transcribe_diarize_get_timings") return _timings_from(tm) - def close(self) -> None: - """Free the session. Idempotent; never waits for the compute lock - (the free runs right after an in-flight call, as in Session).""" - handle = getattr(self, "_handle", None) - if handle is None: - return - self._handle = None - trampoline = getattr(self, "_abort_trampoline", None) - - def free(handle=handle, _pin=trampoline): - _lib.transcribe_diarize_session_free(handle) - - self._model._free_or_defer(free) - - __enter__ = Session.__enter__ - __exit__ = Session.__exit__ - __del__ = Session.__del__ - def transcribe( model: Model | str | os.PathLike, diff --git a/bindings/python/src/transcribe_cpp/errors.py b/bindings/python/src/transcribe_cpp/errors.py index 3dfcac54d..236e94c67 100644 --- a/bindings/python/src/transcribe_cpp/errors.py +++ b/bindings/python/src/transcribe_cpp/errors.py @@ -101,15 +101,8 @@ class InputTooLong(TranscribeError): class Busy(TranscribeError): """A stream is active on this model, so the call was refused instead of - started. - - The C library allows one run / batch / active stream per model across - ALL of its sessions, and an active stream occupies the model from begin - until it ends, not just during a feed. While one is active, - ``Session.run()``, ``run_batch()`` and ``stream()`` on any session of - that model raise this at once. Finalize or reset the stream first, or - give each concurrent worker its own Model. Raised purely on the Python - side, so ``status`` is 0.""" + started (see ``Model``). Finalize or reset the stream first, or use one + Model per worker. Raised by the binding, so ``status`` is 0.""" class Aborted(TranscribeError): diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index 0855f439e..425e937a8 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -1,18 +1,7 @@ -"""Model-wide compute lock and stream lease: deterministic tests that need NO -model. - -The native library allows one compute call in flight per model across all of -its sessions, and an active stream occupies the model from begin until it -ends. The binding enforces that with ``Model._exclusive(kind)`` and the -model-wide stream lease (``Model._stream_lease``). These -tests build Model/Session objects around fake handles and replace the native -entry points the high-level code calls (``transcribe_cpp._lib.``, looked -up at call time) with Python probes, so overlap, waiting, deferred frees and -re-entrancy are observed directly — no GGUF, no native compute. - -Every native call that could touch a fake handle is patched, and the fixture -closes every fake (and collects garbage) BEFORE monkeypatch restores the real -functions, so a fake handle never reaches the real library. +"""Model-wide compute lock and stream lease (see ``Model``), with NO model: +Model/Session objects wrap fake handles and the native entry points are +replaced by Python probes. The fixture closes every fake before monkeypatch +restores the real functions, so a fake handle never reaches the library. """ from __future__ import annotations @@ -75,8 +64,7 @@ def fake(monkeypatch): # The copy-out reads many native accessors; stub it to report which # handle it was given (proves the in-flight call kept its handle). monkeypatch.setattr(t.Session, "_materialize", - lambda self, h=None, utt=None: - ("result", (h or self._h).value, utt)) + lambda self, h, utt=None: ("result", h.value, utt)) def model(): m = t.Model.__new__(t.Model) @@ -86,8 +74,8 @@ def model(): made.append(m) return m - def session(m, track=True): - s = t.Session.__new__(t.Session) + def session(m, cls=t.Session, track=True): + s = cls.__new__(cls) s._model = m s._handle = ctypes.c_void_p(next(ids)) s._cancel = threading.Event() @@ -97,17 +85,7 @@ def session(m, track=True): made.append(s) return s - def diarize(m): - d = t.DiarizeSession.__new__(t.DiarizeSession) - d._model = m - d._handle = ctypes.c_void_p(next(ids)) - d._cancel = threading.Event() - d._abort_trampoline = None - m._sessions.add(d) - made.append(d) - return d - - yield SimpleNamespace(model=model, session=session, diarize=diarize, + yield SimpleNamespace(model=model, session=session, frees=frees, resets=resets, state=state) for obj in reversed(made): @@ -117,6 +95,7 @@ def diarize(m): PCM = [0.0] * 160 +SESSION_CLS = {"session": t.Session, "diarize": t.DiarizeSession} class _LockSpy: @@ -161,8 +140,7 @@ def __call__(self, *args): def test_calls_on_two_sessions_never_overlap(fake, monkeypatch): m = fake.model() - s1, s2, s3 = fake.session(m), fake.session(m), fake.session(m) - stream = t.Stream(s3) + sessions = [fake.session(m) for _ in range(3)] guard = threading.Lock() state = {"active": 0, "max": 0, "calls": 0} @@ -178,7 +156,6 @@ def probe(*args): return 0 monkeypatch.setattr(t._lib, "transcribe_run", probe) - monkeypatch.setattr(t._lib, "transcribe_stream_feed", probe) start = threading.Barrier(3) n = 5 @@ -187,48 +164,13 @@ def runs(session): start.wait(TIMEOUT) return [session.run(PCM) for _ in range(n)] - def feeds(): - start.wait(TIMEOUT) - return [stream.feed(PCM) for _ in range(n)] - - workers = [_in_thread(runs, s1), _in_thread(runs, s2), _in_thread(feeds)] + workers = [_in_thread(runs, s) for s in sessions] for th, _ in workers: _join(th, "worker") - for _, box in workers: - assert "error" not in box, box["error"] - assert state["calls"] == 3 * n assert state["max"] == 1, "compute calls on one model overlapped" - assert workers[0][1]["value"] == [("result", s1._handle.value, None)] * n - assert workers[1][1]["value"] == [("result", s2._handle.value, None)] * n - - -def test_contended_call_waits_instead_of_raising(fake, monkeypatch): - m = fake.model() - s1, s2 = fake.session(m), fake.session(m) - gate = _Gate() - second_entered = threading.Event() - - def probe(h, *rest): - if h.value == s1._handle.value: - return gate(h) - second_entered.set() - return 0 - - monkeypatch.setattr(t._lib, "transcribe_run", probe) - try: - a, a_box = _in_thread(s1.run, PCM) - assert gate.entered.wait(TIMEOUT) - b, b_box = _in_thread(s2.run, PCM) - # B must be parked on the lock: not in native, not failed, not done. - assert not second_entered.wait(0.3), "second call overlapped the first" - assert b.is_alive() and not b_box, f"contended call did not wait: {b_box}" - finally: - gate.release.set() - _join(a, "first run") - _join(b, "second run") - assert "error" not in a_box and "error" not in b_box, (a_box, b_box) - assert b_box["value"] == ("result", s2._handle.value, None) + for s, (_, box) in zip(sessions, workers): + assert box.get("value") == [("result", s._handle.value, None)] * n, box def test_lock_is_per_model(fake, monkeypatch): @@ -272,7 +214,7 @@ def probe(*args): holds("transcribe_batch_n_results", ret=1)) real_materialize = t.Session._materialize - def materialize(self, h=None, utt=None): + def materialize(self, h, utt=None): seen.append(("copy-out", m._compute_lock.locked())) return real_materialize(self, h, utt) @@ -311,11 +253,13 @@ def test_lock_released_after_native_error(fake, monkeypatch): # --- cancellation ------------------------------------------------------------- -def test_cancel_while_queued_is_kept(fake, monkeypatch): +@pytest.mark.parametrize("kind", ["session", "diarize"]) +def test_cancel_while_queued_is_kept(fake, diarize_native, monkeypatch, kind): # cancel() is lock-free and the flag is cleared BEFORE the lock wait, so # a cancel issued while a call is queued is still set when it starts. m = fake.model() - s1, s2 = fake.session(m), fake.session(m) + s1, s2 = fake.session(m), fake.session(m, cls=SESSION_CLS[kind]) + diarize_native.owner["model"] = m gate = _Gate() flag_at_start: list = [] @@ -326,6 +270,7 @@ def probe(h, *rest): return 0 monkeypatch.setattr(t._lib, "transcribe_run", probe) + monkeypatch.setattr(t._lib, "transcribe_diarize_run", probe) try: a, _ = _in_thread(s1.run, PCM) assert gate.entered.wait(TIMEOUT) @@ -343,12 +288,17 @@ def probe(h, *rest): # --- close / GC during an in-flight call ------------------------------------------ -def test_session_close_during_in_flight_call_defers_free(fake, monkeypatch): +@pytest.mark.parametrize("kind", ["session", "diarize"]) +def test_session_close_during_in_flight_call_defers_free(fake, diarize_native, + monkeypatch, kind): m = fake.model() - s = fake.session(m) + s = fake.session(m, cls=SESSION_CLS[kind]) + diarize_native.owner["model"] = m handle = s._handle.value gate = _Gate() monkeypatch.setattr(t._lib, "transcribe_run", gate) + monkeypatch.setattr(t._lib, "transcribe_diarize_run", + lambda *a: gate() or diarize_native.run(*a)) try: a, a_box = _in_thread(s.run, PCM) assert gate.entered.wait(TIMEOUT) @@ -359,9 +309,11 @@ def test_session_close_during_in_flight_call_defers_free(fake, monkeypatch): gate.release.set() _join(a, "in-flight run") # The in-flight call finished its copy-out on the handle it captured... - assert a_box.get("value") == ("result", handle, None), a_box + expected = ROWS if kind == "diarize" else ("result", handle, None) + assert a_box.get("value") == expected, a_box + assert all(c[1] == handle for c in diarize_native.calls) # ...and the deferred free ran exactly once, after it. - assert fake.frees == [("session", handle)] + assert fake.frees == [(kind, handle)] with pytest.raises(t.TranscribeError, match="closed"): s.run(PCM) @@ -425,23 +377,6 @@ def probe(*args): assert fake.frees.count(victim) == 1 -def test_close_from_holder_thread_does_not_deadlock(fake, monkeypatch): - m = fake.model() - s, other = fake.session(m), fake.session(m) - hother = other._handle.value - - def probe(*args): - other.close() # same thread as the lock holder: must return at once - assert fake.frees == [] - return 0 - - monkeypatch.setattr(t._lib, "transcribe_run", probe) - th, box = _in_thread(s.run, PCM) - _join(th, "run closing another session from inside the call") - assert "error" not in box, box - assert fake.frees == [("session", hother)] - - def test_reentrant_compute_raises_instead_of_deadlocking(fake, monkeypatch): # e.g. a log handler that calls run() from inside a native call. m = fake.model() @@ -465,11 +400,6 @@ def probe(h, *rest): # --- stream lease --------------------------------------------------------------- -# -# From a successful stream begin until that stream ends, run / run_batch / -# stream on ANY session of the model raise Busy at once (the C contract -# forbids a run between the feeds of an active stream on another session). -# The stream itself keeps feeding. These mirror the Rust/Swift/TS bindings. RUN_BUSY = "a stream is active on this model; finish or drop it before run()" BATCH_BUSY = ("a stream is active on this model; " @@ -540,13 +470,6 @@ def test_active_stream_refuses_sibling_and_same_session_calls(fake, native): _assert_free(s1) -def test_busy_is_exported(): - assert "Busy" in t.__all__ - assert t.Busy is t.errors.Busy - assert issubclass(t.Busy, t.TranscribeError) - assert not issubclass(t.Busy, t.InvalidArgument) - - def test_lease_is_per_model(fake, native): s1 = fake.session(fake.model()) other = fake.session(fake.model()) @@ -560,13 +483,12 @@ def test_failed_begin_takes_no_lease(fake, native): native.status["begin"] = _generated.TRANSCRIBE_ERR_NOT_IMPLEMENTED with pytest.raises(t.NotImplementedByModel): s1.stream() - assert m._stream_lease.owner is None + assert m._stream_owner is None _assert_free(s2) @pytest.mark.parametrize("ending", [ - "finalize", "finalize_error", "reset", "context_exit", "gc", - "session_close", + "finalize", "finalize_error", "reset", "gc", "session_close", ]) def test_lease_released_when_stream_ends(fake, native, ending): m = fake.model() @@ -585,10 +507,6 @@ def test_lease_released_when_stream_ends(fake, native, ending): elif ending == "reset": stream.reset() assert fake.resets == [h1] - elif ending == "context_exit": - with stream: - pass - assert fake.resets == [h1] elif ending == "gc": stream._cycle = stream # only the cyclic GC can reclaim it del stream @@ -598,7 +516,7 @@ def test_lease_released_when_stream_ends(fake, native, ending): s1.close() assert fake.frees == [("session", h1)] - assert m._stream_lease.owner is None + assert m._stream_owner is None _assert_free(s2) if ending == "session_close": # The stream outlived its session: dropping it now must not reset @@ -613,7 +531,7 @@ def test_model_close_releases_lease(fake, native): s1 = fake.session(m) stream = s1.stream() m.close() - assert m._stream_lease.owner is None + assert m._stream_owner is None with pytest.raises(t.TranscribeError, match="closed"): stream.feed(PCM) del stream @@ -652,7 +570,7 @@ def test_feed_failure_that_ends_stream_releases_lease(fake, native): fake.state["value"] = _generated.TRANSCRIBE_STREAM_FAILED with pytest.raises(t.Aborted): stream.feed(PCM) - assert m._stream_lease.owner is None + assert m._stream_owner is None _assert_free(s2) @@ -739,13 +657,13 @@ def run(h, *rest): # Simulate a stream on victim_session that holds the lease and is # dropped mid-call (a real begin here would be a re-entrant call). lease = t._StreamLease(victim_session._handle) - m._stream_lease.owner = lease + m._stream_owner = lease victim = t.Stream(victim_session, _lease=lease) victim._cycle = victim del victim gc.collect() # runs Stream.__del__ on THIS thread, lock held during["resets"] = list(fake.resets) - during["owner"] = m._stream_lease.owner + during["owner"] = m._stream_owner return 0 monkeypatch.setattr(t._lib, "transcribe_run", run) @@ -755,7 +673,7 @@ def run(h, *rest): assert during["resets"] == [], "stream reset under the in-flight call" assert during["owner"] is not None, "lease released under the lock early" assert fake.resets == [hv] - assert m._stream_lease.owner is None + assert m._stream_owner is None monkeypatch.setattr(t._lib, "transcribe_run", lambda *a: 0) _assert_free(s) @@ -781,7 +699,7 @@ def run(h, *rest): a, a_box = _in_thread(stream.feed, PCM) assert gate.entered.wait(TIMEOUT) s1.close() # never waits; the free is queued behind the feed - assert fake.frees == [] and m._stream_lease.owner is not None + assert fake.frees == [] and m._stream_owner is not None spy = _LockSpy(m) b, b_box = _in_thread(s2.run, PCM) assert spy.waiting.wait(TIMEOUT) @@ -792,17 +710,13 @@ def run(h, *rest): assert "error" not in a_box, a_box assert b_box.get("value") == ("result", s2._handle.value, None), b_box assert seen["frees"] == [("session", h1)] - assert m._stream_lease.owner is None + assert m._stream_owner is None del stream gc.collect() assert fake.resets == [] -# --- DiarizeSession.run ----------------------------------------------------------- -# -# The DIARIZE role's run goes through the same Model._exclusive bridge as -# Session.run: model-wide lock, Busy under a stream lease, copy-out under the -# lock on the captured handle, deferred frees, lock-free cancel. +# --- DiarizeSession (close / cancel are parametrized into the tests above) ---- DIARIZE_BUSY = ("a stream is active on this model; " "finish or drop it before diarize run()") @@ -845,7 +759,7 @@ def get_segment(h, i, out): def test_diarize_run_and_copy_out_hold_the_lock(fake, diarize_native): m = fake.model() - d = fake.diarize(m) + d = fake.session(m, cls=t.DiarizeSession) diarize_native.owner["model"] = m assert d.run(PCM) == ROWS names = [c[0] for c in diarize_native.calls] @@ -885,29 +799,9 @@ def test_diarize_session_init_failure_raises_unsupported_role(fake, monkeypatch) assert not list(m._sessions) -def test_diarize_run_waits_for_session_run(fake, diarize_native, monkeypatch): - m = fake.model() - s, d = fake.session(m), fake.diarize(m) - diarize_native.owner["model"] = m - gate = _Gate() - monkeypatch.setattr(t._lib, "transcribe_run", gate) - try: - a, a_box = _in_thread(s.run, PCM) - assert gate.entered.wait(TIMEOUT) - spy = _LockSpy(m) - b, b_box = _in_thread(d.run, PCM) - assert spy.waiting.wait(TIMEOUT) # parked on the lock, not failed - assert b.is_alive() and not b_box and diarize_native.calls == [] - finally: - gate.release.set() - _join(a, "session run") - _join(b, "queued diarize run") - assert "error" not in a_box and b_box.get("value") == ROWS, (a_box, b_box) - - def test_diarize_run_busy_while_stream_active(fake, native, diarize_native): m = fake.model() - s, d = fake.session(m), fake.diarize(m) + s, d = fake.session(m), fake.session(m, cls=t.DiarizeSession) diarize_native.owner["model"] = m stream = s.stream() with pytest.raises(t.Busy) as ei: @@ -916,84 +810,3 @@ def test_diarize_run_busy_while_stream_active(fake, native, diarize_native): assert diarize_native.calls == [] and not m._compute_lock.locked() stream.finalize() assert d.run(PCM) == ROWS - - -def _gated_diarize_run(diarize_native, monkeypatch): - gate = _Gate() - - def run(*args): - gate() - return diarize_native.run(*args) - - monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) - return gate - - -def test_diarize_close_during_in_flight_run_defers_free(fake, diarize_native, - monkeypatch): - m = fake.model() - d = fake.diarize(m) - diarize_native.owner["model"] = m - hd = d._handle.value - gate = _gated_diarize_run(diarize_native, monkeypatch) - try: - a, a_box = _in_thread(d.run, PCM) - assert gate.entered.wait(TIMEOUT) - closer, _ = _in_thread(d.close) - _join(closer, "diarize close() during an in-flight run") - assert fake.frees == [], "diarize session freed under an in-flight run" - finally: - gate.release.set() - _join(a, "in-flight diarize run") - assert a_box.get("value") == ROWS, a_box - assert all(c[1] == hd for c in diarize_native.calls) # the captured handle - assert fake.frees == [("diarize", hd)] - with pytest.raises(t.TranscribeError, match="closed"): - d.run(PCM) - - -def test_model_close_during_in_flight_diarize_run_frees_session_first( - fake, diarize_native, monkeypatch): - m = fake.model() - d = fake.diarize(m) - diarize_native.owner["model"] = m - hd, hm = d._handle.value, m._handle.value - gate = _gated_diarize_run(diarize_native, monkeypatch) - try: - a, a_box = _in_thread(d.run, PCM) - assert gate.entered.wait(TIMEOUT) - closer, _ = _in_thread(m.close) - _join(closer, "model.close() during an in-flight diarize run") - assert fake.frees == [] - finally: - gate.release.set() - _join(a, "in-flight diarize run") - assert a_box.get("value") == ROWS, a_box - assert fake.frees == [("diarize", hd), ("model", hm)] - - -def test_diarize_cancel_while_queued_is_kept(fake, diarize_native, monkeypatch): - m = fake.model() - s, d = fake.session(m), fake.diarize(m) - diarize_native.owner["model"] = m - gate = _Gate() - flag_at_start: list = [] - - def run(*args): - flag_at_start.append(d._cancel.is_set()) - return diarize_native.run(*args) - - monkeypatch.setattr(t._lib, "transcribe_run", gate) - monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) - try: - a, _ = _in_thread(s.run, PCM) - assert gate.entered.wait(TIMEOUT) - spy = _LockSpy(m) - b, _ = _in_thread(d.run, PCM) - assert spy.waiting.wait(TIMEOUT) # cleared its flag and is parked - d.cancel() # must not block although the lock is held - finally: - gate.release.set() - _join(a, "session run") - _join(b, "queued diarize run") - assert flag_at_start == [True] diff --git a/bindings/python/tests/test_diarize.py b/bindings/python/tests/test_diarize.py index fa6e2e000..967af8d35 100644 --- a/bindings/python/tests/test_diarize.py +++ b/bindings/python/tests/test_diarize.py @@ -62,8 +62,6 @@ def test_diarize_run_golden_segments(sortformer_model_path, mix_pcm, preset): with model.diarize_session() as d: rows = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset=preset)) timings = d.timings - assert isinstance(rows, list) - assert all(isinstance(r, t.SpeakerSegment) for r in rows) assert all(math.isnan(r.p) for r in rows) # Sortformer has no per-turn p assert _turns(rows) == GOLDEN[preset] assert timings.encode_ms > 0 @@ -79,8 +77,6 @@ def test_bad_preset_rejected(sortformer_model_path, mix_pcm): class OutOfRange(t.SortformerDiarizeOptions): _presets = {**t.SortformerDiarizeOptions._presets, "bogus": 99} - with pytest.raises(ValueError, match="preset"): - t.SortformerDiarizeOptions(preset="bogus") # type: ignore[arg-type] with t.Model(sortformer_model_path) as model, model.diarize_session() as d: with pytest.raises(t.InvalidArgument): d.run(mix_pcm, family=OutOfRange(preset="bogus")) # type: ignore[arg-type] diff --git a/bindings/python/tests/test_errors.py b/bindings/python/tests/test_errors.py index 04e63b3c4..1c7b64d71 100644 --- a/bindings/python/tests/test_errors.py +++ b/bindings/python/tests/test_errors.py @@ -77,12 +77,7 @@ def test_output_repetition_is_an_output_truncated(): def test_unsupported_role_is_its_own_type(): # A role mismatch is a property of the loaded model, not of a run option: # it must not be caught by an UnsupportedRequest handler. - exc = errors.exception_for_status(errors.ERR_UNSUPPORTED_ROLE, "no role") - assert type(exc) is t.UnsupportedRole - assert not isinstance(exc, t.UnsupportedRequest) - assert exc.status == errors.ERR_UNSUPPORTED_ROLE == 20 - assert "UnsupportedRole" in t.__all__ - assert t.UnsupportedRole is errors.UnsupportedRole + assert not issubclass(t.UnsupportedRole, t.UnsupportedRequest) def test_exception_for_status_builds_without_raising(): @@ -100,7 +95,6 @@ def test_status_constants_come_from_generated_layer(): assert errors.ERR_ABORTED == _generated.TRANSCRIBE_ERR_ABORTED assert errors.ERR_OUTPUT_TRUNCATED == _generated.TRANSCRIBE_ERR_OUTPUT_TRUNCATED assert errors.OK == _generated.TRANSCRIBE_OK - assert errors.ERR_UNSUPPORTED_ROLE == _generated.TRANSCRIBE_ERR_UNSUPPORTED_ROLE # --- integration: real native calls raise the mapped classes ---------------- @@ -119,54 +113,6 @@ def test_junk_model_file_raises_model_load_error(tmp_path): t.Model(junk) -def _fake_model(monkeypatch): - """A Model around a fake handle (no GGUF). transcribe_model_free is - patched to a no-op; the caller must close() it while that patch is still - active so the fake handle never reaches the real library.""" - import ctypes - import weakref - - monkeypatch.setattr(t._lib, "transcribe_model_free", lambda h: None) - m = t.Model.__new__(t.Model) - m._sessions = weakref.WeakSet() - m._init_compute_state() - m._handle = ctypes.c_void_p(0x1000) - return m - - -def test_capabilities_on_model_without_asr_role_raises_unsupported_role( - monkeypatch, -): - # transcribe_model_get_capabilities returns UNSUPPORTED_ROLE for a model - # without the ASR role; Model.capabilities surfaces it as the typed error. - monkeypatch.setattr(t._lib, "transcribe_model_get_capabilities", - lambda h, caps: errors.ERR_UNSUPPORTED_ROLE) - m = _fake_model(monkeypatch) - try: - with pytest.raises(t.UnsupportedRole) as ei: - m.capabilities - assert ei.value.status == errors.ERR_UNSUPPORTED_ROLE - assert "reading capabilities" in str(ei.value) - finally: - m.close() - - -def test_session_on_model_without_asr_role_raises_unsupported_role(monkeypatch): - # transcribe_session_init returns UNSUPPORTED_ROLE (no handle) for a model - # without the ASR role; opening a session surfaces the typed error. - monkeypatch.setattr(t._lib, "transcribe_session_init", - lambda m, params, out: errors.ERR_UNSUPPORTED_ROLE) - m = _fake_model(monkeypatch) - try: - with pytest.raises(t.UnsupportedRole) as ei: - m.session() - assert ei.value.status == errors.ERR_UNSUPPORTED_ROLE - assert "opening session" in str(ei.value) - assert not list(m._sessions) - finally: - m.close() - - def test_invalid_spec_k_drafts_rejected_before_native_call(): from transcribe_cpp import _build_run_params diff --git a/bindings/python/tests/test_family_ext.py b/bindings/python/tests/test_family_ext.py index 2f8d13619..62cd8a2a0 100644 --- a/bindings/python/tests/test_family_ext.py +++ b/bindings/python/tests/test_family_ext.py @@ -80,8 +80,6 @@ def test_sortformer_unknown_preset_rejected(): def test_sortformer_diarize_preset_maps_to_enum_value(): - built = t.SortformerDiarizeOptions(preset="low_latency")._build() - assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY built = t.SortformerDiarizeOptions(preset="very_high_latency")._build() assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY # None keeps the init default (DEFAULT = GGUF-shipped cfg). diff --git a/bindings/python/tests/test_lifetime.py b/bindings/python/tests/test_lifetime.py index c6aecc487..fb6dffbef 100644 --- a/bindings/python/tests/test_lifetime.py +++ b/bindings/python/tests/test_lifetime.py @@ -76,20 +76,14 @@ def test_gc_order_session_keeps_model_alive(model_path, audio_pcm): # --- multi-session use of one model ------------------------------------------ # -# Natively, at most one compute call may be in flight across the sessions of -# one model — they share its compute backend and some family state, so -# overlapping runs race (observed: corrupted whisper decodes on CPU, -# command-buffer failures on Metal). The binding enforces that itself with a -# model-wide compute lock: concurrent calls from several threads serialize -# (they wait; only an active stream makes them raise Busy), so callers no -# longer need their own lock. The deterministic, model-free lock and stream -# lease tests live in test_compute_lock.py. +# Sessions of one model share its compute backend, so the binding serializes +# their calls with a model-wide lock (see Model). The model-free lock and +# stream lease tests live in test_compute_lock.py. def test_serial_sessions_across_threads(model_path, audio_pcm): # Two threads, each with its own session, runs serialized by a caller - # lock: the session-pool pattern written before the binding locked - # internally. Must keep working (the two locks nest without deadlock). + # lock: must keep working (it nests with the binding's lock). import threading lock = threading.Lock() @@ -160,9 +154,8 @@ def worker(tag, model): def test_concurrent_sessions_on_shared_model(model_path, audio_path): - # Was an xfail (overlapping runs raced and corrupted decodes) until the - # binding's model-wide compute lock: unsynchronized callers on two - # sessions of one model now wait for each other and both decode cleanly. + # Unsynchronized callers on two sessions of one model wait for each + # other and both decode cleanly. import os import subprocess import sys diff --git a/bindings/python/tests/test_prompting.py b/bindings/python/tests/test_prompting.py index ca08cbc09..51e3ebd44 100644 --- a/bindings/python/tests/test_prompting.py +++ b/bindings/python/tests/test_prompting.py @@ -113,4 +113,4 @@ def test_control_token_literal_rejected(model_path, audio_pcm): first = session.run(audio_pcm) with pytest.raises(t.InvalidArgument): session.run(audio_pcm, prompt="hello <|endoftext|>") - assert session._materialize().text == first.text + assert session._materialize(session._h).text == first.text diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index be3ba50c0..4f87ed21d 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -163,29 +163,6 @@ def test_second_stream_while_active_rejected(streaming_model_path, audio_pcm): assert stream.state == "active" -def test_run_while_stream_active_rejected(streaming_model_path, audio_pcm): - # While a stream is active, the stream lease refuses run/run_batch on - # ANY session of the model with Busy (the C contract forbids a run - # between another session's feeds), the stream stays usable, and once - # it is finalized a sibling run goes through. - with t.Model(streaming_model_path) as model, model.session() as session: - with model.session() as sibling, session.stream() as stream: - stream.feed(audio_pcm[:16000]) - for target in (session, sibling): - with pytest.raises(t.Busy, match=r"before run\(\)"): - target.run(audio_pcm[:16000]) - with pytest.raises(t.Busy, match=r"before run_batch\(\)"): - target.run_batch([audio_pcm[:16000]]) - for i in range(16000, len(audio_pcm), 16000): - stream.feed(audio_pcm[i : i + 16000]) - stream.finalize() - assert stream.state == "finished" - committed = stream.text().committed - ran = sibling.run(audio_pcm).text - assert "country" in committed.lower(), committed - assert "country" in ran.lower(), ran - - def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): # Non-finite PCM is rejected BEFORE the family hook: the feed raises # InvalidArgument but the native stream stays ACTIVE (only failures @@ -203,10 +180,6 @@ def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): assert not model._compute_lock.locked() with pytest.raises(t.Busy): sibling.run(audio_pcm[:16000]) - with pytest.raises(t.Busy): - sibling.stream() - with pytest.raises(t.Busy): - session.run(audio_pcm[:16000]) for i in range(16000, len(audio_pcm), 16000): stream.feed(audio_pcm[i : i + 16000]) update = stream.finalize() @@ -233,28 +206,16 @@ def test_hook_failure_feed_releases_lease(streaming_model_path, audio_pcm): assert "country" in ran.lower(), ran -@pytest.mark.parametrize("ending", ["reset", "gc", "session_close"]) -def test_stream_lease_released_without_finalize(streaming_model_path, audio_pcm, - ending): - # Abandoning an active stream (reset, dropping it, or closing its - # session) frees the model for other sessions. - import gc - +def test_stream_lease_released_without_finalize(streaming_model_path, audio_pcm): + # Closing the session of an active stream frees the model for others. with t.Model(streaming_model_path) as model, model.session() as sibling: session = model.session() stream = session.stream() stream.feed(audio_pcm[:16000]) with pytest.raises(t.Busy): sibling.run(audio_pcm[:16000]) - if ending == "reset": - stream.reset() - elif ending == "gc": - del stream - gc.collect() - else: - session.close() - ran = sibling.run(audio_pcm).text session.close() + ran = sibling.run(audio_pcm).text assert "country" in ran.lower(), ran From f2d39ef66bfa5e1653226502ed7207553b19cfea Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:04:42 +0800 Subject: [PATCH 43/51] review fixes: CI sortformer canary, deterministic concurrency test, restored tests, stale Python Stream, CLI diarize -o/--batch, Rust non_exhaustive --- .github/actions/fetch-canary/action.yml | 12 +++++- .../python/src/transcribe_cpp/__init__.py | 13 ++++-- bindings/python/tests/test_streaming.py | 25 +++++++++++ bindings/rust/transcribe-cpp/src/session.rs | 14 ++++++ bindings/rust/transcribe-cpp/src/types.rs | 8 ++++ .../rust/transcribe-cpp/tests/no_model.rs | 3 ++ bindings/typescript/test/common.mjs | 2 +- .../typescript/test/compute-rules.test.mjs | 28 +++++++++++- bindings/typescript/test/diarize.test.mjs | 20 ++++++++- docs/migrating-to-0.4.md | 7 +-- examples/cli/asr.cpp | 43 +++++++++++-------- examples/cli/cli.h | 9 +++- examples/cli/diarize.cpp | 16 +++++-- examples/cli/main.cpp | 2 +- tests/CMakeLists.txt | 1 + tests/cli_diarize_smoke.cmake | 23 +++++++++- tests/nonfinite_input_unit.cpp | 30 ++++++++++++- tests/sortformer_diarize_unit.cpp | 19 ++++++-- 18 files changed, 233 insertions(+), 42 deletions(-) diff --git a/.github/actions/fetch-canary/action.yml b/.github/actions/fetch-canary/action.yml index 5a6ee6cd1..ad1e83bd1 100644 --- a/.github/actions/fetch-canary/action.yml +++ b/.github/actions/fetch-canary/action.yml @@ -6,7 +6,8 @@ description: > secret): the model tests then skip, exactly as before this action existed. Always fetches ALL CI canaries (whisper-tiny + moonshine-streaming ~95 MB, Canary+SenseVoice text-control canaries ~290 MB, plus the two parakeet 0.6b - family-extension canaries — cache-aware + buffered — at Q4_K_M ~0.95 GB) + family-extension canaries — cache-aware + buffered — at Q4_K_M ~0.95 GB, + plus the Sortformer DIARIZE canary at Q8_0 ~140 MB) under one cache key — a per-consumer subset would let one job save the shared key with only its subset in it, and every other consumer would re-download forever (exact-key hits are never re-saved), which is also why the key carries @@ -35,7 +36,7 @@ runs: uses: actions/cache@v5 with: path: canary - key: canary-models-v3 + key: canary-models-v4 - name: Fetch canary models (cache miss only) if: inputs.hf-token != '' shell: bash @@ -67,6 +68,12 @@ runs: uvx --from huggingface_hub hf download \ handy-computer/parakeet-unified-en-0.6b-gguf \ parakeet-unified-en-0.6b-Q4_K_M.gguf --local-dir canary + # Sortformer DIARIZE canary. Q8_0 (~140 MB) reproduces the F32 CPU + # golden segments the binding diarize tests assert. + [ -f canary/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf ] || \ + uvx --from huggingface_hub hf download \ + handy-computer/diar_streaming_sortformer_4spk-v2.1-gguf \ + diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf --local-dir canary # GITHUB_WORKSPACE (not bash's $PWD) so Windows exports a path # Python can open rather than an MSYS one. prefix='${{ inputs.model-path-prefix }}' @@ -77,3 +84,4 @@ runs: echo "TRANSCRIBE_SMOKE_ITN_MODEL=$prefix/canary/SenseVoiceSmall-Q4_K_M.gguf" >> "$GITHUB_ENV" echo "TRANSCRIBE_SMOKE_PARAKEET_STREAM_MODEL=$prefix/canary/nemotron-speech-streaming-en-0.6b-Q4_K_M.gguf" >> "$GITHUB_ENV" echo "TRANSCRIBE_SMOKE_PARAKEET_BUFFERED_MODEL=$prefix/canary/parakeet-unified-en-0.6b-Q4_K_M.gguf" >> "$GITHUB_ENV" + echo "TRANSCRIBE_SMOKE_SORTFORMER_MODEL=$prefix/canary/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf" >> "$GITHUB_ENV" diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index c8c23c770..e3c85dff1 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -1311,6 +1311,8 @@ class Session(_SessionBase): def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0): self._model = model # keep the model alive for the session's lifetime + # Lease of the newest Stream begun here; an older Stream is superseded. + self._stream_lease: Optional[_StreamLease] = None params = _SessionParams() _lib.transcribe_session_params_init(_byref(params)) params.n_threads = n_threads @@ -1527,6 +1529,7 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, # Claim the lease only now that the native begin succeeded. lease = _StreamLease(h) self._model._stream_owner = lease + self._stream_lease = lease # The C contract says everything passed to begin may be freed once # it returns (strings are copied into session-owned storage). The # Stream still pins the params structs until reset() as defense in @@ -1665,6 +1668,8 @@ def _release_lease_locked(self) -> None: def _h(self) -> ctypes.c_void_p: if not self._active: raise TranscribeError("stream has been reset") + if self._session._stream_lease is not self._lease: + raise TranscribeError("stream superseded by a newer stream on this session") return self._session._h def feed(self, pcm: PCMLike) -> StreamUpdate: @@ -1738,11 +1743,13 @@ def last_status(self) -> TranscribeError | None: def reset(self) -> None: """Return the session to idle, discarding stream state, and release - the model's stream lease. Idempotent.""" + the model's stream lease. Idempotent; a no-op once a newer stream has + begun on the session.""" if self._active: with self._session._model._exclusive("stream_reset"): - _lib.transcribe_stream_reset(self._session._h) - self._release_lease_locked() + if self._session._stream_lease is self._lease: + _lib.transcribe_stream_reset(self._session._h) + self._release_lease_locked() self._active = False self._keepalive = None diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index 4f87ed21d..78d306d4b 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -132,6 +132,31 @@ def test_stream_use_after_reset_rejected(streaming_model_path, audio_pcm): stream.snapshot() +def test_superseded_stream_cannot_touch_the_new_one(streaming_model_path, audio_pcm): + # A finalized Stream stays readable until the session begins another; then + # it is superseded: feed raises and reset is a no-op on the new stream. + with t.Model(streaming_model_path) as model, model.session() as session: + old = session.stream() + old.feed(audio_pcm[:16000]) + old.finalize() + assert old.state == "finished" + + new = session.stream() + new.feed(audio_pcm[:16000]) + revision = new.revision + with pytest.raises(t.TranscribeError, match="superseded"): + old.feed(audio_pcm[16000:32000]) + with pytest.raises(t.TranscribeError, match="superseded"): + old.text() + old.reset() + assert new.state == "active" + assert new.revision == revision + with model.session() as sibling, pytest.raises(t.Busy): + sibling.run(audio_pcm[:16000]) # the new stream still holds the lease + new.finalize() + new.reset() + + def test_stream_reset_idempotent_and_session_reusable( streaming_model_path, audio_pcm): with t.Model(streaming_model_path) as model, model.session() as session: diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index bded8aa2a..4c2607893 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -517,6 +517,20 @@ fn attach_partial(err: Error, partial: Box) -> Error { /// [`Stream::text`], and end input with [`Stream::finalize`]. Dropping the /// `Stream` (without finalizing) abandons it and returns the session to idle. /// `Send` but not `Sync`, like the session it borrows. +/// +/// The borrow means a session can't be freed while its stream (or any other +/// call on it) is still in use, so no native free ever races a compute: +/// +/// ```compile_fail,E0505 +/// use transcribe_cpp::{Model, RunOptions, StreamOptions}; +/// let model = Model::load("model.gguf").unwrap(); +/// let mut session = model.session().unwrap(); +/// let mut stream = session +/// .stream(&RunOptions::default(), &StreamOptions::default()) +/// .unwrap(); +/// drop(session); // error[E0505]: cannot move out of `session` while borrowed +/// stream.feed(&[0.0; 160]).unwrap(); +/// ``` pub struct Stream<'a> { session: &'a mut Session, // True while this stream holds the model's compute lease. Set at begin, diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index f9088bb76..a4981cae7 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -303,6 +303,7 @@ impl StreamState { /// A public ABI struct, for size/alignment introspection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum AbiStruct { ModelLoadParams, SessionParams, @@ -319,6 +320,9 @@ pub enum AbiStruct { Ext, DeviceInfo, SpeakerSegment, + DiarizeInfo, + DiarizeSessionParams, + DiarizeParams, } impl AbiStruct { @@ -340,12 +344,16 @@ impl AbiStruct { AbiStruct::Ext => A::TRANSCRIBE_ABI_EXT, AbiStruct::DeviceInfo => A::TRANSCRIBE_ABI_DEVICE_INFO, AbiStruct::SpeakerSegment => A::TRANSCRIBE_ABI_SPEAKER_SEGMENT, + AbiStruct::DiarizeInfo => A::TRANSCRIBE_ABI_DIARIZE_INFO, + AbiStruct::DiarizeSessionParams => A::TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS, + AbiStruct::DiarizeParams => A::TRANSCRIBE_ABI_DIARIZE_PARAMS, } } } /// The slot a family extension is pointed at. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum ExtSlot { /// `transcribe_run_params::family`. diff --git a/bindings/rust/transcribe-cpp/tests/no_model.rs b/bindings/rust/transcribe-cpp/tests/no_model.rs index 307795aa8..c9c57c44d 100644 --- a/bindings/rust/transcribe-cpp/tests/no_model.rs +++ b/bindings/rust/transcribe-cpp/tests/no_model.rs @@ -34,6 +34,9 @@ fn abi_struct_sizes_are_live() { AbiStruct::Segment, AbiStruct::SpeakerSegment, AbiStruct::SessionLimits, + AbiStruct::DiarizeInfo, + AbiStruct::DiarizeSessionParams, + AbiStruct::DiarizeParams, ] { assert!(abi_struct_size(which) > 0, "{which:?} reported size 0"); } diff --git a/bindings/typescript/test/common.mjs b/bindings/typescript/test/common.mjs index 7dcaa9d13..de5e68e0e 100644 --- a/bindings/typescript/test/common.mjs +++ b/bindings/typescript/test/common.mjs @@ -22,7 +22,7 @@ export const ITN_MODEL = // Voxtral realtime is local-only (~2.5 GB+, too heavy for the CI canary set): // its env var is NOT exported by fetch-canary, so this skips cleanly in CI. export const VOXTRAL_MODEL = process.env.TRANSCRIBE_SMOKE_VOXTRAL_MODEL || ""; -// Sortformer (DIARIZE role only) is local-only too. +// Sortformer (DIARIZE role only): fetch-canary exports the Q8_0 canary. export const SORTFORMER_MODEL = process.env.TRANSCRIBE_SMOKE_SORTFORMER_MODEL || ""; export const SORTFORMER_AUDIO = path.resolve(HERE, "../../../samples/sortformer-2spk-mix.wav"); diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs index bcb4cd219..d1a523509 100644 --- a/bindings/typescript/test/compute-rules.test.mjs +++ b/bindings/typescript/test/compute-rules.test.mjs @@ -38,7 +38,7 @@ modelTest("compute is exclusive model-wide: a sibling run waits its turn", MODEL } }); -modelTest("runBatch marks the session in flight with its own label", STREAMING_MODEL, async () => { +modelTest("runBatch and finalize mark the session in flight with their own label", STREAMING_MODEL, async () => { const m = await TranscribeModel.load(STREAMING_MODEL); try { const s = m.createSession(); @@ -47,6 +47,16 @@ modelTest("runBatch marks the session in flight with its own label", STREAMING_M assert.throws(() => s.limits, /runBatch\(\).*in flight/); await pending; assert.doesNotThrow(() => s.limits); + + const stream = await s.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(second()); + const fin = stream.finalize(); + await Promise.resolve(); + assert.throws(() => stream.text, /feed\(\)\/finalize\(\).*in flight/); + assert.throws(() => s.wasAborted, /feed\(\)\/finalize\(\).*in flight/); + await fin; + assert.doesNotThrow(() => stream.text); + stream.reset(); s.dispose(); } finally { m.dispose(); @@ -295,6 +305,14 @@ modelTest("session dispose during an in-flight run defers the free; the result s assert.match(r.text, /ask not what your country/i); assert.ok(r.segments.length >= 1); assert.throws(() => s.limits, /disposed/); + + const s2 = m.createSession(); + const pb = s2.runBatch([jfk(), half()]); + await Promise.resolve(); + s2.dispose(); + const items = await pb; + assert.equal(items.length, 2); + assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); } finally { m.dispose(); } @@ -325,6 +343,14 @@ modelTest("model dispose during an in-flight run keeps the model alive for the c const r = await p; assert.match(r.text, /ask not what your country/i); assert.throws(() => s.limits, /disposed/); + + const m2 = await TranscribeModel.load(MODEL); + const s2 = m2.createSession(); + const pb = s2.runBatch([jfk()]); + await Promise.resolve(); + m2.dispose(); + const items = await pb; + assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); }); modelTest("model dispose during an in-flight feed keeps the model alive for the call", STREAMING_MODEL, async () => { diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs index eade7f13f..ff700be60 100644 --- a/bindings/typescript/test/diarize.test.mjs +++ b/bindings/typescript/test/diarize.test.mjs @@ -2,13 +2,31 @@ // rules a diarize run shares with Session (lock, in-flight mark, abort, // disposed recheck, deferred free). +import { test } from "node:test"; import assert from "node:assert/strict"; import { getEventListeners } from "node:events"; import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav } from "./common.mjs"; -import { TranscribeModel, Aborted, InvalidArgument, TranscribeError, UnsupportedRole } from "../dist/index.js"; +import { + TranscribeModel, + DiarizeSession, + Aborted, + InvalidArgument, + TranscribeError, + UnsupportedRole, +} from "../dist/index.js"; const mix = () => readWav(SORTFORMER_AUDIO); +// Invariant 11: a role session never exposes its native handle. +test("DiarizeSession exposes only its public surface", () => { + assert.deepEqual(Object.getOwnPropertyNames(DiarizeSession.prototype).sort(), [ + "constructor", + "dispose", + "run", + "timings", + ]); +}); + modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MODEL, async () => { const m = await TranscribeModel.load(MODEL); try { diff --git a/docs/migrating-to-0.4.md b/docs/migrating-to-0.4.md index a17a5ed94..cc15a5e54 100644 --- a/docs/migrating-to-0.4.md +++ b/docs/migrating-to-0.4.md @@ -19,7 +19,7 @@ them, and `transcribe_session_init` / `transcribe_open` return | `transcribe_n_speaker_segments` / `transcribe_get_speaker_segment` | `transcribe_diarize_n_segments` / `transcribe_diarize_get_segment` (same `transcribe_speaker_segment` rows) | | `transcribe_sortformer_stream_ext` (`SFST`, RUN slot) | `transcribe_sortformer_diarize_ext` (`SFDR`, DIARIZE_RUN slot); same preset enum | | `transcribe_model_supports(sortformer, TRANSCRIBE_FEATURE_DIARIZATION)` was true | false; check `transcribe_model_roles()` instead | -| `transcribe-cli --batch ... --batch-jsonl` "speakers" for Sortformer | `transcribe-cli -m sortformer.gguf audio.wav` prints the segments | +| `transcribe-cli --batch ... --batch-jsonl` "speakers" for Sortformer | `transcribe-cli -m sortformer.gguf audio.wav` prints the segments (`-o PATH` writes them); `--batch` is ASR-only | Segments are byte-identical to 0.3 for the same audio, preset and backend. Speaker-attributing ASR models (granite, moss, multitalker parakeet) are @@ -51,8 +51,9 @@ from the role's own query (`transcribe_diarize_get_info`). Sortformer ASR-path extension (`SortformerStreamOptions` and equivalents) is removed. - **Rust:** `Model::capabilities()` returns `Result` - (`Err(Error::UnsupportedRole)` on a model without ASR), and `ExtSlot` - gains `DiarizeRun`. + (`Err(Error::UnsupportedRole)` on a model without ASR). `ExtSlot` gains + `DiarizeRun` and `AbiStruct` gains the three diarize structs; both are now + `#[non_exhaustive]`, so later roles add variants without another break. - **Swift:** `Model.capabilities` is `get throws`. - **Python:** calls on one model now serialize (they used to race), and a run / run_batch / second stream on any session of a model with an active diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index 0ef2cb778..b3adb8b21 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -51,23 +51,6 @@ std::string json_escape(const char * s) { return out; } -bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { - if (output == nullptr) { - return true; - } - const char * value = text != nullptr ? text : ""; - *output << value; - if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { - *output << '\n'; - } - output->flush(); - if (!*output) { - std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); - return false; - } - return true; -} - // Shared row formatter for segments_json / batch_segments_json below. std::string segment_row_json(const struct transcribe_segment & seg) { std::string out; @@ -238,6 +221,23 @@ void apply_prompting(const cli_args & args, transcribe_run_params & rp, std::vec namespace transcribe_cli { +bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { + if (output == nullptr) { + return true; + } + const char * value = text != nullptr ? text : ""; + *output << value; + if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { + *output << '\n'; + } + output->flush(); + if (!*output) { + std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); + return false; + } + return true; +} + int run_asr_batch(const cli_args & args, std::ofstream * output) { bool output_ok = true; if (args.model_path.empty()) { @@ -285,6 +285,13 @@ int run_asr_batch(const cli_args & args, std::ofstream * output) { std::fprintf(stderr, "model load: %s\n", transcribe_status_string(load_st)); return EXIT_FAILURE; } + if ((transcribe_model_roles(model) & TRANSCRIBE_ROLE_ASR) == 0) { + std::fprintf(stderr, + "error: --batch is ASR-only and this model has no ASR role; " + "run it on one file instead: transcribe-cli -m MODEL audio.wav\n"); + transcribe_model_free(model); + return EXIT_FAILURE; + } struct transcribe_session_params cp; transcribe_session_params_init(&cp); @@ -692,7 +699,7 @@ int run_asr_file(const cli_args & args, std::ofstream * output) { const uint32_t roles = transcribe_model_roles(model); if ((roles & TRANSCRIBE_ROLE_ASR) == 0) { - return transcribe_cli::run_diarize_file(args, model, pcm, duration_s); + return transcribe_cli::run_diarize_file(args, model, pcm, duration_s, output); } struct transcribe_session_params cp; diff --git a/examples/cli/cli.h b/examples/cli/cli.h index b13f84f10..5224b1168 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -104,16 +104,21 @@ struct cli_args { int spec_k_drafts = -1; }; +// -o/--output: write `text` (newline-terminated) to `output` if non-null. +// Returns false (after an error message naming `path`) on a write failure. +bool write_output_file(std::ofstream * output, const std::string & path, const char * text); + // ASR drivers (asr.cpp). Each loads the model, runs, prints, and returns the // process exit code. int run_asr_batch(const cli_args & args, std::ofstream * output); int run_asr_file(const cli_args & args, std::ofstream * output); // DIARIZE driver (diarize.cpp): runs on a model run_asr_file already loaded -// and printed. Takes ownership of the model. +// and printed. Takes ownership of the model. With -o, writes the segment lines. int run_diarize_file(const cli_args & args, transcribe_model * model, const std::vector & pcm, - double duration_s); + double duration_s, + std::ofstream * output); } // namespace transcribe_cli diff --git a/examples/cli/diarize.cpp b/examples/cli/diarize.cpp index f811edf99..b779ab7a7 100644 --- a/examples/cli/diarize.cpp +++ b/examples/cli/diarize.cpp @@ -7,12 +7,15 @@ #include #include +#include +#include #include int transcribe_cli::run_diarize_file(const cli_args & args, transcribe_model * model, const std::vector & pcm, - double duration_s) { + double duration_s, + std::ofstream * output) { transcribe_diarize_session_params sp; transcribe_diarize_session_params_init(&sp); sp.n_threads = args.n_threads; @@ -26,15 +29,22 @@ int transcribe_cli::run_diarize_file(const cli_args & args, st = transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), nullptr); std::printf("run: %s\n", transcribe_status_string(st)); + bool output_ok = true; if (st == TRANSCRIBE_OK) { const int n = transcribe_diarize_n_segments(session); std::printf("speaker segments: %d\n", n); + std::string lines; for (int i = 0; i < n; ++i) { transcribe_speaker_segment row; transcribe_speaker_segment_init(&row); transcribe_diarize_get_segment(session, i, &row); - std::printf(" [%7.2f -> %7.2f] S%d\n", row.t0_ms / 1000.0, row.t1_ms / 1000.0, row.speaker_id); + char line[64]; + std::snprintf(line, sizeof(line), "[%7.2f -> %7.2f] S%d\n", row.t0_ms / 1000.0, row.t1_ms / 1000.0, + row.speaker_id); + std::printf(" %s", line); + lines += line; } + output_ok = write_output_file(output, args.output_path, lines.c_str()); } transcribe_timings tm; @@ -47,5 +57,5 @@ int transcribe_cli::run_diarize_file(const cli_args & args, transcribe_diarize_session_free(session); transcribe_model_free(model); - return st == TRANSCRIBE_OK ? EXIT_SUCCESS : EXIT_FAILURE; + return st == TRANSCRIBE_OK && output_ok ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 1a44392bb..40e2da5b0 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -53,7 +53,7 @@ void print_usage(const char * argv0) { " --target-language ISO target language for translation (e.g. de, es, fr)\n" " -q, --quiet suppress library log output\n" " -r, --repeat N run N times per file (benchmark)\n" - " -o, --output PATH write transcribed text to PATH (stdout unchanged)\n" + " -o, --output PATH write text or speaker segments to PATH (stdout unchanged)\n" " --threads N CPU threads (default: all cores)\n" " --n-ctx N session context/KV cap in tokens (bounds decoder\n" " KV memory; cannot extend the model): 0 = model\n" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ea1319c5..e593e0702 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1521,6 +1521,7 @@ if(TRANSCRIBE_BUILD_EXAMPLES) COMMAND ${CMAKE_COMMAND} -DCLI=$ -DWAV=${CMAKE_SOURCE_DIR}/samples/sortformer-2spk-mix.wav + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/cli-diarize-smoke -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_diarize_smoke.cmake) set_tests_properties(transcribe_cli_diarize_smoke PROPERTIES SKIP_REGULAR_EXPRESSION "SKIP:") diff --git a/tests/cli_diarize_smoke.cmake b/tests/cli_diarize_smoke.cmake index a60fd78c1..83fa2c85c 100644 --- a/tests/cli_diarize_smoke.cmake +++ b/tests/cli_diarize_smoke.cmake @@ -1,13 +1,18 @@ # transcribe-cli routes a DIARIZE model to the diarize path and prints speaker -# segments for the 2-speaker oracle mix. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). +# segments for the 2-speaker oracle mix, writes them with -o, and refuses +# --batch. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") message("SKIP: TRANSCRIBE_SORTFORMER_GGUF not set") return() endif() +file(REMOVE_RECURSE "${TEST_DIR}") +file(MAKE_DIRECTORY "${TEST_DIR}") +set(_output "${TEST_DIR}/segments.txt") + execute_process( - COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" "${WAV}" + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" -o "${_output}" "${WAV}" RESULT_VARIABLE rc OUTPUT_VARIABLE out ERROR_VARIABLE err) @@ -17,3 +22,17 @@ endif() if(NOT out MATCHES "speaker segments: [1-9].*S2") message(FATAL_ERROR "unexpected output:\n${out}") endif() +file(READ "${_output}" written) +if(NOT written MATCHES "^\\[ +0\\.[0-9]+ -> +[0-9.]+\\] S[0-9].*S2\n$") + message(FATAL_ERROR "-o wrote unexpected segments:\n${written}") +endif() + +file(WRITE "${TEST_DIR}/list.txt" "${WAV}\n") +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --batch "${TEST_DIR}/list.txt" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(rc EQUAL 0 OR NOT err MATCHES "--batch is ASR-only") + message(FATAL_ERROR "--batch on a diarize model: rc=${rc}\n${err}") +endif() diff --git a/tests/nonfinite_input_unit.cpp b/tests/nonfinite_input_unit.cpp index 5eb9688af..38503873d 100644 --- a/tests/nonfinite_input_unit.cpp +++ b/tests/nonfinite_input_unit.cpp @@ -1,6 +1,6 @@ // nonfinite_input_unit.cpp - every ASR PCM entry point rejects NaN / Inf // samples with INVALID_ARG before any result or stream state is modified -// and before the family hook runs. +// and before the family hook runs (D16a). Silence stays valid input. #include "transcribe-arch.h" #include "transcribe-model.h" @@ -109,6 +109,18 @@ void test_run_rejects_nonfinite() { } } +void test_run_accepts_silence() { + transcribe_model model; + model.arch = &k_arch; + transcribe_session s; + seed(s, model); + std::vector pcm(1600, 0.0f); + g_run_calls = 0; + CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + CHECK(g_run_calls == 1); + CHECK(s.full_text == "fresh"); +} + void test_batch_rejects_nonfinite_whole_batch() { transcribe_model model; model.arch = &k_arch; @@ -128,6 +140,20 @@ void test_batch_rejects_nonfinite_whole_batch() { } } +void test_batch_accepts_silence() { + transcribe_model model; + model.arch = &k_arch; + transcribe_session s; + seed(s, model); + std::vector a(800, 0.0f), b(800, 0.0f); + const float * pcm[] = { a.data(), b.data() }; + const int n_samples[] = { 800, 800 }; + g_run_batch_calls = 0; + CHECK(transcribe_run_batch(&s, pcm, n_samples, 2, nullptr) == TRANSCRIBE_OK); + CHECK(g_run_batch_calls == 1); + CHECK(transcribe_batch_n_results(&s) == 2); +} + void test_stream_feed_rejects_nonfinite() { transcribe_model model; model.arch = &k_arch; @@ -163,7 +189,9 @@ int main() { transcribe_log_set(nullptr, nullptr); test_run_rejects_nonfinite(); + test_run_accepts_silence(); test_batch_rejects_nonfinite_whole_batch(); + test_batch_accepts_silence(); test_stream_feed_rejects_nonfinite(); if (g_failures != 0) { diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp index 3ef858e0f..e742d6668 100644 --- a/tests/sortformer_diarize_unit.cpp +++ b/tests/sortformer_diarize_unit.cpp @@ -63,12 +63,22 @@ std::vector diarize(transcribe_diarize_session * s, return diarize_rows(s); } +// The ASR leg runs on jfk.wav: whisper with NULL params seeds temperature +// fallback randomly, and the overlapping 2-speaker mix triggers fallback, so +// its text is not stable across runs. void test_cross_role_concurrency(transcribe_model * sf, const std::vector & pcm) { const char * whisper_path = std::getenv("TRANSCRIBE_WHISPER_GGUF"); if (whisper_path == nullptr || whisper_path[0] == '\0') { std::fprintf(stderr, "sortformer_diarize_unit: TRANSCRIBE_WHISPER_GGUF not set; skipping concurrency leg\n"); return; } + std::vector asr_pcm; + std::string err; + if (!transcribe_cli::load_wav_mono_16k(std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/jfk.wav", asr_pcm, err)) { + std::fprintf(stderr, "FAIL: jfk.wav: %s\n", err.c_str()); + ++g_failures; + return; + } transcribe_model * asr_model = load_cpu(whisper_path); CHECK(asr_model != nullptr); if (asr_model == nullptr) { @@ -79,12 +89,13 @@ void test_cross_role_concurrency(transcribe_model * sf, const std::vector CHECK(transcribe_session_init(asr_model, nullptr, &asr) == TRANSCRIBE_OK); CHECK(transcribe_diarize_session_init(sf, nullptr, &dia) == TRANSCRIBE_OK); - CHECK(transcribe_run(asr, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); - const std::string text_seq = transcribe_full_text(asr); + CHECK(transcribe_run(asr, asr_pcm.data(), static_cast(asr_pcm.size()), nullptr) == TRANSCRIBE_OK); + const std::string text_seq = transcribe_full_text(asr); + CHECK(!text_seq.empty()); const std::vector rows_seq = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); - transcribe_status asr_st = TRANSCRIBE_ERR_BACKEND; - std::thread t([&] { asr_st = transcribe_run(asr, pcm.data(), static_cast(pcm.size()), nullptr); }); + transcribe_status asr_st = TRANSCRIBE_ERR_BACKEND; + std::thread t([&] { asr_st = transcribe_run(asr, asr_pcm.data(), static_cast(asr_pcm.size()), nullptr); }); const std::vector rows_par = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); t.join(); From 2165ed31f113ba5fc8df4e5d62352438380641fd Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:55:45 +0800 Subject: [PATCH 44/51] tests: trim C unit tests to the behavior this PR adds - sortformer_diarize_unit: drop test_cross_role_concurrency (two separate models, overlap never forced, needs a second GGUF). - run_on_threads_unit: 4 tests (every tid once, worker exception rethrown after join, launch failure joins and rethrows, parallel_for_all propagates). - nonfinite_input_unit: drop the silence controls and the position loop. - stream_hook_throw_unit: one loop over begin/feed/finalize; drop the feed-after-FAILED tail already covered by stream_dispatch_unit. - role_resolve_unit: table-driven resolve_roles cases; add the diarize-only arch cases (roles=DIARIZE ok, roles=0 rejected). - diarize_dispatch_unit: drop asserts on session internals. --- tests/CMakeLists.txt | 4 +- tests/diarize_dispatch_unit.cpp | 2 +- tests/nonfinite_input_unit.cpp | 48 ++++------------- tests/role_resolve_unit.cpp | 88 +++++++++++++++++-------------- tests/run_on_threads_unit.cpp | 61 ++------------------- tests/sortformer_diarize_unit.cpp | 50 ------------------ tests/stream_hook_throw_unit.cpp | 59 +++++++-------------- 7 files changed, 82 insertions(+), 230 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e593e0702..26752ae9b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1323,8 +1323,8 @@ if(TRANSCRIBE_BUILD_REAL_MODEL_TESTS) set_tests_properties(transcribe_qwen3_asr_e2e_smoke PROPERTIES SKIP_RETURN_CODE 77) - # Sortformer on the DIARIZE role: golden segments per preset, pre-clear - # rejection, and (with TRANSCRIBE_WHISPER_GGUF) cross-role concurrency. + # Sortformer on the DIARIZE role: golden segments per preset and pre-clear + # rejection. # Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). add_executable(transcribe_sortformer_diarize_unit sortformer_diarize_unit.cpp) diff --git a/tests/diarize_dispatch_unit.cpp b/tests/diarize_dispatch_unit.cpp index ebadba14f..7c8d9ecc7 100644 --- a/tests/diarize_dispatch_unit.cpp +++ b/tests/diarize_dispatch_unit.cpp @@ -140,7 +140,7 @@ void test_role_checks() { CHECK(s == nullptr); sp.n_threads = 2; CHECK(transcribe_diarize_session_init(&model, &sp, &s) == TRANSCRIBE_OK); - CHECK(s != nullptr && s->model == &model && s->n_threads == 2); + CHECK(s != nullptr); transcribe_diarize_session_free(s); transcribe_diarize_session_free(nullptr); } diff --git a/tests/nonfinite_input_unit.cpp b/tests/nonfinite_input_unit.cpp index 38503873d..169e8bb2e 100644 --- a/tests/nonfinite_input_unit.cpp +++ b/tests/nonfinite_input_unit.cpp @@ -1,6 +1,6 @@ // nonfinite_input_unit.cpp - every ASR PCM entry point rejects NaN / Inf // samples with INVALID_ARG before any result or stream state is modified -// and before the family hook runs (D16a). Silence stays valid input. +// and before the family hook runs (D16a). #include "transcribe-arch.h" #include "transcribe-model.h" @@ -96,31 +96,17 @@ void test_run_rejects_nonfinite() { transcribe_model model; model.arch = &k_arch; for (float bad : k_bads) { - for (size_t pos : { size_t{ 0 }, size_t{ 799 }, size_t{ 1599 } }) { - transcribe_session s; - seed(s, model); - std::vector pcm(1600, 0.1f); - pcm[pos] = bad; - g_run_calls = 0; - CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(g_run_calls == 0); - CHECK(prior_intact(s)); - } + transcribe_session s; + seed(s, model); + std::vector pcm(1600, 0.1f); + pcm[799] = bad; + g_run_calls = 0; + CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_run_calls == 0); + CHECK(prior_intact(s)); } } -void test_run_accepts_silence() { - transcribe_model model; - model.arch = &k_arch; - transcribe_session s; - seed(s, model); - std::vector pcm(1600, 0.0f); - g_run_calls = 0; - CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); - CHECK(g_run_calls == 1); - CHECK(s.full_text == "fresh"); -} - void test_batch_rejects_nonfinite_whole_batch() { transcribe_model model; model.arch = &k_arch; @@ -140,20 +126,6 @@ void test_batch_rejects_nonfinite_whole_batch() { } } -void test_batch_accepts_silence() { - transcribe_model model; - model.arch = &k_arch; - transcribe_session s; - seed(s, model); - std::vector a(800, 0.0f), b(800, 0.0f); - const float * pcm[] = { a.data(), b.data() }; - const int n_samples[] = { 800, 800 }; - g_run_batch_calls = 0; - CHECK(transcribe_run_batch(&s, pcm, n_samples, 2, nullptr) == TRANSCRIBE_OK); - CHECK(g_run_batch_calls == 1); - CHECK(transcribe_batch_n_results(&s) == 2); -} - void test_stream_feed_rejects_nonfinite() { transcribe_model model; model.arch = &k_arch; @@ -189,9 +161,7 @@ int main() { transcribe_log_set(nullptr, nullptr); test_run_rejects_nonfinite(); - test_run_accepts_silence(); test_batch_rejects_nonfinite_whole_batch(); - test_batch_accepts_silence(); test_stream_feed_rejects_nonfinite(); if (g_failures != 0) { diff --git a/tests/role_resolve_unit.cpp b/tests/role_resolve_unit.cpp index e8c766fce..e39917b19 100644 --- a/tests/role_resolve_unit.cpp +++ b/tests/role_resolve_unit.cpp @@ -1,6 +1,7 @@ // role_resolve_unit.cpp - load-time role validation (transcribe::resolve_roles) and the public role API (transcribe_model_roles, UNSUPPORTED_ROLE). #include "transcribe-arch.h" +#include "transcribe-diarize.h" #include "transcribe-model.h" #include "transcribe.h" @@ -45,39 +46,54 @@ const transcribe::Arch k_no_role_arch = { /* .name = */ "fake_no_role", }; -transcribe_status resolve(const transcribe::Arch & arch, uint32_t roles, uint32_t * out_roles) { - transcribe_model model; - model.arch = &arch; - model.roles = roles; - const transcribe_status st = transcribe::resolve_roles(&model); - *out_roles = model.roles; - return st; -} - -void test_asr_default_and_explicit() { - uint32_t roles = 0; - CHECK(resolve(k_asr_arch, 0, &roles) == TRANSCRIBE_OK); - CHECK(roles == TRANSCRIBE_ROLE_ASR); - CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR, &roles) == TRANSCRIBE_OK); - CHECK(roles == TRANSCRIBE_ROLE_ASR); -} - -void test_no_role_rejected() { - uint32_t roles = 0; - CHECK(resolve(k_no_role_arch, 0, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - CHECK(resolve(k_half_asr_arch, 0, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); -} +// Diarize-only: a diarize ops table and no ASR hooks. resolve_roles only +// checks the table is present, so an empty one suffices. +const transcribe::DiarizeOps k_diarize_ops = {}; -void test_unbacked_role_rejected() { - uint32_t roles = 0; - CHECK(resolve(k_half_asr_arch, TRANSCRIBE_ROLE_ASR, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_DIARIZE, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE, &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); +transcribe::Arch make_diarize_arch() { + transcribe::Arch a = {}; + a.name = "fake_diarize"; + a.diarize = &k_diarize_ops; + return a; } -void test_unknown_bit_rejected() { - uint32_t roles = 0; - CHECK(resolve(k_asr_arch, TRANSCRIBE_ROLE_ASR | (1u << 31), &roles) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); +const transcribe::Arch k_diarize_arch = make_diarize_arch(); + +void test_resolve_roles() { + constexpr uint32_t ASR = TRANSCRIBE_ROLE_ASR; + constexpr uint32_t DIA = TRANSCRIBE_ROLE_DIARIZE; + + struct Case { + const transcribe::Arch * arch; + uint32_t roles_in; + transcribe_status want; + uint32_t roles_out; // checked on OK only + }; + + const Case cases[] = { + { &k_asr_arch, 0, TRANSCRIBE_OK, ASR }, // defaults to ASR + { &k_asr_arch, ASR, TRANSCRIBE_OK, ASR }, + { &k_diarize_arch, DIA, TRANSCRIBE_OK, DIA }, + { &k_diarize_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, // no default for non-ASR roles + { &k_no_role_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_half_asr_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_half_asr_arch, ASR, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, ASR | DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_diarize_arch, ASR | DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, ASR | (1u << 31), TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, // unknown bit + { nullptr, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + }; + for (const Case & c : cases) { + transcribe_model model; + model.arch = c.arch; + model.roles = c.roles_in; + const transcribe_status st = transcribe::resolve_roles(&model); + CHECK(st == c.want); + if (st == TRANSCRIBE_OK) { + CHECK(model.roles == c.roles_out); + } + } } // ASR entry points refuse a model that does not serve ASR, before @@ -108,22 +124,12 @@ void test_asr_entry_points_check_role() { CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_ASR); } -void test_null_model() { - CHECK(transcribe::resolve_roles(nullptr) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); - transcribe_model model; - CHECK(transcribe::resolve_roles(&model) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); -} - } // namespace int main() { transcribe_log_set(nullptr, nullptr); // the rejections log at ERROR - test_asr_default_and_explicit(); - test_no_role_rejected(); - test_unbacked_role_rejected(); - test_unknown_bit_rejected(); - test_null_model(); + test_resolve_roles(); test_asr_entry_points_check_role(); if (g_failures != 0) { diff --git a/tests/run_on_threads_unit.cpp b/tests/run_on_threads_unit.cpp index 244b632ec..98c898ca1 100644 --- a/tests/run_on_threads_unit.cpp +++ b/tests/run_on_threads_unit.cpp @@ -68,24 +68,7 @@ void test_every_tid_runs_once() { } } -void test_worker_throw_on_caller_thread() { - std::atomic done{ 0 }; - bool caught = false; - try { - transcribe::run_on_threads(4, [&](int tid) { - if (tid == 0) { - throw std::runtime_error("tid0"); - } - done.fetch_add(1); - }); - } catch (const std::runtime_error &) { - caught = true; - } - CHECK(caught); - CHECK(done.load() == 3); // every pool thread ran to completion and was joined -} - -void test_worker_throw_on_pool_thread() { +void test_worker_throw_rethrown_after_join() { std::atomic done{ 0 }; bool caught = false; try { @@ -99,21 +82,7 @@ void test_worker_throw_on_pool_thread() { caught = true; // exception type preserved across the thread boundary } CHECK(caught); - CHECK(done.load() == 3); -} - -void test_lowest_tid_exception_wins() { - int which = -1; - try { - transcribe::run_on_threads(4, [&](int tid) { - if (tid == 1 || tid == 3) { - throw std::runtime_error(tid == 1 ? "1" : "3"); - } - }); - } catch (const std::runtime_error & e) { - which = e.what()[0] - '0'; - } - CHECK(which == 1); + CHECK(done.load() == 3); // every other thread ran to completion and was joined } void test_launch_failure_joins_launched_threads() { @@ -139,26 +108,6 @@ void test_launch_failure_joins_launched_threads() { CHECK(ran.load() == 2); } -void test_first_launch_failure() { - reset_flaky(0); - bool caught = false; - try { - transcribe::run_on_threads(3, [](int) {}); - } catch (const std::system_error &) { - caught = true; - } - CHECK(caught); - CHECK(FlakyThread::joins.load() == 0); -} - -void test_launch_ok_with_seam() { - reset_flaky(-1); - std::atomic ran{ 0 }; - transcribe::run_on_threads(4, [&](int) { ran.fetch_add(1); }); - CHECK(ran.load() == 4); - CHECK(FlakyThread::joins.load() == 3); -} - void test_parallel_for_all_propagates() { bool caught = false; try { @@ -178,12 +127,8 @@ void test_parallel_for_all_propagates() { int main() { test_every_tid_runs_once(); - test_worker_throw_on_caller_thread(); - test_worker_throw_on_pool_thread(); - test_lowest_tid_exception_wins(); + test_worker_throw_rethrown_after_join(); test_launch_failure_joins_launched_threads(); - test_first_launch_failure(); - test_launch_ok_with_seam(); test_parallel_for_all_propagates(); if (g_failures != 0) { diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp index e742d6668..d1361d348 100644 --- a/tests/sortformer_diarize_unit.cpp +++ b/tests/sortformer_diarize_unit.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include namespace { @@ -63,51 +62,6 @@ std::vector diarize(transcribe_diarize_session * s, return diarize_rows(s); } -// The ASR leg runs on jfk.wav: whisper with NULL params seeds temperature -// fallback randomly, and the overlapping 2-speaker mix triggers fallback, so -// its text is not stable across runs. -void test_cross_role_concurrency(transcribe_model * sf, const std::vector & pcm) { - const char * whisper_path = std::getenv("TRANSCRIBE_WHISPER_GGUF"); - if (whisper_path == nullptr || whisper_path[0] == '\0') { - std::fprintf(stderr, "sortformer_diarize_unit: TRANSCRIBE_WHISPER_GGUF not set; skipping concurrency leg\n"); - return; - } - std::vector asr_pcm; - std::string err; - if (!transcribe_cli::load_wav_mono_16k(std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/jfk.wav", asr_pcm, err)) { - std::fprintf(stderr, "FAIL: jfk.wav: %s\n", err.c_str()); - ++g_failures; - return; - } - transcribe_model * asr_model = load_cpu(whisper_path); - CHECK(asr_model != nullptr); - if (asr_model == nullptr) { - return; - } - transcribe_session * asr = nullptr; - transcribe_diarize_session * dia = nullptr; - CHECK(transcribe_session_init(asr_model, nullptr, &asr) == TRANSCRIBE_OK); - CHECK(transcribe_diarize_session_init(sf, nullptr, &dia) == TRANSCRIBE_OK); - - CHECK(transcribe_run(asr, asr_pcm.data(), static_cast(asr_pcm.size()), nullptr) == TRANSCRIBE_OK); - const std::string text_seq = transcribe_full_text(asr); - CHECK(!text_seq.empty()); - const std::vector rows_seq = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); - - transcribe_status asr_st = TRANSCRIBE_ERR_BACKEND; - std::thread t([&] { asr_st = transcribe_run(asr, asr_pcm.data(), static_cast(asr_pcm.size()), nullptr); }); - const std::vector rows_par = diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); - t.join(); - - CHECK(asr_st == TRANSCRIBE_OK); - CHECK(text_seq == transcribe_full_text(asr)); - CHECK(rows_par == rows_seq); - - transcribe_session_free(asr); - transcribe_diarize_session_free(dia); - transcribe_model_free(asr_model); -} - } // namespace int main() { @@ -179,10 +133,6 @@ int main() { CHECK(transcribe_diarize_n_segments(dia) == n_before); transcribe_diarize_session_free(dia); - - // 4. Cross-role concurrency on two models. - test_cross_role_concurrency(model, pcm); - transcribe_model_free(model); if (g_failures != 0) { std::fprintf(stderr, "%d failure(s)\n", g_failures); diff --git a/tests/stream_hook_throw_unit.cpp b/tests/stream_hook_throw_unit.cpp index 9b0fc53ab..ea159ed02 100644 --- a/tests/stream_hook_throw_unit.cpp +++ b/tests/stream_hook_throw_unit.cpp @@ -101,42 +101,25 @@ void expect_failed(const transcribe_session & s, transcribe_status st, Throw t) CHECK(transcribe_stream_last_status(&s) == want); } -void test_begin_throw() { - for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { - reset_globals(); - Fixture f; - g_begin = t; - const transcribe_status st = transcribe_stream_begin(&f.session, nullptr, nullptr); - expect_failed(f.session, st, t); - // A failed stream accepts a fresh begin. - g_begin = Throw::None; - CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); - CHECK(transcribe_stream_get_state(&f.session) == TRANSCRIBE_STREAM_ACTIVE); - } -} - -void test_feed_throw() { - for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { - reset_globals(); - Fixture f; - CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); - g_feed = t; - expect_failed(f.session, f.feed(), t); - // No longer ACTIVE, so further feeds are refused as on any failed stream. - g_feed = Throw::None; - CHECK(f.feed() == TRANSCRIBE_ERR_INVALID_ARG); - } -} - -void test_finalize_throw() { - for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { - reset_globals(); - Fixture f; - CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); - CHECK(f.feed() == TRANSCRIBE_OK); - g_finalize = t; - const transcribe_status st = transcribe_stream_finalize(&f.session, nullptr); - expect_failed(f.session, st, t); +// Each of begin / feed / finalize, throwing bad_alloc or a runtime_error, +// leaves the stream FAILED; a failed stream accepts a fresh begin. +void test_hook_throw_fails_stream() { + Throw * hooks[] = { &g_begin, &g_feed, &g_finalize }; + for (Throw * hook : hooks) { + for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { + reset_globals(); + Fixture f; + if (hook != &g_begin) { + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + } + *hook = t; + const transcribe_status st = hook == &g_begin ? transcribe_stream_begin(&f.session, nullptr, nullptr) : + hook == &g_feed ? f.feed() : + transcribe_stream_finalize(&f.session, nullptr); + expect_failed(f.session, st, t); + *hook = Throw::None; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + } } } @@ -162,9 +145,7 @@ void test_reset_throw_still_idles() { int main() { transcribe_log_set(nullptr, nullptr); // the guard logs each caught exception - test_begin_throw(); - test_feed_throw(); - test_finalize_throw(); + test_hook_throw_fails_stream(); test_reset_throw_still_idles(); if (g_failures != 0) { From d43e279f846985d1d85d8aef8f1590c20800277f Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:55:57 +0800 Subject: [PATCH 45/51] rust: trim tests to the binding's own surface - cancel.rs: drop cancel_token_is_retained_until_cleared (pre-existing behavior, unrelated to roles); file back to its pre-PR form. - streaming.rs: drop option_validation_precedes_busy_check (pins which of two errors wins). - diarize.rs: exact segments live in tests/sortformer_diarize_unit.cpp; the binding test now checks well-formed turns and that LowLatency changes the result (preset reaches native). --- bindings/rust/transcribe-cpp/tests/cancel.rs | 39 +------------ bindings/rust/transcribe-cpp/tests/diarize.rs | 55 ++++++++----------- .../rust/transcribe-cpp/tests/streaming.rs | 36 ------------ 3 files changed, 26 insertions(+), 104 deletions(-) diff --git a/bindings/rust/transcribe-cpp/tests/cancel.rs b/bindings/rust/transcribe-cpp/tests/cancel.rs index c4645a8ed..9b842f09a 100644 --- a/bindings/rust/transcribe-cpp/tests/cancel.rs +++ b/bindings/rust/transcribe-cpp/tests/cancel.rs @@ -1,6 +1,6 @@ -//! Cancellation: a clean baseline run, a cross-thread timer cancel of an -//! in-flight run on long (tiled) audio (ports the Python C5 case; skips the -//! abort assertion if the run wins the race), and token retention. +//! Cancellation: a clean baseline run, and a cross-thread timer cancel of an +//! in-flight run on long (tiled) audio. Ports the Python C5 case. Skips the +//! abort assertion if the machine wins the race (run finishes before cancel). mod common; @@ -72,36 +72,3 @@ fn cross_thread_cancel_of_in_flight_run() { Err(other) => panic!("unexpected error: {other:?}"), } } - -#[test] -fn cancel_token_is_retained_until_cleared() { - // The session keeps the installed token's flag alive (the caller may drop - // every handle) and keeps consulting it on later runs until cleared. - let Some((model_path, pcm)) = common::smoke_fixtures("cancel_token_is_retained_until_cleared") - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - if !model.supports(Feature::Cancellation) { - eprintln!("skip: model does not support cancellation"); - return; - } - let mut session = model.session().unwrap(); - { - let token = CancelToken::new(); - token.cancel(); - session.set_cancel_token(&token); - } // every caller-side handle to the flag is gone - - for attempt in 0..2 { - match session.run(&pcm, &RunOptions::default()) { - Err(Error::Aborted { .. }) => assert!(session.was_aborted()), - other => panic!("attempt {attempt}: pre-cancelled token must abort, got {other:?}"), - } - } - - session.clear_cancel_token(); - let result = session.run(&pcm, &RunOptions::default()).unwrap(); - assert!(result.text.to_lowercase().contains("country")); - assert!(!session.was_aborted()); -} diff --git a/bindings/rust/transcribe-cpp/tests/diarize.rs b/bindings/rust/transcribe-cpp/tests/diarize.rs index dfc5b4f41..eff77fd46 100644 --- a/bindings/rust/transcribe-cpp/tests/diarize.rs +++ b/bindings/rust/transcribe-cpp/tests/diarize.rs @@ -30,51 +30,42 @@ fn sortformer_is_diarize_only() { } #[test] -fn diarize_run_matches_cpu_golden_segments() { +fn diarize_run_returns_turns_and_applies_preset() { let Some((model_path, pcm)) = - common::smoke_sortformer_fixtures("diarize_run_matches_cpu_golden_segments") + common::smoke_sortformer_fixtures("diarize_run_returns_turns_and_applies_preset") else { return; }; - // Goldens are recorded on CPU (tests/sortformer_diarize_unit.cpp). + // Exact segments are pinned in C (tests/sortformer_diarize_unit.cpp); here + // the binding only has to return well-formed turns and pass the preset on. let cpu = ModelOptions { backend: Backend::Cpu, ..Default::default() }; let model = Model::load_with(&model_path, &cpu).unwrap(); + let max_speakers = model.diarize_info().unwrap().max_speakers; let mut diarize = model.diarize_session().unwrap(); - let golden_default = [ - (320, 2400, 1), - (7360, 9360, 1), - (10240, 10640, 1), - (4240, 6640, 2), - (9760, 12000, 2), - ]; - let golden_low_latency = [ - (320, 2480, 1), - (7360, 9360, 1), - (10240, 10640, 1), - (4160, 6640, 2), - (9760, 12000, 2), - ]; - for (preset, golden) in [ - (None, &golden_default), - (Some(SortformerPreset::Default), &golden_default), - (Some(SortformerPreset::LowLatency), &golden_low_latency), - ] { - let opts = DiarizeOptions { - family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { - preset, - })), - }; - let turns = diarize.run(&pcm, &opts).unwrap(); - let rows: Vec<_> = turns + let rows = |turns: Vec| -> Vec<_> { + turns .iter() .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id)) - .collect(); - assert_eq!(rows, golden.to_vec(), "{preset:?}"); - assert!(diarize.timings().encode_ms > 0.0); + .collect() + }; + + let default = rows(diarize.run(&pcm, &DiarizeOptions::default()).unwrap()); + assert!(!default.is_empty()); + for &(t0, t1, speaker) in &default { + assert!(t0 < t1, "{t0} >= {t1}"); + assert!((1..=max_speakers).contains(&speaker), "speaker {speaker}"); } + assert!(diarize.timings().encode_ms > 0.0); + + let low_latency = DiarizeOptions { + family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { + preset: Some(SortformerPreset::LowLatency), + })), + }; + assert_ne!(rows(diarize.run(&pcm, &low_latency).unwrap()), default); } #[test] diff --git a/bindings/rust/transcribe-cpp/tests/streaming.rs b/bindings/rust/transcribe-cpp/tests/streaming.rs index 53494a6c1..86b2ee6a8 100644 --- a/bindings/rust/transcribe-cpp/tests/streaming.rs +++ b/bindings/rust/transcribe-cpp/tests/streaming.rs @@ -284,42 +284,6 @@ fn stream_family_extension_accepted_or_rejected() { assert!(matches!(err, Err(Error::InvalidArgument(_))), "{err:?}"); } -#[test] -fn option_validation_precedes_busy_check() { - // Run options are marshalled before the compute lock is taken, so a bad - // option is reported as itself even while another session streams. - let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) - else { - return; - }; - let model = Model::load(&model_path).unwrap(); - let mut s1 = model.session().unwrap(); - let mut s2 = model.session().unwrap(); - let _stream1 = s1 - .stream(&RunOptions::default(), &StreamOptions::default()) - .unwrap(); - - let bad = RunOptions { - language: Some("e\0n".into()), - ..Default::default() - }; - assert!( - matches!(s2.run(&pcm, &bad), Err(Error::Nul(_))), - "run: NUL option must win over Busy" - ); - assert!( - matches!(s2.run_batch(&[&pcm], &bad), Err(Error::Nul(_))), - "run_batch: NUL option must win over Busy" - ); - assert!( - matches!( - s2.stream(&bad, &StreamOptions::default()), - Err(Error::Nul(_)) - ), - "stream: NUL option must win over Busy" - ); -} - #[test] fn ended_stream_never_releases_another_sessions_lease() { // After finalize() the lease belongs to whoever takes it next; a later From a1cca90a1d162c5794f927c5f73b24840a76a57d Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:56:20 +0800 Subject: [PATCH 46/51] python: trim compute-lock and diarize tests test_compute_lock.py keeps 12 tests for the model-wide lock and stream lease: serialization, every compute site locked, cancel while queued, close / model close / GC during an in-flight call, re-entrancy, Busy, failed begin, lease release per ending, ended stream vs a later lease, diarize Busy. Dropped tests that were true by construction (per-model lock/lease), fake twins of real-model tests, internal-ordering pins, and the diarize variants of _SessionBase paths. test_streaming.py: drop the session-close and keep-alive duplicates; the hook-failure lease check moves into the existing cancellation test. test_diarize.py: exact segments live in tests/sortformer_diarize_unit.cpp; check well-formed turns and that low_latency changes the result. --- bindings/python/tests/test_compute_lock.py | 309 +-------------------- bindings/python/tests/test_diarize.py | 29 +- bindings/python/tests/test_streaming.py | 53 +--- 3 files changed, 28 insertions(+), 363 deletions(-) diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index 425e937a8..40054519a 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -95,7 +95,6 @@ def session(m, cls=t.Session, track=True): PCM = [0.0] * 160 -SESSION_CLS = {"session": t.Session, "diarize": t.DiarizeSession} class _LockSpy: @@ -173,26 +172,6 @@ def runs(session): assert box.get("value") == [("result", s._handle.value, None)] * n, box -def test_lock_is_per_model(fake, monkeypatch): - s1 = fake.session(fake.model()) - s2 = fake.session(fake.model()) - gate = _Gate() - - def probe(h, *rest): - return gate(h) if h.value == s1._handle.value else 0 - - monkeypatch.setattr(t._lib, "transcribe_run", probe) - try: - a, _ = _in_thread(s1.run, PCM) - assert gate.entered.wait(TIMEOUT) - b, b_box = _in_thread(s2.run, PCM) - _join(b, "run on another model") # must not wait behind model 1 - assert "error" not in b_box, b_box - finally: - gate.release.set() - _join(a, "first run") - - def test_every_compute_site_holds_the_lock(fake, monkeypatch): m = fake.model() s = fake.session(m) @@ -238,28 +217,14 @@ def materialize(self, h, utt=None): assert not m._compute_lock.locked() -def test_lock_released_after_native_error(fake, monkeypatch): - m = fake.model() - s = fake.session(m) - monkeypatch.setattr(t._lib, "transcribe_run", - lambda *a: _generated.TRANSCRIBE_ERR_INVALID_ARG) - with pytest.raises(t.InvalidArgument): - s.run(PCM) - assert not m._compute_lock.locked() - monkeypatch.setattr(t._lib, "transcribe_run", lambda *a: 0) - assert s.run(PCM) == ("result", s._handle.value, None) - - # --- cancellation ------------------------------------------------------------- -@pytest.mark.parametrize("kind", ["session", "diarize"]) -def test_cancel_while_queued_is_kept(fake, diarize_native, monkeypatch, kind): +def test_cancel_while_queued_is_kept(fake, monkeypatch): # cancel() is lock-free and the flag is cleared BEFORE the lock wait, so # a cancel issued while a call is queued is still set when it starts. m = fake.model() - s1, s2 = fake.session(m), fake.session(m, cls=SESSION_CLS[kind]) - diarize_native.owner["model"] = m + s1, s2 = fake.session(m), fake.session(m) gate = _Gate() flag_at_start: list = [] @@ -270,7 +235,6 @@ def probe(h, *rest): return 0 monkeypatch.setattr(t._lib, "transcribe_run", probe) - monkeypatch.setattr(t._lib, "transcribe_diarize_run", probe) try: a, _ = _in_thread(s1.run, PCM) assert gate.entered.wait(TIMEOUT) @@ -288,17 +252,12 @@ def probe(h, *rest): # --- close / GC during an in-flight call ------------------------------------------ -@pytest.mark.parametrize("kind", ["session", "diarize"]) -def test_session_close_during_in_flight_call_defers_free(fake, diarize_native, - monkeypatch, kind): +def test_session_close_during_in_flight_call_defers_free(fake, monkeypatch): m = fake.model() - s = fake.session(m, cls=SESSION_CLS[kind]) - diarize_native.owner["model"] = m + s = fake.session(m) handle = s._handle.value gate = _Gate() monkeypatch.setattr(t._lib, "transcribe_run", gate) - monkeypatch.setattr(t._lib, "transcribe_diarize_run", - lambda *a: gate() or diarize_native.run(*a)) try: a, a_box = _in_thread(s.run, PCM) assert gate.entered.wait(TIMEOUT) @@ -309,11 +268,9 @@ def test_session_close_during_in_flight_call_defers_free(fake, diarize_native, gate.release.set() _join(a, "in-flight run") # The in-flight call finished its copy-out on the handle it captured... - expected = ROWS if kind == "diarize" else ("result", handle, None) - assert a_box.get("value") == expected, a_box - assert all(c[1] == handle for c in diarize_native.calls) + assert a_box.get("value") == ("result", handle, None), a_box # ...and the deferred free ran exactly once, after it. - assert fake.frees == [(kind, handle)] + assert fake.frees == [("session", handle)] with pytest.raises(t.TranscribeError, match="closed"): s.run(PCM) @@ -470,13 +427,6 @@ def test_active_stream_refuses_sibling_and_same_session_calls(fake, native): _assert_free(s1) -def test_lease_is_per_model(fake, native): - s1 = fake.session(fake.model()) - other = fake.session(fake.model()) - s1.stream() - _assert_free(other) - - def test_failed_begin_takes_no_lease(fake, native): m = fake.model() s1, s2 = fake.session(m), fake.session(m) @@ -526,54 +476,6 @@ def test_lease_released_when_stream_ends(fake, native, ending): assert fake.resets == [] -def test_model_close_releases_lease(fake, native): - m = fake.model() - s1 = fake.session(m) - stream = s1.stream() - m.close() - assert m._stream_owner is None - with pytest.raises(t.TranscribeError, match="closed"): - stream.feed(PCM) - del stream - gc.collect() - assert fake.resets == [] - - -def test_rejected_feed_keeps_lease_while_stream_active(fake, native): - # A feed rejected before the family hook (NaN/Inf -> InvalidArgument) - # leaves the native stream ACTIVE: the lease stays with it. - m = fake.model() - s1, s2 = fake.session(m), fake.session(m) - stream = s1.stream() - native.status["feed"] = _generated.TRANSCRIBE_ERR_INVALID_ARG - fake.state["value"] = _generated.TRANSCRIBE_STREAM_ACTIVE - with pytest.raises(t.InvalidArgument): - stream.feed(PCM) - assert not m._compute_lock.locked() - _assert_busy(s2, ran=native.calls) - _assert_busy(s1, ran=native.calls) - # A later valid feed and finalize then release it. - native.status["feed"] = 0 - stream.feed(PCM) - _assert_busy(s2, ran=native.calls) - stream.finalize() - _assert_free(s2) - - -def test_feed_failure_that_ends_stream_releases_lease(fake, native): - # A failure inside the family hook leaves the stream FAILED, which is no - # longer an active stream: the lease is released with the error. - m = fake.model() - s1, s2 = fake.session(m), fake.session(m) - stream = s1.stream() - native.status["feed"] = _generated.TRANSCRIBE_ERR_ABORTED - fake.state["value"] = _generated.TRANSCRIBE_STREAM_FAILED - with pytest.raises(t.Aborted): - stream.feed(PCM) - assert m._stream_owner is None - _assert_free(s2) - - def test_ended_stream_never_clears_a_later_lease(fake, native): m = fake.model() sa, sb, sc = fake.session(m), fake.session(m), fake.session(m) @@ -607,206 +509,23 @@ def test_ended_stream_never_clears_a_later_lease(fake, native): _assert_free(sc) -@pytest.mark.parametrize("begin_ok", [True, False]) -def test_busy_check_happens_after_acquiring_the_lock(fake, native, monkeypatch, - begin_ok): - # A run that is already waiting for the lock when a stream begins must - # see that stream's lease once it gets the lock; if the begin fails, - # no lease was taken and the queued run proceeds. - m = fake.model() - s1, s2 = fake.session(m), fake.session(m) - gate = _Gate() - begin_status = 0 if begin_ok else _generated.TRANSCRIBE_ERR_INVALID_ARG - - def begin(h, *rest): - gate(h) - return begin_status - - monkeypatch.setattr(t._lib, "transcribe_stream_begin", begin) - try: - a, a_box = _in_thread(s1.stream) - assert gate.entered.wait(TIMEOUT) - spy = _LockSpy(m) - b, b_box = _in_thread(s2.run, PCM) - assert spy.waiting.wait(TIMEOUT) # parked on the lock, before the lease - finally: - gate.release.set() - _join(a, "stream begin") - _join(b, "queued run") - if begin_ok: - assert isinstance(a_box.get("value"), t.Stream), a_box - assert isinstance(b_box.get("error"), t.Busy), b_box - assert native.calls == [] # the queued run never reached native - a_box["value"].reset() - else: - assert isinstance(a_box.get("error"), t.InvalidArgument), a_box - assert b_box.get("value") == ("result", s2._handle.value, None), b_box - assert not m._compute_lock.locked() - - -def test_stream_gc_on_holder_thread_defers_reset(fake, native, monkeypatch): - # GC can finalize an abandoned stream on the thread that holds the lock. - # Its reset + lease release must neither deadlock nor run under the - # in-flight call; it runs right after, and frees the model. - m = fake.model() - s, victim_session = fake.session(m), fake.session(m) - hv = victim_session._handle.value - during: dict = {} - - def run(h, *rest): - # Simulate a stream on victim_session that holds the lease and is - # dropped mid-call (a real begin here would be a re-entrant call). - lease = t._StreamLease(victim_session._handle) - m._stream_owner = lease - victim = t.Stream(victim_session, _lease=lease) - victim._cycle = victim - del victim - gc.collect() # runs Stream.__del__ on THIS thread, lock held - during["resets"] = list(fake.resets) - during["owner"] = m._stream_owner - return 0 - - monkeypatch.setattr(t._lib, "transcribe_run", run) - th, box = _in_thread(s.run, PCM) - _join(th, "run with a stream GC'd on the holder thread") - assert "error" not in box, box - assert during["resets"] == [], "stream reset under the in-flight call" - assert during["owner"] is not None, "lease released under the lock early" - assert fake.resets == [hv] - assert m._stream_owner is None - monkeypatch.setattr(t._lib, "transcribe_run", lambda *a: 0) - _assert_free(s) - - -def test_session_close_releases_lease_after_free(fake, native, monkeypatch): - # Closing the stream's session while another call holds the lock defers - # the free; the lease is released only after it, so the next call never - # starts while the stream's native session still exists. - m = fake.model() - s1, s2 = fake.session(m), fake.session(m) - h1 = s1._handle.value - stream = s1.stream() - gate = _Gate() - seen: dict = {} - - def run(h, *rest): - seen["frees"] = list(fake.frees) - return 0 - - monkeypatch.setattr(t._lib, "transcribe_stream_feed", gate) - monkeypatch.setattr(t._lib, "transcribe_run", run) - try: - a, a_box = _in_thread(stream.feed, PCM) - assert gate.entered.wait(TIMEOUT) - s1.close() # never waits; the free is queued behind the feed - assert fake.frees == [] and m._stream_owner is not None - spy = _LockSpy(m) - b, b_box = _in_thread(s2.run, PCM) - assert spy.waiting.wait(TIMEOUT) - finally: - gate.release.set() - _join(a, "in-flight feed") - _join(b, "queued run") - assert "error" not in a_box, a_box - assert b_box.get("value") == ("result", s2._handle.value, None), b_box - assert seen["frees"] == [("session", h1)] - assert m._stream_owner is None - del stream - gc.collect() - assert fake.resets == [] - - -# --- DiarizeSession (close / cancel are parametrized into the tests above) ---- +# --- DiarizeSession ------------------------------------------------------------- DIARIZE_BUSY = ("a stream is active on this model; " "finish or drop it before diarize run()") -@pytest.fixture -def diarize_native(fake, monkeypatch): - """Probes for the diarize entry points. ``calls`` records (name, handle, - whether this thread held ``owner["model"]``'s lock); two rows per run.""" - calls: list = [] - owner: dict = {} - - def held(): - m = owner["model"] - return m._compute_lock.locked() and m._compute_owner == threading.get_ident() - - def run(h, pcm, n, params): - calls.append(("run", h.value, held())) - return 0 - - def n_segments(h): - calls.append(("n_segments", h.value, held())) - return 2 - - def get_segment(h, i, out): - calls.append(("get_segment", h.value, held())) - row = out._obj - row.t0_ms, row.t1_ms, row.speaker_id, row.p = 100 * i, 100 * i + 50, i + 1, 0.5 - return 0 - - monkeypatch.setattr(t._lib, "transcribe_diarize_run", run) - monkeypatch.setattr(t._lib, "transcribe_diarize_n_segments", n_segments) - monkeypatch.setattr(t._lib, "transcribe_diarize_get_segment", get_segment) - return SimpleNamespace(calls=calls, owner=owner, run=run) - - -ROWS = [t.SpeakerSegment(t0_ms=0, t1_ms=50, speaker_id=1, p=0.5), - t.SpeakerSegment(t0_ms=100, t1_ms=150, speaker_id=2, p=0.5)] - - -def test_diarize_run_and_copy_out_hold_the_lock(fake, diarize_native): - m = fake.model() - d = fake.session(m, cls=t.DiarizeSession) - diarize_native.owner["model"] = m - assert d.run(PCM) == ROWS - names = [c[0] for c in diarize_native.calls] - assert names == ["run", "n_segments", "get_segment", "get_segment"] - assert all(c[1] == d._handle.value and c[2] for c in diarize_native.calls) - assert not m._compute_lock.locked() - - -def test_diarize_session_init_wires_cancel_and_registers(fake, monkeypatch): - m = fake.model() - hm = m._handle.value - cbs: list = [] - - def init(model_h, params, out): - out._obj.value = 0x9000 - return 0 - - monkeypatch.setattr(t._lib, "transcribe_diarize_session_init", init) - monkeypatch.setattr(t._lib, "transcribe_diarize_set_abort_callback", - lambda h, cb, ud: cbs.append((h.value, cb))) - d = m.diarize_session(n_threads=2) - assert d in set(m._sessions) - [(h, cb)] = cbs - assert h == 0x9000 and cb(None) is False - d.cancel() # lock-free; flips what the native abort callback polls - assert cb(None) is True - m.close() # closes the diarize session before freeing the model - assert fake.frees == [("diarize", 0x9000), ("model", hm)] - - -def test_diarize_session_init_failure_raises_unsupported_role(fake, monkeypatch): - m = fake.model() - monkeypatch.setattr(t._lib, "transcribe_diarize_session_init", - lambda mh, p, out: _generated.TRANSCRIBE_ERR_UNSUPPORTED_ROLE) - with pytest.raises(t.UnsupportedRole): - m.diarize_session() - assert not list(m._sessions) - - -def test_diarize_run_busy_while_stream_active(fake, native, diarize_native): +def test_diarize_run_busy_while_stream_active(fake, native, monkeypatch): m = fake.model() s, d = fake.session(m), fake.session(m, cls=t.DiarizeSession) - diarize_native.owner["model"] = m + calls: list = [] + monkeypatch.setattr(t._lib, "transcribe_diarize_run", + lambda *a: calls.append("run") or 0) + monkeypatch.setattr(t._lib, "transcribe_diarize_n_segments", lambda h: 0) stream = s.stream() with pytest.raises(t.Busy) as ei: d.run(PCM) assert str(ei.value) == DIARIZE_BUSY - assert diarize_native.calls == [] and not m._compute_lock.locked() + assert calls == [] and not m._compute_lock.locked() stream.finalize() - assert d.run(PCM) == ROWS + assert d.run(PCM) == [] and calls == ["run"] diff --git a/bindings/python/tests/test_diarize.py b/bindings/python/tests/test_diarize.py index 967af8d35..e5fe9deb6 100644 --- a/bindings/python/tests/test_diarize.py +++ b/bindings/python/tests/test_diarize.py @@ -47,32 +47,21 @@ def test_asr_only_model_rejects_diarize(model_path): model.diarize_session() -# CPU goldens on the oracle mix, (t0_ms, t1_ms, speaker_id) grouped by speaker. -GOLDEN = { - "default": [(320, 2400, 1), (7360, 9360, 1), (10240, 10640, 1), - (4240, 6640, 2), (9760, 12000, 2)], - "low_latency": [(320, 2480, 1), (7360, 9360, 1), (10240, 10640, 1), - (4160, 6640, 2), (9760, 12000, 2)], -} - - -@pytest.mark.parametrize("preset", sorted(GOLDEN)) -def test_diarize_run_golden_segments(sortformer_model_path, mix_pcm, preset): +def test_diarize_run_returns_turns_per_preset(sortformer_model_path, mix_pcm): + # Exact segments are pinned in C (tests/sortformer_diarize_unit.cpp); here + # only that rows come back well-formed and the preset reaches the run. with t.Model(sortformer_model_path, backend="cpu") as model: with model.diarize_session() as d: - rows = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset=preset)) + default = d.run(mix_pcm) + low = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset="low_latency")) timings = d.timings - assert all(math.isnan(r.p) for r in rows) # Sortformer has no per-turn p - assert _turns(rows) == GOLDEN[preset] + assert default + for r in default: + assert 1 <= r.speaker_id <= 4 and r.t0_ms < r.t1_ms and math.isnan(r.p) + assert _turns(low) != _turns(default) assert timings.encode_ms > 0 -def test_diarize_run_without_extension(sortformer_model_path, mix_pcm): - with t.Model(sortformer_model_path) as model, model.diarize_session() as d: - assert _turns(d.run(mix_pcm)) == _turns( - d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset="default"))) - - def test_bad_preset_rejected(sortformer_model_path, mix_pcm): class OutOfRange(t.SortformerDiarizeOptions): _presets = {**t.SortformerDiarizeOptions._presets, "bogus": 99} diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index 78d306d4b..f26901a46 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -97,13 +97,16 @@ def test_cancellation_aborts_pending_feed(streaming_model_path, audio_pcm): # Pre-set the cancel flag, then feed: the abort callback fires at the # first poll inside the feed. (True cross-thread mid-run cancellation is # covered by TestCancellation in test_transcribe.py.) Must surface as - # the dedicated Aborted class, not a generic TranscribeError. + # the dedicated Aborted class, not a generic TranscribeError. The stream + # is then FAILED, so it releases the model's stream lease. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: session.cancel() with pytest.raises(t.Aborted): stream.feed(audio_pcm) assert session.was_aborted, "was_aborted should be True after cancel()" + assert stream.state == "failed" + sibling.run(audio_pcm[:16000]) # not Busy # --- stream lifecycle edges --------------------------------------------------- @@ -215,35 +218,6 @@ def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): assert "country" in ran.lower(), ran -def test_hook_failure_feed_releases_lease(streaming_model_path, audio_pcm): - # A feed that fails INSIDE the family hook (here: a pending cancel the - # hook polls -> Aborted) leaves the stream FAILED, which is no longer - # active: the lease goes with it and a sibling run proceeds at once. - with t.Model(streaming_model_path) as model, model.session() as session: - with model.session() as sibling, session.stream() as stream: - stream.feed(audio_pcm[:16000]) - session.cancel() - with pytest.raises(t.Aborted): - stream.feed(audio_pcm[16000:32000]) - assert stream.state == "failed" - assert isinstance(stream.last_status, t.Aborted) - ran = sibling.run(audio_pcm).text - assert "country" in ran.lower(), ran - - -def test_stream_lease_released_without_finalize(streaming_model_path, audio_pcm): - # Closing the session of an active stream frees the model for others. - with t.Model(streaming_model_path) as model, model.session() as sibling: - session = model.session() - stream = session.stream() - stream.feed(audio_pcm[:16000]) - with pytest.raises(t.Busy): - sibling.run(audio_pcm[:16000]) - session.close() - ran = sibling.run(audio_pcm).text - assert "country" in ran.lower(), ran - - def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): # A cancel() requested before stream() is cleared by the begin: the new # stream's feeds are not aborted by a stale flag. (A cancel() AFTER begin @@ -254,20 +228,3 @@ def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): stream.feed(audio_pcm[:16000]) assert not session.was_aborted assert stream.state == "active" - - -def test_stream_keeps_session_and_model_alive(streaming_model_path, audio_pcm): - # The Stream holds a strong reference to its Session (which holds the - # Model): dropping every other reference must not free native handles - # under the stream. - import gc - - stream = t.Model(streaming_model_path).session().stream() - gc.collect() - for i in range(0, len(audio_pcm), 16000): - stream.feed(audio_pcm[i : i + 16000]) - stream.finalize() - assert "country" in stream.text().committed.lower() - stream.reset() - del stream - gc.collect() From 64f1387163935c6c51851b095e8af35f0914b47b Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:56:56 +0800 Subject: [PATCH 47/51] swift: drop ComputeRulesTests, keep the Busy checks ComputeRulesTests pinned pre-existing locking through internal state and 0.3 s lock-wait timeouts. The two checks worth keeping move to existing files: StreamingTests.testActiveStreamRefusesSiblingCompute (restores the Busy coverage deleted from StreamingTests) and DiarizeTests.testDiarizeRunIsBusyWhileAStreamIsActive. DiarizeTests: exact segments live in tests/sortformer_diarize_unit.cpp; check well-formed turns and that lowLatency changes the result. The status-mapping test compares case names instead of a 20-line switch. --- .../TranscribeCppTests/CancelTests.swift | 1 - .../ComputeRulesTests.swift | 250 ------------------ .../TranscribeCppTests/DiarizeTests.swift | 49 ++-- .../TranscribeCppTests/NoModelTests.swift | 27 +- .../TranscribeCppTests/StreamingTests.swift | 27 ++ 5 files changed, 59 insertions(+), 295 deletions(-) delete mode 100644 bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift diff --git a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift index 024b217e3..c571419b5 100644 --- a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift @@ -30,7 +30,6 @@ final class CancelTests: XCTestCase { } } XCTAssertTrue(session.wasAborted) - XCTAssertTrue(session.cancelToken === token, "a sync run leaves the caller's token installed") } func testCrossThreadCancelOfInFlightRun() throws { diff --git a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift b/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift deleted file mode 100644 index 99eb88422..000000000 --- a/bindings/swift/Tests/TranscribeCppTests/ComputeRulesTests.swift +++ /dev/null @@ -1,250 +0,0 @@ -import Foundation -import XCTest - -@testable import TranscribeCpp - -/// Every compute site waits on the model-wide lock; option checks run before it, the busy check after. -final class ComputeRulesTests: XCTestCase { - // MARK: - Helpers - - private final class Box { var value: T? } - - /// Hold `model.runLock` on this thread, run `op` on a background queue, - /// and require it to still be blocked after a short wait; then release the - /// lock and require it to finish. Bounded waits, so a regression fails - /// rather than hanging the suite. - private func assertWaitsOnModelLock( - _ model: Model, _ what: String, - file: StaticString = #filePath, line: UInt = #line, - _ op: @escaping () -> Void - ) { - let done = DispatchSemaphore(value: 0) - model.runLock.lock() - var held = true - defer { if held { model.runLock.unlock() } } - DispatchQueue.global().async { op(); done.signal() } - let early = done.wait(timeout: .now() + 0.3) - XCTAssertEqual( - early, .timedOut, "\(what) must wait on the model compute lock", - file: file, line: line) - model.runLock.unlock() - held = false - if early == .success { return } - XCTAssertEqual( - done.wait(timeout: .now() + 60), .success, - "\(what) must proceed once the model lock is free", file: file, line: line) - } - - /// Hold `model.runLock` on this thread and require `op` (on a background - /// queue) to finish anyway: it must not touch the lock. - private func assertSkipsModelLock( - _ model: Model, _ what: String, - file: StaticString = #filePath, line: UInt = #line, - _ op: @escaping () -> Void - ) { - let done = DispatchSemaphore(value: 0) - model.runLock.lock() - DispatchQueue.global().async { op(); done.signal() } - let result = done.wait(timeout: .now() + 10) - model.runLock.unlock() - XCTAssertEqual( - result, .success, "\(what) must not wait on the model compute lock", - file: file, line: line) - if result == .timedOut { _ = done.wait(timeout: .now() + 60) } - } - - private func busyMessage(_ error: Error?) -> String? { - guard case TranscribeError.busy(let message)? = error else { return nil } - return message - } - - private func isInvalidArgument(_ error: Error?) -> Bool { - guard case TranscribeError.invalidArgument? = error else { return false } - return true - } - - // MARK: - Model-wide compute exclusion - - func testRunAndRunBatchWaitOnTheModelLock() throws { - let (path, pcm) = try Fixtures.modelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - - let run = Box() - assertWaitsOnModelLock(model, "run") { run.value = try? s1.run(pcm) } - XCTAssertTrue(run.value?.text.lowercased().contains("country") == true) - - // A sibling session of the same model waits on the same lock. - let batch = Box<[Result]>() - assertWaitsOnModelLock(model, "runBatch") { batch.value = try? s2.runBatch([pcm]) } - XCTAssertEqual(batch.value?.count, 1) - XCTAssertTrue( - (try? batch.value?.first?.get().text.lowercased().contains("country")) == true) - } - - func testEveryStreamSiteWaitsOnTheModelLock() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let model = try Model(path: path) - let session = try model.session() - let chunk = Array(pcm.prefix(1600)) - let stream = Box() - - assertWaitsOnModelLock(model, "stream begin") { stream.value = try? session.stream() } - XCTAssertNotNil(stream.value) - XCTAssertTrue(model.streamActive) - - let fed = Box() - assertWaitsOnModelLock(model, "feed") { fed.value = try? stream.value?.feed(chunk) } - XCTAssertNotNil(fed.value) - - let finalized = Box() - assertWaitsOnModelLock(model, "finalize") { - finalized.value = try? stream.value?.finalize() - } - XCTAssertEqual(finalized.value?.isFinal, true) - XCTAssertFalse(model.streamActive) - stream.value = nil - - stream.value = try session.stream() - _ = try stream.value?.feed(chunk) - let state = Box() - assertWaitsOnModelLock(model, "reset") { state.value = stream.value?.reset() } - XCTAssertEqual(state.value, .idle) - XCTAssertFalse(model.streamActive) - stream.value = nil - - // A dropped ACTIVE stream resets under the lock in deinit. The last - // reference is released on the background thread so the test thread - // (which holds the lock) never runs the deinit itself. - stream.value = try session.stream() - _ = try stream.value?.feed(chunk) - XCTAssertTrue(model.streamActive) - assertWaitsOnModelLock(model, "Stream deinit") { stream.value = nil } - XCTAssertFalse(model.streamActive) - XCTAssertEqual(try session.stream().reset(), .idle) - } - - /// A diarize run follows the Session rules: it keeps its model alive, - /// waits on the model lock, and refuses a held stream lease with `.busy` - /// only once the lock is taken. - func testDiarizeRunSharesTheModelLockAndLease() throws { - let (path, pcm) = try Fixtures.sortformerModelAndAudio() - weak var weakModel: Model? - let session: DiarizeSession = try { - let model = try Model(path: path) - weakModel = model - return try model.diarizeSession() - }() - XCTAssertNotNil(weakModel, "a live DiarizeSession must keep its Model alive") - let model = session.model - - let rows = Box<[SpeakerSegment]>() - assertWaitsOnModelLock(model, "diarize run") { rows.value = try? session.run(pcm) } - XCTAssertEqual(rows.value?.isEmpty, false) - - // Sortformer cannot stream, so the lease a stream would hold is set directly. - model.withCompute { model.streamActive = true } - defer { model.withCompute { model.streamActive = false } } - let error = Box() - assertWaitsOnModelLock(model, "diarize run while streaming") { - do { _ = try session.run(pcm) } catch let e { error.value = e } - } - XCTAssertEqual( - busyMessage(error.value), - "a stream is active on this model; finish or drop it before diarize run()") - } - - // MARK: - Active-stream rejection and its ordering - - /// The `.busy` check runs only once the lock is held: with a stream active - /// and the lock held elsewhere, run / runBatch / stream block first and - /// then refuse. Also pins the exact busy messages. - func testActiveStreamIsRefusedAfterTakingTheLock() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - let active = try s1.stream() - _ = try active.feed(Array(pcm.prefix(1600))) - - let runError = Box() - assertWaitsOnModelLock(model, "run while streaming") { - do { _ = try s2.run(pcm) } catch { runError.value = error } - } - XCTAssertEqual( - busyMessage(runError.value), - "a stream is active on this model; finish or drop it before run()") - - let batchError = Box() - assertWaitsOnModelLock(model, "runBatch while streaming") { - do { _ = try s2.runBatch([pcm]) } catch { batchError.value = error } - } - XCTAssertEqual( - busyMessage(batchError.value), - "a stream is active on this model; finish or drop it before runBatch()") - - let streamError = Box() - assertWaitsOnModelLock(model, "stream while streaming") { - do { _ = try s2.stream() } catch { streamError.value = error } - } - XCTAssertEqual(busyMessage(streamError.value), "a stream is already active on this model") - - active.reset() - } - - /// String-option validation runs before the lock and before the busy - /// check: with a stream active AND the lock held, a NUL in an option is - /// reported as `.invalidArgument` without waiting. - func testOptionValidationPrecedesLockAndBusyCheck() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - let active = try s1.stream() - let bad = RunOptions(language: "e\0n") - - let runError = Box() - assertSkipsModelLock(model, "run option check") { - do { _ = try s2.run(pcm, options: bad) } catch { runError.value = error } - } - XCTAssertTrue(isInvalidArgument(runError.value), "\(String(describing: runError.value))") - - let batchError = Box() - assertSkipsModelLock(model, "runBatch option check") { - do { _ = try s2.runBatch([pcm], options: bad) } catch { batchError.value = error } - } - XCTAssertTrue(isInvalidArgument(batchError.value), "\(String(describing: batchError.value))") - - let streamError = Box() - assertSkipsModelLock(model, "stream option check") { - do { _ = try s2.stream(bad) } catch { streamError.value = error } - } - XCTAssertTrue( - isInvalidArgument(streamError.value), "\(String(describing: streamError.value))") - - active.reset() - } - - // MARK: - Keep-alive - - func testStreamKeepsSessionAndModelAlive() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - weak var weakModel: Model? - weak var weakSession: Session? - var stream: TranscribeCpp.Stream? = try { - let model = try Model(path: path) - let session = try model.session() - weakModel = model - weakSession = session - return try session.stream() - }() - XCTAssertNotNil(weakSession, "a live Stream must keep its Session alive") - XCTAssertNotNil(weakModel, "a live Stream must keep its Model alive") - try Fixtures.drive(stream!, pcm: pcm) - XCTAssertTrue(stream!.text.full.lowercased().contains("country")) - stream = nil - XCTAssertNil(weakSession) - XCTAssertNil(weakModel) - } -} diff --git a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift index 6b07754da..1315e240b 100644 --- a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift @@ -2,8 +2,7 @@ import XCTest @testable import TranscribeCpp -/// DIARIZE role (`DiarizeSession`). Lock / lease / keep-alive rules are in -/// ComputeRulesTests. +/// DIARIZE role (`DiarizeSession`). final class DiarizeTests: XCTestCase { /// `p` is NaN on diarize rows, so compare the turns, not `SpeakerSegment`s. private func turns(_ rows: [SpeakerSegment]) -> [[Int64]] { @@ -43,31 +42,24 @@ final class DiarizeTests: XCTestCase { ("session", { _ = try model.session() }), ]) - // (t0Ms, t1Ms, speakerId) on samples/sortformer-2spk-mix.wav. Only - // .lowLatency moves the turns here, so it shows the preset reaches the - // native run. - let golden: [(SortformerPreset, [[Int64]])] = [ - (.default, [[320, 2400, 1], [7360, 9360, 1], [10240, 10640, 1], - [4240, 6640, 2], [9760, 12000, 2]]), - (.lowLatency, [[320, 2480, 1], [7360, 9360, 1], [10240, 10640, 1], - [4160, 6640, 2], [9760, 12000, 2]]), - ] + // Segment values are pinned by the C test; here, check the rows are sane + // and that the preset reaches the native run (.lowLatency moves turns). let session = try model.diarizeSession() - for (preset, want) in golden { - let rows = try session.run( - pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: preset)))) - XCTAssertEqual(turns(rows), want, "\(preset)") + let base = try session.run(pcm) + XCTAssertFalse(base.isEmpty) + for row in base { + XCTAssertTrue((1...4).contains(row.speakerId), "\(row)") + XCTAssertLessThan(row.t0Ms, row.t1Ms, "\(row)") } XCTAssertGreaterThan(session.timings.encodeMs, 0) - XCTAssertEqual(turns(try session.run(pcm)), golden[0].1, "no extension == .default") + let low = try session.run( + pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: .lowLatency)))) + XCTAssertNotEqual(turns(low), turns(base), ".lowLatency must reach the native run") } func testCancelledRunAbortsAndRecovers() throws { let (path, pcm) = try Fixtures.sortformerModelAndAudio() let session = try Model(path: path).diarizeSession() - let first = try session.run(pcm) - XCTAssertFalse(first.isEmpty) - let token = CancellationToken() token.cancel() session.setCancellationToken(token) @@ -77,6 +69,23 @@ final class DiarizeTests: XCTestCase { } } session.clearCancellationToken() - XCTAssertEqual(turns(try session.run(pcm)), turns(first)) + XCTAssertFalse(try session.run(pcm).isEmpty) + } + + /// A diarize run shares the model's stream lease with ASR sessions. + func testDiarizeRunIsBusyWhileAStreamIsActive() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + let model = try Model(path: path) + let session = try model.diarizeSession() + // Sortformer cannot stream, so set the lease a stream would hold directly. + model.withCompute { model.streamActive = true } + defer { model.withCompute { model.streamActive = false } } + XCTAssertThrowsError(try session.run(pcm)) { error in + guard case TranscribeError.busy(let message) = error else { + return XCTFail("expected .busy, got \(error)") + } + XCTAssertEqual( + message, "a stream is active on this model; finish or drop it before diarize run()") + } } } diff --git a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift index 56e21e173..77d5d0f5a 100644 --- a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift @@ -49,8 +49,7 @@ final class NoModelTests: XCTestCase { XCTAssertNil(TranscribeError.inputTooLong("").partial) } - // Every native status maps to its case. `label` has no `default`, so a new - // TranscribeError case does not compile until this table is updated; the + // Every native status maps to its case (compared by case name); the // "unknown status" check below flags a newly appended C status. func testEveryStatusMapsToItsCase() { let expected: [Int32: String] = [ @@ -66,33 +65,13 @@ final class NoModelTests: XCTestCase { for raw in 1...20 { let status = transcribe_status(rawValue: UInt32(raw)) XCTAssertNotEqual(Transcribe.statusString(Int32(raw)), "unknown status", "status \(raw)") - XCTAssertEqual(label(TranscribeError.make(status)), expected[Int32(raw)], "status \(raw)") + let name = String(describing: TranscribeError.make(status)).prefix { $0 != "(" } + XCTAssertEqual(String(name), expected[Int32(raw)], "status \(raw)") } XCTAssertEqual(Transcribe.statusString(21), "unknown status", "a new status was appended; map it in TranscribeError.make") } - private func label(_ error: TranscribeError) -> String { - switch error { - case .invalidArgument: return "invalidArgument" - case .notImplemented: return "notImplemented" - case .modelFileNotFound: return "modelFileNotFound" - case .modelLoad: return "modelLoad" - case .outOfMemory: return "outOfMemory" - case .backend: return "backend" - case .unsupported: return "unsupported" - case .badStructSize: return "badStructSize" - case .inputTooLong: return "inputTooLong" - case .aborted: return "aborted" - case .outputTruncated: return "outputTruncated" - case .outputRepetition: return "outputRepetition" - case .unsupportedRole: return "unsupportedRole" - case .versionMismatch: return "versionMismatch" - case .busy: return "busy" - case .other: return "other" - } - } - func testAtLeastOneDevice() { XCTAssertGreaterThanOrEqual(Transcribe.devices().count, 1) } diff --git a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift index 1d72d5f8c..d0476924a 100644 --- a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift @@ -131,4 +131,31 @@ final class StreamingTests: XCTestCase { _ = stream1 } } + + /// While a stream is active, run / runBatch / stream on any session of the + /// model throw `.busy`. + func testActiveStreamRefusesSiblingCompute() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + let active = try s1.stream() + _ = try active.feed(Array(pcm.prefix(1600))) + + let cases: [(String, () throws -> Void)] = [ + ("a stream is active on this model; finish or drop it before run()", { _ = try s2.run(pcm) }), + ("a stream is active on this model; finish or drop it before runBatch()", + { _ = try s2.runBatch([pcm]) }), + ("a stream is already active on this model", { _ = try s2.stream() }), + ] + for (message, call) in cases { + XCTAssertThrowsError(try call()) { error in + guard case TranscribeError.busy(let got) = error else { + return XCTFail("expected .busy, got \(error)") + } + XCTAssertEqual(got, message) + } + } + active.reset() + } } From 4723ae9de0fb7d7168c7df138275ac825bebaa96 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 09:57:39 +0800 Subject: [PATCH 48/51] ts: drop compute-rules.test.mjs, keep the two lease fixes compute-rules.test.mjs pinned pre-existing #exclusive behavior. The two tests for what this PR changed (a pre-hook feed rejection keeps the stream lease; an in-hook feed failure releases it) move to streaming.test.mjs next to the other lease tests. diarize.test.mjs: drop the public-surface, abort-listener and dispose tests (same shared SessionCore path); exact segments live in tests/sortformer_diarize_unit.cpp, so check well-formed turns and that low_latency changes the result. errors.test.mjs: keep the status table and unknown-status tests only. --- .../typescript/test/compute-rules.test.mjs | 385 ------------------ bindings/typescript/test/diarize.test.mjs | 110 +---- bindings/typescript/test/errors.test.mjs | 12 - bindings/typescript/test/streaming.test.mjs | 75 +++- 4 files changed, 91 insertions(+), 491 deletions(-) delete mode 100644 bindings/typescript/test/compute-rules.test.mjs diff --git a/bindings/typescript/test/compute-rules.test.mjs b/bindings/typescript/test/compute-rules.test.mjs deleted file mode 100644 index d1a523509..000000000 --- a/bindings/typescript/test/compute-rules.test.mjs +++ /dev/null @@ -1,385 +0,0 @@ -// The rules every native compute call obeys: model-wide exclusion, Busy and -// disposed refusals, abort install/cleanup, deferred frees, and copy-out. - -import assert from "node:assert/strict"; -import { getEventListeners } from "node:events"; -import koffi from "koffi"; -import { modelTest, MODEL, STREAMING_MODEL, jfk } from "./common.mjs"; -import { TranscribeModel, Aborted, Busy, InvalidArgument, libraryPath } from "../dist/index.js"; - -const second = () => jfk().subarray(0, 16000); -const half = () => { - const pcm = jfk(); - return pcm.subarray(0, pcm.length / 2); -}; - -// ---- model-wide exclusion -------------------------------------------------- - -modelTest("compute is exclusive model-wide: a sibling run waits its turn", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const a = m.createSession(); - const b = m.createSession(); - const order = []; - const pa = a.run(jfk()).then((r) => (order.push("a"), r)); - const pb = b.run(jfk()).then((r) => (order.push("b"), r)); - await Promise.resolve(); // a's body reaches its worker; b's is queued - // Only the lock holder is marked in flight; the queued sibling is not. - assert.throws(() => a.limits, /run\(\).*in flight/); - assert.doesNotThrow(() => b.limits); - const [ra, rb] = await Promise.all([pa, pb]); - assert.deepEqual(order, ["a", "b"], "FIFO: the queued sibling runs after"); - assert.match(ra.text, /ask not/i); - assert.match(rb.text, /ask not/i); - a.dispose(); - b.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("runBatch and finalize mark the session in flight with their own label", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const s = m.createSession(); - const pending = s.runBatch([second()]); - await Promise.resolve(); - assert.throws(() => s.limits, /runBatch\(\).*in flight/); - await pending; - assert.doesNotThrow(() => s.limits); - - const stream = await s.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - const fin = stream.finalize(); - await Promise.resolve(); - assert.throws(() => stream.text, /feed\(\)\/finalize\(\).*in flight/); - assert.throws(() => s.wasAborted, /feed\(\)\/finalize\(\).*in flight/); - await fin; - assert.doesNotThrow(() => stream.text); - stream.reset(); - s.dispose(); - } finally { - m.dispose(); - } -}); - -// ---- active-stream rejection ----------------------------------------------- - -modelTest("an active stream refuses every offline call and begin, same or sibling session", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const a = m.createSession(); - const b = m.createSession(); - const stream = await a.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - for (const s of [a, b]) { - await assert.rejects(() => s.run(second()), (e) => e instanceof Busy && /^cannot run:/.test(e.message)); - await assert.rejects( - () => s.runBatch([second()]), - (e) => e instanceof Busy && /^cannot runBatch:/.test(e.message), - ); - await assert.rejects( - () => s.stream(), - (e) => e instanceof Busy && /^cannot begin a stream:/.test(e.message), - ); - } - // The stream holding the lease keeps feeding through all of that. - await stream.feed(second()); - await stream.finalize(); - stream.reset(); - a.dispose(); - b.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("disposal is checked before the stream lease inside the lock", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const a = m.createSession(); - const stream = await a.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - // Each call is queued while a's stream holds the lease, then its session - // is disposed before the queued body runs: the caller sees "disposed", - // not Busy. - for (const call of [(s) => s.run(second()), (s) => s.runBatch([second()]), (s) => s.stream()]) { - const b = m.createSession(); - const p = call(b); - b.dispose(); - await assert.rejects(p, (e) => !(e instanceof Busy) && /disposed/i.test(e.message)); - } - stream.reset(); - a.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("a feed or finalize queued before session dispose still runs", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - // feed/finalize check disposal only before queuing, not inside the lock: - // a call already queued when the session is disposed runs to completion - // (the native free is queued behind it). - const a = m.createSession(); - const sa = await a.stream({ commitPolicy: "stable_prefix" }); - const feed = sa.feed(second()); - a.dispose(); - const u = await feed; - assert.equal(typeof u.revision, "number"); - - const b = m.createSession(); - const sb = await b.stream({ commitPolicy: "stable_prefix" }); - await sb.feed(second()); - const fin = sb.finalize(); - b.dispose(); - const f = await fin; - assert.equal(f.isFinal, true); - - // Both leases are gone: a fresh session can stream. - const c = m.createSession(); - const sc = await c.stream({ commitPolicy: "stable_prefix" }); - sc.reset(); - c.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("a feed rejected before the native hook keeps the stream lease", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const a = m.createSession(); - const b = m.createSession(); - const stream = await a.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - const bad = new Float32Array(1600); - bad[5] = Number.NaN; - // Native rejects non-finite samples before the family hook and leaves the - // stream ACTIVE, so the binding keeps the model-wide lease. - await assert.rejects(() => stream.feed(bad), InvalidArgument); - assert.equal(stream.state, "active"); - await assert.rejects(() => b.run(second()), Busy); - - // The stream is still usable, and finalize releases the lease as usual. - const u = await stream.feed(jfk().subarray(16000, 32000)); - assert.equal(typeof u.revision, "number"); - await stream.finalize(); - const r = await b.run(second()); - assert.equal(typeof r.text, "string"); - stream.reset(); - a.dispose(); - b.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("a feed that fails inside the native hook releases the stream lease", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const a = m.createSession(); - const b = m.createSession(); - const stream = await a.stream({ commitPolicy: "stable_prefix" }); - await stream.feed(second()); - - // Provoke a hook failure: an abort callback that always fires makes the - // streaming hook return ABORTED, which moves the native stream to FAILED. - // feed() has no signal option, so install it on the native session - // directly (test-only; the same library instance the binding loaded). - const lib = koffi.load(libraryPath()); - const proto = koffi.proto("bool ComputeRulesTestAbortCb(void *udata)"); - const setAbort = lib.func("transcribe_set_abort_callback", "void", [ - "void *", - koffi.pointer(proto), - "void *", - ]); - const cb = koffi.register(() => true, koffi.pointer(proto)); - setAbort(a.handle, cb, null); - try { - await assert.rejects(() => stream.feed(second()), Aborted); - } finally { - setAbort(a.handle, null, null); - koffi.unregister(cb); - } - assert.equal(stream.state, "failed"); - assert.ok(stream.lastStatus instanceof Aborted); - - // FAILED is no longer an active stream: the lease is free for a sibling. - const r = await b.run(second()); - assert.equal(typeof r.text, "string"); - stream.reset(); - a.dispose(); - b.dispose(); - } finally { - m.dispose(); - } -}); - -// ---- cancel install / cleanup ---------------------------------------------- - -modelTest("the abort listener lives only for the call and is removed after", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const s = m.createSession(); - const ac = new AbortController(); - const p = s.run(jfk(), { signal: ac.signal }); - await Promise.resolve(); - assert.equal(getEventListeners(ac.signal, "abort").length, 1, "installed for the run"); - const r = await p; - assert.equal(r.aborted, false); - assert.equal(getEventListeners(ac.signal, "abort").length, 0, "removed after the run"); - - const pb = s.runBatch([second()], { signal: ac.signal }); - await Promise.resolve(); - assert.equal(getEventListeners(ac.signal, "abort").length, 1, "installed for the batch"); - await pb; - assert.equal(getEventListeners(ac.signal, "abort").length, 0, "removed after the batch"); - s.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("an aborted call uninstalls its callback; the next call is not aborted", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const s = m.createSession(); - const ac = new AbortController(); - ac.abort(); - const items = await s.runBatch([jfk(), second()], { signal: ac.signal }); - assert.equal(items.length, 2); - assert.ok(items.every((it) => it.error instanceof Aborted)); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - await assert.rejects(() => s.run(jfk(), { signal: ac.signal }), Aborted); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - - const r = await s.run(jfk()); - assert.equal(r.aborted, false); - assert.equal(s.wasAborted, false); - assert.match(r.text, /ask not/i); - s.dispose(); - } finally { - m.dispose(); - } -}); - -modelTest("a refused call never installs its abort listener", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - try { - const a = m.createSession(); - const b = m.createSession(); - const stream = await a.stream({ commitPolicy: "stable_prefix" }); - const ac = new AbortController(); - await assert.rejects(() => b.run(second(), { signal: ac.signal }), Busy); - await assert.rejects(() => b.runBatch([second()], { signal: ac.signal }), Busy); - assert.equal(getEventListeners(ac.signal, "abort").length, 0, "Busy is decided before install"); - - const c = m.createSession(); - const p = c.run(second(), { signal: ac.signal }); - c.dispose(); - await assert.rejects(p, /disposed/i); - assert.equal(getEventListeners(ac.signal, "abort").length, 0, "disposed is decided before install"); - - stream.reset(); - a.dispose(); - b.dispose(); - } finally { - m.dispose(); - } -}); - -// ---- dispose racing an in-flight call; keepalive; copy-out ----------------- - -modelTest("session dispose during an in-flight run defers the free; the result survives", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const s = m.createSession(); - const p = s.run(jfk()); - await Promise.resolve(); // the run is on its worker - s.dispose(); - // The in-flight guard is checked before disposal on reads. - assert.throws(() => s.limits, /run\(\).*in flight/); - const r = await p; - assert.match(r.text, /ask not what your country/i); - assert.ok(r.segments.length >= 1); - assert.throws(() => s.limits, /disposed/); - - const s2 = m.createSession(); - const pb = s2.runBatch([jfk(), half()]); - await Promise.resolve(); - s2.dispose(); - const items = await pb; - assert.equal(items.length, 2); - assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); - } finally { - m.dispose(); - } -}); - -modelTest("dispose during a run with a signal returns the result and removes the listener", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const s = m.createSession(); - const ac = new AbortController(); - const p = s.run(jfk(), { signal: ac.signal }); - await Promise.resolve(); - s.dispose(); - assert.match((await p).text, /ask not what your country/i); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - } finally { - m.dispose(); - } -}); - -modelTest("model dispose during an in-flight run keeps the model alive for the call", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - const s = m.createSession(); - const p = s.run(jfk()); - await Promise.resolve(); - m.dispose(); // disposes s too; both native frees queue behind the run - assert.throws(() => m.capabilities, /disposed/); - const r = await p; - assert.match(r.text, /ask not what your country/i); - assert.throws(() => s.limits, /disposed/); - - const m2 = await TranscribeModel.load(MODEL); - const s2 = m2.createSession(); - const pb = s2.runBatch([jfk()]); - await Promise.resolve(); - m2.dispose(); - const items = await pb; - assert.ok(items[0].ok && /ask not what your country/i.test(items[0].result.text)); -}); - -modelTest("model dispose during an in-flight feed keeps the model alive for the call", STREAMING_MODEL, async () => { - const m = await TranscribeModel.load(STREAMING_MODEL); - const s = m.createSession(); - const stream = await s.stream({ commitPolicy: "stable_prefix" }); - const p = stream.feed(second()); - await Promise.resolve(); - m.dispose(); - const u = await p; - assert.equal(typeof u.revision, "number"); - assert.throws(() => stream.text, /disposed/i); -}); - -modelTest("results are copied out before the next call on the session overwrites them", MODEL, async () => { - const m = await TranscribeModel.load(MODEL); - try { - const s = m.createSession(); - // Both run on the same session back to back; the second overwrites the - // session's native result storage as soon as it gets the lock. - const [full, part] = await Promise.all([s.run(jfk()), s.run(half())]); - assert.match(full.text, /ask what you can do for your country/i); - assert.doesNotMatch(part.text, /ask what you can do for your country/i); - - const [b1, b2] = await Promise.all([s.runBatch([jfk()]), s.runBatch([half()])]); - assert.match(b1[0].result.text, /ask what you can do for your country/i); - assert.doesNotMatch(b2[0].result.text, /ask what you can do for your country/i); - s.dispose(); - } finally { - m.dispose(); - } -}); diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs index ff700be60..6e6981216 100644 --- a/bindings/typescript/test/diarize.test.mjs +++ b/bindings/typescript/test/diarize.test.mjs @@ -1,32 +1,13 @@ -// DIARIZE role: model.roles, diarizeInfo, DiarizeSession, and the compute -// rules a diarize run shares with Session (lock, in-flight mark, abort, -// disposed recheck, deferred free). +// DIARIZE role: model.roles, diarizeInfo and DiarizeSession wiring. Segment +// values are pinned by the C test (sortformer_diarize_unit); here we check the +// binding returns well-formed rows and passes the preset through. -import { test } from "node:test"; import assert from "node:assert/strict"; -import { getEventListeners } from "node:events"; import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav } from "./common.mjs"; -import { - TranscribeModel, - DiarizeSession, - Aborted, - InvalidArgument, - TranscribeError, - UnsupportedRole, -} from "../dist/index.js"; +import { TranscribeModel, InvalidArgument, TranscribeError, UnsupportedRole } from "../dist/index.js"; const mix = () => readWav(SORTFORMER_AUDIO); -// Invariant 11: a role session never exposes its native handle. -test("DiarizeSession exposes only its public surface", () => { - assert.deepEqual(Object.getOwnPropertyNames(DiarizeSession.prototype).sort(), [ - "constructor", - "dispose", - "run", - "timings", - ]); -}); - modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MODEL, async () => { const m = await TranscribeModel.load(MODEL); try { @@ -52,22 +33,24 @@ modelTest("sortformer serves only diarize; diarizeInfo; ASR calls are refused", } }); -// Goldens are CPU (as sortformer_diarize_unit): (t0Ms, t1Ms, speakerId), -// grouped by speaker and time-ordered within one. -const GOLDEN = { - default: [[320, 2400, 1], [7360, 9360, 1], [10240, 10640, 1], [4240, 6640, 2], [9760, 12000, 2]], - low_latency: [[320, 2480, 1], [7360, 9360, 1], [10240, 10640, 1], [4160, 6640, 2], [9760, 12000, 2]], -}; - -modelTest("diarize run returns the golden speaker turns per preset", SORTFORMER_MODEL, async () => { +modelTest("diarize run returns speaker turns and honors the preset", SORTFORMER_MODEL, async () => { const m = await TranscribeModel.load(SORTFORMER_MODEL, { backend: "cpu" }); try { const pcm = mix(); const d = m.createDiarizeSession({ nThreads: 4 }); - for (const [preset, want] of Object.entries(GOLDEN)) { - const rows = await d.run(pcm, { family: { kind: "sortformer_diarize", preset } }); - assert.deepEqual(rows.map((r) => [r.t0Ms, r.t1Ms, r.speakerId]), want, preset); + const turns = async (preset) => + (await d.run(pcm, { family: { kind: "sortformer_diarize", preset } })).map((r) => [ + r.t0Ms, + r.t1Ms, + r.speakerId, + ]); + const def = await turns("default"); + assert.ok(def.length > 0); + for (const [t0, t1, spk] of def) { + assert.ok(t0 < t1, `${t0} < ${t1}`); + assert.ok(spk >= 1 && spk <= m.diarizeInfo.maxSpeakers, `speaker ${spk}`); } + assert.notDeepEqual(await turns("low_latency"), def, "preset reaches the native run"); assert.ok(d.timings.encodeMs > 0); d.dispose(); } finally { @@ -95,62 +78,3 @@ modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, m.dispose(); } }); - -// ---- compute rules shared with Session ------------------------------------- -// No Busy test: no model serves both a stream lease and the diarize role. - -modelTest("the abort listener lives only for the call; a pre-aborted run raises Aborted", SORTFORMER_MODEL, async () => { - const m = await TranscribeModel.load(SORTFORMER_MODEL); - try { - const d = m.createDiarizeSession(); - const ac = new AbortController(); - const p = d.run(mix(), { signal: ac.signal }); - await Promise.resolve(); - assert.equal(getEventListeners(ac.signal, "abort").length, 1); - assert.ok((await p).length > 0); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - - ac.abort(); - await assert.rejects(() => d.run(mix(), { signal: ac.signal }), Aborted); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - assert.ok((await d.run(mix())).length > 0, "the next run is not aborted"); - } finally { - m.dispose(); - } -}); - -modelTest("a diarize run queued before its dispose is rejected as disposed", SORTFORMER_MODEL, async () => { - const m = await TranscribeModel.load(SORTFORMER_MODEL); - try { - const running = m.createDiarizeSession().run(mix()); - const d = m.createDiarizeSession(); - const ac = new AbortController(); - const queued = d.run(mix(), { signal: ac.signal }); - d.dispose(); - await assert.rejects(queued, /disposed/i); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - await running; - } finally { - m.dispose(); - } -}); - -modelTest("dispose during an in-flight diarize run defers the free; the rows survive", SORTFORMER_MODEL, async () => { - const m = await TranscribeModel.load(SORTFORMER_MODEL); - const d = m.createDiarizeSession(); - const ac = new AbortController(); - const p = d.run(mix(), { signal: ac.signal }); - await Promise.resolve(); - d.dispose(); - assert.throws(() => d.timings, /run\(\).*in flight/); - assert.ok((await p).length > 0); - assert.equal(getEventListeners(ac.signal, "abort").length, 0); - assert.throws(() => d.timings, /disposed/); - - const d2 = m.createDiarizeSession(); - const p2 = d2.run(mix()); - await Promise.resolve(); - m.dispose(); // disposes d2 too; both native frees queue behind the run - assert.throws(() => m.diarizeInfo, /disposed/); - assert.ok((await p2).length > 0); -}); diff --git a/bindings/typescript/test/errors.test.mjs b/bindings/typescript/test/errors.test.mjs index eafec6dc3..0c0d13a2c 100644 --- a/bindings/typescript/test/errors.test.mjs +++ b/bindings/typescript/test/errors.test.mjs @@ -68,11 +68,6 @@ test("every transcribe_status maps to its documented error class", () => { ); } assert.equal(Object.getPrototypeOf(UnsupportedRole), TranscribeError); - assert.equal(g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE, 20); -}); - -test("OK does not throw", () => { - assert.equal(raiseForStatus(g.TRANSCRIBE_OK, "ok"), undefined); }); test("unknown status degrades to the base class", () => { @@ -80,10 +75,3 @@ test("unknown status degrades to the base class", () => { assert.equal(e.constructor, TranscribeError); assert.equal(e.status, 999); }); - -test("OutputRepetition is an OutputTruncated", () => { - const e = exceptionForStatus(g.TRANSCRIBE_ERR_OUTPUT_REPETITION, "looped", "run"); - assert.ok(e instanceof OutputRepetition); - assert.ok(e instanceof OutputTruncated); - assert.equal(e.partialResult, undefined); -}); diff --git a/bindings/typescript/test/streaming.test.mjs b/bindings/typescript/test/streaming.test.mjs index 7c38b1716..28e515f55 100644 --- a/bindings/typescript/test/streaming.test.mjs +++ b/bindings/typescript/test/streaming.test.mjs @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; +import koffi from "koffi"; import { modelTest, MODEL, STREAMING_MODEL, jfk, feedChunks } from "./common.mjs"; -import { TranscribeModel, Busy } from "../dist/index.js"; +import { TranscribeModel, Aborted, Busy, InvalidArgument, libraryPath } from "../dist/index.js"; + +const oneSecond = () => jfk().subarray(0, 16000); modelTest("streaming commits text and finalizes", STREAMING_MODEL, async () => { const m = await TranscribeModel.load(STREAMING_MODEL); @@ -249,6 +252,76 @@ modelTest("a stale stream wrapper cannot reset a newer same-session stream", STR } }); +modelTest("a feed rejected before the native hook keeps the stream lease", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(oneSecond()); + const bad = new Float32Array(1600); + bad[5] = Number.NaN; + // Native rejects non-finite samples before the family hook and leaves the + // stream ACTIVE, so the binding keeps the model-wide lease. + await assert.rejects(() => stream.feed(bad), InvalidArgument); + assert.equal(stream.state, "active"); + await assert.rejects(() => b.run(oneSecond()), Busy); + + // The stream is still usable, and finalize releases the lease as usual. + const u = await stream.feed(jfk().subarray(16000, 32000)); + assert.equal(typeof u.revision, "number"); + await stream.finalize(); + const r = await b.run(oneSecond()); + assert.equal(typeof r.text, "string"); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a feed that fails inside the native hook releases the stream lease", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(oneSecond()); + + // Provoke a hook failure: an abort callback that always fires makes the + // streaming hook return ABORTED, which moves the native stream to FAILED. + // feed() has no signal option, so install it on the native session + // directly (test-only; the same library instance the binding loaded). + const lib = koffi.load(libraryPath()); + const proto = koffi.proto("bool ComputeRulesTestAbortCb(void *udata)"); + const setAbort = lib.func("transcribe_set_abort_callback", "void", [ + "void *", + koffi.pointer(proto), + "void *", + ]); + const cb = koffi.register(() => true, koffi.pointer(proto)); + setAbort(a.handle, cb, null); + try { + await assert.rejects(() => stream.feed(oneSecond()), Aborted); + } finally { + setAbort(a.handle, null, null); + koffi.unregister(cb); + } + assert.equal(stream.state, "failed"); + assert.ok(stream.lastStatus instanceof Aborted); + + // FAILED is no longer an active stream: the lease is free for a sibling. + const r = await b.run(oneSecond()); + assert.equal(typeof r.text, "string"); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + modelTest("a non-streaming model rejects stream begin", MODEL, async () => { const m = await TranscribeModel.load(MODEL); try { From cb7145ad80b194189c6fba05ddb56402f3c012ba Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 4 Oct 2026 19:11:37 +0800 Subject: [PATCH 49/51] review --- .github/actions/fetch-canary/action.yml | 4 +- .../python/src/transcribe_cpp/__init__.py | 143 +++-- bindings/python/tests/test_compute_lock.py | 23 +- bindings/rust/transcribe-cpp/README.md | 3 +- bindings/rust/transcribe-cpp/src/diarize.rs | 1 + bindings/rust/transcribe-cpp/src/session.rs | 19 +- bindings/rust/transcribe-cpp/src/types.rs | 5 + bindings/swift/README.md | 14 +- .../swift/Sources/TranscribeCpp/Backend.swift | 6 + .../Sources/TranscribeCpp/Cancellation.swift | 10 +- .../swift/Sources/TranscribeCpp/Diarize.swift | 22 + .../Sources/TranscribeCpp/Streaming.swift | 19 +- bindings/typescript/README.md | 18 +- bindings/typescript/src/index.ts | 31 +- bindings/typescript/test/cancel.test.mjs | 16 + bindings/typescript/test/streaming.test.mjs | 2 +- docs/bindings.md | 16 + docs/migrating-to-0.4.md | 23 +- examples/cli/asr.cpp | 562 ++++++++---------- examples/cli/cli.h | 24 +- examples/cli/main.cpp | 52 +- src/arch/sortformer/model.cpp | 27 +- tests/sortformer_diarize_unit.cpp | 1 + 23 files changed, 625 insertions(+), 416 deletions(-) diff --git a/.github/actions/fetch-canary/action.yml b/.github/actions/fetch-canary/action.yml index ad1e83bd1..9cdeabf43 100644 --- a/.github/actions/fetch-canary/action.yml +++ b/.github/actions/fetch-canary/action.yml @@ -68,8 +68,8 @@ runs: uvx --from huggingface_hub hf download \ handy-computer/parakeet-unified-en-0.6b-gguf \ parakeet-unified-en-0.6b-Q4_K_M.gguf --local-dir canary - # Sortformer DIARIZE canary. Q8_0 (~140 MB) reproduces the F32 CPU - # golden segments the binding diarize tests assert. + # Sortformer DIARIZE canary (Q8_0, ~140 MB) for the binding diarize + # tests. [ -f canary/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf ] || \ uvx --from huggingface_hub hf download \ handy-computer/diar_streaming_sortformer_4spk-v2.1-gguf \ diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index e3c85dff1..91d9937c8 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -953,8 +953,9 @@ class Model: of a model, so ``Session.run()`` / ``run_batch()`` / ``stream()``, ``Stream.feed()`` / ``finalize()`` / ``reset()`` and ``DiarizeSession.run()`` hold a model-wide lock for the native call and its copy-out. Calls from - other threads wait (still cancellable via ``cancel()``); load one Model - per worker for parallelism. An active stream holds the model's stream + other threads wait; a ``cancel()`` on a queued call is kept, and it + aborts at its first poll after acquiring the lock. Load one Model per + worker for parallelism. An active stream holds the model's stream lease from ``stream()`` until it is finalized, reset, fails, is garbage-collected or its session/model is closed. Meanwhile ``run()``, ``run_batch()``, ``stream()`` and diarize ``run()`` on any session raise @@ -1008,8 +1009,10 @@ def _h(self) -> ctypes.c_void_p: # One plain Lock per model serializes native compute calls and copy-out. # Not reentrant: the log callback can fire inside a native call, so a # nested compute is detected via _compute_owner and raises instead. - # Native frees never wait: they queue FIFO on _deferred (sessions before - # their model) and the lock holder runs them before releasing. + # Native frees never wait: they queue FIFO on _deferred and the lock + # holder runs them before releasing. close() queues every live session's + # free before the model's, from _live (strong records, not the weak + # _sessions, which the cycle collector clears before finalizers run). # _stream_owner (the active stream's _StreamLease) is written only under it. def _init_compute_state(self) -> None: @@ -1017,6 +1020,8 @@ def _init_compute_state(self) -> None: self._compute_owner: Optional[int] = None # thread ident while held self._deferred: collections.deque = collections.deque() self._stream_owner: Optional[_StreamLease] = None + # id(handle) -> (handle, native free name, pin) per open session. + self._live: dict = {} @contextmanager def _exclusive(self, kind: str, busy: Optional[str] = None): @@ -1029,21 +1034,25 @@ def _exclusive(self, kind: str, busy: Optional[str] = None): f"cannot start {kind}: this thread is already inside a " "compute call on this model (a re-entrant call from a " "callback?)") - self._compute_lock.acquire() - self._compute_owner = me try: - self._h # raises if the model is closed - if busy is not None and self._stream_owner is not None: - raise Busy(busy) - yield + # `with`: no gap between acquire and the release guard in which + # a KeyboardInterrupt could leak the lock. + with self._compute_lock: + try: + self._compute_owner = me + self._h # raises if the model is closed + if busy is not None and self._stream_owner is not None: + raise Busy(busy) + yield + finally: + try: + self._drain_deferred_locked() + finally: + # Owner stays set while draining: a native free can + # log, and the re-entrancy check must still see this + # thread. + self._compute_owner = None finally: - try: - self._drain_deferred_locked() - finally: - # Owner stays set while draining: a native free can log, and - # the re-entrancy check must still see this thread. - self._compute_owner = None - self._compute_lock.release() self._try_drain() def _drain_deferred_locked(self) -> None: @@ -1058,14 +1067,19 @@ def _try_drain(self) -> None: """Run queued frees now if the lock is free, else leave them to the holder. Never blocks; loops so a free queued during release is run.""" while self._deferred: - if not self._compute_lock.acquire(blocking=False): - return - self._compute_owner = threading.get_ident() + got = False try: + # Acquired inside the try, so the release guard covers all but + # the call's own return (no non-blocking `with` exists). + got = self._compute_lock.acquire(False) + if not got: + return + self._compute_owner = threading.get_ident() self._drain_deferred_locked() finally: - self._compute_owner = None - self._compute_lock.release() + if got: + self._compute_owner = None + self._compute_lock.release() def _free_or_defer(self, free) -> None: """Run *free* behind any in-flight compute call without waiting for @@ -1073,6 +1087,29 @@ def _free_or_defer(self, free) -> None: self._deferred.append(free) self._try_drain() + def _track(self, handle: ctypes.c_void_p, free_fn: str, pin=None) -> None: + """Record an open session's native handle (and *pin*, kept alive + until its free) so close() frees it even if its wrapper is gone.""" + self._live[id(handle)] = (handle, free_fn, pin) + + def _free_session(self, handle: ctypes.c_void_p) -> None: + """Free a tracked session behind any in-flight call. The record is + popped first, so the session's own close() and the model's close() + free it exactly once, whichever runs first.""" + record = self._live.pop(id(handle), None) + if record is None: + return + handle, free_fn, pin = record + + def free(_pin=pin): + getattr(_lib, free_fn)(handle) + # Release this session's stream lease only after the free, in the + # same locked step, so a queued run cannot start before it. + if getattr(self._stream_owner, "handle", None) is handle: + self._stream_owner = None + + self._free_or_defer(free) + @property def arch(self) -> str: return _decode(_lib.transcribe_model_arch_string(self._h)) @@ -1192,6 +1229,8 @@ def close(self) -> None: self._handle = None for session in list(getattr(self, "_sessions", ()) or ()): session.close() + for record in list(getattr(self, "_live", {}).values()): + self._free_session(record[0]) # wrappers already collected self._free_or_defer(lambda: _lib.transcribe_model_free(handle)) def __enter__(self) -> "Model": @@ -1223,6 +1262,8 @@ def _arm_abort(self, set_cb) -> None: self._cancel = threading.Event() event = self._cancel self._abort_trampoline = _ABORT_CFUNC(lambda _ud: event.is_set()) + # The record pins the trampoline the native session points at. + self._model._track(self._handle, self._free_fn, self._abort_trampoline) set_cb(self._handle, self._abort_trampoline, None) # Registered last, once fully constructed: Model.close() closes any # session still tracked here before freeing the model. @@ -1273,19 +1314,7 @@ def close(self) -> None: if handle is None: return self._handle = None - # The deferred free pins the abort trampoline the native session - # points at, and never references self (close() may run from __del__). - trampoline = getattr(self, "_abort_trampoline", None) - free_fn, model = self._free_fn, self._model - - def free(_pin=trampoline): - getattr(_lib, free_fn)(handle) - # Release this session's stream lease only after the free, in the - # same locked step, so a queued run cannot start before it. - if getattr(model._stream_owner, "handle", None) is handle: - model._stream_owner = None - - model._free_or_defer(free) + self._model._free_session(handle) # a no-op if the model freed it def __enter__(self: _S) -> _S: return self @@ -1539,11 +1568,13 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, return Stream(self, _keepalive=(run_params, sp), _lease=lease) def _materialize(self, h: ctypes.c_void_p, utt: int | None = None) -> Result: - """Copy out one result from session handle *h* (captured under the - compute lock, so a concurrent close() cannot pull it away mid-copy). - utt is None for the single-result accessors, or an utterance index for - the batch accessors (index 0 aliases the single result after a plain - run, so both paths share this code).""" + """Copy out one result from session handle *h*. The run paths call + this under the compute lock with *h* captured there, so a concurrent + close() cannot pull it away mid-copy; ``Stream.snapshot()`` is an + unlocked read (see ``Session``). utt is None for the single-result + accessors, or an utterance index for the batch accessors (index 0 + aliases the single result after a plain run, so both paths share + this code).""" if utt is None: n_seg = lambda: _lib.transcribe_n_segments(h) get_seg = lambda j, out: _lib.transcribe_get_segment(h, j, out) @@ -1664,6 +1695,13 @@ def _release_lease_locked(self) -> None: if model._stream_owner is self._lease: model._stream_owner = None + def _release_lease_if_ended_locked(self, h: ctypes.c_void_p) -> None: + """After a feed/finalize: release the lease once the native stream + is no longer ACTIVE (FINISHED or FAILED). A call rejected before the + model saw it (e.g. NaN/Inf samples) leaves it ACTIVE and keeps it.""" + if _lib.transcribe_stream_get_state(h) != _generated.TRANSCRIBE_STREAM_ACTIVE: + self._release_lease_locked() + @property def _h(self) -> ctypes.c_void_p: if not self._active: @@ -1680,12 +1718,7 @@ def feed(self, pcm: PCMLike) -> StreamUpdate: with self._session._model._exclusive("stream_feed"): # no busy check h = self._h status = _lib.transcribe_stream_feed(h, array, n_samples, _byref(update)) - # A failure that leaves the stream no longer ACTIVE ends it: release - # the lease. A pre-hook rejection (e.g. NaN) keeps it ACTIVE. - if (status != _generated.TRANSCRIBE_OK - and _lib.transcribe_stream_get_state(h) - != _generated.TRANSCRIBE_STREAM_ACTIVE): - self._release_lease_locked() + self._release_lease_if_ended_locked(h) _check(status, "transcribe_stream_feed") return _stream_update_from(update) @@ -1695,12 +1728,8 @@ def finalize(self) -> StreamUpdate: _lib.transcribe_stream_update_init(_byref(update)) with self._session._model._exclusive("stream_finalize"): h = self._h - try: - status = _lib.transcribe_stream_finalize(h, _byref(update)) - finally: - # Finalize ends the active stream (FINISHED, or FAILED on - # error): release the lease either way. - self._release_lease_locked() + status = _lib.transcribe_stream_finalize(h, _byref(update)) + self._release_lease_if_ended_locked(h) _check(status, "transcribe_stream_finalize") return _stream_update_from(update) @@ -1756,8 +1785,14 @@ def reset(self) -> None: def __enter__(self) -> "Stream": return self - def __exit__(self, *exc) -> None: - self.reset() + def __exit__(self, exc_type, *exc) -> None: + try: + self.reset() + except TranscribeError: + # e.g. the session or model was closed in the block: never mask + # the exception that is already propagating. + if exc_type is None: + raise def __del__(self): # Abandoned while holding the lease: reset and release it behind any diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index 40054519a..7c4ed2486 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -80,6 +80,7 @@ def session(m, cls=t.Session, track=True): s._handle = ctypes.c_void_p(next(ids)) s._cancel = threading.Event() s._abort_trampoline = None + m._track(s._handle, cls._free_fn) # as _arm_abort does m._sessions.add(s) if track: made.append(s) @@ -117,6 +118,12 @@ def acquire(self, blocking=True, timeout=-1): def release(self): self._lock.release() + def __enter__(self): + return self.acquire() + + def __exit__(self, *exc): + self.release() + def locked(self): return self._lock.locked() @@ -175,6 +182,7 @@ def runs(session): def test_every_compute_site_holds_the_lock(fake, monkeypatch): m = fake.model() s = fake.session(m) + d = fake.session(m, cls=t.DiarizeSession) seen: list = [] def holds(name, ret=0): @@ -187,7 +195,8 @@ def probe(*args): for name in ("transcribe_run", "transcribe_run_batch", "transcribe_stream_begin", "transcribe_stream_feed", "transcribe_stream_finalize", "transcribe_stream_reset", - "transcribe_batch_status"): + "transcribe_batch_status", "transcribe_diarize_run", + "transcribe_diarize_n_segments"): monkeypatch.setattr(t._lib, name, holds(name)) monkeypatch.setattr(t._lib, "transcribe_batch_n_results", holds("transcribe_batch_n_results", ret=1)) @@ -205,12 +214,14 @@ def materialize(self, h, utt=None): stream.feed(PCM) stream.finalize() stream.reset() + assert d.run(PCM) == [] names = [n for n, _ in seen] for site in ("transcribe_run", "transcribe_run_batch", "transcribe_batch_n_results", "transcribe_batch_status", "transcribe_stream_begin", "transcribe_stream_feed", "transcribe_stream_finalize", "transcribe_stream_reset", + "transcribe_diarize_run", "transcribe_diarize_n_segments", "copy-out"): assert site in names, f"{site} never reached" assert all(ok for _, ok in seen), [n for n, ok in seen if not ok] @@ -368,13 +379,21 @@ def probe(h, *rest): def native(fake, monkeypatch): """Probes for the native compute entry points. ``calls`` records (name, session handle) for every native compute call that ran; - ``status`` sets what each one returns.""" + ``status`` sets what each one returns. A successful begin makes the + stream ACTIVE; finalize ends it (FINISHED, or FAILED on error).""" calls: list = [] status = {"run": 0, "begin": 0, "feed": 0, "finalize": 0} + ends = {"begin": (_generated.TRANSCRIBE_STREAM_ACTIVE, None), + "finalize": (_generated.TRANSCRIBE_STREAM_FINISHED, + _generated.TRANSCRIBE_STREAM_FAILED)} def probe(name, key): def fn(h, *rest): calls.append((name, h.value)) + ok, failed = ends.get(key, (None, None)) + new = ok if status[key] == 0 else failed + if new is not None: + fake.state["value"] = new return status[key] return fn diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index 086ecbcd6..dd12b78c7 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -215,6 +215,7 @@ fn main() { - `Session` is `Send` but not `Sync`; mutating calls take `&mut self`. - In 0.x the C library allows at most one in-flight run across all sessions of a model; this crate enforces it with a per-model mutex, so concurrent calls - queue rather than race. For real parallelism, use one `Model` per worker. + queue rather than race. `DiarizeSession`s share the same lock. For real + parallelism, use one `Model` per worker. - License: MIT diff --git a/bindings/rust/transcribe-cpp/src/diarize.rs b/bindings/rust/transcribe-cpp/src/diarize.rs index 8ae20f588..fe343bd1a 100644 --- a/bindings/rust/transcribe-cpp/src/diarize.rs +++ b/bindings/rust/transcribe-cpp/src/diarize.rs @@ -16,6 +16,7 @@ use crate::session::clamp_len; /// Static facts about a diarization model ([`Model::diarize_info`]). #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct DiarizeInfo { /// Input PCM rate (16000). diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index 4c2607893..017c38d9c 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -562,10 +562,23 @@ impl Stream<'_> { let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; // This stream already holds the model's compute lease (it is in - // flight); the lock here just serializes the native call. - let status = self.session.model.with_compute(None, |_| unsafe { - sys::transcribe_stream_feed(self.session.ptr, pcm.as_ptr(), n, &mut update) + // flight); the lock here just serializes the native call. A feed that + // ends the stream (FAILED) frees the lease, as finalize does; a feed + // rejected before the native hook (e.g. NaN) leaves it ACTIVE and keeps it. + let ptr = self.session.ptr; + let held = self.holds_lease; + let (status, ended) = self.session.model.with_compute(None, |lease| { + let st = unsafe { sys::transcribe_stream_feed(ptr, pcm.as_ptr(), n, &mut update) }; + let ended = unsafe { sys::transcribe_stream_get_state(ptr) } + == sys::transcribe_stream_state::TRANSCRIBE_STREAM_FAILED; + if held && ended { + *lease = false; + } + (st, ended) })?; + if ended { + self.holds_lease = false; + } check(status, "stream feed")?; Ok(StreamUpdate::from_raw(&update)) } diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index a4981cae7..30af14822 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -400,4 +400,9 @@ impl Roles { }; self.0 & bit.0 != 0 } + + /// The raw `transcribe_role` bitmask (`TRANSCRIBE_ROLE_*` bits). + pub fn bits(self) -> u32 { + self.0 + } } diff --git a/bindings/swift/README.md b/bindings/swift/README.md index c8e1ce376..8a5bed710 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -5,12 +5,15 @@ a C/C++ speech-to-text library built on ggml. Native code ships as a prebuilt `.xcframework` SwiftPM `binaryTarget`, with Metal embedded on supported Apple slices. -> Status: in development (0.2.0). Core model, session, run, stream, +> Status: in development (0.3.0). Core model, session, run, stream, > cancellation, backend, and family-extension APIs are implemented and tested. Upgrading from 0.1? See the [0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), -including the replacement of `gpuDevice` with exact `Device` values. +including the replacement of `gpuDevice` with exact `Device` values. From 0.3, +see the +[0.4 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.4.md) +(roles, `DiarizeSession`, throwing `capabilities`). ## Install @@ -23,7 +26,7 @@ custom artifact path through `TRANSCRIBE_XCFRAMEWORK_PATH`. The standalone SwiftPM mirror is planned but not published yet: ```swift -.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.2.0") +.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.3.0") ``` Until that mirror repo and tag exist, use the release xcframework directly when @@ -32,7 +35,7 @@ you only need the raw C module: ```swift .binaryTarget( name: "CTranscribe", - url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.2.0/TranscribeCpp.xcframework.zip", + url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.3.0/TranscribeCpp.xcframework.zip", checksum: "" ) ``` @@ -134,7 +137,8 @@ back to another primary device. runs queue; load one `Model` per worker for true parallelism. - `Session` is single-threaded. Use one session from one thread at a time. - An active `Stream` holds the model's compute lease until `finalize`, `reset`, - or drop. Other runs/streams on that model fail with `TranscribeError.busy`. + drop, or a `feed` that fails the stream. Other runs/streams on that model + fail with `TranscribeError.busy`. - `Transcribe.setLogHandler` is best installed at startup. Repeated calls are safe; they swap the Swift handler behind one native trampoline. - On Metal, do not keep models in globals in short-lived programs. Scope models diff --git a/bindings/swift/Sources/TranscribeCpp/Backend.swift b/bindings/swift/Sources/TranscribeCpp/Backend.swift index 9aeef03f1..9eab54f63 100644 --- a/bindings/swift/Sources/TranscribeCpp/Backend.swift +++ b/bindings/swift/Sources/TranscribeCpp/Backend.swift @@ -110,6 +110,9 @@ public enum AbiStruct: Sendable { case sessionLimits case ext case deviceInfo + case diarizeInfo + case diarizeSessionParams + case diarizeParams var cValue: transcribe_abi_struct { switch self { @@ -127,6 +130,9 @@ public enum AbiStruct: Sendable { case .sessionLimits: return TRANSCRIBE_ABI_SESSION_LIMITS case .ext: return TRANSCRIBE_ABI_EXT case .deviceInfo: return TRANSCRIBE_ABI_DEVICE_INFO + case .diarizeInfo: return TRANSCRIBE_ABI_DIARIZE_INFO + case .diarizeSessionParams: return TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS + case .diarizeParams: return TRANSCRIBE_ABI_DIARIZE_PARAMS } } } diff --git a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift index 4b47e2e11..fe5221932 100644 --- a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift +++ b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift @@ -1,11 +1,11 @@ import CTranscribe import os -/// A thread-safe cancellation flag. Install it on a `Session` with -/// `setCancellationToken`; the native abort callback (polled between decode -/// steps / chunks) reads it, so `cancel()` may be called from any thread to -/// abort an in-flight run/stream. The run then throws `.aborted` with the -/// partial transcript preserved. +/// A thread-safe cancellation flag. Install it on a `Session` or a +/// `DiarizeSession` with `setCancellationToken`; the native abort callback +/// (polled between decode steps / chunks) reads it, so `cancel()` may be called +/// from any thread to abort an in-flight run/stream. The run then throws +/// `.aborted` (a `Session` run with the partial transcript preserved). public final class CancellationToken: @unchecked Sendable { private let cancelled = OSAllocatedUnfairLock(initialState: false) diff --git a/bindings/swift/Sources/TranscribeCpp/Diarize.swift b/bindings/swift/Sources/TranscribeCpp/Diarize.swift index 82ae05325..2bb6be4ce 100644 --- a/bindings/swift/Sources/TranscribeCpp/Diarize.swift +++ b/bindings/swift/Sources/TranscribeCpp/Diarize.swift @@ -1,4 +1,5 @@ import CTranscribe +import Foundation /// Static facts about a diarization model. public struct DiarizeInfo: Sendable, Equatable { @@ -79,6 +80,27 @@ public final class DiarizeSession { } } + /// `run` hopped off the caller's thread/actor onto a background queue, with + /// Swift task cancellation bridged to the native abort. Same contract as + /// `Session.run(_:options:) async`: a caller-installed token takes + /// precedence, and the bridged token is removed when the call returns. + public func run(_ pcm: [Float], options: DiarizeOptions = .init()) async throws -> [SpeakerSegment] { + nonisolated(unsafe) let this = self + let bridged = (cancelToken == nil) ? CancellationToken() : nil + if let bridged { setCancellationToken(bridged) } + defer { if bridged != nil { clearCancellationToken() } } + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { + (cont: CheckedContinuation<[SpeakerSegment], Error>) in + DispatchQueue.global().async { + cont.resume(with: Result { try this.run(pcm, options: options) }) + } + } + } onCancel: { + bridged?.cancel() + } + } + /// Timings from the most recent run (`decodeMs` is 0). public var timings: Timings { var t = transcribe_timings(); transcribe_timings_init(&t) diff --git a/bindings/swift/Sources/TranscribeCpp/Streaming.swift b/bindings/swift/Sources/TranscribeCpp/Streaming.swift index 2097759bd..f278ead2d 100644 --- a/bindings/swift/Sources/TranscribeCpp/Streaming.swift +++ b/bindings/swift/Sources/TranscribeCpp/Streaming.swift @@ -98,11 +98,12 @@ public struct StreamUpdate: Sendable, Equatable { public final class Stream { private let session: Session /// True while this stream holds the model's compute lease (set at begin, - /// cleared at finalize/reset/deinit). Tracked per-stream so `deinit` never - /// releases a lease a *different* session has since acquired — mirrors the - /// Rust binding's `holds_lease`. Mutated only inside `model.withCompute`; the - /// `deinit` guard reads it on the deallocating thread, where no other - /// reference to this `Stream` can exist (so no concurrent mutation). + /// cleared at a failing feed, finalize, reset, or deinit). Tracked + /// per-stream so `deinit` never releases a lease a *different* session has + /// since acquired — mirrors the Rust binding's `holds_lease`. Mutated only + /// inside `model.withCompute`; the `deinit` guard reads it on the + /// deallocating thread, where no other reference to this `Stream` can exist + /// (so no concurrent mutation). var holdsLease = true init(_ session: Session) { self.session = session } @@ -120,6 +121,9 @@ public final class Stream { } /// Feed a PCM frame (16 kHz mono float32). Returns per-call change metadata. + /// A feed that ends the stream (FAILED) releases the model's compute lease; + /// one rejected before the model saw it (e.g. NaN/Inf) leaves it ACTIVE + /// and keeps the lease. public func feed(_ frame: [Float]) throws -> StreamUpdate { try session.model.withCompute { var update = transcribe_stream_update() @@ -127,6 +131,9 @@ public final class Stream { let status = frame.withUnsafeBufferPointer { transcribe_stream_feed(session.ptr, $0.baseAddress, Int32($0.count), &update) } + if holdsLease && transcribe_stream_get_state(session.ptr) == TRANSCRIBE_STREAM_FAILED { + session.model.streamActive = false; holdsLease = false + } try TranscribeError.check(status, context: "stream_feed") return StreamUpdate(update) } @@ -187,7 +194,7 @@ extension Session { /// `.notImplemented`) and be idle/finished/failed (not already streaming). /// Claims the model's compute lease for the whole stream lifetime: a second /// stream — or an offline run — on ANY session of the same model is refused - /// with `.busy` until this stream finalizes, resets, or is dropped. + /// with `.busy` until this stream finalizes, fails, resets, or is dropped. public func stream( _ runOptions: RunOptions = .init(), _ streamOptions: StreamOptions = .init() ) throws -> Stream { diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 6eecb8d1d..0a33edcf8 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -123,8 +123,9 @@ const turns = await diarizer.run(pcm, { for (const t of turns) console.log(t.speakerId, t.t0Ms, t.t1Ms); ``` -`run` takes a `signal` like `Session.run`; `diarizer.timings` reports the last -run. Diarize runs share the model's one-at-a-time rule and `Busy` refusal below. +`run` accepts a `signal`; cancelling throws `Aborted` (with no partial result). +`diarizer.timings` reports the last run. Diarize runs wait on the same +model-wide lock as other compute calls (see below). ### Resource management @@ -189,17 +190,18 @@ the lease held; a failure inside the model makes it `"failed"` and frees the lea Because the compute is genuinely on another thread, **do not touch a session while a call against it is in flight** — it is single-threaded in the C library: -- Reading a stream's `text`/`snapshot`/`state`/`revision`/`lastStatus`, or a - session's `limits`/`wasAborted`, during an un-awaited - `feed`/`finalize`/`run`/`runBatch` **throws**. +- Reading a stream's `text`/`snapshot`/`state`/`revision`/`lastStatus`, a + session's `limits`/`wasAborted`, or a `DiarizeSession`'s `timings`, during an + un-awaited `feed`/`finalize`/`run`/`runBatch` **throws**. - `reset()` and `dispose()` are safe to call any time: the native teardown is deferred behind any in-flight call, so it never frees a session mid-compute. - Disposing a `Session` or `TranscribeModel` while a stream is still active releases the lease and invalidates the stream — its later calls throw rather than touch the freed handle. -- The **input PCM is borrowed, not copied**: `run`/`runBatch`/`feed` hand the - buffer to native code that reads it on the worker thread, so do not mutate it - (e.g. reuse a scratch/capture buffer) until the returned promise resolves. +- The **input PCM is borrowed, not copied**: `run`/`runBatch`/`feed` (and + `DiarizeSession.run`) hand the buffer to native code that reads it on the + worker thread, so do not mutate it (e.g. reuse a scratch/capture buffer) + until the returned promise resolves. Pass a fresh buffer per call, or `await` before overwriting. The normal pattern is safe — `await` first, then read: diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index a03ac9bb2..5a67fe6e6 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -658,7 +658,7 @@ const SORTFORMER_PRESET: Record = { * Build a native ext-struct buffer for a family extension and return the koffi * pointer to assign to `params.family`. Validates the slot and that the model * accepts the kind. The returned buffer must be kept alive (held via the params - * object) until the native call returns. + * object) until the native call returns, then freed by freeRunParams. */ function buildFamily( n: Native, @@ -687,7 +687,12 @@ function buildFamily( if (v !== undefined) ext[k] = v; } const buf = n.koffi.alloc(n.T[reg.type], 1); - n.koffi.encode(buf, n.T[reg.type], ext); + try { + n.koffi.encode(buf, n.T[reg.type], ext); + } catch (e) { + n.koffi.free(buf); + throw e; + } return buf; } @@ -698,13 +703,20 @@ function cstr(value: string, name: string): string { return value; } -/** Free what #buildRunParams allocated; call once the native call returns. */ +/** + * Free what #buildRunParams / buildFamily allocated into a params struct; call + * only once the native call has returned (never while a worker reads it). + */ function freeRunParams(n: Native, p: any): void { if (p.vocabulary) { n.koffi.free(p.vocabulary); p.vocabulary = null; p.n_vocabulary = 0; } + if (p.family) { + n.koffi.free(p.family); + p.family = null; + } } function toStreamUpdate(u: any): StreamUpdate { @@ -987,8 +999,6 @@ export class Session { if (opts.keepSpecialTags !== undefined) p.keep_special_tags = opts.keepSpecialTags; if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts; - if (opts.family) - p.family = buildFamily(n, this.#model.handle, opts.family, "run"); if (opts.vocabulary !== undefined) { const terms = opts.vocabulary; if (!Array.isArray(terms) || !terms.every((t) => typeof t === "string")) @@ -1005,6 +1015,9 @@ export class Session { } if (opts.prompt !== undefined) p.prompt = cstr(opts.prompt, "prompt"); if (opts.prefix !== undefined) p.prefix = cstr(opts.prefix, "prefix"); + // Last, so no later validation throw can strand the allocation. + if (opts.family) + p.family = buildFamily(n, this.#model.handle, opts.family, "run"); return p; } @@ -1116,7 +1129,11 @@ export class Session { if (!control) throw new TranscribeError("session control is missing"); control.replaceCurrentStream(stream); return stream; - }).finally(() => freeRunParams(n, rp)); // begin copied the prompting strings + }).finally(() => { + // begin copied the prompting strings and the family extension + freeRunParams(n, rp); + freeRunParams(n, sp); + }); } get wasAborted(): boolean { @@ -1383,7 +1400,7 @@ export class DiarizeSession { return readSpeakerSegments(n, F.diarizeNSegments(h), (i, o) => F.diarizeGetSegment(h, i, o), ); - }); + }).finally(() => freeRunParams(n, p)); } /** load_ms plus the last run's mel / encode time. */ diff --git a/bindings/typescript/test/cancel.test.mjs b/bindings/typescript/test/cancel.test.mjs index 7f9e3beb6..d8b22fcaa 100644 --- a/bindings/typescript/test/cancel.test.mjs +++ b/bindings/typescript/test/cancel.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { getEventListeners } from "node:events"; import { modelTest, MODEL, jfk } from "./common.mjs"; import { TranscribeModel, Aborted } from "../dist/index.js"; @@ -27,3 +28,18 @@ modelTest("a pre-aborted signal raises Aborted with a partial result", MODEL, as m.dispose(); } }); + +modelTest("dispose during a run with a signal returns the result and removes the listener", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const ac = new AbortController(); + const p = s.run(jfk(), { signal: ac.signal }); + await Promise.resolve(); + s.dispose(); + assert.match((await p).text, /ask not what your country/i); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + } finally { + m.dispose(); + } +}); diff --git a/bindings/typescript/test/streaming.test.mjs b/bindings/typescript/test/streaming.test.mjs index 28e515f55..9ad775001 100644 --- a/bindings/typescript/test/streaming.test.mjs +++ b/bindings/typescript/test/streaming.test.mjs @@ -294,7 +294,7 @@ modelTest("a feed that fails inside the native hook releases the stream lease", // feed() has no signal option, so install it on the native session // directly (test-only; the same library instance the binding loaded). const lib = koffi.load(libraryPath()); - const proto = koffi.proto("bool ComputeRulesTestAbortCb(void *udata)"); + const proto = koffi.proto("bool StreamingTestAbortCb(void *udata)"); const setAbort = lib.func("transcribe_set_abort_callback", "void", [ "void *", koffi.pointer(proto), diff --git a/docs/bindings.md b/docs/bindings.md index e92fa9019..9440f3621 100644 --- a/docs/bindings.md +++ b/docs/bindings.md @@ -98,6 +98,22 @@ objects own copies of every row and remain valid after the next run. That contract is for ASR models that attribute speakers inside a transcript. +## Compute lock and stream lease + +The C library allows one compute in flight per model and leaves enforcing it +to the caller. Every first-class binding enforces it with a model-wide lock +shared by every session of every role: a call waits behind any other compute +on the same model. + +The bindings are also stricter than C about streams. `transcribe.h` lets +several sessions of one model each hold an active stream and interleave +their feeds; a binding allows one active stream per model. Starting a stream +takes the model's stream lease, and ending it (finalize, reset, a feed that +leaves the stream FAILED, or dropping / closing the stream) releases it. +While the lease is held, `run`, `run_batch`, a new stream and a diarize run +on any session of that model raise `Busy` instead of waiting. A feed +rejected before the native call (e.g. NaN input) keeps the lease. + ## Roles and the DIARIZE session Each binding exposes the role mask as `Model.roles` (Python `frozenset[Role]`, diff --git a/docs/migrating-to-0.4.md b/docs/migrating-to-0.4.md index cc15a5e54..42c8ed57f 100644 --- a/docs/migrating-to-0.4.md +++ b/docs/migrating-to-0.4.md @@ -49,7 +49,23 @@ from the role's own query (`transcribe_diarize_get_info`). - New `Model.roles`, `DiarizeSession` and an `UnsupportedRole` error for status 20 (`.unsupportedRole` in Swift); see `docs/bindings.md`. The Sortformer ASR-path extension (`SortformerStreamOptions` and equivalents) - is removed. + is removed. Sortformer runs through a diarize session instead: + + | Binding | 0.3 | 0.4 | + | --- | --- | --- | + | Python | `model.session()` + `SortformerStreamOptions` | `model.diarize_session()` + `SortformerDiarizeOptions` | + | Rust | `RunExtension::Sortformer(SortformerStreamOptions)` | `model.diarize_session()` + `DiarizeOptions` with `DiarizeExtension::Sortformer(SortformerDiarizeOptions)` | + | Swift | `RunExtension.sortformer(SortformerStreamOptions)` | `model.diarizeSession()` + `DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(...)))` | + | TypeScript | `{ kind: "sortformer" }` on the `run` slot | `model.createDiarizeSession()` + `{ kind: "sortformer_diarize" }` on the new `"diarize_run"` slot | + + In every binding, `capabilities` on a model without ASR (Sortformer) + raises / throws / returns `UnsupportedRole` instead of reporting a zeroed + struct, and `supports(diarization)` is false for Sortformer; use + `Model.roles`. +- A stream that ends FAILED inside a feed (abort, or a native error) + releases the model's stream lease in every binding, so other sessions on + the model can run without `Busy`. A feed rejected before the native call + (e.g. NaN) keeps the lease and the stream stays usable. - **Rust:** `Model::capabilities()` returns `Result` (`Err(Error::UnsupportedRole)` on a model without ASR). `ExtSlot` gains `DiarizeRun` and `AbiStruct` gains the three diarize structs; both are now @@ -59,6 +75,5 @@ from the role's own query (`transcribe_diarize_get_info`). run / run_batch / second stream on any session of a model with an active stream raises `transcribe_cpp.Busy`, matching the other bindings. On the stream's own session these raised `InvalidArgument` in 0.3. -- **TypeScript:** a feed rejected before the native hook (e.g. NaN) keeps the - model's stream lease; the stream stays usable. An unknown Sortformer preset - string is rejected instead of silently using the default. +- **TypeScript:** an unknown Sortformer preset string is rejected instead of + silently using the default. diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index b3adb8b21..071ba338b 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -659,341 +659,301 @@ int run_asr_batch(const cli_args & args, std::ofstream * output) { return n_fail > 0 || !output_ok ? EXIT_FAILURE : EXIT_SUCCESS; } -int run_asr_file(const cli_args & args, std::ofstream * output) { - bool output_ok = true; - std::vector pcm; - std::string load_err; - if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { - std::fprintf(stderr, "wav: %s\n", load_err.c_str()); +int run_asr_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output) { + bool output_ok = true; + + struct transcribe_session_params cp; + transcribe_session_params_init(&cp); + cp.n_threads = args.n_threads; + cp.n_ctx = args.n_ctx; + cp.kv_type = args.kv_type; + struct transcribe_session * ctx = nullptr; + const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); + if (init_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); + transcribe_model_free(model); return EXIT_FAILURE; } - const double duration_s = static_cast(pcm.size()) / 16000.0; - std::printf("audio: %s\n", args.wav_path.c_str()); - std::printf(" samples: %zu\n", pcm.size()); - std::printf(" duration: %.3f s\n", duration_s); - std::printf(" sample rate 16000 Hz mono float32\n"); - - if (!args.model_path.empty()) { - struct transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; - if (args.device_index >= 0 && mp.device == nullptr) { - std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); - return EXIT_FAILURE; - } - struct transcribe_model * model = nullptr; - const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); - std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); - if (st != TRANSCRIBE_OK) { - return EXIT_FAILURE; - } - std::printf(" backend: %s\n", transcribe_model_backend(model)); - if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { - std::printf(" name: %s\n", dn); - } - if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { - std::printf(" license: %s\n", lic); - } - - const uint32_t roles = transcribe_model_roles(model); - if ((roles & TRANSCRIBE_ROLE_ASR) == 0) { - return transcribe_cli::run_diarize_file(args, model, pcm, duration_s, output); - } - - struct transcribe_session_params cp; - transcribe_session_params_init(&cp); - cp.n_threads = args.n_threads; - cp.n_ctx = args.n_ctx; - cp.kv_type = args.kv_type; - struct transcribe_session * ctx = nullptr; - const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); - if (init_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - // Surface the effective input-length limit so it's obvious how much - // audio this session accepts (reflects --n-ctx). 0 means "no practical - // limit", which covers two different families: one that chunks long - // audio internally (FEATURE_LONG_FORM) and one that is genuinely - // unbounded and encodes the clip in a single pass. Distinguish them - // rather than asserting the first. See docs/input-limits.md. - { - struct transcribe_session_limits lim; - transcribe_session_limits_init(&lim); - if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { - if (lim.effective_max_audio_ms > 0) { - std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); - if (lim.effective_n_ctx > 0) { - std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, - (long long) (lim.max_kv_bytes >> 20)); - } - std::printf("\n"); - } else if (lim.effective_n_ctx > 0) { - // Capped family whose context is too small to fit any audio - // plus a prompt (e.g. an aggressively low --n-ctx). - std::printf( - " max audio: ~0 s (context %d tok too small for " - "audio + prompt)\n", - lim.effective_n_ctx); - } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { - std::printf(" max audio: unbounded (long audio chunked internally)\n"); - } else { - // No context cap and no chunker: the family encodes the - // whole clip in one pass (e.g. block-local attention, - // where cost is linear in audio length). - std::printf(" max audio: unbounded (whole clip in one pass)\n"); + // Surface the effective input-length limit so it's obvious how much + // audio this session accepts (reflects --n-ctx). 0 means "no practical + // limit", which covers two different families: one that chunks long + // audio internally (FEATURE_LONG_FORM) and one that is genuinely + // unbounded and encodes the clip in a single pass. Distinguish them + // rather than asserting the first. See docs/input-limits.md. + { + struct transcribe_session_limits lim; + transcribe_session_limits_init(&lim); + if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { + if (lim.effective_max_audio_ms > 0) { + std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); + if (lim.effective_n_ctx > 0) { + std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, + (long long) (lim.max_kv_bytes >> 20)); } + std::printf("\n"); + } else if (lim.effective_n_ctx > 0) { + // Capped family whose context is too small to fit any audio + // plus a prompt (e.g. an aggressively low --n-ctx). + std::printf( + " max audio: ~0 s (context %d tok too small for " + "audio + prompt)\n", + lim.effective_n_ctx); + } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { + std::printf(" max audio: unbounded (long audio chunked internally)\n"); + } else { + // No context cap and no chunker: the family encodes the + // whole clip in one pass (e.g. block-local attention, + // where cost is linear in audio length). + std::printf(" max audio: unbounded (whole clip in one pass)\n"); } } + } - struct transcribe_run_params rp; - transcribe_run_params_init(&rp); - if (args.translate) { - rp.task = TRANSCRIBE_TASK_TRANSLATE; - } - std::vector vocabulary_ptrs; - apply_prompting(args, rp, vocabulary_ptrs); - if (!args.language.empty()) { - rp.language = args.language.c_str(); - } - if (!args.target_language.empty()) { - rp.target_language = args.target_language.c_str(); - } - rp.timestamps = args.timestamps; - rp.spec_k_drafts = args.spec_k_drafts; + struct transcribe_run_params rp; + transcribe_run_params_init(&rp); + if (args.translate) { + rp.task = TRANSCRIBE_TASK_TRANSLATE; + } + std::vector vocabulary_ptrs; + apply_prompting(args, rp, vocabulary_ptrs); + if (!args.language.empty()) { + rp.language = args.language.c_str(); + } + if (!args.target_language.empty()) { + rp.target_language = args.target_language.c_str(); + } + rp.timestamps = args.timestamps; + rp.spec_k_drafts = args.spec_k_drafts; + + if (args.itn_set) { + rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + } + if (args.canary_pnc_set) { + rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + } + if (args.diarize_set) { + rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + } - if (args.itn_set) { - rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + struct transcribe_whisper_run_ext wx; + transcribe_whisper_run_ext_init(&wx); + if (args.whisper_set) { + if (!args.initial_prompt.empty()) { + wx.initial_prompt = args.initial_prompt.c_str(); } - if (args.canary_pnc_set) { - rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + wx.condition_on_prev_tokens = args.condition_on_prev_tokens; + if (args.temperature_set) { + wx.temperature = args.temperature; } - if (args.diarize_set) { - rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + wx.prompt_condition = args.prompt_condition; + if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { + rp.family = &wx.ext; } + } - struct transcribe_whisper_run_ext wx; - transcribe_whisper_run_ext_init(&wx); - if (args.whisper_set) { - if (!args.initial_prompt.empty()) { - wx.initial_prompt = args.initial_prompt.c_str(); - } - wx.condition_on_prev_tokens = args.condition_on_prev_tokens; - if (args.temperature_set) { - wx.temperature = args.temperature; - } - wx.prompt_condition = args.prompt_condition; - if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { - rp.family = &wx.ext; - } - } + if (args.keep_special_tags) { + rp.keep_special_tags = true; + } - if (args.keep_special_tags) { - rp.keep_special_tags = true; + // Streaming demo: drive transcribe_stream_begin/feed/finalize + // with fixed-size PCM chunks. Families with true per-feed + // partial decoding (moonshine_streaming) flip + // update.result_changed whenever the transcript advances; the + // CLI prints the live tentative text on each such feed. + // Families that only commit at finalize keep result_changed + // false until the finalize call. + transcribe_status run_st = TRANSCRIBE_OK; + if (args.stream_chunk_ms > 0) { + struct transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); + if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { + std::fprintf(stderr, + "stream: model does not advertise " + "supports_streaming; use a streaming-capable " + "model or drop --stream-chunk-ms\n"); + transcribe_session_free(ctx); + transcribe_model_free(model); + return EXIT_FAILURE; } - // Streaming demo: drive transcribe_stream_begin/feed/finalize - // with fixed-size PCM chunks. Families with true per-feed - // partial decoding (moonshine_streaming) flip - // update.result_changed whenever the transcript advances; the - // CLI prints the live tentative text on each such feed. - // Families that only commit at finalize keep result_changed - // false until the finalize call. - transcribe_status run_st = TRANSCRIBE_OK; - if (args.stream_chunk_ms > 0) { - struct transcribe_capabilities caps; - transcribe_capabilities_init(&caps); - const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); - if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { - std::fprintf(stderr, - "stream: model does not advertise " - "supports_streaming; use a streaming-capable " - "model or drop --stream-chunk-ms\n"); - transcribe_session_free(ctx); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); - std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); - - struct transcribe_stream_params sp; - transcribe_stream_params_init(&sp); - struct transcribe_parakeet_stream_ext pkt_sp; - transcribe_parakeet_stream_ext_init(&pkt_sp); - struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; - transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); - struct transcribe_voxtral_realtime_stream_ext vx_sp; - transcribe_voxtral_realtime_stream_ext_init(&vx_sp); - const bool want_cache_aware = (args.stream_att_right >= 0); - const bool want_buffered = - args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; - if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { - pkt_sp.att_context_right = args.stream_att_right; - sp.family = &pkt_sp.ext; - std::printf("stream: att_context_right=%d\n", args.stream_att_right); - } else if (want_buffered && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { - pkt_buf_sp.left_ms = args.stream_buf_left_ms; - pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; - pkt_buf_sp.right_ms = args.stream_buf_right_ms; - sp.family = &pkt_buf_sp.ext; - std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, - args.stream_buf_chunk_ms, args.stream_buf_right_ms); - } else if (args.stream_voxtral_delay != -1 && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { - vx_sp.num_delay_tokens = args.stream_voxtral_delay; - sp.family = &vx_sp.ext; - std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); - } - run_st = transcribe_stream_begin(ctx, &rp, &sp); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); - } else { - size_t pos = 0; - int feed_n = 0; - while (pos < pcm.size()) { - const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); - struct transcribe_stream_update upd; - transcribe_stream_update_init(&upd); - run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); - break; - } - pos += take; - std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, - (long long) upd.input_received_ms, (long long) upd.buffered_ms); - if (upd.result_changed) { - const char * partial = transcribe_full_text(ctx); - std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); - } - std::printf("\n"); - ++feed_n; - } - if (run_st == TRANSCRIBE_OK) { - struct transcribe_stream_update fin_upd; - transcribe_stream_update_init(&fin_upd); - run_st = transcribe_stream_finalize(ctx, &fin_upd); - std::printf( - " finalize: status=%s " - "revision=%d input=%lld ms committed=%lld ms\n", - transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, - (long long) fin_upd.audio_committed_ms); - } - } + const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); + std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); + + struct transcribe_stream_params sp; + transcribe_stream_params_init(&sp); + struct transcribe_parakeet_stream_ext pkt_sp; + transcribe_parakeet_stream_ext_init(&pkt_sp); + struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; + transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); + struct transcribe_voxtral_realtime_stream_ext vx_sp; + transcribe_voxtral_realtime_stream_ext_init(&vx_sp); + const bool want_cache_aware = (args.stream_att_right >= 0); + const bool want_buffered = + args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; + if (want_cache_aware && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { + pkt_sp.att_context_right = args.stream_att_right; + sp.family = &pkt_sp.ext; + std::printf("stream: att_context_right=%d\n", args.stream_att_right); + } else if (want_buffered && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { + pkt_buf_sp.left_ms = args.stream_buf_left_ms; + pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; + pkt_buf_sp.right_ms = args.stream_buf_right_ms; + sp.family = &pkt_buf_sp.ext; + std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, args.stream_buf_chunk_ms, + args.stream_buf_right_ms); + } else if (args.stream_voxtral_delay != -1 && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { + vx_sp.num_delay_tokens = args.stream_voxtral_delay; + sp.family = &vx_sp.ext; + std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); + } + run_st = transcribe_stream_begin(ctx, &rp, &sp); + if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); } else { - // --repeat N runs transcribe_run() N times for steady-state - // perf measurements. - for (int r = 0; r < args.repeat; ++r) { - run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + size_t pos = 0; + int feed_n = 0; + while (pos < pcm.size()) { + const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); + struct transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); break; } + pos += take; + std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, (long long) upd.input_received_ms, + (long long) upd.buffered_ms); + if (upd.result_changed) { + const char * partial = transcribe_full_text(ctx); + std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); + } + std::printf("\n"); + ++feed_n; } - } - std::printf("run: %s\n", transcribe_status_string(run_st)); - // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but - // preserve the partial transcript (see docs/input-limits.md), so show - // the result for them too — just flagged. - const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; - if (result_present) { - const char * text = transcribe_full_text(ctx); - std::printf("text: %s\n", (text && *text) ? text : "(empty)"); - output_ok = write_output_file(output, args.output_path, text) && output_ok; - - // A decode cut short before end-of-stream; the text above is - // incomplete. - if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - std::printf( - " note: decode stopped when the output began repeating " - "itself (repeats dropped); transcript is incomplete\n"); - } else if (transcribe_was_truncated(ctx)) { + if (run_st == TRANSCRIBE_OK) { + struct transcribe_stream_update fin_upd; + transcribe_stream_update_init(&fin_upd); + run_st = transcribe_stream_finalize(ctx, &fin_upd); std::printf( - " note: output truncated (hit the model's " - "context/generation cap before end-of-stream); " - "transcript is incomplete\n"); + " finalize: status=%s " + "revision=%d input=%lld ms committed=%lld ms\n", + transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, + (long long) fin_upd.audio_committed_ms); } - - const char * dl = transcribe_detected_language(ctx); - if (dl && *dl) { - std::printf("detected-language: %s\n", dl); + } + } else { + // --repeat N runs transcribe_run() N times for steady-state + // perf measurements. + for (int r = 0; r < args.repeat; ++r) { + run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + if (run_st != TRANSCRIBE_OK) { + break; } + } + } + std::printf("run: %s\n", transcribe_status_string(run_st)); + // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but + // preserve the partial transcript (see docs/input-limits.md), so show + // the result for them too — just flagged. + const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; + if (result_present) { + const char * text = transcribe_full_text(ctx); + std::printf("text: %s\n", (text && *text) ? text : "(empty)"); + output_ok = write_output_file(output, args.output_path, text) && output_ok; + + // A decode cut short before end-of-stream; the text above is + // incomplete. + if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + std::printf( + " note: decode stopped when the output began repeating " + "itself (repeats dropped); transcript is incomplete\n"); + } else if (transcribe_was_truncated(ctx)) { + std::printf( + " note: output truncated (hit the model's " + "context/generation cap before end-of-stream); " + "transcript is incomplete\n"); + } - const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); - const int n_seg = transcribe_n_segments(ctx); - const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; - if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { - std::printf("segments: %d\n", n_seg); - for (int i = 0; i < n_seg; ++i) { - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_get_segment(ctx, i, &seg); - char spk[16] = ""; - if (seg.speaker_id > 0) { - std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); - } - if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { - std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, - (seg.text != nullptr) ? seg.text : ""); - } else { - // Speaker-attributed turns without timing (granite SAA). - std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); - } + const char * dl = transcribe_detected_language(ctx); + if (dl && *dl) { + std::printf("detected-language: %s\n", dl); + } + + const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); + const int n_seg = transcribe_n_segments(ctx); + const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; + if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { + std::printf("segments: %d\n", n_seg); + for (int i = 0; i < n_seg; ++i) { + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_get_segment(ctx, i, &seg); + char spk[16] = ""; + if (seg.speaker_id > 0) { + std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); } - } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_wrd = transcribe_n_words(ctx); - std::printf("words: %d\n", n_wrd); - for (int i = 0; i < n_wrd; ++i) { - struct transcribe_word wrd; - transcribe_word_init(&wrd); - (void) transcribe_get_word(ctx, i, &wrd); - std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, - (wrd.text != nullptr) ? wrd.text : ""); + if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { + std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, + (seg.text != nullptr) ? seg.text : ""); + } else { + // Speaker-attributed turns without timing (granite SAA). + std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); } } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_tok = transcribe_n_tokens(ctx); - std::printf("tokens: %d\n", n_tok); - for (int i = 0; i < n_tok; ++i) { - struct transcribe_token tok; - transcribe_token_init(&tok); - (void) transcribe_get_token(ctx, i, &tok); - std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, - (tok.text != nullptr) ? tok.text : ""); - } + } + if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_wrd = transcribe_n_words(ctx); + std::printf("words: %d\n", n_wrd); + for (int i = 0; i < n_wrd; ++i) { + struct transcribe_word wrd; + transcribe_word_init(&wrd); + (void) transcribe_get_word(ctx, i, &wrd); + std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, + (wrd.text != nullptr) ? wrd.text : ""); } } - - transcribe_print_timings(ctx); - - { - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_get_timings(ctx, &tm); - const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; - if (total_ms > 0.0 && duration_s > 0.0) { - std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, - duration_s); + if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_tok = transcribe_n_tokens(ctx); + std::printf("tokens: %d\n", n_tok); + for (int i = 0; i < n_tok; ++i) { + struct transcribe_token tok; + transcribe_token_init(&tok); + (void) transcribe_get_token(ctx, i, &tok); + std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, + (tok.text != nullptr) ? tok.text : ""); } } + } - transcribe_session_free(ctx); - transcribe_model_free(model); + transcribe_print_timings(ctx); - if (run_st != TRANSCRIBE_OK || !output_ok) { - return EXIT_FAILURE; + { + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_get_timings(ctx, &tm); + const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; + if (total_ms > 0.0 && duration_s > 0.0) { + std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, + duration_s); } - } else { - std::printf("model: (none specified, skipping load)\n"); + } + + transcribe_session_free(ctx); + transcribe_model_free(model); + + if (run_st != TRANSCRIBE_OK || !output_ok) { + return EXIT_FAILURE; } return EXIT_SUCCESS; diff --git a/examples/cli/cli.h b/examples/cli/cli.h index 5224b1168..fbe4b5b50 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -1,8 +1,9 @@ // cli.h - shared declarations for the transcribe-cli example. // -// main.cpp parses arguments and owns process setup (log sink, output file); -// the per-role drivers (asr.cpp, diarize.cpp) do the work. Shared helpers live in -// namespace transcribe_cli next to the WAV loader in examples/common. +// main.cpp parses arguments, owns process setup (log sink, output file), and +// routes each model to its role's driver (asr.cpp, diarize.cpp). Shared +// helpers live in namespace transcribe_cli next to the WAV loader in +// examples/common. #pragma once @@ -108,13 +109,20 @@ struct cli_args { // Returns false (after an error message naming `path`) on a write failure. bool write_output_file(std::ofstream * output, const std::string & path, const char * text); -// ASR drivers (asr.cpp). Each loads the model, runs, prints, and returns the -// process exit code. +// ASR batch driver (asr.cpp): loads the model, runs every listed file, prints, +// and returns the process exit code. int run_asr_batch(const cli_args & args, std::ofstream * output); -int run_asr_file(const cli_args & args, std::ofstream * output); -// DIARIZE driver (diarize.cpp): runs on a model run_asr_file already loaded -// and printed. Takes ownership of the model. With -o, writes the segment lines. +// Single-file drivers, one per role. main.cpp loads the audio and the model, +// prints them, and dispatches on transcribe_model_roles(); the driver takes +// ownership of the model and returns the process exit code. +int run_asr_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output); + +// With -o, writes the segment lines. int run_diarize_file(const cli_args & args, transcribe_model * model, const std::vector & pcm, diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 40e2da5b0..0d5f4c0b9 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -6,6 +6,7 @@ // Run with --help for the full option list. #include "cli.h" +#include "wav.h" #include #include @@ -539,6 +540,55 @@ void log_cb(transcribe_log_level level, const char * msg, void * userdata) { std::fprintf(stderr, "%s %s%s", prefix, msg, (msg && *msg && msg[std::strlen(msg) - 1] == '\n') ? "" : "\n"); } +// One file: load the audio and (with -m) the model, print both, then hand the +// model to the driver for its role. The driver takes ownership of the model. +int run_file(const cli_args & args, std::ofstream * output) { + std::vector pcm; + std::string load_err; + if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { + std::fprintf(stderr, "wav: %s\n", load_err.c_str()); + return EXIT_FAILURE; + } + + const double duration_s = static_cast(pcm.size()) / 16000.0; + std::printf("audio: %s\n", args.wav_path.c_str()); + std::printf(" samples: %zu\n", pcm.size()); + std::printf(" duration: %.3f s\n", duration_s); + std::printf(" sample rate 16000 Hz mono float32\n"); + + if (args.model_path.empty()) { + std::printf("model: (none specified, skipping load)\n"); + return EXIT_SUCCESS; + } + + struct transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } + struct transcribe_model * model = nullptr; + const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); + std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); + if (st != TRANSCRIBE_OK) { + return EXIT_FAILURE; + } + std::printf(" backend: %s\n", transcribe_model_backend(model)); + if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { + std::printf(" name: %s\n", dn); + } + if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { + std::printf(" license: %s\n", lic); + } + + if ((transcribe_model_roles(model) & TRANSCRIBE_ROLE_ASR) != 0) { + return transcribe_cli::run_asr_file(args, model, pcm, duration_s, output); + } + return transcribe_cli::run_diarize_file(args, model, pcm, duration_s, output); +} + } // namespace int main(int argc, char ** argv) { @@ -579,5 +629,5 @@ int main(int argc, char ** argv) { return transcribe_cli::run_asr_batch(args, output); } - return transcribe_cli::run_asr_file(args, output); + return run_file(args, output); } diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index 403d878fb..b9d11f87b 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -12,6 +12,7 @@ #include "ggml.h" #include "gguf.h" #include "sortformer.h" +#include "transcribe-abi.h" #include "transcribe-arch.h" #include "transcribe-backend.h" #include "transcribe-batch-util.h" @@ -610,7 +611,9 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } - m->roles = TRANSCRIBE_ROLE_DIARIZE; + m->roles = TRANSCRIBE_ROLE_DIARIZE; + // The abort callback is honored (transcribe_diarize_set_abort_callback). + transcribe::set_feature(m.get(), TRANSCRIBE_FEATURE_CANCELLATION, true); m->t_load_us = ggml_time_us() - t_load_start; *out_model = m.release(); return TRANSCRIBE_OK; @@ -1002,15 +1005,23 @@ static bool accepts_ext_kind(const transcribe_model * model, transcribe_ext_slot kind == TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; } -static transcribe_status check_preset(transcribe_sortformer_preset preset) { +// The caller's preset, read as a raw int (see transcribe::enum_field_raw): +// an out-of-range value must be rejected before it is loaded as the enum. +static int ext_preset_raw(const transcribe_diarize_params * params) { + return transcribe::enum_field_raw( + &reinterpret_cast(params->family)->preset); +} + +static transcribe_status check_preset(int preset) { switch (preset) { case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: case TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY: case TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY: case TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY: return TRANSCRIBE_OK; + default: + return TRANSCRIBE_ERR_INVALID_ARG; } - return TRANSCRIBE_ERR_INVALID_ARG; } // ---- DIARIZE role ---- @@ -1029,7 +1040,7 @@ static transcribe_status diarize_run_validate(const transcribe_diarize_params * st != TRANSCRIBE_OK) { return st; } - return check_preset(reinterpret_cast(params->family)->preset); + return check_preset(ext_preset_raw(params)); } static transcribe_status diarize_run(transcribe_diarize_session * session, @@ -1041,10 +1052,10 @@ static transcribe_status diarize_run(transcribe_diarize_session * session, if (pc->poll_abort()) { return TRANSCRIBE_ERR_ABORTED; } - const transcribe_sortformer_preset preset = - params->family != nullptr ? - reinterpret_cast(params->family)->preset : - TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; + // Range-checked by diarize_run_validate before the dispatcher got here. + const transcribe_sortformer_preset preset = params->family != nullptr ? + static_cast(ext_preset_raw(params)) : + TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; return diarize_pcm(pc, static_cast(session->model), pcm, n_samples, preset, out); } diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp index d1361d348..cc582757b 100644 --- a/tests/sortformer_diarize_unit.cpp +++ b/tests/sortformer_diarize_unit.cpp @@ -88,6 +88,7 @@ int main() { // 1. Roles, info, extension probe. CHECK(transcribe_model_roles(model) == TRANSCRIBE_ROLE_DIARIZE); CHECK(!transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION)); + CHECK(transcribe_model_supports(model, TRANSCRIBE_FEATURE_CANCELLATION)); transcribe_diarize_info info; transcribe_diarize_info_init(&info); CHECK(transcribe_diarize_get_info(model, &info) == TRANSCRIBE_OK); From ae03b6851cb505d11efbabbf5765d31b5931fd89 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 5 Oct 2026 09:15:37 +0800 Subject: [PATCH 50/51] review fixes: Python close race, TS param leak, diarize CLI --repeat / --stream-chunk-ms --- .../python/src/transcribe_cpp/__init__.py | 24 ++++++++++--- bindings/python/tests/test_compute_lock.py | 23 ++++++++++++ bindings/typescript/src/index.ts | 35 +++++++++++++------ bindings/typescript/test/prompting.test.mjs | 18 ++++++++++ examples/cli/diarize.cpp | 17 ++++++++- tests/cli_diarize_smoke.cmake | 22 ++++++++++-- 6 files changed, 121 insertions(+), 18 deletions(-) diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index e15a8b1d6..a77f20225 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -1013,6 +1013,10 @@ def _h(self) -> ctypes.c_void_p: # holder runs them before releasing. close() queues every live session's # free before the model's, from _live (strong records, not the weak # _sessions, which the cycle collector clears before finalizers run). + # _free_order makes "pop a session's _live record + queue its free" one + # step against close() queuing the model's free, so a session closed on + # another thread can never land behind the model free. An RLock: a GC + # finalizer can close a session on the thread already holding it. # _stream_owner (the active stream's _StreamLease) is written only under it. def _init_compute_state(self) -> None: @@ -1022,6 +1026,7 @@ def _init_compute_state(self) -> None: self._stream_owner: Optional[_StreamLease] = None # id(handle) -> (handle, native free name, pin) per open session. self._live: dict = {} + self._free_order = threading.RLock() @contextmanager def _exclusive(self, kind: str, busy: Optional[str] = None): @@ -1096,6 +1101,12 @@ def _free_session(self, handle: ctypes.c_void_p) -> None: """Free a tracked session behind any in-flight call. The record is popped first, so the session's own close() and the model's close() free it exactly once, whichever runs first.""" + with self._free_order: + self._queue_session_free(handle) + self._try_drain() + + def _queue_session_free(self, handle: ctypes.c_void_p) -> None: + """Pop *handle*'s record and queue its free. Caller holds _free_order.""" record = self._live.pop(id(handle), None) if record is None: return @@ -1108,7 +1119,7 @@ def free(_pin=pin): if getattr(self._stream_owner, "handle", None) is handle: self._stream_owner = None - self._free_or_defer(free) + self._deferred.append(free) @property def arch(self) -> str: @@ -1229,9 +1240,14 @@ def close(self) -> None: self._handle = None for session in list(getattr(self, "_sessions", ()) or ()): session.close() - for record in list(getattr(self, "_live", {}).values()): - self._free_session(record[0]) # wrappers already collected - self._free_or_defer(lambda: _lib.transcribe_model_free(handle)) + # One step under _free_order: every session free still pending + # (wrappers already collected, or a close() racing on another thread) + # is queued ahead of the model's. + with self._free_order: + for record in list(self._live.values()): + self._queue_session_free(record[0]) + self._deferred.append(lambda: _lib.transcribe_model_free(handle)) + self._try_drain() def __enter__(self) -> "Model": return self diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py index 7c4ed2486..ef65ddf8a 100644 --- a/bindings/python/tests/test_compute_lock.py +++ b/bindings/python/tests/test_compute_lock.py @@ -318,6 +318,29 @@ def probe(h, *rest): assert fake.frees[2:] == [("model", hm)] +def test_model_close_racing_session_close_frees_session_first(fake): + # Model.close() on another thread lands between a session close() popping + # its _live record and queuing its free: the model free must still queue + # behind the session's. + m = fake.model() + s = fake.session(m) + hs, hm = s._handle.value, m._handle.value + closer: dict = {} + + class _RacingLive(dict): + def pop(self, key, *default): + record = super().pop(key, *default) + if not closer: + closer["thread"], _ = _in_thread(m.close) + closer["thread"].join(0.2) # let it run as far as it can + return record + + m._live = _RacingLive(m._live) + s.close() + _join(closer["thread"], "model.close() racing session.close()") + assert fake.frees == [("session", hs), ("model", hm)] + + def test_gc_on_holder_thread_defers_free(fake, monkeypatch): # A finalizer can run on the thread that holds the lock (GC mid copy-out). # It must neither deadlock nor free under the in-flight call. diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index 5a67fe6e6..32adc1eaf 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -985,6 +985,16 @@ export class Session { const n = this.#n; const p: any = {}; n.F.runParamsInit(p); + try { + this.#fillRunParams(n, p, opts); + } catch (e) { + freeRunParams(n, p); // the caller never gets p, so free what was allocated + throw e; + } + return p; + } + + #fillRunParams(n: Native, p: any, opts: TranscribeOptions): void { p.task = lookup(TASKS, opts.task ?? "transcribe", "task"); // Default "auto" mirrors the C transcribe_run_params_init default: // whisper resolves it to "segment" (its robust path), no-timestamp @@ -1015,10 +1025,8 @@ export class Session { } if (opts.prompt !== undefined) p.prompt = cstr(opts.prompt, "prompt"); if (opts.prefix !== undefined) p.prefix = cstr(opts.prefix, "prefix"); - // Last, so no later validation throw can strand the allocation. if (opts.family) p.family = buildFamily(n, this.#model.handle, opts.family, "run"); - return p; } /** @@ -1107,16 +1115,21 @@ export class Session { }); const sp: any = {}; F.streamParamsInit(sp); - sp.commit_policy = lookup( - COMMIT_POLICIES, - opts.commitPolicy ?? "auto", - "commitPolicy", - ); - if (opts.stablePrefixAgreementN !== undefined) { - sp.stable_prefix_agreement_n = opts.stablePrefixAgreementN; + try { + sp.commit_policy = lookup( + COMMIT_POLICIES, + opts.commitPolicy ?? "auto", + "commitPolicy", + ); + if (opts.stablePrefixAgreementN !== undefined) { + sp.stable_prefix_agreement_n = opts.stablePrefixAgreementN; + } + if (opts.family) + sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); + } catch (e) { + freeRunParams(n, rp); // sp holds nothing yet: buildFamily frees on throw + throw e; } - if (opts.family) - sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); // Begin is a synchronous native call, so no in-flight window (no call()). return this.#core.exclusive("begin a stream", async () => { diff --git a/bindings/typescript/test/prompting.test.mjs b/bindings/typescript/test/prompting.test.mjs index fdac27bca..fe60aeb18 100644 --- a/bindings/typescript/test/prompting.test.mjs +++ b/bindings/typescript/test/prompting.test.mjs @@ -2,6 +2,7 @@ // streaming canaries. Mirrors bindings/python/tests/test_prompting.py. import assert from "node:assert/strict"; +import koffi from "koffi"; import { modelTest, MODEL, STREAMING_MODEL, jfk, feedChunks } from "./common.mjs"; import { TranscribeModel, InvalidArgument, UnsupportedRequest } from "../dist/index.js"; @@ -33,6 +34,23 @@ modelTest("vocabulary and prompt reach run and runBatch", MODEL, async () => { }); }); +modelTest("a rejected family option frees the vocabulary it followed", MODEL, async () => { + await withSession(MODEL, async (_m, s) => { + const { alloc, free } = koffi; + let live = 0; + koffi.alloc = (...a) => (live++, alloc(...a)); + koffi.free = (...a) => (live--, free(...a)); + try { + await assert.rejects(s.run(jfk(), { vocabulary: TERMS, family: { kind: "nope" } })); + await assert.rejects(s.stream({ vocabulary: TERMS, commitPolicy: "nope" })); + } finally { + koffi.alloc = alloc; + koffi.free = free; + } + assert.equal(live, 0, "koffi allocations leaked"); + }); +}); + modelTest("prefix: text is the continuation, rawText leads with it", MODEL, async () => { await withSession(MODEL, async (m, s) => { if (!m.supports("transcript_prefix")) return; diff --git a/examples/cli/diarize.cpp b/examples/cli/diarize.cpp index b779ab7a7..15a3a21ad 100644 --- a/examples/cli/diarize.cpp +++ b/examples/cli/diarize.cpp @@ -16,6 +16,14 @@ int transcribe_cli::run_diarize_file(const cli_args & args, const std::vector & pcm, double duration_s, std::ofstream * output) { + // The DIARIZE role has no push-audio entry point yet; refuse rather than + // silently diarizing the whole file. + if (args.stream_chunk_ms > 0) { + std::fprintf(stderr, "stream: the diarize path has no streaming entry point; drop --stream-chunk-ms\n"); + transcribe_model_free(model); + return EXIT_FAILURE; + } + transcribe_diarize_session_params sp; transcribe_diarize_session_params_init(&sp); sp.n_threads = args.n_threads; @@ -27,7 +35,14 @@ int transcribe_cli::run_diarize_file(const cli_args & args, return EXIT_FAILURE; } - st = transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), nullptr); + // --repeat N runs transcribe_diarize_run() N times for steady-state perf + // measurements. + for (int r = 0; r < args.repeat; ++r) { + st = transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), nullptr); + if (st != TRANSCRIBE_OK) { + break; + } + } std::printf("run: %s\n", transcribe_status_string(st)); bool output_ok = true; if (st == TRANSCRIBE_OK) { diff --git a/tests/cli_diarize_smoke.cmake b/tests/cli_diarize_smoke.cmake index 83fa2c85c..f0e326a8b 100644 --- a/tests/cli_diarize_smoke.cmake +++ b/tests/cli_diarize_smoke.cmake @@ -1,6 +1,6 @@ # transcribe-cli routes a DIARIZE model to the diarize path and prints speaker -# segments for the 2-speaker oracle mix, writes them with -o, and refuses -# --batch. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). +# segments for the 2-speaker oracle mix, writes them with -o, honours --repeat, +# and refuses --batch and --stream-chunk-ms. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") message("SKIP: TRANSCRIBE_SORTFORMER_GGUF not set") @@ -36,3 +36,21 @@ execute_process( if(rc EQUAL 0 OR NOT err MATCHES "--batch is ASR-only") message(FATAL_ERROR "--batch on a diarize model: rc=${rc}\n${err}") endif() + +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --repeat 2 "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(NOT rc EQUAL 0 OR NOT out MATCHES "speaker segments: [1-9].*S2") + message(FATAL_ERROR "--repeat 2 on a diarize model: rc=${rc}\n${out}\n${err}") +endif() + +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --stream-chunk-ms 100 "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(rc EQUAL 0 OR NOT err MATCHES "no streaming entry point") + message(FATAL_ERROR "--stream-chunk-ms on a diarize model: rc=${rc}\n${err}") +endif() From f846d7012127f7a46ecff04586294f0297088362 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 5 Oct 2026 11:00:52 +0800 Subject: [PATCH 51/51] review fixes --- docs/environment-variables.md | 2 +- include/transcribe/diarize.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 14d794602..053b0f7e6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -89,7 +89,7 @@ its var is unset. Convention: `TRANSCRIBE__GGUF`. | `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_diarize_unit`, `cli_diarize_smoke` | | `TRANSCRIBE_COHERE_GGUF` | `cohere_real_smoke`, `cohere_e2e_smoke` | | `TRANSCRIBE_GRANITE5_CTC_GGUF` | `granite5_ctc_real_smoke`, `granite5_ctc_e2e_smoke` | -| `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity`, `sortformer_diarize_unit` (concurrency leg) | +| `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity` | | `TRANSCRIBE_QWEN3_ASR_GGUF` (+ `_0_6B_GGUF` / `_1_7B_GGUF`) | qwen3_asr smokes / parity | | `TRANSCRIBE_MOONSHINE_STREAMING_TINY_GGUF` | moonshine_streaming smokes, `stream_offline_interleave_smoke` | | `TRANSCRIBE_VOXTRAL_REALTIME_GGUF` | `voxtral_realtime_real_smoke`, `stream_offline_interleave_smoke` | diff --git a/include/transcribe/diarize.h b/include/transcribe/diarize.h index 0168614d7..4a4c2793a 100644 --- a/include/transcribe/diarize.h +++ b/include/transcribe/diarize.h @@ -74,10 +74,10 @@ TRANSCRIBE_API transcribe_status transcribe_diarize_run(struct transcribe_diariz const struct transcribe_diarize_params * params); /* - * Speaker segments of the last successful run, grouped by speaker and - * time-ordered within a speaker; segments of different speakers may - * overlap. 0 before any run. An out-of-range index returns OK with a - * zeroed row. + * Speaker segments of the last run, grouped by speaker and time-ordered + * within a speaker; segments of different speakers may overlap. 0 before any + * run, and 0 after a run that passed input validation but then failed + * (aborted, backend error). An out-of-range index returns OK with a zeroed row. */ TRANSCRIBE_API int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session); TRANSCRIBE_API transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session,