diff --git a/README.md b/README.md index 2f5e6487f..1393cc9f2 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,12 @@ C/C++ speech-to-text inference library. Runs diverse STT model families via [GGU | Multitalker Parakeet Streaming 0.6B v1 | `multitalker-parakeet-streaming-0.6b-v1` | diarize, streaming, token timestamps | [docs/models/multitalker-parakeet-streaming-0.6b-v1.md](docs/models/multitalker-parakeet-streaming-0.6b-v1.md) | | Nemotron 3.5 ASR Streaming 0.6B | `nemotron-3.5-asr-streaming-0.6b` | streaming, token timestamps | [docs/models/nemotron-3.5-asr-streaming-0.6b.md](docs/models/nemotron-3.5-asr-streaming-0.6b.md) | | Nemotron Speech Streaming EN 0.6B | `nemotron-speech-streaming-en-0.6b` | streaming, token timestamps | [docs/models/nemotron-speech-streaming-en-0.6b.md](docs/models/nemotron-speech-streaming-en-0.6b.md) | -| Parakeet | `parakeet-ctc-0.6b`, `parakeet-ctc-1.1b`, `parakeet-primeline`, `parakeet-rnnt-0.6b`, `parakeet-rnnt-1.1b`, `parakeet-tdt-0.6b-v2`, `parakeet-tdt-0.6b-v3`, `parakeet-tdt-1.1b`, `parakeet-tdt_ctc-1.1b`, `parakeet-tdt_ctc-110m`, `parakeet-ultra`, `parakeet-unified-en-0.6b` | streaming, token timestamps | [docs/models/parakeet.md](docs/models/parakeet.md) | +| Parakeet | `parakeet-ctc-0.6b`, `parakeet-ctc-1.1b`, `parakeet-primeline`, `parakeet-rnnt-0.6b`, `parakeet-rnnt-1.1b`, `parakeet-tdt-0.6b-v2`, `parakeet-tdt-0.6b-v3`, `parakeet-tdt-1.1b`, `parakeet-tdt_ctc-1.1b`, `parakeet-tdt_ctc-110m`, `parakeet-ultra`, `parakeet-unified-en-0.6b`, `orukeet` | streaming, token timestamps | [docs/models/parakeet.md](docs/models/parakeet.md) | | Qwen3-ASR | `qwen3-asr-0.6b`, `qwen3-asr-1.7b` | - | [docs/models/qwen3-asr.md](docs/models/qwen3-asr.md) | | SenseVoice Small | `sensevoice-small` | - | [docs/models/sensevoice-small.md](docs/models/sensevoice-small.md) | | Voxtral (2507) | `voxtral-mini-3b-2507`, `voxtral-small-24b-2507` | translate | [docs/models/voxtral.md](docs/models/voxtral.md) | | Voxtral Realtime (2602) | `voxtral-mini-4b-realtime-2602` | streaming | [docs/models/voxtral-realtime.md](docs/models/voxtral-realtime.md) | -| Whisper | `breeze-asr-25`, `whisper-base`, `whisper-base.en`, `whisper-large`, `whisper-large-v2`, `whisper-large-v3`, `whisper-large-v3-turbo`, `whisper-medium`, `whisper-medium.en`, `whisper-small`, `whisper-small.en`, `whisper-tiny`, `whisper-tiny.en` | segment timestamps, translate | [docs/models/whisper.md](docs/models/whisper.md) | +| Whisper | `whisper-base`, `whisper-base.en`, `whisper-large`, `whisper-large-v2`, `whisper-large-v3`, `whisper-large-v3-turbo`, `whisper-medium`, `whisper-medium.en`, `whisper-small`, `whisper-small.en`, `whisper-tiny`, `whisper-tiny.en`, `breeze-asr-25` | segment timestamps, translate | [docs/models/whisper.md](docs/models/whisper.md) | **Speaker diarization models** (no transcription; verified by DER/JER rather than WER): diff --git a/catalog/orukeet.json b/catalog/orukeet.json new file mode 100644 index 000000000..44c0de7be --- /dev/null +++ b/catalog/orukeet.json @@ -0,0 +1,98 @@ +{ + "schema": "transcribe-catalog-v1", + "variant": "orukeet", + "family": "parakeet", + "display_name": "Orukeet", + "params": 627052166, + "license": { + "spdx": "cc-by-sa-4.0", + "display": "CC-BY-SA-4.0" + }, + "upstream_repo": "oruk/orukeet", + "upstream_commit": "b59c13a", + "published_repo": "handy-computer/orukeet-gguf", + "docs_page": "parakeet.md", + "languages": [ + "bg", "hr", "cs", "da", "nl", "en", "et", "fi", "fr", "de", "el", "hu", + "it", "lv", "lt", "mt", "pl", "pt", "ro", "ru", "sk", "sl", "es", "sv", + "uk" + ], + "language_tag_form": "bare-bcp47", + "long_form_strategy": "chunked-unbounded", + "capabilities": { + "transcribe": {"supported":true,"verified":false}, + "translate": {"supported":false}, + "lang_detect": {"supported":true,"verified":false}, + "timestamps": {"supported":true,"verified":false,"granularities":["token"]}, + "streaming": {"supported":false}, + "diarize": {"supported":false}, + "batching": {"supported":true,"verified":true} + }, + "downloads": [ + {"quant":"F32","filename":"orukeet-F32.gguf","size_bytes":2508435744}, + {"quant":"F16","filename":"orukeet-F16.gguf","size_bytes":1255869984}, + {"quant":"Q8_0","filename":"orukeet-Q8_0.gguf","size_bytes":739508704}, + {"quant":"Q6_K","filename":"orukeet-Q6_K.gguf","size_bytes":610342368}, + {"quant":"Q5_K_M","filename":"orukeet-Q5_K_M.gguf","size_bytes":548946400}, + {"quant":"Q4_K_M","filename":"orukeet-Q4_K_M.gguf","size_bytes":485425632} + ], + "accuracy_benchmarks": [ + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"F32","metric":"wer","err_pct":1.86,"ci95":[1.71,2.01],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":742,"del":145,"ins":101},"empty_hyp":0,"utts_over_50pct":6}, + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"F16","metric":"wer","err_pct":1.86,"ci95":[1.71,2.01],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":742,"del":145,"ins":100},"empty_hyp":0,"utts_over_50pct":6}, + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":1.86,"ci95":[1.71,2.02],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":741,"del":146,"ins":100},"empty_hyp":0,"utts_over_50pct":6}, + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"Q6_K","metric":"wer","err_pct":1.86,"ci95":[1.7,2.01],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":742,"del":143,"ins":99},"empty_hyp":0,"utts_over_50pct":6}, + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"Q5_K_M","metric":"wer","err_pct":1.83,"ci95":[1.68,1.98],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":738,"del":142,"ins":91},"empty_hyp":0,"utts_over_50pct":5}, + {"dataset":"librispeech","split":"test-clean","language":"en","language_hint":"en","backend":"cuda","quant":"Q4_K_M","metric":"wer","err_pct":1.95,"ci95":[1.79,2.11],"n_utts":2620,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":783,"del":145,"ins":105},"empty_hyp":0,"utts_over_50pct":6}, + {"dataset":"fleurs","split":"test","language":"bg","language_hint":"bg","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":11.14,"ci95":[10.42,11.88],"n_utts":658,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1244,"del":234,"ins":100},"empty_hyp":0,"utts_over_50pct":3}, + {"dataset":"fleurs","split":"test","language":"hr","language_hint":"hr","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":11.26,"ci95":[10.58,11.96],"n_utts":914,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1615,"del":213,"ins":171},"empty_hyp":0,"utts_over_50pct":4}, + {"dataset":"fleurs","split":"test","language":"cs","language_hint":"cs","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":10.22,"ci95":[9.41,11.01],"n_utts":723,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1166,"del":121,"ins":112},"empty_hyp":0,"utts_over_50pct":9}, + {"dataset":"fleurs","split":"test","language":"da","language_hint":"da","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":16.58,"ci95":[15.84,17.27],"n_utts":930,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":2649,"del":381,"ins":312},"empty_hyp":1,"utts_over_50pct":15}, + {"dataset":"fleurs","split":"test","language":"nl","language_hint":"nl","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":6.93,"ci95":[6.13,7.7],"n_utts":364,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":440,"del":71,"ins":71},"empty_hyp":0,"utts_over_50pct":0}, + {"dataset":"fleurs","split":"test","language":"en","language_hint":"en","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":4.43,"ci95":[3.99,4.9],"n_utts":647,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":446,"del":115,"ins":84},"empty_hyp":0,"utts_over_50pct":0}, + {"dataset":"fleurs","split":"test","language":"et","language_hint":"et","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":15.16,"ci95":[14.39,15.99],"n_utts":893,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1686,"del":289,"ins":281},"empty_hyp":0,"utts_over_50pct":10}, + {"dataset":"fleurs","split":"test","language":"fi","language_hint":"fi","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":11.88,"ci95":[11.11,12.67],"n_utts":918,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1412,"del":203,"ins":143},"empty_hyp":0,"utts_over_50pct":10}, + {"dataset":"fleurs","split":"test","language":"fr","language_hint":"fr","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":6.17,"ci95":[5.62,6.79],"n_utts":676,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":812,"del":113,"ins":185},"empty_hyp":0,"utts_over_50pct":3}, + {"dataset":"fleurs","split":"test","language":"de","language_hint":"de","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":5.64,"ci95":[5.13,6.17],"n_utts":862,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":794,"del":133,"ins":128},"empty_hyp":0,"utts_over_50pct":3}, + {"dataset":"fleurs","split":"test","language":"el","language_hint":"el","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":33.12,"ci95":[31.97,34.29],"n_utts":650,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":4622,"del":257,"ins":147},"empty_hyp":0,"utts_over_50pct":78}, + {"dataset":"fleurs","split":"test","language":"hu","language_hint":"hu","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":13.53,"ci95":[12.84,14.26],"n_utts":905,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1852,"del":239,"ins":220},"empty_hyp":0,"utts_over_50pct":8}, + {"dataset":"fleurs","split":"test","language":"it","language_hint":"it","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":3.11,"ci95":[2.81,3.43],"n_utts":865,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":508,"del":74,"ins":79},"empty_hyp":0,"utts_over_50pct":0}, + {"dataset":"fleurs","split":"test","language":"lv","language_hint":"lv","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":19.53,"ci95":[18.61,20.48],"n_utts":851,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":2642,"del":188,"ins":180},"empty_hyp":0,"utts_over_50pct":20}, + {"dataset":"fleurs","split":"test","language":"lt","language_hint":"lt","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":17.72,"ci95":[16.97,18.46],"n_utts":986,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":2700,"del":208,"ins":128},"empty_hyp":0,"utts_over_50pct":16}, + {"dataset":"fleurs","split":"test","language":"mt","language_hint":"mt","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":17.18,"ci95":[16.41,17.93],"n_utts":926,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":3114,"del":395,"ins":310},"empty_hyp":0,"utts_over_50pct":12}, + {"dataset":"fleurs","split":"test","language":"pl","language_hint":"pl","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":7.35,"ci95":[6.76,7.97],"n_utts":758,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":847,"del":88,"ins":123},"empty_hyp":0,"utts_over_50pct":3}, + {"dataset":"fleurs","split":"test","language":"pt","language_hint":"pt","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":4.56,"ci95":[4.09,5.09],"n_utts":919,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":676,"del":97,"ins":207},"empty_hyp":0,"utts_over_50pct":4}, + {"dataset":"fleurs","split":"test","language":"ro","language_hint":"ro","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":10.86,"ci95":[10.24,11.49],"n_utts":883,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1744,"del":334,"ins":181},"empty_hyp":0,"utts_over_50pct":7}, + {"dataset":"fleurs","split":"test","language":"ru","language_hint":"ru","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":6.7,"ci95":[6.13,7.27],"n_utts":775,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":849,"del":76,"ins":87},"empty_hyp":0,"utts_over_50pct":2}, + {"dataset":"fleurs","split":"test","language":"sk","language_hint":"sk","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":8.99,"ci95":[8.24,9.82],"n_utts":792,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1080,"del":185,"ins":105},"empty_hyp":4,"utts_over_50pct":9}, + {"dataset":"fleurs","split":"test","language":"sl","language_hint":"sl","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":24.0,"ci95":[22.62,25.53],"n_utts":834,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":3211,"del":415,"ins":337},"empty_hyp":0,"utts_over_50pct":47}, + {"dataset":"fleurs","split":"test","language":"es","language_hint":"es","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":3.75,"ci95":[3.4,4.12],"n_utts":908,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":645,"del":94,"ins":136},"empty_hyp":0,"utts_over_50pct":1}, + {"dataset":"fleurs","split":"test","language":"sv","language_hint":"sv","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":13.26,"ci95":[12.5,13.97],"n_utts":759,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":1575,"del":315,"ins":150},"empty_hyp":0,"utts_over_50pct":8}, + {"dataset":"fleurs","split":"test","language":"uk","language_hint":"uk","backend":"cuda","quant":"Q8_0","metric":"wer","err_pct":6.53,"ci95":[5.98,7.11],"n_utts":750,"batch_size":8,"timestamps":"none","engine_sha":"37273409","publication_profile":"asr-publication-v2","measured_on":null,"errors":{"sub":740,"del":88,"ins":111},"empty_hyp":0,"utts_over_50pct":3} + ], + "headline_benchmark": { + "dataset": "librispeech", + "split": "test-clean", + "language": "en", + "metric": "wer", + "batch_size": null, + "timestamps": "none" + }, + "speed_benchmarks": [ + {"machine":"ryzen-4750u","backend":"cpu","quant":"Q4_K_M","sample":"dots","sample_duration_s":35.332,"total_ms":2950.8,"xrt_compute":11.97,"wall_ms":2977.2,"xrt_wall":11.87,"load_ms":659.1,"mel_ms":56.8,"encode_ms":2562.6,"decode_ms":331.5,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"cpu","quant":"Q4_K_M","sample":"jfk","sample_duration_s":11.0,"total_ms":775.7,"xrt_compute":14.18,"wall_ms":785.3,"xrt_wall":14.01,"load_ms":661.2,"mel_ms":18.8,"encode_ms":683.9,"decode_ms":73.0,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"cpu","quant":"Q8_0","sample":"dots","sample_duration_s":35.332,"total_ms":2903.0,"xrt_compute":12.17,"wall_ms":2929.4,"xrt_wall":12.06,"load_ms":839.2,"mel_ms":57.8,"encode_ms":2518.4,"decode_ms":326.8,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"cpu","quant":"Q8_0","sample":"jfk","sample_duration_s":11.0,"total_ms":729.9,"xrt_compute":15.07,"wall_ms":739.6,"xrt_wall":14.87,"load_ms":835.6,"mel_ms":18.3,"encode_ms":638.9,"decode_ms":72.8,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"vulkan","quant":"Q4_K_M","sample":"dots","sample_duration_s":35.332,"total_ms":1380.5,"xrt_compute":25.59,"wall_ms":1405.1,"xrt_wall":25.15,"load_ms":368.3,"mel_ms":59.4,"encode_ms":1004.8,"decode_ms":316.3,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"vulkan","quant":"Q4_K_M","sample":"jfk","sample_duration_s":11.0,"total_ms":455.1,"xrt_compute":24.17,"wall_ms":470.2,"xrt_wall":23.39,"load_ms":372.4,"mel_ms":18.4,"encode_ms":364.3,"decode_ms":72.4,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"vulkan","quant":"Q8_0","sample":"dots","sample_duration_s":35.332,"total_ms":1359.5,"xrt_compute":25.99,"wall_ms":1381.3,"xrt_wall":25.58,"load_ms":466.5,"mel_ms":58.2,"encode_ms":983.2,"decode_ms":318.1,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"ryzen-4750u","backend":"vulkan","quant":"Q8_0","sample":"jfk","sample_duration_s":11.0,"total_ms":454.5,"xrt_compute":24.2,"wall_ms":471.1,"xrt_wall":23.35,"load_ms":480.3,"mel_ms":18.0,"encode_ms":364.3,"decode_ms":72.1,"engine_sha":"8bab590e","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"cpu","quant":"Q4_K_M","sample":"dots","sample_duration_s":35.332,"total_ms":1030.3,"xrt_compute":34.29,"wall_ms":1033.7,"xrt_wall":34.18,"load_ms":234.1,"mel_ms":2.3,"encode_ms":978.0,"decode_ms":50.1,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"cpu","quant":"Q4_K_M","sample":"jfk","sample_duration_s":11.0,"total_ms":305.3,"xrt_compute":36.03,"wall_ms":307.1,"xrt_wall":35.82,"load_ms":183.1,"mel_ms":0.9,"encode_ms":292.8,"decode_ms":11.6,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"cpu","quant":"Q8_0","sample":"dots","sample_duration_s":35.332,"total_ms":969.4,"xrt_compute":36.45,"wall_ms":972.8,"xrt_wall":36.32,"load_ms":238.4,"mel_ms":2.3,"encode_ms":918.2,"decode_ms":48.9,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"cpu","quant":"Q8_0","sample":"jfk","sample_duration_s":11.0,"total_ms":304.1,"xrt_compute":36.17,"wall_ms":306.0,"xrt_wall":35.95,"load_ms":237.2,"mel_ms":0.9,"encode_ms":284.6,"decode_ms":18.6,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"metal","quant":"Q4_K_M","sample":"dots","sample_duration_s":35.332,"total_ms":163.7,"xrt_compute":215.85,"wall_ms":171.1,"xrt_wall":206.47,"load_ms":154.8,"mel_ms":2.2,"encode_ms":112.2,"decode_ms":49.3,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"metal","quant":"Q4_K_M","sample":"jfk","sample_duration_s":11.0,"total_ms":64.7,"xrt_compute":170.12,"wall_ms":68.5,"xrt_wall":160.64,"load_ms":148.4,"mel_ms":0.8,"encode_ms":47.7,"decode_ms":16.1,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"metal","quant":"Q8_0","sample":"dots","sample_duration_s":35.332,"total_ms":159.3,"xrt_compute":221.73,"wall_ms":166.9,"xrt_wall":211.63,"load_ms":204.4,"mel_ms":2.1,"encode_ms":110.6,"decode_ms":46.6,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null}, + {"machine":"m4-max","backend":"metal","quant":"Q8_0","sample":"jfk","sample_duration_s":11.0,"total_ms":59.2,"xrt_compute":185.68,"wall_ms":63.0,"xrt_wall":174.46,"load_ms":200.6,"mel_ms":0.7,"encode_ms":46.6,"decode_ms":11.9,"engine_sha":"d91374b3","publication_profile":"asr-publication-v2","measured_on":"2026-10-06","thermal_gated":null} + ] +} diff --git a/docs/models/orukeet.md b/docs/models/orukeet.md new file mode 100644 index 000000000..861b6432b --- /dev/null +++ b/docs/models/orukeet.md @@ -0,0 +1,192 @@ +# Orukeet + + +Upstream: [`oruk/orukeet`](https://huggingface.co/oruk/orukeet) at [`b59c13a`](https://huggingface.co/oruk/orukeet/commit/b59c13a). + +A tuned version of parakeet-tdt-0.6b-v3 from Oruk AI. Turns speech into +text in 25 European languages, works offline, and runs as fast as v3. Give +it a 16 kHz mono WAV and it gives you back text with punctuation and +capitals. Swap it in anywhere you use v3. Does not stream and does not +translate. + + +## What it does + +Speech to text in 25 European languages, offline. It is +`parakeet-tdt-0.6b-v3` with new weights from Oruk AI: same size, same speed, +same 25 languages, slightly more accurate. If you already run v3, point at +this file instead and you are done. + +It does not stream and it does not translate. + + +Licensed CC-BY-SA-4.0. Ported from upstream commit [`b59c13a`](https://huggingface.co/oruk/orukeet/commit/b59c13a), pinned 2026-10-06. Validated against the NeMo reference at transcribe.cpp commit [`65856fac`](https://github.com/handy-computer/transcribe.cpp/tree/65856fac) on 2026-10-06. + + +## Download + + +| Quantization | Download | Size | WER (LibriSpeech test-clean) | +| --- | --- | ---: | ---: | +| F32 | [orukeet-F32.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-F32.gguf) | 2.51 GB | 1.86% | +| F16 | [orukeet-F16.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-F16.gguf) | 1.26 GB | 1.86% | +| Q8_0 | [orukeet-Q8_0.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-Q8_0.gguf) | 740 MB | 1.86% | +| Q6_K | [orukeet-Q6_K.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-Q6_K.gguf) | 610 MB | 1.86% | +| Q5_K_M | [orukeet-Q5_K_M.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-Q5_K_M.gguf) | 549 MB | 1.83% | +| Q4_K_M | [orukeet-Q4_K_M.gguf](https://huggingface.co/handy-computer/orukeet-gguf/resolve/main/orukeet-Q4_K_M.gguf) | 485 MB | 1.95% | + + + +WER on the full LibriSpeech test-clean split (2,620 utterances), batch size 8, timestamps none, language hint `en`, decoded on cuda. Measured at transcribe.cpp `37273409`. + + + +A bit more accurate than v3 on English and on 20 of the 25 languages. +parakeet-ultra is more accurate than both on almost everything. + + + +**FLEURS test** + +| Language | Metric | Q8_0 | +| --- | --- | ---: | +| bg | WER | 11.14% | +| cs | WER | 10.22% | +| da | WER | 16.58% | +| de | WER | 5.64% | +| el | WER | 33.12% | +| en | WER | 4.43% | +| es | WER | 3.75% | +| et | WER | 15.16% | +| fi | WER | 11.88% | +| fr | WER | 6.17% | +| hr | WER | 11.26% | +| hu | WER | 13.53% | +| it | WER | 3.11% | +| lt | WER | 17.72% | +| lv | WER | 19.53% | +| mt | WER | 17.18% | +| nl | WER | 6.93% | +| pl | WER | 7.35% | +| pt | WER | 4.56% | +| ro | WER | 10.86% | +| ru | WER | 6.70% | +| sk | WER | 8.99% | +| sl | WER | 24.00% | +| sv | WER | 13.26% | +| uk | WER | 6.53% | + + +## Quick Start + +```bash +cmake -B build +cmake --build build + +build/bin/transcribe-cli \ + -m models/orukeet/orukeet-Q8_0.gguf \ + samples/jfk.wav +``` + +Audio has to be 16 kHz mono WAV. Convert anything else first: + +```bash +ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav +``` + +It figures out the language on its own. Add `--language de` (or any of the +25 codes) if you want to force one. + +## Performance + +Same speed as v3. + +### Apple M4 Max + + +Compute latency (mel + encode + decode), speedup over realtime in parentheses; profile `asr-publication-v2`: mean over 3 iterations after 1 warmup. + +| Backend | Sample | Q8_0 | Q4_K_M | +| ------- | ------------ | ---------------: | ---------------: | +| Metal | jfk (11.0s) | 59 ms (185.68×) | 65 ms (170.12×) | +| Metal | dots (35.3s) | 159 ms (221.73×) | 164 ms (215.85×) | +| CPU | jfk (11.0s) | 304 ms (36.17×) | 305 ms (36.03×) | +| CPU | dots (35.3s) | 969 ms (36.45×) | 1.03 s (34.29×) | + +Apple M4 Max: transcribe.cpp `d91374b3` on 2026-10-06. + + +### AMD Ryzen 7 PRO 4750U + + +Compute latency (mel + encode + decode), speedup over realtime in parentheses; profile `asr-publication-v2`: mean over 3 iterations after 1 warmup. + +| Backend | Sample | Q8_0 | Q4_K_M | +| ------- | ------------ | --------------: | --------------: | +| Vulkan | jfk (11.0s) | 454 ms (24.20×) | 455 ms (24.17×) | +| Vulkan | dots (35.3s) | 1.36 s (25.99×) | 1.38 s (25.59×) | +| CPU | jfk (11.0s) | 730 ms (15.07×) | 776 ms (14.18×) | +| CPU | dots (35.3s) | 2.90 s (12.17×) | 2.95 s (11.97×) | + +AMD Ryzen 7 PRO 4750U (Radeon RADV RENOIR): transcribe.cpp `8bab590e` on 2026-10-06. + + +Rerun the benchmark yourself: + +```bash +uv run scripts/bench/run.py --profile --models orukeet +``` + +## How we checked it + +Compared tensor by tensor against NeMo running Oruk's own `.nemo` file, +using the same tolerances as v3. Everything matches and the test transcript +is identical. Batches of 2, 4 and 8 give the same text as one at a time. + +| Field | Value | +| --- | --- | +| Reference | NeMo, `oruk/orukeet` `orukeet-v0.1.0.nemo` | +| Dump script | `scripts/dump_reference_parakeet_nemo.py` | +| Manifest | `tests/golden/parakeet/orukeet.manifest.json` | +| Tolerances | `tests/tolerances/parakeet.json` | +| Command | `uv run scripts/validate.py all --family parakeet --variant orukeet --model models/orukeet/orukeet-v0.1.0.nemo` | + +## Build it yourself + +```bash +# get the source checkpoint +hf download oruk/orukeet orukeet-v0.1.0.nemo --local-dir models/orukeet + +# convert to F32 GGUF +uv run --project scripts/envs/parakeet \ + scripts/convert-parakeet.py models/orukeet/orukeet-v0.1.0.nemo \ + models/orukeet/orukeet-F32.gguf --repo-id oruk/orukeet + +# make the smaller files +uv run scripts/quantize-all.py models/orukeet/orukeet-F32.gguf + +# check it against NeMo +uv run scripts/validate.py all --family parakeet --variant orukeet \ + --model models/orukeet/orukeet-v0.1.0.nemo + +# measure WER +uv run scripts/wer/ingest.py librispeech +uv run scripts/wer/run.py \ + --model models/orukeet/orukeet-F32.gguf \ + --manifest samples/wer/librispeech-test-clean.manifest.jsonl \ + --out reports/wer/orukeet-F32.librispeech-test-clean.b1.jsonl +uv run scripts/wer/score.py reports/wer/orukeet-F32.librispeech-test-clean.b1.jsonl +``` + +Oruk also ships a `transcribe-cpp/orukeet-Q8_0.gguf` in their own repo. +It was made with this same converter and works too; ours is just rebuilt +from the F32 so all the sizes come from one place. + +## Limits + +- No streaming. +- No translation. +- Timestamps are per token, not per word. +- Licence is CC-BY-SA-4.0. The rest of the parakeet family is CC-BY-4.0. + If you make a derivative of the weights, it has to carry the same licence. +- Long files go through in one pass. Nothing cuts them up for you. diff --git a/docs/models/parakeet.md b/docs/models/parakeet.md index 1e43bfd55..9a1b36700 100644 --- a/docs/models/parakeet.md +++ b/docs/models/parakeet.md @@ -6,8 +6,9 @@ of three decoder heads — TDT (transducer with a duration prediction head), classical RNN-T, or CTC — and a TDT+CTC hybrid that ships both heads in one checkpoint. All variants take 16 kHz mono PCM through a mel frontend; English-only across the family, except -`parakeet-tdt-0.6b-v3`, Moondream's post-trained `parakeet-ultra` and the -German fine-tune `parakeet-primeline`, which cover 25 European languages. +`parakeet-tdt-0.6b-v3`, Moondream's post-trained `parakeet-ultra`, Oruk's +retune `orukeet` and the German fine-tune `parakeet-primeline`, which +cover 25 European languages. For the architecture deep-dive, validation contract, and porting notes, see the family doc at @@ -24,6 +25,9 @@ Most users want one of three: and on every FLEURS language. A drop-in replacement for `parakeet-tdt-0.6b-v3`, which remains available. See [parakeet-ultra.md](parakeet-ultra.md) for how it segments long audio. +- **Multilingual, alternative → `orukeet`.** Oruk AI's retune of v3. Same + size and speed, a bit more accurate than v3, less accurate than + `parakeet-ultra`. CC-BY-SA-4.0 weights. See [orukeet.md](orukeet.md). - **German → `parakeet-primeline`.** primeLine's German fine-tune of v3. Same size and speed; tuned for German while keeping the other 24 v3 languages usable. @@ -54,12 +58,13 @@ WER is on LibriSpeech test-clean for the **Q8_0** preset, measured by transcribe.cpp's WER pipeline. See each per-variant doc for the full quant matrix and the comparison to NVIDIA's self-reported numbers. - + | Variant | Params | Languages | Q8_0 size | Benchmark | Q8_0 | Capabilities | Doc | | --- | ---: | --- | ---: | --- | ---: | --- | --- | | `parakeet-tdt-0.6b-v2` | 618M | en | 730 MB | LibriSpeech test-clean (WER) | 1.69% | token timestamps | [parakeet-tdt-0.6b-v2.md](parakeet-tdt-0.6b-v2.md) | | `parakeet-ultra` | 627M | 25 languages + auto-detect | 740 MB | LibriSpeech test-clean (WER) | 1.80% | token timestamps | [parakeet-ultra.md](parakeet-ultra.md) | | `parakeet-tdt-0.6b-v3` | 627M | 25 languages + auto-detect | 740 MB | LibriSpeech test-clean (WER) | 1.94% | token timestamps | [parakeet-tdt-0.6b-v3.md](parakeet-tdt-0.6b-v3.md) | +| `orukeet` | 627M | 25 languages + auto-detect | 740 MB | LibriSpeech test-clean (WER) | 1.86% | token timestamps | [orukeet.md](orukeet.md) | | `parakeet-primeline` | 627M | 25 languages + auto-detect | 740 MB | FLEURS de (WER) | 5.98% | token timestamps | [parakeet-primeline.md](parakeet-primeline.md) | | `parakeet-tdt-1.1b` | 1.1B | en | 1.27 GB | LibriSpeech test-clean (WER) | 1.38% | token timestamps | [parakeet-tdt-1.1b.md](parakeet-tdt-1.1b.md) | | `parakeet-tdt_ctc-110m` | 114M | en | 135 MB | LibriSpeech test-clean (WER) | 2.43% | token timestamps | [parakeet-tdt_ctc-110m.md](parakeet-tdt_ctc-110m.md) | diff --git a/docs/porting/families/parakeet.md b/docs/porting/families/parakeet.md index 955668829..255a7ada4 100644 --- a/docs/porting/families/parakeet.md +++ b/docs/porting/families/parakeet.md @@ -197,6 +197,28 @@ uv run --project scripts/envs/parakeet-kestrel scripts/wer/run_reference_parakee | Speaker diarization | n/a | n/a | n/a | OUT OF SCOPE — not a diarizer | SKIP — not exposed by runtime | | Translation | n/a | n/a | n/a | OUT OF SCOPE — not advertised | SKIP — not exposed by runtime | +### orukeet (oruk/orukeet) + +Weights-only fine-tune of `parakeet-tdt-0.6b-v3` (Oruk AI, r3 checkpoint). +Half of the encoder's 9-tap depthwise conv taps are frozen fitted Gabor +values, stored as ordinary F32 conv weights; architecture, tokenizer and +frontend are byte-for-byte v3, so no runtime work. Reference is NeMo on +the `.nemo` archive (the publisher's benchmark pipeline), same env as v3. +License is CC-BY-SA-4.0, not CC-BY-4.0. Intake: +`reports/porting/parakeet/orukeet/intake.json`. + +| Capability | Mode | Command / test | Expected observable | Target | Status | +|---|---|---|---|---|---| +| Transcribe | explicit en | `build/bin/transcribe-cli -m models/orukeet/orukeet-F32.gguf --language en samples/jfk.wav` | English transcript with PnC; LibriSpeech test-clean WER within Stage 7 band of the NeMo oracle | MUST PASS | PASS — jfk byte-equal to NeMo (validate.py compare 18/18 within the finalized parakeet tolerances, transcript exact); LibriSpeech test-clean gate in Stage 7 | +| Transcribe | auto | `build/bin/transcribe-cli -m models/orukeet/orukeet-F32.gguf samples/jfk.wav` | English transcript, no hint | MUST PASS | PASS — no-hint jfk byte-equal to NeMo | +| Transcribe | explicit non-English (de/ru/uk clips) | `… --language de samples/german.wav` | transcript in source language with correct script | MUST PASS | PASS — german.wav identical with and without --language de; ru-short / uk-short correct Cyrillic transcripts | +| Language detection | auto, non-English | `build/bin/transcribe-cli -m models/orukeet/orukeet-F32.gguf samples/ru-short.wav` (no `--language`) | Russian transcript without a hint; same dispatcher path as v3 | MUST PASS | PASS — ru-short / uk-short / german transcribed in-language with no hint (same dispatcher path as v3) | +| Offline batch | batch | `uv run scripts/batch_parity.py --model --list --batch-sizes 2,4,8 --backend cpu` | per-item transcripts equal to serial | MUST PASS | PASS — text byte-equal serial vs 2/4/8 on 7 clips incl. dots (35 s), CPU (golden tests/golden/batch/orukeet.cpu.json, list tests/golden/batch/orukeet.list). product-names.wav (56 s) flips one near-tie token ('QuidQuil' vs 'Quid Quill') at batch 4/8 in a mixed-length batch; parakeet-tdt-0.6b-v3 Q8_0 shows the identical flip on the same clip, so it is a pre-existing runtime property of the family on long padded clips, not this checkpoint, and the clip is excluded from the golden | +| Timestamps (token/word/segment) | output | `… --timestamps word samples/jfk.wav` | monotonic word times, same TDT duration math as v3 | MUST PASS | PASS — 22 monotonic word spans on jfk, same TDT duration math as v3 | +| Streaming | streaming | n/a (offline full-context model, `streaming: false`) | n/a | OUT OF SCOPE — no streaming training | SKIP — not exposed by runtime (offline-only model) | +| Speaker diarization | n/a | n/a | n/a | OUT OF SCOPE — not a diarizer | SKIP — not exposed by runtime | +| Translation | n/a | n/a | n/a | OUT OF SCOPE — not advertised | SKIP — not exposed by runtime | + ## Open decisions before Stage 3 (convert) These decisions block converter design for the new variants and should diff --git a/reports/porting/parakeet/forward-map.md b/reports/porting/parakeet/forward-map.md index 511ed6653..ab95c2909 100644 --- a/reports/porting/parakeet/forward-map.md +++ b/reports/porting/parakeet/forward-map.md @@ -133,3 +133,4 @@ cache to mis-index. - **TDT global symbol budget** (`stt.parakeet.tdt.symbol_budget="global"`; ref `ParakeetTdt.generate_encoded`): one budget of `max_symbols * T_valid` steps, blank+duration-0 advances one frame, no per-frame cap. - **VAD head + long-form segmenter** (`vad.{proj,ctx,out}`, `stt.parakeet.vad.*`, `stt.parakeet.segmenter.*`; ref `vad.py`, `segment.py`, `asr/audio.py` `AudioChunks.chunks`): `build_vad_graph` (subsampler -> proj 1x1 + SiLU -> ctx k=5 + SiLU -> out 1x1 -> sigmoid) per 120 s scan block with per-block mel normalization; host `longform::pause_segments` (float64, ties-to-even round); `run_longform` decodes each segment like a short clip and stitches (space-joined text, timestamps shifted by segment start and clamped to its end, one result segment per decoded segment). Gate: `vad.b.{mel,pre_encode,proj,ctx,prob}`, `vad.prob`, `longform.segments` (exact), `seg..enc.{mel.in,final}`, stitched transcript exact, word timestamps. - **Correctness regime**: `TRANSCRIBE_NO_FLASH=1` via manifest `cpp_env` (flash casts the rel-pos score bias to F16). +- **`orukeet`** (Oruk AI, reference NeMo): weights-only fine-tune of `parakeet-tdt-0.6b-v3`. Identical geometry, KVs, tokenizer and frontend (24 layers, d_model=1024, n_mels=128, `use_bias=false`, TDT head, pred_n_layers=2, SPM 8192+blank byte-identical to v3). 12,288 of the 24,576 encoder depthwise-conv taps are frozen fitted Gabor values stored as ordinary F32 conv weights, so the forward map is v3's with no deltas; the only converter change is the `orukeet` VARIANT_PROFILES entry (`stt.variant=tdt-0.6b-orukeet`, cc-by-sa-4.0). Validated with the finalized family tolerances unchanged (18/18 on jfk, transcript exact). diff --git a/reports/porting/parakeet/orukeet/intake.json b/reports/porting/parakeet/orukeet/intake.json new file mode 100644 index 000000000..17b3c98b1 --- /dev/null +++ b/reports/porting/parakeet/orukeet/intake.json @@ -0,0 +1,193 @@ +{ + "schema_version": "transcribe-intake-v1", + "family": "parakeet", + "hf_repo": "oruk/orukeet", + "hf_revision": "b59c13a733fce5cf193230d0fa317ee7f145a108", + "sources": { + "config": { + "kind": "hf_file", + "path": "config.json", + "status": "found" + }, + "preprocessor": { + "kind": "nemo_archive", + "path": "orukeet-v0.1.0.nemo:model_config.yaml#preprocessor", + "status": "found", + "detail": "AudioToMelSpectrogramPreprocessor: 16 kHz, 128 mels, n_fft 512, win 0.025 s, hop 0.01 s, hann, per_feature, dither 1e-5. Section is identical to nvidia/parakeet-tdt-0.6b-v3." + }, + "tokenizer_config": { + "kind": "hf_file", + "path": "tokenizer_config.json", + "status": "found" + }, + "tokenizer_json": { + "kind": "hf_file", + "path": "tokenizer.json", + "status": "found" + }, + "generation_config": { + "kind": "hf_file", + "path": "generation_config.json", + "status": "found" + }, + "safetensors_metadata": { + "kind": "hf_api", + "path": "HfApi.get_safetensors_metadata", + "status": "found", + "detail": "header-only floating dtype distribution; no tensor payloads downloaded" + } + }, + "variants": [ + { + "name": "orukeet", + "memory_gb": 2.5, + "files": [ + "orukeet-v0.1.0.nemo", + "model.safetensors", + "config.json", + "tokenizer.json" + ] + } + ], + "config": { + "architecture_candidates": [ + "encoder-transducer" + ], + "key_fields": { + "architectures": [ + "ParakeetForTDT" + ], + "hidden_act": "relu", + "model_type": "parakeet_tdt", + "vocab_size": 8193 + } + }, + "dtype": { + "expected": "float32", + "source": "config", + "evidence": "config.dtype=float32; safetensors header F32=699 / I64=24, identical distribution to v3. .nemo model_weights.ckpt is the F32 source; converted GGUF is orukeet-F32.gguf.", + "details": { + "config_declared": "float32", + "header_distribution": { + "I64": 24, + "F32": 699 + } + } + }, + "frontend": { + "sample_rate": 16000, + "n_mels": 128, + "hop_length": 160, + "fft_size": 512, + "window": "hann_periodic", + "normalization": "per_feature", + "preemphasis": 0.97, + "dither": 1e-05, + "center": true, + "padding_mode": "reflect", + "mel_filterbank_norm": "slaney" + }, + "tokenizer": { + "type": "sentencepiece", + "vocab_size": 8192, + "special_tokens": { + "pad": 2, + "blank": 8192, + "decoder_start": 8192 + }, + "has_language_tokens": false, + "vocab_sha256": "7c21c04ff6ff2f806ca43a8420479091479c805d2b7952825ae2acc4fec9424d" + }, + "capabilities": { + "languages": [ + "bg", + "hr", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "hu", + "it", + "lv", + "lt", + "mt", + "pl", + "pt", + "ro", + "ru", + "sk", + "sl", + "es", + "sv", + "uk" + ], + "language_detection": true, + "translation": false, + "timestamps": [ + "token", + "word", + "segment" + ], + "streaming": false, + "speaker_diarization": false + }, + "upstream_benchmarks": [ + { + "dataset": "LibriSpeech test-clean", + "language": "en", + "metric": "wer", + "score": 1.46, + "score_unit": "percent", + "source": "https://huggingface.co/oruk/orukeet", + "notes": "NeMo greedy-batch TDT, FP32 weights + BF16 CUDA autocast, publisher's pinned normalizer. Parakeet v3 = 1.53 on the same pipeline. Acceptance dataset." + }, + { + "dataset": "LibriSpeech test-other", + "language": "en", + "metric": "wer", + "score": 2.86, + "score_unit": "percent", + "source": "https://huggingface.co/oruk/orukeet", + "notes": "v3 = 3.14. CAUTION: the r3 final adaptation trained on 2,939 LibriSpeech test-other recordings and used test-other for checkpoint selection, so this split is not held out." + }, + { + "dataset": "FLEURS test, English", + "language": "en", + "metric": "wer", + "score": 3.82, + "score_unit": "percent", + "source": "https://huggingface.co/oruk/orukeet", + "notes": "v3 = 4.28" + }, + { + "dataset": "FLEURS test, 25 languages (pooled)", + "language": null, + "metric": "wer", + "score": 9.85, + "score_unit": "percent", + "source": "https://huggingface.co/oruk/orukeet", + "notes": "Pooled (errors / reference words), not averaged; v3 = 11.01 on the same pipeline. Per-language table in docs/current-checkpoint-benchmarks.md on the repo." + } + ], + "reference_framework": "nemo", + "reference_rationale": "Orukeet is a weights-only fine-tune of nvidia/parakeet-tdt-0.6b-v3 (model_config.yaml encoder/decoder/joint/preprocessor/decoding sections and the SentencePiece tokenizer are byte-for-byte those of v3). The publisher's benchmark numbers are produced with NeMo (nemo_toolkit[asr] 3.0.0, ASRModel.restore_from on orukeet-v0.1.0.nemo, greedy-batch TDT), so NeMo on the .nemo archive is the oracle, same as the v3 port. The repo also ships a transformers ParakeetForTDT export, a sherpa-onnx INT8 export, Core ML bundles and a 'native' NeMo-Speech.cpp GGUF; those are downstream exports with their own validation, not the reference. The publisher also ships a transcribe.cpp Q8_0 GGUF built with our own convert-parakeet.py at 585b98f7 plus a 20-line profile patch; that file is a cross-check, not an input.", + "architecture_pattern": "encoder-transducer", + "known_risks": [ + "Weights-only fine-tune of v3: architecture, hyperparameters, tokenizer and frontend are identical, so src/arch/parakeet needs no changes. The only converter change is a VARIANT_PROFILES entry (slug 'orukeet' from --repo-id oruk/orukeet). Any Stage 4 tensor drift beyond the v3 tolerances is therefore a converter or reference-dump problem, not a runtime one.", + "Tokenizer: SentencePiece BPE 8192 + blank. tokenizer.model / tokenizer.vocab / vocab.txt inside the .nemo are byte-identical (SHA-256) to nvidia/parakeet-tdt-0.6b-v3. No tokenizer work.", + "12,288 of the 24,576 encoder depthwise-conv taps (9-tap, 1024 ch x 24 layers) are frozen fitted Gabor values rather than trained weights. They are stored as ordinary F32 conv weights, so no runtime support is needed, but quantization of conv_dw tensors could move these taps more than trained ones would; the Q4_K_M/Q5_K_M quant WER read in Stage 5/7 is the check.", + "Checkpoint selection and the final 168-step adaptation used LibriSpeech test-other, so test-other is contaminated. test-clean (the acceptance dataset) was not used for training per the card, but treat the publisher's test-other number as non-held-out and do not use test-other as a gate.", + "Publisher's own 120-clip FLEURS check with our runtime (transcribe-cpp/validation.json) shows Q8_0 worse than stock v3 on de (4.07 vs 3.56), fr (4.50 vs 4.07) and uk (11.60 vs 9.40) and better on en/es/ru. The card's '61 of 74 splits' claim is on their NeMo pipeline; our full FLEURS Q8_0 sweep decides what the model card says.", + "License is CC-BY-SA-4.0 (ShareAlike), unlike the CC-BY-4.0 of every other parakeet variant we ship. The HF card YAML, catalog record and GGUF general.license must carry cc-by-sa-4.0 and retain the NVIDIA attribution from NOTICE.md.", + "Publisher reports four sub-block tensor probes exceeding strict tolerances at sequence boundaries (attributed to padding-path differences) while standard probes pass v3 tolerances. Unverified; Stage 4 validate.py against a NeMo dump settles it.", + "The .nemo is 2.5 GB; use the converter's direct-archive path (prefer_direct_load) to avoid NeMo's extract-to-tmp doubling disk use. Spillover dir is /Volumes/port/models/transcribe.cpp/orukeet.", + "Multilingual SentencePiece vocab (8192 + blank). Some scripts (Cyrillic, Greek) have higher byte-fallback rates; tokenizer round-trip tests should cover non-Latin samples, not just English.", + "Long-form audio: v3 model card recommends chunking >35 s clips; the C++ port handles long audio via internal chunking. Boundary handling differs from NeMo's chunked decoder; transcripts can drift at chunk edges if overlap is wrong." + ], + "intake_gaps": [] +} diff --git a/scripts/catalog/render.py b/scripts/catalog/render.py index 3a38d4f47..76e1bf0fb 100755 --- a/scripts/catalog/render.py +++ b/scripts/catalog/render.py @@ -386,13 +386,15 @@ def block_family_index(records: dict[str, dict], attrs: dict[str, str]) -> list[ title = record["display_name"] link = f"[{record['published_repo']}](https://huggingface.co/{record['published_repo']})" group = groups.setdefault(key, {"title": title, "link": link, "variants": [], "caps": set()}) - group["variants"].append(variant) + # Variants named after the family sort first; third-party names + # (orukeet under parakeet) go after them instead of leading the row. + group["variants"].append((0 if variant.startswith(record.get("family", "")) else 1, variant)) group["caps"].update(c for c in common.capabilities_summary(record).split(", ") if c != "-") if not groups: raise RenderError("no models matched") body = [] for group in sorted(groups.values(), key=lambda g: g["title"].lower()): - body.append([group["title"], ", ".join(f"`{v}`" for v in sorted(group["variants"])), + body.append([group["title"], ", ".join(f"`{v}`" for _, v in sorted(group["variants"])), ", ".join(sorted(group["caps"])) or "-", group["link"]]) return common.render_table(["Family", "Variants", "Available capabilities", "Docs"], ["l", "l", "l", "l"], body) diff --git a/scripts/convert-parakeet.py b/scripts/convert-parakeet.py index ff03f4ffa..c3e249317 100644 --- a/scripts/convert-parakeet.py +++ b/scripts/convert-parakeet.py @@ -130,6 +130,22 @@ ] VARIANT_PROFILES: dict[str, dict] = { + "orukeet": { + "variant": "tdt-0.6b-orukeet", + "display_name": "Orukeet", + "version": "v0.1.0", + "size_label": "0.6B", + "head_kind": "tdt", + "expected_vocab_size": 8192, + "languages": V3_LANGUAGES, + "lang_detect": True, + "author": "Oruk", + "organization": "oruk", + "prefer_direct_load": True, + "license": "cc-by-sa-4.0", + "license_name": "Creative Commons Attribution-ShareAlike 4.0", + "license_link": "https://creativecommons.org/licenses/by-sa/4.0/", + }, # v2: 0.6B English-only TDT. "parakeet-tdt-0.6b-v2": { "variant": "tdt-0.6b-v2", @@ -555,8 +571,6 @@ def load_nemo_model(model_spec: str, prefer_direct: bool = False): transiently doubles disk usage; the direct path streams entries out of the archive at near-zero transient cost. """ - from nemo.collections.asr.models import ASRModel - local = Path(model_spec).expanduser() if prefer_direct: @@ -567,6 +581,8 @@ def load_nemo_model(model_spec: str, prefer_direct: bool = False): return _DirectNemoArchive(cfg, sd, sp_proto) # else fall through to NeMo path + from nemo.collections.asr.models import ASRModel + nemo_path: Path | None = None if local.exists(): print(f"Loading Parakeet from local path: {local}") diff --git a/scripts/hf_cards/orukeet.yaml b/scripts/hf_cards/orukeet.yaml new file mode 100644 index 000000000..8c6b6adb3 --- /dev/null +++ b/scripts/hf_cards/orukeet.yaml @@ -0,0 +1,33 @@ +# Spec for the HF README of handy-computer/orukeet-gguf. +# Prose only; numbers and metadata come from catalog/.json. See README.md. + +pin_date: 2026-10-06 + +validation: + reference: NeMo + commit: 65856fac + date: 2026-10-06 + +pipeline_tag: automatic-speech-recognition +tags: + - gguf + - transcribe.cpp + - asr + - speech-to-text + - parakeet + - conformer + - tdt + - multilingual + - gabor + +summary: | + A tuned version of parakeet-tdt-0.6b-v3 from Oruk AI. Turns speech into + text in 25 European languages, works offline, and runs as fast as v3. Give + it a 16 kHz mono WAV and it gives you back text with punctuation and + capitals. Swap it in anywhere you use v3. Does not stream and does not + translate. + +wer: + notes: | + A bit more accurate than v3 on English and on 20 of the 25 languages. + parakeet-ultra is more accurate than both on almost everything. diff --git a/scripts/wer/run_reference_parakeet_nemo.py b/scripts/wer/run_reference_parakeet_nemo.py index 24f2b643d..c6cf9bc07 100644 --- a/scripts/wer/run_reference_parakeet_nemo.py +++ b/scripts/wer/run_reference_parakeet_nemo.py @@ -122,6 +122,16 @@ def main() -> int: # fallbacks when the abstract dispatcher fails. from nemo.collections.asr.models import ASRModel local = Path(args.model).expanduser() + # `org/repo:path/to/file.nemo` pins one .nemo inside an HF repo whose + # layout NeMo's from_pretrained cannot resolve (it wants `.nemo` + # at the root, else it snapshots the whole repo). Needed for publisher + # repos that bundle ONNX / Core ML / GGUF exports next to the .nemo + # (oruk/orukeet). + if not local.exists() and args.model.count(":") == 1 and args.model.endswith(".nemo"): + from huggingface_hub import hf_hub_download + repo_id, filename = args.model.split(":", 1) + local = Path(hf_hub_download(repo_id=repo_id, filename=filename)) + print(f"resolved {args.model} -> {local}") is_local = local.exists() loader = "restore_from" if is_local else "from_pretrained" try: diff --git a/tests/golden/batch/orukeet.cpu.json b/tests/golden/batch/orukeet.cpu.json new file mode 100644 index 000000000..789344377 --- /dev/null +++ b/tests/golden/batch/orukeet.cpu.json @@ -0,0 +1,14 @@ +{ + "model": "models/orukeet/orukeet-F32.gguf", + "backend": "cpu", + "language": null, + "texts": { + "samples/jfk.wav": "And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.", + "samples/german.wav": "Am Strand, der Badeanzug, die Badehose, die Sandalen, die Luftmatratze, das Handtuch, das Eis, der Ball, die Sonne, das Meer, die Wellen.", + "samples/ru-short.wav": "Он добавил, что их, однако, не следует просить брать на себя обязательства, которые выходят за рамки их стадий разработки, ответственности и возможностей.", + "samples/uk-short.wav": "Управління надзвичайними ситуаціями Північних Маріанських островів інформувало, що в Україні не було повідомлення про жодні шкоди.", + "samples/dots.wav": "Of course, it was impossible to connect the dots looking forward when I was in college, but it was very, very clear looking backwards ten years later. Again, you can't connect the dots looking forward. You can only connect them looking backwards. So you have to trust that the dots will somehow connect in your future. You have to trust in something your gut, destiny, life, karma, whatever. Because believing that the dots will connect down the road will give you the confidence to follow your heart, even when it leads you off the well worn path, and that will make all the difference.", + "samples/love-loss.wav": "My second story is about love and loss. I was lucky. I found what I love to do early in life. Waz and I started Apple in my parents' garage when I was twenty. We worked hard, and in ten years, Apple had grown from just the two of us in a garage into a two billion dollar company with over 4,000 employees. We just released our finest creation, the Macintosh, a year earlier, and I just turned 30. And then I got fired. How can you get fired from a company you started? Well, as Apple grew, we hired someone who I thought was very talented to run the company with me. And for the first year or so, things went well. But then our visions of the future began to diverge, and eventually we had a falling out. When we did, our board of directors sided with him. And so at 30, I was out and very publicly out. What had been the focus of my entire adult life was gone, and it was devastating. I really didn't know what to do for a few months. I felt that I had let the previous generation of entrepreneurs down, that I had dropped the baton as it was being passed to me. I met with David Packard and Bob Noyce and tried to apologize for screwing up so badly. I was a very public failure, and I even thought about running away from the valley. But something slowly began to dawn on me. I still loved what I did. The turn of events at Apple had not changed that one bit. I'd been rejected, but I was still in love. And so I decided to start over. I didn't see it then, but it turned out that getting fired from Apple was the best thing that could have ever happened to me. The heaviness of being successful was replaced by the lightness of being a beginner again, less sure about everything. It freed me to enter one of the most creative periods of my life. During the next five years, I started a company named Next, another company named Pixar, and fell in love with an amazing woman who would become my wife. Pixar went on to create the world's first computer animated feature film, Toy Story, and is now the most successful animation studio in the world. In a remarkable turn of events, Apple bought next, and I returned to Apple, and the technology we developed at Next is at the heart of Apple's current renaissance. And Lorene and I have a wonderful family together. I'm pretty sure none of this would have happened if I hadn't been fired from Apple. It was awful tasting medicine, but I guess the patient needed it. Sometime life sometimes life's gonna hit you in the head with a brick. Don't lose faith. I'm convinced that the only thing that kept me going was that I loved what I did. You've got to find what you love, and that is as true for work as it is for your lovers. Your work is going to fill a large part of your life, and the only way to be truly satisfied is to do what you believe is great work. And the only way to do great work is to love what you do. If you haven't found it yet, keep looking and don't settle. As with all matters of the heart, you'll know when you find it. And like any great relationship, it just gets better and better as the years roll on. So keep looking. Don't settle.", + "samples/whole-earth.wav": "When I was young, there was an amazing publication called The Whole Earth Catalogue, which was one of the Bibles of my generation. It was created by a fellow named Stuart Brand, not far from here in Menlo Park, and he brought it to life with his poetic touch. This was in the late 60s, before personal computers and desktop publishing. So it was all made with typewriters, scissors, and Polaroid cameras. It was sort of like Google in paperback form 35 years before Google came along. It was idealistic, overflowing with neat tools and great notions. Stuart and his team put out several issues of the Whole Earth catalogue. And then, when it had run its course, they put out a final issue. It was the mid-1970s, and I was your age. On the back cover of their final issue was a photograph of an early morning country road, the kind you might find yourself hitchhiking on if you were so adventurous. Beneath it were the words, stay hungry, stay foolish. It was their farewell message as they signed off. Stay hungry, stay foolish. And I have always wished that for myself. And now, as you graduate to begin anew, I wish that for you. Stay hungry, stay foolish. Thank you all very much." + } +} diff --git a/tests/golden/batch/orukeet.list b/tests/golden/batch/orukeet.list new file mode 100644 index 000000000..e425a6531 --- /dev/null +++ b/tests/golden/batch/orukeet.list @@ -0,0 +1,7 @@ +samples/jfk.wav +samples/german.wav +samples/ru-short.wav +samples/uk-short.wav +samples/dots.wav +samples/love-loss.wav +samples/whole-earth.wav diff --git a/tests/golden/parakeet/orukeet.manifest.json b/tests/golden/parakeet/orukeet.manifest.json new file mode 100644 index 000000000..59104c8b4 --- /dev/null +++ b/tests/golden/parakeet/orukeet.manifest.json @@ -0,0 +1,74 @@ +{ + "schema": "transcribe-golden-manifest-v1", + "family": "parakeet", + "variant": "orukeet", + "source_model": { + "hf_repo": "oruk/orukeet", + "hf_revision": "b59c13a733fce5cf193230d0fa317ee7f145a108" + }, + "expected_dtype": "float32", + "dtype_source": "config", + "reference": { + "kind": "nemo", + "source": "https://github.com/NVIDIA-NeMo/NeMo", + "revision": "v2.7.2", + "entrypoint": "scripts/dump_reference_parakeet_nemo.py" + }, + "frontend": { + "sample_rate": 16000, + "n_mels": 128, + "hop_length": 160, + "fft_size": 512, + "win_length": 400, + "window": "hann_periodic", + "normalization": "per_feature", + "preemphasis": 0.97, + "dither": 1e-05 + }, + "tokenizer_summary": { + "type": "bpe", + "vocab_size": 8193, + "special_tokens": {} + }, + "capabilities": { + "languages": [ + "bg", + "hr", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "hu", + "it", + "lv", + "lt", + "mt", + "pl", + "pt", + "ro", + "ru", + "sk", + "sl", + "es", + "sv", + "uk" + ], + "language_detection": true, + "translation": false, + "timestamps": [ + "word", + "token" + ], + "streaming": false, + "speaker_diarization": false + }, + "tolerance_file": "tests/tolerances/parakeet.json", + "cases": [ + "jfk" + ] +}