From 0df45632a2a9fef41d65ce61eafe18ddb6e77bb5 Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 25 Sep 2026 06:31:40 -0700 Subject: [PATCH 1/2] feat(carmenta): close the commercial gaps: forms, scanned filings, PDFs (0.11) A real caller ran Carmenta on scanned financial-disclosure filings and hit a run of production gaps (docs/plans/commercial-gaps.md). This batch closes them. Carmenta 0.11: - offline weights: manifests ship in the crate, offline()/preload(), and resolution that needs only the runtime files - charset-constrained CTC decoding, and confidence that stays unrenormalized - quarter-turn orientation (144/144 on turned pages, 48% of a read) - opt-in form-rule removal - full-width to ASCII fold on non-CJK lines - lexicon matching with look-alike-weighted edit distance - checkbox detection (100% recall and state on the corpus) - label-to-value pairing (220/221) ffai-media 0.6.7: - decode_image(&[u8]) - opt-in pdf feature: pages are composited as a viewer shows them, so redactions and overlays are painted in rather than read through - redaction_risk on text layers - every allocation from untrusted PDF values is bounded CLI 0.7: PDF input, --charset, --remove-rules, --auto-orient, --checkboxes, and `ffai models --verify`. Bench: digit error rate in OCR notes. Disclosure: corrects the public 0.041 Text^Edit figure. That number was Unlimited-OCR's; Carmenta's is 0.108 on the same subset. The top-level README now uses official-evaluator numbers throughout, replacing those from the withdrawn biased scorer. Versions: core 0.8.0, carmenta 0.11.0, cli 0.7.0, and patch bumps for the dependents (see CHANGELOG). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 143 ++ Cargo.lock | 195 +- Cargo.toml | 18 +- README.md | 70 +- ROADMAP.md | 24 +- corpora/carmenta-checkbox-v1.toml | 643 +++++ corpora/carmenta-form-v1.toml | 2163 +++++++++++++++++ crates/ffai-argus-wasm/Cargo.toml | 4 +- crates/ffai-argus/Cargo.toml | 2 +- crates/ffai-bench/Cargo.toml | 2 +- crates/ffai-bench/src/metrics.rs | 86 + crates/ffai-bench/src/runner.rs | 32 +- crates/ffai-carmenta-wasm/Cargo.toml | 4 +- crates/ffai-carmenta/Cargo.toml | 2 +- crates/ffai-carmenta/README.md | 78 +- .../ffai-carmenta/examples/checkbox_bench.rs | 167 ++ crates/ffai-carmenta/examples/fold_census.rs | 128 + crates/ffai-carmenta/examples/form_bench.rs | 226 ++ crates/ffai-carmenta/examples/orient_bench.rs | 125 + crates/ffai-carmenta/models/craft-mlt.toml | 20 + .../ffai-carmenta/models/crnn-english-g2.toml | 17 + .../ffai-carmenta/models/crnn-zh-sim-g2.toml | 40 + crates/ffai-carmenta/models/parseq-tiny.toml | 16 + .../models/ppocrv5-mobile-det.toml | 30 + .../models/ppocrv5-mobile-rec.toml | 40 + crates/ffai-carmenta/src/checkbox.rs | 336 +++ crates/ffai-carmenta/src/crnn.rs | 144 +- crates/ffai-carmenta/src/engine.rs | 476 +++- crates/ffai-carmenta/src/fold.rs | 102 + crates/ffai-carmenta/src/forms.rs | 214 ++ crates/ffai-carmenta/src/lexicon.rs | 181 ++ crates/ffai-carmenta/src/lib.rs | 7 + crates/ffai-carmenta/src/manifests.rs | 95 + crates/ffai-carmenta/src/orient.rs | 198 ++ crates/ffai-carmenta/src/rules.rs | 274 +++ crates/ffai-carmenta/src/svtr.rs | 68 +- crates/ffai-carmenta/tests/production.rs | 264 ++ crates/ffai-cli/Cargo.toml | 4 +- crates/ffai-cli/src/main.rs | 105 +- crates/ffai-core/Cargo.toml | 2 +- crates/ffai-core/src/engine.rs | 30 + crates/ffai-diana/Cargo.toml | 4 +- crates/ffai-diana/README.md | 2 +- crates/ffai-media/Cargo.toml | 12 +- crates/ffai-media/README.md | 37 +- crates/ffai-media/src/lib.rs | 105 +- crates/ffai-media/src/pdf.rs | 1190 +++++++++ crates/ffai-mercury-wasm/Cargo.toml | 4 +- crates/ffai-mercury/Cargo.toml | 2 +- crates/ffai-mercury/README.md | 2 +- crates/ffai-models/Cargo.toml | 2 +- crates/ffai-models/src/lib.rs | 207 +- crates/ffai-py/Cargo.toml | 2 +- crates/ffai-wasm/Cargo.toml | 2 +- docs/plans/benching-history-made.md | 13 + docs/plans/commercial-gaps.md | 676 ++++++ models/ppocrv5-mobile-rec.toml | 10 +- tools/carmenta_checkbox_corpus.py | 141 ++ tools/carmenta_form_corpus.py | 138 ++ 59 files changed, 9168 insertions(+), 156 deletions(-) create mode 100644 corpora/carmenta-checkbox-v1.toml create mode 100644 corpora/carmenta-form-v1.toml create mode 100644 crates/ffai-carmenta/examples/checkbox_bench.rs create mode 100644 crates/ffai-carmenta/examples/fold_census.rs create mode 100644 crates/ffai-carmenta/examples/form_bench.rs create mode 100644 crates/ffai-carmenta/examples/orient_bench.rs create mode 100644 crates/ffai-carmenta/models/craft-mlt.toml create mode 100644 crates/ffai-carmenta/models/crnn-english-g2.toml create mode 100644 crates/ffai-carmenta/models/crnn-zh-sim-g2.toml create mode 100644 crates/ffai-carmenta/models/parseq-tiny.toml create mode 100644 crates/ffai-carmenta/models/ppocrv5-mobile-det.toml create mode 100644 crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml create mode 100644 crates/ffai-carmenta/src/checkbox.rs create mode 100644 crates/ffai-carmenta/src/fold.rs create mode 100644 crates/ffai-carmenta/src/forms.rs create mode 100644 crates/ffai-carmenta/src/lexicon.rs create mode 100644 crates/ffai-carmenta/src/manifests.rs create mode 100644 crates/ffai-carmenta/src/orient.rs create mode 100644 crates/ffai-carmenta/src/rules.rs create mode 100644 crates/ffai-carmenta/tests/production.rs create mode 100644 crates/ffai-media/src/pdf.rs create mode 100644 docs/plans/commercial-gaps.md create mode 100644 tools/carmenta_checkbox_corpus.py create mode 100644 tools/carmenta_form_corpus.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 01840352..19627cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,149 @@ mechanics are in `.github/workflows/release.yml`. ## [Unreleased] +### Versions + +| crate | from | to | why | +|---|---|---|---| +| `ffai-core` | 0.7.1 | **0.8.0** | `OcrOptions` gains `charset`, `remove_rules`, `auto_orient`: new public fields break struct-literal construction | +| `ffai-carmenta` | 0.10.1 | **0.11.0** | orientation, checkboxes, forms, lexicon, offline weights; follows core 0.8 | +| `ffai-cli` | 0.6.8 | **0.7.0** | `ffai ocr` takes PDFs, `--charset`, `--remove-rules`, `--auto-orient`, `--checkboxes`; `ffai models --verify` | +| `ffai-media` | 0.6.6 | 0.6.7 | `decode_image`, opt-in `pdf` feature (additive) | +| `ffai-models` | 0.6.4 | 0.6.5 | `resolve_only_in`, `verify` (additive) | +| `ffai-bench` | 0.7.5 | 0.7.6 | digit error rate in OCR notes | +| `ffai-carmenta-wasm`, `ffai-diana`, `ffai-mercury`, `ffai-wasm`, `ffai-mercury-wasm`, `ffai-argus-wasm`, `ffai-py` | — | patch | dependency bumps only (core 0.8 / carmenta 0.11) | + +A dependent that exposes `ffai-core` types in its own API takes a patch bump +here, following release-plz's convention. Strictly, a type from core 0.8 is not +the same type as one from core 0.7, so a caller that names both will see a +mismatch until it moves to 0.8 as well. + +### Production gaps found by a real caller: batch 1 + +`docs/plans/commercial-gaps.md` records what broke when `fraud-alert` used +Carmenta on scanned financial-disclosure filings. This batch closes the gaps +that needed no new model. + +- **`ffai-core` — BREAKING (a new public field).** `OcrOptions::charset: + Option` restricts recognition to a character set, for example + `"0123456789$,.-"` for an amount field. Every in-repo construction already + used `..Default::default()`. A downstream struct literal that lists every + field will not compile until it adds the field or uses the default. +- **`ffai-carmenta` — charset-constrained decoding** for the CRNN and SVTR + recognizers: + - The CTC argmax is taken only over blank plus the allowed classes. + - The reported confidence is the model's probability for the emitted + character over the full distribution, not renormalised. A line the model + wanted to read as letters comes back less confident, not as a confident + digit. + - PARSeq, and routing via `FFAI_ROUTE`, return an error when a charset is + set rather than ignoring it. So does a character the model cannot emit, + and the error names it. + - With every character allowed, the output equals the free decode (tested on + a real page). +- **`ffai-carmenta` — no `models/` directory needed.** The six OCR manifests + are compiled in (`manifests::EMBEDDED`). A `models/` directory in the working + directory, or one passed to `with_manifest_dir` / `variant_in`, still + overrides them by model name. Before this, a crates.io user got + `i/o error: The system cannot find the path specified`. +- **`ffai-carmenta` — `CraftCrnn::offline(rec, det, weights_root)`** for batch + and offline jobs: + - It uses the compiled-in manifests and reads weights only from + `weights_root//`. It never touches the cache or the network. + - `preload()` loads eagerly, so a missing file stops the job before the first + page. The error names the exact expected path. +- **`ffai-models`:** + - `ModelManifest::resolve_in(root)` does strict offline resolution with + checksums. + - `ModelManifest::verify()` checks presence and checksums without + downloading. + - `load_dir` names the directory when it cannot read it. +- **`ffai-media` — `decode_image(&[u8])`** decodes bytes already in memory, + such as pages pulled from a PDF, with the same decoders and output contract + as `load_image`. Unrecognised bytes are an error that shows the leading + bytes. +- **`ffai-cli`:** + - `ffai ocr --charset`. + - `ffai models --verify ` checks without downloading and exits + non-zero on failure. + +### Production gaps: batches 2 and 3 + +- **`ffai-core` — two more `OcrOptions` fields, in the same breaking + release:** + - `remove_rules`: erase form rules before reading. + - `auto_orient`: detect which quarter turn is upright and read it that way. + Boxes stay in the input image's coordinates. + + Engines that cannot honour either must return an error. +- **`ffai-carmenta` — `rules`:** horizontal-rule removal. It keeps any pixel + whose vertical dark run is tall, so glyph strokes survive and filled boxes + (redactions) are never touched. + - On the new `carmenta-form-v1` corpus it takes `mobiledet-crnn` from 13-20 % + to 70 % exact on underlined values, matching its no-rule control. + - It is neutral on `mobiledet-svtr`, and worse on strikethroughs (77-87 % to + 73 %). Opt-in. +- **`ffai-carmenta` — `orient` and `CraftCrnn::detect_orientation`:** + - One detection pass decides the axis from box aspect. + - A dozen of the largest boxes are then read in both candidate turns; the + turn with higher recognition confidence wins. +- **`ffai-carmenta` — `fold`:** full-width ASCII forms that SVTR emits into + Latin lines (`"$25，０00"`, `"12／31／2025"`) are folded to ASCII. Lines + containing any CJK character are left exactly as read. `FFAI_NO_FOLD` + disables it. This changes default output only on Latin lines that carried a + full-width form. +- **`ffai-carmenta` — `FFAI_CONF_REDUCE=min`** (experimental): report a + line's weakest character probability instead of the mean. It is the + calibration experiment for callers that gate on confidence. The default is + unchanged and byte-identical. +- **`ffai-media` — `pdf` feature, `pdf::pdf_pages(bytes)`:** + - Every page comes back as a viewer shows it: the scan composited with + filled paths, later images, form `XObjects`, and `Redact` / `Square` / + `Circle` / `Polygon` / `Stamp` annotations, then turned upright. + - A page that cannot be walked returns no image, never the raw scan. + - Decoders: DCT, CCITT G4 and G3-1D (via the pure-Rust `fax` crate), and + Flate/raw 1- and 8-bit samples in gray, RGB or CMYK, including stencil + masks. + - Depends on `lopdf` (default features off) and `fax`; both MIT. +- **`ffai-cli`:** `ffai ocr -i file.pdf` reads PDFs page by page. + `--remove-rules` and `--auto-orient` are new. +- **Corpora and benches:** + - `tools/carmenta_form_corpus.py` builds `carmenta-form-v1`: 240 clips of + invented form values, ground truth by construction. + - Three new examples in `ffai-carmenta`: `form_bench`, `orient_bench` and + `fold_census`. + +### Security + +- **`ffai-media::pdf` parses untrusted files and is bounded against them:** + - images are capped at 100 MP; + - image decompression is capped at the size the declared dimensions justify; + - form-content decompression is capped at 64 MiB; + - form nesting is capped at 8; + - ICC `/N`, bits per component and CCITT `/Columns` are validated before any + size arithmetic. + + Two of these (the unbounded `/N` feeding the row-size arithmetic, and + uncapped form decompression) were found while writing the module and fixed + before it shipped. Numeric conversions are checked, or individually + justified, rather than silenced wholesale. +- **Redactions:** the module exists to stop OCR reading through them. It + composites overlays in, and withholds a page it cannot walk. Its tests + include the reported incident (a white patch image over a scan must come out + black). They also caught `lopdf`'s annotation helper silently skipping inline + annotations, which would have read through an inline `Redact`. +- No `unsafe` added. No change to what personal data is processed. + +### Security + +- **Changed:** `ModelManifest::resolve_in` and `CraftCrnn::offline` verify + every weight file against the checksum **compiled into the crate**, and never + download. For an offline deployment, what is trusted is now the release's + pinned hashes plus the caller's directory, not a cache a second process might + also write. +- No `unsafe` added or changed. No change to what personal data is processed. + No advisory resolved or waived. + ### `ffai-carmenta` 0.10.0 — the detector's short side gains a ceiling - **Changed** — `mobiledet_input` floored the short side at 736 and capped only diff --git a/Cargo.lock b/Cargo.lock index 09093302..551652b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + [[package]] name = "ahash" version = "0.8.12" @@ -252,6 +263,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -463,6 +483,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.0" @@ -517,6 +546,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + [[package]] name = "clap" version = "4.6.4" @@ -688,6 +727,12 @@ dependencies = [ "futures-io", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1072,6 +1117,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" +[[package]] +name = "ecb" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f2a8b3e564eba0877223dc343703ad0385794e882e6d13f3a4dd5c6b1f41ac" +dependencies = [ + "cipher", +] + [[package]] name = "either" version = "1.17.0" @@ -1155,6 +1209,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ed8646f3993cad0585c6669ffba6f7e5440a0bfb0a1f16600cae33ce7cf307" + [[package]] name = "fdeflate" version = "0.3.7" @@ -1166,7 +1226,7 @@ dependencies = [ [[package]] name = "ffai-argus" -version = "0.7.3" +version = "0.7.4" dependencies = [ "candle-core", "candle-nn", @@ -1184,7 +1244,7 @@ dependencies = [ [[package]] name = "ffai-argus-wasm" -version = "0.1.1" +version = "0.1.2" dependencies = [ "ffai-argus", "ffai-core", @@ -1196,7 +1256,7 @@ dependencies = [ [[package]] name = "ffai-bench" -version = "0.7.5" +version = "0.7.6" dependencies = [ "clap", "csv", @@ -1214,7 +1274,7 @@ dependencies = [ [[package]] name = "ffai-carmenta" -version = "0.10.1" +version = "0.11.0" dependencies = [ "candle-core", "candle-nn", @@ -1235,7 +1295,7 @@ dependencies = [ [[package]] name = "ffai-carmenta-wasm" -version = "0.2.1" +version = "0.2.2" dependencies = [ "ffai-carmenta", "ffai-core", @@ -1250,7 +1310,7 @@ dependencies = [ [[package]] name = "ffai-cli" -version = "0.6.8" +version = "0.7.0" dependencies = [ "anyhow", "clap", @@ -1269,7 +1329,7 @@ dependencies = [ [[package]] name = "ffai-core" -version = "0.7.1" +version = "0.8.0" dependencies = [ "candle-core", "rayon", @@ -1278,7 +1338,7 @@ dependencies = [ [[package]] name = "ffai-demo" -version = "0.7.3" +version = "0.7.4" dependencies = [ "ffai-argus", "ffai-bench", @@ -1291,7 +1351,7 @@ dependencies = [ [[package]] name = "ffai-diana" -version = "0.7.6" +version = "0.7.7" dependencies = [ "candle-core", "candle-nn", @@ -1311,11 +1371,13 @@ dependencies = [ [[package]] name = "ffai-media" -version = "0.6.6" +version = "0.6.7" dependencies = [ + "fax", "ffai-core", "getrandom 0.3.4", "hound", + "lopdf", "png", "rff-codec", "rff-core", @@ -1331,7 +1393,7 @@ dependencies = [ [[package]] name = "ffai-mercury" -version = "1.0.3" +version = "1.0.4" dependencies = [ "candle-nn", "candle-transformers", @@ -1350,7 +1412,7 @@ dependencies = [ [[package]] name = "ffai-mercury-wasm" -version = "0.1.1" +version = "0.1.2" dependencies = [ "ffai-core", "ffai-mercury", @@ -1361,7 +1423,7 @@ dependencies = [ [[package]] name = "ffai-models" -version = "0.6.4" +version = "0.6.5" dependencies = [ "dirs", "ffai-core", @@ -1373,7 +1435,7 @@ dependencies = [ [[package]] name = "ffai-py" -version = "0.1.0" +version = "0.1.1" dependencies = [ "ffai-core", "ffai-diana", @@ -1384,7 +1446,7 @@ dependencies = [ [[package]] name = "ffai-wasm" -version = "0.1.3" +version = "0.1.4" dependencies = [ "ffai-core", "ffai-diana", @@ -2348,6 +2410,16 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding", + "hybrid-array", +] + [[package]] name = "intel-mkl-src" version = "0.8.1" @@ -2571,6 +2643,32 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lopdf" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2ec995d822e05cabc3f06d196ee43650af3fe4fe38012cacb35e0c3d113b68" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "log", + "md-5", + "nom 8.0.0", + "rand 0.10.2", + "rangemap", + "sha2 0.11.0", + "stringprep", + "thiserror 2.0.19", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2630,6 +2728,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if 1.0.4", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2766,6 +2874,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -3453,6 +3570,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -4248,7 +4371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" dependencies = [ "base64 0.13.1", - "nom", + "nom 7.1.3", "serde", "unicode-segmentation", ] @@ -4281,6 +4404,17 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4433,7 +4567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4975,12 +5109,27 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-normalization-alignments" version = "0.1.12" @@ -4990,6 +5139,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -5255,6 +5410,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index a4b0515c..96d22280 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = ["crates/*"] exclude = ["demo-ui"] [workspace.package] -version = "0.7.3" +version = "0.7.4" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" @@ -16,14 +16,14 @@ authors = ["Remade With Rust"] [workspace.dependencies] # ---- internal crates ---- -ffai-core = { path = "crates/ffai-core", version = "0.7.1" } -ffai-media = { path = "crates/ffai-media", version = "0.6.6" } -ffai-models = { path = "crates/ffai-models", version = "0.6.4" } -ffai-mercury = { path = "crates/ffai-mercury", version = "1.0.3" } -ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.10.1" } -ffai-diana = { path = "crates/ffai-diana", version = "0.7.6" } -ffai-argus = { path = "crates/ffai-argus", version = "0.7.3" } -ffai-bench = { path = "crates/ffai-bench", version = "0.7.5" } +ffai-core = { path = "crates/ffai-core", version = "0.8.0" } +ffai-media = { path = "crates/ffai-media", version = "0.6.7" } +ffai-models = { path = "crates/ffai-models", version = "0.6.5" } +ffai-mercury = { path = "crates/ffai-mercury", version = "1.0.4" } +ffai-carmenta = { path = "crates/ffai-carmenta", version = "0.11.0" } +ffai-diana = { path = "crates/ffai-diana", version = "0.7.7" } +ffai-argus = { path = "crates/ffai-argus", version = "0.7.4" } +ffai-bench = { path = "crates/ffai-bench", version = "0.7.6" } # ---- tensor spine ---- # Candle is FFai's PyTorch: every engine shares its Tensor/Device types so diff --git a/README.md b/README.md index a4722351..b0795860 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ ffai models # list model manifests, licenses, cache status | Component | Crate | Task | Namesake | Compare | |---|---|---|---|---| | **Mercury** | `ffai-mercury` | ASR + TTS | Roman god of language and messages | **ASR live**: full WhisperX layer (VAD · word timestamps · diarization) in pure Rust, **all four gates PASS vs whisper.cpp on both holdouts** — and at matched model size ahead on WER, CER *and* speed. Sizes tiny→medium, beam search, 0.84–0.92× its memory. **TTS live**: piper's own voices on candle, oracle-exact vs piper's runtime, **quality parity** through a frozen judge (5.49 % vs 5.27 % WER), **1.58× faster wall-clock at 5 % less CPU**, 10× faster load, and byte-identical output per seed — which piper structurally cannot offer ([Status](#status)) | -| **Carmenta** | `ffai-carmenta` | OCR | Roman goddess who adapted the Greek alphabet into Latin letters | **OCR live**, with a LIVE streaming mode no mainstream tool ships: change-gated, **zero churn across 156 unchanged frames** where stateless Tesseract churns 24 times. On the full **OmniDocBench** holdout: **20.3 % CER, 236/236 correctness**, reading order computed by projection rather than learned — and **89 % of the remaining gap to PP-StructureV3 is sequence, not characters** (order-free CER within 1.40 pp). Against Baidu Unlimited-OCR: 25.9 % vs 15.5 % on a matched 43-page subset, at **17x the throughput on CPU** from 4.7 MB of detector weights against 6.4 GB. Photo accuracy still trails PaddleOCR, causes diagnosed ([Status](#status)) | +| **Carmenta** | `ffai-carmenta` | OCR | Roman goddess who adapted the Greek alphabet into Latin letters | **OCR live**, with a LIVE streaming mode no mainstream tool ships: change-gated, **zero churn across 156 unchanged frames** where stateless Tesseract churns 24 times. On all **1 651 OmniDocBench pages, scored by its own evaluator**: text edit distance **0.116**, reading order **0.204**, with reading order computed rather than learned. Clearly behind Baidu's Unlimited-OCR (0.041 vs 0.108 text on a 236-page subset), at roughly **9× its throughput on CPU**. Earlier figures from our own scorer are withdrawn: it was 2.8× biased in our favour. **New in 0.11, for forms and scanned PDFs:** quarter-turn orientation (144/144), checkboxes, label → value pairing, and PDF pages read as a viewer shows them, redactions included ([Status](#status)). Photo accuracy still trails PaddleOCR | | **Diana** | `ffai-diana` | Object detection | Roman goddess of the hunt — fast, precise detection | **Detection, tracking, and the browser.** YOLO26 on candle from official Ultralytics `.pt` — all five tiers from one tier-agnostic graph, no ONNX. **Every detection identical to PyTorch at n, m, l and x**, at **1.6–5.6× less memory** and up to **10× faster load**. **3.71× less CPU per frame than Ultralytics**, and **1.92× faster under load** — Diana is a ~2.4-core workload against their ~7.9. **ByteTrack** with no appearance model and no second weight file: IDF1 35.93 / MOTA 24.91 against 36.92 / 27.38 on identical weights and frames, with **218 ID switches to their 800**. **Runs in a browser** — [`ffai-wasm`](https://crates.io/crates/ffai-wasm) compiles the whole graph to WebAssembly with no ONNX runtime, agreeing with native to display precision. One failing gate, stated: **2.89× slower than ONNX Runtime**, which Diana beats on accuracy (0.7014 vs 0.6865). ([Status](#status)) | | **Argus** | `ffai-argus` | VLM captioning / video understanding | Argus Panoptes, the all-seeing watchman | **VLM live.** `SmolVLM-256M-Instruct` on candle — `SigLIP` tower, pixel-shuffle connector, Llama decoder, ported tensor by tensor, plus **our own `SmolLM2` text tower** and our own `SigLIP` encoder. The gate is the strong form: from a **raw image file** through our resize, tiling, tower, prompt assembly and decode loop, the caption is **byte-identical** to the reference — **32/32 tokens**, six stages each gated in isolation. Scored **525/1000 on OCRBench** through VLMEvalKit against the checkpoint's **published 526**. **1.20x off PyTorch end to end** (10 918 vs 9 106 ms, same image, idle box, both arms repeated), with the deficit concentrated in the vision tower — down from **2.4x** across three optimization rounds: prefill **3.07x**, generate **2.05x**, the connector's `broadcast_matmul` **14.1x**, the patch embedding as a matmul **2.6x**. A fourth round then took the **vision tower itself 1.199x (16.6 %)**: `candle_nn::Linear`'s bias is a *separate single-threaded pass* that evicted every GEMM's working set, and folding the four biases into ops that already touch every element restored the projections from **288-347 to 500-563 GF/s**; softmax's normalising divide moved past `attn.v` (786 K divides instead of 12.6 M), and the connector's two-permute pixel shuffle became one bit-identical pass. Eight counter-attempts were refuted, including a hand-written GEMM at **0.11x** — candle's GEMM rewards large batched calls, and every trade of call size for locality lost. A fifth round then found a **quadratic in the KV cache**: `Tensor::cat` recopied the whole history every step (**52.7 MB per token**, growing with position), which a preallocated in-place append took to **46 KB — decode 53.9 -> 42.6 ms/token, 1.27x**, worth ~900 ms at the reference 64-token budget and more for video. Decode now sits at **78-92 % of its memory floor** (591 MB read per token), so what remains there is a dtype decision, not an optimisation. Footprint **0.71x** PyTorch's — **contradicted by the two most recent gate runs, which measure 1.29x and 1.54x the reference's memory; see [Status](#status)**. Video captions to `.srt`/`.vtt`/`.json` at constant memory per window; **no video quality claim is made** — the checkpoint is an image model with no published video row ([Status](#status)) | @@ -472,14 +472,15 @@ above traces to a line in [`bench/ledger.jsonl`](bench/ledger.jsonl). ### Carmenta OCR: live, measured against PaddleOCR — the honest split -Four engines run today, from two detector lineages crossed with two +Five engines run today, from two detector lineages crossed with three recognizers, all oracle-matched against their references. The EasyOCR lineage on candle (detection maps to <5e-3, recognition to the exact -per-step argmax) gives `craft-crnn` (line-level CTC, the default) and +per-step argmax) gives `craft-crnn` (line-level CTC, the LIVE default) and `craft-parseq` (word-level AR with the refinement pass). Swapping in a PP-OCRv5 mobile detector — DBNet on PP-LCNetV3, 4.7 MB, reproducing paddle's own exported program to **zero binarised disagreement** across a -pinned page — gives `mobiledet-crnn` and `mobiledet-parseq`. +pinned page — gives `mobiledet-crnn` and `mobiledet-parseq`, and with +PP-OCRv5's own SVTR recognizer `mobiledet-svtr`, the **document default**. The two detector lineages trade against each other rather than ranking, and the measurement says so plainly. On real-photo receipts mobile-det is **3× @@ -504,21 +505,54 @@ corpus class, not by leaderboard. pipeline on pages** (0.73 % vs 3.65 %) — line-level composition dodges its word-segmentation errors. -**Documents, against the best in class.** On 43 pages of -[OmniDocBench](https://github.com/opendatalab/OmniDocBench) (Apache-2.0 — the -benchmark Baidu's Unlimited-OCR states its record on), identical pixels, every -number a ledger line: +**Documents, scored by the benchmark's own evaluator.** Every document number +on this page comes from +[OmniDocBench](https://github.com/opendatalab/OmniDocBench)'s official harness. +An earlier edition of this section quoted CER from our own +concatenate-then-edit-distance scorer (23.76 % against Unlimited-OCR's 15.51 % +on 43 pages). Run beside the official evaluator, that scorer inflated +Carmenta's error 1.58× and Unlimited-OCR's 4.37×, a 2.8× bias in our favour +(`docs/Carmenta-mission-plan.md` §8.173). Every competitive claim made with it +is withdrawn. + +| all 1 651 pages of v1.6, official metric (lower is better) | `mobiledet-svtr`, CPU | published range | +|---|---:|---| +| Text edit distance | **0.116** | 0.033 (PaddleOCR-VL) – 0.157 (Marker) | +| Reading-order edit distance | **0.204** | 0.116 – 0.243 | -| 43 real document pages | CER | pages/s | memory | -|---|---:|---:|---:| -| Unlimited-OCR (Baidu, 3B MoE, **GPU**) | **15.51 %** | 0.01 | 8745 MiB peak | -| PP-StructureV3 | 19.14 % | 0.02 | 1481 MiB steady | -| **`mobiledet-crnn` (ours, CPU)** | **23.76 %** | **0.15** | **425 MiB steady** | - -**10.4 points behind the model that holds the record, at 17× its throughput on -a machine with no GPU**, from 4.7 MB of detector weights against 6.4 GB — -`correctness PASS · quality FAIL · speed PASS · footprint PASS`. Not parity. -The same order of magnitude, in a deployment class neither reference can enter. +Head to head with Baidu's Unlimited-OCR (a 3B MoE on a GPU), on the 236-page +English subset without tables or formulas: + +| | Text | Reading order | +|---|---:|---:| +| Unlimited-OCR | **0.041** | **0.052** | +| Carmenta | 0.108 | 0.175 | +| gap, 95 % CI | +0.068 [+0.049, +0.088] | +0.123 [+0.096, +0.151] | + +Clearly behind the model that holds the record. The remaining loss is mostly +content Carmenta does not emit yet (inline math is 36 % of it) and recognition +substitutions, measured block by block in `docs/plans/benching-history-made.md` +§57. Carmenta does it on a CPU at roughly **9× that model's measured +throughput** on the same machine, from megabytes of weights against 6.4 GB. +The full-corpus row includes the opt-in table/formula routing +(`FFAI_ROUTE=1`); the crate README says what that needs. + +**Forms, scanned filings and PDFs (0.11).** Built after a real caller got +confident wrong values back from scanned disclosure forms. These are measured +on synthetic corpora with ground truth by construction, not yet on real +filings. Details and caveats are in the +[`ffai-carmenta` README](crates/ffai-carmenta/README.md) and +[`docs/plans/commercial-gaps.md`](docs/plans/commercial-gaps.md). + +- On typed values sitting on a form's underline, `mobiledet-svtr` reads + **93 %** exactly where `mobiledet-crnn` reads 13-20 %. +- `--auto-orient` undid **144 / 144** quarter turns, at 48 % of a page read. +- Checkboxes: **100 %** found and **100 %** marked/empty state on 960 + synthetic boxes. +- Label → value pairing: **220 / 221** correctly-read values paired. +- `ffai ocr -i file.pdf` reads each page as a viewer shows it: redaction + patches are painted in, never read through, and a born-digital text layer + with something drawn over it is withheld rather than printed. Full campaign history: [docs/Carmenta-mission-plan.md](docs/Carmenta-mission-plan.md) §8; every diff --git a/ROADMAP.md b/ROADMAP.md index dbe6d6a4..428fc87f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # FFai Roadmap -> **Reconciled against the README and `bench/ledger.jsonl` on 2026-09-19.** +> **Reconciled against the README and `bench/ledger.jsonl` on 2026-09-25.** > Every ✅ below traces to a claim the README evidences; every ⬜ is open > work. If this line is old, distrust the boxes — the README is the source of > truth for what is measured. @@ -98,7 +98,16 @@ participates only as an out-of-process test oracle; nothing GPL ships. - ✅ Oracle gate on public ground truth: **OmniDocBench v1.6 scored by their own evaluator**, all 1 651 pages — Text^Edit 0.1157, ReadOrder^Edit 0.2039 ([docs/plans/restarting-carmenta-doc.md](docs/finished/restarting-carmenta-doc.md)). +- ✅ Forms and scanned filings (0.11, [docs/plans/commercial-gaps.md](docs/plans/commercial-gaps.md)): + quarter-turn orientation (144/144 on turned pages), checkbox detection, + label → value pairing, charset and lexicon constraints, form-rule removal, + offline weights, and PDF input that composites redactions rather than + reading through them. Measured on synthetic corpora with ground truth by + construction; not yet on real filings. +- ⬜ JBIG2 and CCITT Group 3 2-D decoding; tables and free-hand ticks in the + OCR output; hosted converted weights so manifests have a download source. - ⬜ Photo accuracy still trails PaddleOCR; causes are diagnosed, not fixed. + Small-angle deskew is still open (`auto_orient` handles quarter turns only). - ⬜ Evaluate pure-Rust `ocrs` as a zero-setup baseline engine. - ⬜ rff image decoders (PNG/JPEG/WebP). @@ -124,14 +133,17 @@ Remaining: It needs a video-capable checkpoint first. - **Speed.** Now **1.20x** off the PyTorch reference end to end (10 918 vs 9 106 ms, same image, idle box), down from 2.4x across five optimization - rounds; quality is an exact tie and footprint is 0.71x. The deficit is + rounds. The deficit is concentrated in the vision tower, which is 75 % of a caption and 77 % matmul — the elementwise phase is spent. Blocked attention has been refuted five times, the last with candle's own GEMM. -- **Re-run the `ffai bench vlm` gate.** Its recorded row still reads - `speed FAIL` at 2.4x and is stale. Re-running is blocked by a harness defect, - not an engine one: the engine arm segfaults on the second `describe_image` in - one process, and the crash reproduces with the optimizations reverted. +- **Re-measure the `ffai bench vlm` gate on a quiet box.** It has been re-run + (ledger `bench-vlm-1789863219`, `-1789866779`; the old segfault no longer + reproduces). Speed now PASSes and footprint FAILs (1.29x, 1.54x the + reference's memory, contradicting the older 0.71x claim), but the two runs + disagree on magnitude. Quality is SKIPPED until VLMEvalKit is reinstalled in + `.venv-argus`. It needs an interleaved harness and an idle machine before a + number is claimed. - `mistralrs` behind the reserved `mistralrs-backend` feature, for the serving concerns it owns: quantized weights and grammar-constrained JSON decoding. Blocked on a crates.io release that can load `SmolVLM` — the working version diff --git a/corpora/carmenta-checkbox-v1.toml b/corpora/carmenta-checkbox-v1.toml new file mode 100644 index 00000000..7165e240 --- /dev/null +++ b/corpora/carmenta-checkbox-v1.toml @@ -0,0 +1,643 @@ +name = "carmenta-checkbox" +version = 1 +task = "ocr" + +[[clips]] +id = "cb-00-gray" +path = "clips/carmenta-checkbox/cb-00-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "451039ef160ae93318e583f69f6efd8ac0f3151936360137fb9b29a1e6d1c642" + +[[clips]] +id = "cb-00-bw" +path = "clips/carmenta-checkbox/cb-00-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "232587e9fee23238776ecd9203a21313cc59707ff2f29c6a5e1dcecb628c47fa" + +[[clips]] +id = "cb-01-gray" +path = "clips/carmenta-checkbox/cb-01-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "4cf3a42d18627c1551809d2dcbfb519a38a644d014881004ef567d57ea3e68f6" + +[[clips]] +id = "cb-01-bw" +path = "clips/carmenta-checkbox/cb-01-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "f8bf2f2043b9c513b009b29d151066416cda8e8bffd7bb3a8a1cd820759566aa" + +[[clips]] +id = "cb-02-gray" +path = "clips/carmenta-checkbox/cb-02-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "8f7382e76bc8f3d4b4aeb843ec7fe289f2fbacd09ef6ae9f4ae002dc2fdce05a" + +[[clips]] +id = "cb-02-bw" +path = "clips/carmenta-checkbox/cb-02-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "f47e907d0c9e81ac1b09bb76ca124db3fc6454539a6ee6d2ec179f22295802ba" + +[[clips]] +id = "cb-03-gray" +path = "clips/carmenta-checkbox/cb-03-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "10b4230c9a9c1c0607266c890107db0fe7e4c2444545723fb26947c5d18a7a4f" + +[[clips]] +id = "cb-03-bw" +path = "clips/carmenta-checkbox/cb-03-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "0be72a7c169605ba7d38bb0cb0bae7640607beb47198745d427c8cb702aaff84" + +[[clips]] +id = "cb-04-gray" +path = "clips/carmenta-checkbox/cb-04-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "2cdf65049e533e6a635d8a051795f17f03d7b8fec27bdc637c239c7eed07bfdd" + +[[clips]] +id = "cb-04-bw" +path = "clips/carmenta-checkbox/cb-04-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "65169f25a4ba24bc0cce3f6a6c86e10797ead1a11fe0ead1685b8fe12158bd14" + +[[clips]] +id = "cb-05-gray" +path = "clips/carmenta-checkbox/cb-05-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "54daa33d9e893035fb3d85c52c05d124f029919dcfa6f2e499b97da94f30019a" + +[[clips]] +id = "cb-05-bw" +path = "clips/carmenta-checkbox/cb-05-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "6da2defcbc07a68301be911ee692ebad7485f8b0975971d8c4e75d501ce5aa4c" + +[[clips]] +id = "cb-06-gray" +path = "clips/carmenta-checkbox/cb-06-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "79a34dd46d91e801d2ee1622646b54f000ea4ce55567a85347ef573c70eadb8c" + +[[clips]] +id = "cb-06-bw" +path = "clips/carmenta-checkbox/cb-06-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "7b002a592ec0a30c615b9059cbd67597be50602b5f4a3876b40fd7d80998b350" + +[[clips]] +id = "cb-07-gray" +path = "clips/carmenta-checkbox/cb-07-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "3992dcce69810971d1afb78402c0a14edbf044c4736a32411e9b6fad6adf209b" + +[[clips]] +id = "cb-07-bw" +path = "clips/carmenta-checkbox/cb-07-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "d77e4a4744ebf1d5bce370982a6e3f489b36b6c7feb9b16fa585823b5ca29782" + +[[clips]] +id = "cb-08-gray" +path = "clips/carmenta-checkbox/cb-08-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "5f5ec221353f3a9d62b78e9afba90da965c3c974e2d1ab5a50fadec5365d2ee2" + +[[clips]] +id = "cb-08-bw" +path = "clips/carmenta-checkbox/cb-08-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "1638d44ec653ce368fcd48ec735216dffe1fecde4144b6ca98b568dabb0f4bac" + +[[clips]] +id = "cb-09-gray" +path = "clips/carmenta-checkbox/cb-09-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "ca7e71a5b8dc0ef9c19218de846020b7c647d6d414b2fefa2cd8dde964cbee8a" + +[[clips]] +id = "cb-09-bw" +path = "clips/carmenta-checkbox/cb-09-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "4b3b6d22b4785eb32401375f286d532d680f4526b9c64a21ae756ee89f4032d4" + +[[clips]] +id = "cb-10-gray" +path = "clips/carmenta-checkbox/cb-10-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "041eb618d7675a3e3a5a94531939848cc08287425abf496c89155b5008617daa" + +[[clips]] +id = "cb-10-bw" +path = "clips/carmenta-checkbox/cb-10-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "34f8af83f7caac32fc8e3f9d0b8fca4cb54b57e9adaf531c47ae17fd67c5c9be" + +[[clips]] +id = "cb-11-gray" +path = "clips/carmenta-checkbox/cb-11-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "5ae818f594eee2c3885ad067d49a20c7abe75a4aba09b5ba4412617dde2b981f" + +[[clips]] +id = "cb-11-bw" +path = "clips/carmenta-checkbox/cb-11-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "fbc5b28246c89b685a0dfb476f0633377fd730c7f588aab684fd5116ca8fd167" + +[[clips]] +id = "cb-12-gray" +path = "clips/carmenta-checkbox/cb-12-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "c7b1ba4078ebaecac750688d3af1aa4e4a90c2ca1de2526e529d0eeac2f16a3a" + +[[clips]] +id = "cb-12-bw" +path = "clips/carmenta-checkbox/cb-12-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "6ac5ed530cc36c3a83f384002742c079f8bdfa48290f64ea386125c7d234e9a5" + +[[clips]] +id = "cb-13-gray" +path = "clips/carmenta-checkbox/cb-13-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "694d143f6940ccd06c931b626acb4888562961e4d483845a00d8f9a3db743fb1" + +[[clips]] +id = "cb-13-bw" +path = "clips/carmenta-checkbox/cb-13-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "c61ccea6996bb0b5d89c262f7bc5329ee19d9b9035936e26e3bee567c36572e2" + +[[clips]] +id = "cb-14-gray" +path = "clips/carmenta-checkbox/cb-14-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "43058e393be4f107a725e201afb156218b2df2512387c7ae0adf3cd5003bbfcb" + +[[clips]] +id = "cb-14-bw" +path = "clips/carmenta-checkbox/cb-14-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "51ec211ad0677c2168096fae1a57adba576e201bde637f694a84e1c17e387519" + +[[clips]] +id = "cb-15-gray" +path = "clips/carmenta-checkbox/cb-15-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "9e7c13f7f121fd07c76cd7cd26304f1be1a9971aded5be510163de04768216dc" + +[[clips]] +id = "cb-15-bw" +path = "clips/carmenta-checkbox/cb-15-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "534582559780d6b1e6b1e3105f99e01ff079bbbdd2e876e31d20cc0681f48029" + +[[clips]] +id = "cb-16-gray" +path = "clips/carmenta-checkbox/cb-16-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "ef3a27f194fa6c2bbe2d0d6f15fb96c4594e7d695bd559f8d7ce4bc6f84224a7" + +[[clips]] +id = "cb-16-bw" +path = "clips/carmenta-checkbox/cb-16-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "47598597fcffd344014b30e0c84434b42818a101debdf81cd71fde4d73f1b476" + +[[clips]] +id = "cb-17-gray" +path = "clips/carmenta-checkbox/cb-17-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "f9b581813b2d8c1037da65910691673f05119d3d4997275fac4599c7577dd1a3" + +[[clips]] +id = "cb-17-bw" +path = "clips/carmenta-checkbox/cb-17-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "5380d8cd674854a83809cdb738a11f4989e49d80692cc085336d67ad8d3f6d67" + +[[clips]] +id = "cb-18-gray" +path = "clips/carmenta-checkbox/cb-18-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "16f48b1d8a23146a83bc681187eabf8a9d414f1e6ffb3b32d71a0babcd9b8612" + +[[clips]] +id = "cb-18-bw" +path = "clips/carmenta-checkbox/cb-18-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "aea5a2793864b4ee712d313abd50f868c0219f1b0977151d1275052da51a4149" + +[[clips]] +id = "cb-19-gray" +path = "clips/carmenta-checkbox/cb-19-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "d5f483a3d24a75717fd3d00a15033398e559aaaec3ec4d1243476ed5ad7ea7eb" + +[[clips]] +id = "cb-19-bw" +path = "clips/carmenta-checkbox/cb-19-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "df581cacee04c8923da67786811ccfada36714328c4ca38691a2690fb6988d99" + +[[clips]] +id = "cb-20-gray" +path = "clips/carmenta-checkbox/cb-20-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "da33f2d52d1ae1b3aee1083b8d7b0bc26a13914e5835c682b0b876543abefdaa" + +[[clips]] +id = "cb-20-bw" +path = "clips/carmenta-checkbox/cb-20-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "14eae0cf69b760e56c6d025d28510e0ab81b0db4f47eb7a02fd523e7c3d618ec" + +[[clips]] +id = "cb-21-gray" +path = "clips/carmenta-checkbox/cb-21-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "9a1bb6f5ee51155ed5f8b8b8f40f533be95f8287d0a6dfeb22c5850a636675b1" + +[[clips]] +id = "cb-21-bw" +path = "clips/carmenta-checkbox/cb-21-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "a76fc93a343a78d10de4cf2abd713acc9fb5b59ef9f3e3e3a793003b9f3d787c" + +[[clips]] +id = "cb-22-gray" +path = "clips/carmenta-checkbox/cb-22-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "03f18f676ff2a344c61c9d1284fb46382bbb103c36a48d53fe3aecf707cb2993" + +[[clips]] +id = "cb-22-bw" +path = "clips/carmenta-checkbox/cb-22-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "c581b8e1b31b9698531d5433acede69bd334f8072679dd2d1e73ed8e6b8a1060" + +[[clips]] +id = "cb-23-gray" +path = "clips/carmenta-checkbox/cb-23-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "e7946830db55b8d61bde4e56c0320a100f54ea25d62481f2bd69687c55f864a2" + +[[clips]] +id = "cb-23-bw" +path = "clips/carmenta-checkbox/cb-23-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "f7dc7373821e0907c4aac01ab6c0bf9a80e37813635921a0ec20105ad8816ba5" + +[[clips]] +id = "cb-24-gray" +path = "clips/carmenta-checkbox/cb-24-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "264aabd0715125e8cb6d436b4171715d987cefba958765f54d47b52f2e738131" + +[[clips]] +id = "cb-24-bw" +path = "clips/carmenta-checkbox/cb-24-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "bff1587a46e7c19bef52f9cbf6d3f04619de306cebbc6b17c73da31115afd891" + +[[clips]] +id = "cb-25-gray" +path = "clips/carmenta-checkbox/cb-25-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "cd78d4037f78a591f591687831cb85ef9a28363143dd2ddaa624c108be3c73e1" + +[[clips]] +id = "cb-25-bw" +path = "clips/carmenta-checkbox/cb-25-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "565e7a2ddb40173c09151f55d1ad4299e2dca39834205fb41451483121a1df18" + +[[clips]] +id = "cb-26-gray" +path = "clips/carmenta-checkbox/cb-26-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "e78cfafc9a1d6ec111d27601559be3b7014cea364447bac9c4b9ecb69fb2be4b" + +[[clips]] +id = "cb-26-bw" +path = "clips/carmenta-checkbox/cb-26-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "3230332d5b5aac98061aaac082ef463bc372a6c327ca03ca3b50712619b42b69" + +[[clips]] +id = "cb-27-gray" +path = "clips/carmenta-checkbox/cb-27-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "7e10b2b7fe177b11ed726ae7e6507ae2a007b56dc801616c61b791522f180a09" + +[[clips]] +id = "cb-27-bw" +path = "clips/carmenta-checkbox/cb-27-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "c48da6967ca664a30716f79293864e5a8af1b407fd85c2290e5d8c644a7e5fab" + +[[clips]] +id = "cb-28-gray" +path = "clips/carmenta-checkbox/cb-28-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "5bb331811fed063fc9bfe608bbaac67d8cf97e977dd26a30ea3368dc671d63b1" + +[[clips]] +id = "cb-28-bw" +path = "clips/carmenta-checkbox/cb-28-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "6e4ed0cbc4b9c5a8c82c094dfd4e53fff65584949a912154d4548cca2f056d90" + +[[clips]] +id = "cb-29-gray" +path = "clips/carmenta-checkbox/cb-29-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "03a7df0ceb270f2c49bf16db7ce7059d756c0fd9c94db75101a7f904d0e1dd9a" + +[[clips]] +id = "cb-29-bw" +path = "clips/carmenta-checkbox/cb-29-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "80fa7d203f9cb73ae3698d61933c156fdce7e319f63a755294e39121c5783255" + +[[clips]] +id = "cb-30-gray" +path = "clips/carmenta-checkbox/cb-30-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "7752a3964aae8ceca61c85a26851e26f6826d60f3253887d7211bca32ac0afe3" + +[[clips]] +id = "cb-30-bw" +path = "clips/carmenta-checkbox/cb-30-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "437e10cb8c3628f663df1b6b76bccf460262af997adbf7010a3bbd5a50c4d97f" + +[[clips]] +id = "cb-31-gray" +path = "clips/carmenta-checkbox/cb-31-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "da83630e72fbe31476820973f3b3702eb8b418fcf35f1d1f19cc965f5fcd2dca" + +[[clips]] +id = "cb-31-bw" +path = "clips/carmenta-checkbox/cb-31-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "9a6d8ba08aa5e338ff2453fe3a59c131b133f5e21f967c0a3c6717d44d42068a" + +[[clips]] +id = "cb-32-gray" +path = "clips/carmenta-checkbox/cb-32-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "33d3b911084df647924bcbb7c94fe4fe0059f6e85bc8588937c2d986afcb6863" + +[[clips]] +id = "cb-32-bw" +path = "clips/carmenta-checkbox/cb-32-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "de21e38682c218c1af39701aa468072770ad02bdc717743a93794a46981912db" + +[[clips]] +id = "cb-33-gray" +path = "clips/carmenta-checkbox/cb-33-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "7d71f57cf87a8ca7c80eae35ce868f4e97f437a17aeebe2809f7769a3b9253cc" + +[[clips]] +id = "cb-33-bw" +path = "clips/carmenta-checkbox/cb-33-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "02c00f16cfebaeecd02d0f30c2899c5b322144654997f99bc9b1f4598fe2210e" + +[[clips]] +id = "cb-34-gray" +path = "clips/carmenta-checkbox/cb-34-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "999ccb9d6ded0c0fff3c73378901f2cf2a55eb75f42117548309be9041d3ac2e" + +[[clips]] +id = "cb-34-bw" +path = "clips/carmenta-checkbox/cb-34-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "7d67572e6445fc24210a91e61a89e21ce5825d74bad9d2b8af6f7fe0aa1a1b8f" + +[[clips]] +id = "cb-35-gray" +path = "clips/carmenta-checkbox/cb-35-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "85fed12fe13b72f8da03d748a6fa446684feabb25aeced9aa5887b53f9deb0f0" + +[[clips]] +id = "cb-35-bw" +path = "clips/carmenta-checkbox/cb-35-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "b5e703dbb3625ec010e26e7e7306fc84b8252b719bdbac5b51478396c06903b0" + +[[clips]] +id = "cb-36-gray" +path = "clips/carmenta-checkbox/cb-36-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "fc259587c05aae5c4c24931da0cd566d7eb9d02c87f0737beeacbb4d459097ed" + +[[clips]] +id = "cb-36-bw" +path = "clips/carmenta-checkbox/cb-36-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "a7a4fedfc2af23ae0d3449738d19e0c1944f4d5f9c00dcb56fa4befacb8843ae" + +[[clips]] +id = "cb-37-gray" +path = "clips/carmenta-checkbox/cb-37-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "31db9d562fd4e31d7b8fd997fd3a5db966122d0e30a238cc6abeabf870f78939" + +[[clips]] +id = "cb-37-bw" +path = "clips/carmenta-checkbox/cb-37-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "dc1c231c3f547853edc973b8936a7f4a1213f73c7d351e04b349a0bf1f820bfa" + +[[clips]] +id = "cb-38-gray" +path = "clips/carmenta-checkbox/cb-38-gray.png" +class = "checkbox-gray" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "5f94c1c8b5510d6ec4a4f3136054430fa3e68cee3195efcdd76de67a6d774926" + +[[clips]] +id = "cb-38-bw" +path = "clips/carmenta-checkbox/cb-38-bw.png" +class = "checkbox-bw" +split = "train" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "e71f95b3637a0939e03f617474c80e404bf79f467d0dbb09e43f1bc618f76bf3" + +[[clips]] +id = "cb-39-gray" +path = "clips/carmenta-checkbox/cb-39-gray.png" +class = "checkbox-gray" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "c3b900d877fb62e479d710e6b0b524d9a5361be595ec4bc6ae79344c190a2d1b" + +[[clips]] +id = "cb-39-bw" +path = "clips/carmenta-checkbox/cb-39-bw.png" +class = "checkbox-bw" +split = "holdout" +license = "CC0-1.0 (render of invented form rows)" +sha256 = "93656fad98f211163c16992cf0292cc845f1638dde26c01235ef09bd4575a753" diff --git a/corpora/carmenta-form-v1.toml b/corpora/carmenta-form-v1.toml new file mode 100644 index 00000000..b321f3fb --- /dev/null +++ b/corpora/carmenta-form-v1.toml @@ -0,0 +1,2163 @@ +name = "carmenta-form" +version = 1 +task = "ocr" + +[[clips]] +id = "form-00-none-gray" +path = "clips/carmenta-form/form-00-none-gray.png" +ground_truth = "clips/carmenta-form/form-00-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f1cfe6d5c8fe968c2492e528a4af212e6624701bd934c84d8bf623cce0c3e0de" + +[[clips]] +id = "form-00-none-bw" +path = "clips/carmenta-form/form-00-none-bw.png" +ground_truth = "clips/carmenta-form/form-00-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "dcba52d9e53fecb2736ac42f01e8390dc21e408b624f069e511689c6707cd6d8" + +[[clips]] +id = "form-00-under-gray" +path = "clips/carmenta-form/form-00-under-gray.png" +ground_truth = "clips/carmenta-form/form-00-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "bb609c6cc2577058edb0a62b131c0748c66c29a7a5b847d6a8c1d8d622fbeb89" + +[[clips]] +id = "form-00-under-bw" +path = "clips/carmenta-form/form-00-under-bw.png" +ground_truth = "clips/carmenta-form/form-00-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5296f4725d322ec50944b48d414eef1c1c349799c3a9c90c93be659a9f4715e3" + +[[clips]] +id = "form-00-desc-gray" +path = "clips/carmenta-form/form-00-desc-gray.png" +ground_truth = "clips/carmenta-form/form-00-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8306079208d451036b4a00e855dcd5e70dae2cca03bf603debbd8132b6a9f5d1" + +[[clips]] +id = "form-00-desc-bw" +path = "clips/carmenta-form/form-00-desc-bw.png" +ground_truth = "clips/carmenta-form/form-00-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "57dc0734b04932d30f150ab4e65f24ad3425a6cb364db91c9a8158357c704b5c" + +[[clips]] +id = "form-00-strike-gray" +path = "clips/carmenta-form/form-00-strike-gray.png" +ground_truth = "clips/carmenta-form/form-00-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f36db11e532108a11517b7c54a30c1cce261916bf68ca30ed24f889d92560282" + +[[clips]] +id = "form-00-strike-bw" +path = "clips/carmenta-form/form-00-strike-bw.png" +ground_truth = "clips/carmenta-form/form-00-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b438603d32bc3e196ee16bde967f15c5f0899d5eb18d9a84b44985d341ba7f46" + +[[clips]] +id = "form-01-none-gray" +path = "clips/carmenta-form/form-01-none-gray.png" +ground_truth = "clips/carmenta-form/form-01-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f6e499bc98b08d1d650c245e45c62fb9e4f2f6c7c86453bac80166eb51053051" + +[[clips]] +id = "form-01-none-bw" +path = "clips/carmenta-form/form-01-none-bw.png" +ground_truth = "clips/carmenta-form/form-01-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3eae9b05439a1d5970e44255e0b23cef9fe6259368e4e33cce3a5ffd4ce41d1c" + +[[clips]] +id = "form-01-under-gray" +path = "clips/carmenta-form/form-01-under-gray.png" +ground_truth = "clips/carmenta-form/form-01-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a91d7e318985ffc8b5eb8efac24c3da315235511a40faf8a60d67b23fad959e5" + +[[clips]] +id = "form-01-under-bw" +path = "clips/carmenta-form/form-01-under-bw.png" +ground_truth = "clips/carmenta-form/form-01-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5fc3f17ec540d7190684f4734b58271f67c45306b2c6c299eaf319bd27f95c34" + +[[clips]] +id = "form-01-desc-gray" +path = "clips/carmenta-form/form-01-desc-gray.png" +ground_truth = "clips/carmenta-form/form-01-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "509532c3c6f3311b8ed803c4687d04c7a70f3f5b0688b4fdaeb8b245f29fbae5" + +[[clips]] +id = "form-01-desc-bw" +path = "clips/carmenta-form/form-01-desc-bw.png" +ground_truth = "clips/carmenta-form/form-01-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b639c00eb6ae964deeb54035f800cd4291187edb24062ead18cb37dca753be6f" + +[[clips]] +id = "form-01-strike-gray" +path = "clips/carmenta-form/form-01-strike-gray.png" +ground_truth = "clips/carmenta-form/form-01-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7556729e85e5282b40dfc7d79141df1c1b3bc0cbb6920ef13307bf9e45964a39" + +[[clips]] +id = "form-01-strike-bw" +path = "clips/carmenta-form/form-01-strike-bw.png" +ground_truth = "clips/carmenta-form/form-01-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ada1143e4e968edf3708faac0d2847e73f3689d2b69f125a90b43244c1d0e8e2" + +[[clips]] +id = "form-02-none-gray" +path = "clips/carmenta-form/form-02-none-gray.png" +ground_truth = "clips/carmenta-form/form-02-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ec013fd0ad79f3a7d62dde0c2d556fd7882948667106021196c7713a601bc444" + +[[clips]] +id = "form-02-none-bw" +path = "clips/carmenta-form/form-02-none-bw.png" +ground_truth = "clips/carmenta-form/form-02-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8b7fe9bbc700b87f37b0649be377df5544e5607fb718d902c37a912db86cc2ac" + +[[clips]] +id = "form-02-under-gray" +path = "clips/carmenta-form/form-02-under-gray.png" +ground_truth = "clips/carmenta-form/form-02-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3faa61d501f329dcd65805c74beb6f97c2e19e335be8086492bc95dcf85c11ac" + +[[clips]] +id = "form-02-under-bw" +path = "clips/carmenta-form/form-02-under-bw.png" +ground_truth = "clips/carmenta-form/form-02-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "74c158a5446f5498d74e47fb54dfa960faa492ba4c0a790c3a1c9771988fc39d" + +[[clips]] +id = "form-02-desc-gray" +path = "clips/carmenta-form/form-02-desc-gray.png" +ground_truth = "clips/carmenta-form/form-02-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0bcab3f7569f9c8c987c6a89eb3afe89986438fcfd1cefec36c2f68d2113af40" + +[[clips]] +id = "form-02-desc-bw" +path = "clips/carmenta-form/form-02-desc-bw.png" +ground_truth = "clips/carmenta-form/form-02-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "4c92deeb6338dba28035ce7806c528a5f8b2876955156195bb465930c87de334" + +[[clips]] +id = "form-02-strike-gray" +path = "clips/carmenta-form/form-02-strike-gray.png" +ground_truth = "clips/carmenta-form/form-02-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8e1765dd0c7a0ed7cb3cb2010fe1efbcf736d62a8d429c1b474b532dc5dd6a1b" + +[[clips]] +id = "form-02-strike-bw" +path = "clips/carmenta-form/form-02-strike-bw.png" +ground_truth = "clips/carmenta-form/form-02-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f2f6dab74b1cf653d36acc8efd5a3888a89869e3e4756c8756916a040436dc9d" + +[[clips]] +id = "form-03-none-gray" +path = "clips/carmenta-form/form-03-none-gray.png" +ground_truth = "clips/carmenta-form/form-03-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c4dcbc81551d0519d28d02492bf4963269cbcab3e4a40c98cf48bc94735bb88f" + +[[clips]] +id = "form-03-none-bw" +path = "clips/carmenta-form/form-03-none-bw.png" +ground_truth = "clips/carmenta-form/form-03-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "68a1cd2b3bec3ee80e4ac657c342a19657648befb63aa66578802aacbb957081" + +[[clips]] +id = "form-03-under-gray" +path = "clips/carmenta-form/form-03-under-gray.png" +ground_truth = "clips/carmenta-form/form-03-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "21a099628b796164eefdd5d9a4884ecddb3ed1a235d08d0e2b3f541b09c897e2" + +[[clips]] +id = "form-03-under-bw" +path = "clips/carmenta-form/form-03-under-bw.png" +ground_truth = "clips/carmenta-form/form-03-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a43911929a2d0e0d253e4c978420ab780f506637bbfca3badff8930ff8200227" + +[[clips]] +id = "form-03-desc-gray" +path = "clips/carmenta-form/form-03-desc-gray.png" +ground_truth = "clips/carmenta-form/form-03-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2ef11cebc68d5b343a32e8f2b09cd78f70e9311cf9cdf79969a08b46cf484b99" + +[[clips]] +id = "form-03-desc-bw" +path = "clips/carmenta-form/form-03-desc-bw.png" +ground_truth = "clips/carmenta-form/form-03-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b5f5693762331ece24311df096ba195176f41c40ccde20eeb50e131d6b9f3c97" + +[[clips]] +id = "form-03-strike-gray" +path = "clips/carmenta-form/form-03-strike-gray.png" +ground_truth = "clips/carmenta-form/form-03-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9b934f11e89314b4ba7613397cf3a085a42d7dacb3aefb10f38cf8195a279728" + +[[clips]] +id = "form-03-strike-bw" +path = "clips/carmenta-form/form-03-strike-bw.png" +ground_truth = "clips/carmenta-form/form-03-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "cb1b5f2d31e86e82d6423c1b6f6e5adedf5726e4a42dab95cc9a3293160b29e2" + +[[clips]] +id = "form-04-none-gray" +path = "clips/carmenta-form/form-04-none-gray.png" +ground_truth = "clips/carmenta-form/form-04-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "507ee0bdf21bff5f8cbc93591869cb209cf5e64ec506dc94491859615f29f8e7" + +[[clips]] +id = "form-04-none-bw" +path = "clips/carmenta-form/form-04-none-bw.png" +ground_truth = "clips/carmenta-form/form-04-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "fc95944eaa999e8f252dd2298f1a2a3247976058aa00156dae0ec715eace178a" + +[[clips]] +id = "form-04-under-gray" +path = "clips/carmenta-form/form-04-under-gray.png" +ground_truth = "clips/carmenta-form/form-04-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7780cdd04d8f2c17ca26778ca31bc672042e680a38a58aabd7e02650f89ae269" + +[[clips]] +id = "form-04-under-bw" +path = "clips/carmenta-form/form-04-under-bw.png" +ground_truth = "clips/carmenta-form/form-04-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1cd48fa515dab7bfd2007d9453486a60dfffec95b181756f1e436cb55ba28696" + +[[clips]] +id = "form-04-desc-gray" +path = "clips/carmenta-form/form-04-desc-gray.png" +ground_truth = "clips/carmenta-form/form-04-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "603d5511a42fda216bb76fa1b1e5695c1c0119a2e30de11513d01c9140ee40fa" + +[[clips]] +id = "form-04-desc-bw" +path = "clips/carmenta-form/form-04-desc-bw.png" +ground_truth = "clips/carmenta-form/form-04-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "fa3535bfe559fe22069dd553ceab9c4ac7eecaa713a64a17462fcc76ddaf809d" + +[[clips]] +id = "form-04-strike-gray" +path = "clips/carmenta-form/form-04-strike-gray.png" +ground_truth = "clips/carmenta-form/form-04-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "990b56f1518384805a067df40ba8f6390e4418ee61f339486219b3526b5409fa" + +[[clips]] +id = "form-04-strike-bw" +path = "clips/carmenta-form/form-04-strike-bw.png" +ground_truth = "clips/carmenta-form/form-04-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "697f6dc9c09e054459a4264694237bb705371adec281233e8f4e87d9bd60dd34" + +[[clips]] +id = "form-05-none-gray" +path = "clips/carmenta-form/form-05-none-gray.png" +ground_truth = "clips/carmenta-form/form-05-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "47b7d5f56c4bb69440cc34c5885ad77282fe197e20e4a58dbee98aa29ca9d68b" + +[[clips]] +id = "form-05-none-bw" +path = "clips/carmenta-form/form-05-none-bw.png" +ground_truth = "clips/carmenta-form/form-05-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a763c167c78b7fd54da941046d528c3a9d59d80f70896a67f9da630493ac9080" + +[[clips]] +id = "form-05-under-gray" +path = "clips/carmenta-form/form-05-under-gray.png" +ground_truth = "clips/carmenta-form/form-05-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5149c416220ad9839e7eda7839cbd7d2fe64b23105528224e791e4d3050c0cdf" + +[[clips]] +id = "form-05-under-bw" +path = "clips/carmenta-form/form-05-under-bw.png" +ground_truth = "clips/carmenta-form/form-05-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "4f8273c09a606fd42e78b36b21aa3f1317359d6df336a68c768733cf456802c2" + +[[clips]] +id = "form-05-desc-gray" +path = "clips/carmenta-form/form-05-desc-gray.png" +ground_truth = "clips/carmenta-form/form-05-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9e6927ec6637dd6fb9e679cee3072ae640adcedd1b30eeb9eacbfc54f4252c41" + +[[clips]] +id = "form-05-desc-bw" +path = "clips/carmenta-form/form-05-desc-bw.png" +ground_truth = "clips/carmenta-form/form-05-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1295b8d4169a848019d70bf56e67321fc77c83dd91037ef347324fb8a58b648d" + +[[clips]] +id = "form-05-strike-gray" +path = "clips/carmenta-form/form-05-strike-gray.png" +ground_truth = "clips/carmenta-form/form-05-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "d2acef581204b7de92060a9533e59d8506b9d95e9979c3eca47a79b62f11c504" + +[[clips]] +id = "form-05-strike-bw" +path = "clips/carmenta-form/form-05-strike-bw.png" +ground_truth = "clips/carmenta-form/form-05-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1d3c96518445325a561acc6c84f4c873775e917e475c834b4f8cc80a8f9a5a2c" + +[[clips]] +id = "form-06-none-gray" +path = "clips/carmenta-form/form-06-none-gray.png" +ground_truth = "clips/carmenta-form/form-06-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7b170241f6b9858e1b6aa25014f3e6e026bf0c414d40098256df465082b23113" + +[[clips]] +id = "form-06-none-bw" +path = "clips/carmenta-form/form-06-none-bw.png" +ground_truth = "clips/carmenta-form/form-06-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c35735818b1a33e64fd4ecc70d6ef8d16781e6a5253d83df30a1b9ce9d93726e" + +[[clips]] +id = "form-06-under-gray" +path = "clips/carmenta-form/form-06-under-gray.png" +ground_truth = "clips/carmenta-form/form-06-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5a71e5a1aaf890d5688bb9a4015184a454e1f92528504c915bc37d6a5cc45085" + +[[clips]] +id = "form-06-under-bw" +path = "clips/carmenta-form/form-06-under-bw.png" +ground_truth = "clips/carmenta-form/form-06-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "121ee8781e4ea7712a6bf65e09263d9147b559b3c1031526e3a3a4c4eb68c2e8" + +[[clips]] +id = "form-06-desc-gray" +path = "clips/carmenta-form/form-06-desc-gray.png" +ground_truth = "clips/carmenta-form/form-06-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3bdfb25a6115368b54d2316b919283909cabd258da8a8280eb51fba1d2fb33d1" + +[[clips]] +id = "form-06-desc-bw" +path = "clips/carmenta-form/form-06-desc-bw.png" +ground_truth = "clips/carmenta-form/form-06-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5ab87b7eb85d76d8b227f12d4279ad2a0264d7eee4ceb6fbc146a9cd424a8c30" + +[[clips]] +id = "form-06-strike-gray" +path = "clips/carmenta-form/form-06-strike-gray.png" +ground_truth = "clips/carmenta-form/form-06-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "153fba47ae0c3c26c04608312aa40127425a1bfbfb3c5d62da98ec352d9dc0ab" + +[[clips]] +id = "form-06-strike-bw" +path = "clips/carmenta-form/form-06-strike-bw.png" +ground_truth = "clips/carmenta-form/form-06-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "560ff4f6830b129e44f504e31521d8ee8fc8a7b7a96db1f041070d65ab5b4495" + +[[clips]] +id = "form-07-none-gray" +path = "clips/carmenta-form/form-07-none-gray.png" +ground_truth = "clips/carmenta-form/form-07-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3baedb59c91188646e75c14b8a77351b4d696db39a4907e49843b37cc07c59d3" + +[[clips]] +id = "form-07-none-bw" +path = "clips/carmenta-form/form-07-none-bw.png" +ground_truth = "clips/carmenta-form/form-07-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "893211aa7351c62d63bbb3b8d58b48c1067c98d3f9c18b0c58a010607e7a7392" + +[[clips]] +id = "form-07-under-gray" +path = "clips/carmenta-form/form-07-under-gray.png" +ground_truth = "clips/carmenta-form/form-07-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c5e3023f0e4ca23198e538cd4ead61ab6194fca005a1d068ae5da7c2fcb03b46" + +[[clips]] +id = "form-07-under-bw" +path = "clips/carmenta-form/form-07-under-bw.png" +ground_truth = "clips/carmenta-form/form-07-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "65be5a7c337d95950829a5582ea664f2bb014a68cbec496a7cfc10cd2b89882e" + +[[clips]] +id = "form-07-desc-gray" +path = "clips/carmenta-form/form-07-desc-gray.png" +ground_truth = "clips/carmenta-form/form-07-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "991465b6ed9af0bbfb7cfe32d33dbc765b69140d0c8e285136e2bc8b368e8d82" + +[[clips]] +id = "form-07-desc-bw" +path = "clips/carmenta-form/form-07-desc-bw.png" +ground_truth = "clips/carmenta-form/form-07-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f7614f819fc8adb7fac098bcbf7a3dd47322da8cccb37e69bc922ccc48f59382" + +[[clips]] +id = "form-07-strike-gray" +path = "clips/carmenta-form/form-07-strike-gray.png" +ground_truth = "clips/carmenta-form/form-07-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "e72e02d165928f83c7e77ba80ddf71d31fc7f4065dc687b78a6e835837c5e646" + +[[clips]] +id = "form-07-strike-bw" +path = "clips/carmenta-form/form-07-strike-bw.png" +ground_truth = "clips/carmenta-form/form-07-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "dc404dd41ca20987d5030422121c10ff9f45fac772108a76cf9ec9ef4fda5251" + +[[clips]] +id = "form-08-none-gray" +path = "clips/carmenta-form/form-08-none-gray.png" +ground_truth = "clips/carmenta-form/form-08-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7f38eea8986499f582e7ae34285dca2ef124fa0f6ce6ba7971e6cb411264aa10" + +[[clips]] +id = "form-08-none-bw" +path = "clips/carmenta-form/form-08-none-bw.png" +ground_truth = "clips/carmenta-form/form-08-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "69d597cc5b655cf577c7473e8993608cdb31cf445653a191ca88fb0065804c16" + +[[clips]] +id = "form-08-under-gray" +path = "clips/carmenta-form/form-08-under-gray.png" +ground_truth = "clips/carmenta-form/form-08-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "147a4f56c58e069138a0a16022c113b4144f0ca46584cd8a33c190a6211993af" + +[[clips]] +id = "form-08-under-bw" +path = "clips/carmenta-form/form-08-under-bw.png" +ground_truth = "clips/carmenta-form/form-08-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "d50d5367318c9faa232b838a0f2c94ff02d53f525a0521f3745bd8b9e7e3790b" + +[[clips]] +id = "form-08-desc-gray" +path = "clips/carmenta-form/form-08-desc-gray.png" +ground_truth = "clips/carmenta-form/form-08-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "67bd48dd0b39ea11c834690776f6adab7a322afc12cc124adf8b05b0e0ee9b90" + +[[clips]] +id = "form-08-desc-bw" +path = "clips/carmenta-form/form-08-desc-bw.png" +ground_truth = "clips/carmenta-form/form-08-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a7ed5f1980379445d130cf899c389bf965b8fe8529eed7c31487e1275a0f851c" + +[[clips]] +id = "form-08-strike-gray" +path = "clips/carmenta-form/form-08-strike-gray.png" +ground_truth = "clips/carmenta-form/form-08-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "fab798701befebd454cc4c4b176751493615ee734d3fe1f5562eec2254144d07" + +[[clips]] +id = "form-08-strike-bw" +path = "clips/carmenta-form/form-08-strike-bw.png" +ground_truth = "clips/carmenta-form/form-08-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "fdd746f9e1ed81d0ddbb8548d2c26add15124eedbb0877ce1242fb38a6ee5b92" + +[[clips]] +id = "form-09-none-gray" +path = "clips/carmenta-form/form-09-none-gray.png" +ground_truth = "clips/carmenta-form/form-09-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6d4ef7501f89c1e98a9a3d2aa4e12680d56eb78ec498e983d69f37b4653d84e2" + +[[clips]] +id = "form-09-none-bw" +path = "clips/carmenta-form/form-09-none-bw.png" +ground_truth = "clips/carmenta-form/form-09-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "44d29eb526efc0c09d339bf80e085879a1cb142d022721771e07234f1e9f8660" + +[[clips]] +id = "form-09-under-gray" +path = "clips/carmenta-form/form-09-under-gray.png" +ground_truth = "clips/carmenta-form/form-09-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f3417cbed6bb496d7ae13a26789bb463e0021b032b67cbc423b36cf248cb330f" + +[[clips]] +id = "form-09-under-bw" +path = "clips/carmenta-form/form-09-under-bw.png" +ground_truth = "clips/carmenta-form/form-09-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7194f10412f19d2179d695663ab89af4283076a18cd6d7270b3a6a1ce672d93f" + +[[clips]] +id = "form-09-desc-gray" +path = "clips/carmenta-form/form-09-desc-gray.png" +ground_truth = "clips/carmenta-form/form-09-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ad91be5e77c5bf70ae6e5b478ef644a1f19d432800cb352d2539a5cecc0a0537" + +[[clips]] +id = "form-09-desc-bw" +path = "clips/carmenta-form/form-09-desc-bw.png" +ground_truth = "clips/carmenta-form/form-09-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "670f7420c6e972acb13da87ac54bce4737770c7c739f2924821208add4dec4b9" + +[[clips]] +id = "form-09-strike-gray" +path = "clips/carmenta-form/form-09-strike-gray.png" +ground_truth = "clips/carmenta-form/form-09-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f2f0c66dc674c0d0a0d845e3162c0515232f1bc4b5e3098ecd7b70ea76180cef" + +[[clips]] +id = "form-09-strike-bw" +path = "clips/carmenta-form/form-09-strike-bw.png" +ground_truth = "clips/carmenta-form/form-09-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7b53b91988021a9c015d90df6150e342a655bedfcc0f05acb805230605e61488" + +[[clips]] +id = "form-10-none-gray" +path = "clips/carmenta-form/form-10-none-gray.png" +ground_truth = "clips/carmenta-form/form-10-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "792543e8d7f35b1c96941bee2b18f7bcdcfd30b7462a3568c64a3c77a9ee98e1" + +[[clips]] +id = "form-10-none-bw" +path = "clips/carmenta-form/form-10-none-bw.png" +ground_truth = "clips/carmenta-form/form-10-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "71e7af8e79a23feec6dd8d234621bf4c805b1cd9822cb9da2f6737080e9ea010" + +[[clips]] +id = "form-10-under-gray" +path = "clips/carmenta-form/form-10-under-gray.png" +ground_truth = "clips/carmenta-form/form-10-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8208d7f0f9918440c49b7c126fede111dd1147061a371f55fae3aabe47e49f21" + +[[clips]] +id = "form-10-under-bw" +path = "clips/carmenta-form/form-10-under-bw.png" +ground_truth = "clips/carmenta-form/form-10-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "83ca7f8b0410566ca8c9485dc4bc8b55e684f944a210f79f353600108e795cb6" + +[[clips]] +id = "form-10-desc-gray" +path = "clips/carmenta-form/form-10-desc-gray.png" +ground_truth = "clips/carmenta-form/form-10-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "569219073da19b9f1fed2cbb95765be8ebbe432f5fd5f5ab257faf332e1265be" + +[[clips]] +id = "form-10-desc-bw" +path = "clips/carmenta-form/form-10-desc-bw.png" +ground_truth = "clips/carmenta-form/form-10-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b255cf841fffe35d1371ce04c715d759196c594b76f9abc264663689b0d7562a" + +[[clips]] +id = "form-10-strike-gray" +path = "clips/carmenta-form/form-10-strike-gray.png" +ground_truth = "clips/carmenta-form/form-10-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ff52403beba0a02805b8bd2cea86aa91297b0d1b1a85cf2af33cbd083d6bb8a1" + +[[clips]] +id = "form-10-strike-bw" +path = "clips/carmenta-form/form-10-strike-bw.png" +ground_truth = "clips/carmenta-form/form-10-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "128a2ad5c9864c70fa164a3b17e97da5bbb89c51316c7d3dd2c1a8f8b82a5de7" + +[[clips]] +id = "form-11-none-gray" +path = "clips/carmenta-form/form-11-none-gray.png" +ground_truth = "clips/carmenta-form/form-11-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3b3e50b6652150a0753b2e2358195c5ea71f0cb709496229a356399cdff97cb4" + +[[clips]] +id = "form-11-none-bw" +path = "clips/carmenta-form/form-11-none-bw.png" +ground_truth = "clips/carmenta-form/form-11-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "bc855262f48c0c31a476725bc577306849c1b47e5157a9967130c03835ca8c91" + +[[clips]] +id = "form-11-under-gray" +path = "clips/carmenta-form/form-11-under-gray.png" +ground_truth = "clips/carmenta-form/form-11-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6868c2e6ed3a68c7e7fcbcd35e6e8df07d137f6ddccf4960a1387c93b2621fc5" + +[[clips]] +id = "form-11-under-bw" +path = "clips/carmenta-form/form-11-under-bw.png" +ground_truth = "clips/carmenta-form/form-11-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "020b62f772e4d8bea6cb7e3668c6491ac78d1f7ee87f46b2e5351e2104e8b9e6" + +[[clips]] +id = "form-11-desc-gray" +path = "clips/carmenta-form/form-11-desc-gray.png" +ground_truth = "clips/carmenta-form/form-11-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "14b889185210b7f2f43efa73bc37cb1e106a398ab43d71d264c9935d68a5d2db" + +[[clips]] +id = "form-11-desc-bw" +path = "clips/carmenta-form/form-11-desc-bw.png" +ground_truth = "clips/carmenta-form/form-11-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9554a2a8f6f6b9b991e50a006b49f924dda803ac58d6f1dfd157d42bbb6972d5" + +[[clips]] +id = "form-11-strike-gray" +path = "clips/carmenta-form/form-11-strike-gray.png" +ground_truth = "clips/carmenta-form/form-11-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "77dca60cc62c2e7e78053d5090be3a2658ad2c3c96f934bbe462dfb903d91aed" + +[[clips]] +id = "form-11-strike-bw" +path = "clips/carmenta-form/form-11-strike-bw.png" +ground_truth = "clips/carmenta-form/form-11-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ebfc15daec70f869af59ca2ac63a41d4cfc1ee13c166375ac5fe1dd289911170" + +[[clips]] +id = "form-12-none-gray" +path = "clips/carmenta-form/form-12-none-gray.png" +ground_truth = "clips/carmenta-form/form-12-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2dc345f8ce2c5dc4d58f9fc7c53578f2100094f5df2a99dff52577acef0fac19" + +[[clips]] +id = "form-12-none-bw" +path = "clips/carmenta-form/form-12-none-bw.png" +ground_truth = "clips/carmenta-form/form-12-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c9a8779c14eb4a9c3f75a429f9f4d9cfa44d6853b38130f981d800a7ecd7cffe" + +[[clips]] +id = "form-12-under-gray" +path = "clips/carmenta-form/form-12-under-gray.png" +ground_truth = "clips/carmenta-form/form-12-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b4c0043e48bbe99f1c72d553619b4baac52516e9f58524f07ccb7ee12bb327e7" + +[[clips]] +id = "form-12-under-bw" +path = "clips/carmenta-form/form-12-under-bw.png" +ground_truth = "clips/carmenta-form/form-12-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "e19f431461530471e58a09d340fd253f8d728e9cbc55da63a9fdb80f2e9f41b8" + +[[clips]] +id = "form-12-desc-gray" +path = "clips/carmenta-form/form-12-desc-gray.png" +ground_truth = "clips/carmenta-form/form-12-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "22606226bcd3acebfd1eacbab6c00581e8748fe54ccdce37ff36159bb4b8f213" + +[[clips]] +id = "form-12-desc-bw" +path = "clips/carmenta-form/form-12-desc-bw.png" +ground_truth = "clips/carmenta-form/form-12-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "031bf7b9612624344fad4eec24cf8ac98a35896759a9c5ea33c6ab793d8fa7fc" + +[[clips]] +id = "form-12-strike-gray" +path = "clips/carmenta-form/form-12-strike-gray.png" +ground_truth = "clips/carmenta-form/form-12-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5619607db3d36828ce0b87821030618c886a81f43a8974978dacc55884ffc2eb" + +[[clips]] +id = "form-12-strike-bw" +path = "clips/carmenta-form/form-12-strike-bw.png" +ground_truth = "clips/carmenta-form/form-12-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "483681e3cb00acc00ff52ba73acfb63fb37d5d43c34cc74752494f96137c72ff" + +[[clips]] +id = "form-13-none-gray" +path = "clips/carmenta-form/form-13-none-gray.png" +ground_truth = "clips/carmenta-form/form-13-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "199e761f055919568a1b2dc3b64d22cd045bc5c004267a14a5c8010eead857ec" + +[[clips]] +id = "form-13-none-bw" +path = "clips/carmenta-form/form-13-none-bw.png" +ground_truth = "clips/carmenta-form/form-13-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ee0da72a51b2aa303c3001630706b67643a3aece70b999fe6572cfb94e23717d" + +[[clips]] +id = "form-13-under-gray" +path = "clips/carmenta-form/form-13-under-gray.png" +ground_truth = "clips/carmenta-form/form-13-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0ac5579d8550ab82374535f0e85835b90178d1d6bb713b3487be2b6bc3a57f20" + +[[clips]] +id = "form-13-under-bw" +path = "clips/carmenta-form/form-13-under-bw.png" +ground_truth = "clips/carmenta-form/form-13-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b46deb332d1707447145aed7114b4180911be42045b70c2eee065b0cc8f9ac67" + +[[clips]] +id = "form-13-desc-gray" +path = "clips/carmenta-form/form-13-desc-gray.png" +ground_truth = "clips/carmenta-form/form-13-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2c66f834a3d1f7cc0a93685adf462edca41200fffe70c7a25a39843ff3e3c7f5" + +[[clips]] +id = "form-13-desc-bw" +path = "clips/carmenta-form/form-13-desc-bw.png" +ground_truth = "clips/carmenta-form/form-13-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "031f3f74f2511836b1141818e9e3df06e8e208a6c3e2561616aef4c868e19b87" + +[[clips]] +id = "form-13-strike-gray" +path = "clips/carmenta-form/form-13-strike-gray.png" +ground_truth = "clips/carmenta-form/form-13-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ddceb86a1f208da7d7e8d1b359ac6f8381545c4dd15365e98895dd49d57b7846" + +[[clips]] +id = "form-13-strike-bw" +path = "clips/carmenta-form/form-13-strike-bw.png" +ground_truth = "clips/carmenta-form/form-13-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a1aa9028a140d46f995542f8ac0fcc92e15ebadc28d128e5642cda7f2b653bf6" + +[[clips]] +id = "form-14-none-gray" +path = "clips/carmenta-form/form-14-none-gray.png" +ground_truth = "clips/carmenta-form/form-14-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3a88a882c733539a3a7ef531f4e09980176385c6e50569083a595f2b14a6a2e4" + +[[clips]] +id = "form-14-none-bw" +path = "clips/carmenta-form/form-14-none-bw.png" +ground_truth = "clips/carmenta-form/form-14-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5526e84c1edde9238836dce81e53276828681522a0bc99baab82fed7fc461eb9" + +[[clips]] +id = "form-14-under-gray" +path = "clips/carmenta-form/form-14-under-gray.png" +ground_truth = "clips/carmenta-form/form-14-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7d7e45056dd0c92200706faa6c8c2486fd23f65b18dd4685e17aeec8c0ec020f" + +[[clips]] +id = "form-14-under-bw" +path = "clips/carmenta-form/form-14-under-bw.png" +ground_truth = "clips/carmenta-form/form-14-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "75cb808c7df785d1ea8768998396a5e9a7ed3b8f48aaeb3e996fa95402976a88" + +[[clips]] +id = "form-14-desc-gray" +path = "clips/carmenta-form/form-14-desc-gray.png" +ground_truth = "clips/carmenta-form/form-14-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "72e0b23e25583b58a0022c63aa430f405ba766b9fb452ad6f38e77980c642a73" + +[[clips]] +id = "form-14-desc-bw" +path = "clips/carmenta-form/form-14-desc-bw.png" +ground_truth = "clips/carmenta-form/form-14-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "741b9a6bde7ec841e531382f2737bdff2ebc25ae1e7ebcbed612236df16c43a3" + +[[clips]] +id = "form-14-strike-gray" +path = "clips/carmenta-form/form-14-strike-gray.png" +ground_truth = "clips/carmenta-form/form-14-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "200cff7279541255d987c4fbfdfa65dfd503b6245e8005f20084103e48ba0798" + +[[clips]] +id = "form-14-strike-bw" +path = "clips/carmenta-form/form-14-strike-bw.png" +ground_truth = "clips/carmenta-form/form-14-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "991c236aab477b61a1d46afff660cdd30e1a3fa13b51f56e1d236f5cae65d4fa" + +[[clips]] +id = "form-15-none-gray" +path = "clips/carmenta-form/form-15-none-gray.png" +ground_truth = "clips/carmenta-form/form-15-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "676d17818b474d7981b92bd060ba611772c9fd06db2b0e1b7ae2c622db393c86" + +[[clips]] +id = "form-15-none-bw" +path = "clips/carmenta-form/form-15-none-bw.png" +ground_truth = "clips/carmenta-form/form-15-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9b1ab4ffb3e38c69818d936ece15bb82d29aecf34e3378d1e08380bfd975c99f" + +[[clips]] +id = "form-15-under-gray" +path = "clips/carmenta-form/form-15-under-gray.png" +ground_truth = "clips/carmenta-form/form-15-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "e3b0cf90d9f10be44551024b01890fd0725d2909d67f0cad2ec1b7ff981d0114" + +[[clips]] +id = "form-15-under-bw" +path = "clips/carmenta-form/form-15-under-bw.png" +ground_truth = "clips/carmenta-form/form-15-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "35c1456b610ed4e21529875def322ab24c659d753db836164d12c7d22f073883" + +[[clips]] +id = "form-15-desc-gray" +path = "clips/carmenta-form/form-15-desc-gray.png" +ground_truth = "clips/carmenta-form/form-15-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0d3b2670ad2f926cb948b2fc61790fa9c7243295dacd9fc83c683483b8a4c6cf" + +[[clips]] +id = "form-15-desc-bw" +path = "clips/carmenta-form/form-15-desc-bw.png" +ground_truth = "clips/carmenta-form/form-15-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f1e00177acd3c71d75cb494099a2bb664023d56b48313615a3d6e03fe477f3d9" + +[[clips]] +id = "form-15-strike-gray" +path = "clips/carmenta-form/form-15-strike-gray.png" +ground_truth = "clips/carmenta-form/form-15-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "57e3dd079dd182b4f1a3d56b3797589f3912aaf20fbc65a54b56683ef015c3de" + +[[clips]] +id = "form-15-strike-bw" +path = "clips/carmenta-form/form-15-strike-bw.png" +ground_truth = "clips/carmenta-form/form-15-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "edc04e79734c5721aca3f62d030df1dbafa7e5fd8d7c6dc5b32797157b762939" + +[[clips]] +id = "form-16-none-gray" +path = "clips/carmenta-form/form-16-none-gray.png" +ground_truth = "clips/carmenta-form/form-16-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2a8a393a857d21c1e54d8313446e706f261be6a0bb5bb4dff2b3f7b246b22576" + +[[clips]] +id = "form-16-none-bw" +path = "clips/carmenta-form/form-16-none-bw.png" +ground_truth = "clips/carmenta-form/form-16-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8da6eba2dd75b8f022213fa43cceb6648de395f1647032dc50cd9b875cf3c35a" + +[[clips]] +id = "form-16-under-gray" +path = "clips/carmenta-form/form-16-under-gray.png" +ground_truth = "clips/carmenta-form/form-16-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "76f6f09d90b35893373b139725e81dc02f32b7a361ba56f9599ad7d1e9a98de2" + +[[clips]] +id = "form-16-under-bw" +path = "clips/carmenta-form/form-16-under-bw.png" +ground_truth = "clips/carmenta-form/form-16-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b286550c657afea8d2b36b6f0e75c1bfdc42383cacac50185e30a3ccab066a78" + +[[clips]] +id = "form-16-desc-gray" +path = "clips/carmenta-form/form-16-desc-gray.png" +ground_truth = "clips/carmenta-form/form-16-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "d93ae8f97216839bc471abaf1fa0244e2fb3204f0df642bf2c0613c77aa83bf9" + +[[clips]] +id = "form-16-desc-bw" +path = "clips/carmenta-form/form-16-desc-bw.png" +ground_truth = "clips/carmenta-form/form-16-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "301356832e8e19089c8721442d3e3ad9d6b4e670481107b922de7a734d02b9c4" + +[[clips]] +id = "form-16-strike-gray" +path = "clips/carmenta-form/form-16-strike-gray.png" +ground_truth = "clips/carmenta-form/form-16-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3bea1f0b8c74faa30e1e82437421bacf488a66b3b64e87ed6211e2d779e7b6e2" + +[[clips]] +id = "form-16-strike-bw" +path = "clips/carmenta-form/form-16-strike-bw.png" +ground_truth = "clips/carmenta-form/form-16-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2d8e520c6abdbe5798a7719dfc08e7d3896466e12620536234cfb5a30cea3812" + +[[clips]] +id = "form-17-none-gray" +path = "clips/carmenta-form/form-17-none-gray.png" +ground_truth = "clips/carmenta-form/form-17-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b64cd97b91e7adc4b9a6a3cde29ff674a6a1c495adba8edf003853c304875e55" + +[[clips]] +id = "form-17-none-bw" +path = "clips/carmenta-form/form-17-none-bw.png" +ground_truth = "clips/carmenta-form/form-17-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "698b3ee03c5a077064835294b43c1d92fbde9c4227c2e19ddc1b1b700621ecf9" + +[[clips]] +id = "form-17-under-gray" +path = "clips/carmenta-form/form-17-under-gray.png" +ground_truth = "clips/carmenta-form/form-17-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0ed0b59f7c464025179717fc0d8a9dae057f201c5ab1d71b6a7c87f457b14e80" + +[[clips]] +id = "form-17-under-bw" +path = "clips/carmenta-form/form-17-under-bw.png" +ground_truth = "clips/carmenta-form/form-17-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8f8e65f0e1f94bdd7a52bacbdeffa5efecd99476669cd0ae5e110d3fc24f7ed4" + +[[clips]] +id = "form-17-desc-gray" +path = "clips/carmenta-form/form-17-desc-gray.png" +ground_truth = "clips/carmenta-form/form-17-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "48067b24991a5a58f1470a2e9b2bc92d14ed9b033b0e61d27c05b5ffdb869e5a" + +[[clips]] +id = "form-17-desc-bw" +path = "clips/carmenta-form/form-17-desc-bw.png" +ground_truth = "clips/carmenta-form/form-17-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2db261ad05abb5b9d5c3fa6046571ce1026f3fea07441c4b15fe5fbb669988ea" + +[[clips]] +id = "form-17-strike-gray" +path = "clips/carmenta-form/form-17-strike-gray.png" +ground_truth = "clips/carmenta-form/form-17-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "650416e514ec77d9298fab01549dc67a094a175e30dc0ac506f17c28627d28cb" + +[[clips]] +id = "form-17-strike-bw" +path = "clips/carmenta-form/form-17-strike-bw.png" +ground_truth = "clips/carmenta-form/form-17-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5dc43dcf05b5416984deb68e9cfd7ffa21402e83df1327ed7fd34c0f85bf99c3" + +[[clips]] +id = "form-18-none-gray" +path = "clips/carmenta-form/form-18-none-gray.png" +ground_truth = "clips/carmenta-form/form-18-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "59dbea96d74e979c1454fa6bc034eab24b1f1325d74257250505a0e6e027634e" + +[[clips]] +id = "form-18-none-bw" +path = "clips/carmenta-form/form-18-none-bw.png" +ground_truth = "clips/carmenta-form/form-18-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5dc3be80479d74da4ba5318954656d9bb953b72a20f38498abce16748670fe4b" + +[[clips]] +id = "form-18-under-gray" +path = "clips/carmenta-form/form-18-under-gray.png" +ground_truth = "clips/carmenta-form/form-18-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "4674b487114f422bc92db515d4e81d0a7887d6cb81a7a8ed11776a5a64c36f54" + +[[clips]] +id = "form-18-under-bw" +path = "clips/carmenta-form/form-18-under-bw.png" +ground_truth = "clips/carmenta-form/form-18-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "784a5fe4467782929dfcf5a723182dcf81128c69f870ed7164583b07b04d37cf" + +[[clips]] +id = "form-18-desc-gray" +path = "clips/carmenta-form/form-18-desc-gray.png" +ground_truth = "clips/carmenta-form/form-18-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0219b1f0757ca717e6236ce138afd77ce42fb7e0c5db2410c5cd9c1a202839ae" + +[[clips]] +id = "form-18-desc-bw" +path = "clips/carmenta-form/form-18-desc-bw.png" +ground_truth = "clips/carmenta-form/form-18-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9bb20618046c666fbbb96b72b0e359fb16c34ce563cbbdb82aa307e1f6a498dc" + +[[clips]] +id = "form-18-strike-gray" +path = "clips/carmenta-form/form-18-strike-gray.png" +ground_truth = "clips/carmenta-form/form-18-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1222b174adf84657c328bc672595950d21ca743e7de20059bf0491771d7e5c47" + +[[clips]] +id = "form-18-strike-bw" +path = "clips/carmenta-form/form-18-strike-bw.png" +ground_truth = "clips/carmenta-form/form-18-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "89ef2922617e7c5732a51b993a8815242d16d09ab4d58cbad2e37567f51d7d09" + +[[clips]] +id = "form-19-none-gray" +path = "clips/carmenta-form/form-19-none-gray.png" +ground_truth = "clips/carmenta-form/form-19-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a91e637ab820e4fc997d32c90539d5d78e558f6392e4ed8ef54460d9f58dc2e9" + +[[clips]] +id = "form-19-none-bw" +path = "clips/carmenta-form/form-19-none-bw.png" +ground_truth = "clips/carmenta-form/form-19-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3fc8b6d3b61a1a05c0f792181eec43da44892f2b03388db2afdcca86ac0b73fb" + +[[clips]] +id = "form-19-under-gray" +path = "clips/carmenta-form/form-19-under-gray.png" +ground_truth = "clips/carmenta-form/form-19-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "fce69861fce2560653b00aaf7fdbd04c88e64600df485ac0dd0493c3eeb01d55" + +[[clips]] +id = "form-19-under-bw" +path = "clips/carmenta-form/form-19-under-bw.png" +ground_truth = "clips/carmenta-form/form-19-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5f80e8c99ae914bb620927544b6b5e2ec1a63615ebedab9430815e3430e87f49" + +[[clips]] +id = "form-19-desc-gray" +path = "clips/carmenta-form/form-19-desc-gray.png" +ground_truth = "clips/carmenta-form/form-19-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "624bcfeffe9ce4ae5d8fdd37f437e2b7833db900ab5b3853baebcf8e908d26c4" + +[[clips]] +id = "form-19-desc-bw" +path = "clips/carmenta-form/form-19-desc-bw.png" +ground_truth = "clips/carmenta-form/form-19-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8e703a461e08620a7474530ce2d57c68254bbab3a49379be77ad4ddd2a4c87e9" + +[[clips]] +id = "form-19-strike-gray" +path = "clips/carmenta-form/form-19-strike-gray.png" +ground_truth = "clips/carmenta-form/form-19-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "f225c32b7818cfe711f0e348d60db3fea66e06ae095945d2de21e24f82b402a0" + +[[clips]] +id = "form-19-strike-bw" +path = "clips/carmenta-form/form-19-strike-bw.png" +ground_truth = "clips/carmenta-form/form-19-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6c80a9515705930308c23c5c02a33d9634fee7ce0fcf1b449f201c35a94f1775" + +[[clips]] +id = "form-20-none-gray" +path = "clips/carmenta-form/form-20-none-gray.png" +ground_truth = "clips/carmenta-form/form-20-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c0e88f6c174ff853e4f7a12d42d97fe989649dec96717eb484edfa6a319ecb74" + +[[clips]] +id = "form-20-none-bw" +path = "clips/carmenta-form/form-20-none-bw.png" +ground_truth = "clips/carmenta-form/form-20-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7abc9a705e6a17351d6626f50e73dd7ae201b5360b4ebbe4a3c82d9a4799fe41" + +[[clips]] +id = "form-20-under-gray" +path = "clips/carmenta-form/form-20-under-gray.png" +ground_truth = "clips/carmenta-form/form-20-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "40fd4f08d1855a040c56de6083fbd26f09315ead19055cb5335197507f80b3f7" + +[[clips]] +id = "form-20-under-bw" +path = "clips/carmenta-form/form-20-under-bw.png" +ground_truth = "clips/carmenta-form/form-20-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "df3f80590dda431011d5c7791f19810a0ab1f8c8854bd05f98513d5d59b995aa" + +[[clips]] +id = "form-20-desc-gray" +path = "clips/carmenta-form/form-20-desc-gray.png" +ground_truth = "clips/carmenta-form/form-20-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b138706101c669ad54c54a727cc6864d9d491724a916084f6398a96d6274f100" + +[[clips]] +id = "form-20-desc-bw" +path = "clips/carmenta-form/form-20-desc-bw.png" +ground_truth = "clips/carmenta-form/form-20-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0c714da55c5d35323f5cdaa4422b8f50dddf3955e883199d43fcb89dcfeacf8a" + +[[clips]] +id = "form-20-strike-gray" +path = "clips/carmenta-form/form-20-strike-gray.png" +ground_truth = "clips/carmenta-form/form-20-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5e1169f47aff1097f4291efee0576745d55297364a30528a986b053d8831030f" + +[[clips]] +id = "form-20-strike-bw" +path = "clips/carmenta-form/form-20-strike-bw.png" +ground_truth = "clips/carmenta-form/form-20-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8dc6e09e5b5d49b59102013a4e303c4db945ee278c8dfcc0253be85e3a9757e5" + +[[clips]] +id = "form-21-none-gray" +path = "clips/carmenta-form/form-21-none-gray.png" +ground_truth = "clips/carmenta-form/form-21-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5b9d19266f7d1f59580cba3a4435fa34a77e20390ca085d98ffb03b5298c2433" + +[[clips]] +id = "form-21-none-bw" +path = "clips/carmenta-form/form-21-none-bw.png" +ground_truth = "clips/carmenta-form/form-21-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b3adab76985123acd1f9a8aaea43a1229fff99470ff73d322919d9a8fdc2bdb0" + +[[clips]] +id = "form-21-under-gray" +path = "clips/carmenta-form/form-21-under-gray.png" +ground_truth = "clips/carmenta-form/form-21-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a20c6f59c67a34234ef05e25736718e02af0e3cb92828cb9a951a5c4c17b4477" + +[[clips]] +id = "form-21-under-bw" +path = "clips/carmenta-form/form-21-under-bw.png" +ground_truth = "clips/carmenta-form/form-21-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "02f0cac1bc5a89d8c6dc456fb2250c3f77254615de2ac41f9abab5904e65cc13" + +[[clips]] +id = "form-21-desc-gray" +path = "clips/carmenta-form/form-21-desc-gray.png" +ground_truth = "clips/carmenta-form/form-21-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "367218771d15e01d8fce6547fe322184839b8039c5ee05b2a091fe6fae778817" + +[[clips]] +id = "form-21-desc-bw" +path = "clips/carmenta-form/form-21-desc-bw.png" +ground_truth = "clips/carmenta-form/form-21-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "15cf246c173c9f2d41f1c8e6bf8e68762b33e6909ec19692183e5f5b202cdd9e" + +[[clips]] +id = "form-21-strike-gray" +path = "clips/carmenta-form/form-21-strike-gray.png" +ground_truth = "clips/carmenta-form/form-21-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "da5ce17c2338c38d206741bace5e3f45262ec4073c1e8eb84d37e5de6e4850cb" + +[[clips]] +id = "form-21-strike-bw" +path = "clips/carmenta-form/form-21-strike-bw.png" +ground_truth = "clips/carmenta-form/form-21-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3cd575563a10c8248a706d50d8d9dcef438f50e2c099d3a2ef86abffd4fbf12e" + +[[clips]] +id = "form-22-none-gray" +path = "clips/carmenta-form/form-22-none-gray.png" +ground_truth = "clips/carmenta-form/form-22-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "943dd80668ec8803ede89b5b91fad7adc31fd7c567958eaae7f3a3ed7e8b25b1" + +[[clips]] +id = "form-22-none-bw" +path = "clips/carmenta-form/form-22-none-bw.png" +ground_truth = "clips/carmenta-form/form-22-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "37f4f277b8dd11976750f1dc9ae9eda80e0d1e93ddd570fa12d8f94a3e6a7208" + +[[clips]] +id = "form-22-under-gray" +path = "clips/carmenta-form/form-22-under-gray.png" +ground_truth = "clips/carmenta-form/form-22-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5f935ee2a9142520a8d20fda02d3088b9a822fea268d2701004512571bbe22b1" + +[[clips]] +id = "form-22-under-bw" +path = "clips/carmenta-form/form-22-under-bw.png" +ground_truth = "clips/carmenta-form/form-22-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6d11d5c503748a8dbd67c2bda7cc4506d08c0de31e46591b3beab173c79d3053" + +[[clips]] +id = "form-22-desc-gray" +path = "clips/carmenta-form/form-22-desc-gray.png" +ground_truth = "clips/carmenta-form/form-22-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "89cc20436da9e0ecc064db9f0dbdb2eafb0eef0cb9ba17a36aac0bcced277d83" + +[[clips]] +id = "form-22-desc-bw" +path = "clips/carmenta-form/form-22-desc-bw.png" +ground_truth = "clips/carmenta-form/form-22-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5faea46b0757090b7d63e6a9d1dac1106d1c3c1bb228ffc1c6f416400df16ffa" + +[[clips]] +id = "form-22-strike-gray" +path = "clips/carmenta-form/form-22-strike-gray.png" +ground_truth = "clips/carmenta-form/form-22-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6a2e03a95d16357d2560c90cd314927bb2a7d61dbb081dc9ace5fa0919cf62e4" + +[[clips]] +id = "form-22-strike-bw" +path = "clips/carmenta-form/form-22-strike-bw.png" +ground_truth = "clips/carmenta-form/form-22-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "79e335a3b52962f6eb0d32892801815a641b88de4c4e5ed431cd208691eef38c" + +[[clips]] +id = "form-23-none-gray" +path = "clips/carmenta-form/form-23-none-gray.png" +ground_truth = "clips/carmenta-form/form-23-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0d54db555fc398784ddb53a3807c5d355dc69d5a3712f60338d1cb6c725cf5ed" + +[[clips]] +id = "form-23-none-bw" +path = "clips/carmenta-form/form-23-none-bw.png" +ground_truth = "clips/carmenta-form/form-23-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "ce8709f3ff42467e8dec4309ea366c43aaedc1f14bee7540557d8f34c4689cdd" + +[[clips]] +id = "form-23-under-gray" +path = "clips/carmenta-form/form-23-under-gray.png" +ground_truth = "clips/carmenta-form/form-23-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "52c0049a50416c34124a1e39ceefad8b3ac19698d522250fad62926ce9f4d78f" + +[[clips]] +id = "form-23-under-bw" +path = "clips/carmenta-form/form-23-under-bw.png" +ground_truth = "clips/carmenta-form/form-23-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "d4916aad4d0d1937138aef4b240d70fb41893ae7162008fa0793af4b3cb812c6" + +[[clips]] +id = "form-23-desc-gray" +path = "clips/carmenta-form/form-23-desc-gray.png" +ground_truth = "clips/carmenta-form/form-23-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1e695f67e5f57d9bb784d5a4c8f439ce7fb3fba88acd22f5777746d4fa87062f" + +[[clips]] +id = "form-23-desc-bw" +path = "clips/carmenta-form/form-23-desc-bw.png" +ground_truth = "clips/carmenta-form/form-23-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "34e7992d9b94b270e6a0cb0a6c29773a2f07d66a8778fc4aee5ed4f7222d52a2" + +[[clips]] +id = "form-23-strike-gray" +path = "clips/carmenta-form/form-23-strike-gray.png" +ground_truth = "clips/carmenta-form/form-23-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5974865a3128eca523c373d47f67781db2733e858b79c231c5917bbc55b89ff3" + +[[clips]] +id = "form-23-strike-bw" +path = "clips/carmenta-form/form-23-strike-bw.png" +ground_truth = "clips/carmenta-form/form-23-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "580fdd9c93e4327c21a291f87ee0112e1cd0471d75018b14e652eb10f9730732" + +[[clips]] +id = "form-24-none-gray" +path = "clips/carmenta-form/form-24-none-gray.png" +ground_truth = "clips/carmenta-form/form-24-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "73ff94b56fa25ac1b6f5157fc60585e0797ea729363ea59c3f227e7825c30749" + +[[clips]] +id = "form-24-none-bw" +path = "clips/carmenta-form/form-24-none-bw.png" +ground_truth = "clips/carmenta-form/form-24-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "de8c64f2f6eddaeb6b3c2618dc788ce68f2e3cf1fa3ebdd8af26acbf7985a0b9" + +[[clips]] +id = "form-24-under-gray" +path = "clips/carmenta-form/form-24-under-gray.png" +ground_truth = "clips/carmenta-form/form-24-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "a144f02e45844baca8a8e2c2d903fc7c7000bbb407136a1cbe62df7ba85caa0c" + +[[clips]] +id = "form-24-under-bw" +path = "clips/carmenta-form/form-24-under-bw.png" +ground_truth = "clips/carmenta-form/form-24-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c5914164a705b5c6546ced3242ec7c402c9835724ac46de810680e138f0c286e" + +[[clips]] +id = "form-24-desc-gray" +path = "clips/carmenta-form/form-24-desc-gray.png" +ground_truth = "clips/carmenta-form/form-24-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "d34d3532fc421f3f97b827740256055bd2e18e900d38fe320d0a65850e92cf95" + +[[clips]] +id = "form-24-desc-bw" +path = "clips/carmenta-form/form-24-desc-bw.png" +ground_truth = "clips/carmenta-form/form-24-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c41c1e8fe187da958b7431359a2da928e058f7cacc71fb21f845bfc6179005d1" + +[[clips]] +id = "form-24-strike-gray" +path = "clips/carmenta-form/form-24-strike-gray.png" +ground_truth = "clips/carmenta-form/form-24-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "65f587e36ba0e9c7d4635d376363c95de52abe263a4965d5be524326a269638d" + +[[clips]] +id = "form-24-strike-bw" +path = "clips/carmenta-form/form-24-strike-bw.png" +ground_truth = "clips/carmenta-form/form-24-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "367bb5cc38f6651bbca414bf7d747cda4fb71af9367a6b8c8af9ab0c43093436" + +[[clips]] +id = "form-25-none-gray" +path = "clips/carmenta-form/form-25-none-gray.png" +ground_truth = "clips/carmenta-form/form-25-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "54b635acb75e28a08bcf7daa55b473d1a17b27d8c54b5ee061f31dbf7c944385" + +[[clips]] +id = "form-25-none-bw" +path = "clips/carmenta-form/form-25-none-bw.png" +ground_truth = "clips/carmenta-form/form-25-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "5457c18c080f604871b03dc4316081aecac222bf159b5188a79a234b33ed2379" + +[[clips]] +id = "form-25-under-gray" +path = "clips/carmenta-form/form-25-under-gray.png" +ground_truth = "clips/carmenta-form/form-25-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "32d3a3ec0bd40c3474a145515080987166ab87e8179f55f7f17060fcc3785612" + +[[clips]] +id = "form-25-under-bw" +path = "clips/carmenta-form/form-25-under-bw.png" +ground_truth = "clips/carmenta-form/form-25-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8f1c83554432d373257a4ef3068443256a22652acd00fcf22fa070112f89877b" + +[[clips]] +id = "form-25-desc-gray" +path = "clips/carmenta-form/form-25-desc-gray.png" +ground_truth = "clips/carmenta-form/form-25-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7a7aeaa62e5afed1098a11875be3102f9096904e117cb9d3cfbf74c852e0d65e" + +[[clips]] +id = "form-25-desc-bw" +path = "clips/carmenta-form/form-25-desc-bw.png" +ground_truth = "clips/carmenta-form/form-25-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "661f94a110baeb2f0ba0adff0f9d382e1c06eb4b3286054a138ff601a32640ec" + +[[clips]] +id = "form-25-strike-gray" +path = "clips/carmenta-form/form-25-strike-gray.png" +ground_truth = "clips/carmenta-form/form-25-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6a3ce7c70adc47fc0c5b3ac4245f46bc9f5e8b03bb025627558ed5d902d5a9f0" + +[[clips]] +id = "form-25-strike-bw" +path = "clips/carmenta-form/form-25-strike-bw.png" +ground_truth = "clips/carmenta-form/form-25-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6805eb859d4e02ef55e7d3e2e0797485ad170e8594730fea1d5d0ded1ef2d856" + +[[clips]] +id = "form-26-none-gray" +path = "clips/carmenta-form/form-26-none-gray.png" +ground_truth = "clips/carmenta-form/form-26-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8e64ade86a10f50c64054214a50f10b7d9582d996b83254e22be3b342e01a4bb" + +[[clips]] +id = "form-26-none-bw" +path = "clips/carmenta-form/form-26-none-bw.png" +ground_truth = "clips/carmenta-form/form-26-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "60a4256591be67ce04a5cc7a12f23ab2d3ec8899a9d27f9c98edc272141c5018" + +[[clips]] +id = "form-26-under-gray" +path = "clips/carmenta-form/form-26-under-gray.png" +ground_truth = "clips/carmenta-form/form-26-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b3354da2acce672e879387a7334ec2305c3bae1b6a4f0f1f915dfb3e11cc826c" + +[[clips]] +id = "form-26-under-bw" +path = "clips/carmenta-form/form-26-under-bw.png" +ground_truth = "clips/carmenta-form/form-26-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "0bdca2432f084244656a73a553b659dc98c82c85b9c3efc5573218bd9978b9fc" + +[[clips]] +id = "form-26-desc-gray" +path = "clips/carmenta-form/form-26-desc-gray.png" +ground_truth = "clips/carmenta-form/form-26-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "e00d2aea3918e00c893fc59361a749d7ec2e895e405ad402276162801a8e5ef7" + +[[clips]] +id = "form-26-desc-bw" +path = "clips/carmenta-form/form-26-desc-bw.png" +ground_truth = "clips/carmenta-form/form-26-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "9733cd31a99241b43a45c0010e95284bf416db18ca5b9510910b8473e09775be" + +[[clips]] +id = "form-26-strike-gray" +path = "clips/carmenta-form/form-26-strike-gray.png" +ground_truth = "clips/carmenta-form/form-26-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "470c060da3442d1c1217fcb562087c4f35e41b2c72ee5f8bc514528c21f472f1" + +[[clips]] +id = "form-26-strike-bw" +path = "clips/carmenta-form/form-26-strike-bw.png" +ground_truth = "clips/carmenta-form/form-26-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "48b8996b3943587f0086b14e168a4aa86be36e957e4c7e65005655be906e99a8" + +[[clips]] +id = "form-27-none-gray" +path = "clips/carmenta-form/form-27-none-gray.png" +ground_truth = "clips/carmenta-form/form-27-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "795d614efd70fc48374592cd815bace7d2de82ab1105fe53b5da085b331e3f21" + +[[clips]] +id = "form-27-none-bw" +path = "clips/carmenta-form/form-27-none-bw.png" +ground_truth = "clips/carmenta-form/form-27-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "86fee0090e8d4baf12805a9e6e07bd9fe6e3a01a12d2a3e97f29abd3a845f53b" + +[[clips]] +id = "form-27-under-gray" +path = "clips/carmenta-form/form-27-under-gray.png" +ground_truth = "clips/carmenta-form/form-27-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c3b1f096130d20df9384c0bcd5faced8289d59159a4c1fa831887aa26448e1d8" + +[[clips]] +id = "form-27-under-bw" +path = "clips/carmenta-form/form-27-under-bw.png" +ground_truth = "clips/carmenta-form/form-27-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "1568903e7d49d26acfbca34383d867455d85e214ec72a7509b7cb1200cba1754" + +[[clips]] +id = "form-27-desc-gray" +path = "clips/carmenta-form/form-27-desc-gray.png" +ground_truth = "clips/carmenta-form/form-27-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "55b70fb58e205da86d5cb6426393ce4e157d754d1c00966f8b11bc105b3f885a" + +[[clips]] +id = "form-27-desc-bw" +path = "clips/carmenta-form/form-27-desc-bw.png" +ground_truth = "clips/carmenta-form/form-27-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "21ff3b923c4c8b787654957da283b1af907e54b9c75da7ee0c099364de35e405" + +[[clips]] +id = "form-27-strike-gray" +path = "clips/carmenta-form/form-27-strike-gray.png" +ground_truth = "clips/carmenta-form/form-27-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "09f6fbbf5eb6b3673b75a70b6136382ff011c247154da0700471f9a02e411b8f" + +[[clips]] +id = "form-27-strike-bw" +path = "clips/carmenta-form/form-27-strike-bw.png" +ground_truth = "clips/carmenta-form/form-27-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "99374ecc609edafa7aa84537644d3fdea0aa8caadcfcfebc05ed67d17b44a0ae" + +[[clips]] +id = "form-28-none-gray" +path = "clips/carmenta-form/form-28-none-gray.png" +ground_truth = "clips/carmenta-form/form-28-none-gray.txt" +class = "form-none-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "56fcd87d11fa19f2dfadc342c5a059a90cc9636b032039c5c47f9f93584c4695" + +[[clips]] +id = "form-28-none-bw" +path = "clips/carmenta-form/form-28-none-bw.png" +ground_truth = "clips/carmenta-form/form-28-none-bw.txt" +class = "form-none-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "3660f17fa63d1afe483a2dd4aad02a91d8a36a041918d2b85c5039220cce3493" + +[[clips]] +id = "form-28-under-gray" +path = "clips/carmenta-form/form-28-under-gray.png" +ground_truth = "clips/carmenta-form/form-28-under-gray.txt" +class = "form-under-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "56f10ded930ee02ae7d0ed62443a00ffc20cfb0921c71ec93a06359b7021c5d8" + +[[clips]] +id = "form-28-under-bw" +path = "clips/carmenta-form/form-28-under-bw.png" +ground_truth = "clips/carmenta-form/form-28-under-bw.txt" +class = "form-under-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7a9b905d206cc25448f56a55250c57e959597fd6e2e11737c8a82646a2366e3a" + +[[clips]] +id = "form-28-desc-gray" +path = "clips/carmenta-form/form-28-desc-gray.png" +ground_truth = "clips/carmenta-form/form-28-desc-gray.txt" +class = "form-desc-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "aaaec9b418956017252c72a5464a7b6f9832955d08f29670815726b98d56452e" + +[[clips]] +id = "form-28-desc-bw" +path = "clips/carmenta-form/form-28-desc-bw.png" +ground_truth = "clips/carmenta-form/form-28-desc-bw.txt" +class = "form-desc-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "8fa24ab367524be9e28ae3fd67781a0d5cd8807f1522cac97e0aa8b26c6b7d8e" + +[[clips]] +id = "form-28-strike-gray" +path = "clips/carmenta-form/form-28-strike-gray.png" +ground_truth = "clips/carmenta-form/form-28-strike-gray.txt" +class = "form-strike-gray" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6be93d4d8aab3666d5427628e7893f3844c07bbbe974e2a58ea01072090fd39a" + +[[clips]] +id = "form-28-strike-bw" +path = "clips/carmenta-form/form-28-strike-bw.png" +ground_truth = "clips/carmenta-form/form-28-strike-bw.txt" +class = "form-strike-bw" +split = "train" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2ea54c62d508614e8817c60d560dba094a8053a6ac37deeaea9cee0c9b64592f" + +[[clips]] +id = "form-29-none-gray" +path = "clips/carmenta-form/form-29-none-gray.png" +ground_truth = "clips/carmenta-form/form-29-none-gray.txt" +class = "form-none-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "b791dd456ec3e95f5018e01d7272f19f7ad8feda2407474dd1b5470bb3cc0ef3" + +[[clips]] +id = "form-29-none-bw" +path = "clips/carmenta-form/form-29-none-bw.png" +ground_truth = "clips/carmenta-form/form-29-none-bw.txt" +class = "form-none-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "6232c2c9d6283fa7f3808a9c54e9ff8a23f5ad4400f14d3ae3a02e8b3c7df478" + +[[clips]] +id = "form-29-under-gray" +path = "clips/carmenta-form/form-29-under-gray.png" +ground_truth = "clips/carmenta-form/form-29-under-gray.txt" +class = "form-under-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "2b866455828fe71842e9be47bbb90ae3d58a164f61420a4bff4b364a6504bd2c" + +[[clips]] +id = "form-29-under-bw" +path = "clips/carmenta-form/form-29-under-bw.png" +ground_truth = "clips/carmenta-form/form-29-under-bw.txt" +class = "form-under-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "7bf3e896601ef35de7c764830ce210eddb0a2931e5bccd485fffe2c16cdcfb80" + +[[clips]] +id = "form-29-desc-gray" +path = "clips/carmenta-form/form-29-desc-gray.png" +ground_truth = "clips/carmenta-form/form-29-desc-gray.txt" +class = "form-desc-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "df30da2458af8b7a1d125b3189c9c2c08d9b8334ea440c07d133ad7e4b9def90" + +[[clips]] +id = "form-29-desc-bw" +path = "clips/carmenta-form/form-29-desc-bw.png" +ground_truth = "clips/carmenta-form/form-29-desc-bw.txt" +class = "form-desc-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c8f3321d4393c9f0850f91d7f4f1cb2ed5cb5d19b9b5178b5324b8ed5508e6d4" + +[[clips]] +id = "form-29-strike-gray" +path = "clips/carmenta-form/form-29-strike-gray.png" +ground_truth = "clips/carmenta-form/form-29-strike-gray.txt" +class = "form-strike-gray" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "609e84d1f8e0009fc22eedfaefb94d16d2ac698cecac9b355ed67754a11137d8" + +[[clips]] +id = "form-29-strike-bw" +path = "clips/carmenta-form/form-29-strike-bw.png" +ground_truth = "clips/carmenta-form/form-29-strike-bw.txt" +class = "form-strike-bw" +split = "holdout" +license = "CC0-1.0 (render of invented values in Windows system fonts)" +sha256 = "c3d1e8239b5330a2e4a4c79363e041970ebb9cabef38b80373169997f66d05c4" diff --git a/crates/ffai-argus-wasm/Cargo.toml b/crates/ffai-argus-wasm/Cargo.toml index c2236f00..0f520ff4 100644 --- a/crates/ffai-argus-wasm/Cargo.toml +++ b/crates/ffai-argus-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-argus-wasm" description = "Argus in the browser — pure-Rust SmolVLM image captioning compiled to WebAssembly, no ONNX runtime, no Python" -version = "0.1.1" +version = "0.1.2" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -25,7 +25,7 @@ ffai-core = { workspace = true } # **0.7.2, not 0.7.1** — see the note in `ffai-mercury-wasm`. `SmolVlm::from_bytes` # and `ArgusBytes` first exist in 0.7.2, and a tarball verification resolves # this from crates.io rather than from the path. -ffai-argus = { path = "../ffai-argus", version = "0.7.3", default-features = false } +ffai-argus = { path = "../ffai-argus", version = "0.7.4", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-argus/Cargo.toml b/crates/ffai-argus/Cargo.toml index 9cb94914..b9873509 100644 --- a/crates/ffai-argus/Cargo.toml +++ b/crates/ffai-argus/Cargo.toml @@ -19,7 +19,7 @@ ffai-core = { workspace = true } # their own weights; they must not inherit fetch just because Argus is linked. # Not `workspace = true`: cargo forbids overriding default-features on an # inherited dep. -ffai-models = { path = "../ffai-models", version = "0.6.4", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.6.5", default-features = false } # The tokenizer is the ENGINE's business now, not just the step-4 test's: the # assembled prompt has to become ids and the generated ids have to become text. tokenizers = { workspace = true } diff --git a/crates/ffai-bench/Cargo.toml b/crates/ffai-bench/Cargo.toml index c0dbc4df..dbf81b3e 100644 --- a/crates/ffai-bench/Cargo.toml +++ b/crates/ffai-bench/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai's analyzer — one call to compare any engine against world # the runner records a clip failure and continues instead of aborting the # whole corpus, and decode is wrapped so a panicking dependency costs one # page rather than the run. -version = "0.7.5" +version = "0.7.6" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-bench/src/metrics.rs b/crates/ffai-bench/src/metrics.rs index 9310cfbe..c371405c 100644 --- a/crates/ffai-bench/src/metrics.rs +++ b/crates/ffai-bench/src/metrics.rs @@ -46,6 +46,63 @@ pub fn cer_with(reference: &str, hypothesis: &str, mode: Mode) -> f64 { error_rate(&r, &h) } +/// Digit errors and reference digits, for a DIGIT ERROR RATE — the number a +/// financial-forms buyer asks for first (docs/plans/commercial-gaps.md, +/// gap 3). A page CER of 5 % says nothing about whether "$150,000" came back +/// right, and one wrong digit makes the value worthless. +/// +/// Both strings are normalized under `mode` and aligned by a minimal-edit +/// Levenshtein alignment. Counted as digit errors: +/// * a substitution where EITHER side is a digit — a digit misread +/// (`1` -> `T`) and a phantom digit (`$` -> `8`, which turns "$10,000" +/// into "810,000") both corrupt the number; +/// * a reference digit deleted; +/// * a hypothesis digit inserted where the reference had nothing. +/// +/// Returns `(errors, reference_digits)`, so a corpus total is summed +/// (micro-averaged) rather than a mean of per-page rates dominated by pages +/// with two digits. +#[must_use] +pub fn digit_errors_with(reference: &str, hypothesis: &str, mode: Mode) -> (usize, usize) { + let truth: Vec = normalize(reference, mode).chars().collect(); + let read: Vec = normalize(hypothesis, mode).chars().collect(); + let digits = truth.iter().filter(|c| c.is_ascii_digit()).count(); + // Full DP table, then walk back one minimal alignment. + let mut table = vec![vec![0usize; read.len() + 1]; truth.len() + 1]; + for (row_idx, row) in table.iter_mut().enumerate() { + row[0] = row_idx; + } + for (col_idx, cell) in table[0].iter_mut().enumerate() { + *cell = col_idx; + } + for ti in 1..=truth.len() { + for ri in 1..=read.len() { + table[ti][ri] = (table[ti - 1][ri - 1] + usize::from(truth[ti - 1] != read[ri - 1])) + .min(table[ti - 1][ri] + 1) + .min(table[ti][ri - 1] + 1); + } + } + let (mut ti, mut ri, mut errors) = (truth.len(), read.len(), 0usize); + while ti > 0 || ri > 0 { + let diagonal = ti > 0 + && ri > 0 + && table[ti][ri] == table[ti - 1][ri - 1] + usize::from(truth[ti - 1] != read[ri - 1]); + if diagonal { + let (t, r) = (truth[ti - 1], read[ri - 1]); + errors += usize::from(t != r && (t.is_ascii_digit() || r.is_ascii_digit())); + ti -= 1; + ri -= 1; + } else if ti > 0 && table[ti][ri] == table[ti - 1][ri] + 1 { + errors += usize::from(truth[ti - 1].is_ascii_digit()); // deleted + ti -= 1; + } else { + errors += usize::from(read[ri - 1].is_ascii_digit()); // inserted + ri -= 1; + } + } + (errors, digits) +} + /// Levenshtein distance / reference length. Empty reference: 0.0 if the /// hypothesis is empty too, else 1.0 per hypothesis token (capped at 1.0 by /// convention would hide gross over-generation, so we don't cap). @@ -108,6 +165,35 @@ mod tests { assert!((c - 1.0 / 6.0).abs() < 1e-12); } + /// The field misreads that motivated the metric: every one is a digit + /// error, and letter errors elsewhere on the line are not. + #[test] + fn digit_errors_count_only_digits() { + let m = Mode::Ocr; + assert_eq!( + digit_errors_with("$10,000", "810,000", m), + (1, 5), + "$ -> 8 invents a digit" + ); + assert_eq!(digit_errors_with("1117 Oak", "TTT7 Oak", m), (3, 4)); + assert_eq!( + digit_errors_with("#305 Main", "#30S Mian", m), + (1, 3), + "a letter swap is not counted" + ); + assert_eq!( + digit_errors_with("Unit 7", "Unit 77", m), + (1, 1), + "an invented digit counts" + ); + assert_eq!( + digit_errors_with("Total 42", "Total", m), + (2, 2), + "dropped digits count" + ); + assert_eq!(digit_errors_with("no digits", "n0 digits", m), (1, 0)); + } + #[test] fn empty_reference_with_output_is_penalized() { assert!(wer("", "spurious words") > 0.0); diff --git a/crates/ffai-bench/src/runner.rs b/crates/ffai-bench/src/runner.rs index a99ee569..9f2b3ce8 100644 --- a/crates/ffai-bench/src/runner.rs +++ b/crates/ffai-bench/src/runner.rs @@ -19,7 +19,19 @@ use ffai_core::registry::EngineRegistry; use crate::corpus::{ClipEntry, Manifest}; use crate::gate::{GateKind, GateOutcome, GateReport, GateResult}; use crate::ledger::{BenchRecord, Environment, LEDGER_SCHEMA, RunSummary, append}; -use crate::metrics::{cer_with, wer_with}; +use crate::metrics::{cer_with, digit_errors_with, wer_with}; + +/// The digit error rate as a ledger note, micro-averaged over the corpus +/// (docs/plans/commercial-gaps.md gap 3). `None` for a corpus with no digits +/// in its ground truth, where the rate is undefined rather than zero. +fn digit_note(errors: usize, digits: usize) -> Option { + (digits > 0).then(|| { + format!( + "digit error rate {:.2} % over {digits} reference digits", + 100.0 * errors as f64 / digits as f64 + ) + }) +} use crate::normalize::Mode; use crate::reference::ReferenceSpec; use crate::speed::{best_of_n, real_time_factor}; @@ -997,16 +1009,23 @@ fn run_ocr_engine( stats.best_secs + summary.load_secs.unwrap_or(0.0), )); let (mut wers, mut cers) = (Vec::new(), Vec::new()); + let (mut digit_errors, mut digits) = (0usize, 0usize); for (id, text) in &texts { if let Some(clip) = holdout.iter().find(|c| &c.id == id) && let Some(truth) = manifest.ground_truth(clip)? { wers.push(wer_with(&truth, text, Mode::Ocr)); cers.push(cer_with(&truth, text, Mode::Ocr)); + let (e, n) = digit_errors_with(&truth, text, Mode::Ocr); + digit_errors += e; + digits += n; } } summary.wer = mean(&wers); summary.cer = mean(&cers); + if let Some(note) = digit_note(digit_errors, digits) { + summary.notes.push(note); + } } // FRONT, not back. The correctness gate reports `notes.first()` as the // "first failure", and by this point `notes` already holds @@ -1100,6 +1119,7 @@ fn run_reference( }; let (mut wers, mut cers) = (Vec::new(), Vec::new()); + let (mut digit_errors, mut digits) = (0usize, 0usize); for (clip, path) in holdout.iter().zip(paths) { match batch.text_for(path) { Some(hypothesis) => { @@ -1107,6 +1127,9 @@ fn run_reference( if let Some(truth) = manifest.ground_truth(clip)? { wers.push(wer_with(&truth, hypothesis, mode)); cers.push(cer_with(&truth, hypothesis, mode)); + let (e, n) = digit_errors_with(&truth, hypothesis, mode); + digit_errors += e; + digits += n; } } None => summary @@ -1117,6 +1140,13 @@ fn run_reference( summary.wer = mean(&wers); summary.cer = mean(&cers); + // OCR only: the reference arm scores digits exactly as the engine arm does, + // so the two digit rates are comparable. ASR ledger lines stay unchanged. + if matches!(mode, Mode::Ocr) + && let Some(note) = digit_note(digit_errors, digits) + { + summary.notes.push(note); + } if let Some(unit) = per_item_unit { let mut times: Vec = batch diff --git a/crates/ffai-carmenta-wasm/Cargo.toml b/crates/ffai-carmenta-wasm/Cargo.toml index 159b176a..a9a4c7d2 100644 --- a/crates/ffai-carmenta-wasm/Cargo.toml +++ b/crates/ffai-carmenta-wasm/Cargo.toml @@ -5,7 +5,7 @@ description = "Carmenta in the browser — pure-Rust OCR compiled to WebAssembly # slice. The JS signature is unchanged — it is still a `Uint8Array` — but the # Rust signature is, so this is a breaking change for a Rust consumer even # though a browser sees nothing. Adds `setDetMaxShort` and `linearMemoryBytes`. -version = "0.2.1" +version = "0.2.2" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -26,7 +26,7 @@ ffai-core = { workspace = true } # release calls `reqwest::blocking` — a module that does not exist on wasm32. # Nothing in a browser can fetch weights through the model cache anyway: the # caller supplies bytes. -ffai-carmenta = { path = "../ffai-carmenta", version = "0.10.1", default-features = false } +ffai-carmenta = { path = "../ffai-carmenta", version = "0.11.0", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/ffai-carmenta/Cargo.toml b/crates/ffai-carmenta/Cargo.toml index 47c86459..268b84eb 100644 --- a/crates/ffai-carmenta/Cargo.toml +++ b/crates/ffai-carmenta/Cargo.toml @@ -38,7 +38,7 @@ description = "Carmenta — pure-Rust OCR: documents, screens, and a change-gate # (cord +0.0033 with a +0.1928 clip, doc +0.0395 with a +0.2920 clip) — the same # failure the 544x960 arm produced, smaller. 1280 is the only arm that improves # both with no badly-regressing clip. The table lives on `DET_MAX_SHORT_DEFAULT`. -version = "0.10.1" +version = "0.11.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-carmenta/README.md b/crates/ffai-carmenta/README.md index 81125efd..fd80ff56 100644 --- a/crates/ffai-carmenta/README.md +++ b/crates/ffai-carmenta/README.md @@ -39,8 +39,8 @@ cargo add ffai-carmenta ffai-core ffai-media ```toml [dependencies] -ffai-carmenta = "0.9" -ffai-core = "0.6" +ffai-carmenta = "0.11" +ffai-core = "0.8" ffai-media = "0.6" ``` @@ -101,12 +101,21 @@ which is the one thing between a default install and the number above. Everything else in it is on by default as of 0.9.0. Ahead of published pipelines on reading order's worst rows, well behind the -VLM leaders on text — and the gap is *characterised*, not guessed. On the -236-page English holdout with no tables or formulas (content the pipeline -fully represents) Carmenta reads **0.041**, between PaddleOCR-VL's 0.033 -(a 3B-class model on GPU) and PP-StructureV3's 0.079 (the full Paddle -pipeline). Where the rest of the loss lives is measured block by block from -the evaluator's own match records, and it reconstructs the headline exactly: +VLM leaders on text — and the gap is *characterised*, not guessed. + +**Correction, 2026-09-25.** Earlier versions of this page said that on the +236-page English holdout with no tables or formulas, Carmenta reads **0.041**, +between PaddleOCR-VL's 0.033 and PP-StructureV3's 0.079. **That was wrong: 0.041 +is Unlimited-OCR's score on that holdout**, misattributed to Carmenta in the +campaign log (§44). Carmenta's measured score on the same 236 pages was +**0.108 text / 0.175 reading order**. That was measured through the official +evaluator (§8.173), before the later campaign wins that took the full corpus +from 0.128 to 0.116. The subset has not been re-scored since, so no newer +number is claimed for it. The "near the VLM leader on content we can represent" +reading that followed from the wrong number is withdrawn with it. + +Where the remaining loss lives is measured block by block from the evaluator's +own match records, and it reconstructs the full-corpus headline exactly: | remaining loss | share | |---|---:| @@ -116,13 +125,61 @@ the evaluator's own match records, and it reconstructs the headline exactly: | recognition substitutions | 20 % | | detection misses · sequence errors · harness | 10 % | -So this is a **coverage and character** problem, not a layout one — the +So the remaining loss is mostly **coverage and characters**, not layout: the reading-order and detection stages this crate spent its campaign on are down to a combined 4 % of remaining error. All of this on CPU at ~9–17 s/page — roughly 9× the reference VLM's measured throughput on the same machine, from megabytes of weights against gigabytes. +## Forms, scanned filings and PDFs (0.11) + +Built after a real caller read state financial-disclosure filings and got +confident wrong values back. The full record, including what did not work, is +[`docs/plans/commercial-gaps.md`](https://github.com/Remade-With-Rust/FFAI/blob/master/docs/plans/commercial-gaps.md). +**Every number below is on synthetic corpora with ground truth by +construction** (`tools/carmenta_form_corpus.py`, +`tools/carmenta_checkbox_corpus.py`), or on OmniDocBench. None is yet a claim +about real filings. + +**Use `mobiledet-svtr` for forms.** 240 invented form values, rendered with no +rule and with three kinds of form rule, share read exactly: + +| condition | `mobiledet-crnn` | `mobiledet-svtr` | +|---|---:|---:| +| no rule | 70 % | 93 % | +| value on an underline | 13-20 % | **93 %** | +| rule through the descenders | 47-50 % | **97 %** | +| struck through | 7-10 % | **83-87 %** | +| values read wrong, of 240 | 154 | **19** | +| ...of which still passed a 0.85 confidence gate | 128 | 19 | + +What else 0.11 adds, each measured: + +| capability | API | measured | +|---|---|---| +| orientation, 0/90/180/270 | `OcrOptions::auto_orient`, `detect_orientation` | **144 / 144** turns undone on document pages and single form lines; a decision costs **48 %** of one page read | +| checkboxes, frame + marked/empty | `checkbox::find` | **100 %** recall and **100 %** state on 960 synthetic boxes (stroked and glyph boxes; X, overflowing tick, fill). Known false positives: the square ideograph `口`, and small square plot markers | +| label → value pairs | `forms::pair_fields` | **220 / 221** correctly-read values paired with their label | +| restrict a field's characters | `OcrOptions::charset` | forced characters show in the confidence: a prose page read as digits drops from 0.918 to 0.781 mean | +| snap to known values | `lexicon::Lexicon` | every recorded field misread snaps to its entry ("Trish HTlI" → Irish Hill); ties are reported, not guessed | +| erase form rules | `OcrOptions::remove_rules` | CRNN on underlined values: 13-20 % → **70 %**. **Not for SVTR**: on strikethroughs it goes 77-87 % → 73 %. Opt-in | +| full-width → ASCII on Latin lines | on by default (`FFAI_NO_FOLD=1` disables) | OmniDocBench, 316 pages: fires on 35 of 34 852 lines; **5 pages better, 1 worse** (a full-width comma in the ground truth itself) | +| offline weights, fail fast | `CraftCrnn::offline(rec, det, dir)`, `preload()` | no cache, no network; a missing file names its exact path | + +Two measured non-results, recorded so nobody has to re-run them: + +- **Carmenta's output is deterministic.** The same 120-line page, read three + times across two engine instances, gives identical text and boxes and a + confidence drift of **0**. +- **Reporting a line's weakest character instead of its mean confidence is + not a better gate.** AUROC goes 0.746 → 0.764 on SVTR and 0.873 → 0.828 on + CRNN; the sign flips by engine. The mean stays. + +PDFs are read by [`ffai-media`](https://crates.io/crates/ffai-media)'s `pdf` +feature, which returns each page **as a viewer shows it**: redaction patches +and boxes drawn over a scan are painted in, never read through. + ## LIVE: point it at a screen `LiveSession` wraps **any** `OcrEngine` and adds what a stream needs: a change @@ -190,7 +247,8 @@ turns it off. **Real photographs remain the open front.** On photographed receipts the full pipeline still trails PaddleOCR (20.9 % vs 15.6 % CER) even though our recognition stage beats theirs on identical crops — the diagnosed cause is -tilt-sensitive line grouping, with deskew as the named fix. +tilt-sensitive line grouping, with deskew as the named fix. `auto_orient` +handles quarter turns, not small tilt: deskew is still open. ## Status: `experimental`, honestly diff --git a/crates/ffai-carmenta/examples/checkbox_bench.rs b/crates/ffai-carmenta/examples/checkbox_bench.rs new file mode 100644 index 00000000..72f858db --- /dev/null +++ b/crates/ffai-carmenta/examples/checkbox_bench.rs @@ -0,0 +1,167 @@ +//! Checkbox bench over `carmenta-checkbox-v1` (docs/plans/commercial-gaps.md +//! gaps 4 and 14): are the boxes found, is anything else taken for a box, and +//! is each box's marked/empty state right? +//! +//! ```text +//! python tools/carmenta_checkbox_corpus.py +//! cargo run --release -p ffai-carmenta --example checkbox_bench -- [--misses] +//! ``` +//! +//! A found box matches a ground-truth box at IoU >= 0.5. Precision counts +//! found boxes that match nothing (letters, ideographs, table cells) as false +//! positives; recall counts ground-truth boxes nothing matched. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use ffai_carmenta::checkbox::{CheckboxParams, find}; +use ffai_core::types::BoundingBox; + +fn iou(a: &BoundingBox, b: [f32; 4]) -> f32 { + let (bx, by, bw, bh) = (b[0], b[1], b[2], b[3]); + let ix = (a.x + a.width).min(bx + bw) - a.x.max(bx); + let iy = (a.y + a.height).min(by + bh) - a.y.max(by); + if ix <= 0.0 || iy <= 0.0 { + return 0.0; + } + let inter = ix * iy; + inter / (a.width * a.height + bw * bh - inter) +} + +#[derive(Default)] +struct Tally { + gt: usize, + matched: usize, + state_right: usize, +} + +fn main() { + let misses = std::env::args().any(|a| a == "--misses"); + let root = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpora/clips/carmenta-checkbox"); + let mut metas: Vec = std::fs::read_dir(&root) + .expect("corpus: run tools/carmenta_checkbox_corpus.py") + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "json")) + .collect(); + metas.sort(); + + let mut by_ink: BTreeMap = BTreeMap::new(); + let mut by_mark: BTreeMap = BTreeMap::new(); + let (mut found_total, mut false_pos) = (0usize, 0usize); + for m in &metas { + let meta: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(m).unwrap()).unwrap(); + let ink = meta["ink"].as_str().unwrap().to_string(); + let img = ffai_media::load_image(&m.with_extension("png")).unwrap(); + let found = find(&img, CheckboxParams::default()); + found_total += found.len(); + let gts: Vec<(BoundingBox, [f32; 4], bool, String)> = meta["boxes"] + .as_array() + .unwrap() + .iter() + .map(|b| { + let v: Vec = b["bbox"] + .as_array() + .unwrap() + .iter() + .map(|x| x.as_f64().unwrap() as f32) + .collect(); + let bb = BoundingBox { + x: v[0], + y: v[1], + width: v[2], + height: v[3], + }; + let mark = b["mark"].as_str().unwrap().to_string(); + let glyph = b["glyph"].as_bool().unwrap_or(false); + ( + bb, + [v[0], v[1], v[2], v[3]], + b["checked"].as_bool().unwrap(), + format!("{mark}{}", if glyph { "/glyph" } else { "" }), + ) + }) + .collect(); + let mut used = vec![false; found.len()]; + for (_, frame, checked, mark) in >s { + let best = found + .iter() + .enumerate() + .filter(|(i, _)| !used[*i]) + .map(|(i, f)| (i, iou(&f.bbox, *frame))) + .max_by(|a, b| a.1.total_cmp(&b.1)); + for t in [ + by_ink.entry(ink.clone()).or_default(), + by_mark.entry(mark.clone()).or_default(), + ] { + t.gt += 1; + } + match best { + Some((i, v)) if v >= 0.5 => { + used[i] = true; + let right = found[i].checked == *checked; + for t in [ + by_ink.get_mut(&ink).unwrap(), + by_mark.get_mut(mark).unwrap(), + ] { + t.matched += 1; + t.state_right += usize::from(right); + } + if misses && !right { + eprintln!( + "STATE {} {mark}: truth {checked}, read {} (fill {:.3})", + m.file_stem().unwrap().to_string_lossy(), + found[i].checked, + found[i].fill + ); + } + } + _ => { + if misses { + eprintln!( + "MISSED {} {mark} at {frame:?}", + m.file_stem().unwrap().to_string_lossy() + ); + } + } + } + } + for (i, f) in found.iter().enumerate() { + if !used[i] { + false_pos += 1; + if misses { + eprintln!( + "FALSE {} at ({:.0},{:.0}) {:.0}x{:.0} checked {}", + m.file_stem().unwrap().to_string_lossy(), + f.bbox.x, + f.bbox.y, + f.bbox.width, + f.bbox.height, + f.checked + ); + } + } + } + } + let print = |title: &str, t: &BTreeMap| { + println!( + "\n{title:<14} {:>5} {:>8} {:>12}", + "boxes", "recall", "state right" + ); + for (k, v) in t { + println!( + "{k:<14} {:>5} {:>7.1}% {:>11.1}%", + v.gt, + 100.0 * v.matched as f32 / v.gt.max(1) as f32, + 100.0 * v.state_right as f32 / v.matched.max(1) as f32 + ); + } + }; + print("ink", &by_ink); + print("mark", &by_mark); + println!( + "\nfound {found_total}, false positives {false_pos} (precision {:.1}%)", + 100.0 * (found_total - false_pos) as f32 / found_total.max(1) as f32 + ); +} diff --git a/crates/ffai-carmenta/examples/fold_census.rs b/crates/ffai-carmenta/examples/fold_census.rs new file mode 100644 index 00000000..73dd68b1 --- /dev/null +++ b/crates/ffai-carmenta/examples/fold_census.rs @@ -0,0 +1,128 @@ +//! Census for the full-width fold (`ffai_carmenta::fold`): on real +//! documents, how many lines would it change, what are they, and does page +//! CER against ground truth move? +//! +//! ```text +//! FFAI_NO_FOLD=1 cargo run --release -p ffai-carmenta --example fold_census -- [--pages 316] +//! ``` +//! +//! The engine runs with the fold OFF (the env var), and the fold is applied +//! here, offline, to every line — so one pass yields both arms, exactly +//! paired. CER is our whitespace-collapsed edit distance over the page's +//! ground-truth text: a direction signal, not the official evaluator. + +use std::path::PathBuf; + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_carmenta::fold::fold_latin_fullwidth; +use ffai_core::engine::{OcrEngine, OcrOptions}; + +fn edit(a: &[char], b: &[char]) -> usize { + let mut prev: Vec = (0..=b.len()).collect(); + for (i, ca) in a.iter().enumerate() { + let mut cur = vec![i + 1; b.len() + 1]; + for (j, cb) in b.iter().enumerate() { + cur[j + 1] = (prev[j] + usize::from(ca != cb)) + .min(prev[j + 1] + 1) + .min(cur[j] + 1); + } + prev = cur; + } + prev[b.len()] +} + +fn norm(s: &str) -> Vec { + s.split_whitespace() + .collect::>() + .join(" ") + .chars() + .collect() +} + +fn main() { + assert!( + std::env::var_os("FFAI_NO_FOLD").is_some(), + "run with FFAI_NO_FOLD=1 so the engine's own fold is off and this census sees raw output" + ); + let args: Vec = std::env::args().skip(1).collect(); + let n: usize = args + .iter() + .position(|a| a == "--pages") + .and_then(|i| args.get(i + 1)) + .and_then(|v| v.parse().ok()) + .unwrap_or(316); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let e = CraftCrnn::variant_in(RecStage::Svtr, DetStage::MobileDet, &root.join("models")); + let dir = root.join("corpora/clips/carmenta-omnidoc"); + let mut pages: Vec = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "png")) + .collect(); + pages.sort(); + pages.truncate(n); + + let (mut lines_total, mut lines_changed, mut pages_changed) = (0usize, 0usize, 0usize); + let (mut better, mut worse, mut same) = (0usize, 0usize, 0usize); + let (mut edits_off, mut edits_on, mut chars) = (0usize, 0usize, 0usize); + for p in &pages { + let Ok(gt) = std::fs::read_to_string(p.with_extension("txt")) else { + continue; + }; + let img = ffai_media::load_image(p).unwrap(); + let out = e.recognize(&img, &OcrOptions::default()).unwrap(); + let raw: Vec = out + .blocks + .iter() + .flat_map(|b| &b.lines) + .map(|l| l.text.clone()) + .collect(); + let mut changed_here = 0; + let folded: Vec = raw + .iter() + .map(|l| { + lines_total += 1; + fold_latin_fullwidth(l).map_or_else( + || l.clone(), + |f| { + changed_here += 1; + println!( + "{}: {l:?} -> {f:?}", + p.file_name().unwrap().to_string_lossy() + ); + f + }, + ) + }) + .collect(); + lines_changed += changed_here; + let g = norm(>); + let off = edit(&norm(&raw.join("\n")), &g); + let on = if changed_here > 0 { + edit(&norm(&folded.join("\n")), &g) + } else { + off + }; + edits_off += off; + edits_on += on; + chars += g.len(); + if changed_here > 0 { + pages_changed += 1; + match on.cmp(&off) { + std::cmp::Ordering::Less => better += 1, + std::cmp::Ordering::Greater => worse += 1, + std::cmp::Ordering::Equal => same += 1, + } + } + } + println!( + "\n{} pages, {lines_total} lines: fold fires on {lines_changed} lines across {pages_changed} pages", + pages.len() + ); + println!("changed pages: {better} better, {worse} worse, {same} unchanged in CER"); + println!( + "corpus CER: fold off {:.4} % fold on {:.4} %", + 100.0 * edits_off as f64 / chars.max(1) as f64, + 100.0 * edits_on as f64 / chars.max(1) as f64 + ); +} diff --git a/crates/ffai-carmenta/examples/form_bench.rs b/crates/ffai-carmenta/examples/form_bench.rs new file mode 100644 index 00000000..24a4db50 --- /dev/null +++ b/crates/ffai-carmenta/examples/form_bench.rs @@ -0,0 +1,226 @@ +//! Form-value bench over `carmenta-form-v1` (docs/plans/commercial-gaps.md, +//! gap 12): how often a typed value on a ruled line is read EXACTLY, and how +//! often a WRONG value still passes a caller's confidence gate. +//! +//! ```text +//! cargo run --release -p ffai-carmenta --example form_bench -- \ +//! [--engine mobiledet-crnn|mobiledet-svtr] [--split train|holdout|all] \ +//! [--gate 0.85] [--remove-rules] [--charset-by-kind] +//! ``` +//! +//! Scoring is by VALUE, because a value is what a caller stores: +//! * `exact`: the value string appears, whitespace-collapsed, in the output. +//! * `CER`: edit distance from the value to the output line nearest to it, +//! over the value's length. +//! * `wrong>gate`: the value was not read exactly AND the line carrying it +//! passed the confidence gate. This is the count that embarrassed a caller. +//! +//! Every rule condition is paired with its `none` control on the same value, +//! font and size, so a difference between rows is the rule's doing. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_core::engine::{OcrEngine, OcrOptions}; + +/// Smallest edit distance from `value` to any substring of `text` (the +/// value's own span inside a line that also carries the label). +fn substring_distance(value: &[char], text: &[char]) -> usize { + let mut prev = vec![0usize; text.len() + 1]; + for (i, cv) in value.iter().enumerate() { + let mut cur = vec![i + 1; text.len() + 1]; + for (j, ct) in text.iter().enumerate() { + cur[j + 1] = (prev[j] + usize::from(cv != ct)) + .min(prev[j + 1] + 1) + .min(cur[j] + 1); + } + prev = cur; + } + prev.into_iter().min().unwrap_or(value.len()) +} + +fn collapse(s: &str) -> String { + s.split_whitespace().collect::>().join(" ") +} + +#[derive(Default)] +struct Row { + n: usize, + exact: usize, + edits: usize, + chars: usize, + wrong_passed: usize, + conf_sum: f32, + /// `forms::pair_fields` returned this clip's label with its exact value. + paired: usize, +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let get = |k: &str| { + args.iter() + .position(|a| a == k) + .and_then(|i| args.get(i + 1)) + .cloned() + }; + let has = |k: &str| args.iter().any(|a| a == k); + let engine = get("--engine").unwrap_or_else(|| "mobiledet-crnn".into()); + let split = get("--split").unwrap_or_else(|| "all".into()); + let gate: f32 = get("--gate").and_then(|g| g.parse().ok()).unwrap_or(0.85); + let remove_rules = has("--remove-rules"); + let charset_by_kind = has("--charset-by-kind"); + let misses = has("--misses"); + + let rec = match engine.as_str() { + "mobiledet-svtr" => RecStage::Svtr, + "mobiledet-crnn" => RecStage::Crnn, + other => panic!("unsupported engine {other}"), + }; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let e = CraftCrnn::variant_in(rec, DetStage::MobileDet, &root.join("models")); + + let clips = root.join("corpora/clips/carmenta-form"); + let mut metas: Vec = std::fs::read_dir(&clips) + .expect("corpus: run tools/carmenta_form_corpus.py") + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "json")) + .collect(); + metas.sort(); + + // (confidence of the value's line, read exactly?) for every clip: the + // calibration question is how well the first predicts the second. + let mut scored: Vec<(f32, bool)> = Vec::new(); + let mut rows: BTreeMap<(String, String), Row> = BTreeMap::new(); + let mut kinds: BTreeMap<(String, String), Row> = BTreeMap::new(); + for meta_path in &metas { + let meta: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(meta_path).unwrap()).unwrap(); + let idx = meta["index"].as_u64().unwrap(); + let in_split = match split.as_str() { + "train" => idx % 2 == 0, + "holdout" => idx % 2 == 1, + _ => true, + }; + if !in_split { + continue; + } + let value = meta["value"].as_str().unwrap().to_string(); + let kind = meta["kind"].as_str().unwrap().to_string(); + let rule = meta["rule"].as_str().unwrap().to_string(); + let ink = meta["ink"].as_str().unwrap().to_string(); + let img = ffai_media::load_image(&meta_path.with_extension("png")).unwrap(); + + // The per-kind charset a form reader would pass for this field. + let charset = charset_by_kind.then(|| match kind.as_str() { + "amount" => "0123456789$,.- to".to_string(), + _ => String::new(), + }); + let opts = OcrOptions { + charset: charset.filter(|c| !c.is_empty()), + remove_rules, + ..OcrOptions::default() + }; + let out = e + .recognize(&img, &opts) + .unwrap_or_else(|err| panic!("{}: {err}", meta_path.display())); + + let v: Vec = value.chars().collect(); + let lines: Vec<(String, Option)> = out + .blocks + .iter() + .flat_map(|b| &b.lines) + .map(|l| (collapse(&l.text), l.confidence)) + .collect(); + let best = lines + .iter() + .map(|(t, c)| (substring_distance(&v, &t.chars().collect::>()), *c)) + .min_by_key(|(d, _)| *d); + let (dist, conf) = best.unwrap_or((v.len(), None)); + let exact = collapse( + &lines + .iter() + .map(|(t, _)| t.as_str()) + .collect::>() + .join(" "), + ) + .contains(&collapse(&value)); + scored.push((conf.unwrap_or(0.0), exact)); + // Field pairing: the clip's label (without its colon) must come back + // with the clip's exact value. + let label = meta["label"].as_str().unwrap().trim_end_matches(':').to_string(); + let paired = ffai_carmenta::forms::pair_fields(&out) + .iter() + .any(|f| f.label == label && collapse(&f.value) == collapse(&value)); + if misses && !exact { + let got = lines + .iter() + .min_by_key(|(t, _)| substring_distance(&v, &t.chars().collect::>())) + .map_or(String::new(), |(t, c)| { + format!("{t:?} conf {:.3}", c.unwrap_or(0.0)) + }); + eprintln!("MISS {rule}/{ink} {value:?} -> {got}"); + } + for row in [ + rows.entry((rule.clone(), ink.clone())).or_default(), + kinds.entry((rule.clone(), kind.clone())).or_default(), + ] { + row.n += 1; + row.exact += usize::from(exact); + row.edits += dist; + row.chars += v.len(); + row.conf_sum += conf.unwrap_or(0.0); + row.wrong_passed += usize::from(!exact && conf.is_some_and(|c| c >= gate)); + row.paired += usize::from(paired); + } + } + + println!( + "engine {engine} | split {split} | gate {gate} | remove_rules {remove_rules} | charset_by_kind {charset_by_kind}" + ); + let print = |title: &str, table: &BTreeMap<(String, String), Row>| { + println!( + "\n{title:<16} {:>4} {:>8} {:>8} {:>10} {:>9} {:>8}", + "n", "exact", "CER", "wrong>gate", "mean conf", "paired" + ); + for ((a, b), r) in table { + println!( + "{:<16} {:>4} {:>7.1}% {:>7.2}% {:>10} {:>9.3} {:>7.1}%", + format!("{a}/{b}"), + r.n, + 100.0 * r.exact as f32 / r.n.max(1) as f32, + 100.0 * r.edits as f32 / r.chars.max(1) as f32, + r.wrong_passed, + r.conf_sum / r.n.max(1) as f32, + 100.0 * r.paired as f32 / r.n.max(1) as f32 + ); + } + }; + print("rule/ink", &rows); + print("rule/kind", &kinds); + + // Calibration: AUROC = P(a right value's confidence > a wrong value's), + // ties counted half. 1.0 separates perfectly; 0.5 is a coin. + let right: Vec = scored.iter().filter(|s| s.1).map(|s| s.0).collect(); + let wrong: Vec = scored.iter().filter(|s| !s.1).map(|s| s.0).collect(); + let mut wins = 0f64; + for r in &right { + for w in &wrong { + wins += if r > w { + 1.0 + } else if (r - w).abs() < f32::EPSILON { + 0.5 + } else { + 0.0 + }; + } + } + let auroc = wins / (right.len() * wrong.len()).max(1) as f64; + let kept_right = right.iter().filter(|&&c| c >= gate).count(); + let kept_wrong = wrong.iter().filter(|&&c| c >= gate).count(); + println!( + "\ncalibration: AUROC {auroc:.3} | at gate {gate}: kept {kept_right}/{} right, {kept_wrong}/{} wrong", + right.len(), + wrong.len() + ); +} diff --git a/crates/ffai-carmenta/examples/orient_bench.rs b/crates/ffai-carmenta/examples/orient_bench.rs new file mode 100644 index 00000000..d5eddc95 --- /dev/null +++ b/crates/ffai-carmenta/examples/orient_bench.rs @@ -0,0 +1,125 @@ +//! Orientation bench (docs/plans/commercial-gaps.md, gap 5): turn upright +//! pages by a KNOWN quarter turn and check `detect_orientation` returns the +//! turn that undoes it. Ground truth by construction, no labelling. +//! +//! ```text +//! cargo run --release -p ffai-carmenta --example orient_bench -- \ +//! [--engine mobiledet-svtr|mobiledet-crnn] [--pages 24] [--forms 12] +//! ``` +//! +//! Sources: OmniDocBench pages (dense documents) and `carmenta-form-v1` +//! control clips (a single sparse form line, the hard case for an axis vote). +//! Also prints the cost of a decision against one full-page read. + +use std::path::PathBuf; +use std::time::Instant; + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_carmenta::orient::rotate; +use ffai_core::engine::{OcrEngine, OcrOptions}; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let get = |k: &str| { + args.iter() + .position(|a| a == k) + .and_then(|i| args.get(i + 1)) + .cloned() + }; + let engine = get("--engine").unwrap_or_else(|| "mobiledet-svtr".into()); + let n_pages: usize = get("--pages").and_then(|v| v.parse().ok()).unwrap_or(24); + let n_forms: usize = get("--forms").and_then(|v| v.parse().ok()).unwrap_or(12); + let rec = match engine.as_str() { + "mobiledet-svtr" => RecStage::Svtr, + "mobiledet-crnn" => RecStage::Crnn, + other => panic!("unsupported engine {other}"), + }; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let e = CraftCrnn::variant_in(rec, DetStage::MobileDet, &root.join("models")); + e.preload().expect("weights"); + + let list = |dir: &str, filter: &dyn Fn(&str) -> bool, n: usize| -> Vec { + let mut v: Vec = std::fs::read_dir(root.join(dir)) + .map(|d| d.filter_map(|e| e.ok().map(|e| e.path())).collect()) + .unwrap_or_default(); + v.retain(|p| { + p.extension().is_some_and(|x| x == "png") + && filter(p.file_name().unwrap().to_str().unwrap()) + }); + v.sort(); + v.truncate(n); + v + }; + let sources = [ + ( + "omnidoc", + list("corpora/clips/carmenta-omnidoc", &|_| true, n_pages), + ), + ( + "form", + list( + "corpora/clips/carmenta-form", + &|f| f.contains("-none-gray"), + n_forms, + ), + ), + ]; + + println!("engine {engine}"); + println!( + "{:<8} {:>5} {:>8} {:>8} {:>8} {:>8} {:>9}", + "source", "n", "t=0", "t=90", "t=180", "t=270", "margin" + ); + for (name, pages) in &sources { + let mut correct = [0usize; 4]; + let mut margins = Vec::new(); + let (mut t_orient, mut t_full, mut reads) = (0f64, 0f64, 0usize); + for p in pages { + let upright = ffai_media::load_image(p).unwrap(); + let t0 = Instant::now(); + let _ = e.recognize(&upright, &OcrOptions::default()).unwrap(); + t_full += t0.elapsed().as_secs_f64(); + for t in 0..4u8 { + let turned = rotate(&upright, t); + let t1 = Instant::now(); + let o = e.detect_orientation(&turned).unwrap(); + t_orient += t1.elapsed().as_secs_f64(); + reads += 1; + let want = (4 - t) % 4; + if o.turns == want { + correct[t as usize] += 1; + } else { + eprintln!( + "MISS {} turned {}: got {} (h {:.0} v {:.0} conf {:.3} vs {:.3}, n {})", + p.file_name().unwrap().to_string_lossy(), + u32::from(t) * 90, + u32::from(o.turns) * 90, + o.horizontal_area, + o.vertical_area, + o.chosen_confidence, + o.other_confidence, + o.sampled + ); + } + margins.push(o.chosen_confidence - o.other_confidence); + } + } + let n = pages.len(); + margins.sort_by(f32::total_cmp); + println!( + "{name:<8} {n:>5} {:>7}/{n} {:>6}/{n} {:>6}/{n} {:>6}/{n} {:>9.3}", + correct[0], + correct[1], + correct[2], + correct[3], + margins.first().copied().unwrap_or(0.0) + ); + println!( + " decision {:.2} s vs full read {:.2} s ({:.0} % of a read)", + t_orient / reads.max(1) as f64, + t_full / n.max(1) as f64, + 100.0 * (t_orient / reads.max(1) as f64) / (t_full / n.max(1) as f64).max(1e-9) + ); + } + println!("\nmargin = smallest (chosen - other) confidence gap across all decisions"); +} diff --git a/crates/ffai-carmenta/models/craft-mlt.toml b/crates/ffai-carmenta/models/craft-mlt.toml new file mode 100644 index 00000000..e5dbc319 --- /dev/null +++ b/crates/ffai-carmenta/models/craft-mlt.toml @@ -0,0 +1,20 @@ +# CRAFT text detection (Baek et al. 2019), weights `craft_mlt_25k`. +# +# License: MIT (clovaai/CRAFT-pytorch). The AUTHORS' official weight links +# are Google Drive (click-gated — disqualified as a fetch source, plan +# §7.1); the clean ungated host is JaidedAI/EasyOCR's GitHub release, which +# is where these bytes come from. +# +# The safetensors file is converted locally from the released .pth by +# `tools/carmenta_m_c1_prepare.py` (strips the DataParallel `module.` +# prefix, verified under load_state_dict(strict=True)). A hosted safetensors +# mirror so end users skip the torch conversion is open work tracked for +# Carmenta `stable` (M-C6). +name = "craft-mlt" +task = "ocr" +description = "CRAFT text detector (VGG16-BN U-Net, region+affinity maps) — craft_mlt_25k" +license = "MIT" + +[[files]] +name = "craft.safetensors" +sha256 = "131c7db96edeb7ae49c4ce31f43261aab4e80006eb124ef8fbb134755fc3f604" diff --git a/crates/ffai-carmenta/models/crnn-english-g2.toml b/crates/ffai-carmenta/models/crnn-english-g2.toml new file mode 100644 index 00000000..b924f2de --- /dev/null +++ b/crates/ffai-carmenta/models/crnn-english-g2.toml @@ -0,0 +1,17 @@ +# EasyOCR generation-2 English recognizer (`english_g2`): VGG feature +# extractor + 2x bidirectional LSTM + CTC head, 96-char mixed-case charset +# (digits, punctuation, upper AND lower case — the property that put it in +# this seat; see the M-C1 log's trocr pivot). +# +# License: Apache-2.0 (JaidedAI/EasyOCR); weights fetch ungated from the +# EasyOCR GitHub releases. Converted locally from english_g2.pth by +# `tools/carmenta_crnn_prepare.py`, strict-load verified. Hosted safetensors +# mirror tracked for M-C6, as with craft-mlt. +name = "crnn-english-g2" +task = "ocr" +description = "CRNN recognizer english_g2 (VGG + BiLSTM + CTC, 96-char mixed-case) — EasyOCR lineage" +license = "Apache-2.0" + +[[files]] +name = "crnn.safetensors" +sha256 = "577ff8c53d62ee03a99ded316d45140b84cacad182f1dbbd9fd72eb965278a0b" diff --git a/crates/ffai-carmenta/models/crnn-zh-sim-g2.toml b/crates/ffai-carmenta/models/crnn-zh-sim-g2.toml new file mode 100644 index 00000000..9f257148 --- /dev/null +++ b/crates/ffai-carmenta/models/crnn-zh-sim-g2.toml @@ -0,0 +1,40 @@ +# EasyOCR generation-2 Simplified-Chinese recognizer (`zh_sim_g2`): the SAME +# network as `crnn-english-g2` — VGG feature extractor + 2x bidirectional LSTM +# + CTC head, 44 tensors under identical names — differing in exactly one place: +# +# Prediction.weight en (97, 256) -> zh (6719, 256) +# Prediction.bias en (97,) -> zh (6719,) +# +# So `crnn.rs` runs it unchanged; what differs is the head width and the +# charset. That charset is 6 718 characters — 95 ASCII, 6 614 CJK, 9 other — +# and INCLUDES the full Latin range, so this model reads English pages too. +# +# OPT-IN, NOT THE DEFAULT. `FFAI_REC_LANG=zh` selects it; unset stays English. +# §8.142 measured 11 holdout pages of 236 as 51 % of the competitive gap against +# Unlimited-OCR purely because they contain Chinese and a 96-class CTC head +# cannot emit a character it has no class for. But a 6 719-class head is +# normally weaker on Latin than a 97-class specialist, and trading English CER +# for CJK coverage is a bad bargain until measured — so English remains the +# default AND the oracle, the shape `--features mimalloc` already uses. +# +# License: Apache-2.0 (JaidedAI/EasyOCR); weights fetch ungated from the EasyOCR +# v1.3 release (`zh_sim_g2.zip`). Converted locally by +# `tools/carmenta_crnn_zh_prepare.py`, strict-load verified, and the charset is +# read from the installed easyocr package rather than retyped — an off-by-one +# there shifts EVERY decoded character, which is §8.113's failure in a form far +# harder to spot, so the tool refuses unless charset+1 == head classes. +name = "crnn-zh-sim-g2" +task = "ocr" +description = "CRNN recognizer zh_sim_g2 (VGG + BiLSTM + CTC, 6718-char ASCII+CJK) — EasyOCR lineage, opt-in via FFAI_REC_LANG=zh" +license = "Apache-2.0" + +[[files]] +name = "crnn.safetensors" +sha256 = "4644161d0c78fb76ceadf0658f8020779b20a007dbf19afee93e120975f83dd2" + +# The charset ships beside the weights rather than compiled in: 6 718 characters +# is 20 KB of UTF-8, and a generated `const` that large is a merge hazard for no +# benefit. `charset_for` reads it and the loader sizes the head from its length. +[[files]] +name = "charset.txt" +sha256 = "ea93e357dff635633aea8e467194b5f73abd2fd302366657d4ec45f909d4343a" diff --git a/crates/ffai-carmenta/models/parseq-tiny.toml b/crates/ffai-carmenta/models/parseq-tiny.toml new file mode 100644 index 00000000..da273d6d --- /dev/null +++ b/crates/ffai-carmenta/models/parseq-tiny.toml @@ -0,0 +1,16 @@ +# PARSeq-tiny (baudm/parseq, Apache-2.0): the audit-cleared successor +# recognition lineage (plan §7.1) — 94-char mixed-case charset, 24 MB, +# fetched ungated from the parseq GitHub v1.0.0 release. +# +# Status: weights converted and shape-mapped (tools/carmenta_parseq_prepare.py, +# corpora/refs/fixtures/parseq_tiny_shapes.json); the candle port + forward +# oracle are the open M-C1 quality-gate brick. Until the port lands, this +# manifest documents provenance for the weights already in the cache. +name = "parseq-tiny" +task = "ocr" +description = "PARSeq-tiny scene-text recognizer (ViT + AR decoder, 94-char mixed-case) — quality-gate successor to crnn-english-g2" +license = "Apache-2.0" + +[[files]] +name = "parseq-tiny.safetensors" +sha256 = "72d6f2dbc0716af1967cb62d6ad2e01e4525c521a59e3e5b7cb7a413d73c6701" diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml new file mode 100644 index 00000000..bba4797e --- /dev/null +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-det.toml @@ -0,0 +1,30 @@ +# PP-OCRv5 mobile text detector (DBNet / PP-LCNetV3, Apache-2.0, plan §7.1 +# audit) — the speed-margin lineage for LIVE. Converted from the paddlex +# cache by tools/carmenta_mobiledet_prepare.py (234 tensors, shape map in +# corpora/refs/fixtures/ppocrv5_mobile_det_shapes.json). Candle port = the +# open campaign brick. +name = "ppocrv5-mobile-det" +task = "ocr" +description = "PP-OCRv5 mobile det (DBNet/PP-LCNetV3) — staged detector lineage" +license = "Apache-2.0" + +[[files]] +name = "det.safetensors" +sha256 = "abe92a954293370d2dc078bceae0c316f6c3bd02542c84ff8be166e7b797c97d" + +# Training checkpoint (STRUCTURAL names: backbone.blocksN / neck / head — +# the export-fused inference program has only op-indexed names, which +# blocks a direct port). Source for the fusion below; not loaded at runtime. +[[files]] +name = "det-train.safetensors" +sha256 = "d7188218e5666327c0006ed149a2305da6cedae15132a19c0e657c45fb9df052" + +# What the candle port actually loads. `tools/carmenta_mobiledet_fuse.py` +# collapses every LearnableRepLayer branch and every BatchNorm into plain +# convolutions offline — 905 tensors down to 168 — so `mobiledet.rs` sees +# nothing but convs. The fusion is not self-checked: it is rejected unless it +# reproduces paddle's own exported program (currently 7.3e-04 max logit +# error, 100% binarised agreement at the 0.3 threshold). +[[files]] +name = "det-fused.safetensors" +sha256 = "7aa0fe33ac821b4df1c26c216a715a8e82f0cbf8df42321a959a25fb944a8d6d" diff --git a/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml new file mode 100644 index 00000000..c9b07bb1 --- /dev/null +++ b/crates/ffai-carmenta/models/ppocrv5-mobile-rec.toml @@ -0,0 +1,40 @@ +# PP-OCRv5 mobile text recognizer (SVTR) — the MATCHED HALF of the detector we +# already ship. Today we pair a 2024 PP-OCRv5 detector with EasyOCR's 2019 +# VGG-CRNN; §8.165 measured the mismatch with detection held constant and only +# the recognizer swapped: +# +# 6 hard stylized pages CRNN 44.13 % -> SVTR 40.12 % (-4.01 pp) +# 3 clean controls CRNN 3.20 % -> SVTR 2.66 % (-0.53 pp) +# +# It wins on BOTH halves, which refutes the risk this model opened with: an +# 18,383-class multilingual head does not confuse English against a 97-class +# English-only one. It reads it better. +# +# THE DOCUMENT DEFAULT since §8.171 (`ffai_carmenta::DOC_DEFAULT` = +# `mobiledet-svtr`): the 236-page engine A/B returned +1.521 pp macro, CI +# [+1.068, +2.016], at 1.91x the wall time. It was opt-in until then, which is +# what this comment used to say. LIVE still resolves the registry default +# (`craft-crnn`) until it has its own live_bench/live_soak verdicts. +# +# Converted from the paddlex cache by `tools/carmenta_svtr_prepare.py` +# (234 tensors). The candle port in `svtr.rs` is pinned by +# `examples/svtr_oracle.rs` against the recorded fixture: max abs diff 8.857e-5, +# argmax agreement 100 % (§8.168). +name = "ppocrv5-mobile-rec" +task = "ocr" +description = "PP-OCRv5 mobile rec (SVTR, 18385-class CTC) — opt-in via engine name mobiledet-svtr" +license = "Apache-2.0" + +[[files]] +name = "rec.safetensors" +sha256 = "04da812bb245cb92ea384e8d15899628d85b243b48e44755e9dc6d4296c874d1" + +# Head order: index 0 is the implicit CTC blank, lines 1..18384 are these +# entries — PaddleOCR's label list is [blank] + dict + [space]. The prepare tool +# READS THE FILE BACK and refuses unless it holds 18,384 entries ending in a +# space: an earlier edit meant to append that space silently did not apply, and +# an off-by-one here shifts every decoded character (§8.113's failure mode, in a +# form that only shows up as subtly wrong text). +[[files]] +name = "charset.txt" +sha256 = "9a5b0e675766282fa1042f6ff7e153b11a3433cf29c053412d27438f11af1920" diff --git a/crates/ffai-carmenta/src/checkbox.rs b/crates/ffai-carmenta/src/checkbox.rs new file mode 100644 index 00000000..e953d5b3 --- /dev/null +++ b/crates/ffai-carmenta/src/checkbox.rs @@ -0,0 +1,336 @@ +//! Checkboxes on forms: where they are and whether they are marked +//! (docs/plans/commercial-gaps.md, gaps 4 and 14). +//! +//! Read as text, a checkbox comes back as a character — `@`, `0`, `D`, `B` — +//! at the confidence of text, so a caller cannot tell a ticked box from a +//! letter, and a form whose one value field is a set of ticks is unreadable. +//! This finds the boxes directly, from the pixels, with no model: +//! +//! 1. **A frame is four sides.** A candidate starts at a horizontal dark run +//! of checkbox size (the top edge). It is kept when a bottom edge sits +//! roughly one side-length below it and the left and right edges are each +//! at least [`SIDE_COVERAGE`] covered. Testing the frame's SIDES, rather +//! than a connected blob's outline, is what keeps a tick that overflows +//! the box from hiding the box. +//! 2. **Marked is ink inside.** The interior, inset past the frame's own +//! thickness, is marked when its dark fraction reaches [`MARK_FILL`] — an +//! X, a tick and a solid fill all qualify; an empty box does not. +//! +//! Size is bounded (`min_side..=max_side`), so a table's cells and a page's +//! border are not checkboxes. Square character grids (one-letter-per-box +//! fields) will be found as boxes; a caller can drop them by position. + +use ffai_core::types::{BoundingBox, ImageBuffer}; + +/// Fraction of a frame side that must be dark for the side to count. +pub const SIDE_COVERAGE: f32 = 0.85; +/// Dark fraction of the interior at which a box counts as marked. +pub const MARK_FILL: f32 = 0.06; +/// Dark fraction of each corner square for the corner to count as sharp. +pub const CORNER_FILL: f32 = 0.5; +/// Largest dark fraction allowed along the line just OUTSIDE each side. +pub const OUTSIDE_INK: f32 = 0.25; + +/// One checkbox. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Checkbox { + /// The frame, in image pixels. + pub bbox: BoundingBox, + /// Whether the interior carries a mark. + pub checked: bool, + /// Dark fraction of the interior, the evidence for `checked`. + pub fill: f32, +} + +/// Size bounds for [`find`], in pixels. +#[derive(Debug, Clone, Copy)] +pub struct CheckboxParams { + pub min_side: usize, + pub max_side: usize, +} + +impl Default for CheckboxParams { + /// 10-150 px: a 3-12 mm box at 300 ppi, down to a small box at 150 ppi. + fn default() -> Self { + Self { + min_side: 10, + max_side: 150, + } + } +} + +/// Every checkbox on the page, top to bottom, left to right. +#[must_use] +pub fn find(img: &ImageBuffer, p: CheckboxParams) -> Vec { + let (w, h) = (img.width as usize, img.height as usize); + if w == 0 || h == 0 { + return Vec::new(); + } + let lum = crate::rules::luma(img); + let (t, bg) = crate::rules::otsu(&lum); + if bg <= t { + return Vec::new(); + } + let dark: Vec = lum.iter().map(|&v| v <= t).collect(); + let at = |x: usize, y: usize| dark[y * w + x]; + + // Fraction of the row segment [x0, x1) at `y` that is dark. + let row_cov = |y: usize, x0: usize, x1: usize| { + (x0..x1).filter(|&x| at(x, y)).count() as f32 / (x1 - x0).max(1) as f32 + }; + // A vertical side at columns [c0, c1) over rows [y0, y1): the fraction of + // rows with any dark pixel in the band (a side has thickness). + let col_cov = |c0: usize, c1: usize, y0: usize, y1: usize| { + (y0..y1).filter(|&y| (c0..c1).any(|x| at(x, y))).count() as f32 / (y1 - y0).max(1) as f32 + }; + + let mut found: Vec = Vec::new(); + for y in 0..h { + let mut x = 0; + while x < w { + if !at(x, y) { + x += 1; + continue; + } + let x0 = x; + while x < w && at(x, y) { + x += 1; + } + let x1 = x; + let side = x1 - x0; + if side < p.min_side || side > p.max_side { + continue; + } + // Already inside a found box: a thick top edge produces a run on + // each of its rows. + if found.iter().any(|b| contains(&b.bbox, x0, y)) { + continue; + } + if let Some(cb) = frame_at(x0, x1, y, w, h, &row_cov, &col_cov, &at) { + found.push(cb); + } + } + } + found +} + +/// Is there a square frame whose top edge is the run `[x0, x1)` at row `y`? +#[allow(clippy::too_many_arguments)] +fn frame_at( + x0: usize, + x1: usize, + y: usize, + w: usize, + h: usize, + row_cov: &dyn Fn(usize, usize, usize) -> f32, + col_cov: &dyn Fn(usize, usize, usize, usize) -> f32, + at: &dyn Fn(usize, usize) -> bool, +) -> Option { + let side = x1 - x0; + // Bottom edge: a well-covered row between 0.8 and 1.25 side-lengths down. + let lo = y + side * 4 / 5; + let hi = (y + side * 5 / 4).min(h - 1); + if lo >= h { + return None; + } + let bottom = (lo..=hi).find(|&yy| row_cov(yy, x0, x1) >= SIDE_COVERAGE)?; + // The bottom edge may be several rows thick: take its last row. + let mut y1 = bottom; + while y1 + 1 < h && y1 + 1 <= hi && row_cov(y1 + 1, x0, x1) >= SIDE_COVERAGE { + y1 += 1; + } + let height = y1 + 1 - y; + // A mark that overflows the box (a tick flicked past its top-right + // corner) joins the top edge and stretches the run past the frame. The + // frame's right side is the rightmost column that is a full vertical + // line over the frame's height: trim back to it. + let full = |c: usize| col_cov(c, c + 1, y, y1 + 1) >= SIDE_COVERAGE; + let x1 = (x0 + side * 2 / 3..x1) + .rev() + .find(|&c| full(c)) + .map_or(x1, |c| c + 1); + let side = x1 - x0; + // Side thickness allowance: a 2-4 px pen at 300 ppi, proportionally less + // on a small box. Taken from the TRIMMED side. + let band = (side / 8).clamp(2, 6); + // Near-square only: a wide rectangle is a field, not a box. + if (height as f32) < side as f32 * 0.8 || (height as f32) > side as f32 * 1.25 { + return None; + } + let left = col_cov(x0, (x0 + band).min(w), y, y1 + 1); + let right = col_cov(x1.saturating_sub(band), x1, y, y1 + 1); + if left < SIDE_COVERAGE || right < SIDE_COVERAGE { + return None; + } + // A printed box has SHARP corners; `o`, `e`, `a`, `0`, `8` have rounded + // ones, and `D` and `B` round on the right. Each corner's band x band + // square must be mostly ink — an L of two frame sides meeting. + let corner = |cx: usize, cy: usize| { + let (cx0, cy0) = ( + cx.min(w.saturating_sub(band)), + cy.min(h.saturating_sub(band)), + ); + let dark = (cy0..cy0 + band) + .flat_map(|yy| (cx0..cx0 + band).map(move |xx| (xx, yy))) + .filter(|&(xx, yy)| at(xx, yy)) + .count(); + dark as f32 / (band * band) as f32 >= CORNER_FILL + }; + let (xr, yb) = (x1.saturating_sub(band), (y1 + 1).saturating_sub(band)); + if !(corner(x0, y) && corner(xr, y) && corner(x0, yb) && corner(xr, yb)) { + return None; + } + // ...but a run taken from a round letter's own contour ALSO has dark ends, + // so the corner test alone passes `o`, `d`, `g`, `R`. What a letter cannot + // fake is CLEAR SPACE just outside the frame: a box's top edge is its + // topmost ink and its sides its outermost, while a curve bulges past any + // run cut from it. Measured over the middle of each side, so a tick + // overflowing one corner does not disqualify its box. + let clear = |dark: usize, n: usize| (dark as f32) <= OUTSIDE_INK * n.max(1) as f32; + let (my0, my1) = (y + band, (y1 + 1).saturating_sub(band)); + let (mx0, mx1) = (x0 + band, x1.saturating_sub(band)); + let outside_left = x0 == 0 + || clear( + (my0..my1).filter(|&yy| at(x0 - 1, yy)).count(), + my1.saturating_sub(my0), + ); + let outside_right = x1 >= w + || clear( + (my0..my1).filter(|&yy| at(x1, yy)).count(), + my1.saturating_sub(my0), + ); + let outside_top = y == 0 + || clear( + (mx0..mx1).filter(|&xx| at(xx, y - 1)).count(), + mx1.saturating_sub(mx0), + ); + let outside_bottom = y1 + 1 >= h + || clear( + (mx0..mx1).filter(|&xx| at(xx, y1 + 1)).count(), + mx1.saturating_sub(mx0), + ); + if !(outside_left && outside_right && outside_top && outside_bottom) { + return None; + } + // Interior, inset past the frame. Measure the frame's real thickness from + // the top edge so a heavy pen does not read as a mark. + let mut thick = 1; + while thick < band * 2 && y + thick < y1 && row_cov(y + thick, x0, x1) >= SIDE_COVERAGE { + thick += 1; + } + let inset = thick + 2; + let (ix0, ix1, iy0, iy1) = ( + x0 + inset, + x1.saturating_sub(inset), + y + inset, + (y1 + 1).saturating_sub(inset), + ); + if ix1 <= ix0 || iy1 <= iy0 { + return None; + } + let interior = (ix1 - ix0) * (iy1 - iy0); + let ink = (iy0..iy1) + .map(|yy| (ix0..ix1).filter(|&xx| at(xx, yy)).count()) + .sum::(); + let fill = ink as f32 / interior as f32; + Some(Checkbox { + bbox: BoundingBox { + x: x0 as f32, + y: y as f32, + width: side as f32, + height: height as f32, + }, + checked: fill >= MARK_FILL, + fill, + }) +} + +fn contains(b: &BoundingBox, x: usize, y: usize) -> bool { + let (x, y) = (x as f32, y as f32); + x >= b.x - 2.0 && x <= b.x + b.width + 2.0 && y >= b.y - 2.0 && y <= b.y + b.height + 2.0 +} + +#[cfg(test)] +mod tests { + use super::*; + use ffai_core::types::PixelFormat; + + struct Page { + w: usize, + d: Vec, + } + impl Page { + fn new(w: usize, h: usize) -> Self { + Self { + w, + d: vec![255; w * h], + } + } + fn fill(&mut self, x0: usize, y0: usize, x1: usize, y1: usize) { + for y in y0..y1 { + self.d[y * self.w + x0..y * self.w + x1].fill(0); + } + } + fn frame(&mut self, x: usize, y: usize, s: usize, t: usize) { + self.fill(x, y, x + s, y + t); + self.fill(x, y + s - t, x + s, y + s); + self.fill(x, y, x + t, y + s); + self.fill(x + s - t, y, x + s, y + s); + } + fn img(self) -> ImageBuffer { + let h = self.d.len() / self.w; + ImageBuffer { + width: self.w as u32, + height: h as u32, + format: PixelFormat::Gray8, + data: self.d, + } + } + } + + #[test] + fn an_empty_and_a_marked_box_are_found_and_told_apart() { + let mut p = Page::new(300, 100); + p.frame(20, 20, 40, 3); // empty + p.frame(120, 20, 40, 3); + for i in 0..28 { + // an X inside the second + p.fill(126 + i, 26 + i, 128 + i, 28 + i); + p.fill(152 - i, 26 + i, 154 - i, 28 + i); + } + let found = find(&p.img(), CheckboxParams::default()); + assert_eq!(found.len(), 2, "{found:?}"); + assert!(!found[0].checked, "{:?}", found[0]); + assert!(found[1].checked, "{:?}", found[1]); + } + + #[test] + fn a_tick_overflowing_the_box_does_not_hide_it() { + let mut p = Page::new(200, 120); + p.frame(40, 40, 40, 3); + // a stroke from inside the box up and out past its top-right corner + for i in 0..50 { + p.fill(50 + i, 70 - i, 53 + i, 73 - i); + } + let found = find(&p.img(), CheckboxParams::default()); + assert_eq!(found.len(), 1, "{found:?}"); + assert!(found[0].checked); + } + + #[test] + fn letters_rules_and_wide_fields_are_not_boxes() { + let mut p = Page::new(400, 120); + p.fill(10, 100, 390, 103); // an underline rule + p.fill(20, 20, 60, 24); // a 'T' top bar... + p.fill(38, 20, 42, 60); // ...and its stem + p.frame(100, 20, 40, 3); // a box... + p.fill(100, 20, 140, 60); // ...filled solid: a redaction square, still a box + p.fill(200, 20, 390, 23); // a wide field frame 190x40: not square + p.fill(200, 57, 390, 60); + p.fill(200, 20, 203, 60); + p.fill(387, 20, 390, 60); + let found = find(&p.img(), CheckboxParams::default()); + assert_eq!(found.len(), 1, "only the square: {found:?}"); + assert!(found[0].checked, "a solid square reads as marked"); + } +} diff --git a/crates/ffai-carmenta/src/crnn.rs b/crates/ffai-carmenta/src/crnn.rs index 61906f4f..e3231898 100644 --- a/crates/ffai-carmenta/src/crnn.rs +++ b/crates/ffai-carmenta/src/crnn.rs @@ -102,6 +102,16 @@ impl RecLang { Self::ChineseSimplified => "crnn-zh-sim-g2", } } + + /// The files a loader opens for this variant: English's charset is + /// compiled in, Chinese reads `charset.txt` beside the weights. + #[must_use] + pub fn runtime_files(self) -> &'static [&'static str] { + match self { + Self::English => &["crnn.safetensors"], + Self::ChineseSimplified => &["crnn.safetensors", "charset.txt"], + } + } } /// The charset a variant decodes with. @@ -175,6 +185,29 @@ impl Crnn { ctc_greedy_with(logits, &self.charset) } + /// CTC greedy decode restricted to the classes `mask` allows — see + /// [`ctc_greedy_masked`]. Build the mask with [`Self::class_mask`]. + pub fn decode_masked(&self, logits: &Tensor, mask: &[bool]) -> Result<(String, Option)> { + ctc_greedy_masked(logits, &self.charset, mask) + } + + /// Per-class allow mask for `allowed`: index 0 (blank) always true, class + /// `i` true when `charset[i - 1]` is in `allowed`. A requested character + /// this model cannot emit is an error naming it — silently dropping it + /// would make the constraint mean something other than what was asked. + pub fn class_mask(&self, allowed: &str) -> Result> { + let missing: String = allowed.chars().filter(|c| !self.charset.contains(c)).collect(); + if !missing.is_empty() { + return Err(candle_core::Error::Msg(format!( + "charset constraint: this recognizer cannot emit {missing:?}" + ))); + } + let mut mask = Vec::with_capacity(self.charset.len() + 1); + mask.push(true); + mask.extend(self.charset.iter().map(|c| allowed.contains(*c))); + Ok(mask) + } + /// How many characters this instance can emit — 96 for English, 6 718 for /// Chinese. Exposed so a caller can assert which model it actually got. #[must_use] @@ -247,18 +280,117 @@ pub fn ctc_greedy_with(logits: &Tensor, charset: &[char]) -> Result<(String, Opt } prev = id; } - let conf = if confs.is_empty() { - None - } else { - Some(confs.iter().sum::() / confs.len() as f32) - }; - Ok((out, conf)) + Ok((out, line_confidence(&confs))) +} + +/// How a line's per-character probabilities become one confidence. +/// +/// The shipped reduction is the MEAN. `FFAI_CONF_REDUCE=min` reports the +/// weakest character instead — the calibration experiment of +/// docs/plans/commercial-gaps.md gap 12: a line with one badly-read character +/// averages out to a confident mean, and a caller gating on confidence keeps +/// it. Experimental until the form bench says which separates right from +/// wrong better. +#[must_use] +pub fn line_confidence(confs: &[f32]) -> Option { + static MIN: std::sync::OnceLock = std::sync::OnceLock::new(); + if confs.is_empty() { + return None; + } + if *MIN.get_or_init(|| std::env::var("FFAI_CONF_REDUCE").is_ok_and(|v| v == "min")) { + return confs.iter().copied().reduce(f32::min); + } + Some(confs.iter().sum::() / confs.len() as f32) +} + +/// [`ctc_greedy_with`], with the argmax at each step taken only over the +/// classes `mask` allows (`mask[0]`, the blank, must be allowed). +/// +/// The kept confidence is the step's softmax probability for the chosen class +/// over the FULL distribution — deliberately not renormalised over the allowed +/// set. Renormalising would report a forced digit with high confidence even +/// where the model put nearly all its mass on a letter; unrenormalised, that +/// line comes back with low confidence, which is the signal a caller gating on +/// confidence needs. +/// +/// With every class allowed this is `ctc_greedy_with`, character for +/// character (`masked_decode_with_everything_allowed_is_the_free_decode`). +pub fn ctc_greedy_masked( + logits: &Tensor, + charset: &[char], + mask: &[bool], +) -> Result<(String, Option)> { + let probs = candle_nn::ops::softmax(logits, 1)?.to_vec2::()?; + if mask.len() != charset.len() + 1 || !mask[0] { + return Err(candle_core::Error::Msg(format!( + "class mask has {} entries with blank {}; the head has {} classes", + mask.len(), + if mask.first() == Some(&true) { "allowed" } else { "DISALLOWED" }, + charset.len() + 1 + ))); + } + let mut out = String::new(); + let mut confs = Vec::new(); + let mut prev = 0usize; + for row in &probs { + // First maximum among allowed classes, matching argmax's tie rule. + let (id, p) = row + .iter() + .enumerate() + .filter(|(i, _)| mask[*i]) + .fold((0usize, f32::NEG_INFINITY), |best, (i, &p)| if p > best.1 { (i, p) } else { best }); + if id != 0 && id != prev { + out.push(charset[id - 1]); + confs.push(p); + } + prev = id; + } + Ok((out, line_confidence(&confs))) } #[cfg(test)] mod tests { use super::*; + /// Logits for a toy 3-char charset "abc": steps favour a, a, blank, c, b. + fn toy_logits() -> Tensor { + let rows: Vec = vec![ + 0.0, 5.0, 1.0, 0.0, // a + 0.0, 5.0, 1.0, 0.0, // a (repeat, collapsed) + 5.0, 0.0, 0.0, 0.0, // blank + 0.0, 0.0, 1.0, 4.0, // c + 0.0, 1.0, 3.0, 2.0, // b + ]; + Tensor::from_vec(rows, (5, 4), &candle_core::Device::Cpu).unwrap() + } + + #[test] + fn masked_decode_with_everything_allowed_is_the_free_decode() { + let cs = ['a', 'b', 'c']; + let free = ctc_greedy_with(&toy_logits(), &cs).unwrap(); + let masked = ctc_greedy_masked(&toy_logits(), &cs, &[true; 4]).unwrap(); + assert_eq!(free.0, "acb"); + assert_eq!(free, masked); + } + + /// Disallowing `a` must substitute the best ALLOWED class at those steps, + /// and the confidence must fall — the forced choice is reported honestly. + #[test] + fn masked_decode_substitutes_and_reports_the_lower_confidence() { + let cs = ['a', 'b', 'c']; + let free = ctc_greedy_with(&toy_logits(), &cs).unwrap(); + let masked = ctc_greedy_masked(&toy_logits(), &cs, &[true, false, true, true]).unwrap(); + assert_eq!(masked.0, "bcb"); + assert!(masked.1.unwrap() < free.1.unwrap(), "{masked:?} vs {free:?}"); + } + + #[test] + fn a_mask_that_disallows_blank_or_mis_sizes_the_head_is_refused() { + let cs = ['a', 'b', 'c']; + assert!(ctc_greedy_masked(&toy_logits(), &cs, &[false, true, true, true]).is_err()); + assert!(ctc_greedy_masked(&toy_logits(), &cs, &[true; 3]).is_err()); + } + /// The switch is OFF unless asked for, and only for the values documented. #[test] fn rec_lang_defaults_to_english() { diff --git a/crates/ffai-carmenta/src/engine.rs b/crates/ffai-carmenta/src/engine.rs index 775aa109..b6648689 100644 --- a/crates/ffai-carmenta/src/engine.rs +++ b/crates/ffai-carmenta/src/engine.rs @@ -97,11 +97,28 @@ struct Models { device: Device, } +/// Where an engine's manifests and weights come from. +#[derive(Clone)] +enum Source { + /// `./models` when it exists — a repository checkout — overlaid on the + /// manifests compiled into the crate; otherwise the compiled-in ones alone. + /// Weights resolve through the cache and download when `fetch` is on. + Default, + /// A caller-named manifest directory, which MUST exist, overlaid on the + /// compiled-in manifests by model name. Weights as for `Default`. + ManifestDir(std::path::PathBuf), + /// The compiled-in manifests, and weights ONLY from `root//`. + /// Never touches the cache or the network. + Offline(std::path::PathBuf), + /// Built by [`CraftCrnn::from_bytes`]; there is nothing to load. + Bytes, +} + pub struct CraftCrnn { /// Lazy: weights load on first `recognize`, matching Mercury's contract /// (the bench harness warms and records this separately). models: OnceLock>, - manifest_dir: std::path::PathBuf, + source: Source, rec: RecStage, det: DetStage, /// §48: the three ONNX stages, loaded on first ROUTED page. Separate from @@ -110,9 +127,12 @@ pub struct CraftCrnn { } impl CraftCrnn { - #[must_use] + /// CRAFT + CRNN. Manifests come from `./models` when it exists and from + /// the copies compiled into this crate otherwise, so a crates.io user + /// needs no manifest directory at all — only the weights. + #[must_use] pub fn new() -> Self { - Self::with_manifest_dir(Path::new("models")) + Self::variant(RecStage::Crnn, DetStage::Craft) } /// The `craft-parseq` variant: same detection, PARSeq-tiny recognition. @@ -135,24 +155,42 @@ impl CraftCrnn { } fn variant(rec: RecStage, det: DetStage) -> Self { - Self { - models: OnceLock::new(), - manifest_dir: Path::new("models").to_path_buf(), - rec, - det, - router: OnceLock::new(), - } + Self::with_source(rec, det, Source::Default) } - #[must_use] + fn with_source(rec: RecStage, det: DetStage, source: Source) -> Self { + Self { models: OnceLock::new(), source, rec, det, router: OnceLock::new() } + } + + /// CRAFT + CRNN with manifests read from `dir`, which must exist. + #[must_use] pub fn with_manifest_dir(dir: &Path) -> Self { - Self { - models: OnceLock::new(), - manifest_dir: dir.to_path_buf(), - rec: RecStage::Crnn, - det: DetStage::Craft, - router: OnceLock::new(), - } + Self::variant_in(RecStage::Crnn, DetStage::Craft, dir) + } + + /// Any det/rec pair for an OFFLINE, scheduled or production job: the + /// manifests compiled into this crate, and weights read ONLY from + /// `weights_root//`. + /// + /// Nothing is looked up in the cache and nothing is downloaded, ever. A + /// missing or corrupt file is an error naming the exact path that was + /// expected and the checksum it failed (docs/plans/commercial-gaps.md, + /// gap 7). Call [`Self::preload`] at startup to fail before the first page + /// rather than on it. + /// + /// The layout under `weights_root` is the cache's own layout, so a + /// populated cache directory (`ffai models` prints its root) can be copied + /// or mounted as-is. + #[must_use] + pub fn offline(rec: RecStage, det: DetStage, weights_root: &Path) -> Self { + Self::with_source(rec, det, Source::Offline(weights_root.to_path_buf())) + } + + /// Load every model now, returning the error instead of deferring it to + /// the first `recognize`. For a batch job, a missing weight file should + /// stop the job before page one, not after it has started writing output. + pub fn preload(&self) -> Result<()> { + self.models().map(|_| ()) } /// Any det/rec pair, with manifests read from `dir`. @@ -164,17 +202,11 @@ impl CraftCrnn { /// the other constructors assume does not resolve. #[must_use] pub fn variant_in(rec: RecStage, det: DetStage, dir: &Path) -> Self { - Self { - models: OnceLock::new(), - manifest_dir: dir.to_path_buf(), - rec, - det, - router: OnceLock::new(), - } + Self::with_source(rec, det, Source::ManifestDir(dir.to_path_buf())) } fn router(&self) -> Result<&crate::route::Router> { - match self.router.get_or_init(|| crate::route::Router::new().map_err(|e| e.to_string())) { + match self.router.get_or_init(|| crate::route::Router::new().map_err(cached_message)) { Ok(r) => Ok(r), Err(e) => Err(Error::Model(e.clone())), } @@ -184,7 +216,7 @@ impl CraftCrnn { let (rec, det) = (self.rec, self.det); let loaded = self .models - .get_or_init(|| load_models(&self.manifest_dir, rec, det).map_err(|e| e.to_string())); + .get_or_init(|| load_models(&self.source, rec, det).map_err(cached_message)); match loaded { Ok(m) => Ok(m), Err(e) => Err(Error::Model(e.clone())), @@ -192,6 +224,16 @@ impl CraftCrnn { } } +/// The message a cached load failure is re-raised with as `Error::Model`. +/// Taking `to_string()` of an `Error::Model` and wrapping it again printed +/// "model error: model error: ..."; this keeps the prefix single. +fn cached_message(e: Error) -> String { + match e { + Error::Model(msg) => msg, + other => other.to_string(), + } +} + impl Default for CraftCrnn { fn default() -> Self { Self::new() @@ -244,7 +286,7 @@ impl CraftCrnn { }, rec, det, - manifest_dir: std::path::PathBuf::new(), + source: Source::Bytes, // The routing stages are ONNX files read from disk, so a // from-bytes caller cannot have them. Left uninitialised: routing // is opt-in (`FFAI_ROUTE=1`) and a page that asks for it here gets @@ -320,14 +362,55 @@ fn build_models(det: DetStage, rec: RecStage, w: WeightBytes) -> Result Ok(Models { craft, mobiledet, crnn, parseq, svtr, device }) } -fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { +/// The manifests a [`Source`] sees, and where its weights resolve. +/// +/// Disk manifests REPLACE compiled-in ones of the same name, so a repository +/// checkout (or a caller's directory) still wins — which is what lets a model +/// be re-converted and re-pinned without a release. Offline resolution uses +/// only the compiled-in manifests: its point is that the checksums a job runs +/// against are the ones this release was gated on. +fn manifests_for(source: &Source) -> Result<(Vec, String)> { + let mut manifests = crate::manifests::embedded()?; + let overlay = |manifests: &mut Vec, dir: &Path| -> Result<()> { + for m in ffai_models::load_dir(dir)? { + manifests.retain(|e| e.name != m.name); + manifests.push(m); + } + Ok(()) + }; + let described = match source { + Source::Default => { + let dir = Path::new("models"); + if dir.is_dir() { + overlay(&mut manifests, dir)?; + format!("{} and the manifests compiled into ffai-carmenta", dir.display()) + } else { + "the manifests compiled into ffai-carmenta".to_string() + } + } + Source::ManifestDir(dir) => { + overlay(&mut manifests, dir)?; + format!("{} and the manifests compiled into ffai-carmenta", dir.display()) + } + Source::Offline(_) => "the manifests compiled into ffai-carmenta".to_string(), + Source::Bytes => { + return Err(Error::Model( + "engine was built from bytes and has no manifest source to load from".into(), + )); + } + }; + Ok((manifests, described)) +} + +fn load_models(source: &Source, rec: RecStage, det: DetStage) -> Result { let device = Device::Cpu; - let manifests = ffai_models::load_dir(dir)?; - let find = |name: &str| { + let (manifests, searched) = manifests_for(source)?; + let find = |name: &str| -> Result> { manifests .iter() .find(|m| m.name == name) - .ok_or_else(|| Error::Model(format!("no model manifest named `{name}` in {}", dir.display()))) + .map(|m| Resolver { manifest: m, source }) + .ok_or_else(|| Error::Model(format!("no model manifest named `{name}` in {searched}"))) }; let load_vb = |file: std::path::PathBuf| { unsafe { VarBuilder::from_mmaped_safetensors(&[file], DType::F32, &device) } @@ -335,13 +418,13 @@ fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { }; let (craft, mobiledet) = match det { DetStage::Craft => { - let f = find("craft-mlt")?.fetch()?; + let f = find("craft-mlt")?.fetch(&["craft.safetensors"])?; let m = Craft::new(load_vb(f.file("craft.safetensors")?.to_path_buf())?) .map_err(image::candle_err)?; (Some(m), None) } DetStage::MobileDet => { - let f = find("ppocrv5-mobile-det")?.fetch()?; + let f = find("ppocrv5-mobile-det")?.fetch(&["det-fused.safetensors"])?; let m = crate::mobiledet::MobileDet::new(load_vb( f.file("det-fused.safetensors")?.to_path_buf(), )?) @@ -349,10 +432,10 @@ fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { (None, Some(m)) } DetStage::Composed => { - let cf = find("craft-mlt")?.fetch()?; + let cf = find("craft-mlt")?.fetch(&["craft.safetensors"])?; let c = Craft::new(load_vb(cf.file("craft.safetensors")?.to_path_buf())?) .map_err(image::candle_err)?; - let df = find("ppocrv5-mobile-det")?.fetch()?; + let df = find("ppocrv5-mobile-det")?.fetch(&["det-fused.safetensors"])?; let d = crate::mobiledet::MobileDet::new(load_vb( df.file("det-fused.safetensors")?.to_path_buf(), )?) @@ -369,7 +452,7 @@ fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { // otherwise, so the same code path runs both. English is the // default and remains the oracle. let lang = crate::crnn::RecLang::from_env(); - let f = find(lang.model_name())?.fetch()?; + let f = find(lang.model_name())?.fetch(lang.runtime_files())?; let weights = f.file("crnn.safetensors")?.to_path_buf(); let charset = crate::crnn::charset_for(lang, weights.parent()) .map_err(image::candle_err)?; @@ -378,13 +461,13 @@ fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { (Some(m), None) } RecStage::Parseq => { - let f = find("parseq-tiny")?.fetch()?; + let f = find("parseq-tiny")?.fetch(&["parseq-tiny.safetensors"])?; let m = crate::parseq::Parseq::new(load_vb(f.file("parseq-tiny.safetensors")?.to_path_buf())?) .map_err(image::candle_err)?; (None, Some(m)) } RecStage::Svtr => { - let f = find("ppocrv5-mobile-rec")?.fetch()?; + let f = find("ppocrv5-mobile-rec")?.fetch(&["rec.safetensors", "charset.txt"])?; let weights = f.file("rec.safetensors")?.to_path_buf(); let charset = crate::svtr::load_charset(f.file("charset.txt")?) .map_err(image::candle_err)?; @@ -399,6 +482,223 @@ fn load_models(dir: &Path, rec: RecStage, det: DetStage) -> Result { Ok(Models { craft, mobiledet, crnn, parseq, svtr, device }) } +/// A manifest bound to the weight-resolution rule of its [`Source`]. +struct Resolver<'a> { + manifest: &'a ffai_models::ModelManifest, + source: &'a Source, +} + +impl Resolver<'_> { + /// Resolve the files this loader will open. Offline resolution asks for + /// exactly those, so a deployment never has to ship files a manifest lists + /// only for provenance (a training checkpoint, say). The cache path keeps + /// its existing behaviour of resolving every declared file. + fn fetch(&self, needed: &[&str]) -> Result { + match self.source { + Source::Offline(root) => self.manifest.resolve_only_in(root, needed), + _ => self.manifest.fetch(), + } + } +} + +/// What [`CraftCrnn::detect_orientation`] decided, and the evidence for it. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Orientation { + /// Clockwise quarter turns to APPLY to the image to make it upright (0-3). + pub turns: u8, + /// Detected text area in boxes wider than tall (lines running across). + pub horizontal_area: f32, + /// Detected text area in boxes taller than wide (lines running down). + pub vertical_area: f32, + /// Mean recognition confidence of the sampled boxes at the chosen turn. + pub chosen_confidence: f32, + /// ...and at the other candidate on the same axis. The gap between the + /// two is how sure the decision is. + pub other_confidence: f32, + /// Boxes sampled for the confidence comparison. + pub sampled: usize, +} + +/// How many of the largest boxes are read to decide which way up. +const ORIENT_SAMPLES: usize = 12; +/// Short side the page is decimated toward for the orientation axis vote. +const ORIENT_DET_SHORT: usize = 640; +/// A box counts toward an axis when its long side is this many times its +/// short side; near-square boxes (single glyphs, logos) vote for neither. +const ORIENT_ASPECT: f32 = 1.5; + +fn crop(img: &ImageBuffer, x0: usize, y0: usize, x1: usize, y1: usize) -> ImageBuffer { + let (w, h) = (img.width as usize, img.height as usize); + let (x0, y0, x1, y1) = (x0.min(w), y0.min(h), x1.min(w), y1.min(h)); + let bpp = img.format.bytes_per_pixel(); + let (cw, ch) = (x1.saturating_sub(x0), y1.saturating_sub(y0)); + let mut data = Vec::with_capacity(cw * ch * bpp); + for y in y0..y1 { + data.extend_from_slice(&img.data[(y * w + x0) * bpp..(y * w + x1) * bpp]); + } + ImageBuffer { width: cw as u32, height: ch as u32, format: img.format, data } +} + +impl CraftCrnn { + /// Decide which clockwise quarter turn makes `img` upright. + /// + /// One detection pass decides the AXIS: text lines produce boxes wider + /// than tall when they run across the image and taller than wide when they + /// run down it. Then the largest boxes on that axis are read in both + /// candidate turns, and the turn with the higher mean recognition + /// confidence wins — an upside-down or mirrored-axis line reads with + /// clearly lower confidence. About one detection pass plus two dozen + /// small crop reads, against three or four full-page reads for trying + /// every turn. + /// + /// A page with no detected text returns `turns: 0` and `sampled: 0`. + pub fn detect_orientation(&self, img: &ImageBuffer) -> Result { + let m = self.models()?; + // The axis vote needs box SHAPES, not fine geometry: detect on a + // decimated page (short side near ORIENT_DET_SHORT) and scale the + // boxes back, so the crops read below still come from full-resolution + // pixels. + let short = img.width.min(img.height) as usize; + let f = (short / ORIENT_DET_SHORT).max(1); + let boxes: Vec = self + .raw_boxes(m, &crate::orient::decimate(img, f))? + .into_iter() + .map(|b| boxes::DetBox { x0: b.x0 * f, y0: b.y0 * f, x1: b.x1 * f, y1: b.y1 * f, score: b.score }) + .collect(); + let (mut horizontal_area, mut vertical_area) = (0f32, 0f32); + let mut across = Vec::new(); + let mut down = Vec::new(); + for b in boxes { + let (bw, bh) = ((b.x1 - b.x0) as f32, (b.y1 - b.y0) as f32); + if bw >= ORIENT_ASPECT * bh { + horizontal_area += bw * bh; + across.push(b); + } else if bh >= ORIENT_ASPECT * bw { + vertical_area += bw * bh; + down.push(b); + } + } + let (candidates, mut sample) = if vertical_area > horizontal_area { + ([1u8, 3u8], down) + } else { + ([0u8, 2u8], across) + }; + sample.sort_by_key(|b| std::cmp::Reverse((b.x1 - b.x0) * (b.y1 - b.y0))); + sample.truncate(ORIENT_SAMPLES); + if sample.is_empty() { + return Ok(Orientation { + turns: 0, + horizontal_area, + vertical_area, + chosen_confidence: 0.0, + other_confidence: 0.0, + sampled: 0, + }); + } + let mean_at = |turns: u8| -> Result { + use crate::par::prelude::*; + // The crops are independent reads: fan them across cores, the way + // `recognize` fans out lines. + let confs = sample + .par_iter() + .map(|b| { + let c = crate::orient::rotate(&crop(img, b.x0, b.y0, b.x1, b.y1), turns); + read_confidence(m, self.rec, &c).map(|c| c.unwrap_or(0.0)) + }) + .collect::>>()?; + Ok(confs.iter().sum::() / sample.len() as f32) + }; + let (a, b) = (mean_at(candidates[0])?, mean_at(candidates[1])?); + let (turns, chosen_confidence, other_confidence) = + if b > a { (candidates[1], b, a) } else { (candidates[0], a, b) }; + Ok(Orientation { + turns, + horizontal_area, + vertical_area, + chosen_confidence, + other_confidence, + sampled: sample.len(), + }) + } + + /// Detection boxes in image coordinates, before any line grouping, + /// splitting or reading-order logic — the raw geometry the axis vote needs. + fn raw_boxes(&self, m: &Models, img: &ImageBuffer) -> Result> { + match self.det { + DetStage::MobileDet | DetStage::Composed => { + let (input, sx, sy) = image::mobiledet_input(img, mobiledet_min_side(), &m.device)?; + let det = m.mobiledet.as_ref().expect("mobiledet loaded for a mobile-det stage"); + let prob = det.forward(&input).map_err(image::candle_err)?; + let (_, _, ph, pw) = image::ok(prob.dims4())?; + let flat = image::ok(prob.flatten_all()?.to_vec1::())?; + Ok(crate::mobiledet::boxes_from_probability( + &flat, + pw, + ph, + sx, + sy, + crate::mobiledet::UNCLIP_LINE, + )) + } + DetStage::Craft => { + let (input, scale) = image::craft_input_color(img, &m.device)?; + let craft = m.craft.as_ref().expect("craft loaded for DetStage::Craft"); + let maps = craft.forward(&input).map_err(image::candle_err)?; + let (mh, mw, _) = image::ok(maps.dims3())?; + let flat = image::ok(maps.flatten_all()?.to_vec1::())?; + let region: Vec = flat.iter().step_by(2).copied().collect(); + let affinity: Vec = flat.iter().skip(1).step_by(2).copied().collect(); + let k = 2.0 / scale; + let to = |v: usize| (v as f32 * k).round() as usize; + Ok(boxes::extract_boxes(®ion, &affinity, mw, mh) + .into_iter() + .map(|b| boxes::DetBox { + x0: to(b.x0), + y0: to(b.y0), + x1: to(b.x1), + y1: to(b.y1), + score: b.score, + }) + .collect()) + } + } + } +} + +/// Mean recognition confidence of a whole crop, read as one line by this +/// engine's recognizer. `None` when nothing was read or the crop is degenerate. +fn read_confidence(m: &Models, rec: RecStage, img: &ImageBuffer) -> Result> { + let (w, h) = (img.width as usize, img.height as usize); + if w < 2 || h < 2 { + return Ok(None); + } + match rec { + RecStage::Svtr => { + let (svtr, charset) = m.svtr.as_ref().expect("svtr loaded for RecStage::Svtr"); + let Ok(x) = crate::svtr::svtr_input(img, 0, 0, w, h, &m.device) else { + return Ok(None); + }; + let probs = svtr.forward(&x).map_err(image::candle_err)?; + Ok(crate::svtr::ctc_greedy(&probs, charset).map_err(image::candle_err)?.1) + } + RecStage::Crnn => { + let crnn = m.crnn.as_ref().expect("crnn loaded for RecStage::Crnn"); + let gray = image::to_gray_f32(img)?; + let Ok(x) = image::crnn_input(&gray, w, h, 0, 0, w, h, &m.device) else { + return Ok(None); + }; + let logits = crnn.forward(&x).map_err(image::candle_err)?; + Ok(crnn.decode(&logits).map_err(image::candle_err)?.1) + } + RecStage::Parseq => { + let parseq = m.parseq.as_ref().expect("parseq loaded for RecStage::Parseq"); + let gray = image::to_gray_f32(img)?; + let x = crate::parseq::parseq_input(&gray, w, h, &m.device).map_err(image::candle_err)?; + Ok(parseq.recognize(&x).map_err(image::candle_err)?.1) + } + } +} + impl OcrEngine for CraftCrnn { fn info(&self) -> EngineInfo { let (name, description) = match (self.det, self.rec) { @@ -444,8 +744,88 @@ impl OcrEngine for CraftCrnn { } fn recognize(&self, img: &ImageBuffer, opts: &OcrOptions) -> Result { + // Orientation first: rule removal and everything after it assume the + // lines run across the image. The inner call reads the upright page; + // its boxes are mapped back so the caller's coordinates never change + // meaning underneath them. + if opts.auto_orient { + let o = self.detect_orientation(img)?; + let inner = OcrOptions { auto_orient: false, ..opts.clone() }; + if o.turns == 0 { + return self.recognize(img, &inner); + } + let upright = crate::orient::rotate(img, o.turns); + let mut out = self.recognize(&upright, &inner)?; + let (w, h) = (img.width as f32, img.height as f32); + for line in out.blocks.iter_mut().flat_map(|b| b.lines.iter_mut()) { + if let Some(b) = line.bbox { + line.bbox = Some(crate::orient::unrotate_box(b, o.turns, w, h)); + } + for word in &mut line.words { + if let Some(b) = word.bbox { + word.bbox = Some(crate::orient::unrotate_box(b, o.turns, w, h)); + } + } + } + for block in &mut out.blocks { + if let Some(b) = block.bbox { + block.bbox = Some(crate::orient::unrotate_box(b, o.turns, w, h)); + } + } + return Ok(out); + } let m = self.models()?; + // Form rules come out BEFORE detection as well as recognition: a rule + // joins the boxes DBNet draws, and the crop the recognizer reads is cut + // from the same pixels. Boxes are unaffected in coordinates — the image + // keeps its size. + let cleaned; + let img = if opts.remove_rules { + let params = crate::rules::RuleParams::for_width(img.width as usize); + cleaned = crate::rules::remove_horizontal_rules(img, params).0; + &cleaned + } else { + img + }; + + // Charset constraint (`OcrOptions::charset`): one allow-mask per call, + // over the recognizer's own classes. An engine that cannot honour it + // refuses — reading freely while the caller believes the output is + // constrained is the failure this option exists to prevent. + let mask: Option> = match (opts.charset.as_deref(), self.rec) { + (None, _) => None, + (Some(_), RecStage::Parseq) => { + return Err(Error::Other( + "charset constraint is not supported by the PARSeq recognizer (its \ + autoregressive decoder is not masked); use a CRNN or SVTR engine" + .into(), + )); + } + (Some(_), _) if crate::route::enabled() => { + return Err(Error::Other( + "charset constraint cannot be combined with region routing (FFAI_ROUTE): \ + routed regions are rendered by other models" + .into(), + )); + } + (Some(cs), RecStage::Crnn) => Some( + m.crnn + .as_ref() + .expect("crnn loaded for RecStage::Crnn") + .class_mask(cs) + .map_err(image::candle_err)?, + ), + (Some(cs), RecStage::Svtr) => { + let (_, charset) = m.svtr.as_ref().expect("svtr loaded for RecStage::Svtr"); + Some(crate::svtr::class_mask(charset, cs).map_err(image::candle_err)?) + } + }; + let crnn_decode = |crnn: &Crnn, logits: &candle_core::Tensor| match &mask { + Some(mk) => crnn.decode_masked(logits, mk), + None => crnn.decode(logits), + }; + let (w, h) = (img.width as usize, img.height as usize); // --psm-7 analog: the caller vouches the image is ONE text line @@ -471,8 +851,9 @@ impl OcrEngine for CraftCrnn { })?; let logits = crate::profile::timed(|p| &p.rec_fwd, || crnn.forward(&crop)) .map_err(image::candle_err)?; - let (text, confidence) = crate::profile::timed(|p| &p.decode, || crnn.decode(&logits)) - .map_err(image::candle_err)?; + let (text, confidence) = + crate::profile::timed(|p| &p.decode, || crnn_decode(crnn, &logits)) + .map_err(image::candle_err)?; let mut lines = Vec::new(); if !text.is_empty() { lines.push(OcrLine { @@ -725,10 +1106,15 @@ impl OcrEngine for CraftCrnn { }; let probs = crate::profile::timed(|p| &p.rec_fwd, || svtr.forward(&crop)) .map_err(image::candle_err)?; - crate::profile::timed(|p| &p.decode, || { - crate::svtr::ctc_greedy(&probs, charset) + let (mut text, conf) = crate::profile::timed(|p| &p.decode, || match &mask { + Some(mk) => crate::svtr::ctc_greedy_masked(&probs, charset, mk), + None => crate::svtr::ctc_greedy(&probs, charset), }) - .map_err(image::candle_err)? + .map_err(image::candle_err)?; + // SVTR's multilingual charset can put a full-width + // `０` or `，` into a Latin line; see `fold`. + crate::fold::apply(&mut text); + (text, conf) } RecStage::Crnn => { let crnn = m.crnn.as_ref().expect("crnn loaded for RecStage::Crnn"); @@ -740,7 +1126,7 @@ impl OcrEngine for CraftCrnn { }; let logits = crate::profile::timed(|p| &p.rec_fwd, || crnn.forward(&crop)) .map_err(image::candle_err)?; - crate::profile::timed(|p| &p.decode, || crnn.decode(&logits)) + crate::profile::timed(|p| &p.decode, || crnn_decode(crnn, &logits)) .map_err(image::candle_err)? } RecStage::Parseq => { diff --git a/crates/ffai-carmenta/src/fold.rs b/crates/ffai-carmenta/src/fold.rs new file mode 100644 index 00000000..9a0c97a4 --- /dev/null +++ b/crates/ffai-carmenta/src/fold.rs @@ -0,0 +1,102 @@ +//! Fold stray full-width forms to ASCII on Latin lines +//! (docs/plans/commercial-gaps.md, gap 12). +//! +//! The SVTR recognizer's charset is multilingual: it contains both `0` and the +//! full-width `０` (U+FF10), both `,` and `，` (U+FF0C). On a Latin line it +//! occasionally emits the full-width twin — `"$25，０00"`, `"12／31／2025"` — +//! which looks right to a person and is rejected, or worse silently dropped, +//! by any number or date parser downstream. +//! +//! In CJK text the full-width forms are correct and must stay. So the fold is +//! decided per LINE: a line with no CJK character at all is a Latin line, and +//! its full-width ASCII variants (U+FF01..=U+FF5E) and ideographic space are +//! mapped to their ASCII equivalents. A line with any CJK character is left +//! exactly as read. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Lines the fold has changed in this process — the census that says how +/// often it fires on a corpus before anyone decides it is harmless. +pub static FIRED: AtomicUsize = AtomicUsize::new(0); + +/// Fold `line` in place when [`fold_latin_fullwidth`] would change it, +/// counting the firing. `FFAI_NO_FOLD` turns the fold off for A/B runs. +pub fn apply(line: &mut String) { + static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); + if *OFF.get_or_init(|| std::env::var_os("FFAI_NO_FOLD").is_some()) { + return; + } + if let Some(folded) = fold_latin_fullwidth(line) { + *line = folded; + FIRED.fetch_add(1, Ordering::Relaxed); + } +} + +/// True for characters that mark a line as CJK text: CJK ideographs, kana, +/// Hangul, and CJK punctuation (but NOT the full-width ASCII block, which is +/// what the fold targets). +const fn is_cjk(c: char) -> bool { + matches!(c, + // CJK symbols and punctuation, except U+3000 (the ideographic space), + // which a Latin line can carry and the fold converts + '\u{3001}'..='\u{303F}' + | '\u{3040}'..='\u{30FF}' // hiragana, katakana + | '\u{3100}'..='\u{312F}' // bopomofo + | '\u{3400}'..='\u{4DBF}' // CJK extension A + | '\u{4E00}'..='\u{9FFF}' // CJK unified ideographs + | '\u{AC00}'..='\u{D7AF}' // Hangul syllables + | '\u{F900}'..='\u{FAFF}' // CJK compatibility ideographs + | '\u{FF65}'..='\u{FFDC}' // half-width katakana, Hangul + ) +} + +/// The line with full-width ASCII variants folded, when it contains no CJK +/// character; otherwise the line unchanged. `None` when nothing would change, +/// so a caller can count how often the fold fires. +#[must_use] +pub fn fold_latin_fullwidth(line: &str) -> Option { + if line.chars().any(is_cjk) { + return None; + } + if !line + .chars() + .any(|c| matches!(c, '\u{FF01}'..='\u{FF5E}' | '\u{3000}')) + { + return None; + } + Some( + line.chars() + .map(|c| match c { + // U+FF01..U+FF5E are U+0021..U+007E shifted by 0xFEE0. + '\u{FF01}'..='\u{FF5E}' => char::from_u32(c as u32 - 0xFEE0).unwrap_or(c), + '\u{3000}' => ' ', + other => other, + }) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::fold_latin_fullwidth as fold; + + #[test] + fn stray_full_width_forms_on_a_latin_line_become_ascii() { + assert_eq!(fold("$25，０00").as_deref(), Some("$25,000")); + assert_eq!(fold("12／31／2025").as_deref(), Some("12/31/2025")); + assert_eq!(fold("Ａmount：１０").as_deref(), Some("Amount:10")); + } + + #[test] + fn a_line_with_cjk_text_is_left_exactly_as_read() { + assert_eq!(fold("金额：１０，０００元"), None); + assert_eq!(fold("東京（ＴＯＫＹＯ）"), None); + assert_eq!(fold("서울，２０２５"), None); + } + + #[test] + fn a_plain_line_is_not_touched() { + assert_eq!(fold("$25,000"), None); + assert_eq!(fold(""), None); + } +} diff --git a/crates/ffai-carmenta/src/forms.rs b/crates/ffai-carmenta/src/forms.rs new file mode 100644 index 00000000..72843946 --- /dev/null +++ b/crates/ffai-carmenta/src/forms.rs @@ -0,0 +1,214 @@ +//! Label → value pairs from a read form page (docs/plans/commercial-gaps.md, +//! gap 4). +//! +//! `OcrOutput` is lines with boxes. A form reader wants FIELDS: "Name of +//! Creditor" → "Merrill Lynch". The layout rule printed forms follow, and the +//! one a caller's born-digital reader already relies on, is: +//! +//! * a **label** is a line that ends in `:`; +//! * its **value** is the nearest unlabelled line to its RIGHT on the same +//! baseline, or, failing that, the nearest one directly BELOW it; +//! * a single line reading `Label: value` (the detector drew one box around +//! both) is split at its first `:`. +//! +//! Each line is used as a value at most once, closest label first, so two +//! labels never claim the same value. A label with no value is returned with +//! an empty value rather than dropped: "this field is blank" is information. + +use ffai_core::types::{BoundingBox, OcrOutput}; + +/// One field. +#[derive(Debug, Clone, PartialEq)] +pub struct Field { + /// The label, without its trailing `:`. + pub label: String, + /// The value, or empty when none was found. + pub value: String, + pub label_bbox: Option, + pub value_bbox: Option, + /// The lower of the two lines' confidences, when both report one. + pub confidence: Option, +} + +struct Line { + text: String, + bbox: BoundingBox, + confidence: Option, +} + +fn center_y(b: &BoundingBox) -> f32 { + b.y + b.height / 2.0 +} + +/// Every labelled field on the page, in reading order of the labels. +#[must_use] +pub fn pair_fields(out: &OcrOutput) -> Vec { + let lines: Vec = out + .blocks + .iter() + .flat_map(|b| &b.lines) + .filter_map(|l| { + let text = l.text.trim().to_string(); + (!text.is_empty()).then_some(Line { + text, + bbox: l.bbox?, + confidence: l.confidence, + }) + }) + .collect(); + + let mut fields = Vec::new(); + // Labels waiting for a value on another line: (field index, label line). + let mut open: Vec<(usize, usize)> = Vec::new(); + let mut is_label = vec![false; lines.len()]; + for (i, l) in lines.iter().enumerate() { + if let Some(label) = l.text.strip_suffix(':') { + is_label[i] = true; + fields.push(Field { + label: label.trim().to_string(), + value: String::new(), + label_bbox: Some(l.bbox), + value_bbox: None, + confidence: l.confidence, + }); + open.push((fields.len() - 1, i)); + } else if let Some((label, value)) = l.text.split_once(':') + && !label.trim().is_empty() + && !value.trim().is_empty() + // A time ("10:30") or a ratio is not a field. + && !label.trim().ends_with(|c: char| c.is_ascii_digit()) + { + is_label[i] = true; + fields.push(Field { + label: label.trim().to_string(), + value: value.trim().to_string(), + label_bbox: Some(l.bbox), + value_bbox: Some(l.bbox), + confidence: l.confidence, + }); + } + } + + // Candidate (distance, field, value line), searched right first, then below. + let mut candidates: Vec<(f32, usize, usize)> = Vec::new(); + for &(f, li) in &open { + let lb = lines[li].bbox; + for (vi, v) in lines.iter().enumerate() { + if is_label[vi] { + continue; + } + let vb = v.bbox; + let same_row = (center_y(&vb) - center_y(&lb)).abs() <= lb.height * 0.6; + if same_row && vb.x >= lb.x + lb.width - lb.height * 0.5 { + candidates.push((vb.x - (lb.x + lb.width), f, vi)); + } else { + let below = vb.y - (lb.y + lb.height); + let overlaps = vb.x < lb.x + lb.width + lb.height * 2.0 && vb.x + vb.width > lb.x; + if below >= -lb.height * 0.3 && below <= lb.height * 1.5 && overlaps { + // Below ranks after every same-row candidate. + candidates.push((10_000.0 + below, f, vi)); + } + } + } + } + candidates.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut taken = vec![false; lines.len()]; + let mut filled = vec![false; fields.len()]; + for (_, f, vi) in candidates { + if taken[vi] || filled[f] { + continue; + } + taken[vi] = true; + filled[f] = true; + let v = &lines[vi]; + fields[f].value.clone_from(&v.text); + fields[f].value_bbox = Some(v.bbox); + fields[f].confidence = match (fields[f].confidence, v.confidence) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + } + fields +} + +#[cfg(test)] +mod tests { + use super::*; + use ffai_core::types::{OcrBlock, OcrLine}; + + fn line(text: &str, x: f32, y: f32, w: f32) -> OcrLine { + OcrLine { + text: text.into(), + words: Vec::new(), + bbox: Some(BoundingBox { + x, + y, + width: w, + height: 30.0, + }), + confidence: Some(0.95), + } + } + + fn page(lines: Vec) -> OcrOutput { + OcrOutput { + blocks: vec![OcrBlock { lines, bbox: None }], + } + } + + #[test] + fn a_value_to_the_right_on_the_same_row_is_paired() { + let f = pair_fields(&page(vec![ + line("Name of Creditor:", 40.0, 100.0, 300.0), + line("Merrill Lynch", 560.0, 102.0, 240.0), + line("Amount:", 40.0, 160.0, 140.0), + line("$10,000", 560.0, 158.0, 150.0), + ])); + assert_eq!(f.len(), 2); + assert_eq!( + (f[0].label.as_str(), f[0].value.as_str()), + ("Name of Creditor", "Merrill Lynch") + ); + assert_eq!( + (f[1].label.as_str(), f[1].value.as_str()), + ("Amount", "$10,000") + ); + } + + #[test] + fn a_value_below_the_label_is_paired_when_nothing_is_to_its_right() { + let f = pair_fields(&page(vec![ + line("Business Address:", 40.0, 100.0, 300.0), + line("1117 Oak Street", 40.0, 140.0, 280.0), + ])); + assert_eq!(f[0].value, "1117 Oak Street"); + } + + #[test] + fn a_label_and_value_in_one_line_are_split_and_times_are_not_fields() { + let f = pair_fields(&page(vec![ + line("Amount: $2,500", 40.0, 100.0, 300.0), + line("Meeting at 10:30 today", 40.0, 200.0, 400.0), + ])); + assert_eq!(f.len(), 1); + assert_eq!( + (f[0].label.as_str(), f[0].value.as_str()), + ("Amount", "$2,500") + ); + } + + #[test] + fn a_blank_field_is_reported_blank_and_values_are_not_shared() { + let f = pair_fields(&page(vec![ + line("Spouse:", 40.0, 100.0, 120.0), + line("Employer:", 40.0, 300.0, 160.0), + line("Acme Corp", 560.0, 300.0, 200.0), + ])); + assert_eq!(f.len(), 2); + assert_eq!( + f[0].value, "", + "nothing near Spouse: it is blank, not given Employer's value" + ); + assert_eq!(f[1].value, "Acme Corp"); + } +} diff --git a/crates/ffai-carmenta/src/lexicon.rs b/crates/ffai-carmenta/src/lexicon.rs new file mode 100644 index 00000000..f750d4fa --- /dev/null +++ b/crates/ffai-carmenta/src/lexicon.rs @@ -0,0 +1,181 @@ +//! Snap a read to a caller's lexicon (docs/plans/commercial-gaps.md, gap 12). +//! +//! A form field often has a known vocabulary — the state's counties, the +//! banks a filer has used before, the form's own printed choices. A caller +//! reading "Trish HTlI" where the lexicon holds "Irish Hill" should be told +//! so, with the evidence, rather than storing the misread or discarding the +//! line. +//! +//! The distance is Levenshtein with one change: substituting one member of a +//! LOOK-ALIKE class for another costs half an edit. The classes are the +//! confusions measured on real filings and on `carmenta-form-v1`: +//! `1 l I i | T`, `0 O o D U Q`, `5 S s`, `8 B $`, `2 Z z`, `6 G b`, `9 g q`, +//! `C G`, `c e`, `, .`. So "112l" is closer to "1121" than any word that +//! differs by a real letter. +//! +//! A snap is returned only when it is UNAMBIGUOUS: within `max_distance`, and +//! the runner-up is at least `margin` further away. Otherwise the caller gets +//! the candidates and decides. Nothing here changes what the engine read; it +//! is advice with its evidence attached. + +/// Look-alike classes: substituting within one costs half an edit. +const LOOK_ALIKES: &[&str] = &[ + "1lIi|T", "0OoDUQ", "5Ss", "8B$", "2Zz", "6Gb", "9gq", "CG", "ce", ",.", +]; + +fn substitution_cost(a: char, b: char) -> f32 { + if a == b { + 0.0 + } else if a.eq_ignore_ascii_case(&b) + || LOOK_ALIKES + .iter() + .any(|class| class.contains(a) && class.contains(b)) + { + 0.5 + } else { + 1.0 + } +} + +/// Edit distance with look-alike substitutions at half cost. Insertions and +/// deletions cost 1, so a dropped or invented character is never cheap. +#[must_use] +pub fn distance(a: &str, b: &str) -> f32 { + let (a, b): (Vec, Vec) = (a.chars().collect(), b.chars().collect()); + let mut prev: Vec = (0..=b.len()).map(|j| j as f32).collect(); + for (i, &ca) in a.iter().enumerate() { + let mut cur = vec![(i + 1) as f32; b.len() + 1]; + for (j, &cb) in b.iter().enumerate() { + cur[j + 1] = (prev[j] + substitution_cost(ca, cb)) + .min(prev[j + 1] + 1.0) + .min(cur[j] + 1.0); + } + prev = cur; + } + prev[b.len()] +} + +/// The outcome of [`Lexicon::snap`]. +#[derive(Debug, Clone, PartialEq)] +pub enum Snap { + /// The read is already an entry, exactly. + Exact(String), + /// One entry is clearly closest. + Snapped { entry: String, distance: f32 }, + /// Nothing close enough, or two entries too close to call. The nearest + /// candidates, closest first, for the caller to judge. + Ambiguous(Vec<(String, f32)>), +} + +/// A caller-supplied vocabulary for one field. +#[derive(Debug, Clone)] +pub struct Lexicon { + entries: Vec, + /// Largest distance, as a fraction of the entry's length, that may snap. + pub max_distance: f32, + /// How much further the runner-up must be for a snap to be unambiguous. + pub margin: f32, +} + +impl Lexicon { + /// A lexicon with conservative defaults: snap within 25 % of the entry's + /// length, and only when the runner-up is at least 1.0 edit further. + #[must_use] + pub fn new(entries: impl IntoIterator>) -> Self { + Self { + entries: entries.into_iter().map(Into::into).collect(), + max_distance: 0.25, + margin: 1.0, + } + } + + /// Match `read` against the lexicon; see [`Snap`]. + #[must_use] + pub fn snap(&self, read: &str) -> Snap { + let read = read.trim(); + if let Some(e) = self.entries.iter().find(|e| e.as_str() == read) { + return Snap::Exact(e.clone()); + } + let mut scored: Vec<(String, f32)> = self + .entries + .iter() + .map(|e| (e.clone(), distance(read, e))) + .collect(); + scored.sort_by(|x, y| x.1.total_cmp(&y.1)); + let Some((best, d)) = scored.first().cloned() else { + return Snap::Ambiguous(Vec::new()); + }; + let limit = self.max_distance * best.chars().count().max(1) as f32; + let runner_up = scored.get(1).map_or(f32::INFINITY, |s| s.1); + if d <= limit && runner_up - d >= self.margin { + Snap::Snapped { + entry: best, + distance: d, + } + } else { + scored.truncate(3); + Snap::Ambiguous(scored) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn look_alike_substitutions_cost_half() { + assert!((distance("112l", "1121") - 0.5).abs() < 1e-6); + assert!((distance("TTT7", "1117") - 1.5).abs() < 1e-6); + assert!((distance("810,000", "$10,000") - 0.5).abs() < 1e-6); + assert!( + (distance("cat", "cut") - 1.0).abs() < 1e-6, + "a real letter change costs a full edit" + ); + assert!( + (distance("abc", "abcd") - 1.0).abs() < 1e-6, + "insertions are never cheap" + ); + } + + /// The misreads recorded on real filings snap to the right entries. + #[test] + fn field_misreads_snap_to_their_entries() { + let lex = Lexicon::new([ + "Irish Hill", + "Gulf Shores", + "Merrill Lynch", + "Cabell County", + "Tri-State Bank", + ]); + for (read, want) in [ + ("Trish HTlI", "Irish Hill"), + ("GuIF Shores", "Gulf Shores"), + ("MerrITLynch", "Merrill Lynch"), + ("Cabe]l County", "Cabell County"), + ("Tr1-State Bank", "Tri-State Bank"), + ] { + match lex.snap(read) { + Snap::Snapped { entry, .. } => assert_eq!(entry, want, "{read}"), + other => panic!("{read}: expected a snap to {want}, got {other:?}"), + } + } + } + + #[test] + fn exact_reads_and_far_reads_are_reported_as_such() { + let lex = Lexicon::new(["Irish Hill", "Gulf Shores"]); + assert_eq!(lex.snap(" Irish Hill "), Snap::Exact("Irish Hill".into())); + assert!(matches!( + lex.snap("Something Else Entirely"), + Snap::Ambiguous(_) + )); + } + + /// Two entries equally close must not be guessed between. + #[test] + fn a_close_call_is_ambiguous_not_a_guess() { + let lex = Lexicon::new(["Unit 10", "Unit 1O"]); + assert!(matches!(lex.snap("Unit 1o"), Snap::Ambiguous(_))); + } +} diff --git a/crates/ffai-carmenta/src/lib.rs b/crates/ffai-carmenta/src/lib.rs index 0e818b51..7d8bc0ce 100644 --- a/crates/ffai-carmenta/src/lib.rs +++ b/crates/ffai-carmenta/src/lib.rs @@ -33,6 +33,13 @@ pub mod content; pub mod craft; pub mod crnn; pub mod engine; +pub mod manifests; +pub mod checkbox; +pub mod fold; +pub mod forms; +pub mod lexicon; +pub mod orient; +pub mod rules; /// Great Gate feature tap (plan §11 W1). Off unless `FFAI_GATE_HARVEST` is set; /// offline analysis only, never consulted by the engine's own decisions. pub mod harvest; diff --git a/crates/ffai-carmenta/src/manifests.rs b/crates/ffai-carmenta/src/manifests.rs new file mode 100644 index 00000000..6055db4d --- /dev/null +++ b/crates/ffai-carmenta/src/manifests.rs @@ -0,0 +1,95 @@ +//! The OCR model manifests, compiled into the crate. +//! +//! Every other constructor reads manifests from a `models/` directory on disk, +//! and that directory exists only in the `FFai` repository — not in the crate +//! crates.io ships. A user who added `ffai-carmenta` as a dependency got +//! `i/o error: The system cannot find the path specified` until they copied the +//! TOMLs in by hand (docs/plans/commercial-gaps.md, gap 10). +//! +//! The manifests are small, fixed per release, and carry the one thing a +//! caller must not get wrong — the SHA-256 of each weight file — so they belong +//! in the binary. Weights do not: they are licensed data, fetched or placed by +//! the caller, and never vendored. +//! +//! The copies under `crates/ffai-carmenta/models/` must stay byte-identical to +//! the repository's `models/`; `embedded_manifests_match_the_repository` fails +//! the moment they drift. + +use ffai_core::error::{Error, Result}; +use ffai_models::ModelManifest; + +/// `(model name, manifest TOML)` for every model a Carmenta engine can load. +pub const EMBEDDED: &[(&str, &str)] = &[ + ("craft-mlt", include_str!("../models/craft-mlt.toml")), + ( + "crnn-english-g2", + include_str!("../models/crnn-english-g2.toml"), + ), + ( + "crnn-zh-sim-g2", + include_str!("../models/crnn-zh-sim-g2.toml"), + ), + ("parseq-tiny", include_str!("../models/parseq-tiny.toml")), + ( + "ppocrv5-mobile-det", + include_str!("../models/ppocrv5-mobile-det.toml"), + ), + ( + "ppocrv5-mobile-rec", + include_str!("../models/ppocrv5-mobile-rec.toml"), + ), +]; + +/// Every embedded manifest, parsed. +pub fn embedded() -> Result> { + EMBEDDED + .iter() + .map(|(name, toml)| { + ModelManifest::from_toml(toml) + .map_err(|e| Error::Model(format!("embedded manifest `{name}`: {e}"))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each entry's key is the manifest's own `name`, so a lookup by key can + /// never return a different model's checksums. + #[test] + fn embedded_manifests_parse_and_are_keyed_by_their_own_name() { + let parsed = embedded().unwrap(); + assert_eq!(parsed.len(), EMBEDDED.len()); + for ((key, _), m) in EMBEDDED.iter().zip(&parsed) { + assert_eq!(*key, m.name); + assert_eq!(m.task, "ocr"); + assert!(!m.files.is_empty(), "{key}: no files"); + assert!( + m.files.iter().all(|f| f.sha256.is_some()), + "{key}: every embedded file must carry a checksum" + ); + } + } + + /// The crate's copies are the repository's manifests, byte for byte. Run + /// only where the repository exists — a crate unpacked from crates.io has + /// no `../../models` and has nothing to drift from. + #[test] + fn embedded_manifests_match_the_repository() { + let repo = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../models"); + if !repo.is_dir() { + eprintln!("SKIP embedded manifest sync: no repository models/ directory"); + return; + } + for (name, toml) in EMBEDDED { + let on_disk = std::fs::read_to_string(repo.join(format!("{name}.toml"))) + .unwrap_or_else(|e| panic!("repository models/{name}.toml: {e}")); + assert!( + on_disk == *toml, + "crates/ffai-carmenta/models/{name}.toml has drifted from models/{name}.toml — \ + copy the repository file over the crate's" + ); + } + } +} diff --git a/crates/ffai-carmenta/src/orient.rs b/crates/ffai-carmenta/src/orient.rs new file mode 100644 index 00000000..8d48e60b --- /dev/null +++ b/crates/ffai-carmenta/src/orient.rs @@ -0,0 +1,198 @@ +//! Page orientation: which quarter turn makes a page upright +//! (docs/plans/commercial-gaps.md, gap 5). +//! +//! Scanners store portrait forms sideways, and phone photos arrive in any of +//! four turns. Read sideways, a clean page came back with a fraction of its +//! lines at 0.38-0.58 confidence, with no error to say why. The engine-side +//! decision ([`crate::engine::CraftCrnn::detect_orientation`]) uses two +//! signals the models already produce: +//! +//! * **axis**, from detection: the detector draws WIDE boxes around text lines +//! that run horizontally and TALL ones around lines that run vertically, so +//! box aspect, weighted by area, separates {0, 180} from {90, 270}; +//! * **which way up**, from recognition: a sample of the largest boxes is read +//! in both candidate turns, and the upright turn reads with clearly higher +//! confidence (0.15 or more on every page a caller measured). +//! +//! This module holds the pure geometry: rotation and mapping boxes back. + +use ffai_core::types::{BoundingBox, ImageBuffer}; + +/// Rotate an image `turns` quarter turns CLOCKWISE (0-3; taken mod 4). +#[must_use] +pub fn rotate(img: &ImageBuffer, turns: u8) -> ImageBuffer { + let turns = turns % 4; + if turns == 0 { + return img.clone(); + } + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let (nw, nh) = if turns == 2 { (w, h) } else { (h, w) }; + let mut out = vec![0u8; w * h * bpp]; + for y in 0..h { + let src_row = &img.data[y * w * bpp..(y + 1) * w * bpp]; + for x in 0..w { + let (nx, ny) = match turns { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + _ => (y, w - 1 - x), + }; + let d = (ny * nw + nx) * bpp; + out[d..d + bpp].copy_from_slice(&src_row[x * bpp..(x + 1) * bpp]); + } + } + ImageBuffer { + width: nw as u32, + height: nh as u32, + format: img.format, + data: out, + } +} + +/// Keep every `f`-th pixel in each direction (nearest-neighbour decimation). +/// For the axis vote only: box ASPECT survives a 2-3x reduction intact, and +/// detection cost falls with the square of it. +#[must_use] +pub fn decimate(img: &ImageBuffer, f: usize) -> ImageBuffer { + if f <= 1 { + return img.clone(); + } + let (w, h) = (img.width as usize, img.height as usize); + let (nw, nh) = (w.div_ceil(f), h.div_ceil(f)); + let bpp = img.format.bytes_per_pixel(); + let mut data = Vec::with_capacity(nw * nh * bpp); + for y in (0..h).step_by(f) { + for x in (0..w).step_by(f) { + let s = (y * w + x) * bpp; + data.extend_from_slice(&img.data[s..s + bpp]); + } + } + ImageBuffer { + width: u32::try_from(nw).unwrap_or(u32::MAX), + height: u32::try_from(nh).unwrap_or(u32::MAX), + format: img.format, + data, + } +} + +/// Map a box found in the image rotated `turns` clockwise back to the frame +/// of the ORIGINAL `(orig_w, orig_h)` image. Boxes are edge coordinates +/// (`x .. x + width`), so a quarter turn maps them exactly. +#[must_use] +pub fn unrotate_box(b: BoundingBox, turns: u8, orig_w: f32, orig_h: f32) -> BoundingBox { + let turns = turns % 4; + // Size of the rotated frame the box lives in. + let (rw, rh) = if turns % 2 == 1 { + (orig_h, orig_w) + } else { + (orig_w, orig_h) + }; + // Rotating the rotated frame by (4 - turns) clockwise quarter turns + // returns to the original. One clockwise turn of a frame (W, H) sends the + // point (x, y) to (H - y, x). + let mut corners = [(b.x, b.y), (b.x + b.width, b.y + b.height)]; + let (mut fw, mut fh) = (rw, rh); + for _ in 0..(4 - turns) % 4 { + for c in &mut corners { + *c = (fh - c.1, c.0); + } + (fw, fh) = (fh, fw); + } + let (x0, x1) = ( + corners[0].0.min(corners[1].0), + corners[0].0.max(corners[1].0), + ); + let (y0, y1) = ( + corners[0].1.min(corners[1].1), + corners[0].1.max(corners[1].1), + ); + BoundingBox { + x: x0, + y: y0, + width: x1 - x0, + height: y1 - y0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ffai_core::types::PixelFormat; + + fn img(w: u32, h: u32) -> ImageBuffer { + ImageBuffer { + width: w, + height: h, + format: PixelFormat::Gray8, + data: (0..w * h).map(|i| i as u8).collect(), + } + } + + #[test] + fn four_quarter_turns_are_the_identity_and_two_are_a_half_turn() { + let a = img(5, 3); + let mut r = a.clone(); + for _ in 0..4 { + r = rotate(&r, 1); + } + assert_eq!(r.data, a.data); + assert_eq!(rotate(&rotate(&a, 1), 1).data, rotate(&a, 2).data); + assert_eq!(rotate(&rotate(&a, 3), 1).data, a.data); + let q = rotate(&a, 1); + assert_eq!((q.width, q.height), (3, 5)); + } + + #[test] + fn a_clockwise_turn_moves_the_top_left_pixel_to_the_top_right() { + // 2x1 [10, 20] turned clockwise is 1x2 with 10 on top. + let a = ImageBuffer { + width: 2, + height: 1, + format: PixelFormat::Gray8, + data: vec![10, 20], + }; + assert_eq!(rotate(&a, 1).data, vec![10, 20]); + assert_eq!(rotate(&a, 3).data, vec![20, 10]); + } + + /// A box drawn around specific pixels in a rotated image maps back onto + /// the same pixels in the original, for every turn. + #[test] + fn unrotate_box_lands_on_the_same_pixels() { + let (w, h) = (7usize, 4usize); + // Mark a 2x1 block of distinct values at (x 2..4, y 1..2). + let mut data = vec![0u8; w * h]; + data[w + 2] = 200; + data[w + 3] = 201; + let orig = ImageBuffer { + width: w as u32, + height: h as u32, + format: PixelFormat::Gray8, + data, + }; + for turns in 0..4u8 { + let r = rotate(&orig, turns); + let (rw, rh) = (r.width as usize, r.height as usize); + let marked: Vec<(usize, usize)> = (0..rh) + .flat_map(|y| (0..rw).map(move |x| (x, y))) + .filter(|&(x, y)| r.data[y * rw + x] >= 200) + .collect(); + let x0 = marked.iter().map(|p| p.0).min().unwrap(); + let x1 = marked.iter().map(|p| p.0).max().unwrap() + 1; + let y0 = marked.iter().map(|p| p.1).min().unwrap(); + let y1 = marked.iter().map(|p| p.1).max().unwrap() + 1; + let b = BoundingBox { + x: x0 as f32, + y: y0 as f32, + width: (x1 - x0) as f32, + height: (y1 - y0) as f32, + }; + let back = unrotate_box(b, turns, w as f32, h as f32); + assert_eq!( + (back.x, back.y, back.width, back.height), + (2.0, 1.0, 2.0, 1.0), + "turns {turns}" + ); + } + } +} diff --git a/crates/ffai-carmenta/src/rules.rs b/crates/ffai-carmenta/src/rules.rs new file mode 100644 index 00000000..a1e447aa --- /dev/null +++ b/crates/ffai-carmenta/src/rules.rs @@ -0,0 +1,274 @@ +//! Remove horizontal form rules before reading a page +//! (docs/plans/commercial-gaps.md, gap 12). +//! +//! A typed value sitting on a form's field line, or a line a filer struck +//! through, fuses the rule into the glyphs: `1` gains a foot and reads as `T` +//! or `I`, `0` reads as `U`, and the recognizer is confident about it. The +//! rule carries no text, so erasing it before detection and recognition gives +//! the models the glyphs alone. +//! +//! What counts as a rule pixel, and why each condition is there: +//! +//! * **dark**, by an Otsu threshold over the page; +//! * **in a long horizontal dark run** (at least [`RuleParams::min_len`]) — +//! text breaks between words and inside most glyphs, a rule does not; +//! * **vertically thin** — the dark run through it, top to bottom, is at most +//! [`RuleParams::max_thick`] pixels. A glyph stroke that crosses the rule, or +//! stands on it, makes its column's vertical run tall, so those pixels are +//! KEPT and the stroke survives intact. A filled box (a redaction patch) is +//! tall everywhere and is never touched, which matters: erasing a redaction +//! would be the opposite of what a privacy-honouring caller wants. +//! +//! Erased pixels take the page's background value (the mean of the light +//! class). Vertical rules are untouched. + +use ffai_core::types::{ImageBuffer, PixelFormat}; + +/// Thresholds for [`remove_horizontal_rules`]. +#[derive(Debug, Clone, Copy)] +pub struct RuleParams { + /// Minimum horizontal dark run, in pixels, to be considered a rule. + pub min_len: usize, + /// Maximum vertical thickness, in pixels, of a pixel that is erased. + pub max_thick: usize, +} + +impl RuleParams { + /// Defaults scaled to the page: a rule is at least 1/16 of the width + /// (and never under 60 px), and at most 8 px thick. A 300 ppi scan draws a + /// field line 2-4 px thick; glyph stems are an x-height tall. + #[must_use] + pub fn for_width(width: usize) -> Self { + let env = |k: &str| std::env::var(k).ok().and_then(|v| v.parse::().ok()); + Self { + min_len: env("FFAI_RULE_MIN_LEN").unwrap_or((width / 16).max(60)), + max_thick: env("FFAI_RULE_MAX_THICK").unwrap_or(8), + } + } +} + +/// What [`remove_horizontal_rules`] did, for instruments and tests. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RuleStats { + /// Pixels erased. + pub erased: usize, + /// Distinct rows that had at least one pixel erased. + pub rows: usize, +} + +pub(crate) fn luma(img: &ImageBuffer) -> Vec { + match img.format { + PixelFormat::Gray8 => img.data.clone(), + PixelFormat::Rgb8 | PixelFormat::Rgba8 => { + let bpp = img.format.bytes_per_pixel(); + img.data + .chunks_exact(bpp) + .map(|p| { + ((77 * u32::from(p[0]) + 150 * u32::from(p[1]) + 29 * u32::from(p[2])) >> 8) + as u8 + }) + .collect() + } + } +} + +/// Otsu threshold and the mean of the light class. +pub(crate) fn otsu(lum: &[u8]) -> (u8, u8) { + let mut hist = [0u64; 256]; + for &v in lum { + hist[v as usize] += 1; + } + let total = lum.len() as f64; + let sum_all: f64 = hist + .iter() + .enumerate() + .map(|(i, &c)| i as f64 * c as f64) + .sum(); + let (mut w_b, mut sum_b, mut best, mut t) = (0f64, 0f64, -1f64, 128u8); + for (i, &c) in hist.iter().enumerate() { + w_b += c as f64; + if w_b == 0.0 { + continue; + } + let w_f = total - w_b; + if w_f == 0.0 { + break; + } + sum_b += i as f64 * c as f64; + let (m_b, m_f) = (sum_b / w_b, (sum_all - sum_b) / w_f); + let between = w_b * w_f * (m_b - m_f) * (m_b - m_f); + if between > best { + best = between; + t = i as u8; + } + } + let light: (f64, f64) = hist + .iter() + .enumerate() + .skip(t as usize + 1) + .fold((0.0, 0.0), |(s, n), (i, &c)| { + (s + i as f64 * c as f64, n + c as f64) + }); + let bg = if light.1 > 0.0 { + (light.0 / light.1).round() as u8 + } else { + 255 + }; + (t, bg) +} + +/// Erase horizontal rules, returning the cleaned image and what was erased. +/// The input is unchanged; the output has the same size and format. +#[must_use] +pub fn remove_horizontal_rules(img: &ImageBuffer, p: RuleParams) -> (ImageBuffer, RuleStats) { + let (w, h) = (img.width as usize, img.height as usize); + let mut out = img.clone(); + if w == 0 || h == 0 { + return (out, RuleStats::default()); + } + let lum = luma(img); + let (t, bg) = otsu(&lum); + // A page with no contrast has nothing to separate. + if bg <= t { + return (out, RuleStats::default()); + } + let dark: Vec = lum.iter().map(|&v| v <= t).collect(); + + // Vertical dark-run length through every dark pixel, column by column. + let mut vrun = vec![0u16; w * h]; + for x in 0..w { + let mut y = 0; + while y < h { + if !dark[y * w + x] { + y += 1; + continue; + } + let start = y; + while y < h && dark[y * w + x] { + y += 1; + } + let len = (y - start).min(u16::MAX as usize) as u16; + for yy in start..y { + vrun[yy * w + x] = len; + } + } + } + + let bpp = img.format.bytes_per_pixel(); + let mut stats = RuleStats::default(); + for y in 0..h { + let row = &dark[y * w..(y + 1) * w]; + let mut touched = false; + let mut x = 0; + while x < w { + if !row[x] { + x += 1; + continue; + } + let start = x; + while x < w && row[x] { + x += 1; + } + if x - start < p.min_len { + continue; + } + for xx in start..x { + if (vrun[y * w + xx] as usize) <= p.max_thick { + let o = (y * w + xx) * bpp; + out.data[o..o + bpp.min(3)].fill(bg); + stats.erased += 1; + touched = true; + } + } + } + stats.rows += usize::from(touched); + } + (out, stats) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn page(w: usize, h: usize) -> Vec { + vec![255; w * h] + } + + fn fill(buf: &mut [u8], w: usize, x0: usize, y0: usize, x1: usize, y1: usize) { + for y in y0..y1 { + buf[y * w + x0..y * w + x1].fill(0); + } + } + + fn gray(data: Vec, w: usize, h: usize) -> ImageBuffer { + ImageBuffer { + width: w as u32, + height: h as u32, + format: PixelFormat::Gray8, + data, + } + } + + const P: RuleParams = RuleParams { + min_len: 60, + max_thick: 8, + }; + + #[test] + fn a_rule_is_erased_and_a_stem_standing_on_it_survives() { + let (w, h) = (300, 60); + let mut d = page(w, h); + fill(&mut d, w, 20, 40, 280, 43); // the rule: 3 px thick, 260 long + fill(&mut d, w, 100, 10, 104, 40); // a stem standing on it + let (out, stats) = remove_horizontal_rules(&gray(d, w, h), P); + let at = |x: usize, y: usize| out.data[y * w + x]; + assert_eq!(at(50, 41), 255, "rule pixel away from the stem is erased"); + assert_eq!(at(102, 20), 0, "the stem is untouched"); + assert_eq!(at(102, 41), 0, "the stem's column through the rule is kept"); + assert_eq!(stats.rows, 3); + } + + #[test] + fn a_filled_box_such_as_a_redaction_is_never_touched() { + let (w, h) = (300, 80); + let mut d = page(w, h); + fill(&mut d, w, 20, 10, 280, 60); // 50 px tall black patch + let img = gray(d, w, h); + let (out, stats) = remove_horizontal_rules(&img, P); + assert_eq!(stats.erased, 0); + assert_eq!(out.data, img.data); + } + + #[test] + fn short_dark_runs_such_as_glyph_bars_are_not_rules() { + let (w, h) = (300, 40); + let mut d = page(w, h); + fill(&mut d, w, 20, 10, 60, 13); // a 40 px bar: a 'T' top, not a rule + let (_, stats) = remove_horizontal_rules(&gray(d, w, h), P); + assert_eq!(stats.erased, 0); + } + + #[test] + fn rgb_input_keeps_its_format_and_size() { + let (w, h) = (200, 30); + let mut d = vec![255u8; w * h * 3]; + for x in 10..190 { + for c in 0..3 { + d[(20 * w + x) * 3 + c] = 0; + } + } + let img = ImageBuffer { + width: w as u32, + height: h as u32, + format: PixelFormat::Rgb8, + data: d, + }; + let (out, stats) = remove_horizontal_rules(&img, P); + assert_eq!( + (out.width, out.height, out.format), + (img.width, img.height, img.format) + ); + assert_eq!(stats.erased, 180); + assert!(out.data.iter().all(|&v| v == 255)); + } +} diff --git a/crates/ffai-carmenta/src/svtr.rs b/crates/ffai-carmenta/src/svtr.rs index 057e4da2..09d8b875 100644 --- a/crates/ffai-carmenta/src/svtr.rs +++ b/crates/ffai-carmenta/src/svtr.rs @@ -486,7 +486,7 @@ pub fn ctc_greedy(probs: &Tensor, charset: &[String]) -> Result<(String, Option< let best = probs.argmax(D::Minus1)?.flatten_all()?.to_vec1::()?; let conf = probs.max(D::Minus1)?.flatten_all()?.to_vec1::()?; let mut out = String::new(); - let (mut sum, mut kept) = (0f32, 0usize); + let mut kept = Vec::new(); let mut prev = u32::MAX; for i in 0..t { let k = best[i]; @@ -495,13 +495,73 @@ pub fn ctc_greedy(probs: &Tensor, charset: &[String]) -> Result<(String, Option< if let Some(s) = charset.get(k as usize - 1) { out.push_str(s); } - sum += conf[i]; - kept += 1; + kept.push(conf[i]); } prev = k; } let _ = n; - Ok((out, if kept == 0 { None } else { Some(sum / kept as f32) })) + Ok((out, crate::crnn::line_confidence(&kept))) +} + +/// Per-class allow mask for SVTR's head: index 0 (blank) always true, class +/// `k` true when charset entry `k - 1` is non-empty and every character in it +/// is in `allowed`. Entries can be multi-character, so an entry is allowed +/// only when ALL of it is. A requested character no single-character entry +/// covers is an error naming it. +pub fn class_mask(charset: &[String], allowed: &str) -> Result> { + let missing: String = allowed + .chars() + .filter(|c| !charset.iter().any(|e| e.chars().eq(std::iter::once(*c)))) + .collect(); + if !missing.is_empty() { + return Err(candle_core::Error::Msg(format!( + "charset constraint: this recognizer cannot emit {missing:?}" + ))); + } + let mut mask = Vec::with_capacity(charset.len() + 1); + mask.push(true); + mask.extend( + charset.iter().map(|e| !e.is_empty() && e.chars().all(|c| allowed.contains(c))), + ); + Ok(mask) +} + +/// [`ctc_greedy`] restricted to the classes `mask` allows. As in the CRNN +/// twin, the kept confidence is the class's probability over the full +/// distribution, not renormalised, so a forced character reads as uncertain. +/// With every class allowed it equals `ctc_greedy`. +pub fn ctc_greedy_masked( + probs: &Tensor, + charset: &[String], + mask: &[bool], +) -> Result<(String, Option)> { + let (_, t, n) = probs.dims3()?; + if mask.len() != n || !mask[0] { + return Err(candle_core::Error::Msg(format!( + "class mask has {} entries; the head has {n} classes and the blank must be allowed", + mask.len() + ))); + } + let flat = probs.flatten_all()?.to_vec1::()?; + let mut out = String::new(); + let mut kept = Vec::new(); + let mut prev = usize::MAX; + for i in 0..t { + let row = &flat[i * n..(i + 1) * n]; + let (k, p) = row + .iter() + .enumerate() + .filter(|(j, _)| mask[*j]) + .fold((0usize, f32::NEG_INFINITY), |best, (j, &p)| if p > best.1 { (j, p) } else { best }); + if k != 0 && k != prev { + if let Some(s) = charset.get(k - 1) { + out.push_str(s); + } + kept.push(p); + } + prev = k; + } + Ok((out, crate::crnn::line_confidence(&kept))) } /// One emitted run of characters, with WHERE ON THE LINE it came from. diff --git a/crates/ffai-carmenta/tests/production.rs b/crates/ffai-carmenta/tests/production.rs new file mode 100644 index 00000000..cf6d5427 --- /dev/null +++ b/crates/ffai-carmenta/tests/production.rs @@ -0,0 +1,264 @@ +//! Contracts a production caller relies on, from docs/plans/commercial-gaps.md: +//! +//! * **gap 6: determinism.** The same page read twice gives the same text and +//! the same boxes, so a stored extraction can be re-derived and audited. +//! * **gaps 7 and 10: offline resolution.** `CraftCrnn::offline` needs no +//! manifest directory, never touches the network, and reads exactly what +//! the cache-backed constructor reads. +//! * **gap 3: charset constraint.** With every character allowed the +//! constrained decoder IS the free decoder; with a restricted set, nothing +//! outside it is emitted. +//! +//! Weights are fetched artifacts, so these SKIP loudly when absent, like +//! `oracles.rs` and `from_bytes.rs`. + +use ffai_carmenta::engine::{CraftCrnn, DetStage, RecStage}; +use ffai_core::engine::{OcrEngine, OcrOptions}; +use ffai_core::types::{ImageBuffer, OcrOutput}; +use std::path::PathBuf; + +fn weights_root() -> PathBuf { + ffai_models::cache_dir().join("models") +} + +fn have(model: &str, file: &str) -> bool { + weights_root().join(model).join(file).exists() +} + +fn have_mobiledet_svtr() -> bool { + have("ppocrv5-mobile-det", "det-fused.safetensors") + && have("ppocrv5-mobile-rec", "rec.safetensors") + && have("ppocrv5-mobile-rec", "charset.txt") +} + +fn have_mobiledet_crnn() -> bool { + have("ppocrv5-mobile-det", "det-fused.safetensors") + && have("crnn-english-g2", "crnn.safetensors") +} + +fn manifests() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../models") +} + +/// A real document page with enough lines to take the PARALLEL recognition +/// path (three or more), which is where a non-deterministic reduction would +/// show up if there were one. +fn doc_page() -> Option { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../corpora/clips/carmenta-omnidoc/omni-0000.png"); + p.exists() + .then(|| ffai_media::load_image(&p).expect("doc page loads")) +} + +/// Every line as (text, bbox) — the parts of the output that must be exact. +fn lines(out: &OcrOutput) -> Vec<(String, Option<[i64; 4]>)> { + out.blocks + .iter() + .flat_map(|b| &b.lines) + .map(|l| { + let bb = l + .bbox + .map(|b| [b.x as i64, b.y as i64, b.width as i64, b.height as i64]); + (l.text.clone(), bb) + }) + .collect() +} + +fn confidences(out: &OcrOutput) -> Vec> { + out.blocks + .iter() + .flat_map(|b| &b.lines) + .map(|l| l.confidence) + .collect() +} + +#[test] +fn the_same_page_read_twice_is_identical() { + let Some(img) = doc_page() else { + eprintln!("SKIP determinism: doc page missing"); + return; + }; + if !have_mobiledet_svtr() { + eprintln!("SKIP determinism: mobiledet-svtr weights not in cache"); + return; + } + let opts = OcrOptions::default(); + // Two independent instances as well as two calls on one, so neither lazy + // loading nor any per-instance state can hide a difference. + let a = CraftCrnn::variant_in(RecStage::Svtr, DetStage::MobileDet, &manifests()); + let b = CraftCrnn::variant_in(RecStage::Svtr, DetStage::MobileDet, &manifests()); + let r1 = a.recognize(&img, &opts).unwrap(); + let r2 = a.recognize(&img, &opts).unwrap(); + let r3 = b.recognize(&img, &opts).unwrap(); + + let l1 = lines(&r1); + assert!( + l1.len() >= 3, + "page should exercise the parallel path, got {} lines", + l1.len() + ); + assert_eq!(l1, lines(&r2), "text or boxes changed between two calls"); + assert_eq!( + l1, + lines(&r3), + "text or boxes changed between two instances" + ); + + // Confidences: §8.46 measured ~1e-4 drift when OTHER processes compete for + // the CPU. Measured here rather than assumed, and bounded well below any + // threshold a caller would gate on. + let drift = confidences(&r1) + .iter() + .zip(confidences(&r2).iter().chain(confidences(&r3).iter())) + .filter_map(|(x, y)| Some((x.as_ref()? - y.as_ref()?).abs())) + .fold(0f32, f32::max); + eprintln!( + "determinism: {} lines identical x3, max confidence drift {drift:e}", + l1.len() + ); + assert!( + drift <= 1e-3, + "confidence drift {drift} is large enough to flip a gate" + ); +} + +#[test] +fn offline_reads_what_the_cache_backed_constructor_reads() { + let Some(img) = doc_page() else { + eprintln!("SKIP offline: doc page missing"); + return; + }; + if !have_mobiledet_svtr() { + eprintln!("SKIP offline: mobiledet-svtr weights not in cache"); + return; + } + let opts = OcrOptions::default(); + let cached = CraftCrnn::variant_in(RecStage::Svtr, DetStage::MobileDet, &manifests()); + let offline = CraftCrnn::offline(RecStage::Svtr, DetStage::MobileDet, &weights_root()); + offline + .preload() + .expect("offline engine loads from the cache layout"); + assert_eq!( + lines(&cached.recognize(&img, &opts).unwrap()), + lines(&offline.recognize(&img, &opts).unwrap()), + "offline resolution read different weights or manifests" + ); +} + +#[test] +fn offline_with_no_weights_fails_at_preload_naming_the_path() { + let empty = std::env::temp_dir().join(format!("ffai-offline-empty-{}", std::process::id())); + std::fs::create_dir_all(&empty).unwrap(); + let e = CraftCrnn::offline(RecStage::Svtr, DetStage::MobileDet, &empty); + let msg = match e.preload() { + Ok(()) => panic!("an empty weights directory must not load"), + Err(err) => err.to_string(), + }; + let want = empty + .join("ppocrv5-mobile-det") + .join("det-fused.safetensors"); + assert!( + msg.contains(&want.display().to_string()), + "error must name the expected file {}: {msg}", + want.display() + ); + let _ = std::fs::remove_dir_all(&empty); +} + +#[test] +fn a_fully_permissive_charset_is_the_free_decode() { + let Some(img) = doc_page() else { + eprintln!("SKIP charset: doc page missing"); + return; + }; + if !have_mobiledet_crnn() { + eprintln!("SKIP charset: mobiledet-crnn weights not in cache"); + return; + } + let e = CraftCrnn::variant_in(RecStage::Crnn, DetStage::MobileDet, &manifests()); + let free = e.recognize(&img, &OcrOptions::default()).unwrap(); + let all = OcrOptions { + charset: Some(ffai_carmenta::crnn::CHARSET.to_string()), + ..OcrOptions::default() + }; + let masked = e.recognize(&img, &all).unwrap(); + assert_eq!( + lines(&free), + lines(&masked), + "allowing every class changed the text" + ); + assert_eq!( + confidences(&free), + confidences(&masked), + "allowing every class changed confidence" + ); +} + +#[test] +fn a_digit_charset_emits_nothing_outside_it() { + let Some(img) = doc_page() else { + eprintln!("SKIP charset: doc page missing"); + return; + }; + if !have_mobiledet_svtr() { + eprintln!("SKIP charset: mobiledet-svtr weights not in cache"); + return; + } + let allowed = "0123456789$,.- "; + let e = CraftCrnn::variant_in(RecStage::Svtr, DetStage::MobileDet, &manifests()); + let free = e.recognize(&img, &OcrOptions::default()).unwrap(); + let digits = e + .recognize( + &img, + &OcrOptions { + charset: Some(allowed.into()), + ..OcrOptions::default() + }, + ) + .unwrap(); + let text = digits.text(); + assert!( + text.chars().all(|c| allowed.contains(c) || c == '\n'), + "constrained output contains a character outside {allowed:?}: {text:?}" + ); + // A page of prose forced into digits: the model did not want these + // characters, and the confidence must say so. + let mean = |o: &OcrOutput| { + let c: Vec = confidences(o).into_iter().flatten().collect(); + c.iter().sum::() / c.len().max(1) as f32 + }; + eprintln!( + "charset: free mean conf {:.3}, digits-only {:.3}", + mean(&free), + mean(&digits) + ); + assert!( + mean(&digits) < mean(&free), + "forced characters should read as less confident" + ); +} + +#[test] +fn a_character_the_model_cannot_emit_is_an_error_naming_it() { + let Some(img) = doc_page() else { + eprintln!("SKIP charset: doc page missing"); + return; + }; + if !have_mobiledet_crnn() { + eprintln!("SKIP charset: mobiledet-crnn weights not in cache"); + return; + } + let e = CraftCrnn::variant_in(RecStage::Crnn, DetStage::MobileDet, &manifests()); + // The English CRNN has no CJK classes. + let err = e + .recognize( + &img, + &OcrOptions { + charset: Some("0123中".into()), + ..OcrOptions::default() + }, + ) + .expect_err("an unrepresentable character must be refused") + .to_string(); + assert!(err.contains('中'), "error should name the character: {err}"); +} diff --git a/crates/ffai-cli/Cargo.toml b/crates/ffai-cli/Cargo.toml index 904f8495..df74d0a5 100644 --- a/crates/ffai-cli/Cargo.toml +++ b/crates/ffai-cli/Cargo.toml @@ -2,7 +2,7 @@ name = "ffai-cli" description = "The ffai binary — a thin shell over the FFai library crates" # follows its dependencies. -version = "0.6.8" +version = "0.7.0" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -18,7 +18,7 @@ path = "src/main.rs" [dependencies] ffai-core = { workspace = true } -ffai-media = { workspace = true } +ffai-media = { workspace = true, features = ["pdf"] } ffai-models = { workspace = true } ffai-mercury = { workspace = true } ffai-carmenta = { workspace = true } diff --git a/crates/ffai-cli/src/main.rs b/crates/ffai-cli/src/main.rs index 1e831b3c..5473c7eb 100644 --- a/crates/ffai-cli/src/main.rs +++ b/crates/ffai-cli/src/main.rs @@ -149,6 +149,11 @@ enum Cmd { /// measurement (see docs/benchmarking.md). #[arg(long)] fetch: Option, + /// Check that this model's files are present and match their + /// checksums, WITHOUT downloading anything. Repeatable. Exits non-zero + /// on the first model that fails, so a batch job can gate on it. + #[arg(long)] + verify: Vec, }, /// Transcribe speech to text (Mercury) Asr { @@ -226,6 +231,25 @@ enum Cmd { /// Language hints, repeatable #[arg(long)] language: Vec, + /// Restrict recognition to exactly these characters, e.g. + /// `--charset '0123456789$,.-'` for an amount field. Confidence then + /// reports how much the model wanted a character it was not allowed. + #[arg(long)] + charset: Option, + /// Erase horizontal form rules (field lines, strike-throughs) before + /// reading. Helps the CRNN engines on ruled forms; the SVTR document + /// default reads underlines well without it. + #[arg(long)] + remove_rules: bool, + /// Detect which way up the page is (0/90/180/270) and read it upright. + /// For scans stored sideways and phone photos. + #[arg(long)] + auto_orient: bool, + /// Also report checkboxes: each box's frame and whether it is marked + /// (`checkbox x y w h checked|empty`), from the pixels rather than as + /// the characters a text reader would make of them. + #[arg(long)] + checkboxes: bool, /// LIVE mode: treat the input as a frame sequence and emit a timed /// text track (mission plan §4.1) #[arg(long)] @@ -625,6 +649,23 @@ fn match_candle_threads(cmd: &Cmd) { } } +/// `ffai ocr --checkboxes`: one line per box, frame then state. +fn print_checkboxes(image: &ffai_core::types::ImageBuffer) { + for b in + ffai_carmenta::checkbox::find(image, ffai_carmenta::checkbox::CheckboxParams::default()) + { + println!( + "checkbox {:.0} {:.0} {:.0} {:.0} {} (fill {:.3})", + b.bbox.x, + b.bbox.y, + b.bbox.width, + b.bbox.height, + if b.checked { "checked" } else { "empty" }, + b.fill + ); + } +} + fn main() -> Result<()> { // Parse FIRST: the thread cap is per-command now, so it needs to know // which command. Clap spawns no threads, so the `set_var` inside is still @@ -656,9 +697,28 @@ fn main() -> Result<()> { ); } } - Cmd::Models { dir, fetch } => { + Cmd::Models { dir, fetch, verify } => { let manifests = ffai_models::load_dir(&dir) .with_context(|| format!("reading manifests from {}", dir.display()))?; + if !verify.is_empty() { + for name in &verify { + let manifest = + manifests + .iter() + .find(|m| &m.name == name) + .with_context(|| { + format!("no model manifest named `{name}` in {}", dir.display()) + })?; + let resolved = manifest + .verify() + .with_context(|| format!("verifying `{name}` (no download attempted)"))?; + println!( + "ok {name} ({} files, checksums match)", + resolved.files.len() + ); + } + return Ok(()); + } if let Some(name) = fetch { let manifest = manifests.iter().find(|m| m.name == name).with_context(|| { format!("no model manifest named `{name}` in {}", dir.display()) @@ -811,6 +871,10 @@ fn main() -> Result<()> { input, engine, language, + charset, + remove_rules, + auto_orient, + checkboxes, live, fps, change_fraction, @@ -820,6 +884,9 @@ fn main() -> Result<()> { } => { let opts = OcrOptions { languages: language, + charset, + remove_rules, + auto_orient, ..Default::default() }; // §8.171: the DOCUMENT default is SVTR (+1.521 pp macro on the @@ -918,10 +985,46 @@ fn main() -> Result<()> { } None => print!("{body}"), } + } else if std::fs::read(&input).is_ok_and(|b| b.starts_with(b"%PDF-")) { + // A PDF: every page as a viewer shows it (redactions and + // overlays painted in, turned upright), never the raw scan. + let pages = ffai_media::pdf::pdf_pages(&std::fs::read(&input)?)?; + for page in &pages { + println!("--- page {} ---", page.index + 1); + match &page.image { + Some(image) => { + println!("{}", eng.recognize(image, &opts)?.text()); + if checkboxes { + print_checkboxes(image); + } + } + None if page.has_text_layer && page.redaction_risk => { + // Text under a black box is still in the text + // layer; printing it would read through the + // redaction. Withheld, loudly. + eprintln!( + "page {}: text layer WITHHELD: something drawn over it may be \ + hiding text a viewer does not show (redaction risk)", + page.index + 1 + ); + } + None if page.has_text_layer => { + println!("{}", page.text.as_deref().unwrap_or_default()); + } + None => eprintln!( + "page {}: not read: {}", + page.index + 1, + page.note.as_deref().unwrap_or("no image") + ), + } + } } else { let image = ffai_media::load_image(&input)?; let out = eng.recognize(&image, &opts)?; println!("{}", out.text()); + if checkboxes { + print_checkboxes(&image); + } } if ffai_carmenta::profile::is_enabled() { eprint!("{}", ffai_carmenta::profile::profile().report()); diff --git a/crates/ffai-core/Cargo.toml b/crates/ffai-core/Cargo.toml index 81901c28..506ba3fa 100644 --- a/crates/ffai-core/Cargo.toml +++ b/crates/ffai-core/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai shared types, engine traits, and the engine registry" # published, so the crates.io copy of 0.6.1 cannot build ffai-diana — a gap # invisible locally, where path dependencies mask it, and caught only by # `cargo publish --dry-run`. -version = "0.7.1" +version = "0.8.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-core/src/engine.rs b/crates/ffai-core/src/engine.rs index 1970af29..b7a69d29 100644 --- a/crates/ffai-core/src/engine.rs +++ b/crates/ffai-core/src/engine.rs @@ -243,6 +243,36 @@ pub struct OcrOptions { /// detection becomes async maintenance, recognition the only /// synchronous work. pub single_line: bool, + /// Restrict recognition to exactly these characters — `"0123456789$,.-"` + /// for an amount field, `"0123456789/"` for a date. `None` reads freely. + /// + /// For a field whose content class is known this removes a whole class of + /// look-alike errors (`1`→`T`, `0`→`O`, `5`→`S`) instead of leaving them + /// for the caller to guess at. The reported confidence is the model's own + /// probability for the character it was allowed to emit, NOT renormalised + /// over the allowed set, so a line the model wanted to read as letters + /// comes back with LOW confidence rather than as a confident digit. + /// + /// An engine that cannot constrain its decoder must return an error when + /// this is set — never read freely and ignore it. A character the model + /// cannot emit at all is also an error, naming the character. + pub charset: Option, + /// Erase horizontal form rules (field lines, strike-throughs, table + /// rules) before reading. A typed value sitting on a field line fuses with + /// it — `1` reads as `T`, `0` as `U` — and the recognizer is confident + /// about the result. Off by default: it changes what every ruled page + /// reads, so it is asked for per call, by callers reading forms. + /// + /// Engines that do not implement it must return an error when it is set. + pub remove_rules: bool, + /// Detect which quarter turn makes the page upright, and read it that + /// way. Scanners store portrait forms sideways and phones photograph in + /// any orientation; read sideways, a page silently comes back as a few + /// low-confidence lines. Returned boxes stay in the INPUT image's + /// coordinates. Off by default: it costs an extra detection pass. + /// + /// Engines that do not implement it must return an error when it is set. + pub auto_orient: bool, } /// Detection options (Diana). diff --git a/crates/ffai-diana/Cargo.toml b/crates/ffai-diana/Cargo.toml index b88d16ab..3fea0682 100644 --- a/crates/ffai-diana/Cargo.toml +++ b/crates/ffai-diana/Cargo.toml @@ -3,7 +3,7 @@ name = "ffai-diana" description = "Diana — FFai's detection component (YOLO26 on candle, pure Rust)" # 0.7.3 is a documentation-only republish: README restructured to the shared # Remade With Rust page layout. No code, API or output change. -version = "0.7.6" +version = "0.7.7" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -35,7 +35,7 @@ ffai-core = { workspace = true } # Declared directly rather than inherited: `default-features = false` cannot # override a workspace dependency's defaults, and setting it at the workspace # level would silently strip `fetch` from every other crate that expects it. -ffai-models = { path = "../ffai-models", version = "0.6.4", default-features = false } +ffai-models = { path = "../ffai-models", version = "0.6.5", default-features = false } candle-core = { workspace = true } candle-nn = { workspace = true } serde = { workspace = true } diff --git a/crates/ffai-diana/README.md b/crates/ffai-diana/README.md index ffec741b..18d14040 100644 --- a/crates/ffai-diana/README.md +++ b/crates/ffai-diana/README.md @@ -34,7 +34,7 @@ cargo add ffai-diana ffai-core ffai-media ```toml [dependencies] ffai-diana = "0.7" -ffai-core = "0.6" +ffai-core = "0.8" ffai-media = "0.6" ``` diff --git a/crates/ffai-media/Cargo.toml b/crates/ffai-media/Cargo.toml index dfbf5ff2..8de19c4b 100644 --- a/crates/ffai-media/Cargo.toml +++ b/crates/ffai-media/Cargo.toml @@ -4,7 +4,7 @@ description = "FFai media ingest/egress — demux, decode, resample (rff-backed) # rusty_jpeg 0.1 -> 0.2 picks up the progressive-JPEG fix. A dependency # change with no API change, so a patch bump -- but a load-bearing one: # 0.6.1 cannot decode 35 of OmniDocBench's 236 pages. -version = "0.6.6" +version = "0.6.7" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -58,6 +58,12 @@ rff-codec = "0.2" # house rule, and the reason `ffai-cli` owns that decision. `std` and `asm` # are the two default features worth keeping, so they come back by name. rusty_h264 = { version = "0.16", default-features = false, features = ["std", "asm"] } +# PDF pages for OCR (`pdf` feature; docs/plans/commercial-gaps.md gaps 1, 2, +# 11). Both pure Rust, MIT. lopdf's defaults (chrono, jiff, rayon, time) are +# date handling and parallel parsing this module never uses. `fax` is the +# CCITT Group 3/4 decoder; a house `rusty_ccitt` would replace it. +lopdf = { version = "0.44", default-features = false, optional = true } +fax = { version = "0.3", optional = true } # THE PUBLISHING CONSTRAINT IS RESOLVED, and the note stays because it cost # real time to establish. @@ -83,6 +89,10 @@ rusty_h264 = { version = "0.16", default-features = false, features = ["std", "a # decoder cannot: demux and video codecs. Still a distribution question, no # longer a blocker for still images. remade-ffmpeg = [] +# Read PDF pages as a viewer shows them — scan composited with redactions and +# overlays, turned upright — for OCR. Off by default: it adds a PDF parser to +# a crate most users take for images and audio. +pdf = ["dep:lopdf", "dep:fax"] # wasm32 needs getrandom's browser backend selected explicitly. BOTH halves are # required and the crate errors if either is missing: this feature, plus diff --git a/crates/ffai-media/README.md b/crates/ffai-media/README.md index 1b148896..271468b8 100644 --- a/crates/ffai-media/README.md +++ b/crates/ffai-media/README.md @@ -11,13 +11,42 @@ println!("{:.1}s at {} Hz", audio.duration_secs(), audio.sample_rate); An `AudioBuffer` is always `f32` samples in `[-1.0, 1.0]` with a known sample rate and channel count. Engines resample and downmix from there — `AudioBuffer::to_mono()` averages channels, and ASR engines reject audio at the wrong rate rather than silently resampling badly. -## Where the codecs come from +## Images, from a file or from bytes + +```rust +let page = ffai_media::load_image("page.png")?; // from a path +let page = ffai_media::decode_image(&bytes_from_a_pdf)?; // from memory, same output +``` + +Both dispatch on the bytes' magic, not the file name: 169 of OmniDocBench's English pages are JPEGs named `.png`. `decode_image` exists for callers whose pages come out of a PDF, an archive or a network response, so they need no temporary file and no copy of the grayscale/RGB handling. + +## PDF pages as a viewer shows them (`pdf` feature) + +```toml +ffai-media = { version = "0.6", features = ["pdf"] } +``` -Container and codec work routes through [remade_ffmpeg_rs](https://github.com/Remade-With-Rust/remade_ffmpeg_rs) (`rff-*`), our own pure-Rust ffmpeg — FFai's principle 7, "codecs come from home". Formats land as that project lands them. +```rust +for page in ffai_media::pdf::pdf_pages(&pdf_bytes)? { + if let Some(image) = &page.image { /* OCR it */ } + else if page.has_text_layer && !page.redaction_risk { /* use page.text */ } +} +``` + +**A scanned PDF page is an image with things drawn on top of it, and extracting the image alone reads through redactions.** A real filing hid a home address under a black patch; the patch was a separate object, and OCR of the extracted scan returned the address. `pdf_pages` therefore returns the scan **composited with everything drawn after it**, turned upright: + +- filled boxes, later images (painted black) and nested form objects; +- redaction, square, circle, polygon and stamp annotations. -Today: **WAV** (native), **JPEG** (`rff-codec-jpeg`, through rff's `CodecRegistry`), **PNG** (the `png` crate). PNG has not moved to `rff-codec-png` yet for a measured reason rather than a cautious one — rff expands grayscale to packed `Rgb24` where this path returns `Gray8`, and Carmenta's OCR reads `load_image`, so the swap changes which preprocessing branch every grayscale page takes and needs Carmenta's corpus gates green first. +A page it cannot walk returns **no image** rather than the raw scan. + +A born-digital text layer has the same trap: text under a black box stays in the content stream. The page's `text` comes with a `redaction_risk` flag, set when anything could be hiding part of it. + +Decoders: JPEG, CCITT Group 4 and Group 3 1-D (fax scans), and raw or Flate 1- and 8-bit samples in gray, RGB or CMYK, including stencil masks. JBIG2 and JPEG 2000 are reported per page, not guessed. Every allocation is bounded before it happens: images at 100 MP, decompression at the size the dimensions justify, and form content at 64 MiB. + +## Where the codecs come from -Note on distribution: rff is not on crates.io, and `cargo publish` refuses a git dependency outright. Crates downstream of `ffai-media` therefore cannot be published until rff publishes — a deliberate trade of distribution for owning the stack, made so the deployment is Rust end to end. +Codecs come from home, FFai's principle 7. Still images decode through [`rusty_jpeg`](https://crates.io/crates/rusty_jpeg) and [`rusty_png`](https://crates.io/crates/rusty_png), and video ingest through [remade_ffmpeg_rs](https://github.com/Remade-With-Rust/remade_ffmpeg_rs) (`rff-*`). All are pure Rust and published on crates.io, so everything downstream of this crate is publishable. They are gated against what they replaced: `rusty_png` byte-identical to upstream `png` across every corpus image, and `rusty_jpeg` within 3/255 of libjpeg. The PDF feature adds `lopdf` and the `fax` CCITT decoder, both pure Rust and MIT. ## License diff --git a/crates/ffai-media/src/lib.rs b/crates/ffai-media/src/lib.rs index 73065b21..7d9b1251 100644 --- a/crates/ffai-media/src/lib.rs +++ b/crates/ffai-media/src/lib.rs @@ -9,6 +9,8 @@ //! clear "pending rff integration" error rather than silently failing. pub mod annexb; +#[cfg(feature = "pdf")] +pub mod pdf; use std::path::Path; @@ -142,11 +144,8 @@ pub fn load_image(path: &Path) -> Result { // decoder choice it was making. Reading once and passing the bytes down // costs nothing and removes a whole file read per image. let bytes = std::fs::read(path).map_err(|e| Error::Media(format!("open failed: {e}")))?; - if bytes.starts_with(&[0x89, b'P', b'N', b'G']) { - return decode_png(&bytes); - } - if bytes.starts_with(&[0xFF, 0xD8]) { - return decode_jpeg(bytes); + if let Some(decoded) = decode_by_magic(&bytes) { + return decoded; } let ext = path .extension() @@ -155,7 +154,7 @@ pub fn load_image(path: &Path) -> Result { .to_ascii_lowercase(); match ext.as_str() { "png" => decode_png(&bytes), - "jpg" | "jpeg" => decode_jpeg(bytes), + "jpg" | "jpeg" => decode_jpeg(&bytes), other => Err(Error::Media(format!( "`.{other}` decode is not wired yet — PNG and JPEG are supported; WebP/AVIF \ arrive with the rff image decoders. Convert with `ffmpeg -i in.{other} out.png`" @@ -163,6 +162,43 @@ pub fn load_image(path: &Path) -> Result { } } +/// Decode a still image from bytes already in memory — the same decoders and +/// the same output contract as [`load_image`], with no file involved. +/// +/// This exists because a caller whose pages come out of a PDF, an archive or a +/// network response holds BYTES, and `load_image` only took a path: the +/// choices were a temporary file per page, or calling `rusty_jpeg` directly +/// and re-implementing the grayscale/RGB handling this module already does +/// (docs/plans/commercial-gaps.md, gap 9). +/// +/// Dispatch is by magic bytes only — there is no filename to fall back on, so +/// anything that is not PNG or JPEG is an error naming what was found. +pub fn decode_image(bytes: &[u8]) -> Result { + decode_by_magic(bytes).unwrap_or_else(|| { + let head: Vec = bytes.iter().take(8).map(|b| format!("{b:02x}")).collect(); + Err(Error::Media(format!( + "unrecognised image bytes (first bytes {}) — PNG and JPEG are supported", + if head.is_empty() { + "".to_string() + } else { + head.join(" ") + } + ))) + }) +} + +/// The magic-byte half of both entry points. `None` means "not a format we +/// recognise by content", so the path-based caller can still try the extension. +fn decode_by_magic(bytes: &[u8]) -> Option> { + if bytes.starts_with(&[0x89, b'P', b'N', b'G']) { + return Some(decode_png(bytes)); + } + if bytes.starts_with(&[0xFF, 0xD8]) { + return Some(decode_jpeg(bytes)); + } + None +} + /// JPEG, decoded by **`rusty_jpeg`** — ours, from crates.io. /// /// Moved off `rff-codec-jpeg` for the same reason PNG did: rff is a git @@ -177,7 +213,7 @@ pub fn load_image(path: &Path) -> Result { /// OCR corpus is read through it, and changing an output FORMAT is a /// different decision from changing a decoder. Made separately, or not at /// all. -fn decode_jpeg(data: Vec) -> Result { +fn decode_jpeg(data: &[u8]) -> Result { use ffai_core::types::PixelFormat; use rusty_jpeg::{Decoder, PixelFormat as JpegFormat}; @@ -676,6 +712,43 @@ mod tests { let err = load_audio(Path::new("clip.mp3")).unwrap_err(); assert!(err.to_string().contains("remade_ffmpeg_rs")); } + + /// `decode_image` has no filename to fall back on, so unrecognised bytes + /// must be an error that shows what arrived — not a silent empty image. + #[test] + fn decode_image_refuses_unknown_bytes_and_says_what_it_saw() { + let err = decode_image(b"%PDF-1.7 not an image") + .unwrap_err() + .to_string(); + assert!( + err.contains("25 50 44 46"), + "error should show the leading bytes: {err}" + ); + let err = decode_image(&[]).unwrap_err().to_string(); + assert!(err.contains(""), "empty input should say so: {err}"); + } + + /// A PNG built in memory decodes through `decode_image` with no file and + /// no corpus — the case a PDF-page caller is in. + #[test] + fn decode_image_reads_an_in_memory_png() { + let (w, h) = (5u32, 3u32); + let pixels: Vec = (0..w * h).map(|i| (i * 17) as u8).collect(); + let mut bytes = Vec::new(); + { + let mut enc = png::Encoder::new(&mut bytes, w, h); + enc.set_color(png::ColorType::Grayscale); + enc.set_depth(png::BitDepth::Eight); + enc.write_header() + .unwrap() + .write_image_data(&pixels) + .unwrap(); + } + let img = decode_image(&bytes).unwrap(); + assert_eq!((img.width, img.height), (w, h)); + assert_eq!(img.format, ffai_core::types::PixelFormat::Gray8); + assert_eq!(img.data, pixels); + } } #[cfg(test)] @@ -778,6 +851,24 @@ mod png_oracle { eprintln!("SKIP jpeg/libjpeg twin check: corpus absent"); return; } + // The bytes entry point must be the path entry point with the file + // read removed: identical pixels for both formats. + for f in ["coco-000.src.jpg", "coco-000.png"] { + let p = root.join(f); + if p.exists() { + let by_path = load_image(&p).expect("path decode"); + let by_bytes = decode_image(&std::fs::read(&p).unwrap()).expect("bytes decode"); + assert_eq!( + (by_path.width, by_path.height), + (by_bytes.width, by_bytes.height) + ); + assert_eq!(by_path.format, by_bytes.format, "{f}: format"); + assert!( + by_path.data == by_bytes.data, + "{f}: decode_image differs from load_image" + ); + } + } assert!( worst <= 8, "rusty_jpeg diverges from libjpeg by {worst}/255 over {n} images" diff --git a/crates/ffai-media/src/pdf.rs b/crates/ffai-media/src/pdf.rs new file mode 100644 index 00000000..0a30d6d8 --- /dev/null +++ b/crates/ffai-media/src/pdf.rs @@ -0,0 +1,1190 @@ +//! PDF pages as a viewer shows them — for OCR of scanned documents +//! (docs/plans/commercial-gaps.md, gaps 1, 2 and 11). +//! +//! A scanned PDF page is an image `XObject` with things drawn on top of it. +//! Extracting the image alone is **unsafe**: a published filing hid a home +//! address under a black patch, a viewer showed the patch, and reading the +//! extracted scan returned the address at high confidence, because the patch +//! was a separate object drawn over the scan (gap 11). So this module never +//! returns a raw scan. It returns the scan **composited with everything drawn +//! after it**: +//! +//! * filled paths (a black box, a white-out), painted in their fill colour +//! over the bounding box of the path — over-painting a curved shape's +//! corners is the fail-safe direction; +//! * images drawn after the scan (a patch pasted in a later save), painted +//! black — an overlay may hide a stamp, it must never reveal what it covers; +//! * form `XObjects` drawn after the scan, walked recursively; +//! * `Redact`, `Square`, `Circle`, `Polygon` and `Stamp` annotations, painted +//! black over their rectangle. +//! +//! A page whose drawing cannot be walked returns **no image** and a note — the +//! caller cannot honour redactions it cannot see, so it is not handed pixels. +//! +//! The composited image is then turned upright the way a viewer shows it, +//! from the image's placement matrix and the page's `/Rotate`. +//! +//! Decoders: DCT (JPEG, via `rusty_jpeg`), CCITT Group 3 one-dimensional and +//! Group 4 (via the pure-Rust `fax` crate), and Flate/LZW/raw samples at 1 or +//! 8 bits in gray or RGB, including 1-bit stencil masks. JBIG2 and JPEG 2000 +//! are reported per page as unsupported rather than guessed at. +//! +//! Every decoded image is bounded by [`MAX_PIXELS`] before anything is +//! allocated for it, and decompression is capped at the size the declared +//! dimensions justify, so a hostile PDF cannot drive an unbounded allocation. +//! +//! Numbers read from the file (dimensions, component counts, parameters) go +//! through checked conversions. The casts that remain carry their own +//! `allow` and the reason the value is already bounded. + +// Page geometry is f64 affine arithmetic on coordinates a PDF writer chose. A +// fused multiply-add rounds differently from every other reader of the same +// file, and nothing here needs the extra precision. +#![allow(clippy::suboptimal_flops)] + +use lopdf::{Dictionary, Document, Object, ObjectId}; + +use ffai_core::error::{Error, Result}; +use ffai_core::types::{ImageBuffer, PixelFormat}; + +/// Largest image this module will decode, in pixels: 100 MP, a 600 ppi +/// tabloid page with room to spare. +pub const MAX_PIXELS: usize = 100_000_000; + +/// Deepest form `XObject` nesting walked before a page is refused. +const MAX_FORM_DEPTH: usize = 8; + +/// One page of a PDF. +#[derive(Debug, Clone)] +pub struct PdfPage { + /// Zero-based page index. + pub index: usize, + /// The page's content draws text (a born-digital page, or a scan with an + /// OCR text layer). A caller may prefer the text layer for such a page. + pub has_text_layer: bool, + /// The page's scan as a viewer shows it: overlays and redactions painted + /// in, turned upright. `None` when the page has no image, or when it has + /// one that could not be composited safely — see `note`. + pub image: Option, + /// Why `image` is `None`, or what was approximated. + pub note: Option, + /// The page's text layer, when it has one (`lopdf`'s extractor: text in + /// content-stream order, no positions). **Read `redaction_risk` before + /// using it.** + pub text: Option, + /// The text layer may contain text a viewer does NOT show. A black box + /// drawn over born-digital text hides it on screen and leaves it in the + /// content stream, where a text extractor reads straight through it — the + /// text-layer twin of the scan incident (gap 11). Set when the page carries + /// a `Redact`/`Square`/`Circle`/`Polygon`/`Stamp` annotation, or a dark + /// filled shape is drawn AFTER text was shown. A caller handling private + /// data should OCR the rendered `image` instead, or withhold the page. + pub redaction_risk: bool, +} + +/// Every page of `pdf`, in order. Errors only when the file itself cannot be +/// parsed; a page that cannot be rendered safely carries a note instead. +pub fn pdf_pages(pdf: &[u8]) -> Result> { + let doc = Document::load_mem(pdf).map_err(|e| Error::Media(format!("PDF parse: {e}")))?; + Ok(doc + .get_pages() + .into_iter() + .enumerate() + .map(|(index, (_, id))| page(&doc, index, id)) + .collect()) +} + +fn page(doc: &Document, index: usize, id: ObjectId) -> PdfPage { + let mut out = PdfPage { + index, + has_text_layer: false, + image: None, + note: None, + text: None, + redaction_risk: false, + }; + let walked = match walk_page(doc, id) { + Ok(w) => w, + Err(e) => { + // Unwalkable: nothing can be said about what covers the text + // either, so the risk is the fail-safe answer. + out.redaction_risk = true; + out.note = Some(format!("drawing could not be walked, image withheld: {e}")); + return out; + } + }; + out.has_text_layer = walked.text; + let covering_annotations = annotation_overlays(doc, id).map_or(true, |a| !a.is_empty()); + out.redaction_risk = walked.dark_fill_after_text || covering_annotations; + if walked.text { + // lopdf numbers pages from 1. + let number = u32::try_from(index + 1).unwrap_or(u32::MAX); + out.text = doc + .extract_text_with_limit(&[number], MAX_FORM_CONTENT) + .ok(); + } + let Some(scan) = walked.scan else { + out.note = Some("no image on this page".into()); + return out; + }; + let mut image = match decode_image_xobject(doc, scan.id) { + Ok(img) => img, + Err(e) => { + out.note = Some(format!("scan not decoded: {e}")); + return out; + } + }; + let mut overlays = walked.overlays; + match annotation_overlays(doc, id) { + Ok(a) => overlays.extend(a), + Err(e) => { + out.note = Some(format!( + "annotations could not be read, image withheld: {e}" + )); + return out; + } + } + paint(&mut image, &scan.ctm, &overlays); + let (turns, mirrored) = upright_turns(&scan.ctm, page_rotate(doc, id)); + if mirrored { + image = flip_horizontal(&image); + } + out.image = Some(rotate_cw(&image, turns)); + out +} + +// ---------------------------------------------------------------- drawing walk + +/// A 2-D affine map `[a b c d e f]`: (x, y) -> (a x + c y + e, b x + d y + f). +type Matrix = [f64; 6]; + +const IDENTITY: Matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + +/// `m` applied first, then `n`. +fn concat(m: Matrix, n: Matrix) -> Matrix { + [ + m[0] * n[0] + m[1] * n[2], + m[0] * n[1] + m[1] * n[3], + m[2] * n[0] + m[3] * n[2], + m[2] * n[1] + m[3] * n[3], + m[4] * n[0] + m[5] * n[2] + n[4], + m[4] * n[1] + m[5] * n[3] + n[5], + ] +} + +fn apply(m: &Matrix, x: f64, y: f64) -> (f64, f64) { + (m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5]) +} + +/// A region to paint, in page space, and the gray (0 black .. 1 white). +#[derive(Debug, Clone)] +struct Overlay { + corners: Vec<(f64, f64)>, + gray: f64, +} + +struct Scan { + id: ObjectId, + ctm: Matrix, + area: f64, +} + +struct Walked { + scan: Option, + overlays: Vec, + text: bool, + /// A dark filled shape was drawn after some text was shown: it may be + /// hiding that text on screen while leaving it in the text layer. + dark_fill_after_text: bool, +} + +/// A fill at or below this gray level counts as a box that could hide text. +const DARK_FILL: f64 = 0.2; + +fn real(o: &Object) -> Option { + match o { + // A coordinate or colour beyond 2^52 is not a page; precision there + // is irrelevant. + #[allow(clippy::cast_precision_loss)] + Object::Integer(i) => Some(*i as f64), + Object::Real(r) => Some(f64::from(*r)), + _ => None, + } +} + +fn gray_of(values: &[f64]) -> Option { + match *values { + [g] => Some(g), + [r, g, b] => Some(0.3 * r + 0.59 * g + 0.11 * b), + [c, m, y, k] => Some(1.0 - (0.3 * c + 0.59 * m + 0.11 * y + k).min(1.0)), + _ => None, + } +} + +fn walk_page(doc: &Document, page_id: ObjectId) -> std::result::Result { + let content = doc + .get_and_decode_page_content(page_id) + .map_err(|e| e.to_string())?; + let xobjects = page_xobjects(doc, page_id); + let mut w = Walked { + scan: None, + overlays: Vec::new(), + text: false, + dark_fill_after_text: false, + }; + walk(doc, &content.operations, &xobjects, IDENTITY, &mut w, 0)?; + Ok(w) +} + +/// The page's `XObject` dictionary, own resources first, then inherited. +fn page_xobjects(doc: &Document, page_id: ObjectId) -> Vec { + let Ok((own, inherited)) = doc.get_page_resources(page_id) else { + return Vec::new(); + }; + own.into_iter() + .cloned() + .chain( + inherited + .iter() + .filter_map(|id| doc.get_dictionary(*id).ok().cloned()), + ) + .filter_map(|r| xobject_dict(doc, &r)) + .collect() +} + +fn xobject_dict(doc: &Document, resources: &Dictionary) -> Option { + let x = resources.get(b"XObject").ok()?; + doc.dereference(x) + .ok() + .and_then(|(_, o)| o.as_dict().ok().cloned()) +} + +fn resolve_xobject(dicts: &[Dictionary], name: &[u8]) -> Option { + dicts + .iter() + .find_map(|d| d.get(name).ok()?.as_reference().ok()) +} + +/// Largest form `XObject` content stream decompressed, in bytes. A content +/// stream is drawing operators; 64 MiB of them is far past any real page, and +/// the cap is what stops a small hostile file inflating without bound. +const MAX_FORM_CONTENT: usize = 64 << 20; + +/// `Do`: an image becomes the scan or an overlay; a form is walked. +fn draw_xobject( + doc: &Document, + xobjects: &[Dictionary], + name: &[u8], + ctm: Matrix, + w: &mut Walked, + depth: usize, +) -> std::result::Result<(), String> { + let Some(id) = resolve_xobject(xobjects, name) else { + return Ok(()); + }; + let Ok(stream) = doc.get_object(id).and_then(Object::as_stream) else { + return Ok(()); + }; + match stream + .dict + .get(b"Subtype") + .ok() + .and_then(|t| t.as_name().ok()) + { + Some(b"Image") => { + let area = (ctm[0] * ctm[3] - ctm[1] * ctm[2]).abs(); + let corners = [(0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0)] + .map(|(x, y)| apply(&ctm, x, y)) + .to_vec(); + match &w.scan { + // A bigger image replaces the scan; everything drawn before it + // is underneath and irrelevant. + Some(s) if area <= s.area => w.overlays.push(Overlay { corners, gray: 0.0 }), + _ => { + w.scan = Some(Scan { id, ctm, area }); + w.overlays.clear(); + } + } + } + Some(b"Form") => { + let matrix = stream + .dict + .get(b"Matrix") + .ok() + .and_then(|m| m.as_array().ok()) + .map(|a| a.iter().filter_map(real).collect::>()) + .filter(|v| v.len() == 6) + .map_or(IDENTITY, |v| [v[0], v[1], v[2], v[3], v[4], v[5]]); + let content = if stream.dict.get(b"Filter").is_ok() { + stream + .decompressed_content_with_limit(MAX_FORM_CONTENT) + .map_err(|e| format!("form XObject content: {e}"))? + } else { + stream.content.clone() + }; + let ops = lopdf::content::Content::decode(&content) + .map_err(|e| format!("form XObject decode: {e}"))? + .operations; + let mut inner = xobjects.to_vec(); + if let Some(r) = stream + .dict + .get(b"Resources") + .ok() + .and_then(|r| doc.dereference(r).ok()) + .and_then(|(_, r)| r.as_dict().ok()) + .and_then(|r| xobject_dict(doc, r)) + { + inner.insert(0, r); + } + walk(doc, &ops, &inner, concat(matrix, ctm), w, depth + 1)?; + } + _ => {} + } + Ok(()) +} + +fn walk( + doc: &Document, + ops: &[lopdf::content::Operation], + xobjects: &[Dictionary], + base: Matrix, + w: &mut Walked, + depth: usize, +) -> std::result::Result<(), String> { + if depth > MAX_FORM_DEPTH { + return Err(format!("form XObjects nested deeper than {MAX_FORM_DEPTH}")); + } + let mut ctm = base; + let mut fill = 0.0f64; + let mut stack: Vec<(Matrix, f64)> = Vec::new(); + let mut path: Vec<(f64, f64)> = Vec::new(); + for op in ops { + let args: Vec = op.operands.iter().filter_map(real).collect(); + match op.operator.as_str() { + "q" => stack.push((ctm, fill)), + "Q" => (ctm, fill) = stack.pop().unwrap_or((ctm, fill)), + "cm" if args.len() == 6 => { + ctm = concat([args[0], args[1], args[2], args[3], args[4], args[5]], ctm); + } + "g" | "rg" | "k" | "sc" | "scn" => fill = gray_of(&args).unwrap_or(fill), + "re" if args.len() == 4 => { + let (x, y, rw, rh) = (args[0], args[1], args[2], args[3]); + for (px, py) in [(x, y), (x + rw, y), (x + rw, y + rh), (x, y + rh)] { + path.push(apply(&ctm, px, py)); + } + } + // Every point a path visits, control points included: a curve lies + // inside the hull of its control points, so the bounding box of + // these always contains the filled shape. + "m" | "l" | "c" | "v" | "y" => { + for pair in args.chunks_exact(2) { + path.push(apply(&ctm, pair[0], pair[1])); + } + } + "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => { + if w.text && fill <= DARK_FILL && !path.is_empty() { + w.dark_fill_after_text = true; + } + if w.scan.is_some() && !path.is_empty() { + w.overlays.push(Overlay { + corners: std::mem::take(&mut path), + gray: fill, + }); + } + path.clear(); + } + "n" | "S" | "s" | "W" | "W*" => { + if matches!(op.operator.as_str(), "n" | "S" | "s") { + path.clear(); + } + } + "Tj" | "TJ" | "'" | "\"" => w.text = true, + "Do" => { + if let Some(name) = op.operands.first().and_then(|o| o.as_name().ok()) { + draw_xobject(doc, xobjects, name, ctm, w, depth)?; + } + } + _ => {} + } + } + Ok(()) +} + +/// Opaque annotations drawn over the page, painted black. +/// +/// `/Annots` is resolved here rather than through lopdf's +/// `get_page_annotations`, which follows only indirect references: an +/// annotation stored inline in the array would be skipped, and a skipped +/// redaction is the one failure this module exists to prevent. +fn annotation_overlays( + doc: &Document, + page_id: ObjectId, +) -> std::result::Result, String> { + let page = doc.get_dictionary(page_id).map_err(|e| e.to_string())?; + let Ok(annots) = page.get(b"Annots") else { + return Ok(Vec::new()); + }; + let annots = doc.dereference(annots).map_err(|e| e.to_string())?.1; + let annots = annots + .as_array() + .map_err(|_| "/Annots is not an array".to_string())?; + let mut out = Vec::new(); + for item in annots { + let a = doc + .dereference(item) + .map_err(|e| e.to_string())? + .1 + .as_dict() + .map_err(|_| "an annotation is not a dictionary".to_string())?; + let sub = a.get(b"Subtype").ok().and_then(|s| s.as_name().ok()); + if !matches!( + sub, + Some(b"Redact" | b"Square" | b"Circle" | b"Polygon" | b"Stamp") + ) { + continue; + } + let Some(rect) = a + .get(b"Rect") + .ok() + .and_then(|r| doc.dereference(r).ok()) + .and_then(|(_, r)| r.as_array().ok()) + .map(|v| v.iter().filter_map(real).collect::>()) + .filter(|v| v.len() == 4) + else { + continue; + }; + let (x0, y0, x1, y1) = (rect[0], rect[1], rect[2], rect[3]); + out.push(Overlay { + corners: vec![(x0, y0), (x1, y0), (x1, y1), (x0, y1)], + gray: 0.0, + }); + } + Ok(out) +} + +fn page_rotate(doc: &Document, page_id: ObjectId) -> i64 { + // /Rotate is inheritable: walk up through /Parent. + let mut id = Some(page_id); + for _ in 0..32 { + let Some(dict) = id.and_then(|i| doc.get_dictionary(i).ok()) else { + break; + }; + if let Ok(r) = dict.get(b"Rotate").and_then(Object::as_i64) { + return r; + } + id = dict.get(b"Parent").ok().and_then(|p| p.as_reference().ok()); + } + 0 +} + +// ------------------------------------------------------------- painting + +/// Paint every overlay onto the scan's pixels. `ctm` maps the image's unit +/// square into page space; its inverse takes an overlay back to pixels. +fn paint(image: &mut ImageBuffer, ctm: &Matrix, overlays: &[Overlay]) { + let [a, b, c, d, e, f] = *ctm; + let det = a * d - b * c; + if det.abs() < f64::EPSILON { + return; + } + let (w, h) = (f64::from(image.width), f64::from(image.height)); + // Page space -> unit square -> pixels. The image's first row is the TOP + // of the unit square (v = 1). + let to_px = |(x, y): (f64, f64)| { + let (u, v) = ( + (d * (x - e) - c * (y - f)) / det, + (a * (y - f) - b * (x - e)) / det, + ); + (u * w, (1.0 - v) * h) + }; + let bpp = image.format.bytes_per_pixel(); + let stride = image.width as usize * bpp; + for o in overlays { + let pts: Vec<(f64, f64)> = o.corners.iter().copied().map(to_px).collect(); + // Clamped to [0, image side] first, so the cast cannot truncate or + // lose a sign. A NaN edge (a degenerate overlay) casts to 0. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let clamp = |v: f64, max: f64| v.clamp(0.0, max) as usize; + let x0 = clamp(pts.iter().map(|p| p.0).fold(f64::MAX, f64::min).floor(), w); + let x1 = clamp(pts.iter().map(|p| p.0).fold(f64::MIN, f64::max).ceil(), w); + let y0 = clamp(pts.iter().map(|p| p.1).fold(f64::MAX, f64::min).floor(), h); + let y1 = clamp(pts.iter().map(|p| p.1).fold(f64::MIN, f64::max).ceil(), h); + // In [0, 255] after the clamp. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let v = (o.gray.clamp(0.0, 1.0) * 255.0).round() as u8; + for row in y0..y1 { + let s = row * stride; + image.data[s + x0 * bpp..s + x1 * bpp].fill(v); + } + } +} + +// ------------------------------------------------------------- orientation + +/// Clockwise quarter turns that show the image the way a viewer does, and +/// whether its placement is mirrored. Viewer space is page space with y +/// flipped, then rotated clockwise by `/Rotate`. +fn upright_turns(ctm: &Matrix, rotate: i64) -> (u8, bool) { + // Where the image's +column (right) and +row (down) directions point in + // viewer space. Image rows run from v = 1 down to v = 0. + let right = (ctm[0], -ctm[1]); + let down = (-ctm[2], ctm[3]); + let mirrored = right.0 * down.1 - right.1 * down.0 < 0.0; + let right = if mirrored { + (-right.0, -right.1) + } else { + right + }; + // Dominant direction of the image's right-hand axis, as clockwise turns. + let placement = if right.0.abs() >= right.1.abs() { + if right.0 >= 0.0 { 0 } else { 2 } + } else if right.1 >= 0.0 { + 1 + } else { + 3 + }; + // rem_euclid(360) / 90 is 0..=3; a non-multiple of 90 rounds down. + let page = u8::try_from(rotate.rem_euclid(360) / 90).unwrap_or(0); + ((placement + page) % 4, mirrored) +} + +fn rotate_cw(img: &ImageBuffer, turns: u8) -> ImageBuffer { + let turns = turns % 4; + if turns == 0 { + return img.clone(); + } + let (w, h) = (img.width as usize, img.height as usize); + let bpp = img.format.bytes_per_pixel(); + let (width, height) = if turns == 2 { + (img.width, img.height) + } else { + (img.height, img.width) + }; + let nw = width as usize; + let mut out = vec![0u8; w * h * bpp]; + for y in 0..h { + for x in 0..w { + let (nx, ny) = match turns { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + _ => (y, w - 1 - x), + }; + let (s, d) = ((y * w + x) * bpp, (ny * nw + nx) * bpp); + out[d..d + bpp].copy_from_slice(&img.data[s..s + bpp]); + } + } + ImageBuffer { + width, + height, + format: img.format, + data: out, + } +} + +fn flip_horizontal(img: &ImageBuffer) -> ImageBuffer { + let (w, bpp) = (img.width as usize, img.format.bytes_per_pixel()); + let mut out = img.clone(); + for row in out.data.chunks_exact_mut(w * bpp) { + let src = row.to_vec(); + for x in 0..w { + row[x * bpp..(x + 1) * bpp].copy_from_slice(&src[(w - 1 - x) * bpp..(w - x) * bpp]); + } + } + out +} + +// ------------------------------------------------------------- decoding + +const fn int(o: &Object) -> Option { + match o { + Object::Integer(i) => Some(*i), + // A float where an integer belongs: saturating truncation, and every + // caller range-checks the result before using it as a size. + #[allow(clippy::cast_possible_truncation)] + Object::Real(r) => Some(*r as i64), + _ => None, + } +} + +/// A positive integer entry, as a `usize` — or an error naming the key. +fn dim(dict: &Dictionary, key: &[u8]) -> std::result::Result { + let k = String::from_utf8_lossy(key); + let v = dict + .get(key) + .ok() + .and_then(int) + .ok_or_else(|| format!("missing /{k}"))?; + usize::try_from(v) + .ok() + .filter(|v| *v > 0) + .ok_or_else(|| format!("/{k} {v} is not a positive size")) +} + +fn filters(dict: &Dictionary) -> Vec> { + match dict.get(b"Filter") { + Ok(Object::Name(n)) => vec![n.clone()], + Ok(Object::Array(a)) => a + .iter() + .filter_map(|o| o.as_name().ok().map(<[u8]>::to_vec)) + .collect(), + _ => Vec::new(), + } +} + +/// `/DecodeParms` for the LAST filter (the one that produced the samples). +fn decode_parms(doc: &Document, dict: &Dictionary) -> Option { + let p = dict.get(b"DecodeParms").ok()?; + let p = doc.dereference(p).ok()?.1; + match p { + Object::Dictionary(d) => Some(d.clone()), + Object::Array(a) => a.iter().rev().find_map(|o| { + doc.dereference(o) + .ok() + .and_then(|(_, o)| o.as_dict().ok().cloned()) + }), + _ => None, + } +} + +/// Components per sample of the image's colour space. +fn components(doc: &Document, dict: &Dictionary) -> std::result::Result { + let Ok(cs) = dict.get(b"ColorSpace") else { + return Ok(1); + }; + let cs = doc.dereference(cs).map_err(|e| e.to_string())?.1; + let name = match cs { + Object::Name(n) => n.clone(), + Object::Array(a) => match a.first() { + Some(Object::Name(n)) if n == b"ICCBased" => { + let n = a + .get(1) + .and_then(|s| doc.dereference(s).ok()) + .and_then(|(_, s)| s.as_stream().ok()) + .and_then(|s| s.dict.get(b"N").ok().and_then(int)) + .unwrap_or(3); + // /N comes from the file and feeds the row-size arithmetic: + // only the three sizes a colour space can have are accepted. + return match n { + 1 => Ok(1), + 3 => Ok(3), + 4 => Ok(4), + other => Err(format!("ICC colour space with {other} components")), + }; + } + Some(Object::Name(n)) => n.clone(), + _ => return Err("unrecognised colour space".into()), + }, + _ => return Err("unrecognised colour space".into()), + }; + match name.as_slice() { + b"DeviceGray" | b"CalGray" | b"G" => Ok(1), + b"DeviceRGB" | b"CalRGB" | b"RGB" => Ok(3), + b"DeviceCMYK" | b"CMYK" => Ok(4), + other => Err(format!( + "colour space {} not supported", + String::from_utf8_lossy(other) + )), + } +} + +fn decode_image_xobject(doc: &Document, id: ObjectId) -> std::result::Result { + let stream = doc + .get_object(id) + .and_then(Object::as_stream) + .map_err(|e| e.to_string())?; + let dict = &stream.dict; + let (w, h) = (dim(dict, b"Width")?, dim(dict, b"Height")?); + let pixels = w.checked_mul(h).ok_or("image dimensions overflow")?; + if pixels > MAX_PIXELS { + return Err(format!("{w}x{h} exceeds the {MAX_PIXELS}-pixel limit")); + } + // Both sides are now at most MAX_PIXELS, which fits a u32. + let (w32, h32) = ( + u32::try_from(w).map_err(|_| "width out of range")?, + u32::try_from(h).map_err(|_| "height out of range")?, + ); + let mask = matches!(dict.get(b"ImageMask"), Ok(Object::Boolean(true))); + let invert = dict + .get(b"Decode") + .ok() + .and_then(|d| d.as_array().ok()) + .and_then(|a| a.first()) + .and_then(real) + .is_some_and(|first| first > 0.5); + let fs = filters(dict); + let last = fs.last().map(Vec::as_slice); + + let gray = |data: Vec| ImageBuffer { + width: w32, + height: h32, + format: PixelFormat::Gray8, + data, + }; + + match last { + Some(b"DCTDecode") => { + let img = crate::decode_image(&stream.content).map_err(|e| e.to_string())?; + if (img.width as usize, img.height as usize) != (w, h) { + return Err("JPEG dimensions disagree with /Width and /Height".into()); + } + Ok(img) + } + Some(b"CCITTFaxDecode") => { + let parms = decode_parms(doc, dict); + let get = |k: &[u8]| parms.as_ref().and_then(|p| p.get(k).ok()).and_then(int); + let k = get(b"K").unwrap_or(0); + // /Columns sizes the decoder's line buffers; it must be a real + // row width, not whatever the file says. + let columns = match get(b"Columns") { + None => w, + Some(c) => usize::try_from(c) + .ok() + .filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c)) + .ok_or_else(|| format!("CCITT /Columns {c} out of range"))?, + }; + let black_is_1 = matches!( + parms.as_ref().and_then(|p| p.get(b"BlackIs1").ok()), + Some(Object::Boolean(true)) + ); + let mut bits = ccitt(&stream.content, k, columns, w, h)?; + // `ccitt` yields 0 for black. BlackIs1 and a [1 0] /Decode each + // flip what a sample means; two flips cancel. + if black_is_1 ^ invert { + for v in &mut bits { + *v = 255 - *v; + } + } + Ok(gray(bits)) + } + Some(b"JBIG2Decode") => Err("JBIG2 is not supported yet".into()), + Some(b"JPXDecode") => Err("JPEG 2000 is not supported".into()), + _ => { + let bpc = match dict + .get(b"BitsPerComponent") + .ok() + .and_then(int) + .unwrap_or(1) + { + 1 => 1usize, + 8 => 8, + other => return Err(format!("{other} bits per component is not supported")), + }; + let comps = if mask { 1 } else { components(doc, dict)? }; + let row_bytes = (w * comps * bpc).div_ceil(8); + let expected = row_bytes.checked_mul(h).ok_or("image size overflows")?; + let raw = if fs.is_empty() { + stream.content.clone() + } else { + stream + .decompressed_content_with_limit(expected + 1024) + .map_err(|e| format!("decompress: {e}"))? + }; + if raw.len() < expected { + return Err(format!("{} bytes of samples, {expected} needed", raw.len())); + } + samples_to_image(&raw, (w, w32), (h, h32), comps, bpc, invert) + } + } +} + +/// Widest CCITT row accepted: 100 000 pixels, far past a 600 ppi tabloid. +const MAX_CCITT_COLUMNS: usize = 100_000; + +/// Raw samples to an 8-bit image. `invert_one_bit`: for 1-bit data, whether a +/// 1 bit is dark. By default a 1 bit is white for BOTH a 1-bit gray image and +/// a stencil mask (a mask paints where the sample is 0), so the mask flag does +/// not change the mapping; only a `[1 0]` /Decode flips it. +fn samples_to_image( + raw: &[u8], + (w, width): (usize, u32), + (h, height): (usize, u32), + comps: usize, + bpc: usize, + invert_one_bit: bool, +) -> std::result::Result { + let row_bytes = (w * comps * bpc).div_ceil(8); + let image = |format, data| { + Ok(ImageBuffer { + width, + height, + format, + data, + }) + }; + match (comps, bpc) { + (1, 1) => { + let mut data = Vec::with_capacity(w * h); + for y in 0..h { + let row = &raw[y * row_bytes..(y + 1) * row_bytes]; + for x in 0..w { + let bit = (row[x / 8] >> (7 - (x % 8))) & 1 == 1; + // Default gray: 1 is white. For a stencil mask, a 0 bit is + // painted (black) — also "1 is white". An invert flips both. + let white = bit != invert_one_bit; + data.push(if white { 255 } else { 0 }); + } + } + image(PixelFormat::Gray8, data) + } + (1, 8) => image( + PixelFormat::Gray8, + (0..h) + .flat_map(|y| raw[y * row_bytes..y * row_bytes + w].iter().copied()) + .collect(), + ), + (3, 8) => image( + PixelFormat::Rgb8, + (0..h) + .flat_map(|y| raw[y * row_bytes..y * row_bytes + 3 * w].iter().copied()) + .collect(), + ), + (4, 8) => { + // CMYK to RGB, naive: enough for OCR, which reads luminance. + let mut data = Vec::with_capacity(w * h * 3); + for y in 0..h { + for px in raw[y * row_bytes..y * row_bytes + 4 * w].chunks_exact(4) { + let k = 255 - u16::from(px[3]); + for c in &px[..3] { + // (255 - c) * k / 255 is at most 255. + data.push(u8::try_from((255 - u16::from(*c)) * k / 255).unwrap_or(u8::MAX)); + } + } + } + image(PixelFormat::Rgb8, data) + } + (c, b) => Err(format!( + "{c} components at {b} bits per component is not supported" + )), + } +} + +/// CCITT Group 3 (1-D) or Group 4 to 8-bit gray, 0 for black. +fn ccitt( + data: &[u8], + k: i64, + columns: usize, + w: usize, + h: usize, +) -> std::result::Result, String> { + let mut out = Vec::with_capacity(w * h); + let mut line_cb = |transitions: &[u32]| { + if out.len() >= w * h { + return; + } + let mut line = vec![255u8; w]; + let (mut black, mut x) = (false, 0usize); + for &t in transitions { + let t = (t as usize).min(w); + if black { + line[x..t].fill(0); + } + x = t; + black = !black; + } + if black { + line[x.min(w)..].fill(0); + } + out.extend_from_slice(&line); + }; + let columns = u32::try_from(columns).map_err(|_| "CCITT /Columns out of range")?; + let rows = u32::try_from(h).map_err(|_| "image height out of range")?; + let ok = match k.cmp(&0) { + std::cmp::Ordering::Less => { + fax::decoder::decode_g4(data.iter().copied(), columns, Some(rows), &mut line_cb) + } + std::cmp::Ordering::Equal => fax::decoder::decode_g3(data.iter().copied(), &mut line_cb), + std::cmp::Ordering::Greater => { + return Err("CCITT Group 3 two-dimensional (K > 0) is not supported yet".into()); + } + }; + if ok.is_none() && out.is_empty() { + return Err("CCITT data did not decode".into()); + } + out.resize(w * h, 255); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use lopdf::{Stream, dictionary}; + + /// A one-page PDF: `content` drawn with `images` as named XObjects, and + /// any extra page-dictionary entries (`/Rotate`, `/Annots`). + fn pdf(content: &[u8], images: Vec<(&str, Stream)>, extra: Vec<(&str, Object)>) -> Vec { + let mut doc = Document::with_version("1.5"); + let mut xobjects = Dictionary::new(); + for (name, stream) in images { + let id = doc.add_object(stream); + xobjects.set(name, Object::Reference(id)); + } + let content_id = doc.add_object(Stream::new(Dictionary::new(), content.to_vec())); + let pages_id = doc.new_object_id(); + let mut page = dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + "Resources" => dictionary! { "XObject" => xobjects }, + }; + for (k, v) in extra { + page.set(k, v); + } + let page_id = doc.add_object(page); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![page_id.into()], + "Count" => 1, + }), + ); + let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id }); + doc.trailer.set("Root", catalog); + let mut out = Vec::new(); + doc.save_to(&mut out).unwrap(); + out + } + + /// An uncompressed 8-bit gray image XObject. + fn gray_image(w: usize, h: usize, value: impl Fn(usize, usize) -> u8) -> Stream { + let data: Vec = (0..h) + .flat_map(|y| (0..w).map(move |x| (x, y))) + .map(|(x, y)| value(x, y)) + .collect(); + Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", + "Width" => w as i64, "Height" => h as i64, + "ColorSpace" => "DeviceGray", "BitsPerComponent" => 8, + }, + data, + ) + } + + fn only_page(bytes: &[u8]) -> PdfPage { + let mut pages = pdf_pages(bytes).unwrap(); + assert_eq!(pages.len(), 1); + pages.remove(0) + } + + fn at(img: &ImageBuffer, x: usize, y: usize) -> u8 { + img.data[y * img.width as usize + x] + } + + /// Gap 11, the incident: a patch image pasted over the scan. The scan is + /// 100x50 drawn over page (10,20)-(210,120); the patch covers page + /// (60,70)-(110,120), i.e. pixels x 25..50, rows 0..25. + #[test] + fn a_patch_drawn_over_the_scan_is_painted_black() { + let bytes = pdf( + b"q 200 0 0 100 10 20 cm /Scan Do Q q 50 0 0 50 60 70 cm /Patch Do Q", + vec![ + ("Scan", gray_image(100, 50, |_, _| 200)), + ("Patch", gray_image(4, 4, |_, _| 255)), + ], + vec![], + ); + let p = only_page(&bytes); + let img = p.image.expect("composited scan"); + assert_eq!((img.width, img.height), (100, 50)); + assert_eq!( + at(&img, 30, 10), + 0, + "under the patch: black, even though the patch is white" + ); + assert_eq!(at(&img, 30, 40), 200, "below the patch"); + assert_eq!(at(&img, 10, 10), 200, "left of the patch"); + assert!(!p.has_text_layer); + } + + #[test] + fn filled_rectangles_after_the_scan_are_painted_in_their_colour_and_before_it_are_not() { + let bytes = pdf( + b"0 g 10 20 200 100 re f q 200 0 0 100 10 20 cm /Scan Do Q \ + 0 g 60 70 50 50 re f 1 g 10 20 20 20 re f", + vec![("Scan", gray_image(100, 50, |_, _| 128))], + vec![], + ); + let img = only_page(&bytes).image.unwrap(); + assert_eq!(at(&img, 30, 10), 0, "black box over the scan"); + assert_eq!(at(&img, 2, 45), 255, "white-out box over the scan"); + assert_eq!( + at(&img, 80, 40), + 128, + "the box drawn BEFORE the scan is under it" + ); + } + + #[test] + fn a_redaction_annotation_is_painted() { + let redact = Object::Dictionary(dictionary! { + "Type" => "Annot", "Subtype" => "Redact", + "Rect" => vec![60.into(), 70.into(), 110.into(), 120.into()], + }); + let bytes = pdf( + b"q 200 0 0 100 10 20 cm /Scan Do Q", + vec![("Scan", gray_image(100, 50, |_, _| 200))], + vec![("Annots", Object::Array(vec![redact]))], + ); + let img = only_page(&bytes).image.unwrap(); + assert_eq!(at(&img, 30, 10), 0); + assert_eq!(at(&img, 30, 40), 200); + } + + /// A page stored with `/Rotate 90` comes back the way a viewer shows it: + /// turned a quarter clockwise, so the scan's top-left pixel is top-right. + #[test] + fn page_rotate_is_applied() { + let bytes = pdf( + b"q 200 0 0 100 10 20 cm /Scan Do Q", + vec![( + "Scan", + gray_image(100, 50, |x, y| if x == 0 && y == 0 { 7 } else { 200 }), + )], + vec![("Rotate", Object::Integer(90))], + ); + let img = only_page(&bytes).image.unwrap(); + assert_eq!((img.width, img.height), (50, 100)); + assert_eq!(at(&img, 49, 0), 7); + } + + /// An image placed with a rotating matrix: its columns run DOWN the page, + /// which on screen is down — shown upright, it is turned a quarter + /// clockwise, so its top-left pixel lands top-right. + #[test] + fn a_rotated_placement_is_turned_upright() { + let bytes = pdf( + b"q 0 -100 200 0 10 200 cm /Scan Do Q", + vec![( + "Scan", + gray_image(100, 50, |x, y| if x == 0 && y == 0 { 7 } else { 200 }), + )], + vec![], + ); + let img = only_page(&bytes).image.unwrap(); + assert_eq!((img.width, img.height), (50, 100)); + assert_eq!(at(&img, 49, 0), 7); + } + + #[test] + fn ccitt_group4_round_trips_and_black_is_black() { + use fax::{Color, VecWriter, encoder::Encoder}; + let (w, h) = (64usize, 8usize); + let black = |x: usize, y: usize| (8..24).contains(&x) && y >= 2; + let mut enc = Encoder::new(VecWriter::new()); + for y in 0..h { + enc.encode_line( + (0..w).map(|x| { + if black(x, y) { + Color::Black + } else { + Color::White + } + }), + w as u32, + ) + .unwrap(); + } + let data = enc.finish().unwrap().finish(); + let stream = Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", + "Width" => w as i64, "Height" => h as i64, + "ColorSpace" => "DeviceGray", "BitsPerComponent" => 1, + "Filter" => "CCITTFaxDecode", + "DecodeParms" => dictionary! { "K" => -1, "Columns" => w as i64 }, + }, + data, + ); + let bytes = pdf( + b"q 64 0 0 8 0 0 cm /Scan Do Q", + vec![("Scan", stream)], + vec![], + ); + let img = only_page(&bytes).image.expect("CCITT G4 decodes"); + for y in 0..h { + for x in 0..w { + assert_eq!(at(&img, x, y) == 0, black(x, y), "pixel ({x},{y})"); + } + } + } + + /// A black box drawn over born-digital text hides it on screen and leaves + /// it in the content stream. The text is still extracted, so the page + /// must say it is a redaction risk — and a plain text page must not. + #[test] + fn text_under_a_black_box_is_flagged_as_a_redaction_risk() { + let font = "BT /F1 12 Tf 10 700 Td (123 Secret Lane) Tj ET"; + let plain = only_page(&pdf(font.as_bytes(), vec![], vec![])); + assert!(plain.has_text_layer); + assert!(!plain.redaction_risk, "nothing covers this text"); + + let covered = format!("{font} 0 g 5 695 200 20 re f"); + let p = only_page(&pdf(covered.as_bytes(), vec![], vec![])); + assert!( + p.redaction_risk, + "a black box drawn after the text may be hiding it" + ); + + let white_first = format!("1 g 0 0 612 792 re f {font}"); + let p = only_page(&pdf(white_first.as_bytes(), vec![], vec![])); + assert!( + !p.redaction_risk, + "a background fill BEFORE the text hides nothing" + ); + } + + #[test] + fn a_text_layer_is_reported() { + let bytes = pdf(b"BT /F1 12 Tf 10 10 Td (hello) Tj ET", vec![], vec![]); + let p = only_page(&bytes); + assert!(p.has_text_layer); + assert!(p.image.is_none()); + assert_eq!(p.note.as_deref(), Some("no image on this page")); + } + + #[test] + fn unsupported_and_oversized_images_are_reported_not_guessed() { + let mut jbig2 = gray_image(4, 4, |_, _| 0); + jbig2.dict.set("Filter", "JBIG2Decode"); + let p = only_page(&pdf( + b"q 10 0 0 10 0 0 cm /Scan Do Q", + vec![("Scan", jbig2)], + vec![], + )); + assert!(p.image.is_none()); + assert!(p.note.unwrap().contains("JBIG2")); + + let mut huge = gray_image(1, 1, |_, _| 0); + huge.dict.set("Width", 20_000); + huge.dict.set("Height", 20_000); + let p = only_page(&pdf( + b"q 10 0 0 10 0 0 cm /Scan Do Q", + vec![("Scan", huge)], + vec![], + )); + assert!(p.image.is_none()); + assert!(p.note.unwrap().contains("pixel limit")); + } + + /// A stencil mask (`/ImageMask true`) paints where the sample is 0. + #[test] + fn a_one_bit_stencil_mask_paints_zero_bits_black() { + let stream = Stream::new( + dictionary! { + "Type" => "XObject", "Subtype" => "Image", + "Width" => 8, "Height" => 1, "ImageMask" => true, "BitsPerComponent" => 1, + }, + vec![0b1111_0000], + ); + let img = only_page(&pdf( + b"q 8 0 0 1 0 0 cm /Scan Do Q", + vec![("Scan", stream)], + vec![], + )) + .image + .unwrap(); + assert_eq!(img.data, vec![255, 255, 255, 255, 0, 0, 0, 0]); + } +} diff --git a/crates/ffai-mercury-wasm/Cargo.toml b/crates/ffai-mercury-wasm/Cargo.toml index ad2d352b..2bdbd963 100644 --- a/crates/ffai-mercury-wasm/Cargo.toml +++ b/crates/ffai-mercury-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-mercury-wasm" description = "Mercury in the browser — pure-Rust Whisper speech recognition compiled to WebAssembly, no whisper.cpp, no ONNX runtime" -version = "0.1.1" +version = "0.1.2" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -28,7 +28,7 @@ ffai-core = { workspace = true } # the constructor this crate is built on, first exists in 1.0.2. A `^1.0.1` # requirement resolves happily in the workspace and then fails to compile the # moment it is packaged, which is exactly how it failed. -ffai-mercury = { path = "../ffai-mercury", version = "1.0.3", default-features = false } +ffai-mercury = { path = "../ffai-mercury", version = "1.0.4", default-features = false } wasm-bindgen = "=0.2.121" # pinned to the installed wasm-bindgen-cli; the two must match exactly # The binary is the only thing entitled to choose an allocator, and a cdylib IS diff --git a/crates/ffai-mercury/Cargo.toml b/crates/ffai-mercury/Cargo.toml index 81d9af19..29bb37e2 100644 --- a/crates/ffai-mercury/Cargo.toml +++ b/crates/ffai-mercury/Cargo.toml @@ -6,7 +6,7 @@ description = "Mercury — FFai's voice component: pure-Rust ASR (Whisper/Whispe # to 0.6.1 -- the API is purely additive, but this crate advertises byte-stable # determinism, so a changed waveform is a compatibility break for anyone who # pinned bytes. Its dependencies stay at the published 0.6.1. -version = "1.0.3" +version = "1.0.4" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-mercury/README.md b/crates/ffai-mercury/README.md index 0472d319..bf567988 100644 --- a/crates/ffai-mercury/README.md +++ b/crates/ffai-mercury/README.md @@ -12,7 +12,7 @@ Mercury is [FFai](https://github.com/Remade-With-Rust/FFAI)'s voice component. S ```toml [dependencies] ffai-mercury = "0.7" -ffai-core = "0.6" +ffai-core = "0.8" ffai-media = "0.6" ``` diff --git a/crates/ffai-models/Cargo.toml b/crates/ffai-models/Cargo.toml index 4b45f26d..5c804105 100644 --- a/crates/ffai-models/Cargo.toml +++ b/crates/ffai-models/Cargo.toml @@ -6,7 +6,7 @@ description = "FFai model weight management — manifests, cache, licensing surf # excludes the 0.7.0 every other crate in this release now needs. Left at # 0.6.2 it would put TWO ffai-core versions in every downstream tree and the # shared `ffai_core` types would stop being the same types. -version = "0.6.4" +version = "0.6.5" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-models/src/lib.rs b/crates/ffai-models/src/lib.rs index 5ac4d3ab..bb137784 100644 --- a/crates/ffai-models/src/lib.rs +++ b/crates/ffai-models/src/lib.rs @@ -179,15 +179,115 @@ impl ModelManifest { files, }) } + + /// Resolve every file from `root//` and nowhere else — + /// **no cache lookup, no network, ever.** + /// + /// This is the production form for a scheduled or offline job: the caller + /// states where weights live, a missing file is an error naming the exact + /// path that was expected, and nothing is fetched mid-job + /// (docs/plans/commercial-gaps.md, gap 7). Checksums are verified exactly + /// as [`Self::fetch`] verifies them. + pub fn resolve_in(&self, root: &Path) -> Result { + let all: Vec<&str> = self.files.iter().map(|f| f.name.as_str()).collect(); + self.resolve_only_in(root, &all) + } + + /// [`Self::resolve_in`], for just the named files. + /// + /// A manifest documents provenance as well as runtime needs, so it can list + /// files no engine loads. `ppocrv5-mobile-det`, for example, lists its + /// training checkpoint, the source of the fused weights. An offline + /// deployment must not be made to ship those, so a loader asks for what it + /// will open. A name the manifest does not declare is an error, because it + /// has no checksum to verify against. + pub fn resolve_only_in(&self, root: &Path, needed: &[&str]) -> Result { + let dir = root.join(&self.name); + let mut files = BTreeMap::new(); + for name in needed { + if !self.files.iter().any(|f| f.name == *name) { + return Err(Error::Model(format!( + "model `{}` declares no file `{name}` — it has no checksum to verify against", + self.name + ))); + } + } + for file in self + .files + .iter() + .filter(|f| needed.contains(&f.name.as_str())) + { + let path = dir.join(&file.name); + if !path.is_file() { + return Err(Error::Model(format!( + "model `{}` needs `{}`, expected at {} — not present (offline resolution \ + never downloads)", + self.name, + file.name, + path.display() + ))); + } + verify_checksum(&path, file)?; + files.insert(file.name.clone(), path); + } + Ok(ResolvedModel { + name: self.name.clone(), + license: self.license.clone(), + files, + }) + } + + /// Check that every file is already present locally and matches its + /// checksum, **without downloading anything** — the `verify` half of + /// `prefetch`/`verify`. Resolves exactly where [`Self::fetch`] would look. + pub fn verify(&self) -> Result { + let mut files = BTreeMap::new(); + for file in &self.files { + let path = self.local_path(&file.name).ok_or_else(|| { + Error::Model(format!( + "model `{}` is missing `{}` (looked under {}{})", + self.name, + file.name, + self.cache_path().display(), + if self.hf_repo.is_some() { + " and the Hugging Face cache" + } else { + "" + } + )) + })?; + verify_checksum(&path, file)?; + files.insert(file.name.clone(), path); + } + Ok(ResolvedModel { + name: self.name.clone(), + license: self.license.clone(), + files, + }) + } } /// Load every `*.toml` manifest in a directory (typically `models/`). +/// +/// A missing directory is an error that NAMES the directory. The bare i/o +/// error ("The system cannot find the path specified") is what a crates.io +/// user got when `models/` did not exist relative to their working directory, +/// and it reads like missing weights rather than a missing manifest folder. pub fn load_dir(dir: &Path) -> Result> { let mut out = Vec::new(); - for entry in std::fs::read_dir(dir)? { + let entries = std::fs::read_dir(dir).map_err(|e| { + Error::Model(format!( + "cannot read model manifest directory {}: {e}", + dir.display() + )) + })?; + for entry in entries { let path = entry?.path(); if path.extension().and_then(|e| e.to_str()) == Some("toml") { - out.push(ModelManifest::load(&path)?); + out.push( + ModelManifest::load(&path) + .map_err(|e| Error::Model(format!("{}: {e}", path.display())))?, + ); } } out.sort_by(|a, b| a.task.cmp(&b.task).then_with(|| a.name.cmp(&b.name))); @@ -227,4 +327,107 @@ mod tests { assert_eq!(m.files.len(), 1); assert!(!m.is_cached()); } + + fn scratch(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("ffai-models-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn manifest_for(sha: &str) -> ModelManifest { + ModelManifest::from_toml(&format!( + r#" + name = "tiny-test" + task = "ocr" + license = "MIT" + + [[files]] + name = "w.bin" + sha256 = "{sha}" + "# + )) + .unwrap() + } + + /// sha256("abc") + const ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + + #[test] + fn resolve_in_names_the_exact_missing_path_and_never_fetches() { + let root = scratch("missing"); + let err = manifest_for(ABC).resolve_in(&root).unwrap_err().to_string(); + let want = root.join("tiny-test").join("w.bin"); + assert!( + err.contains(&want.display().to_string()), + "error must name {}: {err}", + want.display() + ); + assert!(err.contains("never downloads"), "{err}"); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn resolve_in_finds_and_checksums_a_placed_file() { + let root = scratch("present"); + std::fs::create_dir_all(root.join("tiny-test")).unwrap(); + std::fs::write(root.join("tiny-test").join("w.bin"), b"abc").unwrap(); + let r = manifest_for(ABC).resolve_in(&root).unwrap(); + assert_eq!( + r.file("w.bin").unwrap(), + root.join("tiny-test").join("w.bin") + ); + + // Same bytes, wrong declared checksum: an error, not a warning. + let bad = manifest_for(&"0".repeat(64)) + .resolve_in(&root) + .unwrap_err() + .to_string(); + assert!(bad.contains("checksum mismatch"), "{bad}"); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn load_dir_names_a_missing_directory() { + let missing = std::env::temp_dir().join("ffai-models-definitely-absent-dir"); + let err = load_dir(&missing).unwrap_err().to_string(); + assert!(err.contains(&missing.display().to_string()), "{err}"); + } + + /// Only the named files are required, so a provenance-only file (a + /// training checkpoint) need not be shipped; but an undeclared name is + /// refused, because there is no checksum to verify it against. + #[test] + fn resolve_only_in_needs_just_the_named_files_and_refuses_undeclared_ones() { + let root = scratch("subset"); + std::fs::create_dir_all(root.join("tiny-test")).unwrap(); + std::fs::write(root.join("tiny-test").join("w.bin"), b"abc").unwrap(); + let m = ModelManifest::from_toml(&format!( + r#" + name = "tiny-test" + task = "ocr" + license = "MIT" + + [[files]] + name = "w.bin" + sha256 = "{ABC}" + + [[files]] + name = "train-only.bin" + "# + )) + .unwrap(); + assert!( + m.resolve_in(&root).is_err(), + "the full set includes an absent file" + ); + let r = m.resolve_only_in(&root, &["w.bin"]).unwrap(); + assert_eq!(r.files.len(), 1); + let err = m + .resolve_only_in(&root, &["nope.bin"]) + .unwrap_err() + .to_string(); + assert!(err.contains("declares no file `nope.bin`"), "{err}"); + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/crates/ffai-py/Cargo.toml b/crates/ffai-py/Cargo.toml index 61e7df25..ae41946a 100644 --- a/crates/ffai-py/Cargo.toml +++ b/crates/ffai-py/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-py" description = "Python bindings for FFai — Diana detection and depth over numpy/PIL/torch arrays" -version = "0.1.0" +version = "0.1.1" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/ffai-wasm/Cargo.toml b/crates/ffai-wasm/Cargo.toml index 78f6b5d2..05df8b19 100644 --- a/crates/ffai-wasm/Cargo.toml +++ b/crates/ffai-wasm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ffai-wasm" description = "Diana in the browser — YOLO26 object detection compiled to WebAssembly, pure Rust, no ONNX runtime" -version = "0.1.3" +version = "0.1.4" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/docs/plans/benching-history-made.md b/docs/plans/benching-history-made.md index d84541f0..ecd2e34d 100644 --- a/docs/plans/benching-history-made.md +++ b/docs/plans/benching-history-made.md @@ -2101,6 +2101,19 @@ remaining points are.** ## §44 — the three missing stages, and why the headline is a COVERAGE gap +> **CORRECTION, 2026-09-25.** This section attributes **0.0406 text / 0.0522 +> order** to Carmenta on the 236-page `unlimited_holdout` subset. Those are +> **Unlimited-OCR's** numbers on that subset (§1, and +> `docs/Carmenta-mission-plan.md` §8.173). Carmenta's measured score there is +> **0.1084 / 0.1749**. The table row "Carmenta, formula/table-free English +> 0.0406", and the conclusion drawn from it ("we already beat PP-StructureV3 +> and sit near the VLM leader"), are therefore **withdrawn**. The section's +> other point, that the full-corpus loss is dominated by content classes +> Carmenta does not emit (§57's ledger measures this independently), does not +> depend on the wrong number and stands. The claim had reached the published +> `ffai-carmenta` README, and was corrected there the same day. The text below +> is left as written, per this log's append-only rule. + **The reframing, from evidence already on disk.** `configs/unlimited_holdout.yaml` scores Carmenta on a 236-page holdout: **English only, ZERO LaTeX-bearing text blocks, ZERO table regions, ZERO isolated equations** (verified by counting the diff --git a/docs/plans/commercial-gaps.md b/docs/plans/commercial-gaps.md new file mode 100644 index 00000000..60ec33d5 --- /dev/null +++ b/docs/plans/commercial-gaps.md @@ -0,0 +1,676 @@ +# Commercial gaps — found while using FFai in a real pipeline + +A running list of gaps found when an application tried to use FFai for real work. Each entry names the use, what was missing or weak, the evidence, and what would close it. + +Evidence labels: + +- **source**: read in FFai's own code or README. +- **observed**: seen in the input documents. +- **measured**: run and counted. + +--- + +## Progress + +Executing the whole list, in three batches. Branch `feat/commercial-gaps`. + +| Batch | Gap | Work | Status | +|---|---|---|---| +| 1 quick | 9 | `ffai_media::decode_image(&[u8])`: same decoders and output as `load_image`, dispatch by magic bytes | **done**: byte-identical to `load_image` on JPEG and PNG, tests pass | +| 1 quick | 7 | `CraftCrnn::offline(rec, det, weights_root)`: weights only from `root//`, no cache, no network. `preload()` fails before page one. `ffai models --verify ` checks without downloading. Errors name the exact expected path | **done**, details below | +| 1 quick | 10 | The six OCR manifests are compiled into `ffai-carmenta` (`manifests::EMBEDDED`), so no `models/` directory is needed. A test keeps the crate's copies byte-identical to `models/`. A missing manifest directory is an error that names it | **done**; `cargo package --list` confirms the manifests ship (hosted weights: batch 3) | +| 1 quick | 3 (part) | `OcrOptions::charset`: decoding restricted to a character set, for CRNN and SVTR. Confidence is not renormalised, so a forced character reads as uncertain. PARSeq and routing refuse rather than ignore it. `ffai ocr --charset` | **done**, details below | +| 1 quick | 6 | Determinism test: same page, two calls and two instances, text and boxes must be identical | **done: already deterministic**, details below | +| 1 quick | 8 | Confidence always reported? | **answered from source**: CRNN and SVTR report a confidence on every emitted line (a line with no kept character is dropped, never emitted with `None`). Only PARSeq can return `None`, when no word carried one | +| 2 measured | 12 | Remove form underlines before recognition; calibrated confidence | **rule removal done**, opt-in: fixes CRNN, neutral or harmful on SVTR (below). **Biggest measured fix: use `mobiledet-svtr`.** Full-width fold done and on by default (OmniDocBench: 5 pages better, 1 worse, the worse one a ground-truth quirk) | +| 2 measured | 5 | Orientation (0/90/180/270) detection | **done: 144/144** on turned pages and form lines, at 48 % of a read per document page. `OcrOptions::auto_orient`, `ffai ocr --auto-orient` | +| — | 13 | Throughput with orientation search | **done via gap 5**: about 1.5 reads per page, against the 3 reads fraud-alert pays | +| 3 build | 1, 11 | PDF ingest returning the page **as rendered** (redactions composited), not the raw scan XObject | **done**: `ffai_media::pdf`, wired into `ffai ocr`; born-digital text layers returned with a `redaction_risk` flag (below) | +| 3 build | 2 | CCITT G3/G4 decoder (then JBIG2) | **G4 and G3-1D done** via the pure-Rust `fax` crate inside `pdf`; a house `rusty_ccitt`, G3-2D and JBIG2 are open | +| 3 build | 4, 14 | Checkbox, table and key-value output | **checkboxes done**: 100 % recall and state on 960 synthetic boxes; `口` is the known false positive. **Key-value pairing done** (`forms::pair_fields`; 220/221 correctly-read fields paired). Free hand ticks and tables in `OcrOutput`: open | +| 3 build | 3, 12 (rest) | Lexicon constraint; confidence calibration; digit CER in the bench | **all three done**: lexicon; calibration measured (min-character confidence changes sign by engine, so it is refuted as a default); digit error rate in every OCR ledger line | +| 3 build | 10 (rest) | Hosted converted weights, so manifests name a download source | not started | + +Out of scope for batch 1, noted: region routing (`FFAI_ROUTE`) still loads its +ONNX stages from `corpora/refs/fixtures`, a repository path. It is opt-in and +off by default. + +### Batch 1 results (2026-09-24) + +All measured on this machine with the cached weights, in +`crates/ffai-carmenta/tests/production.rs`. + +- **Determinism (gap 6): closed; there was nothing to fix.** OmniDocBench page + `omni-0000` has 120 lines, enough for the parallel recognition path. It was + read twice by one `mobiledet-svtr` instance and once by a second instance. + Text and boxes were identical on all three reads, and the largest confidence + difference was **0**. The test now guards it. Caveat, from §8.46: the reads + ran in isolation, and under contention from another process confidences have + been seen to move by about 1e-4. That is far below any gate a caller would + set, and text did not move. +- **Offline resolution (gap 7): it found a defect in its own first version.** + The `ppocrv5-mobile-det` manifest also lists `det.safetensors` and the + training checkpoint `det-train.safetensors`, for provenance. Neither is + loaded at runtime. Strict offline resolution demanded all three files, which + would have forced a deployment to ship two unused checkpoints. Offline + loading now asks only for the files it opens + (`ModelManifest::resolve_only_in`). A name the manifest does not declare is + refused, since it has no checksum. The fix also removed a doubled + "model error: model error:" prefix from load errors. + - Offline output is identical to the cache-backed constructor's on the + 120-line page. + - An empty weights directory fails at `preload()`, naming + `/ppocrv5-mobile-det/det-fused.safetensors`. +- **Charset constraint (gap 3, part).** + - With the full English charset allowed, text and confidences are identical + to the free decode on the real page. + - Restricting to `0123456789$,.- ` emitted nothing outside the set. + - On a page of prose, mean line confidence fell from **0.918 to 0.781**. The + forced characters show up in the confidence, as intended. + - A character the model cannot emit (`中` on the English CRNN) is an error + naming it. + - Not yet measured: whether constraining fixes the underline look-alikes + ("1117" → "TTT7"). That needs the KY crops and belongs with gap 12 in + batch 2. +- **Gates:** the three blocking CI clippy commands, `cargo check --workspace + --all-targets`, rustfmt on `ffai-media` and on every file that was clean + before, and `tools/lint_invariants.py` all pass. `ffai models --verify` + passes for cached models and exits 1 naming the searched path for a missing + one. +- **Breaking change:** the new public field on `OcrOptions` breaks + `ffai-core`'s semver (see CHANGELOG). + +### Batch 2: form rules (gap 12) + +**Corpus.** The Kentucky filings are public records that carry personal +details, and fraud-alert reads them without keeping them, so they are not +test data here. `tools/carmenta_form_corpus.py` builds `carmenta-form-v1` +instead, with ground truth known by construction: + +- 30 invented values: amounts, numbers and names chosen for the confusions + seen in the field. +- Each value is rendered with no rule (the paired control) and three rule + kinds: an underline touching the glyph bottoms, a rule through the + descenders, and a strikethrough. +- Clean grayscale, plus 1-bit to mimic CCITT fax scans. +- Courier, Arial and Times, 10-12 pt at 300 ppi. 240 clips. + +The bench is `examples/form_bench.rs`. It scores the VALUE: exact read, CER, +and **wrong>gate**, meaning the value was wrong but its line passed a 0.85 +confidence gate. + +**Baseline, all 240 clips (measured).** + +| condition | exact, `mobiledet-crnn` | wrong>gate | exact, `mobiledet-svtr` | wrong>gate | +|---|---:|---:|---:|---:| +| no rule | 70 % | 17 / 60 | 93 % | 4 / 60 | +| underline | 13-20 % | 40 / 60 | 93 % | 4 / 60 | +| rule through descenders | 47-50 % | 28 / 60 | 97 % | 2 / 60 | +| strikethrough | 7-10 % | 43 / 60 | 77-87 % | 10 / 60 | +| amounts, no rule | 15 % | 16 / 20 | 85 % | 3 / 20 | + +- **The corpus reproduces the field failure**, and only on the recognizer + fraud-alert uses. `OcrPages::new` builds `variant_in(RecStage::Crnn, + DetStage::MobileDet, ..)`. On that engine, ruled values are read exactly + 7-20 % of the time, and most of the wrong ones pass the gate. +- **`mobiledet-svtr`, the shipped document default since §8.171, is already + robust to underlines and to rules through descenders.** Switching + fraud-alert to it is the largest single fix available, and it is one line in + their code. Strikethrough remains the weak case for both engines. +- CRNN also reads amounts badly **with no rule at all** (15 % exact). That is a + recognizer weakness on `$`-prefixed values, not a rule artefact. + +**Rule removal, `OcrOptions::remove_rules` (measured, all 240 clips).** +`carmenta::rules` erases thin, long horizontal dark runs. It keeps any pixel +whose vertical dark run is tall, so glyph strokes standing on or crossing a +rule survive, and a filled box such as a redaction is never touched. It runs +before detection. + +| condition | CRNN | CRNN + removal | SVTR | SVTR + removal | +|---|---:|---:|---:|---:| +| no rule | 70 % | 70 % | 93 % | 93 % | +| underline | 13-20 % | **70 %** | 93 % | 90-93 % | +| through descenders | 47-50 % | **67-70 %** | 97 % | 93-97 % | +| strikethrough | 7-10 % | **47-50 %** | 77-87 % | 73 % | +| wrong>gate, underline | 40 / 60 | **18 / 60** | 4 / 60 | 5 / 60 | + +- **On CRNN it removes the whole underline penalty.** Underlined values read + exactly as often as the no-rule control. +- **It is a true no-op on unruled text**: the "no rule" rows match the baseline + to the digit. +- **It is neutral-to-harmful on SVTR.** Strikethrough gets worse (77-87 % to + 73 %), because erasing a strike through the middle of the glyphs also erases + the horizontal strokes it coincides with (`e` reads as `c`, `C` as `G`). + +**Verdict: opt-in.** It helps a caller on CRNN, and the better fix is SVTR +without removal. + +**The per-field charset (`--charset-by-kind`, amounts restricted to +`0123456789$,.- to`) changed nothing on SVTR.** Its remaining amount errors +are not look-alikes a charset can exclude. + +**What SVTR still gets wrong, and the next targets:** + +- **Full-width forms.** `"$25，０00"` and `"12／31／2025"`: SVTR's multilingual + charset sometimes emits CJK full-width digits and punctuation (U+FF0C, + U+FF10, U+FF0F). A number parser rejects them, or worse, drops them silently. + The charset constraint excludes them for fields known to be numeric. A + full-width-to-ASCII fold for Latin pages is the general fix; it needs the + OmniDocBench gate because CJK text uses full-width forms correctly. +- **`1` read as `l`** in `"1121 River Road"`, in every condition including no + rule. This is a lexicon or pattern constraint problem (gap 12). +- **A value split by detection.** `"$5,001 - $10,000"` loses its first half, + also with no rule. + +### Batch 2: orientation (gap 5) + +`CraftCrnn::detect_orientation` and `OcrOptions::auto_orient` use no new +model: + +1. One detection pass decides the axis from box aspect, weighted by area. +2. The 12 largest boxes on that axis are read in both candidate turns, and + the higher mean recognition confidence wins. +3. With `auto_orient`, boxes are mapped back to the input's coordinates. + +**Accuracy (measured, `examples/orient_bench.rs`, ground truth by +construction).** 24 OmniDocBench pages and 12 form lines were each turned +0/90/180/270 and had to be turned back. + +| source | 0 | 90 | 180 | 270 | smallest confidence margin | +|---|---:|---:|---:|---:|---:| +| document pages | 24/24 | 24/24 | 24/24 | 24/24 | 0.545 | +| single form lines | 12/12 | 12/12 | 12/12 | 12/12 | 0.148 | + +**144 of 144 correct, including upside-down (180), which fraud-alert's +workaround never tries.** The margin is the gap between the chosen and +rejected turn's confidence. It is wide on pages and narrower, but still +unanimous, on a single sparse line. + +**Cost, first version: missed.** A decision took 122 % of a full read. The +axis vote ran detection at full page resolution, and the sample crops were +read one after another. + +**Second version: fixed.** Detection runs on a page decimated to about a +640 px short side (box aspect survives), and the crops are read in parallel, +on the same pages: + +| source | accuracy | smallest margin | decision cost | +|---|---:|---:|---:| +| document pages | 96/96 | 0.465 | **48 % of a read** (7.8 s vs 16.3 s) | +| single form lines | 48/48 | 0.148 | 104 % of a read (nothing to decimate on a one-line image) | + +`auto_orient` therefore costs about **1.5 reads per document page**. +fraud-alert's workaround reads each landscape page three times, and never +checks upside-down. (Absolute seconds are from a contended machine; only the +ratios are claims.) + +### Batch 3: checkboxes (gaps 4, 14) + +`ffai_carmenta::checkbox::find` and `ffai ocr --checkboxes` find boxes in the +pixels and report each frame with a marked or empty state, instead of the +`@` / `0` / `D` / `B` a text reader makes of them. No model is involved. A +frame is accepted only when all of these hold: + +- four sides, each at least 85 % covered; +- four sharp, inked corners; +- clear space just outside every side: a box's top edge is its topmost ink, + while a round letter bulges past any run cut from its own contour; +- a mark that overflows the box is trimmed back to the frame's real right + side. + +A box counts as marked when its interior, inset past the frame, is at least +6 % ink. + +**Corpus.** `tools/carmenta_checkbox_corpus.py` builds `carmenta-checkbox-v1`: +80 pages and 960 boxes, ground truth by construction. + +- Boxes are drawn both as stroked rectangles and as the font's own `☐` glyph. +- Marks are X strokes, ticks that overflow the box, solid fills and `☒`, or + the box is left empty. +- Distractors: `O D 0 8` in running text, `口 回`, a table, and a rule. +- Every page comes in gray and 1-bit. + +**Measured, in four iterations. Each earlier result stays recorded because +each was wrong in a different, instructive way.** + +| version | recall | state right | false positives | +|---|---:|---:|---:| +| four sides only | 99.4 % | 96.6 % | 1544 | +| + generator fix (Courier questions ran into the "empty" boxes; the detector was right and the ground truth was not) | 98.8 % | **100 %** | 1540 | +| + clear space outside each side (corners alone passed `o d g R`: a run cut from a curve has dark ends by construction) | 98.8 % | 100 % | 240 | +| + trim an overflowing tick back to the frame | **100 %** | **100 %** | 240 | + +- **All 240 remaining false positives are the `口口 回` distractor, three per + page.** Those ideographs are geometrically square frames, and no shape test + can separate `口` from an empty box. On CJK pages, cross-check each box + against the OCR text at its position. That is the known limit, stated. +- **First real page, first new false-positive class.** On a real scientific + page (the smoke test below), two 13x12 px squares were reported as empty + boxes. They are most likely square plot markers or legend symbols, a class + the synthetic corpus does not contain. `CheckboxParams::min_side` is the + knob. The default is left alone rather than tuned to one page. +- **Caveats.** The detector's structure was developed while looking at this + corpus's failures, so these are not yet holdout claims. Nothing has been + run on real scans yet. Hand ticks with no box at all (Indiana's columns, + gap 14) are not handled: this finds boxes, not free marks. + +### Batch 3: lexicon constraint (gap 12) + +`ffai_carmenta::lexicon::Lexicon` takes the values a caller already knows a +field can hold (counties, known creditors) and snaps a read to the nearest +entry. + +- The distance is Levenshtein, except that look-alike substitutions cost half + an edit: `1 l I i | T`, `0 O o D U Q`, `5 S s`, `8 B $` and the rest. +- It snaps **only when unambiguous**: within 25 % of the entry's length, with + the runner-up at least one edit further away. Otherwise it returns the + candidates. +- It never rewrites what the engine read. It is advice with its evidence + attached. +- **Every misread in this document snaps to its entry:** "Trish HTlI" → + Irish Hill, "GuIF Shores" → Gulf Shores, "MerrITLynch" → Merrill Lynch, + "Cabe]l County" → Cabell County, "Tr1-State Bank" → Tri-State Bank. A tie + between "Unit 10" and "Unit 1O" is reported as ambiguous, not guessed. + +### Batch 3: confidence calibration (gap 12), measured on SVTR + +The line's mean character probability is compared with its weakest +character's (`FFAI_CONF_REDUCE=min`), on the 240 form clips (19 wrong values): + +| confidence | AUROC | at the 0.85 gate | +|---|---:|---| +| mean (shipped) | 0.746 | keeps 216/221 right, **19/19 wrong** | +| min character | 0.764 | keeps 125/221 right, 3/19 wrong | + +- **This reproduces the incident exactly.** With mean confidence, a 0.85 gate + keeps every wrong value. +- **Min separates only slightly better** (AUROC 0.746 to 0.764). At the same + gate it removes 16 of 19 wrong values, but also 43 % of the right ones, + because it is a different scale and its gate has to be set separately. +**On CRNN it reverses** (240 clips, 154 wrong values): + +| CRNN confidence | AUROC | at the 0.85 gate | +|---|---:|---| +| mean (shipped) | **0.873** | keeps 86/86 right, 128/154 wrong | +| min character | 0.828 | keeps 57/86 right, 20/154 wrong | + +- **The min reduction's effect changes sign by engine**: slightly better on + SVTR, worse on CRNN. So it is **refuted as a general fix**. The mean stays + the default, and `FFAI_CONF_REDUCE=min` stays an experiment. +- Neither reduction is a calibrated probability, and no single gate protects a + field. The measured protections are: + - a constraint that knows the field (charset or lexicon); + - `mobiledet-svtr`, which makes far fewer wrong reads to gate in the first + place (19 against CRNN's 154 on the same clips). + +### End-to-end smoke test: a sideways scanned PDF + +An OmniDocBench page was rotated 90 degrees and saved as an image-only PDF +with PIL, the way a scanner stores a portrait form landscape. +`ffai ocr -i sideways.pdf`, release build: + +| flags | first lines read | +|---|---| +| `--auto-orient` | "Antibody—antigen interactions I C. Bich et al. 1 Anal. Biochem. 375 (2008) 35–45", then the figure's axis labels | +| none | `3`, `4`, `行`, `o`, `0`, `2`, `3`, `n`, `间`, `m`, `E` | + +The second row is the silent failure fraud-alert hit: no error, just +confident fragments. The first row is the PDF path (`ffai_media::pdf`), the +orientation decision and the document default recognizer working together. + +### Batch 3: the full-width fold, census on OmniDocBench (gap 12) + +`examples/fold_census.rs` runs `mobiledet-svtr` with the fold off +(`FFAI_NO_FOLD=1`), then applies it offline, so both arms come from one +exactly-paired pass. + +**Interim, first ~176 of 316 pages: the fold fired on 32 lines.** Where the +ground truth settles the question (the raw or the folded form appears in it, +not both): + +| verdict | count | examples | +|---|---:|---| +| folded form is in the ground truth | 5 | `"prｅviously"` → `"previously"`, `"corｐorate"` → `"corporate"`, `"water/sewer，"` → `"water/sewer,"`, `"QQ：87124697"` → `"QQ:87124697"` | +| raw form is in the ground truth | 1 | `"e)，and"`: the annotators themselves typed a full-width comma | + +- **SVTR puts full-width LETTERS inside English words** (`prｅviously`, + `corｐorate`). They look identical and break every search, lexicon match and + parser downstream. That is a bigger class than the digits the fold was + written for. +- **The CJK-page worry did not materialise.** `QQ：87124697` sits on a Chinese + page on a line with no CJK characters, and its ground truth uses the ASCII + colon. +**Final, all 316 pages:** + +- The fold fired on **35 of 34,852 lines (0.1 %) across 19 pages**. +- Changed pages: **5 better, 1 worse, 13 unchanged in CER**. The one worse + page is the annotators' full-width comma above. +- Corpus CER: 14.2460 % with the fold off, 14.2453 % with it on. +- On the form corpus it turned `"$25，０00"` and `"12／31／2025"` into correct + reads (strikethrough/gray 76.7 % to 83.3 % exact). + +**Verdict: on by default, earned.** The corpus-wide effect is small, as +expected for something that fires on 0.1 % of lines. It is never worse except +where the ground truth itself is full-width. On the lines where it fires it +turns an invisible, parser-breaking character into the one a reader sees. +`FFAI_NO_FOLD=1` turns it off. + +### Batch 3: label → value pairing (gap 4) + +`ffai_carmenta::forms::pair_fields(&OcrOutput) -> Vec`: + +- A line ending in `:` is a label. +- Its value is the nearest unlabelled line to its right on the same row, + otherwise the nearest one directly below. +- A single `Label: value` line is split at its colon. A time such as `10:30` + is not a field. +- Each value line is claimed once, closest label first. +- A label with nothing near it comes back with an **empty value** rather than + a neighbour's: a blank field is information. + +**Measured on `mobiledet-svtr` over the 240 form clips.** The clip's label had +to come back with its exact value: **220 / 240**, against 221 values read +correctly. So pairing loses **one field out of 221 correct reads (99.5 %)**, a +single amount on a "rule through descenders" gray clip. Every other miss is +the recognizer's, not the pairing's. Per row, pairing equals the exact-read +rate in 7 of 8 conditions. + +### Batch 3: digit error rate in the bench (gap 3) + +`ffai_bench::metrics::digit_errors_with` aligns the reference and the read +and counts only digit errors: + +- a substitution where either side is a digit, so a phantom digit counts as + well as a misread one (`$` → `8` turns "$10,000" into "810,000"); +- a dropped digit; +- an invented digit. + +Every OCR bench run, engine and reference arm alike, now adds `digit error +rate X % over N reference digits`, micro-averaged over the corpus, to its +ledger notes. ASR ledger lines are unchanged. The first version counted only +reference-digit errors. Its own test (`no` → `n0`) showed that misses +phantom digits, and it was corrected before being wired in. + +### Batch 3: born-digital text layers, and their own redaction trap (gaps 1, 11) + +`PdfPage` now carries the page's `text` layer (`lopdf`'s bounded extractor) +and a `redaction_risk` flag. + +**A text layer has the scan incident's twin.** A black box drawn over +born-digital text hides it on screen and leaves it in the content stream, +where any text extractor reads straight through it. That is the classic +failed redaction. `redaction_risk` is set when: + +- the page carries a redaction-style annotation; +- a dark fill is drawn after text was shown; +- the drawing cannot be walked at all (fail-safe). + +A background fill drawn before the text does not set it. `ffai ocr` prints a +page's text layer only when the risk is clear. Otherwise it withholds the text +with a warning, and the rendered image is what gets read. Tested on in-memory +PDFs covering all three cases. + +### Batch 3: PDF ingest (gaps 1, 2, 11) + +`ffai_media::pdf::pdf_pages(bytes)`, behind the opt-in `pdf` feature. +It depends on `lopdf` without its default features, and on `fax` for CCITT; +both are pure Rust and MIT. + +- Every page comes back as a viewer shows it. The scan is composited with + every filled path, image and form `XObject` drawn after it, plus `Redact`, + `Square`, `Circle`, `Polygon` and `Stamp` annotations. It is then turned + upright from its placement matrix and `/Rotate`. +- `has_text_layer` flags born-digital pages, so they can be routed to text + extraction instead of OCR. +- **A page whose drawing cannot be walked returns no image** and a note. It + never falls back to the raw scan. +- Decoders: DCT (JPEG), CCITT G4 and G3-1D, and Flate/raw samples at 1 or 8 + bits in gray, RGB or CMYK, including stencil masks. JBIG2, JPEG 2000 and + G3-2D are reported per page, not guessed. +- Bounded against hostile files: + - images are capped at 100 MP; + - decompression is capped at the size the dimensions justify; + - form content is capped at 64 MiB; + - form nesting is capped at 8; + - `/N`, bits per component and CCITT `/Columns` are validated before any + size arithmetic. +- 9 tests on PDFs built in memory, covering the incident itself: a white patch + image over a scan must come out black. +- **Two defects found by those tests before anything shipped:** + - Stencil-mask polarity was inverted, so every stencil scan would have come + out as a negative. + - `lopdf`'s annotation helper skips annotations stored inline in `/Annots`, + so an inline redaction would have been read through. `/Annots` is now + resolved directly, accepting both forms. +- **Two hostile-input paths found while fixing lints:** an unbounded ICC `/N` + feeding row-size arithmetic, and uncapped form-content decompression. Both + are now bounded. +- Clippy-clean under `ffai-media`'s pedantic and nursery lints. The casts are + checked conversions or individually justified, not a blanket allow (open + audit item R-006 criticises exactly that for parsers). + +--- + +## 2026-09-24 — fraud-alert: reading scanned financial-disclosure filings + +**The use.** `fraud-alert` (F:\coding\fraud-alert) reads state officials' financial disclosure filings for all 50 states. About a dozen states publish them only as scans or phone photos: + +- CO: phone photos. +- KY, WV: copier scans. +- LA: scans with a vendor OCR layer. +- NY, SD: image-only pages. +- IN: legislators' filings are image-only. + +The job is to pull specific fields (income sources, business interests, debts, real estate by county, dollar ranges) into records whose values must be exact. A value that cannot be verified is held back, not published. Carmenta was the intended engine. At the time of writing, Carmenta has been read but not yet run on these documents; measured entries will be added once it has. + +### 1. No PDF ingest (source, observed) + +**Gap.** Neither `ffai-carmenta` nor `ffai-media` opens a PDF; `OcrEngine::recognize` takes an `ImageBuffer`. The caller has to find each page's image XObject, decode it, and handle page rotation and the image's placement on the page itself. + +**Evidence.** `pdfimages -list` on the sample filings shows three encodings: + +| Encoding | Where seen | +|---|---| +| DCTDecode (JPEG), RGB and gray | KY, WV, IN | +| CCITTFaxDecode (Group 4, 1-bit) | KY page 2, IN, WV stencils | +| Raw 1-bit bitmaps (Flate) | IN | + +Page sizes range from 1275×1650 at 150 ppi to 3304×2550 at 300 ppi. KY pages are stored landscape. + +**To close it:** a `ffai-media` (or `ffai-carmenta`) entry point that takes PDF bytes and returns the page images, with orientation, in page order. It could also pass through the text layer of born-digital PDFs, so a caller does not OCR a page that already has text. Among our sample states, LA ships an OCR text layer, and FL, NC, and IA are born-digital. + +### 2. No CCITT Group 3/4 or JBIG2 decoder in the house stack (source) + +**Gap.** Fax-encoded bilevel images are the norm for black-and-white document scanners. The house stack has `rusty_jpeg` and `rusty_png`, but no CCITT or JBIG2 decoder, so a caller has to reach for a third-party crate from crates.io. + +**To close it:** a `rusty_ccitt` (G3/G4) crate, and `rusty_jbig2` later. Both are small, well specified, and C-free. Every scanned-PDF user needs them. + +### 3. Document accuracy is short of the bar for exact values (source) + +**Gap.** The README reports the following. Neither figure distinguishes digits from letters. + +| Benchmark | Carmenta CER | Reference CER | +|---|---|---| +| OmniDocBench English holdout | 18.88 % micro | — | +| 43-page set | 23.76 % | 15.51 % (3B GPU reference) | + +A disclosure field such as "$150,000 to $249,999" is useless unless every digit is right. At roughly a 1-in-5 character error rate, most multi-digit amounts would fail. + +**To close it:** + +- **Digit error rate.** Report digit/numeric-field CER separately (the metric a financial-forms buyer asks for first). +- **Field-level confidence.** Report per-field confidence that a caller can gate on. +- **Constrained decoding.** Allow a numeric-only or charset-restricted mode for fields known to be amounts or dates. + +### 4. No form structure: key-value pairs, checkboxes, tables as output (source, observed) + +**Gap.** `OcrOutput` is blocks → lines → words, with boxes. The `OcrBlock` docs say region classification (title, table, figure) "arrives as typed payloads with the DOCUMENT milestone — deliberately absent from v1". `table.rs` exists (`recognize` → `TableStructure`), but it is not part of `OcrEngine`'s output. + +The disclosure forms are form grids: + +- labelled cells ("Name of Creditor", "Amount of Liability") with bordered rows; +- ☐ / ☒ Yes/No checkboxes that decide whether a section applies. + +**Evidence.** On born-digital forms, the drawn table rules decide row and column membership. The fraud-alert table reader depends on them because centered headings do not line up with left-aligned values. On scans that geometry has to come from the image. + +**To close it:** + +- **Tables in OCR output.** Put table structure (cells with row and column indices) into `OcrOutput`. +- **Checkbox output.** Add a checkbox or mark detector, emitting checked/unchecked state with a box. +- **Key-value pairing.** Emit label → value pairing for form fields. + +These are the features that make a document OCR engine commercially useful for forms. + +### 5. Tilt and phone photos (source, observed) + +**Gap.** The README states the photographed-receipt pipeline trails PaddleOCR (20.9 % vs 15.6 % CER), and names tilt-sensitive line grouping as the cause, with deskew as the fix. CO's filings (2023 on) are phone photos: the PDF producer is "iOS Quartz PDFContext", and the pages carry no text layer. + +**To close it:** deskew plus perspective correction before detection. Rotation detection (0/90/180/270) is also needed, because KY stores portrait forms as landscape pages. Whether Carmenta detects rotation today is **unverified**. + +### 6. Run-to-run non-determinism (source) + +**Gap.** The README says run-to-run CER varies about 0.5 pp on identical inputs, from a non-deterministic parallel reduction. fraud-alert's rule is that re-running an extraction on the same bytes gives the same record, so a stored value can be re-derived and audited. Output that changes between runs breaks that. + +**To close it:** a deterministic mode (fixed reduction order) that yields byte-identical output for identical input and weights, even at some throughput cost. + +**Correction (2026-09-24, from the Carmenta log).** The "~0.5 pp run-to-run CER variance" traces to the mission plan's §8.53, which measured a 17 % **speed** noise floor, not CER variance. What the log does record: + +- §8.46: recognised text is byte-identical when run alone. Only line confidences move, by about 1e-4, when another process competes for the CPU. +- §8.100: text is byte-identical across threading modes. + +So the premise is probably wrong. The batch 1 determinism test measures it directly rather than trusting either reading. + +### 7. Weights fetched at runtime into a relative `models` dir (source) + +**Gap.** `CraftCrnn::new()` uses `Path::new("models")`, relative to the working directory. The default `fetch` feature downloads weights at first use from hash-verified manifests. A scheduled batch job needs to know where weights live, and must fail fast (not download) when it is offline. + +**To close it:** + +- **Explicit weights dir.** Document the manifest dir as a required, explicit parameter in production use. +- **Prefetch.** Offer a `prefetch` / `verify` command. +- **No fetch mid-job.** Allow a build without runtime fetch that errors cleanly when weights are missing. + +### 8. Per-line confidence is optional (source) + +**Gap.** `OcrLine.confidence` and `OcrWord.confidence` are `Option`, "when the engine reports one". A caller that gates on confidence (keep a value only above a threshold, hold the rest) needs to know which engines always report it. For the CRNN recognizer this is **unverified**. + +**To close it:** guarantee confidence from every document engine, and document how it is calibrated. + +### 9. No in-memory image decode in `ffai-media` (source) + +**Gap.** `ffai_media::load_image` takes only a file path. The byte-level `decode_jpeg` / `decode_png` it dispatches to are private. A caller whose pages come out of a PDF as bytes must either write them to a temporary file or call `rusty_jpeg` directly, which duplicates the grayscale/RGB handling `ffai-media` already does. + +**To close it:** a public `decode_image(bytes: &[u8]) -> Result` with the same magic-byte dispatch. + +### 10. Model manifests are not shipped with the published crates (source, measured) + +**Gap.** `CraftCrnn::new_mobiledet` looks for manifests in `./models/*.toml`. Those TOMLs exist only in the FFai repository (`F:\coding\FFai\models`), not in the `ffai-carmenta` or `ffai-models` crates from crates.io. + +Some manifests also name no download source, because the weights are converted locally by FFai's `tools/*.py`, for example `det-fused.safetensors`. On a machine without the FFai repo and a warm model cache, a crates.io user gets `i/o error: The system cannot find the path specified`, which is what fraud-alert got until it copied five manifests by hand. + +**To close it:** + +- **Ship manifests.** Embed the OCR manifests in the crate (`include_str!`). +- **Hosted weights.** Publish converted weights somewhere a manifest can point to. +- **Clear error.** Make the error name the missing manifest file. + +### Measured, 2026-09-24 (Carmenta 0.10.2, `mobiledet-crnn`, CPU) + +The pages came from a Kentucky legislator filing (4 pages, 300 ppi, first page JPEG RGB, the rest CCITT G4) and an Indiana one (6 pages, 200 to 400 ppi). Only counts and confidence were recorded, not text. + +| Page | As stored: lines, mean conf | Rotated 90°: lines, mean conf | Rotated 270°: lines, mean conf | +|---|---|---|---| +| KY p1 (landscape-stored) | 5, 0.575 | 34, 0.831 | 34, **0.976** | +| KY p2 | 3, 0.380 | 26, 0.824 | 26, **0.984** | +| KY p3 | 16, 0.573 | 24, 0.807 | 23, **0.987** | +| KY p4 | 35, 0.577 | 17, 0.803 | 19, **0.953** | + +- **Orientation is the whole difference.** On upright clean scans Carmenta's line confidence is 0.95 to 0.99, with 0 to 1 lines per page under 0.8. On the same pages sideways it is 0.38 to 0.58 and it finds a fraction of the lines. This confirms gap 5: **Carmenta does not detect or correct rotation.** +- **A cheap workaround exists.** The right orientation wins on mean confidence by 0.15 or more on every page. A caller can run 0°, 90°, and 270° and keep the best, at three times the cost. A built-in orientation classifier (or a cheap detector-only pass per orientation) would remove that cost. +- **Confidence is always reported** by `mobiledet-crnn` (every line carried one), which answers gap 8 for this engine. +- **Speed:** about 2 to 3.5 s per page per orientation on CPU for 2550×3300 pages after the first page's model load (17 s cold). +- **Mixed sources within one state (observed):** West Virginia's filings come from very different tools. A pipeline needs to route each file to text extraction or OCR, which is gap 1 again. + + | Producer | Kind of file | + |---|---| + | iOS scanner | photos with a text layer | + | Konica Minolta copier, HP scanner | scans | + | Chrome/Skia print-to-PDF | born-digital text, no images | + | iText | generated | + + A page the probe first reported as undecodable was a Skia text PDF with no image at all, not a JPEG decode failure. + +### 11. Extracting a page's raw image reads through redactions (measured) — privacy + +**Gap.** Gap 1's workaround is to pull the page's image XObject out and OCR it, and that is unsafe. A published Kentucky filing hides the filer's home address under a black patch, and a viewer shows the patch. Extracting the scan and OCRing it returned the full address at high confidence, because the patch is a separate object drawn on top: + +- **What the patch is:** a second, small image placed over the scan in an incremental save, not a vector rectangle. +- **Other ways to redact:** a filled rectangle, or a white-out box. All of these exist in public-records PDFs. + +fraud-alert now: + +1. walks the page's content stream; +2. takes the largest image's transform; +3. paints every rectangle filled after it, and every smaller image drawn after it, onto the pixels (image overlays in black, the fail-safe choice); +4. only then runs OCR. + +A page whose drawing can't be walked is not read. After the fix, the same line reads "Home address:" and nothing more. + +**To close it:** the PDF entry point from gap 1 must return **the page as rendered** (the scan composited with everything drawn over it), never the raw image XObject. Annotations need the same treatment: a `/Redact` or `/Square` annotation with an appearance stream is also drawn over the page. For a product sold to anyone handling public records, legal discovery or HR files, this is the first thing a privacy review will test. + +### 12. Confidence does not flag wrong names (measured) + +**Setup.** One Kentucky legislator's filing (3 pages, JPEG and CCITT, 300 ppi, stored landscape) was read by hand against the Carmenta output. There are 17 answer lines (typed entries under the form's questions), and every one passed the 0.85 line-confidence gate. + +| Measure | Result | +|---|---| +| Lines exactly right | 7 of 17 (41 %) | +| Entity name right, punctuation aside | 13 of 17 (76 %) | +| Wrong entity name | 4 of 17: "401K" → "Z0TK", "Irish Hill" → "Trish HTlI", "I/ONX" → "IJONX", "Gulf Shores" → "GuIF Shores" | +| Symbols in the printed form | "$10,000" → "810,000", "$1000" → "S1OOO", "n/a" → "nla" | +| Clean printed paragraphs | near-perfect | + +- **Where the errors are.** They cluster on typed lines whose underline rule runs through the descenders. That is a form-specific condition, and the detector/recognizer doesn't separate the rule from the text. +- **Confidence doesn't help.** It did not separate these lines from correct ones, so a caller cannot gate names on it. + +**A second filing, read the same way** (18 answer lines, all passing the gate): + +| Measure | Result | +|---|---| +| Entity name right | 14 of 18 | +| Wrong | "Tri-State" → "Tr1-State", "Cabell" → "Cabe]l", "Merrill Lynch" → "MerrITLynch", and one handwritten line turned into noise ("N/A (… permission 2/11/26)" → "Nlt (p mpermssion 4"/a)") | +| Street numbers on struck-through lines | **every one lettered**: "1000" → "TOOU", "1117" → "TTT7", "1121" → "TT2T", "#305" → "#30S" | + +**Across both filings:** 27 of 35 entity names right (77 %); 8 wrong, and none flagged by confidence. + +- **Numbers on underlined forms can't be used.** On lines the form's underline runs through, digits come out as look-alike letters (1 → T/I, 0 → O/U, 5 → S). Any numeric field on an underlined form is unusable as read. +- **Why it matters for privacy.** A digits-based address filter misses these, so a caller has to treat digit look-alikes as digits when deciding what is an address. +- **Handwriting.** Handwritten notes are read as confident noise rather than rejected. A handwriting detector (or a low confidence on them) would let a caller drop them. + +**To close it:** + +- **Rule removal.** Strip horizontal rules from the image before recognition, since form underlines are universal. +- **Calibrated confidence.** Make confidence track character errors, most importantly for the `$`/`S`/`8` and `I`/`l`/`1` confusions. +- **Lexicon constraint.** Allow a caller-supplied lexicon, such as the form's own words or a list of known entity names. + +Until then, fraud-alert stores OCR'd names only as flagged, unverified leads. + +### 13. Throughput with orientation search (measured) + +Three orientation tries on landscape-stored 3-page filings took **23 to 30 s per filing** on CPU, about 8 to 10 s per page. The first filing was 64 s, including the model load. Kentucky's 259 legislator filings come to about 2 hours for a full re-read. That is acceptable for a batch job, but an orientation classifier (gap 5) would cut it by about two-thirds. + +### 14. Checkbox and tick marks read as characters (observed) + +Indiana's statement (a RICOH copier scan at 200 ppi) marks "Your interest / Spouse's interest / over $500,000" with hand ticks in table columns. Carmenta returns them as characters ("@", "0", "D", "B") at the confidence of text, so a caller can't tell a tick from a letter. This is the same form-structure gap as gap 4, measured: **the one value field on Indiana's form is unreadable through Carmenta.** + +**Workaround in use:** orientation found on a document's first page is tried first on the rest, and kept when mean confidence is at least 0.9. That turns three readings per page into about one, because a scanner feeds a document's pages the same way. An engine-level orientation classifier would still remove the first page's extra readings. + +--- + +*Next:* when Carmenta runs on the sample KY, WV, IN, and CO filings, fraud-alert will add measured numbers here: + +- lines per page; +- mean confidence; +- the share of dollar amounts that parse and match the form's printed range choices; +- pages per second on CPU; +- whether rotated KY pages read correctly. diff --git a/models/ppocrv5-mobile-rec.toml b/models/ppocrv5-mobile-rec.toml index 12b4b93c..c9b07bb1 100644 --- a/models/ppocrv5-mobile-rec.toml +++ b/models/ppocrv5-mobile-rec.toml @@ -10,11 +10,11 @@ # 18,383-class multilingual head does not confuse English against a 97-class # English-only one. It reads it better. # -# OPT-IN, NOT THE DEFAULT — the `crnn-zh-sim-g2` shape. Selected by engine name -# (`mobiledet-svtr`); the shipped default stays `mobiledet-crnn` until a 236-page -# engine A/B returns a macro gain whose CI excludes zero. LIVE wraps -# `dyn OcrEngine` and never reaches inside, so it is unaffected either way — but -# a global default flip would need its own live_bench/live_soak verdicts first. +# THE DOCUMENT DEFAULT since §8.171 (`ffai_carmenta::DOC_DEFAULT` = +# `mobiledet-svtr`): the 236-page engine A/B returned +1.521 pp macro, CI +# [+1.068, +2.016], at 1.91x the wall time. It was opt-in until then, which is +# what this comment used to say. LIVE still resolves the registry default +# (`craft-crnn`) until it has its own live_bench/live_soak verdicts. # # Converted from the paddlex cache by `tools/carmenta_svtr_prepare.py` # (234 tensors). The candle port in `svtr.rs` is pinned by diff --git a/tools/carmenta_checkbox_corpus.py b/tools/carmenta_checkbox_corpus.py new file mode 100644 index 00000000..e100636c --- /dev/null +++ b/tools/carmenta_checkbox_corpus.py @@ -0,0 +1,141 @@ +"""carmenta-checkbox-v1: form rows with checkboxes, ground truth by construction. + +Built for docs/plans/commercial-gaps.md gaps 4 and 14: a reader returned +checkbox marks as characters ("@", "0", "D", "B") at text confidence, so a +form whose value is a set of ticks could not be read. + +Each clip is a form fragment: question rows ending in "[ ] Yes [ ] No" +(and some three-way rows), plus DISTRACTORS a naive square-finder would take +for boxes: the letters O, D, 0, 8 and the ideograph 口 in running text, a +small table, and an underline rule. + +Boxes are drawn two ways (a stroked rectangle; the font's own ballot-box +glyph) and marked four ways (X strokes; a tick that overflows the box; a +solid fill; the font's ballot-box-with-X glyph), or left empty. Every clip is +rendered anti-aliased gray and 1-bit (a CCITT-style scan). Ground truth, in a +.json sidecar, is every box's frame and whether it is marked. + +All text is invented. Renders are CC0-1.0. +""" +import hashlib +import json +import random +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +REPO = Path(__file__).resolve().parent.parent +NAME = "carmenta-checkbox" +CLIPS = REPO / "corpora" / "clips" / NAME +TEXT_FONTS = ["C:/Windows/Fonts/arial.ttf", "C:/Windows/Fonts/times.ttf", "C:/Windows/Fonts/cour.ttf"] +SYMBOL_FONT = "C:/Windows/Fonts/seguisym.ttf" +CJK_FONT = "C:/Windows/Fonts/msyh.ttc" +W, H = 2100, 640 +QUESTIONS = [ + "Did you hold a position with a business?", + "Is the income over $500,000?", + "Was any gift received from a lobbyist?", + "Does your spouse hold stock in the company?", + "Real estate owned in this county?", + "Any creditor owed more than $10,000?", + "Interest held jointly with spouse?", + "Was the loan made on ordinary terms?", +] + + +def stroke_box(d, x, y, s, t): + d.rectangle([x, y, x + s - 1, y + s - 1], outline=0, width=t) + + +def mark(d, kind, x, y, s, t): + """Draw a mark in the box at (x, y) of side s. Returns nothing.""" + if kind == "x": + m = s // 5 + d.line([x + m, y + m, x + s - m, y + s - m], fill=0, width=t + 1) + d.line([x + s - m, y + m, x + m, y + s - m], fill=0, width=t + 1) + elif kind == "tick": + # starts inside, ends above and right of the box: an overflowing tick + d.line([x + s // 5, y + s // 2, x + s * 2 // 5, y + s * 4 // 5], fill=0, width=t + 2) + d.line([x + s * 2 // 5, y + s * 4 // 5, x + s * 6 // 5, y - s // 3], fill=0, width=t + 2) + elif kind == "fill": + d.rectangle([x + t + 2, y + t + 2, x + s - t - 3, y + s - t - 3], fill=0) + + +def render(seed, ink): + rnd = random.Random(seed) + img = Image.new("L", (W, H), 255) + d = ImageDraw.Draw(img) + font_path = TEXT_FONTS[seed % len(TEXT_FONTS)] + em = rnd.choice([34, 40, 46]) + font = ImageFont.truetype(font_path, em) + sym = ImageFont.truetype(SYMBOL_FONT, int(em * 1.1)) + boxes = [] + y = 30 + for row in range(5): + q = rnd.choice(QUESTIONS) + d.text((30, y), q, font=font, fill=0) + options = ["Yes", "No"] if row % 3 else ["Self", "Spouse", "Both"] + # Boxes start after the question: a long line in a wide font must + # never put text inside a box that the ground truth calls empty. + x = max(1000, 30 + int(d.textlength(q, font=font)) + 60) + chosen = rnd.randrange(len(options) + 1) # len(options): none marked + for i, opt in enumerate(options): + s = int(em * rnd.choice([0.8, 0.9, 1.0])) + t = max(2, s // 16) + glyph = rnd.random() < 0.3 + by = y + (em - s) // 2 + 4 + marked = i == chosen + kind = rnd.choice(["x", "tick", "fill", "glyph"]) if marked else "none" + if glyph: + ch = "\u2612" if kind in ("glyph", "x") else "\u2610" + d.text((x, by), ch, font=sym, fill=0) + gx0, gy0, gx1, gy1 = d.textbbox((x, by), ch, font=sym) + bx, by2, bs = gx0, gy0, max(gx1 - gx0, gy1 - gy0) + if kind in ("tick", "fill"): + mark(d, kind, gx0, gy0, gx1 - gx0, t) + frame = [gx0, gy0, gx1 - gx0, gy1 - gy0] + else: + stroke_box(d, x, by, s, t) + if marked: + mark(d, "x" if kind == "glyph" else kind, x, by, s, t) + frame = [x, by, s, s] + boxes.append({"bbox": frame, "checked": marked, "mark": kind, "glyph": glyph}) + d.text((frame[0] + frame[2] + 12, y), opt, font=font, fill=0) + x = frame[0] + frame[2] + 30 + int(d.textlength(opt, font=font)) + 40 + y += int(em * 2.1) + # Distractors: square-ish letters, a CJK ideograph, a table, a rule. + d.text((30, y), "O D 0 8 QOD 080 ODD", font=font, fill=0) + cjk = ImageFont.truetype(CJK_FONT, em) + d.text((700, y), "\u53e3\u53e3 \u56de", font=cjk, fill=0) + ty = y + int(em * 1.6) + for c in range(4): + for r in range(2): + d.rectangle([30 + c * 220, ty + r * 70, 30 + (c + 1) * 220, ty + (r + 1) * 70], outline=0, width=2) + d.rectangle([1000, ty + 60, 1650, ty + 62], fill=0) + if ink == "bw": + img = img.point(lambda v: 0 if v < 128 else 255, mode="L") + return img, boxes + + +def main(): + CLIPS.mkdir(parents=True, exist_ok=True) + man = [f'name = "{NAME}"', "version = 1", 'task = "ocr"'] + n = 0 + for seed in range(40): + for ink in ["gray", "bw"]: + img, boxes = render(seed, ink) + cid = f"cb-{seed:02}-{ink}" + png = CLIPS / f"{cid}.png" + img.save(png, optimize=True) + (CLIPS / f"{cid}.json").write_text(json.dumps({"ink": ink, "seed": seed, "boxes": boxes}), encoding="utf-8") + sha = hashlib.sha256(png.read_bytes()).hexdigest() + man += ["", "[[clips]]", f'id = "{cid}"', f'path = "clips/{NAME}/{png.name}"', + f'class = "checkbox-{ink}"', f'split = "{"train" if seed % 2 == 0 else "holdout"}"', + 'license = "CC0-1.0 (render of invented form rows)"', f'sha256 = "{sha}"'] + n += 1 + (REPO / "corpora" / f"{NAME}-v1.toml").write_text("\n".join(man) + "\n", encoding="utf-8") + print(f"{n} clips -> {CLIPS}") + + +if __name__ == "__main__": + main() diff --git a/tools/carmenta_form_corpus.py b/tools/carmenta_form_corpus.py new file mode 100644 index 00000000..e80038f3 --- /dev/null +++ b/tools/carmenta_form_corpus.py @@ -0,0 +1,138 @@ +"""carmenta-form-v1: typed form values on ruled lines, ground truth by construction. + +Built for docs/plans/commercial-gaps.md gap 12. A caller reading scanned +financial-disclosure forms found values on ruled lines read as look-alike +letters ("1117" -> "TTT7", "$10,000" -> "810,000") at confidences that passed +a 0.85 gate. The filings themselves are public records full of personal data +and the caller does not keep them, so this corpus reproduces the CONDITION +instead: the same fictional values rendered with no rule (the control) and +with three kinds of rule, so every error the rule causes is attributable to +it by a paired comparison. + +Axes: + rule none | under (touching the baseline) | desc (through the descender + zone) | strike (through the middle of the glyphs) + ink gray (anti-aliased) | bw (1-bit, the way a CCITT Group 4 scan arrives) + font Courier New, Arial, Times New Roman, cycled per value + size 40 / 46 / 52 px em, i.e. 10-12 pt at 300 ppi + +Values with an even index are split=train (tune thresholds there), odd are +split=holdout (claim there). Each clip is one form line: a printed label and +a typed value; ground truth is "label value". A sidecar .json records the +value and condition so a scorer can measure the VALUE exactly. + +All values are invented. Fonts are the Windows system fonts; the renders are +released CC0-1.0. +""" +import hashlib +import json +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +REPO = Path(__file__).resolve().parent.parent +NAME = "carmenta-form" +CLIPS = REPO / "corpora" / "clips" / NAME +FONTS = [ + ("courier", "C:/Windows/Fonts/cour.ttf"), + ("arial", "C:/Windows/Fonts/arial.ttf"), + ("times", "C:/Windows/Fonts/times.ttf"), +] +SIZES = [40, 46, 52] +W, H = 1500, 150 + +# (label, value, kind). Invented, and chosen for the confusions seen in the +# field: 1/I/l/T, 0/O/U, 5/S, 8/$, and names with descenders. +VALUES = [ + ("Amount:", "$10,000", "amount"), + ("Amount:", "$1,000", "amount"), + ("Amount:", "$150,000 to $249,999", "amount"), + ("Amount:", "$5,001 - $10,000", "amount"), + ("Amount:", "$500,000", "amount"), + ("Amount:", "$1,117.00", "amount"), + ("Amount:", "$25,000", "amount"), + ("Amount:", "$100", "amount"), + ("Amount:", "$2,500", "amount"), + ("Amount:", "$75,000", "amount"), + ("Address:", "1117 Oak Street", "number"), + ("Address:", "1000 Park Avenue", "number"), + ("Address:", "1121 River Road", "number"), + ("Unit:", "#305", "number"), + ("Address:", "Suite 1414", "number"), + ("Address:", "PO Box 1810", "number"), + ("Plan:", "401K Plan", "number"), + ("Date:", "2/11/26", "number"), + ("Date:", "12/31/2025", "number"), + ("Address:", "Unit 7", "number"), + ("Location:", "Gulf Shores", "name"), + ("Location:", "Irish Hill", "name"), + ("Name of Creditor:", "Merrill Lynch", "name"), + ("Name of Creditor:", "Tri-State Bank", "name"), + ("Location:", "Cabell County", "name"), + ("Source of Income:", "Kentucky Utilities", "name"), + ("Name of Creditor:", "First Federal Savings", "name"), + ("Source of Income:", "Blue Grass Energy", "name"), + ("Source of Income:", "Lexington Clinic", "name"), + ("Business:", "Hillcrest Farms", "name"), +] +RULES = ["none", "under", "desc", "strike"] +INKS = ["gray", "bw"] + + +def render(label, value, font_path, em, rule, ink): + img = Image.new("L", (W, H), 255) + d = ImageDraw.Draw(img) + font = ImageFont.truetype(font_path, em) + ascent, descent = font.getmetrics() + baseline = 30 + ascent + x_label, x_value = 40, 560 + d.text((x_label, baseline), label, font=font, fill=0, anchor="ls") + d.text((x_value, baseline), value, font=font, fill=0, anchor="ls") + x0, _, x1, _ = d.textbbox((x_value, baseline), value, font=font, anchor="ls") + t = max(2, round(em / 16)) # rule thickness: 2-3 px at 300 ppi + if rule == "under": + # a form's field line: spans the field, touching the glyph bottoms + d.rectangle([x_value - 20, baseline, W - 40, baseline + t - 1], fill=0) + elif rule == "desc": + y = baseline + round(descent * 0.45) + d.rectangle([x_value - 20, y, W - 40, y + t - 1], fill=0) + elif rule == "strike": + # a filer striking through an entry: only across the typed text + y = baseline - round(ascent * 0.32) + d.rectangle([x0 - 6, y, x1 + 6, y + t - 1], fill=0) + if ink == "bw": + img = img.point(lambda v: 0 if v < 128 else 255, mode="L") + return img + + +def main(): + CLIPS.mkdir(parents=True, exist_ok=True) + man = [f'name = "{NAME}"', "version = 1", 'task = "ocr"'] + n = 0 + for i, (label, value, kind) in enumerate(VALUES): + font_name, font_path = FONTS[i % len(FONTS)] + em = SIZES[(i // len(FONTS)) % len(SIZES)] + split = "train" if i % 2 == 0 else "holdout" + for rule in RULES: + for ink in INKS: + cid = f"form-{i:02}-{rule}-{ink}" + png = CLIPS / f"{cid}.png" + render(label, value, font_path, em, rule, ink).save(png, optimize=True) + (CLIPS / f"{cid}.txt").write_text(f"{label} {value}", encoding="utf-8") + meta = {"value": value, "label": label, "kind": kind, "rule": rule, + "ink": ink, "font": font_name, "em": em, "index": i} + (CLIPS / f"{cid}.json").write_text(json.dumps(meta), encoding="utf-8") + sha = hashlib.sha256(png.read_bytes()).hexdigest() + man += ["", "[[clips]]", f'id = "{cid}"', + f'path = "clips/{NAME}/{png.name}"', + f'ground_truth = "clips/{NAME}/{cid}.txt"', + f'class = "form-{rule}-{ink}"', f'split = "{split}"', + 'license = "CC0-1.0 (render of invented values in Windows system fonts)"', + f'sha256 = "{sha}"'] + n += 1 + (REPO / "corpora" / f"{NAME}-v1.toml").write_text("\n".join(man) + "\n", encoding="utf-8") + print(f"{n} clips -> {CLIPS}") + + +if __name__ == "__main__": + main() From 60d5ceb980eb24f57f38571ac348986fafab185e Mon Sep 17 00:00:00 2001 From: Tim Date: Fri, 25 Sep 2026 09:49:02 -0700 Subject: [PATCH 2/2] fix(supply-chain): cargo-vet exemptions for lopdf and what it brings in The pdf feature adds lopdf 0.44 and fax 0.3. lopdf's non-optional encryption support brings aes, cbc, ecb, cipher, md-5, stringprep and small helpers. All of them are exempted at their locked versions, which is the repo's convention for dependency deltas. The ffai-* rows move to their published versions, and two stale entries (shlex, simdutf8) are pruned. Co-Authored-By: Claude Opus 5.5 (1M context) --- supply-chain/config.toml | 90 +++++++++++++++++++++------- supply-chain/imports.lock | 120 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 21 deletions(-) diff --git a/supply-chain/config.toml b/supply-chain/config.toml index df5b0b57..3b50ea61 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -91,6 +91,10 @@ audit-as-crates-io = true [policy.tokenizers] audit-as-crates-io = true +[[exemptions.aes]] +version = "0.9.3" +criteria = "safe-to-deploy" + [[exemptions.ahash]] version = "0.8.12" criteria = "safe-to-deploy" @@ -131,6 +135,10 @@ criteria = "safe-to-deploy" version = "0.12.1" criteria = "safe-to-deploy" +[[exemptions.block-padding]] +version = "0.4.2" +criteria = "safe-to-deploy" + [[exemptions.bon]] version = "3.9.3" criteria = "safe-to-deploy" @@ -179,6 +187,10 @@ criteria = "safe-to-deploy" version = "0.2.4" criteria = "safe-to-deploy" +[[exemptions.cbc]] +version = "0.2.1" +criteria = "safe-to-deploy" + [[exemptions.cc]] version = "1.4.0" criteria = "safe-to-deploy" @@ -191,6 +203,10 @@ criteria = "safe-to-deploy" version = "0.4.45" criteria = "safe-to-deploy" +[[exemptions.cipher]] +version = "0.5.2" +criteria = "safe-to-deploy" + [[exemptions.colored]] version = "3.1.1" criteria = "safe-to-deploy" @@ -227,6 +243,10 @@ criteria = "safe-to-deploy" version = "0.3.0" criteria = "safe-to-deploy" +[[exemptions.cpubits]] +version = "0.1.1" +criteria = "safe-to-deploy" + [[exemptions.cpufeatures]] version = "0.2.17" criteria = "safe-to-deploy" @@ -347,6 +367,10 @@ criteria = "safe-to-deploy" version = "0.1.3" criteria = "safe-to-deploy" +[[exemptions.ecb]] +version = "0.2.1" +criteria = "safe-to-deploy" + [[exemptions.enum-as-inner]] version = "0.6.1" criteria = "safe-to-deploy" @@ -371,56 +395,60 @@ criteria = "safe-to-deploy" version = "2.5.0" criteria = "safe-to-deploy" +[[exemptions.fax]] +version = "0.3.0" +criteria = "safe-to-deploy" + [[exemptions.ffai-argus]] -version = "0.7.3" +version = "0.7.4" criteria = "safe-to-deploy" [[exemptions.ffai-argus-wasm]] -version = "0.1.1" +version = "0.1.2" criteria = "safe-to-deploy" [[exemptions.ffai-bench]] -version = "0.7.5" +version = "0.7.6" criteria = "safe-to-deploy" [[exemptions.ffai-carmenta]] -version = "0.10.1" +version = "0.10.2" criteria = "safe-to-deploy" [[exemptions.ffai-carmenta-wasm]] -version = "0.2.1" +version = "0.2.2" criteria = "safe-to-deploy" [[exemptions.ffai-cli]] -version = "0.6.8" +version = "0.6.9" criteria = "safe-to-deploy" [[exemptions.ffai-core]] -version = "0.7.1" +version = "0.7.2" criteria = "safe-to-deploy" [[exemptions.ffai-diana]] -version = "0.7.6" +version = "0.7.7" criteria = "safe-to-deploy" [[exemptions.ffai-media]] -version = "0.6.6" +version = "0.6.7" criteria = "safe-to-deploy" [[exemptions.ffai-mercury]] -version = "1.0.3" +version = "1.0.4" criteria = "safe-to-deploy" [[exemptions.ffai-mercury-wasm]] -version = "0.1.1" +version = "0.1.2" criteria = "safe-to-deploy" [[exemptions.ffai-models]] -version = "0.6.4" +version = "0.6.5" criteria = "safe-to-deploy" [[exemptions.ffai-wasm]] -version = "0.1.3" +version = "0.1.4" criteria = "safe-to-deploy" [[exemptions.find-msvc-tools]] @@ -663,6 +691,10 @@ criteria = "safe-to-deploy" version = "0.4.33" criteria = "safe-to-deploy" +[[exemptions.lopdf]] +version = "0.44.0" +criteria = "safe-to-deploy" + [[exemptions.lru-slab]] version = "0.1.2" criteria = "safe-to-deploy" @@ -675,6 +707,10 @@ criteria = "safe-to-deploy" version = "0.3.11" criteria = "safe-to-deploy" +[[exemptions.md-5]] +version = "0.11.0" +criteria = "safe-to-deploy" + [[exemptions.memmap2]] version = "0.9.11" criteria = "safe-to-deploy" @@ -699,6 +735,10 @@ criteria = "safe-to-deploy" version = "0.16.1" criteria = "safe-to-deploy" +[[exemptions.nom]] +version = "8.0.0" +criteria = "safe-to-deploy" + [[exemptions.ntapi]] version = "0.4.3" criteria = "safe-to-deploy" @@ -839,6 +879,10 @@ criteria = "safe-to-deploy" version = "0.10.2" criteria = "safe-to-deploy" +[[exemptions.rangemap]] +version = "1.8.0" +criteria = "safe-to-deploy" + [[exemptions.raw-cpuid]] version = "11.6.0" criteria = "safe-to-deploy" @@ -1035,10 +1079,6 @@ criteria = "safe-to-deploy" version = "3.1.2" criteria = "safe-to-deploy" -[[exemptions.shlex]] -version = "2.0.1" -criteria = "safe-to-deploy" - [[exemptions.simd-adler32]] version = "0.3.10" criteria = "safe-to-deploy" @@ -1047,10 +1087,6 @@ criteria = "safe-to-deploy" version = "1.2.0" criteria = "safe-to-deploy" -[[exemptions.simdutf8]] -version = "0.1.5" -criteria = "safe-to-deploy" - [[exemptions.spm_precompiled]] version = "0.1.4" criteria = "safe-to-deploy" @@ -1067,6 +1103,10 @@ criteria = "safe-to-deploy" version = "0.2.4" criteria = "safe-to-deploy" +[[exemptions.stringprep]] +version = "0.1.5" +criteria = "safe-to-deploy" + [[exemptions.symlink]] version = "0.1.0" criteria = "safe-to-deploy" @@ -1203,6 +1243,10 @@ criteria = "safe-to-deploy" version = "0.1.12" criteria = "safe-to-deploy" +[[exemptions.unicode-properties]] +version = "0.1.4" +criteria = "safe-to-deploy" + [[exemptions.unicode_categories]] version = "0.1.1" criteria = "safe-to-deploy" @@ -1271,6 +1315,10 @@ criteria = "safe-to-deploy" version = "1.0.9" criteria = "safe-to-deploy" +[[exemptions.weezl]] +version = "0.2.1" +criteria = "safe-to-deploy" + [[exemptions.which]] version = "7.0.3" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index b5ba6817..c3a1b4b1 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -1,6 +1,58 @@ # cargo-vet imports lock +[[unpublished.ffai-argus]] +version = "0.7.5" +audited_as = "0.7.4" + +[[unpublished.ffai-argus-wasm]] +version = "0.1.3" +audited_as = "0.1.2" + +[[unpublished.ffai-bench]] +version = "0.7.7" +audited_as = "0.7.6" + +[[unpublished.ffai-carmenta]] +version = "0.11.0" +audited_as = "0.10.2" + +[[unpublished.ffai-carmenta-wasm]] +version = "0.2.3" +audited_as = "0.2.2" + +[[unpublished.ffai-cli]] +version = "0.7.0" +audited_as = "0.6.9" + +[[unpublished.ffai-core]] +version = "0.8.0" +audited_as = "0.7.2" + +[[unpublished.ffai-diana]] +version = "0.7.8" +audited_as = "0.7.7" + +[[unpublished.ffai-media]] +version = "0.6.8" +audited_as = "0.6.7" + +[[unpublished.ffai-mercury]] +version = "1.0.5" +audited_as = "1.0.4" + +[[unpublished.ffai-mercury-wasm]] +version = "0.1.3" +audited_as = "0.1.2" + +[[unpublished.ffai-models]] +version = "0.6.6" +audited_as = "0.6.5" + +[[unpublished.ffai-wasm]] +version = "0.1.5" +audited_as = "0.1.4" + [[publisher.aho-corasick]] version = "1.1.4" when = "2025-10-28" @@ -745,6 +797,13 @@ user-id = 3618 user-login = "dtolnay" user-name = "David Tolnay" +[[publisher.unicode-normalization]] +version = "0.1.25" +when = "2025-10-30" +user-id = 1139 +user-login = "Manishearth" +user-name = "Manish Goregaokar" + [[publisher.unicode-segmentation]] version = "1.13.3" when = "2026-06-01" @@ -1336,6 +1395,12 @@ crate is broadly used throughout the ecosystem and does not contain anything suspicious. """ +[[audits.bytecode-alliance.audits.inout]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "0.1.3" +notes = "A part of RustCrypto/utils, this crate is designed to handle unsafe buffers and carefully documents the safety concerns throughout. Older versions of this tally up to ~130k daily downloads." + [[audits.bytecode-alliance.audits.matchers]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -1435,6 +1500,12 @@ criteria = "safe-to-deploy" version = "0.1.4" notes = "I always really enjoy reading eliza's code, she left perfect comments at every use of unsafe." +[[audits.bytecode-alliance.audits.shlex]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.1.0" +notes = "Only minor `unsafe` code blocks which look valid and otherwise does what it says on the tin." + [[audits.bytecode-alliance.audits.tar]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -1940,6 +2011,13 @@ criteria = "safe-to-deploy" version = "0.1.0" aggregated-from = "https://chromium.googlesource.com/chromiumos/third_party/rust_crates/+/refs/heads/main/cargo-vet/audits.toml?format=TEXT" +[[audits.google.audits.unicode-bidi]] +who = "Manish Goregaokar " +criteria = "safe-to-deploy" +version = "0.3.18" +notes = "Contains one line of repr(transparent) unsafe" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.utf8parse]] who = "David Koloski " criteria = "safe-to-deploy" @@ -1997,6 +2075,11 @@ who = "David Cook " criteria = "safe-to-deploy" delta = "0.2.17 -> 0.3.0" +[[audits.isrg.audits.inout]] +who = "David Cook " +criteria = "safe-to-deploy" +delta = "0.1.4 -> 0.2.2" + [[audits.isrg.audits.rand_chacha]] who = "David Cook " criteria = "safe-to-deploy" @@ -2086,6 +2169,15 @@ end = "2027-09-07" notes = "I, Henri Sivonen, wrote encoding_rs for Gecko and have reviewed contributions by others." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.wildcard-audits.unicode-normalization]] +who = "Manish Goregaokar " +criteria = "safe-to-deploy" +user-id = 1139 # Manish Goregaokar (Manishearth) +start = "2019-11-06" +end = "2027-04-23" +notes = "All code written or reviewed by Manish" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.wildcard-audits.unicode-segmentation]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -2695,6 +2787,28 @@ criteria = "safe-to-deploy" delta = "0.1.4 -> 0.1.7" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.shlex]] +who = "Max Inden " +criteria = "safe-to-deploy" +delta = "1.1.0 -> 1.3.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.shlex]] +who = "Emilio Cobos Álvarez " +criteria = "safe-to-deploy" +delta = "1.3.0 -> 2.0.1" +notes = """ +Mostly removes some deprecated and unsound APIs. +""" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.simdutf8]] +who = "Henri Sivonen " +criteria = "safe-to-deploy" +version = "0.1.5" +notes = "Confidence in correctness of the algorithm is based on fuzzing the SSE 4.2 and AVX2 implementations rather than working through the logic of the code. Audit of aarch64 and Wasm is by comparing the code with the SSE 4.2 case." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.smallvec]] who = "Erich Gubler " criteria = "safe-to-deploy" @@ -2917,6 +3031,12 @@ criteria = "safe-to-deploy" delta = "0.3.13 -> 0.3.14" aggregated-from = "https://raw.githubusercontent.com/zcash/librustzcash/main/supply-chain/audits.toml" +[[audits.zcash.audits.inout]] +who = "Jack Grigg " +criteria = "safe-to-deploy" +delta = "0.1.3 -> 0.1.4" +aggregated-from = "https://raw.githubusercontent.com/zcash/wallet/main/supply-chain/audits.toml" + [[audits.zcash.audits.num-conv]] who = "Kris Nuttycombe " criteria = "safe-to-deploy"