diff --git a/.github/workflows/dyadt-verify.yml b/.github/workflows/dyadt-verify.yml index 7c95b9454..8c9f83235 100644 --- a/.github/workflows/dyadt-verify.yml +++ b/.github/workflows/dyadt-verify.yml @@ -36,6 +36,14 @@ jobs: run: | bash 1-formats/sub-specs/did-you-actually-do-that/spec/conformance/run-conformance.sh + - name: FFP conformance vectors (form-fill provenance) + run: | + # Same shape as DYADT: the spec's own vectors are the gate. A detector + # change that stops reproducing a vector's expected line fails here. + # Reference probe only (bash + awk); a product detector runs the same + # runner via FFP_DETECTOR=. + bash 1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh + - name: Verify this change's own claims (dogfood) env: DYADT_BASE: origin/${{ github.base_ref || 'main' }} diff --git a/.machine_readable/REGISTRY.a2ml b/.machine_readable/REGISTRY.a2ml index 01c85cef7..0f4ca3c25 100644 --- a/.machine_readable/REGISTRY.a2ml +++ b/.machine_readable/REGISTRY.a2ml @@ -20,7 +20,7 @@ version = "1.0.0" generator = "scripts/build-registry.sh" hash_algorithm = "sha256(git ls-files -s ) # local; external: recorded pin" -entry_count = 33 +entry_count = 34 [registry.streams] foundation = "A2ML format family + K9 + contractiles (Stream 1)" @@ -282,6 +282,15 @@ canonical_doc = "1-formats/templates/STATE.a2ml.v2.spec.adoc" source_hash = "sha256:5dbe5d5bef5e084631af8523ba210980b8a21e3969f382b6f0a52ca791609a6a" route = "copy-in templates for the 7 A2ML files" +[[spec]] +id = "form-fill-provenance" +name = "FFP — Form-Fill Provenance" +stream = "foundation" +home = "1-formats/sub-specs/form-fill-provenance/" +canonical_doc = "1-formats/sub-specs/form-fill-provenance/README.adoc" +source_hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +route = "whether a PDF form was machine-filled or printed blank for hand completion, and what a print path must record" + [[spec]] id = "affine-spec" name = "AffineScript .affine (faces / source documents)" diff --git a/.machine_readable/scorecards/form-fill-provenance.scorecard.a2ml b/.machine_readable/scorecards/form-fill-provenance.scorecard.a2ml new file mode 100644 index 000000000..7e8603d35 --- /dev/null +++ b/.machine_readable/scorecards/form-fill-provenance.scorecard.a2ml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: CC-BY-SA-4.0 +# form-fill-provenance.scorecard.a2ml +# Hand-authored source. Regenerate the dashboard with: just scorecards +# Schema: .machine_readable/scorecards/scorecard.schema.json + +[scorecard] +spec_id = "form-fill-provenance" +version = "1.0.0" +assessed_date = "2026-10-04" +assessor = "arena session 01a1046c (standards), on ruling D189" + +[[must]] +id = "M1" +text = "The DETECTION rules MUST be executable and exact: a detector MUST reproduce every conformance vector's expected line, and the vectors MUST be byte-reproducible from their generator." +system = "1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh — 14 vectors, each diffed against expected/, plus a make-fixtures.sh --check reproducibility pass" +status = "pass" +evidence = "just ffp-conformance: 14 passed, 0 failed, 0 orphan expectation(s); run 2026-10-04" +check = "bash 1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh >/dev/null 2>&1" + +[[must]] +id = "M2" +text = "The marker MUST be recognised by namespace, `filledBy=machine` MUST be the only defined value, and an unrecognised declaration MUST be recorded rather than trusted." +system = "vectors machine-filled-declared, machine-filled-declared-generated and declared-unknown-value, diffed by run-conformance.sh" +status = "pass" +evidence = "declared-unknown-value expects FFP-E-DECL-UNRECOGNISED with classification machine-filled-suspected — the unknown declaration does not promote the classification; machine-filled-declared-generated expects machine-filled with appearances=generated, i.e. declaration wins over the structural signature" +check = "grep -q 'FFP-E-DECL-UNRECOGNISED' 1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/declared-unknown-value.expected && grep -q 'form-fill-provenance/1.0/' 1-formats/sub-specs/form-fill-provenance/spec/MARKER.adoc" + +[[must]] +id = "M3" +text = "A print-path consumer MUST classify every entered document, persist the FFP record with the job, keep it in the audit trail, and make it visible/routable (FFP/3 P1-P4) — the D189 obligation in presswerk." +system = "none — presswerk has no FFP code; D189 ruled 2026-09-30 on presswerk#118, implementation issue filed 2026-10-04" +status = "fail" +effects = "A machine-filled form and a blank form to be completed by hand stay byte-indistinguishable in presswerk's queue and audit trail, so routing and audit cannot tell them apart and no record survives the print. The blank-print hazard (values present, appearance streams absent) is also unrecorded, so a job can print empty fields with nothing in the trail to explain it." + +[[must]] +id = "M4" +text = "A form-filling producer MUST declare machine authorship via the FFP marker, and SHOULD generate widget appearance streams (FFP/1)." +system = "none — blocky-writer writes /V and NeedAppearances but no metadata marker; producer issue filed 2026-10-04" +status = "fail" +effects = "blocky-writer output can only ever classify as machine-filled-suspected (weaker evidence, no tool/version attribution), and its reliance on NeedAppearances leaves print output dependent on a flag PDF 2.0 deprecates and that some viewers and rasterisers ignore — the condition FFP/3 P5 exists to warn about." + +[[should]] +id = "S1" +text = "The conformance suite SHOULD run in this repository's CI on every pull request, not only on a developer's workstation." +system = ".github/workflows/dyadt-verify.yml step 'FFP conformance vectors'" +status = "pass" +evidence = "step added 2026-10-04, running the same runner as just ffp-conformance; no new action references, so the actions.lock gate is untouched" +check = "grep -q 'form-fill-provenance/spec/conformance/run-conformance.sh' .github/workflows/dyadt-verify.yml" + +[[should]] +id = "S2" +text = "A product detector SHOULD be testable against the same vectors without adding a runtime to the suite (no Python, no JS)." +system = "run-conformance.sh honours FFP_DETECTOR=; the whole conformance tree is bash + awk" +status = "pass" +evidence = "the reference path itself is invoked as FFP_DETECTOR default ('awk -f probe.awk'); no .py/.js/.ts file exists under the spec home" +check = "test -z \"$(find 1-formats/sub-specs/form-fill-provenance -name '*.py' -o -name '*.js' -o -name '*.ts')\" && grep -q 'FFP_DETECTOR' 1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh" + +[[could]] +id = "C1" +text = "A future revision COULD carry the appearance hazard into the submission gate (hold for confirmation) once a print-path consumer implements FFP/3, so that P5's SHOULD becomes mechanical." +system = "none — deliberately aspirational; P5 is a SHOULD in FFP/3 v1.0.0" +status = "aspirational" +effects = "Until a consumer implements it, the hazard is recorded and surfaced but not gated; a user can still send a form to the printer and receive blank fields." diff --git a/0-canon/COMPLIANCE-DASHBOARD.adoc b/0-canon/COMPLIANCE-DASHBOARD.adoc index 42e8ed586..8ec4d1f28 100644 --- a/0-canon/COMPLIANCE-DASHBOARD.adoc +++ b/0-canon/COMPLIANCE-DASHBOARD.adoc @@ -51,15 +51,16 @@ ____ | `+release-pre-flight+` | ❌ gap | 4/5 | 3/4 | 0/2 | 72% | 7/7 | 2026-07-03 | `+hypatia-rules+` | ❌ gap | 2/4 | 1/3 | 1/3 | 100% | 4/4 | 2026-07-03 | `+a2ml-templates+` | ❌ gap | 1/5 | 1/3 | 0/2 | 20% | 2/2 | 2026-07-03 +| `+form-fill-provenance+` | ❌ gap | 2/4 | 2/2 | 0/1 | 100% | 4/4 | 2026-10-04 |=== == Estate rollup -* *Specs registered (local):* 28 -* *Specs with a scorecard:* 28 / 28 -* *MUST requirements:* 36 passing / 137 total (71 failing) -* *Estate systems coverage:* 66% of 317 graded requirements have a mechanical check -* *Grounded passes:* 65 / 65 (100%) pass rows carry an executable `+check+` run by `+--verify+` +* *Specs registered (local):* 29 +* *Specs with a scorecard:* 29 / 29 +* *MUST requirements:* 38 passing / 141 total (73 failing) +* *Estate systems coverage:* 67% of 324 graded requirements have a mechanical check +* *Grounded passes:* 69 / 69 (100%) pass rows carry an executable `+check+` run by `+--verify+` == How this dashboard stays honest diff --git a/1-formats/sub-specs/form-fill-provenance/README.adoc b/1-formats/sub-specs/form-fill-provenance/README.adoc new file mode 100644 index 000000000..4f8379d26 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/README.adoc @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised) +// +// FFP — Form-Fill Provenance. Normative sub-spec of 1-formats. +// Home: 1-formats/sub-specs/form-fill-provenance/ +// Version: 1.0.0 · Status: ACTIVE · Ratified: 2026-10-04 + += Form-Fill Provenance (FFP) — machine-filled vs hand-completed forms in the print path +:toc: preamble +:icons: font +:revdate: 2026-10-04 +:revnumber: 1.0.0 +:status: ACTIVE + +[.lead] +FFP is the estate standard for *stating, detecting and recording how a PDF form +came to be filled* — by a machine before printing, or by hand after it. It +exists because a filled application form is a print job, and the print path +currently cannot tell the two apart. + +== Why this document exists + +`blocky-writer` fills fixed-layout PDF and application forms. Its `fill_blocks` +returns ordinary PDF bytes — an AcroForm with `/V` values written and +`NeedAppearances` set — with **no marker** distinguishing that output from a +form a human completed. It asked the question deliberately +(https://github.com/hyperpolymath/presswerk/issues/118[presswerk#118], from the +2026-09-27 ecosystem recon in +https://github.com/hyperpolymath/blocky-writer/pull/73[blocky-writer#73]). + +The owner answered it as **D189** (2026-09-30, `scope: repo`, basis +*applied-unless-struck*; surface +https://github.com/hyperpolymath/standards/issues/787#issuecomment-5913118060[standards#787]): + +[quote] +____ +Yes: record machine-filled provenance as metadata in the print path so +audit/routing can distinguish it. +____ + +A ruling that only one repository acts on cannot bind two. *How* the two sides +agree is a cross-repository matter — the marker a producer writes and the +evidence a print path validates — so the normative text lives here, and the +implementations live where the work happens: + +[cols="1,2,2",options="header"] +|=== +| Class | Who | What they owe + +| *Producer* +| `blocky-writer` (and any future form-filling producer) +| Writes the FFP marker (link:spec/MARKER.adoc[MARKER]) so its output is *declared*, not guessed. + +| *Print-path consumer* +| `presswerk` (and any future print router/batch tool) +| Classifies every document that enters the print path (link:spec/DETECTION.adoc[DETECTION]), and +*records* the result on the job and in the audit trail (link:spec/PRINT-PATH.adoc[PRINT-PATH]). + +| *Detector* +| any of the above, or a third party +| Implements link:spec/DETECTION.adoc[DETECTION] exactly; passes the +link:spec/conformance/README.adoc[conformance suite]. +|=== + +[IMPORTANT] +==== +This standard does **not** require a producer to write a marker before a +consumer may classify. It defines *both* legs: a declared marker that is +authoritative when present, and a structural signature that is honest when it +is not. A consumer that cannot see a marker MUST NOT invent one; see +<>. +==== + +== Scope + +*In scope:* PDF documents containing interactive form fields (AcroForm, +ISO 32000-1 §12.7), the provenance metadata a producer may attach to them, the +classification a print path derives from them, and the record it keeps. + +*Out of scope:* XFA-only forms (PDF 2.0 deprecates XFA; XFA provenance is +undefined here), digital signatures, document authorship or identity of the +person who filled a form, and any rendering decision (FFP classifies; it never +re-renders). + +== The spec set + +[cols="1,3",options="header"] +|=== +| Document | What it defines + +| link:spec/MARKER.adoc[MARKER] +| The `ffp:filledBy` XMP marker: namespace, properties, writer obligations, +forward-compatibility, and why XMP rather than `/Info`. + +| link:spec/DETECTION.adoc[DETECTION] +| The classification algorithm: fillable-field counting, value meaningfulness, +appearance detection, the classification lattice, and the evidence codes. + +| link:spec/PRINT-PATH.adoc[PRINT-PATH] +| The obligations of a print-path consumer: record, surface, never silently +normalise, and the blank-print hazard. + +| link:spec/conformance/README.adoc[CONFORMANCE] +| Fourteen vectors + a reference probe + a runner. A conforming detector MUST +pass this suite. +|=== + +== Definitions + +machine-filled:: + A document whose field values were written programmatically, without an + interactive editing session — the `blocky-writer` case. + +hand-completed:: + A document completed by a person on paper, after printing. A print path + usually sees this as a *blank form* (link:spec/DETECTION.adoc[DETECTION]): the values are added + later, by pen. + +interactively-filled:: + A document completed by a person in a PDF viewer/editor. FFP deliberately does + **not** classify this case as `hand` — see <>. + +declared provenance:: + What the document says about itself, via the FFP marker (link:spec/MARKER.adoc[MARKER]). + +inferred provenance:: + What the document's AcroForm structure shows, per the rules in + link:spec/DETECTION.adoc[DETECTION]. + +appearances:: + The widget appearance streams (`/AP` with a normal appearance) that determine + what a viewer or printer actually draws. A field can hold a value and still + print blank when its appearance is missing — this is the reason FFP is a + *print-path* concern, not merely a labelling one. + +== The two axes + +FFP never collapses "is there a form?" with "how was it filled?" into one guess. +It reports both, and derives one classification from them. + +*Axis 1 — form/fill state:* `absent` · `present`, with `filled_fields/total_fields`. +*Axis 2 — provenance evidence:* declared (marker) · inferred (structure) · none. + +=== The classification lattice + +[cols="1,2,3",options="header"] +|=== +| Classification | Set when | Routing meaning + +| `no-form` +| no interactive form, or no fillable terminal fields +| not a form; nothing to distinguish + +| `blank-form` +| form present, *zero* meaningful values +| the print-and-hand-complete case — print blank, fill by hand + +| `machine-filled` +| ≥1 meaningful value **and** the FFP marker declares `filledBy=machine` +| machine-filled, *declared* + +| `machine-filled-suspected` +| ≥1 meaningful value **and** no machine marker **and** +`NeedAppearances=true` **and** appearances incomplete +| machine-filled by structural signature only — treat conservatively, record the weaker evidence + +| `filled-unknown` +| ≥1 meaningful value, none of the above matched +| filled, provenance undetermined — never guess + +| `unreadable` +| the document structure could not be parsed +| unknown; the job MUST still record that FFP ran and could not decide +|=== + +`machine-filled-suspected` exists so a print path never has to choose between +lying and silence: the structural signature is strong evidence, and the +classification says exactly how strong. + +=== Precedence + +1. The declared marker wins over the structural signature when it is present + and recognised (`machine-filled`, even if appearances are complete and + `NeedAppearances` is absent). +2. An unrecognised declaration does not fall back to "trusting" the producer: + it is recorded as evidence and the structural rules decide. +3. `blank-form` is never promoted to `machine-filled*`: a form with zero + meaningful values is blank regardless of flags (a negative control for this + rule is vector `blank-form-need-appearances`). +4. If the document cannot be parsed, the classification is `unreadable`. It is + never `blank-form`. + +[[honesty]] +== The honesty rule (normative) + +A consumer MUST NOT report a provenance the evidence does not support. + +* FFP does not claim to detect "hand-filled in a viewer". `/AP` streams can be + generated by a program just as well as by an editor, so the honest + classification for that case is `filled-unknown`. +* `ffp:filledBy` is a *declaration*, not proof. Consumers MUST NOT present it as + verification, and MUST NOT infer authorship or identity from it: FFP records + **how** values were written, never **who** wrote them. +* A `machine-filled-suspected` result MUST be recorded with the evidence that + produced it, so a later reviewer can re-derive the classification. + +== Relationship to other standards + +* ISO 32000-1 §12.7 (interactive forms), §12.5.5 (`/AP` appearance streams), + §14.3.2 (metadata streams); ISO 16684-1 (XMP). `NeedAppearances` is + deprecated by ISO 32000-2 (PDF 2.0) and PDF/A requires appearances to be + present — which is *why* the structural signature in + link:spec/DETECTION.adoc[DETECTION] is a + reliable, if not sufficient, machine-fill signal. +* `docs/SEAMS-SPEC.adoc` — the blocky-writer → presswerk handoff is a declared + seam (`blocky-writer-presswerk-form-handoff`); the seam-register entry for + each side is in link:spec/PRINT-PATH.adoc[PRINT-PATH] §Seam. +* The precision-document suite's composition contract forbids any component + from silently normalising page geometry. FFP extends the same discipline to + content provenance: a print path may *report* that appearances are missing, + but it may not silently regenerate them. +* link:../../../docs/decisions/[ADR-004] (README discipline) is why this spec + is one README plus normative parts, and why the registry entry is a + `scripts/build-registry.sh` row rather than a hand-kept index. + +== Versioning + +This spec is versioned as a whole, `MAJOR.MINOR.PATCH`, and additionally carries +a *wire* version in the marker namespace (`…/1.0/`) and in the record +(`"ffp": "1.0"`). Adding a classification, an evidence code, or a marker +property is a MINOR change; repurposing an existing value is MAJOR. Consumers +MUST ignore unknown properties and MUST NOT fail a job because a document +declares a version they do not know. diff --git a/1-formats/sub-specs/form-fill-provenance/spec/DETECTION.adoc b/1-formats/sub-specs/form-fill-provenance/spec/DETECTION.adoc new file mode 100644 index 000000000..918a08508 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/DETECTION.adoc @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised) +// +// FFP spec part 2 — the detection algorithm. += FFP/2 — Detection and classification +:toc: preamble +:icons: font + +== Purpose + +This part defines exactly what a detector reads, what it may conclude, and what +it must record. The rules are written against ISO 32000-1 §12.7 (interactive +forms) and §12.5.5 (appearance streams). + +A detector is conforming iff, for every vector in +link:conformance/README.adoc[the conformance suite], it produces the expected +classification, counts, appearance state and evidence codes. + +== The algorithm + +=== Step 1 — parse + +Parse the document structure. If it cannot be parsed — not a PDF, truncated, +or encrypted and not decryptable by the consumer — stop with: + +* classification `unreadable` +* evidence `FFP-E-UNREADABLE` + +MUST NOT guess in this case, and MUST NOT classify an unparsed document as +`blank-form`. An encrypted document the consumer cannot open is also +`unreadable`; provenance detection is not a licence to attempt decryption. + +=== Step 2 — declared marker + +If the catalog has a `/Metadata` stream, parse it as XMP and look for the FFP +namespace (link:MARKER.adoc[MARKER]): + +* `filledBy=machine` present → evidence `FFP-E-DECL-MACHINE`; capture `tool`, + `toolVersion`, `filledAt`, `appearancesGenerated`. +* the FFP namespace present but `filledBy` absent or not `machine` → evidence + `FFP-E-DECL-UNRECOGNISED`; capture the value verbatim in the record. +* the namespace absent → no declaration evidence. +* `/Metadata` present but not parseable XML → evidence `FFP-E-XMP-UNREADABLE`; + continue *without* declaring anything. + +=== Step 3 — fillable fields + +Find `/AcroForm` in the catalog and walk its `/Fields` array recursively. + +An *inheritable* field attribute is resolved from the field dictionary or its +nearest ancestor (`/FT`, `/Ff`, `/V`, `/DA`, `/Opt`). + +A terminal field is a node with no `/Kids`, or whose `/Kids` are all widget +annotations (`/Subtype /Widget`). A node that is not an annotation and has +non-widget kids is an intermediate node: recurse. + +A terminal field is *fillable* iff its resolved `/FT` is one of: + +* `Tx` — text +* `Ch` — choice +* `Btn` — button, **excluding** pushbuttons (the pushbutton bit is set in the + resolved `/Ff` bit position 17, 1-based as defined by ISO 32000-1 Table 226) + +Signature (`Sig`) fields are never fillable. A document with no fillable +terminal field is `no-form` with evidence `FFP-E-NO-ACROFORM`. + +Read-only fields remain in scope: read-only constrains interactive editing, not +machine filling, and a read-only field with a value is still evidence. + +=== Step 4 — meaningful values + +`total_fields` is the count of fillable terminal fields. `filled_fields` is the +count whose resolved value is *meaningful*: + +[cols="1,3",options="header"] +|=== +| Field type | Meaningful value + +| `Tx` +| `/V` resolves to a string containing at least one non-whitespace character. +Whitespace is U+0020, U+0009, U+000D, U+000A. + +| `Ch` +| `/V` resolves to such a string, or to a non-empty array containing at least +one such string (multi-select). + +| `Btn` +| `/V` resolves to a name other than `Off`. A detector MAY additionally require +that the value matches an on-state (a non-`Off` key of `/AP /N`) of at least +one widget kid; if it does not match, the value is not meaningful. +|=== + +If `filled_fields` is zero, the form is blank; emit evidence `FFP-E-NO-VALUES`. +A value on a non-fillable field (pushbutton caption, signature) is ignored. + +=== Step 5 — appearance state + +For each filled terminal field, collect its widgets (the field dictionary +itself when it is a widget, otherwise its `/Subtype /Widget` kids). A filled +widget has a *normal appearance* iff: + +1. it has an `/AP` dictionary containing `/N`; and +2. either `/N` resolves to a stream, or `/N` resolves to a dictionary that + contains the widget's current state (`/AS`, or for `Btn` fields the resolved + `/V`) as a stream. + +Then: + +[cols="1,3",options="header"] +|=== +| `appearances` | When + +| `generated` +| every filled widget has a normal appearance +| `incomplete` +| at least one filled widget does not +| `not-applicable` +| `filled_fields` is zero +| `unknown` +| the structures cannot be resolved from the file +|=== + +`incomplete` emits evidence `FFP-E-AP-INCOMPLETE`. + +`NeedAppearances` (AcroForm `/NeedAppearances` = the boolean `true`) emits +evidence `FFP-E-NEED-APPEARANCES`. It is a document-level key, not inherited. + +=== Step 6 — classify + +Apply the lattice and precedence from the +link:../README.adoc[standard's front page] — marker first, structural signature +second, `blank-form` never promoted, `unreadable` never invented over: + +[cols="1,1",options="header"] +|=== +| Condition | Classification + +| parse failed | `unreadable` +| no fillable terminal field | `no-form` +| `filled_fields` = 0 | `blank-form` +| `FFP-E-DECL-MACHINE` | `machine-filled` +| `FFP-E-NEED-APPEARANCES` **and** `FFP-E-AP-INCOMPLETE` | `machine-filled-suspected` +| otherwise | `filled-unknown` +|=== + +== The record (normative shape) + +A consumer that records a classification MUST persist at least +`classification`; it SHOULD persist the full record. The record is JSON with +this shape (`declared` is `null` when no marker parsed): + +[source,json] +---- +{ + "ffp": "1.0", + "classification": "machine-filled", + "form": "present", + "filled_fields": 2, + "total_fields": 2, + "appearances": "incomplete", + "declared": { + "filledBy": "machine", + "tool": "blocky-writer", + "toolVersion": "0.2.0", + "filledAt": null, + "appearancesGenerated": false + }, + "evidence": ["FFP-E-DECL-MACHINE", "FFP-E-NEED-APPEARANCES", "FFP-E-AP-INCOMPLETE"] +} +---- + +Field notes: + +* `form` ∈ `present` | `absent`. +* `total_fields` counts fillable terminal fields; `0/0` accompanies `no-form`. +* `declared.filledBy` MUST be the verbatim value read from the packet when one + is present, including unrecognised values. +* `evidence` is a set (order-independent, duplicates removed). + +=== Evidence codes + +[cols="1,3",options="header"] +|=== +| Code | Meaning + +| `FFP-E-DECL-MACHINE` | marker present, `filledBy=machine` +| `FFP-E-DECL-UNRECOGNISED` | FFP namespace present, value absent/unknown +| `FFP-E-XMP-UNREADABLE` | `/Metadata` present, not parseable as XMP +| `FFP-E-NEED-APPEARANCES` | AcroForm `NeedAppearances` is `true` +| `FFP-E-AP-INCOMPLETE` | ≥1 filled widget has no normal appearance +| `FFP-E-NO-ACROFORM` | no fillable terminal field +| `FFP-E-NO-VALUES` | form present, zero meaningful values +| `FFP-E-UNREADABLE` | document structure could not be parsed +|=== + +Adding a code is a MINOR change; consumers MUST ignore codes they do not know. + +== Worked examples + +* *blocky-writer output today* — values written, `NeedAppearances=true`, no + `/AP` → `machine-filled-suspected`, evidence + `[FFP-E-NEED-APPEARANCES, FFP-E-AP-INCOMPLETE]`. +* *blocky-writer output with the marker and appearances generated* — as + recommended → `machine-filled`, `appearances=generated`. +* *A blank form downloaded from an agency, printed and filled in by hand* — + zero values → `blank-form`. This is the case the print path must keep + distinct from the one above. +* *A form completed in a desktop editor and saved* — values, `/AP` present, + `NeedAppearances` absent or false → `filled-unknown`, not `hand`. FFP does not + pretend the byte stream can tell you who typed. diff --git a/1-formats/sub-specs/form-fill-provenance/spec/MARKER.adoc b/1-formats/sub-specs/form-fill-provenance/spec/MARKER.adoc new file mode 100644 index 000000000..9e801c9ea --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/MARKER.adoc @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised) +// +// FFP spec part 1 — the declared marker. += FFP/1 — The declared marker +:toc: preamble +:icons: font + +== Purpose + +A producer that fills a form programmatically can *say so* in the document. +That declaration is the only provenance evidence a print path can trust without +inference, so it is the load-bearing half of this standard. + +== Namespace + +[cols="1,3",options="header"] +|=== +| Property | Value + +| Namespace URI | `pass:[https://hyperpolymath.dev/ns/form-fill-provenance/1.0/]` +| Preferred prefix | `ffp` +| Wire version | `1.0` (the trailing path segment) +| Carrier | an XMP packet in the document catalog's `/Metadata` stream +|=== + +The URI is an XML namespace name: it identifies the vocabulary and need not +resolve to a document. A future incompatible revision changes the trailing +segment; the URI is therefore the wire version. + +== Properties + +[cols="1,1,1,3",options="header"] +|=== +| Property | Type | Presence | Meaning + +| `ffp:filledBy` +| string, value `machine` +| MUST, when the marker is present +| The values in this document were written programmatically. `machine` is the +*only* value defined in 1.0. Producers MUST NOT write any other value — see +<>. + +| `ffp:tool` +| string +| SHOULD +| Producing tool identifier, e.g. `blocky-writer`. + +| `ffp:toolVersion` +| string +| MAY +| Producing tool version, e.g. `0.2.0`. + +| `ffp:filledAt` +| string, ISO 8601 date-time with `Z` or an offset +| MAY +| When the values were written. This is *not* a privacy-neutral field; see +<>. + +| `ffp:appearancesGenerated` +| boolean +| SHOULD +| `true` iff the producer generated the `/AP` normal appearance streams for +every filled widget. `false` preserves the print-path hazard described in +link:PRINT-PATH.adoc[PRINT-PATH]. +|=== + +=== The packet + +XMP permits properties as attributes or as child elements; both are conforming +and a detector MUST accept both. The attribute form: + +[source,xml] +---- + + + + + + + +---- + +The `` wrapper is customary, not required: a detector MUST find the +properties whether or not it is present, and MUST NOT depend on the byte-order +mark (U+FEFF) that generators conventionally place inside `begin`. The vectors +in link:conformance/README.adoc[the conformance suite] omit it. + +[[no-human]] +== Why `filledBy` has exactly one value + +FFP defines `human` as *not a value of this vocabulary*. A producer cannot know +that a human filled a form: an interactive editor knows only that *it* wrote the +values, not who typed them, and a printer cannot observe the pen. Offering +`filledBy=human` would invite a claim the producer cannot support and that a +consumer would then repeat as fact. + +The absence of a marker is what hand-completion looks like from the print path +(`blank-form`, or `filled-unknown` if values are nevertheless present). Absence +is honest; a false declaration is not. + +A consumer that reads a marker whose `filledBy` it does not recognise MUST +record it as unrecognised evidence (link:DETECTION.adoc[`FFP-E-DECL-UNRECOGNISED`]) +and let the structural rules decide. It MUST NOT treat an unknown declaration as +`machine`. + +[[privacy]] +== Privacy and retention + +The marker describes the *document*, not a person: it is an authoring-tool +declaration, not an identity. Consumers MUST NOT infer or record authorship, +identity, or location from it. `ffp:filledAt` and `ffp:tool` exist for audit +*of the pipeline*; a consumer that persists them MUST obey the same retention +policy as the document itself, and a producer SHOULD omit `ffp:filledAt` when +the field's own contents are timestamp-bearing anyway. + +== Writer obligations + +A conforming producer: + +1. MUST write `ffp:filledBy=machine` when it wrote any field value. +2. MUST place the packet in the catalog's `/Metadata` stream (not in `/Info`; + ISO 32000-2 deprecates the Info dictionary, and XMP is the designed + extension point — ISO 32000-1 §14.3.2, ISO 16684-1). +3. MUST NOT remove or overwrite an existing XMP packet. Where the packet + already exists, the new `rdf:Description` is added to the existing `rdf:RDF` + (or the existing description gains the `ffp:` properties). +4. MUST set `ffp:appearancesGenerated` truthfully when it writes it. A producer + that writes `/V` without generating `/AP`, relying on `NeedAppearances`, MUST + write `false` (or omit the property, which is recorded as *unknown*). +5. SHOULD generate appearances. `NeedAppearances` is deprecated in PDF 2.0, + PDF/A requires appearances to be present, and numerous viewers and print + rasterisers ignore the flag and print blank fields. Generating appearances is + the correctness fix for the producer side; the marker then declares + `appearancesGenerated=true` and the print path no longer has to warn. +6. MUST NOT set `ffp:filledBy=machine` on a document it did not fill (a + pass-through converter, a merger, a print router copying bytes). The marker + is a record of authorship of the *values*, and pass-through components MUST + preserve it verbatim if present. + +== Producer note: what the marker does not replace + +The marker does not describe *which* fields were filled, does not replace the +`/V` entries, and does not assert the document is valid, printable, or +complete. It records one fact — the values were machine-written — and the +appearances flag. + +== Detector note + +A detector MUST look for the property in the FFP namespace, not for the literal +string `filledBy` anywhere in the packet (an unrelated vocabulary may carry that +name). Namespace-prefix spelling is not normative: `ffp:` is a convention, and +any prefix bound to the FFP namespace URI is equivalent. diff --git a/1-formats/sub-specs/form-fill-provenance/spec/PRINT-PATH.adoc b/1-formats/sub-specs/form-fill-provenance/spec/PRINT-PATH.adoc new file mode 100644 index 000000000..6f7cb3aca --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/PRINT-PATH.adoc @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised) +// +// FFP spec part 3 — obligations on the print path. += FFP/3 — The print path +:toc: preamble +:icons: font + +== Scope of this part + +This part binds a *print-path consumer*: any component that accepts a document +for printing, routing or job management — the `presswerk` class of software. +The obligations are written so that they are implementable in a single intake +function and verifiable from the job record. + +== Why the print path + +A filled application form is usually a print job before it is anything else. +The print path is where the distinction has consequences: + +* *Routing* — a blank form going to a printer to be filled by hand and a + machine-filled form going to a printer to be filed are different jobs with + different handling, and today they are byte-indistinguishable at the queue. +* *Audit* — the audit trail is the record of what was printed, when, and from + what. "Arrived already-filled by a machine" is a fact about the document that + the trail cannot reconstruct afterwards unless it was captured at intake. +* *Hazard* — a document whose filled fields have no appearance streams can + print *blank* on viewers and print rasterisers that ignore + `NeedAppearances`; see <>. + +== Obligations + +[[P1]] +=== P1 — classify at intake (MUST) + +Every document entering the print path MUST be classified per +link:DETECTION.adoc[DETECTION] before the job is submitted to a printer, and +the classification MUST be attached to that job. A print path MAY implement the +detector itself or call a shared one; it MUST NOT skip classification because +"it is probably not a form". + +Network-received jobs (e.g. an embedded IPP print server) are in scope: the +classification is performed on the received payload, at receipt. + +[[P2]] +=== P2 — record (MUST) + +The full FFP record (link:DETECTION.adoc[DETECTION] §The record) MUST be +persisted with the job, alongside the document hash, so a later reader can +distinguish the two cases *without re-parsing the document*. At least: + +* the job's own persistent metadata (the queue/record the job lives in), and +* the audit trail entry for the job's submission. + +`machine-filled` and `machine-filled-suspected` MUST be recorded as distinct +values, and the evidence list MUST be preserved with them. A record that says +only "form: yes" fails this obligation. + +[[P3]] +=== P3 — never silently normalise (MUST) + +Classification MUST NOT change the document bytes on their way to the printer, +and a print path MUST NOT silently generate appearances, rewrite `/V`, or strip +`NeedAppearances` in order to make a job printable. This restates, for +provenance, the suite's existing clause that no component may silently +normalise the document. + +Any repair is an explicit, user-visible operation performed before submission — +never an invisible side effect of intake. + +[[P4]] +=== P4 — surface the distinction (MUST) + +Operator-visible surfaces (job list, job detail, audit view) MUST show the +classification such that `blank-form`, `machine-filled` and +`machine-filled-suspected` are distinguishable without opening the document. + +If the print path consults a routing policy, the classification MUST be +available to that policy as a job attribute. Its *representation* is the +consumer's choice; its *presence* is not. + +[[hazard]] +=== P5 — the blank-print hazard + +If `appearances` is `incomplete` and `filled_fields` is non-zero, the document +may print with empty fields. The print path: + +* MUST record the hazard on the job (evidence `FFP-E-AP-INCOMPLETE` is + sufficient); +* MUST surface it in the job's operator-visible detail; and +* SHOULD require explicit confirmation before submitting such a job + interactively, or hold it for review in an unattended route. + +A print path MUST NOT present such a document as only `filled-unknown` when +`FFP-E-AP-INCOMPLETE` is present: the hazard is about appearances, not about +authorship, and it is actionable in both `machine-filled` and +`machine-filled-suspected`. + +=== P6 — no guessing, no authorship + +`unreadable` is a conforming outcome and MUST be recorded as such rather than +defaulted to `blank-form`. The record MUST NOT contain, and no surface may +display, an inference about *who* filled the form; FFP is about the pipeline, +not the person (link:MARKER.adoc[MARKER] §Privacy). + +=== P7 — retention + +The FFP record is metadata about the document and MUST follow the document's own +retention: deleting the job deletes the record, and exporting/archiving a job +carries its classification with it. It is not a separate personal-data store. + +== Seam declaration + +The handoff this standard governs is a declared seam under +`docs/SEAMS-SPEC.adoc`. Each side records it in its own +`.machine_readable/contractiles/trust/Trustfile.a2ml` `[SEAMS]` section: + +[source,yaml] +---- +seams: + - id: "blocky-writer-presswerk-form-handoff" + description: "A filled AcroForm PDF crosses from a form filler to a print router; FFP classification crosses back as job metadata" + from: "hyperpolymath/blocky-writer (fill_blocks output)" + to: "hyperpolymath/presswerk (print-path intake)" + contract: "1-formats/sub-specs/form-fill-provenance/ (this standard, v1.0.0)" + test_ref: "1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh" + failure_mode: "fail-degrade" + tier: "external-trust" + notes: "The producer side is optional in time: a consumer classifies structurally until producers emit the marker." +---- + +`failure_mode` is `fail-degrade` deliberately: a document that does not parse +must not block printing, and a document without a marker must not be treated as +unfilled. `tier` is `external-trust` because neither side is proven against the +other at compile time; the shared ground truth is the conformance suite, which +both sides run. + +== Conformance for this part + +A print-path consumer conforms to FFP/3 when: + +1. the conformance suite passes for its detector (FFP/2); +2. a test demonstrates the classification is attached to a submitted job and + survives a queue round-trip; and +3. a test demonstrates an `unreadable` document is recorded as `unreadable`, + not as `blank-form`. diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/README.adoc b/1-formats/sub-specs/form-fill-provenance/spec/conformance/README.adoc new file mode 100644 index 000000000..852c22a10 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/README.adoc @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised) += FFP Conformance Vectors +:icons: font + +The suite is the shared ground truth between the two sides of the seam: a +producer's output and a print path's classification. A detector conforms iff it +reproduces every line in `expected/` for the corresponding PDF in `vectors/`. + +== Layout + +[cols="1,3",options="header"] +|=== +| Path | What it is + +| `vectors/*.pdf` +| 14 minimal, uncompressed, ASCII PDFs. Committed; regenerated byte-identically +by `make-fixtures.sh`. +| `expected/*.expected` +| One canonical line per vector: the classification, form state, field counts, +appearance state and evidence codes a conforming detector MUST produce. +| `probe.awk` +| The reference detector, in awk. +| `run-conformance.sh` +| Runs a detector over every vector, diffs against `expected/`, and verifies the +vectors still match the generator. +|=== + +== Running + +[source,bash] +---- +# reference detector +bash run-conformance.sh + +# a product detector (anything that takes a PDF path and prints the line) +FFP_DETECTOR="/path/to/ffp-classify" bash run-conformance.sh +---- + +The scorecard's check for this spec runs the first form; a product detector is +expected to run the second form in its own CI. + +== The canonical line + +---- +classification= form= filled=/ appearances= evidence= +---- + +Field meanings and the allowed values are defined in +link:../DETECTION.adoc[DETECTION]. `evidence` is a comma-separated, sorted set; +an empty set is written as `evidence=`. + +The line is deliberately flat so that a shell runner can diff it without a +parser, in the estate's no-Python, no-JS idiom. + +== What the reference probe is, and is not + +*Is:* an executable statement of the DETECTION rules over the syntax the vectors +use — indirect objects, dictionaries, arrays, literal strings, names, booleans, +`NeedAppearances`, `/AP` normal appearances (stream or state dictionary), and +the FFP XMP property in attribute or element form. + +*Is not:* a general PDF parser. It does not decompress object or content +streams, does not follow cross-reference streams, does not decrypt, does not +resolve references other than `N 0 R`, and handles one level of `Kids` +inheritance shape. It MUST NOT be shipped as a detector. + +A product detector MUST use a real PDF library. It MUST still pass this suite — +that is the point of the vectors: they fix the *semantics*, not the +implementation. + +== The vectors + +[cols="1,2,3",options="header"] +|=== +| Vector | Shape | Expected + +| `no-form` +| a page with no AcroForm +| `no-form` — not every PDF is a form + +| `blank-form` +| 2 text fields, no `/V` +| `blank-form` — the print-and-hand-complete case + +| `blank-form-need-appearances` +| as above, `NeedAppearances=true` +| `blank-form` — the negative control: a flag is not a value + +| `blank-form-empty-values` +| `/V ()` and `/V ( )` +| `blank-form` — empty and whitespace-only values are not values + +| `machine-filled-suspected` +| 2 values, `NeedAppearances=true`, no `/AP` +| `machine-filled-suspected` — the structural signature, which is exactly +`blocky-writer`'s output today + +| `machine-filled-declared` +| as above plus the FFP marker +| `machine-filled` — declaration wins over inference + +| `machine-filled-declared-generated` +| marker, values, `/AP` present, no `NeedAppearances` +| `machine-filled` — the declaration is not downgraded by good appearances + +| `machine-filled-suspected-partial` +| 1 of 2 fields filled, `NeedAppearances=true`, no `/AP` +| `filled=1/2`, `machine-filled-suspected` + +| `machine-filled-suspected-mixed-ap` +| 2 filled, `/AP` on one only +| `appearances=incomplete`, `machine-filled-suspected` + +| `viewer-filled` +| 2 filled, `/AP` on both, no `NeedAppearances` +| `filled-unknown` — the honesty control: FFP does not claim to detect a human + +| `filled-with-ap-need-appearances` +| as above with `NeedAppearances=true` +| `filled-unknown` — incomplete appearances are required for the inference + +| `declared-unknown-value` +| marker `filledBy=robot` +| `machine-filled-suspected` + `FFP-E-DECL-UNRECOGNISED` — an unknown +declaration is recorded, never trusted + +| `button-machine-filled` +| a radio group with two state appearances + an empty text field +| `filled=1/2`, `appearances=generated` — buttons count; `/FT` and `/V` are read +from the field, appearances from each widget kid + +| `unreadable` +| `%PDF-` header, no usable structure +| `unreadable` — never `blank-form` +|=== + +== Vector validity + +Every indirect object in every vector holds exactly one well-formed object: a +key such as `/AP` lives *inside* its dictionary, never as a second dictionary +appended after the closing `>>`. The reference probe would tolerate the +appended spelling — it matches text — but a real PDF parser cannot see it, and +a vector written that way would be unpassable by exactly the product detectors +this suite exists to test. `make-fixtures.sh` enforces the spelling by +construction (`field_tx_ap`); there is no automated check for it yet, so keep it +in mind when adding vectors. + +== Determinism + +`make-fixtures.sh` writes no timestamps and no random data; the XMP packet is a +fixed literal. `make-fixtures.sh --check` rebuilds into a temporary directory +and diffs against `vectors/`, so a hand-edit to a fixture fails the suite rather +than silently changing the ground truth. diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-empty-values.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-empty-values.expected new file mode 100644 index 000000000..e4cdd5ef4 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-empty-values.expected @@ -0,0 +1 @@ +classification=blank-form form=present filled=0/2 appearances=not-applicable evidence=FFP-E-NO-VALUES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-need-appearances.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-need-appearances.expected new file mode 100644 index 000000000..b8378d0aa --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form-need-appearances.expected @@ -0,0 +1 @@ +classification=blank-form form=present filled=0/2 appearances=not-applicable evidence=FFP-E-NEED-APPEARANCES,FFP-E-NO-VALUES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form.expected new file mode 100644 index 000000000..e4cdd5ef4 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/blank-form.expected @@ -0,0 +1 @@ +classification=blank-form form=present filled=0/2 appearances=not-applicable evidence=FFP-E-NO-VALUES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/button-machine-filled.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/button-machine-filled.expected new file mode 100644 index 000000000..7a307059b --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/button-machine-filled.expected @@ -0,0 +1 @@ +classification=filled-unknown form=present filled=1/2 appearances=generated evidence=FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/declared-unknown-value.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/declared-unknown-value.expected new file mode 100644 index 000000000..a68ac8cc4 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/declared-unknown-value.expected @@ -0,0 +1 @@ +classification=machine-filled-suspected form=present filled=2/2 appearances=incomplete evidence=FFP-E-AP-INCOMPLETE,FFP-E-DECL-UNRECOGNISED,FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/filled-with-ap-need-appearances.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/filled-with-ap-need-appearances.expected new file mode 100644 index 000000000..78c66fa1a --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/filled-with-ap-need-appearances.expected @@ -0,0 +1 @@ +classification=filled-unknown form=present filled=2/2 appearances=generated evidence=FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared-generated.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared-generated.expected new file mode 100644 index 000000000..616b9e910 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared-generated.expected @@ -0,0 +1 @@ +classification=machine-filled form=present filled=2/2 appearances=generated evidence=FFP-E-DECL-MACHINE diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared.expected new file mode 100644 index 000000000..c3854e21d --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-declared.expected @@ -0,0 +1 @@ +classification=machine-filled form=present filled=2/2 appearances=incomplete evidence=FFP-E-AP-INCOMPLETE,FFP-E-DECL-MACHINE,FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-mixed-ap.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-mixed-ap.expected new file mode 100644 index 000000000..30b45c924 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-mixed-ap.expected @@ -0,0 +1 @@ +classification=machine-filled-suspected form=present filled=2/2 appearances=incomplete evidence=FFP-E-AP-INCOMPLETE,FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-partial.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-partial.expected new file mode 100644 index 000000000..3ca83dc6a --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected-partial.expected @@ -0,0 +1 @@ +classification=machine-filled-suspected form=present filled=1/2 appearances=incomplete evidence=FFP-E-AP-INCOMPLETE,FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected.expected new file mode 100644 index 000000000..30b45c924 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/machine-filled-suspected.expected @@ -0,0 +1 @@ +classification=machine-filled-suspected form=present filled=2/2 appearances=incomplete evidence=FFP-E-AP-INCOMPLETE,FFP-E-NEED-APPEARANCES diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/no-form.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/no-form.expected new file mode 100644 index 000000000..6350517db --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/no-form.expected @@ -0,0 +1 @@ +classification=no-form form=absent filled=0/0 appearances=not-applicable evidence=FFP-E-NO-ACROFORM diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/unreadable.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/unreadable.expected new file mode 100644 index 000000000..297ff9bc9 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/unreadable.expected @@ -0,0 +1 @@ +classification=unreadable form=unknown filled=0/0 appearances=unknown evidence=FFP-E-UNREADABLE diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/viewer-filled.expected b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/viewer-filled.expected new file mode 100644 index 000000000..728e96247 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/expected/viewer-filled.expected @@ -0,0 +1 @@ +classification=filled-unknown form=present filled=2/2 appearances=generated evidence= diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/make-fixtures.sh b/1-formats/sub-specs/form-fill-provenance/spec/conformance/make-fixtures.sh new file mode 100755 index 000000000..cb3bef647 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/make-fixtures.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# make-fixtures.sh — deterministically build the FFP conformance vectors. +# +# The fixtures are deliberately minimal, uncompressed, ASCII-only PDFs with a +# classic cross-reference table, so that (a) they are readable in a terminal and +# in review, and (b) the reference probe (a small awk reader, not a real PDF +# parser) can classify them without a PDF library. +# +# Usage: +# bash make-fixtures.sh write vectors/*.pdf +# bash make-fixtures.sh --check rebuild into a temp dir and diff against the +# committed vectors; non-zero on drift +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VECTORS="$HERE/vectors" +MODE="write" +[ "${1:-}" = "--check" ] && MODE="check" + +OUTDIR="$VECTORS" +TMP="" +if [ "$MODE" = "check" ]; then + TMP="$(mktemp -d)"; OUTDIR="$TMP/vectors"; mkdir -p "$OUTDIR" + trap 'rm -rf "$TMP"' EXIT +else + mkdir -p "$OUTDIR" +fi + +# --------------------------------------------------------------------------- +# Object assembly. Objects are written sequentially; offsets are the byte +# position of each object header, which is what the xref table records. +# --------------------------------------------------------------------------- +declare -A OFF +MAXID=0 +CUR="" + +new_doc() { + CUR="$1" + OFF=(); MAXID=0 + printf '%%PDF-1.7\n' > "$CUR" +} + +obj() { # obj + local id="$1" body="$2" + OFF[$id]=$(wc -c < "$CUR" | tr -d ' ') + printf '%s 0 obj\n%s\nendobj\n' "$id" "$body" >> "$CUR" + [ "$id" -gt "$MAXID" ] && MAXID="$id" +} + +stream_obj() { # stream_obj + local id="$1" dict="$2" content="$3" + local len=${#content} + obj "$id" "$dict /Length $len >> +stream +$content +endstream" +} + +finish_doc() { # finish_doc + local root="$1" i + local xref_off + xref_off=$(wc -c < "$CUR" | tr -d ' ') + { + printf 'xref\n0 %d\n' $((MAXID + 1)) + printf '0000000000 65535 f \n' + for ((i = 1; i <= MAXID; i++)); do + if [ -n "${OFF[$i]:-}" ]; then + printf '%010d 00000 n \n' "${OFF[$i]}" + else + printf '0000000000 65535 f \n' + fi + done + printf 'trailer\n<< /Size %d /Root %d 0 R >>\nstartxref\n%s\n%%%%EOF\n' \ + $((MAXID + 1)) "$root" "$xref_off" + } >> "$CUR" +} + +# --------------------------------------------------------------------------- +# Shared fragments. +# --------------------------------------------------------------------------- +PAGE="<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Resources << /Font << /Helv 8 0 R >> >>" +FONT='<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>' +FORM_STREAM='BT /Helv 10 Tf 0 0 Td (Jewell) Tj ET' + +# field_tx (value appears as /V only when non-empty) +field_tx() { + local name="$1" value="$2" + local body="<< /Type /Annot /Subtype /Widget /FT /Tx /T ($name) /Rect [50 700 300 730] /P 3 0 R /DA (/Helv 10 Tf 0 g)" + [ -n "$value" ] && body="$body /V ($value)" + printf '%s >>' "$body" +} + +# field_tx_ap +# As field_tx, but with a normal appearance. /AP is a key *inside* the widget +# dictionary: an indirect object holds exactly one object, so appending a second +# dictionary after the widget's closing `>>` would be malformed PDF. The +# reference probe tolerates that spelling because it matches text, but a real +# parser (and therefore any conforming product detector) cannot. +field_tx_ap() { + local name="$1" value="$2" ap="$3" + local body; body="$(field_tx "$name" "$value")" + printf '%s /AP << /N %s 0 R >> >>' "${body% >>}" "$ap" +} + +# xmp +# filledBy-value may be empty (namespace present, no property). +xmp_packet() { + local filled_by="$1" tool="$2" apgen="$3" + local props="" + [ -n "$filled_by" ] && props="$props ffp:filledBy=\"$filled_by\"" + [ -n "$tool" ] && props="$props ffp:tool=\"$tool\"" + [ -n "$apgen" ] && props="$props ffp:appearancesGenerated=\"$apgen\"" + printf '%s' " + + + + + +" +} + +# --------------------------------------------------------------------------- +# The vectors. Each builder writes one fixture; add a row to VECTORS below. +# --------------------------------------------------------------------------- +v_no_form() { + obj 1 '<< /Type /Catalog /Pages 2 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE >>" + obj 8 "$FONT" + finish_doc 1 +} + +v_blank_form() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname '')" + obj 6 "$(field_tx given '')" + obj 8 "$FONT" + finish_doc 1 +} + +v_blank_form_need_appearances() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname '')" + obj 6 "$(field_tx given '')" + obj 8 "$FONT" + finish_doc 1 +} + +v_blank_form_empty_values() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>' + obj 5 '<< /Type /Annot /Subtype /Widget /FT /Tx /T (surname) /Rect [50 700 300 730] /P 3 0 R /V () >>' + obj 6 '<< /Type /Annot /Subtype /Widget /FT /Tx /T (given) /Rect [50 650 300 680] /P 3 0 R /V ( ) >>' + obj 8 "$FONT" + finish_doc 1 +} + +v_machine_filled_suspected() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname Jewell)" + obj 6 "$(field_tx given Jonathan)" + obj 8 "$FONT" + finish_doc 1 +} + +v_machine_filled_declared() { + local packet; packet="$(xmp_packet machine blocky-writer false)" + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R /Metadata 9 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname Jewell)" + obj 6 "$(field_tx given Jonathan)" + obj 8 "$FONT" + stream_obj 9 '<< /Type /Metadata /Subtype /XML' "$packet" + finish_doc 1 +} + +v_machine_filled_declared_generated() { + local packet; packet="$(xmp_packet machine blocky-writer true)" + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R /Metadata 9 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx_ap surname Jewell 10)" + obj 6 "$(field_tx_ap given Jonathan 11)" + obj 8 "$FONT" + stream_obj 9 '<< /Type /Metadata /Subtype /XML' "$packet" + stream_obj 10 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + stream_obj 11 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + finish_doc 1 +} + +v_machine_filled_suspected_partial() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname Jewell)" + obj 6 "$(field_tx given '')" + obj 8 "$FONT" + finish_doc 1 +} + +v_machine_filled_suspected_mixed_ap() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx_ap surname Jewell 10)" + obj 6 "$(field_tx given Jonathan)" + obj 8 "$FONT" + stream_obj 10 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + finish_doc 1 +} + +v_viewer_filled() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx_ap surname Jewell 10)" + obj 6 "$(field_tx_ap given Jonathan 11)" + obj 8 "$FONT" + stream_obj 10 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + stream_obj 11 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + finish_doc 1 +} + +v_filled_with_ap_need_appearances() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx_ap surname Jewell 10)" + obj 6 "$(field_tx_ap given Jonathan 11)" + obj 8 "$FONT" + stream_obj 10 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + stream_obj 11 '<< /Type /XObject /Subtype /Form /BBox [0 0 250 30]' "$FORM_STREAM" + finish_doc 1 +} + +v_declared_unknown_value() { + local packet; packet="$(xmp_packet robot blocky-writer false)" + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R /Metadata 9 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [5 0 R 6 0 R] >>" + obj 4 '<< /Fields [5 0 R 6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + obj 5 "$(field_tx surname Jewell)" + obj 6 "$(field_tx given Jonathan)" + obj 8 "$FONT" + stream_obj 9 '<< /Type /Metadata /Subtype /XML' "$packet" + finish_doc 1 +} + +v_button_machine_filled() { + obj 1 '<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>' + obj 2 '<< /Type /Pages /Kids [3 0 R] /Count 1 >>' + obj 3 "$PAGE /Annots [6 0 R 7 0 R 9 0 R] >>" + obj 4 '<< /Fields [5 0 R 9 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>' + # 5 is a button field with two widget kids; /FT and /V live on the parent, + # and both kids carry the state appearances. 9 is an empty text field. + obj 5 '<< /FT /Btn /Ff 32768 /T (sex) /V /Yes /Kids [6 0 R 7 0 R] >>' + obj 6 '<< /Type /Annot /Subtype /Widget /Parent 5 0 R /Rect [50 600 70 620] /P 3 0 R /AS /Yes /AP << /N << /Off 10 0 R /Yes 11 0 R >> >> >>' + obj 7 '<< /Type /Annot /Subtype /Widget /Parent 5 0 R /Rect [80 600 100 620] /P 3 0 R /AS /Off /AP << /N << /Off 10 0 R /Yes 11 0 R >> >> >>' + obj 8 "$FONT" + obj 9 "$(field_tx surname '')" + stream_obj 10 '<< /Type /XObject /Subtype /Form /BBox [0 0 20 20]' 'q Q' + stream_obj 11 '<< /Type /XObject /Subtype /Form /BBox [0 0 20 20]' 'q Q' + finish_doc 1 +} + +v_unreadable() { + CUR="$1"; MAXID=0; OFF=() + printf '%%PDF-1.7\nthis is not a structurally valid PDF: no xref, no trailer\n' > "$CUR" +} + +# name → builder function +declare -A BUILDERS=( + [no-form]=v_no_form + [blank-form]=v_blank_form + [blank-form-need-appearances]=v_blank_form_need_appearances + [blank-form-empty-values]=v_blank_form_empty_values + [machine-filled-suspected]=v_machine_filled_suspected + [machine-filled-declared]=v_machine_filled_declared + [machine-filled-declared-generated]=v_machine_filled_declared_generated + [machine-filled-suspected-partial]=v_machine_filled_suspected_partial + [machine-filled-suspected-mixed-ap]=v_machine_filled_suspected_mixed_ap + [viewer-filled]=v_viewer_filled + [filled-with-ap-need-appearances]=v_filled_with_ap_need_appearances + [declared-unknown-value]=v_declared_unknown_value + [button-machine-filled]=v_button_machine_filled + [unreadable]=v_unreadable +) + +for name in "${!BUILDERS[@]}"; do + new_doc "$OUTDIR/$name.pdf" + "${BUILDERS[$name]}" "$OUTDIR/$name.pdf" +done + +if [ "$MODE" = "check" ]; then + if diff -rq "$VECTORS" "$OUTDIR" >/dev/null 2>&1 \ + || diff -rq "$VECTORS" "$OUTDIR"; then + echo "make-fixtures --check: committed vectors match the generator" + exit 0 + fi + echo "make-fixtures --check: DRIFT — regenerate with: bash make-fixtures.sh" >&2 + exit 1 +fi + +echo "wrote $(ls "$OUTDIR" | wc -l | tr -d ' ') fixtures to ${OUTDIR#"$HERE"/}" diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/probe.awk b/1-formats/sub-specs/form-fill-provenance/spec/conformance/probe.awk new file mode 100644 index 000000000..56bad2531 --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/probe.awk @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# probe.awk — the FFP reference probe. +# +# WHAT THIS IS +# An executable statement of the DETECTION rules, over the small, uncompressed +# PDF syntax the conformance vectors are written in. It is the oracle the +# vectors are checked against, and the thing a production detector (e.g. +# presswerk's Rust implementation) is compared to. +# +# WHAT THIS IS NOT +# A general PDF parser. It does not decode streams, follow object streams or +# cross-reference streams, decrypt, or accept every legal spelling of every +# construct. Its limits are named in spec/conformance/README.adoc. A +# production detector MUST use a real PDF library; it MUST still produce this +# line for every vector in vectors/. +# +# OUTPUT (one line, stable order) +# classification= form= filled=/ appearances= evidence= +# +# FFP_PROBE_RECORD=1 additionally prints the DETECTION record as JSON. + +{ buf = buf $0 "\n" } + +END { + n = length(buf) + ok = 1 + if (buf !~ /%PDF-/) ok = 0 + if (ok) { + index_objects() + root = trailer_root() + if (root == "" || !(root in OBJ)) ok = 0 + } + if (!ok) { die_unreadable() } else { + catalog = OBJ[root] + + decl = ""; tool = ""; toolver = "" + + # --- Step 2: declared marker ------------------------------------------- + meta = dget(catalog, "Metadata") + if (meta != "") { + payload = stream_payload(deref_text(meta)) + if (payload == "") { + ev["FFP-E-XMP-UNREADABLE"] = 1 + } else if (payload ~ /form-fill-provenance\/1\.0/) { + val = xmp_value(payload, "filledBy") + if (val == "machine") { ev["FFP-E-DECL-MACHINE"] = 1; decl = "machine" } + else { ev["FFP-E-DECL-UNRECOGNISED"] = 1; decl = val } + tool = xmp_value(payload, "tool") + toolver = xmp_value(payload, "toolVersion") + } + } + + # --- Step 3: fillable fields ------------------------------------------- + form = "absent"; total = 0; filled = 0; apmissing = 0 + acro = dget(catalog, "AcroForm") + if (acro == "") { + ev["FFP-E-NO-ACROFORM"] = 1 + } else { + adict = deref_text(acro) + if (adict == "") { + ev["FFP-E-NO-ACROFORM"] = 1 + } else { + if (dget(adict, "NeedAppearances") == "true") ev["FFP-E-NEED-APPEARANCES"] = 1 + fields = dget(adict, "Fields") + ids = array_ids(fields) + cnt = split_ids(ids, fids) + for (i = 1; i <= cnt; i++) walk_field(fids[i], "", "", "", 0) + if (total > 0) form = "present" + if (total == 0) ev["FFP-E-NO-ACROFORM"] = 1 + } + } + + # --- Steps 5/6: appearance state and classification --------------------- + if (form == "absent" || filled == 0) { + appr = "not-applicable" + if (form == "present") ev["FFP-E-NO-VALUES"] = 1 + } else if (apmissing > 0) { + appr = "incomplete"; ev["FFP-E-AP-INCOMPLETE"] = 1 + } else { + appr = "generated" + } + + if (form == "absent") cls = "no-form" + else if (filled == 0) cls = "blank-form" + else if (decl == "machine") cls = "machine-filled" + else if (("FFP-E-NEED-APPEARANCES" in ev) && ("FFP-E-AP-INCOMPLETE" in ev)) cls = "machine-filled-suspected" + else cls = "filled-unknown" + + printf "classification=%s form=%s filled=%d/%d appearances=%s evidence=%s\n", \ + cls, form, filled, total, appr, evcodes() + if (ENVIRON["FFP_PROBE_RECORD"] == "1") + print record_json(cls, form, appr, decl, tool, toolver) + } +} + +# --------------------------------------------------------------------------- +# Object index and reference resolution +# --------------------------------------------------------------------------- +function index_objects( pos, m, s, e, id, body, rest) { + pos = 1 + while (pos <= n) { + rest = substr(buf, pos) + if (!match(rest, /[0-9]+[ \t\r\n]+0[ \t\r\n]+obj/)) break + m = pos + RSTART - 1 + id = substr(rest, RSTART); sub(/[ \t\r\n].*$/, "", id) + s = m + RLENGTH + e = index(substr(buf, s), "endobj") + if (e == 0) e = n - s + 1 + body = substr(buf, s, e - 1) + gsub(/^[ \t\r\n]+/, "", body); gsub(/[ \t\r\n]+$/, "", body) + OBJ[id] = body + pos = s + e + 5 + } +} + +function trailer_root( s) { + if (!match(buf, /\/Root[ \t\r\n]+[0-9]+/)) return "" + s = substr(buf, RSTART, RLENGTH) + sub(/^\/Root[ \t\r\n]+/, "", s) + return s +} + +function deref_text(v, parts, id) { + v = trim(v) + if (v ~ /^[0-9]+[ \t\r\n]+0[ \t\r\n]+R$/) { + split(v, parts, /[ \t\r\n]+/) + id = parts[1] + return (id in OBJ) ? OBJ[id] : "" + } + return v +} + +function stream_payload(t, s, e) { + if (t !~ /stream/) return "" + s = index(t, "stream") + 6 + if (substr(t, s, 1) == "\r") s++ + if (substr(t, s, 1) == "\n") s++ + e = index(substr(t, s), "endstream") + if (e == 0) return "" + return substr(t, s, e - 1) +} + +# --------------------------------------------------------------------------- +# Minimal object grammar +# --------------------------------------------------------------------------- +function trim(s) { gsub(/^[ \t\r\n]+/, "", s); gsub(/[ \t\r\n]+$/, "", s); return s } + +function dget(dict, key, pos, rest, start, after, prev, nxt, tok) { + pos = 1 + while (pos <= length(dict)) { + rest = substr(dict, pos) + tok = "/" key + if (!match(rest, tok)) return "" + start = pos + RSTART - 1 + after = start + 1 + length(key) + prev = (start > 1) ? substr(dict, start - 1, 1) : " " + nxt = substr(dict, after, 1) + if (prev !~ /[A-Za-z0-9]/ && (nxt == "" || nxt ~ /[ \t\r\n\/\[\]<>\(\)]/)) + return value_at(dict, after) + pos = after + } + return "" +} + +function value_at(text, pos, c, c2, endp, tok, p2, e2, t2, p3, e3) { + while (pos <= length(text) && substr(text, pos, 1) ~ /[ \t\r\n]/) pos++ + if (pos > length(text)) return "" + c = substr(text, pos, 1); c2 = substr(text, pos + 1, 1) + if (c == "<" && c2 == "<") { endp = scan_balanced(text, pos); return trim(substr(text, pos, endp - pos)) } + if (c == "[") { endp = scan_balanced(text, pos); return trim(substr(text, pos, endp - pos)) } + if (c == "(") { endp = scan_string(text, pos); return trim(substr(text, pos, endp - pos)) } + if (c == "<") { endp = index(substr(text, pos + 1), ">"); if (endp == 0) return ""; return trim(substr(text, pos, endp + 1)) } + if (c == "/") { # a name: "/" is part of the token + endp = pos + 1 + while (endp <= length(text) && substr(text, endp, 1) !~ /[ \t\r\n\/\[\]<>\(\)]/) endp++ + return trim(substr(text, pos, endp - pos)) + } + + endp = pos + while (endp <= length(text) && substr(text, endp, 1) !~ /[ \t\r\n\/\[\]<>\(\)]/) endp++ + tok = substr(text, pos, endp - pos) + if (tok ~ /^[0-9]+$/) { + p2 = endp + while (p2 <= length(text) && substr(text, p2, 1) ~ /[ \t\r\n]/) p2++ + e2 = p2 + while (e2 <= length(text) && substr(text, e2, 1) !~ /[ \t\r\n\/\[\]<>\(\)]/) e2++ + t2 = substr(text, p2, e2 - p2) + if (t2 == "0") { + p3 = e2 + while (p3 <= length(text) && substr(text, p3, 1) ~ /[ \t\r\n]/) p3++ + e3 = p3 + while (e3 <= length(text) && substr(text, e3, 1) !~ /[ \t\r\n\/\[\]<>\(\)]/) e3++ + if (substr(text, p3, e3 - p3) == "R") tok = substr(text, pos, e3 - pos) + } + } + return trim(tok) +} + +function scan_balanced(text, p, i, ch, stack, top, e) { + i = p; stack = "" + while (i <= length(text)) { + ch = substr(text, i, 1) + if (ch == "(") { i = scan_string(text, i); continue } + if (ch == "<" && substr(text, i + 1, 1) == "<") { stack = stack "d"; i += 2; continue } + if (ch == ">" && substr(text, i + 1, 1) == ">") { + top = substr(stack, length(stack), 1) + if (top == "d") stack = substr(stack, 1, length(stack) - 1) + i += 2 + if (stack == "") return i + continue + } + if (ch == "[") { stack = stack "a"; i++; continue } + if (ch == "]") { + top = substr(stack, length(stack), 1) + if (top == "a") stack = substr(stack, 1, length(stack) - 1) + i++ + if (stack == "") return i + continue + } + if (ch == "<") { e = index(substr(text, i + 1), ">"); i = (e == 0) ? length(text) + 1 : i + e + 1; continue } + i++ + } + return length(text) + 1 +} + +function scan_string(text, p, i, ch, depth) { + i = p; depth = 0 + while (i <= length(text)) { + ch = substr(text, i, 1) + if (ch == "\\") { i += 2; continue } + if (ch == "(") depth++ + if (ch == ")") { depth--; if (depth == 0) return i + 1 } + i++ + } + return length(text) + 1 +} + +# array_ids: "[5 0 R 6 0 R]" -> "5 6" +function array_ids(v, s, out, tok, num, k) { + s = trim(v) + if (substr(s, 1, 1) != "[") return "" + s = trim(substr(s, 2, length(s) - 2)) + out = "" + while (s != "") { + tok = value_at(s, 1) + if (tok == "") break + num = tok + sub(/[ \t\r\n].*$/, "", num) # keep the object number only + out = (out == "") ? num : out " " num + k = index(s, tok) + if (k == 0) break + s = trim(substr(s, k + length(tok))) + } + return out +} + +function split_ids(ids, arr, cnt) { + cnt = split(ids, arr, " ") + return cnt +} + +# --------------------------------------------------------------------------- +# Field walking +# --------------------------------------------------------------------------- +function walk_field(id, inh_ft, inh_ff, inh_v, depth, f, ft, ff, v, kids, kidsids, kcnt, kparts, kf, first, i) { + if (depth > 16) return + if (!(id in OBJ)) return + f = OBJ[id] + + ft = dget(f, "FT"); if (ft == "") ft = inh_ft + ff = dget(f, "Ff"); if (ff == "") ff = inh_ff + v = dget(f, "V"); if (v == "") v = inh_v + kids = dget(f, "Kids") + + if (kids == "") { + register_field(f, id, "", ft, ff, v) + return + } + + kidsids = array_ids(kids) + kcnt = split_ids(kidsids, kparts) + first = (kcnt >= 1) ? kparts[1] : "" + kf = (first in OBJ) ? OBJ[first] : "" + if (dget(kf, "Subtype") == "/Widget") { + register_field(f, id, kidsids, ft, ff, v) + } else { + for (i = 1; i <= kcnt; i++) walk_field(kparts[i], ft, ff, v, depth + 1) + } +} + +# register_field +function register_field(f, fid, wids, ft, ff, v, fnum, wcnt, wparts, i, wid, w, ap, nn, stream, state, substream) { + if (ft !~ /^\/(Tx|Ch|Btn)$/) return + fnum = ff + 0 + if (ft == "/Btn" && int(fnum / 65536) % 2 == 1) return + + total++ + if (!is_meaningful(ft, v)) return + filled++ + + if (wids == "") wids = fid + wcnt = split_ids(wids, wparts) + for (i = 1; i <= wcnt; i++) { + wid = wparts[i] + if (!(wid in OBJ)) { apmissing++; continue } + w = OBJ[wid] + ap = dget(w, "AP") + if (ap == "") { apmissing++; continue } + nn = dget(ap, "N") + if (nn == "") { apmissing++; continue } + stream = deref_text(nn) + if (stream ~ /\/Length/ || stream ~ /\/Subtype[ \t\r\n]+\/Form/) continue + state = dget(w, "AS") + if (state == "") state = dget(f, "AS") + if (state == "" && substr(trim(v), 1, 1) == "/") state = trim(v) + if (state == "") { apmissing++; continue } + substream = dget(stream, substr(state, 2)) + if (substream == "") apmissing++ + } +} + +function is_meaningful(ft, v, name) { + v = trim(v) + if (v == "") return 0 + if (ft == "/Btn") { + name = trim(v) + return (name != "/Off" && substr(name, 1, 1) == "/") + } + if (substr(v, 1, 1) == "[") return array_any_meaningful(v) + return string_meaningful(v) +} + +# array_any_meaningful: true iff at least one element of [..] is meaningful. +function array_any_meaningful(v, s, tok, k, n2) { + s = trim(v) + if (substr(s, 1, 1) != "[") return 0 + s = trim(substr(s, 2, length(s) - 2)) + while (s != "") { + tok = value_at(s, 1) + if (tok == "") return 0 + if (string_meaningful(tok)) return 1 + k = index(s, tok) + if (k == 0) return 0 + s = trim(substr(s, k + length(tok))) + } + return 0 +} + +function string_meaningful(v, s) { + v = trim(v) + if (v == "") return 0 + if (substr(v, 1, 1) == "(") { + s = substr(v, 2, length(v) - 2) + gsub(/\\[()\\]/, "", s) + gsub(/[ \t\r\n]/, "", s) + return (length(s) > 0) + } + if (substr(v, 1, 1) == "<") { + s = substr(v, 2, length(v) - 2) + gsub(/[ \t\r\n0]/, "", s) + return (length(s) > 0) + } + return 0 +} + +# --------------------------------------------------------------------------- +# XMP +# --------------------------------------------------------------------------- +function xmp_value(payload, prop, s) { + if (match(payload, prop "[ \t\r\n]*=[ \t\r\n]*\"[^\"]*\"")) { + s = substr(payload, RSTART, RLENGTH) + sub(/^[^"]*"/, "", s) + sub(/"$/, "", s) + return s + } + if (match(payload, "<[A-Za-z0-9_]*:" prop ">[^<]*<")) { + s = substr(payload, RSTART, RLENGTH) + sub(/^<[^>]*>/, "", s) + sub(/<.*$/, "", s) + return s + } + return "" +} + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- +function evcodes( k, i, j, cnt, keys, out) { + cnt = 0 + for (k in ev) keys[cnt++] = k + for (i = 1; i < cnt; i++) { + k = keys[i]; j = i - 1 + while (j >= 0 && keys[j] > k) { keys[j + 1] = keys[j]; j-- } + keys[j + 1] = k + } + out = "" + for (i = 0; i < cnt; i++) out = (out == "") ? keys[i] : out "," keys[i] + return out +} + +function record_json(cls, form, appr, decl, tool, toolver, codes, cnt, i, arr, out, dv) { + if (decl == "") dv = "null" + else dv = sprintf("{\"filledBy\":\"%s\",\"tool\":%s,\"toolVersion\":%s}", decl, + (tool == "" ? "null" : "\"" tool "\""), + (toolver == "" ? "null" : "\"" toolver "\"")) + codes = evcodes() + cnt = split(codes, arr, ",") + out = "" + for (i = 1; i <= cnt; i++) { + if (arr[i] == "") continue + out = (out == "") ? "\"" arr[i] "\"" : out ",\"" arr[i] "\"" + } + return sprintf("{\"ffp\":\"1.0\",\"classification\":\"%s\",\"form\":\"%s\",\"filled_fields\":%d,\"total_fields\":%d,\"appearances\":\"%s\",\"declared\":%s,\"evidence\":[%s]}", + cls, form, filled, total, appr, dv, out) +} + +function die_unreadable() { + printf "classification=unreadable form=unknown filled=0/0 appearances=unknown evidence=FFP-E-UNREADABLE\n" + if (ENVIRON["FFP_PROBE_RECORD"] == "1") + print "{\"ffp\":\"1.0\",\"classification\":\"unreadable\",\"form\":\"unknown\",\"filled_fields\":0,\"total_fields\":0,\"appearances\":\"unknown\",\"declared\":null,\"evidence\":[\"FFP-E-UNREADABLE\"]}" +} diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh b/1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh new file mode 100755 index 000000000..a20a035ef --- /dev/null +++ b/1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# run-conformance.sh — run a detector over every FFP vector and diff the +# produced line against the .expected file. +# +# A conforming detector MUST pass this suite. The default detector is the +# reference probe (probe.awk); point FFP_DETECTOR at a product detector to test +# it, e.g. +# +# FFP_DETECTOR="./target/debug/ffp-classify" bash run-conformance.sh +# +# The detector is invoked as: +# and MUST print exactly the canonical line (see probe.awk's header). +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VECTORS="$HERE/vectors" +EXPECTED="$HERE/expected" +PROBE="${FFP_DETECTOR:-awk -f $HERE/probe.awk}" + +pass=0; fail=0; missing=0 + +# --- fixture reproducibility ------------------------------------------------- +if bash "$HERE/make-fixtures.sh" --check >/dev/null 2>&1; then + echo " ok vectors are byte-identical to the generator" +else + echo " FAIL vectors have drifted from make-fixtures.sh (run: bash make-fixtures.sh)" + fail=$((fail + 1)) +fi + +# --- every vector has an expectation, every expectation a vector ------------- +for vec in "$VECTORS"/*.pdf; do + name="$(basename "$vec" .pdf)" + [ -f "$EXPECTED/$name.expected" ] || { echo " FAIL $name: no .expected file"; missing=$((missing + 1)); } +done +for exp in "$EXPECTED"/*.expected; do + name="$(basename "$exp" .expected)" + [ -f "$VECTORS/$name.pdf" ] || { echo " FAIL $name.expected: no vector"; missing=$((missing + 1)); } +done + +# --- the vectors ------------------------------------------------------------- +for vec in "$VECTORS"/*.pdf; do + name="$(basename "$vec" .pdf)" + exp="$EXPECTED/$name.expected" + [ -f "$exp" ] || continue + got="$($PROBE "$vec" 2>/dev/null | head -n1)" + want="$(cat "$exp")" + if [ "$got" = "$want" ]; then + printf ' \033[32mok\033[0m %s\n' "$name" + pass=$((pass + 1)) + else + printf ' \033[31mFAIL\033[0m %s\n' "$name" + printf ' want: %s\n' "$want" + printf ' got: %s\n' "$got" + fail=$((fail + 1)) + fi +done + +echo "" +echo "FFP conformance: $pass passed, $fail failed, $missing orphan expectation(s)" +[ "$fail" -eq 0 ] && [ "$missing" -eq 0 ] diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-empty-values.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-empty-values.pdf new file mode 100644 index 000000000..947ef999f Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-empty-values.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-need-appearances.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-need-appearances.pdf new file mode 100644 index 000000000..e3764159e Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form-need-appearances.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form.pdf new file mode 100644 index 000000000..72ef8ddb5 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/blank-form.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/button-machine-filled.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/button-machine-filled.pdf new file mode 100644 index 000000000..184ee4bef Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/button-machine-filled.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/declared-unknown-value.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/declared-unknown-value.pdf new file mode 100644 index 000000000..953738eff Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/declared-unknown-value.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/filled-with-ap-need-appearances.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/filled-with-ap-need-appearances.pdf new file mode 100644 index 000000000..339e6d3cb Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/filled-with-ap-need-appearances.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared-generated.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared-generated.pdf new file mode 100644 index 000000000..b2b45e2c2 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared-generated.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared.pdf new file mode 100644 index 000000000..14b7cc634 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-declared.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-mixed-ap.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-mixed-ap.pdf new file mode 100644 index 000000000..4c73ff7f8 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-mixed-ap.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-partial.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-partial.pdf new file mode 100644 index 000000000..981254d22 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected-partial.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected.pdf new file mode 100644 index 000000000..3fc8dec35 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/machine-filled-suspected.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/no-form.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/no-form.pdf new file mode 100644 index 000000000..41420240d Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/no-form.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/unreadable.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/unreadable.pdf new file mode 100644 index 000000000..165706fd7 Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/unreadable.pdf differ diff --git a/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/viewer-filled.pdf b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/viewer-filled.pdf new file mode 100644 index 000000000..7977ede4f Binary files /dev/null and b/1-formats/sub-specs/form-fill-provenance/spec/conformance/vectors/viewer-filled.pdf differ diff --git a/Justfile b/Justfile index 475a94f89..2dffd72b5 100644 --- a/Justfile +++ b/Justfile @@ -132,6 +132,11 @@ dyadt-conformance: dyadt-test: @bash scripts/tests/wave4-dyadt-test.sh +# FFP: run the form-fill provenance conformance vectors against the reference probe +# (set FFP_DETECTOR to test a product detector instead) +ffp-conformance: + @bash 1-formats/sub-specs/form-fill-provenance/spec/conformance/run-conformance.sh + # Structural lint for per-language testing guides (required sections + R1..R9) language-guides-check: @bash scripts/check-language-guide.sh diff --git a/TOPOLOGY.adoc b/TOPOLOGY.adoc index 28583408c..ab237c23c 100644 --- a/TOPOLOGY.adoc +++ b/TOPOLOGY.adoc @@ -11,7 +11,7 @@ It cannot freeze: every regeneration re-reads ground truth. Do not edit by hand. ____ * *Phase:* active | *Maturity:* experimental | *STATE last-updated:* 2026-09-07T00:00:00Z -* *Registry entries:* 33 specs across 6 streams +* *Registry entries:* 34 specs across 6 streams * *Front door:* human → link:README.adoc[README.adoc]; machine → link:0-AI-MANIFEST.a2ml[0-AI-MANIFEST.a2ml] * *Registry:* link:.machine_readable/REGISTRY.a2ml[.machine_readable/REGISTRY.a2ml] (index + source hashes) · prose: link:REGISTRY.adoc[REGISTRY.adoc] @@ -31,6 +31,7 @@ ____ | NEUROSYM.a2ml spec | link:1-formats/a2ml/neurosym/[`+1-formats/a2ml/neurosym/+`] | symbolic semantics / proof obligations | PLAYBOOK.a2ml spec | link:1-formats/a2ml/playbook/[`+1-formats/a2ml/playbook/+`] | executable operational runbooks | ANCHOR.a2ml spec | link:1-formats/a2ml/anchor/[`+1-formats/a2ml/anchor/+`] | project-recalibration intervention format +| FFP — Form-Fill Provenance | link:1-formats/sub-specs/form-fill-provenance/[`+1-formats/sub-specs/form-fill-provenance/+`] | whether a PDF form was machine-filled or printed blank for hand completion, and what a print path must record | A2ML — Attested Markup Language | https://github.com/hyperpolymath/a2ml/blob/main/README.adoc[`+hyperpolymath/a2ml+`] `+@ v1.0.0+` ⇗ | the typed/verified machine-readable document format; evicted from standards 2026-08-28 per #490 |=== diff --git a/scripts/build-registry.sh b/scripts/build-registry.sh index 9f4e51553..71754b44f 100755 --- a/scripts/build-registry.sh +++ b/scripts/build-registry.sh @@ -87,6 +87,7 @@ publication-pre-flight|governance|3-practice/publication-pre-flight/|Publication release-pre-flight|governance|3-practice/release-pre-flight/|Release Pre-Flight (V1 Gate)|hard v1.0.0 audit requirements hypatia-rules|integration|hypatia-rules/|Standards Hypatia Rules|the dogfooding rules that scan THIS repo (incl. drift detection) a2ml-templates|integration|1-formats/templates/|A2ML Templates|copy-in templates for the 7 A2ML files +form-fill-provenance|foundation|1-formats/sub-specs/form-fill-provenance/|FFP — Form-Fill Provenance|whether a PDF form was machine-filled or printed blank for hand completion, and what a print path must record TSV # ---------------------------------------------------------------------------