diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fb8cd9..896d77b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,21 @@ stability commitment. See [ROADMAP.md](ROADMAP.md) for planned milestones. ## [Unreleased] -## [0.14.0] — 2026-10-07 +### Changed + +- `admina redteam --format md`: each cell gives the recall with the + detected and expected counts (`89% (8/9)`), and a note says that the + figures are indicative (a class often has one to three samples) and that + the labels of the corpora are assigned in this repository. +- The PII corpus has eight hard negatives (`hard_negative`): numbers and + codes that are not personal data. The baseline records the false + positives they produce: the Rust scanner reports a number that fails the + Luhn check as `CREDIT_CARD` and an ISBN and a build number as `PHONE`; + spaCy and Presidio report `PERSON` on two Italian sentences. PII false + positives: Python 8/24, Rust 3/24, Presidio 12/24 (6/16, 0/16 and 9/16 + before). + +## [0.14.0] — 2026-10-08 Minor release with a declared breaking change: Admina is published on PyPI as `admina`. The module (`import admina`), the `admina` command, the extras, diff --git a/MODEL_CARD.md b/MODEL_CARD.md index b0743a9..59659bf 100644 --- a/MODEL_CARD.md +++ b/MODEL_CARD.md @@ -1106,19 +1106,20 @@ silently skipping — so a real regression measured in the wrong mode can never pass. A python detector that ran but is absent from the baseline fails too. **First measured baseline** (Admina's own corpus — **not** a third-party PINT -score; the corpus is small and EN/EU-focused, intended to grow). Injection/loop +score; the corpus is small and EN/EU-focused, intended to grow, and its labels +are assigned in this repository, not adjudicated by a third party). Injection/loop are sample-level recall; PII is type-level recall measured in `nlp:en_core_web_sm` mode (the mode pinned in the baseline): | Detector | Python recall | Rust recall | False positives (py · rust) | |-----------|:---:|:---:|:---:| | injection | 57% (sample-level) | 35% | 0/27 · 0/27 | -| pii | 100% (type-level, nlp) | 66% (type-level) | 6/16 · 0/16 | +| pii | 100% (type-level, nlp) | 66% (type-level) | 8/24 · 3/24 | | loop | 82% (sample-level) | 91% | 0/11 · 0/11 | The optional Presidio PII engine is measured as a third row in the same baseline (`admina/redteam/baselines/baseline.json`): **52%** type-level -recall with **9/16** false positives, pinned to mode +recall with **12/24** false positives, pinned to mode `presidio:2.2.363/en+it`. It is an alternative engine, not an accelerator, so it is reported separately rather than in the Python-vs-Rust matrix above. @@ -1131,7 +1132,12 @@ scanner scores **0%** on IBAN / codice-fiscale / DNI (regex-only, fewer patterns the Python loop-breaker misses counter-reset loops (last-5 window) that the Rust full-window engine catches. The Python PII false positives are spaCy NER mis-firing `PERSON`/`ORG` on non-English negative samples — which is also why -the PII baseline pins the NER mode. These measured gaps are consistent with §1: +the PII baseline pins the NER mode. The PII negatives include eight hard +negatives (`hard_negative`: an order number that fails the Luhn check, a +version, an invoice number and a date, an ISBN, a build number, room and +chapter numbers, two Italian reference codes): the Rust scanner reports the +non-Luhn number as `CREDIT_CARD` and the ISBN and the build number as +`PHONE`; spaCy and Presidio report `PERSON` on the two Italian sentences. These measured gaps are consistent with §1: the two engines are **not** behaviorally equivalent — Python is the higher-recall default, Rust the narrower-coverage opt-in. diff --git a/admina/redteam/baselines/baseline.json b/admina/redteam/baselines/baseline.json index 1a298a7..5f2cad8 100644 --- a/admina/redteam/baselines/baseline.json +++ b/admina/redteam/baselines/baseline.json @@ -14,19 +14,19 @@ "pii": { "python": { "type_recall": 1.0, - "fp": 6, - "fp_samples": 16, + "fp": 8, + "fp_samples": 24, "mode": "nlp:en_core_web_sm@3.8.0" }, "rust": { "type_recall": 0.6551724137931034, - "fp": 0, - "fp_samples": 16 + "fp": 3, + "fp_samples": 24 }, "presidio": { "type_recall": 0.5172413793103449, - "fp": 9, - "fp_samples": 16, + "fp": 12, + "fp_samples": 24, "mode": "presidio:2.2.363/en+it" } }, @@ -42,4 +42,4 @@ "fp_samples": 11 } } -} +} \ No newline at end of file diff --git a/admina/redteam/corpora/SHA256SUMS b/admina/redteam/corpora/SHA256SUMS index 7b5da3b..ba6679f 100644 --- a/admina/redteam/corpora/SHA256SUMS +++ b/admina/redteam/corpora/SHA256SUMS @@ -1,4 +1,4 @@ 5b6904c941e71ca6fee0fcab722124ec1f9b17348648daee678d435ff940566b injection.jsonl -7157269c349894bd07aacceff469124a4444971d34c987eb0505a8c418532c8f pii.jsonl +902f6ada48bfca0aa79cfd0a156b90f5dbfaa6930eafd0de7ee90630e24e2371 pii.jsonl df6727b94272ab72897acd5f71e883ff52711038286b55dfaa026becdc193ab3 loop.jsonl ece92830693b29ac5246f95754bf704092473cf2602bd970355e85a0358bc0d5 coordination.jsonl diff --git a/admina/redteam/corpora/pii.jsonl b/admina/redteam/corpora/pii.jsonl index cf4ddf8..5dcd6f7 100644 --- a/admina/redteam/corpora/pii.jsonl +++ b/admina/redteam/corpora/pii.jsonl @@ -40,3 +40,11 @@ {"id":"pii-neg-014","text":"nos vemos luego para tomar algo","expected_types":[],"lang":"es","tag":"negative"} {"id":"pii-neg-015","text":"das Wetter ist heute wirklich schön","expected_types":[],"lang":"de","tag":"negative"} {"id":"pii-neg-016","text":"wir sehen uns morgen im Büro","expected_types":[],"lang":"de","tag":"negative"} +{"id":"pii-hneg-001","text":"order 4111 1111 1111 1112 was cancelled yesterday","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-002","text":"upgrade the library to version 2.14.3 before Friday","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-003","text":"invoice INV-2026-00123 is due on 2026-10-31","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-004","text":"ISBN 978-3-16-148410-0 is out of print","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-005","text":"build 20261008.1 passed all 4554 checks","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-006","text":"room 4B on floor 12, chapter 3.14 of the manual","expected_types":[],"lang":"en","tag":"hard_negative"} +{"id":"pii-hneg-007","text":"il codice articolo AB-12345-CD è esaurito in magazzino","expected_types":[],"lang":"it","tag":"hard_negative"} +{"id":"pii-hneg-008","text":"la pratica n. 2026/000457 è stata chiusa il 3 ottobre","expected_types":[],"lang":"it","tag":"hard_negative"} diff --git a/admina/redteam/report.py b/admina/redteam/report.py index 4942c5b..91a2dda 100644 --- a/admina/redteam/report.py +++ b/admina/redteam/report.py @@ -123,8 +123,23 @@ def _pct(x: float | None) -> str: return "n/a" if x is None else f"{round(x * 100)}%" +def _hits(metric: dict) -> tuple[int, int]: + """(detected, expected) behind the recall of *metric*: samples for the + binary detectors, PII types for the PII detector.""" + c = metric["counts"] + if "tp" in c: + return c["tp"], c["tp"] + c["fn"] + return c["intersect_total"], c["expected_total"] + + def _cell_recall(metric: dict | None) -> str: - return _pct(recall_item(metric)[1]) if metric else "-" + if not metric: + return "-" + value = recall_item(metric)[1] + if value is None: + return _pct(value) + detected, expected = _hits(metric) + return f"{_pct(value)} ({detected}/{expected})" def to_markdown(scorecard: dict) -> str: @@ -144,7 +159,12 @@ def to_markdown(scorecard: dict) -> str: lines.append( f"| {det} | {tag} | {_cell_recall(py)} | {_cell_recall(rs)} | {_cell_recall(pr)} |" ) - footnotes = [] + footnotes = [ + "_Each cell is the recall with the detected and expected counts. A class " + "often has one to three samples, so the figures are indicative, not a " + "benchmark. The labels of the corpora are assigned in this repository, " + "not adjudicated by a third party._" + ] external = scorecard.get("external_corpora") detector_of = external["corpora"] if external else {} if any(detector_of.get(name, name) == "pii" for name in scorecard["detectors"]): diff --git a/tests/test_redteam_lib.py b/tests/test_redteam_lib.py index 46cd476..66e6042 100644 --- a/tests/test_redteam_lib.py +++ b/tests/test_redteam_lib.py @@ -227,6 +227,18 @@ def test_build_scorecard_and_markdown_have_caveat(): assert "Detector" in md +def test_markdown_cells_show_detected_over_expected(): + results = { + "injection": {"python": _fake_eval_binary()}, + "pii": {"python": _fake_eval_pii()}, + } + md = report.to_markdown(report.build_scorecard(results, rust_version=None)) + assert "| injection | leetspeak | 0% (0/1) |" in md + assert "| pii | email | 100% (4/4) |" in md + assert "indicative" in md + assert "not adjudicated by a third party" in md + + def test_make_baseline_shape(): results = {"injection": {"python": _fake_eval_binary()}} card = report.build_scorecard(results, rust_version=None)