From 0a348f3345ce7e79d3063c95ddf6d81b79900d78 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 11:19:03 +0200 Subject: [PATCH 1/7] fix(core): check the Luhn sum of card numbers and match the phone formats of the Python redactor The Rust PII scanner reports a credit_card match only when its digits pass the Luhn checksum. Its phone pattern is the one of the Python redactor: North American numbers and Italian mobile and landline numbers, with or without +39 or 0039. The regex crate has no lookarounds, so each pattern carries an accept function that checks what the regex cannot: the checksum, and the characters before and after a phone number. redact() masks and counts only the accepted matches, and scan() counts them. An ISBN, a build number and a 16-digit number that fails the Luhn check are no longer reported. --- core-rust/src/pii.rs | 150 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 139 insertions(+), 11 deletions(-) diff --git a/core-rust/src/pii.rs b/core-rust/src/pii.rs index b05f780..9c0c5f8 100644 --- a/core-rust/src/pii.rs +++ b/core-rust/src/pii.rs @@ -13,13 +13,84 @@ // limitations under the License. use pyo3::prelude::*; -use regex::Regex; +use regex::{Captures, Regex}; use std::sync::OnceLock; struct PiiPattern { name: &'static str, regex: Regex, mask: &'static str, + /// Whether a match is reported, given the scanned text: the checks the + /// regex cannot express (a checksum, the characters around the match). + accept: fn(&str, &Captures) -> bool, +} + +fn any_match(_text: &str, _caps: &Captures) -> bool { + true +} + +/// A card number is reported only when its digits pass the Luhn checksum, +/// as in the Python PII redactor. +fn luhn_valid(_text: &str, caps: &Captures) -> bool { + let digits: Vec = caps[0].chars().filter_map(|c| c.to_digit(10)).collect(); + let sum: u32 = digits + .iter() + .rev() + .enumerate() + .map(|(i, &d)| { + if !i.is_multiple_of(2) { + let x = d * 2; + if x > 9 { + x - 9 + } else { + x + } + } else { + d + } + }) + .sum(); + !digits.is_empty() && sum.is_multiple_of(10) +} + +fn is_word(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +/// The boundaries of the phone patterns of the Python PII redactor, which +/// the regex crate cannot express as lookarounds: a North American number +/// is not preceded or followed by a digit; an Italian number is not +/// preceded by a word character or "+", nor followed by a word character. +fn phone_boundaries(text: &str, caps: &Captures) -> bool { + let whole = caps.get(0).unwrap(); + let before = text[..whole.start()].chars().next_back(); + let after = text[whole.end()..].chars().next(); + if caps.name("us").is_some() { + !before.is_some_and(|c| c.is_ascii_digit()) && !after.is_some_and(|c| c.is_ascii_digit()) + } else { + !before.is_some_and(|c| is_word(c) || c == '+') && !after.is_some_and(is_word) + } +} + +// Phone numbers, as in the Python PII redactor: North American format +// (3-3-4 digits, optional country code), and Italian numbers, with +39 or +// 0039 or without: mobile numbers (3 and 8 or 9 more digits, compact or in +// groups) and landline numbers (an area code 02, 06 or 0 followed by 2 or 3 +// digits, then 5 to 8 digits, or 2 or 3 groups of 2 to 4 digits separated +// by spaces; without +39 a space, "/" or "-" follows the area code). +const US_PHONE: &str = r"(?:\+\d{1,3}[\s.\-]?)?\(?\d{3}\)?[\s.\-]\d{3}[\s.\-]\d{4}"; +const IT_MOBILE: &str = r"3[1-9]\d(?:[ \-]?\d{6,7}|[ \-]\d{3}[ \-]\d{3,4})"; +const IT_AREA: &str = r"0(?:[26]|[1-9]\d{1,2})"; +const IT_SUBSCRIBER: &str = r"(?:\d{5,8}|\d{2,4} \d{2,4}(?: \d{2,4})?)"; + +fn phone_regex() -> Regex { + let italian = format!( + "(?:(?:\\+|00)39[ .\\-]?(?:{m}|{a}[ ./\\-]?{s})|{m}|{a}[ /\\-]{s})", + m = IT_MOBILE, + a = IT_AREA, + s = IT_SUBSCRIBER + ); + Regex::new(&format!("(?P{US_PHONE})|(?P{italian})")).unwrap() } static PII_PATTERNS: OnceLock> = OnceLock::new(); @@ -31,29 +102,31 @@ fn get_pii_patterns() -> &'static Vec { name: "email", regex: Regex::new(r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}").unwrap(), mask: "[EMAIL_REDACTED]", + accept: any_match, }, PiiPattern { name: "credit_card", regex: Regex::new(r"\b(?:\d{4}[\s\-]?){3}\d{4}\b").unwrap(), mask: "[CC_REDACTED]", + accept: luhn_valid, }, PiiPattern { name: "ssn", regex: Regex::new(r"\b\d{3}-\d{2}-\d{4}\b").unwrap(), mask: "[SSN_REDACTED]", + accept: any_match, }, PiiPattern { name: "phone", - regex: Regex::new( - r"(?:\+?\d{1,3}[\s\-.]?)?\(?\d{2,4}\)?[\s\-.]?\d{3,4}[\s\-.]?\d{3,4}\b", - ) - .unwrap(), + regex: phone_regex(), mask: "[PHONE_REDACTED]", + accept: phone_boundaries, }, PiiPattern { name: "iban", regex: Regex::new(r"\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z0-9]?){0,16}\b").unwrap(), mask: "[IBAN_REDACTED]", + accept: any_match, }, PiiPattern { name: "ip_address", @@ -62,6 +135,7 @@ fn get_pii_patterns() -> &'static Vec { ) .unwrap(), mask: "[IP_REDACTED]", + accept: any_match, }, ] }) @@ -123,10 +197,18 @@ impl RustPiiScanner { let mut details: Vec<(String, usize)> = Vec::new(); for pattern in patterns.iter() { - let matches: Vec<_> = pattern.regex.find_iter(&result).collect(); - let match_count = matches.len(); + let current = result.clone(); + let mut match_count: usize = 0; + let replaced = pattern.regex.replace_all(¤t, |caps: &Captures| { + if (pattern.accept)(¤t, caps) { + match_count += 1; + pattern.mask.to_string() + } else { + caps[0].to_string() + } + }); if match_count > 0 { - result = pattern.regex.replace_all(&result, pattern.mask).to_string(); + result = replaced.into_owned(); count += match_count; categories.push(pattern.name.to_string()); details.push((pattern.name.to_string(), match_count)); @@ -151,7 +233,11 @@ impl RustPiiScanner { let mut details: Vec<(String, usize)> = Vec::new(); for pattern in patterns.iter() { - let match_count = pattern.regex.find_iter(text).count(); + let match_count = pattern + .regex + .captures_iter(text) + .filter(|caps| (pattern.accept)(text, caps)) + .count(); if match_count > 0 { count += match_count; categories.push(pattern.name.to_string()); @@ -203,8 +289,50 @@ mod tests { #[test] fn test_credit_card_redaction() { let mut s = RustPiiScanner::new(); - let r = s.redact("Card: 4111-2222-3333-4444"); - assert!(!r.redacted_text.contains("4111-2222-3333-4444")); + let r = s.redact("Card: 4111-1111-1111-1111"); + assert!(!r.redacted_text.contains("4111-1111-1111-1111")); + assert!(r.categories.contains(&"credit_card".to_string())); + } + + #[test] + fn test_card_number_failing_luhn_is_not_a_card() { + let mut s = RustPiiScanner::new(); + let r = s.redact("order 4111 1111 1111 1112 was cancelled"); + assert!(r.redacted_text.contains("4111 1111 1111 1112")); + assert_eq!(r.count, 0); + assert_eq!(s.scan("order 4111 1111 1111 1112").count, 0); + } + + #[test] + fn test_phone_numbers_are_redacted() { + let mut s = RustPiiScanner::new(); + for text in [ + "call 555-123-4567 today", + "call +1 555-123-4567 today", + "chiama il 333 1234567", + "chiama +39 06 1234 5678", + "ufficio 02/12345678", + ] { + let r = s.redact(text); + assert!( + r.redacted_text.contains("[PHONE_REDACTED]"), + "{text}: {}", + r.redacted_text + ); + } + } + + #[test] + fn test_codes_that_are_not_phone_numbers() { + let s = RustPiiScanner::new(); + for text in [ + "ISBN 978-3-16-148410-0 is out of print", + "build 20261008.1 passed all 4554 checks", + "upgrade to version 2.14.3", + "id1234567890", + ] { + assert_eq!(s.scan(text).count, 0, "{text}"); + } } #[test] From ff93ea6393409b4ca9965da7344d989486bc9c12 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 11:19:03 +0200 Subject: [PATCH 2/7] fix(domains): mask a PERSON entity only when it reads as a name admina.domains.data_sovereignty.names.is_name_like() holds for a span with no digit and a word with a capital initial. The PII redactor, the spacy-regex engine and the Presidio engine skip the PERSON entities of the NER model for which it does not hold: the English spaCy model labels lowercase phrases of other languages as PERSON ("il codice articolo", "la pratica n. 2026/000457"). A name written all in lowercase is not masked either. --- admina/domains/data_sovereignty/names.py | 32 +++++++++ admina/domains/data_sovereignty/pii.py | 5 ++ admina/engines/presidio.py | 3 + admina/plugins/builtin/pii/spacy_regex.py | 3 + tests/test_person_names.py | 86 +++++++++++++++++++++++ tests/test_presidio_engine.py | 14 ++++ 6 files changed, 143 insertions(+) create mode 100644 admina/domains/data_sovereignty/names.py create mode 100644 tests/test_person_names.py diff --git a/admina/domains/data_sovereignty/names.py b/admina/domains/data_sovereignty/names.py new file mode 100644 index 0000000..24a1fc3 --- /dev/null +++ b/admina/domains/data_sovereignty/names.py @@ -0,0 +1,32 @@ +# Copyright © 2025–2026 Stefano Noferi & Admina contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Whether a PERSON entity of an NER model reads as a name. + +The PII engines mask a PERSON entity only when :func:`is_name_like` holds: +no digit, and at least one word with a capital initial. The English spaCy +model labels lowercase phrases of other languages as PERSON (Italian "il +codice articolo", "la pratica n. 2026/000457"); a name written all in +lowercase is not masked either. +""" + +from __future__ import annotations + + +def is_name_like(text: str) -> bool: + """True when *text* has no digit and a word that starts with a capital.""" + if any(ch.isdigit() for ch in text): + return False + return any(word[:1].isupper() for word in text.split()) diff --git a/admina/domains/data_sovereignty/pii.py b/admina/domains/data_sovereignty/pii.py index 1629245..c65edf1 100644 --- a/admina/domains/data_sovereignty/pii.py +++ b/admina/domains/data_sovereignty/pii.py @@ -30,6 +30,7 @@ placeholder_spans, replace_spans, ) +from admina.domains.data_sovereignty.names import is_name_like # spaCy is part of the [nlp] extra. When absent, PIIRedactor falls back # to regex-only mode (still covers EMAIL/PHONE/SSN/IBAN/IP/credit-card/EU IDs). @@ -301,6 +302,10 @@ def redact(self, text: str, categories: dict | None = None) -> dict: cat_config = active_categories.get(ent.label_, {}) if not cat_config.get("enabled", False): continue + if ent.label_ == "PERSON" and not is_name_like( + redacted[ent.start_char : ent.end_char] + ): + continue mask = self._mask(ent.label_, cat_config) parts = outside_placeholders(ent.start_char, ent.end_char, placeholders, redacted) for start, end in reversed(parts): diff --git a/admina/engines/presidio.py b/admina/engines/presidio.py index 60aa591..7a3cc0f 100644 --- a/admina/engines/presidio.py +++ b/admina/engines/presidio.py @@ -55,6 +55,7 @@ placeholder_spans, replace_spans, ) +from admina.domains.data_sovereignty.names import is_name_like from admina.domains.data_sovereignty.pii import PII_CATEGORIES logger = logging.getLogger("admina.engines.presidio") @@ -272,6 +273,8 @@ def redact(self, text: str) -> dict[str, Any]: cfg = PII_CATEGORIES.get(category, {}) if not cfg.get("enabled", False): continue + if category == "PERSON" and not is_name_like(text[r.start : r.end]): + continue mask = OMISSIS if self.mask_style == "omissis" else cfg.get("mask", f"[{category}]") for start, end in outside_placeholders(r.start, r.end, placeholders, text): spans.append((start, end, category, mask)) diff --git a/admina/plugins/builtin/pii/spacy_regex.py b/admina/plugins/builtin/pii/spacy_regex.py index 787a99e..c11a28b 100644 --- a/admina/plugins/builtin/pii/spacy_regex.py +++ b/admina/plugins/builtin/pii/spacy_regex.py @@ -27,6 +27,7 @@ from admina.domains.data_sovereignty.email_matching import EMAIL_RX, iter_email_matches from admina.domains.data_sovereignty.masking import replace_spans +from admina.domains.data_sovereignty.names import is_name_like from admina.plugins.base import BasePIIEngine logger = logging.getLogger("admina.plugins.pii.spacy_regex") @@ -125,6 +126,8 @@ async def detect( continue if allowed and ent.label_ not in allowed: continue + if ent.label_ == "PERSON" and not is_name_like(ent.text): + continue # Skip if already covered by a regex match (full interval-overlap test) overlap = any( not (ent.end_char <= mm["start"] or ent.start_char >= mm["end"]) diff --git a/tests/test_person_names.py b/tests/test_person_names.py new file mode 100644 index 0000000..7c19352 --- /dev/null +++ b/tests/test_person_names.py @@ -0,0 +1,86 @@ +# Copyright © 2025–2026 Stefano Noferi & Admina contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""A PERSON entity of an NER model is masked only when it reads as a name: +no digit, and at least one word with a capital initial. + +The English spaCy model labels lowercase Italian phrases as PERSON ("il +codice articolo", "la pratica n. 2026/000457"); such spans are not masked, +by the PII redactor, the spaCy + regex engine and the Presidio engine. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from admina.domains.data_sovereignty.names import is_name_like +from admina.domains.data_sovereignty.pii import PIIRedactor + + +@pytest.mark.parametrize( + "text", + ["Mario Rossi", "John Smith", "il signor Rossi", "O'Brien", "JOHN"], +) +def test_names(text): + assert is_name_like(text) + + +@pytest.mark.parametrize( + "text", + ["il codice articolo", "la pratica n. 2026/000457", "John 3", "", " "], +) +def test_not_names(text): + assert not is_name_like(text) + + +class _Nlp: + """Labels each given phrase of the text as PERSON.""" + + def __init__(self, *phrases: str) -> None: + self.phrases = phrases + + def __call__(self, text: str) -> SimpleNamespace: + ents = [] + for phrase in self.phrases: + start = text.find(phrase) + if start >= 0: + ents.append( + SimpleNamespace( + start_char=start, + end_char=start + len(phrase), + text=phrase, + label_="PERSON", + ) + ) + return SimpleNamespace(ents=ents) + + +def test_the_redactor_masks_names_only(): + redactor = PIIRedactor() + redactor.nlp = _Nlp("il codice articolo", "Mario Rossi") + out = redactor.redact("il codice articolo AB-12 è di Mario Rossi") + assert out["redacted_text"] == "il codice articolo AB-12 è di [PERSON]" + + +def test_the_spacy_regex_engine_reports_names_only(): + from admina.plugins.builtin.pii.spacy_regex import SpaCyRegexPIIEngine + + engine = SpaCyRegexPIIEngine() + engine._nlp, engine._nlp_loaded = _Nlp("il codice articolo", "Mario Rossi"), True + found = asyncio.run(engine.detect("il codice articolo AB-12 è di Mario Rossi")) + assert [(m["type"], m["text"]) for m in found] == [("PERSON", "Mario Rossi")] diff --git a/tests/test_presidio_engine.py b/tests/test_presidio_engine.py index ba61347..b78c261 100644 --- a/tests/test_presidio_engine.py +++ b/tests/test_presidio_engine.py @@ -215,3 +215,17 @@ def test_text_with_many_dots_is_analyzed_quickly(): # Every recognizer of Presidio took about a minute on this text; the # mapped ones take a fraction of a second. assert time.perf_counter() - start < 5.0 + + +def test_person_spans_that_are_not_names_are_not_masked(monkeypatch): + presidio = _presidio() + from presidio_analyzer import RecognizerResult + + engine = presidio.PresidioPIIEngine(nlp_models={"it": "blank"}) + text = "il codice articolo AB-12 è di Mario Rossi" + found = [ + RecognizerResult("PERSON", 0, 18, 0.85), # "il codice articolo" + RecognizerResult("PERSON", 30, 41, 0.85), # "Mario Rossi" + ] + monkeypatch.setattr(engine._analyzer, "analyze", lambda text, language, entities: list(found)) + assert engine.redact(text)["redacted_text"] == "il codice articolo AB-12 è di [PERSON]" From 414b5de34e2e4c836385659c5b6c5d1335e07f37 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 11:19:03 +0200 Subject: [PATCH 3/7] test(proxy): bound the event-loop stall by the duration of the slow job The event-loop tests run a 1 s job and accept a stall shorter than the job. A loop that runs the job, or waits for it without yielding, stalls for at least the whole job, so the tests fail for it whatever the load; with the loop free, a stall reaches the bound only if the scheduler holds the loop for a second. Checked with an executor that waits for the worker thread on the loop: the three tests fail with stalls of 1.0 to 8.0 s. --- tests/test_pipeline_executor.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline_executor.py b/tests/test_pipeline_executor.py index fd583a4..09bde65 100644 --- a/tests/test_pipeline_executor.py +++ b/tests/test_pipeline_executor.py @@ -123,13 +123,14 @@ def _blocked(resp: httpx.Response) -> bool: # How long the slow job (a firewall check, a redaction) sleeps in the thread # that runs it, and the largest event loop stall the tests accept. Run on the -# event loop, the job stalls it for its whole duration; run in a worker -# thread, the loop only waits for the scheduler. The bound is half the job: -# well below the stall of a blocked loop, well above the scheduling delays of -# a loaded CI runner (stalls just over 0.1 s were seen on macOS runners with -# the loop free). -SLOW_JOB = 0.6 -MAX_STALL = SLOW_JOB / 2 +# event loop, or awaited there without yielding, the job stalls the loop for +# at least its whole duration: a stall shorter than the job means the loop +# was free. The bound is the job itself, so the test fails for a blocked +# loop whatever the load, and passes with the loop free unless the scheduler +# holds it for a whole second (stalls of up to 0.51 s were seen on loaded +# macOS runners with the loop free). +SLOW_JOB = 1.0 +MAX_STALL = SLOW_JOB async def _largest_gap(stop: asyncio.Event) -> float: From 0726bbaf42c84a68674d9a0674d15469cdaaccd7 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 11:19:03 +0200 Subject: [PATCH 4/7] test(redteam): regenerate the baseline with the PII fixes PII false positives: Python 4/24, Rust 0/24, Presidio 8/24; Presidio type-level recall 0.586. MODEL_CARD gives the numbers, the Luhn check and the phone formats of both engines, the IBAN checks of each engine, and the false positives that remain: ORG/GPE on non-English sentences, PERSON on German sentences, and PERSON on "Friday" and PHONE on an Italian reference number with Presidio. --- MODEL_CARD.md | 37 +++++++++++++++----------- admina/redteam/baselines/baseline.json | 8 +++--- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/MODEL_CARD.md b/MODEL_CARD.md index 59659bf..64b09b5 100644 --- a/MODEL_CARD.md +++ b/MODEL_CARD.md @@ -41,8 +41,8 @@ All Rust components are pure functions exposed via PyO3. Rust is faster, but the two engines are not behaviorally equivalent: on an internal 14-attack evasion corpus the Python firewall blocks all 14 while the Rust firewall blocks 7 (plain-text and single-encoding attacks only). The Rust -PII engine also lacks EU national-ID patterns, spaCy NER, and Luhn -validation. Python is the higher-recall engine; Rust is opt-in for +PII engine also lacks EU national-ID patterns and spaCy NER (it checks the +Luhn sum of card numbers since 0.14.1). Python is the higher-recall engine; Rust is opt-in for latency-sensitive workloads where the narrower coverage is acceptable. Opt-in means the `[rust]` extra of a `pip install`. The official proxy @@ -263,7 +263,7 @@ optionally mirrored into the Rust accelerator at Detects and redacts PII in text. Three modes: - **Regex-only** (default, fast): email, phone, SSN, US credit card - (Luhn-validated — Python engine only; Rust path does not run Luhn), + (Luhn-validated on both engines; the Rust engine since 0.14.1), IBAN, IPv4, Italian codice fiscale, Spanish DNI/NIE, and German Personalausweis (shipped but **disabled by default** — the format is too ambiguous to regex safely). Python engine default; Rust path @@ -290,10 +290,12 @@ Detects and redacts PII in text. Three modes: models it needs. Note that on Admina's own corpus Presidio measures *lower* type-level recall than the default spaCy+regex engine on EU identifiers — see §9. -- **Regex precision varies by category.** Phone-number regex has high - recall but low precision (matches version strings, IDs). Credit-card - regex uses Luhn validation (Python engine) and is reliable. IBAN regex does not - validate the country-specific checksum and may match invalid IBANs. +- **Regex precision varies by category.** Phone numbers are matched in the + North American and Italian formats only, so other national formats are + missed. Card numbers are reported only when they pass the Luhn check (both + engines). The Python engine checks the length and the mod-97 checksum of + an IBAN; the Rust engine matches the IBAN shape only and may report an + invalid IBAN. - **No image or document parsing.** Admina sees text only. PII embedded in images, PDFs, or audio passes through unchanged. Pre-process those upstream. @@ -1114,12 +1116,12 @@ mode (the mode pinned in the baseline): | Detector | Python recall | Rust recall | False positives (py · rust) | |-----------|:---:|:---:|:---:| | injection | 57% (sample-level) | 35% | 0/27 · 0/27 | -| pii | 100% (type-level, nlp) | 66% (type-level) | 8/24 · 3/24 | +| pii | 100% (type-level, nlp) | 66% (type-level) | 4/24 · 0/24 | | loop | 82% (sample-level) | 91% | 0/11 · 0/11 | The optional Presidio PII engine is measured as a third row in the same -baseline (`admina/redteam/baselines/baseline.json`): **52%** type-level -recall with **12/24** false positives, pinned to mode +baseline (`admina/redteam/baselines/baseline.json`): **59%** type-level +recall with **8/24** false positives, pinned to mode `presidio:2.2.363/en+it`. It is an alternative engine, not an accelerator, so it is reported separately rather than in the Python-vs-Rust matrix above. @@ -1135,11 +1137,16 @@ mis-firing `PERSON`/`ORG` on non-English negative samples — which is also why the PII baseline pins the NER mode. The PII negatives include eight hard negatives (`hard_negative`: an order number that fails the Luhn check, a version, an invoice number and a date, an ISBN, a build number, room and -chapter numbers, two Italian reference codes): the Rust scanner reports the -non-Luhn number as `CREDIT_CARD` and the ISBN and the build number as -`PHONE`; spaCy and Presidio report `PERSON` on the two Italian sentences. These measured gaps are consistent with §1: -the two engines are **not** behaviorally equivalent — Python is the -higher-recall default, Rust the narrower-coverage opt-in. +chapter numbers, two Italian reference codes). Since 0.14.1 the Rust scanner +checks the Luhn sum of card numbers and matches the phone formats of the +Python redactor, and the three engines mask a `PERSON` entity only when it +has no digit and a word with a capital initial: the remaining false +positives are `ORG`/`GPE` on non-English sentences, `PERSON` on German +sentences (German capitalises nouns), and, for Presidio, `PERSON` on +"Friday" and `PHONE` on an Italian reference number. These measured gaps +are consistent with §1: the two engines are **not** behaviorally +equivalent — Python is the higher-recall default, Rust the +narrower-coverage opt-in. This replaces the previous "no accuracy benchmark suite" gap. Contributions extending the corpora (more languages, larger adversarial sets, `garak` / diff --git a/admina/redteam/baselines/baseline.json b/admina/redteam/baselines/baseline.json index 5f2cad8..4da67d0 100644 --- a/admina/redteam/baselines/baseline.json +++ b/admina/redteam/baselines/baseline.json @@ -14,18 +14,18 @@ "pii": { "python": { "type_recall": 1.0, - "fp": 8, + "fp": 4, "fp_samples": 24, "mode": "nlp:en_core_web_sm@3.8.0" }, "rust": { "type_recall": 0.6551724137931034, - "fp": 3, + "fp": 0, "fp_samples": 24 }, "presidio": { - "type_recall": 0.5172413793103449, - "fp": 12, + "type_recall": 0.5862068965517241, + "fp": 8, "fp_samples": 24, "mode": "presidio:2.2.363/en+it" } From 9273fdb04653170a24c2e3f2cc00c2b05aec1137 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 11:19:03 +0200 Subject: [PATCH 5/7] =?UTF-8?q?chore:=20release=200.14.1=20=E2=80=94=20ver?= =?UTF-8?q?sion=20bump=20and=20lockfile=20alignment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 44 +++++++++++++++++++++++++++++++++------- ROADMAP.md | 2 +- admina/__init__.py | 2 +- core-rust/Cargo.lock | 2 +- core-rust/Cargo.toml | 2 +- core-rust/pyproject.toml | 2 +- core-rust/uv.lock | 2 +- pyproject.toml | 2 +- uv.lock | 4 ++-- 9 files changed, 46 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 896d77b..efc6b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,29 @@ stability commitment. See [ROADMAP.md](ROADMAP.md) for planned milestones. ## [Unreleased] +## [0.14.1] — 2026-10-08 + +Patch release: fewer PII false positives on numbers, codes and non-English +text, on the three PII engines; the redteam scorecard gives the counts +behind each recall. Upgrading is recommended. + +### Fixed + +- The Rust PII scanner (`admina-core`) reports a card number only when it + passes the Luhn check, as the Python redactor does: a 16-digit order + number that fails the check is no longer masked as `CREDIT_CARD`. +- The Rust PII scanner matches the phone formats of the Python redactor + (North American numbers, and Italian mobile and landline numbers with or + without +39 or 0039, with the same boundaries). It reported an ISBN, a + build number and other digit runs as `PHONE`. +- The PII redactor, the `spacy-regex` engine and the Presidio engine mask a + `PERSON` entity of the NER model only when it has no digit and a word + with a capital initial. The English spaCy model labelled lowercase + phrases of other languages as `PERSON` ("il codice articolo", "la + pratica n. 2026/000457"). A name written all in lowercase is not masked + either. With Presidio, such a span no longer takes over an overlapping + entity of another type. + ### Changed - `admina redteam --format md`: each cell gives the recall with the @@ -20,12 +43,18 @@ stability commitment. See [ROADMAP.md](ROADMAP.md) for planned milestones. figures are indicative (a class often has one to three samples) and that the labels of the corpora are assigned in this repository. - The PII corpus has eight hard negatives (`hard_negative`): numbers and - codes that are not personal data. The baseline records the false - positives they produce: the Rust scanner reports a number that fails the - Luhn check as `CREDIT_CARD` and an ISBN and a build number as `PHONE`; - spaCy and Presidio report `PERSON` on two Italian sentences. PII false - positives: Python 8/24, Rust 3/24, Presidio 12/24 (6/16, 0/16 and 9/16 - before). + codes that are not personal data. The baseline is regenerated with the + fixes above. PII false positives: Python 4/24, Rust 0/24, Presidio 8/24 + (6/16, 0/16 and 9/16 in 0.14.0); Presidio type-level recall 59% (52% in + 0.14.0). The remaining false positives are listed in MODEL_CARD §9. + +### Notes + +- `tests/test_pipeline_executor.py`: the event-loop tests run a 1 s job and + accept a stall shorter than the job. A loop blocked by the job stalls for + at least its whole duration, so the tests still fail for it, and the + scheduling delays of loaded CI runners (up to 0.51 s on macOS) no longer + stop the release test gate. ## [0.14.0] — 2026-10-08 @@ -2280,7 +2309,8 @@ environment in `docker-compose.benchmark.yml`. --- -[Unreleased]: https://github.com/admina-org/admina/compare/v0.14.0...HEAD +[Unreleased]: https://github.com/admina-org/admina/compare/v0.14.1...HEAD +[0.14.1]: https://github.com/admina-org/admina/compare/v0.14.0...v0.14.1 [0.14.0]: https://github.com/admina-org/admina/compare/v0.13.2...v0.14.0 [0.13.2]: https://github.com/admina-org/admina/compare/v0.13.1...v0.13.2 [0.13.1]: https://github.com/admina-org/admina/compare/v0.13.0...v0.13.1 diff --git a/ROADMAP.md b/ROADMAP.md index 956e733..7a91144 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,7 +6,7 @@ not on a calendar. Scope may shift in response to user feedback, security findings, or upstream changes in the governance landscape (EU AI Act implementing acts, new frameworks, new attack classes). -The current release is **0.14.0**. Admina is pre-1.0: the public API may +The current release is **0.14.1**. Admina is pre-1.0: the public API may still evolve before the 1.0 stability commitment, so a minor release may carry a declared breaking change once its replacement is in place. Shipped detail lives in [CHANGELOG.md](CHANGELOG.md). diff --git a/admina/__init__.py b/admina/__init__.py index a9e5ab0..e97fd70 100644 --- a/admina/__init__.py +++ b/admina/__init__.py @@ -23,7 +23,7 @@ from admina.sdk import ComplianceKit, GovernedAgent, GovernedData, GovernedModel -__version__ = "0.14.0" +__version__ = "0.14.1" __all__ = [ "__version__", diff --git a/core-rust/Cargo.lock b/core-rust/Cargo.lock index 3ebecfe..8510fe0 100644 --- a/core-rust/Cargo.lock +++ b/core-rust/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "admina_core" -version = "0.14.0" +version = "0.14.1" dependencies = [ "chrono", "hex", diff --git a/core-rust/Cargo.toml b/core-rust/Cargo.toml index 17ec386..f01d726 100644 --- a/core-rust/Cargo.toml +++ b/core-rust/Cargo.toml @@ -15,7 +15,7 @@ [package] name = "admina_core" -version = "0.14.0" +version = "0.14.1" edition = "2021" description = "Admina governance engines — Rust core with Python bindings" license = "Apache-2.0" diff --git a/core-rust/pyproject.toml b/core-rust/pyproject.toml index bd984ff..a1502d4 100644 --- a/core-rust/pyproject.toml +++ b/core-rust/pyproject.toml @@ -19,7 +19,7 @@ build-backend = "maturin" [project] name = "admina-core" -version = "0.14.0" +version = "0.14.1" description = "Admina governance engines — Rust core with Python bindings (firewall, PII, loop breaker, hash chain)" readme = "README.md" requires-python = ">=3.11" diff --git a/core-rust/uv.lock b/core-rust/uv.lock index 9a206d9..7da6a84 100644 --- a/core-rust/uv.lock +++ b/core-rust/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.11" [[package]] name = "admina-core" -version = "0.14.0" +version = "0.14.1" source = { editable = "." } diff --git a/pyproject.toml b/pyproject.toml index d6d8864..10d6388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ build-backend = "setuptools.build_meta" [project] name = "admina" -version = "0.14.0" +version = "0.14.1" description = "Admina — governed AI development framework" readme = "README.md" requires-python = ">=3.11" diff --git a/uv.lock b/uv.lock index ce5d5a2..0d190fb 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ [[package]] name = "admina" -version = "0.14.0" +version = "0.14.1" source = { editable = "." } dependencies = [ { name = "click" }, @@ -199,7 +199,7 @@ dev = [ [[package]] name = "admina-core" -version = "0.14.0" +version = "0.14.1" source = { directory = "core-rust" } [[package]] From 80478e1cf067493364381a07cc2efeceea421b70 Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 14:50:50 +0200 Subject: [PATCH 6/7] fix(domains): recognise the PERSON spans that are not names by stop words is_name_like() rejects a span with a digit, a character of an e-mail address or a path (@ / \ _ # : =), or a lowercase word that is a stop word of English, Italian, German, French, Spanish or Portuguese (the lists of spaCy, without the words that are also first names: will, may, sara, mia, ali). Capitalised words are not checked. Names written in lowercase, which the English spaCy model labels as PERSON ("john smith", "giovanni esposito"), are masked again; phrases such as "il codice articolo", "ci vediamo domani" or "das Wetter" are not. --- admina/domains/data_sovereignty/names.py | 52 ++++++++++++++++++++---- tests/test_person_names.py | 48 +++++++++++++++++++--- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/admina/domains/data_sovereignty/names.py b/admina/domains/data_sovereignty/names.py index 24a1fc3..6e0c1e6 100644 --- a/admina/domains/data_sovereignty/names.py +++ b/admina/domains/data_sovereignty/names.py @@ -16,17 +16,55 @@ """Whether a PERSON entity of an NER model reads as a name. The PII engines mask a PERSON entity only when :func:`is_name_like` holds: -no digit, and at least one word with a capital initial. The English spaCy -model labels lowercase phrases of other languages as PERSON (Italian "il -codice articolo", "la pratica n. 2026/000457"); a name written all in -lowercase is not masked either. +no digit or character that a name does not hold (``@ / \\ _ # : =``, as in +an e-mail address or a path), and no lowercase word that is a stop word of English, Italian, +German, French, Spanish or Portuguese (the stop-word lists of spaCy, without +the words that are also first names, such as "will", "may" and "sara"). The +English spaCy model labels phrases as PERSON ("il codice articolo", "ci +vediamo domani", "das Wetter", "grab a coffee"); they hold such words. +Names written in lowercase pass ("mario rossi"), and capitalised words are +not checked ("Sara", "Il Signore"). """ from __future__ import annotations +import functools +import importlib + +# The stop-word lists of spaCy whose lowercase words reject a PERSON span. +_STOP_WORD_LANGUAGES = ("en", "it", "de", "fr", "es", "pt") + +# Characters of e-mail addresses, paths and identifiers, not of names. +_NOT_IN_NAMES = frozenset("@/\\_#:=") + +# Stripped from both ends of a word before the comparison. +_PUNCTUATION = ".,;:!?\"'()[]«»“”‘’" + +# Words of those lists that are also first names. +_FIRST_NAMES = frozenset({"ali", "may", "mia", "sara", "will"}) + + +@functools.cache +def _stop_words() -> frozenset[str]: + """The stop words of :data:`_STOP_WORD_LANGUAGES`, without + :data:`_FIRST_NAMES`; empty without spaCy (no NER runs then).""" + words: set[str] = set() + for lang in _STOP_WORD_LANGUAGES: + try: + module = importlib.import_module(f"spacy.lang.{lang}.stop_words") + except ImportError: + return frozenset() + words |= module.STOP_WORDS + return frozenset(words - _FIRST_NAMES) + def is_name_like(text: str) -> bool: - """True when *text* has no digit and a word that starts with a capital.""" - if any(ch.isdigit() for ch in text): + """True when *text* has a word, no digit nor character of + :data:`_NOT_IN_NAMES`, and no lowercase word that is a stop word of + English, Italian, German, French, Spanish or Portuguese.""" + words = text.split() + if not words or any(ch.isdigit() or ch in _NOT_IN_NAMES for ch in text): return False - return any(word[:1].isupper() for word in text.split()) + stop = _stop_words() + bare = (word.strip(_PUNCTUATION) for word in words) + return not any(word in stop for word in bare if word.islower()) diff --git a/tests/test_person_names.py b/tests/test_person_names.py index 7c19352..8679b52 100644 --- a/tests/test_person_names.py +++ b/tests/test_person_names.py @@ -14,11 +14,13 @@ """A PERSON entity of an NER model is masked only when it reads as a name: -no digit, and at least one word with a capital initial. +no digit, and no lowercase word that is a stop word of English, Italian, +German, French, Spanish or Portuguese. -The English spaCy model labels lowercase Italian phrases as PERSON ("il -codice articolo", "la pratica n. 2026/000457"); such spans are not masked, -by the PII redactor, the spaCy + regex engine and the Presidio engine. +The English spaCy model labels phrases as PERSON ("il codice articolo", +"ci vediamo domani", "das Wetter", "grab a coffee"); such spans are not +masked, by the PII redactor, the spaCy + regex engine and the Presidio +engine. Names written in lowercase are masked. """ from __future__ import annotations @@ -34,7 +36,19 @@ @pytest.mark.parametrize( "text", - ["Mario Rossi", "John Smith", "il signor Rossi", "O'Brien", "JOHN"], + [ + "Mario Rossi", + "mario rossi", + "john smith", + "giovanni esposito", + "O'Brien", + "JOHN", + "Sara Rossi", + "sara rossi", # "sara" is an Italian stop word, kept as a name + "Mia", + "will smith", # "will" is an English stop word, kept as a name + "Il Signore", # capitalised words are not checked + ], ) def test_names(text): assert is_name_like(text) @@ -42,7 +56,22 @@ def test_names(text): @pytest.mark.parametrize( "text", - ["il codice articolo", "la pratica n. 2026/000457", "John 3", "", " "], + [ + "il codice articolo", + "la pratica n. 2026/000457", + "ci vediamo domani", + "ci vediamo, domani", + "(il) progetto", + "ho parlato", + "das Wetter", + "il cliente", + "grab a coffee", + "email john.doe@example.org", + "Mario/Rossi", + "John 3", + "", + " ", + ], ) def test_not_names(text): assert not is_name_like(text) @@ -77,6 +106,13 @@ def test_the_redactor_masks_names_only(): assert out["redacted_text"] == "il codice articolo AB-12 è di [PERSON]" +def test_the_redactor_masks_lowercase_names(): + redactor = PIIRedactor() + redactor.nlp = _Nlp("il cliente", "giovanni esposito") + out = redactor.redact("il cliente è giovanni esposito") + assert out["redacted_text"] == "il cliente è [PERSON]" + + def test_the_spacy_regex_engine_reports_names_only(): from admina.plugins.builtin.pii.spacy_regex import SpaCyRegexPIIEngine From 14e6b82e0ae8482ff0804b29b2f5b46c57b5f46f Mon Sep 17 00:00:00 2001 From: Stefano Noferi Date: Thu, 8 Oct 2026 14:50:50 +0200 Subject: [PATCH 7/7] test(redteam): regenerate the baseline with the stop-word rule for PERSON PII false positives: Python 3/24, Rust 0/24, Presidio 9/24; Presidio type-level recall 0.586. MODEL_CARD and the [0.14.1] section of CHANGELOG describe the rule and the remaining false positives. --- CHANGELOG.md | 17 ++++++++++------- MODEL_CARD.md | 14 ++++++++------ admina/redteam/baselines/baseline.json | 4 ++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efc6b53..c14d3dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,12 +29,15 @@ behind each recall. Upgrading is recommended. without +39 or 0039, with the same boundaries). It reported an ISBN, a build number and other digit runs as `PHONE`. - The PII redactor, the `spacy-regex` engine and the Presidio engine mask a - `PERSON` entity of the NER model only when it has no digit and a word - with a capital initial. The English spaCy model labelled lowercase - phrases of other languages as `PERSON` ("il codice articolo", "la - pratica n. 2026/000457"). A name written all in lowercase is not masked - either. With Presidio, such a span no longer takes over an overlapping - entity of another type. + `PERSON` entity of the NER model only when it reads as a name: no digit, + no character of an e-mail address or a path (`@ / \ _ # : =`), and no + lowercase word that is a stop word of English, Italian, German, French, + Spanish or Portuguese (spaCy's lists, without the words that are also + first names: "will", "may", "sara", "mia", "ali"). The English spaCy + model labelled phrases such as "il codice articolo", "ci vediamo + domani" or "das Wetter" as `PERSON`. Names written in lowercase ("mario + rossi") are still masked. With Presidio, such a span no longer takes + over an overlapping entity of another type. ### Changed @@ -44,7 +47,7 @@ behind each recall. Upgrading is recommended. the labels of the corpora are assigned in this repository. - The PII corpus has eight hard negatives (`hard_negative`): numbers and codes that are not personal data. The baseline is regenerated with the - fixes above. PII false positives: Python 4/24, Rust 0/24, Presidio 8/24 + fixes above. PII false positives: Python 3/24, Rust 0/24, Presidio 9/24 (6/16, 0/16 and 9/16 in 0.14.0); Presidio type-level recall 59% (52% in 0.14.0). The remaining false positives are listed in MODEL_CARD §9. diff --git a/MODEL_CARD.md b/MODEL_CARD.md index 64b09b5..bb1eb2e 100644 --- a/MODEL_CARD.md +++ b/MODEL_CARD.md @@ -1116,12 +1116,12 @@ mode (the mode pinned in the baseline): | Detector | Python recall | Rust recall | False positives (py · rust) | |-----------|:---:|:---:|:---:| | injection | 57% (sample-level) | 35% | 0/27 · 0/27 | -| pii | 100% (type-level, nlp) | 66% (type-level) | 4/24 · 0/24 | +| pii | 100% (type-level, nlp) | 66% (type-level) | 3/24 · 0/24 | | loop | 82% (sample-level) | 91% | 0/11 · 0/11 | The optional Presidio PII engine is measured as a third row in the same baseline (`admina/redteam/baselines/baseline.json`): **59%** type-level -recall with **8/24** false positives, pinned to mode +recall with **9/24** false positives, pinned to mode `presidio:2.2.363/en+it`. It is an alternative engine, not an accelerator, so it is reported separately rather than in the Python-vs-Rust matrix above. @@ -1140,10 +1140,12 @@ version, an invoice number and a date, an ISBN, a build number, room and chapter numbers, two Italian reference codes). Since 0.14.1 the Rust scanner checks the Luhn sum of card numbers and matches the phone formats of the Python redactor, and the three engines mask a `PERSON` entity only when it -has no digit and a word with a capital initial: the remaining false -positives are `ORG`/`GPE` on non-English sentences, `PERSON` on German -sentences (German capitalises nouns), and, for Presidio, `PERSON` on -"Friday" and `PHONE` on an Italian reference number. These measured gaps +has no digit, no character of an e-mail address or a path, and no lowercase +stop word of English, Italian, German, French, Spanish or Portuguese (names +written in lowercase are masked): the remaining false positives are +`ORG`/`GPE` on non-English sentences, `PERSON` on a capitalised German noun +("Büro"), and, for Presidio, `PERSON` on "week" and "Friday" and `PHONE` on +an Italian reference number. These measured gaps are consistent with §1: the two engines are **not** behaviorally equivalent — Python is the higher-recall default, Rust the narrower-coverage opt-in. diff --git a/admina/redteam/baselines/baseline.json b/admina/redteam/baselines/baseline.json index 4da67d0..6593b97 100644 --- a/admina/redteam/baselines/baseline.json +++ b/admina/redteam/baselines/baseline.json @@ -14,7 +14,7 @@ "pii": { "python": { "type_recall": 1.0, - "fp": 4, + "fp": 3, "fp_samples": 24, "mode": "nlp:en_core_web_sm@3.8.0" }, @@ -25,7 +25,7 @@ }, "presidio": { "type_recall": 0.5862068965517241, - "fp": 8, + "fp": 9, "fp_samples": 24, "mode": "presidio:2.2.363/en+it" }