diff --git a/ROADMAP.md b/ROADMAP.md index 669f7c9..50e1057 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -368,6 +368,13 @@ this project exists for. Measure that before choosing. Whether OTX exposes anyth The discount exists because malware contacting a DNS resolver fills OTX with noise, and OTX's whitelist is a fair sign of that. The same rule lets an attacker on that whitelist drop from Malicious to Suspicious with no veneer. +### 3.10 Padded pulses hiding OTX-only evidence · S + +- [ ] **PDX-40: Padding can't hide OTX-only evidence silently.** Done when: the dilution test's numbers change or the report plainly says the only evidence was discounted as bulk, and a rescore reports the effect on the benign IPs that discount was built for. + +The adversarial suite shows 5 pulses padded to 20,000 indicators reading Likely benign when no other beacon agrees. The +discount clears sandbox dumps that list DNS resolvers and CDNs, so removing it would bring those false alarms back. + --- ## Phase 4: The detector as a target (AI-security track) @@ -579,7 +586,7 @@ and fix the top issues. Repeat after 5.6. It also doubles as class material. | When | Items | Why first | |---|---|---| | Now | ~~1.1, 1.2, 1.5, 3.2, 5.1~~ (done) | Small, keyless, and fix the biggest measured weakness (the scanner rule's dependence on GreyNoise); 5.1 makes that dependence visible | -| Next | 1.3, 3.1, 3.6, 3.8, 3.9, 4.1, ~~3.5, 4.6, 4.7, 4.8, 4.9, 4.10, 4.5, 1.7, 1.8, 3.7, 1.4, 1.6, 5.2, 5.4~~ (done) | Close the known false positives, build adversarial and hiding-place test sets, test the AI summarizer under attack, and show watch state and claim checks in the UI | +| Next | 1.3, 3.1, 3.6, 3.8, 3.9, 3.10, 4.1, ~~3.5, 4.6, 4.7, 4.8, 4.9, 4.10, 4.5, 1.7, 1.8, 3.7, 1.4, 1.6, 5.2, 5.4~~ (done) | Close the known false positives, build adversarial and hiding-place test sets, test the AI summarizer under attack, and show watch state and claim checks in the UI | | Then | 2.1, 2.3, 3.3, 3.4, 4.2, ~~4.3, 4.4~~ (done), 5.3, 5.5–5.7 | New unmasking signals plus self-measurement once history has accumulated, with the UI restructured to hold them | | Later | 2.2, 2.4, 2.5 | Bigger extensions: domains, campaigns, more beacons | diff --git a/SECURITY.md b/SECURITY.md index 6488f95..3f6eac9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -63,7 +63,7 @@ forge a reverse DNS name. - `tests/test_adversarial.py` pins how far each attack moves a score. With independent beacons, laundering, padding and a forged PTR don't clear a malicious IP. But when OTX is the only witness, [5 new accounts can take a clean IP to Likely malicious](ROADMAP.md#38-smearing-by-new-otx-accounts--m), [OTX's own whitelist can clear real - evidence](ROADMAP.md#39-otxs-whitelist-clearing-its-own-evidence--s), and padding every pulse hides it. + evidence](ROADMAP.md#39-otxs-whitelist-clearing-its-own-evidence--s), and [padding every pulse hides it](ROADMAP.md#310-padded-pulses-hiding-otx-only-evidence--s). - Those tests use synthetic fixtures and attacks we thought of, so they don't show what a real attacker can do. - One-vendor scaling has a cost: an attacker known to exactly one vendor and nothing else can read as Likely benign. - Independence between VirusTotal's vendors can't be shown, because their sources are closed. diff --git a/tests/test_adversarial.py b/tests/test_adversarial.py index 7476384..824d560 100644 --- a/tests/test_adversarial.py +++ b/tests/test_adversarial.py @@ -125,6 +125,22 @@ def test_guarantee_otx_whitelist_doesnt_clear_other_beacons(self): self.assertScore(result, 93.1, 0.765) self.assertTrue(result["veneer"]["active"]) + def test_guarantee_a_label_isnt_contradicted_by_its_own_origin(self): + """Feodo and ThreatFox are both abuse.ch: one source disagreeing with itself isn't independent evidence.""" + label = Signal("feodo", "Feodo", "fast", threat=0.0, confidence=0.9, facts={"benign_label": "abuse.ch vouched"}) + hit = Signal("threatfox", "ThreatFox", "fast", threat=0.9, confidence=0.9, last_seen=RECENT) + self.assertFalse(score(otx(), label, hit)["veneer"]["active"]) + outside = Signal("virustotal", "VirusTotal", "slow", threat=0.9, confidence=0.8, last_seen=RECENT) + self.assertTrue(score(otx(), label, virustotal=outside)["veneer"]["active"]) + + def test_guarantee_stale_evidence_cant_contradict_a_label(self): + """An old report shouldn't paint a label as borrowed; only fresh or live evidence can.""" + old = (NOW - timedelta(days=200)).date().isoformat() + hit = Signal("virustotal", "VirusTotal", "slow", threat=0.9, confidence=0.8, last_seen=old) + self.assertFalse(score(otx(), greynoise=benign_greynoise(), virustotal=hit)["veneer"]["active"]) + hit = Signal("virustotal", "VirusTotal", "slow", threat=0.9, confidence=0.8, last_seen=RECENT) + self.assertTrue(score(otx(), greynoise=benign_greynoise(), virustotal=hit)["veneer"]["active"]) + def test_known_weakness_otx_whitelist_clears_otx_only_evidence(self): """PDX-39. When OTX is the only witness, its own whitelist entry drops it from Severe paradox and Malicious to Coherent and Suspicious, and no veneer fires, because a label can't be contradicted by its own origin.""" @@ -156,8 +172,8 @@ def test_guarantee_padding_cant_clear_independent_evidence(self): self.assertEqual(result["threat_label"], "Malicious", size) def test_known_weakness_padding_hides_otx_only_evidence(self): - """PDX-38's other edge. If every pulse naming an IP is huge, OTX-only evidence fades to Likely benign. That's - the discount working on sandbox dumps, and also what a bad actor's own padded feed would get.""" + """PDX-40. If every pulse naming an IP is huge, OTX-only evidence fades to Likely benign. That's the discount + working on sandbox dumps, and also what an attacker gets if the only reports are ones it padded.""" for size, paradox, threat, threat_label in ((1_000, 60.3, 0.689, "Likely malicious"), (3_000, 38.6, 0.441, "Likely malicious"), (20_000, 12.1, 0.138, "Likely benign"),