diff --git a/libs/openant-core/core/schemas.py b/libs/openant-core/core/schemas.py index 0bb83533..fa302229 100644 --- a/libs/openant-core/core/schemas.py +++ b/libs/openant-core/core/schemas.py @@ -436,7 +436,7 @@ def step_summary(self) -> dict: return verify_step_summary(self) def to_dict(self) -> dict: - return { + d = { "verified_results_path": self.verified_results_path, "findings_input": self.findings_input, "findings_verified": self.findings_verified, @@ -457,6 +457,13 @@ def to_dict(self) -> dict: "upgraded": self.upgraded, "usage": self.usage.to_dict(), } + # #653: the standalone-verify stdout envelope's source — carry the + # attacker_model present-only exactly like verify_step_summary + # (the shared construction, line 387): an omitted key drops the + # methodology line from the standalone lane the shared lane keeps. + if self.attacker_model: + d["attacker_model"] = self.attacker_model + return d # --------------------------------------------------------------------------- diff --git a/libs/openant-core/core/verifier.py b/libs/openant-core/core/verifier.py index 3940562b..291cfa24 100644 --- a/libs/openant-core/core/verifier.py +++ b/libs/openant-core/core/verifier.py @@ -58,7 +58,7 @@ def _verify_template_texts(): never enters the key); the remaining members are the module constants and frozen-fixture renders the None render cannot reach — the two non-None user-prompt personas, the three system-prompt context arms, and the - context-block and full-prompt renders over two frozen fixtures (the + context-block and full-prompt renders over four frozen fixtures (the fixture prompts also carry the ROUTING: a discriminator edit that re-routes a fixture re-pays verify, closing the "the persona existed but was never selected" half of #621 for future edits). diff --git a/libs/openant-core/prompts/verification_prompts.py b/libs/openant-core/prompts/verification_prompts.py index 9fbf4524..a0ffa84b 100644 --- a/libs/openant-core/prompts/verification_prompts.py +++ b/libs/openant-core/prompts/verification_prompts.py @@ -36,6 +36,14 @@ - Being the user who runs the application Then the vulnerability is NOT EXPLOITABLE by you, because local users can already do anything on their own machine.""" +PERSONA_REMOTE_ONLY_WEB = """You are an attacker on the internet. You have a browser and nothing else. +No server access, no admin credentials, no ability to modify files on the server, and NO ABILITY TO RUN CLI COMMANDS. + +You must find a way to trigger this vulnerability REMOTELY, through the web. If the only attack path requires: +- Operator-local access a browser user cannot reach +- Shell access to the server + +Then the vulnerability is NOT EXPLOITABLE by you.""" PERSONA_UNTRUSTED_INPUT = """You are an attacker who supplies the untrusted input this application processes. You can deliver crafted input through: {supply_list}. @@ -57,6 +65,11 @@ IMPORTANT: This is a CLI tool or library. The user running this code has local filesystem access. You must exploit this as a REMOTE attacker. If the only way to trigger the vulnerability is by running CLI commands locally, it is NOT exploitable - the user can already access the filesystem.""" +SYSTEM_ARM_REMOTE_ONLY_WEB = """ + +IMPORTANT: This is a web application. Its remote surface is the browser. +You must exploit this as a REMOTE attacker. If the only way to trigger the vulnerability is by +reaching operator-local access a browser user cannot reach, it is NOT exploitable in this class.""" SYSTEM_ARM_UNTRUSTED_INPUT = """ @@ -111,9 +124,35 @@ def _builtin_context_digest_renders() -> list[str]: trust_boundaries={"digest_source": "untrusted"}, requires_remote_trigger=True, ) + # #653: the third routing class — a degenerate web_app (all-trusted + # boundaries, no remote trigger): _is_untrusted_input_context is False + # (the web_app exclusion) and suppress_local_only is True, but the + # descriptor must NOT call it a CLI tool/library. Without this fixture, + # a routing change re-routing a web_app is invisible to the checkpoint + # fold (the exact #621 failure mode, one class wider). + web_app_fixture = ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "trusted", "http_headers": "trusted"}, + requires_remote_trigger=False, + ) + # #653 §1: the UNTRUSTED web app — the discriminator's own branch. + # The exclusion in _is_untrusted_input_context keeps it on the browser + # persona today; a routing edit that removes the exclusion re-routes it + # to the untrusted-input persona, and THAT edit is what the fold must + # catch (the all-trusted fixture above covers only the degenerate + # class — suppress_local_only fires before the discriminator). + web_app_untrusted_fixture = ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "untrusted"}, + requires_remote_trigger=True, + ) return [ _format_builtin_app_context_for_verification(suppress_fixture), _format_builtin_app_context_for_verification(untrusted_fixture), + _format_builtin_app_context_for_verification(web_app_fixture), + _format_builtin_app_context_for_verification(web_app_untrusted_fixture), ] @@ -137,6 +176,28 @@ def _builtin_persona_digest_renders() -> list[str]: trust_boundaries={"digest_source": "untrusted"}, requires_remote_trigger=True, ) + # #653: the web_app fixture joins the persona renders (three routing + # classes, not two) — the routing-coverage half of the digest fold. + # The web_app SYSTEM ARM + PERSONA render in the same fold: an inline + # literal at the system-prompt site was invisible to templates_sha + # (the T1 round-2 mutation finding — the exact #621 failure mode). + from context.application_context import ApplicationContext + web_app_fixture = ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "trusted", "http_headers": "trusted"}, + requires_remote_trigger=False, + ) + # #653 §1: the untrusted web app joins the persona renders too — the + # routing edit this fixture exists to catch changes WHICH persona the + # class renders, so the user-prompt render must be in the fold for + # the re-route to move the digest. + web_app_untrusted_fixture = ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "untrusted"}, + requires_remote_trigger=True, + ) return [ get_verification_prompt( code="", finding="", attack_vector="", reasoning="", @@ -144,6 +205,15 @@ def _builtin_persona_digest_renders() -> list[str]: get_verification_prompt( code="", finding="", attack_vector="", reasoning="", app_context=untrusted_fixture), + get_verification_prompt( + code="", finding="", attack_vector="", reasoning="", + app_context=web_app_fixture), + get_verification_prompt( + code="", finding="", attack_vector="", reasoning="", + app_context=web_app_untrusted_fixture), + # the web_app system arm + persona join the fold so a wording + # change re-pays verify (the #621 contract, extended to the arm). + get_verification_system_prompt(web_app_fixture), ] @@ -184,12 +254,23 @@ def attacker_model_descriptor(app_context: "ApplicationContext") -> dict: "admin credentials, no CLI access."), } if app_context is not None and app_context.suppress_local_only(): + # #653: the "local access is the operator's own" framing is right + # for a CLI tool/library whose inputs are operator-controlled — a + # web_app reaching this branch (all-trusted boundaries, no remote + # trigger) is DEGENERATE: its remote surface is the browser, not + # an operator's local access, and calling it a CLI tool mis-states + # the methodology on every degenerate web_app scan. + is_web = str(getattr(app_context, "application_type", "")) == "web_app" return { "kind": "remote_only", "attacker": ( "Remote attacker with browser access, no server-side " - "access, no admin credentials; this CLI tool/library's " - "local access is the operator's own."), + "access, no admin credentials" + + (" — this web application's remote surface is the " + "browser; no operator-local access applies." + if is_web else + "; this CLI tool/library's local access is the " + "operator's own.")), } return { "kind": "browser_only", @@ -219,7 +300,13 @@ def get_verification_system_prompt(app_context: "ApplicationContext" = None) -> if app_context and app_context.has_threat_model(): base_prompt += SYSTEM_ARM_THREAT_MODEL elif app_context and app_context.suppress_local_only(): - base_prompt += SYSTEM_ARM_REMOTE_ONLY + if str(getattr(app_context, "application_type", "")) == "web_app": + # #653: the web_app arm — module-level constant so the + # checkpoint fold can hash it (an inline literal is invisible + # to templates_sha; the #621 failure mode, one class wider). + base_prompt += SYSTEM_ARM_REMOTE_ONLY_WEB + else: + base_prompt += SYSTEM_ARM_REMOTE_ONLY elif _is_untrusted_input_context(app_context): # #621: the system prompt mirrors the user prompt's persona lattice # (a supply-persona user prompt under a generic-attacker system @@ -284,9 +371,19 @@ def _format_builtin_app_context_for_verification(app_context: "ApplicationContex lines.append("") if app_context.suppress_local_only(): - lines.append("**CRITICAL:** This is a CLI tool/library. Users have local filesystem access.") - lines.append("A vulnerability requires a REMOTE attacker to exploit it.") - lines.append("If the 'attack' requires running CLI commands locally, it's NOT a vulnerability.") + # #653: the degenerate web_app (all-trusted boundaries, no remote + # trigger) reaches this suppress branch too — but its framing is + # the browser, not the operator's local filesystem. The digest + # moves in the same PR (the third fixture), so the #621 + # keep-the-text-stable rationale does not hold it here. + if str(getattr(app_context, "application_type", "")) == "web_app": + lines.append("**CRITICAL:** This is a web application. Its remote surface is the browser.") + lines.append("A vulnerability requires a REMOTE attacker to exploit it.") + lines.append("If the 'attack' requires operator-local access that a browser user cannot reach, it is out of scope.") + else: + lines.append("**CRITICAL:** This is a CLI tool/library. Users have local filesystem access.") + lines.append("A vulnerability requires a REMOTE attacker to exploit it.") + lines.append("If the 'attack' requires running CLI commands locally, it's NOT a vulnerability.") lines.append("") return "\n".join(lines) @@ -373,8 +470,14 @@ def get_verification_prompt( # pre-#621 render (verify's checkpoint identity hashes this arm). attacker_description = PERSONA_BROWSER_ONLY elif app_context.suppress_local_only(): - # All-trusted CLI/library: byte-identical to the pre-#621 render. - attacker_description = PERSONA_REMOTE_ONLY + # All-trusted CLI/library: byte-identical to the pre-#621 render + # (#653: a web_app reaching this branch gets the web persona — + # the CLI rationale's "being the user who runs the application" + # is false for a web app). + if str(getattr(app_context, "application_type", "")) == "web_app": + attacker_description = PERSONA_REMOTE_ONLY_WEB + else: + attacker_description = PERSONA_REMOTE_ONLY elif _is_untrusted_input_context(app_context): # #621: the untrusted-input class — a parser/CLI/library whose attack # surface IS the attacker-supplied input. The browser-only persona @@ -400,10 +503,15 @@ def get_verification_prompt( local_access_rule = ( "" if (app_context is not None and app_context.has_threat_model()) - else ("\n- If this is a CLI tool/library and the attack requires " - "local access, it is NOT a vulnerability." - if (app_context is None or app_context.suppress_local_only()) - else "") + else ("\n- If this is a web application and the attack requires " + "operator-local access a browser user cannot reach, it is NOT a vulnerability." + if (app_context is not None + and app_context.suppress_local_only() + and str(getattr(app_context, "application_type", "")) == "web_app") + else ("\n- If this is a CLI tool/library and the attack requires " + "local access, it is NOT a vulnerability." + if (app_context is None or app_context.suppress_local_only()) + else "")) ) # `reasoning` is Stage-1 LLM output (untrusted). It was interpolated raw diff --git a/libs/openant-core/tests/test_issue621_verification_persona.py b/libs/openant-core/tests/test_issue621_verification_persona.py index bb8e160c..a5e4c1bf 100644 --- a/libs/openant-core/tests/test_issue621_verification_persona.py +++ b/libs/openant-core/tests/test_issue621_verification_persona.py @@ -423,8 +423,8 @@ def test_verify_template_digest_includes_persona_constants(): # stops moving, so this row is the guard for the fold itself (fable r2 F1). from prompts.verification_prompts import ( # noqa: PLC0415 _builtin_context_digest_renders, _builtin_persona_digest_renders) - assert len(_builtin_context_digest_renders()) == 2 - assert len(_builtin_persona_digest_renders()) == 2 + assert len(_builtin_context_digest_renders()) == 4 # #653: web_app routing classes join the fold (trusted + untrusted) + assert len(_builtin_persona_digest_renders()) == 5 # #653: 4 user prompts (incl. the untrusted web app) + the web_app system arm original_renders = vp._builtin_context_digest_renders try: diff --git a/libs/openant-core/tests/test_issue653_verify_followups.py b/libs/openant-core/tests/test_issue653_verify_followups.py new file mode 100644 index 00000000..099c3a00 --- /dev/null +++ b/libs/openant-core/tests/test_issue653_verify_followups.py @@ -0,0 +1,275 @@ +"""Regression tests for issue #653 — the three live PR-#652 follow-ups. + +Item 1 — the third digest fixture (web_app routing class): #652's digest +helpers cover cli_tool/library (the suppress branch and the untrusted branch). +A web_app with NO untrusted boundaries is the third routing class: its +``_is_untrusted_input_context`` is False (the web_app exclusion) and its +``suppress_local_only`` is True — a degenerate web_app (no untrusted +boundaries) that the descriptor must not call a CLI tool/library. + +Item 2 — the standalone-verify stdout lane drops ``attacker_model``: +``verify_step_summary`` (the shared construction) carries it present-only +(schemas.py:387); ``VerifyResult.to_dict`` (the standalone stdout envelope's +source, cli.py's success()) does not — the reader loses the methodology line. + +Item 3 — the degenerate web-app class: a web_app with all-trusted boundaries +falls to the ``remote_only`` descriptor whose text calls it "this CLI +tool/library" — wrong for a web application. The boundary condition matters: +``requires_remote_trigger=False`` alone does NOT make the suppress/descriptor +class right (a web_app's remote surface is the browser, not the operator's +own local access). +""" +from __future__ import annotations + +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from context.application_context import ApplicationContext +from prompts.verification_prompts import ( + _builtin_context_digest_renders, + _builtin_persona_digest_renders, + attacker_model_descriptor, +) + + +def _degenerate_web_app() -> ApplicationContext: + """The third routing class: a web_app whose boundaries are all trusted.""" + return ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "trusted", "http_headers": "trusted"}, + requires_remote_trigger=False, + ) + + +class TestItem1ThirdDigestFixture: + def test_web_app_digest_fixture_renders_at_base(self): + """RED at base: neither digest helper covers the web_app class.""" + renders = _builtin_context_digest_renders() + assert any("web_app" in r or "web application" in r for r in renders), ( + "the third digest fixture (web_app routing class) must render — " + "a routing change re-routing a web_app is invisible to verify's " + "checkpoint fold without it") + + def test_web_app_persona_digest_renders_at_base(self): + renders = _builtin_persona_digest_renders() + # the web_app fixture's context block renders the literal type + # (the enum value is the routing discriminator) + assert any("web_app" in r for r in renders), ( + "the persona digest must cover the web_app routing class") + + +class TestItem2StdoutAttackerModel: + def _minimal_verify_result(self, **overrides): + """A minimally-constructed VerifyResult (the dataclass fields the + to_dict path reads; attacker_model defaults None).""" + from core.schemas import VerifyResult + defaults = dict( + verified_results_path="/tmp/verified.json", + findings_input=0, + findings_verified=0, + agreed=0, + disagreed=0, + disagreed_inconclusive=0, + disagreed_protected=0, + confirmed_vulnerabilities=0, + needs_review=0, + error_count=0, + units_analyzed_total=0, + ) + defaults.update(overrides) + return VerifyResult(**defaults) + + def test_to_dict_carries_attacker_model(self): + """BEHAVIORAL (the T1 round-1 fix: the source-inspection form was + vacuous — satisfied by the comment alone): a stamped descriptor + rides the standalone stdout envelope; the unstamped case omits + the key (present-only, matching verify_step_summary's truthiness).""" + stamped = self._minimal_verify_result( + attacker_model={"kind": "remote_only", "attacker": "..."}) + assert stamped.to_dict()["attacker_model"] == { + "kind": "remote_only", "attacker": "..."}, ( + "a stamped attacker_model must ride to_dict — the standalone " + "stdout envelope's methodology line") + + def test_to_dict_omits_attacker_model_when_absent(self): + unstamped = self._minimal_verify_result() # attacker_model=None + assert "attacker_model" not in unstamped.to_dict(), ( + "present-only: the unstamped envelope carries no fabricated key") + + def test_to_dict_truthiness_matches_step_summary(self): + """F3: to_dict's presence check uses the same truthiness as + verify_step_summary ({} → omitted, not carried as a fabricated + empty dict).""" + empty_stamped = self._minimal_verify_result(attacker_model={}) + assert "attacker_model" not in empty_stamped.to_dict() + + +class TestItem3DegenerateWebAppDescriptor: + def test_web_app_descriptor_is_not_a_cli_tool(self): + """RED at base: the degenerate web_app's descriptor calls it a CLI + tool/library — wrong for a web application.""" + descriptor = attacker_model_descriptor(_degenerate_web_app()) + assert "CLI" not in descriptor["attacker"], ( + f"a web_app's attacker model must not call it a CLI tool/library; " + f"got: {descriptor['attacker']!r}") + + +def test_four_digest_fixtures_not_three(): + """The digest helpers carry FOUR routing classes after the #653 §1 + follow-up (was: three — the untrusted-web class joined #670's set).""" + assert len(_builtin_context_digest_renders()) == 4 + assert len(_builtin_persona_digest_renders()) == 5 # 4 user prompts + the web_app system arm + + +class TestPromptTextsBehavioral: + """T1 round-2 N2: the three web_app prompt-text branches are PINNED + behaviorally (the round-1 disease one level up — presence-only tests + passed with the branches deleted).""" + + def _web(self): + return ApplicationContext( + application_type="web_app", purpose="digest fixture", + trust_boundaries={"http_body": "trusted"}, + requires_remote_trigger=False) + + def _cli(self): + return ApplicationContext( + application_type="cli_tool", purpose="digest fixture", + trust_boundaries={"x": "trusted"}, + requires_remote_trigger=False) + + def test_web_app_user_prompt_says_web_app(self): + from prompts.verification_prompts import get_verification_prompt + r = get_verification_prompt( + code="", finding="", attack_vector="", reasoning="", + app_context=self._web()) + assert "web application" in r.lower(), "the CRITICAL block must say web application" + assert "CLI tool" not in r, "the web_app render must not carry the CLI-tool framing" + + def test_web_app_system_prompt_says_web_app(self): + from prompts.verification_prompts import ( + get_verification_system_prompt, SYSTEM_ARM_REMOTE_ONLY_WEB) + s = get_verification_system_prompt(self._web()) + assert "web application" in s.lower(), "the system arm must say web application" + assert "CLI tool" not in s, "the web_app system arm must not carry the CLI framing" + assert SYSTEM_ARM_REMOTE_ONLY_WEB in s, "the hoisted constant is the arm actually served" + + def test_web_app_persona_is_web(self): + from prompts.verification_prompts import PERSONA_REMOTE_ONLY_WEB, get_verification_prompt + r = get_verification_prompt( + code="", finding="", attack_vector="", reasoning="", + app_context=self._web()) + assert PERSONA_REMOTE_ONLY_WEB in r, "the web_app persona must be the web constant" + assert "being the user who runs the application" not in r.lower(), ( + "the CLI persona's 'user who runs the application' rationale is " + "false for a web app") + + def test_cli_tool_renders_unchanged(self): + from prompts.verification_prompts import ( + get_verification_prompt, get_verification_system_prompt, + PERSONA_REMOTE_ONLY, SYSTEM_ARM_REMOTE_ONLY) + user = get_verification_prompt( + code="", finding="", attack_vector="", reasoning="", + app_context=self._cli()) + system = get_verification_system_prompt(self._cli()) + assert "CLI tool" in user, "the cli_tool render keeps the CLI framing" + assert SYSTEM_ARM_REMOTE_ONLY in system, "the cli_tool system arm keeps the CLI constant" + assert PERSONA_REMOTE_ONLY in user, "the cli_tool persona keeps the CLI constant" + assert "web application" not in user.lower(), "the cli_tool render gains no web framing" + + def test_web_app_system_arm_in_fold(self): + """N1's pin (mutation-hardened after round 3's N4: the presence-only + form passed with the arm's render deleted): mutating the web_app + arm's CONSTANT moves the folded texts; mutating the web_app PERSONA + moves them too.""" + from prompts import verification_prompts as vp + from core.verifier import _verify_template_texts + base = [r() for r in _verify_template_texts()] + + arm = vp.SYSTEM_ARM_REMOTE_ONLY_WEB + try: + vp.SYSTEM_ARM_REMOTE_ONLY_WEB = arm + "\n# mutated" + mutated = [r() for r in _verify_template_texts()] + finally: + vp.SYSTEM_ARM_REMOTE_ONLY_WEB = arm + assert mutated != base, ( + "mutating the web_app system arm must move the fold — an arm " + "invisible to templates_sha is the #621 failure mode") + + persona = vp.PERSONA_REMOTE_ONLY_WEB + try: + vp.PERSONA_REMOTE_ONLY_WEB = persona + "\n# mutated" + mutated2 = [r() for r in _verify_template_texts()] + finally: + vp.PERSONA_REMOTE_ONLY_WEB = persona + assert mutated2 != base, ( + "mutating the web_app persona must move the fold") + + # the arm text itself appears in the renders (not satisfied by the + # user-prompt render at index 2 — the round-3 escape) + from prompts.verification_prompts import ( + _builtin_persona_digest_renders, SYSTEM_ARM_REMOTE_ONLY_WEB, + PERSONA_REMOTE_ONLY_WEB) + renders = _builtin_persona_digest_renders() + assert any(SYSTEM_ARM_REMOTE_ONLY_WEB in r for r in renders), ( + "the system arm's own text is in the fold renders") + assert any(PERSONA_REMOTE_ONLY_WEB in r for r in renders), ( + "the web persona's own text is in the fold renders") + # the web_app system-prompt render is in the fold: the digest + # moved from the 2-fixture era (the third fixture + the arm join + # in the same fold — the #621 contract extended) + from prompts.verification_prompts import _builtin_persona_digest_renders + renders = _builtin_persona_digest_renders() + assert len(renders) == 5, ( + "the persona renders: 4 user prompts (incl. the untrusted web " + "app) + the web_app system prompt (the fold-visible arm)") + system_in_renders = any("web application" in r.lower() for r in renders) + assert system_in_renders, "the web_app system arm is fold-visible" + + +def test_untrusted_web_app_routing_moves_the_digest(): + """The #653 §1 class: a routing edit that re-routes an UNTRUSTED web + app (the web_app exclusion in _is_untrusted_input_context) must move + templates_sha — resumed scans must not adopt verdicts produced under + the superseded routing. The all-trusted fixture covers the degenerate + class only; this is the discriminator's own branch.""" + from context.application_context import ApplicationContext + from core.backend_identity import templates_digest + from core.verifier import _verify_template_texts + from prompts.verification_prompts import _is_untrusted_input_context + + untrusted_web = ApplicationContext( + application_type="web_app", + purpose="digest fixture", + trust_boundaries={"http_body": "untrusted"}, + requires_remote_trigger=True, + ) + # the live routing today: the web_app exclusion keeps an untrusted + # web app on the browser persona + assert _is_untrusted_input_context(untrusted_web) is False + base = templates_digest([r() for r in _verify_template_texts()]) + + original = _is_untrusted_input_context + + def _mutant(ctx): + # remove the web_app exclusion — the untrusted web app re-routes + # to the untrusted-input persona + return bool(ctx is not None and ctx.untrusted_boundaries()) + + from prompts import verification_prompts as vp + vp._is_untrusted_input_context = _mutant + try: + assert _mutant(untrusted_web) is True, ( + "the mutant really re-routes the untrusted web app") + mutated = templates_digest([r() for r in _verify_template_texts()]) + finally: + vp._is_untrusted_input_context = original + assert mutated != base, ( + "a routing edit that changes an untrusted web app's live persona " + "must move templates_sha (the #653 §1 guard)" + )