From fc9295124afe9c4416bdde02c0f3649ae22e620d Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 17:24:52 -0400 Subject: [PATCH 01/14] =?UTF-8?q?feat(eventing):=20Phase=203=20step=201=20?= =?UTF-8?q?=E2=80=94=20tenancy=20keys=20and=20declarative=20agent=20specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN_PHASE3.md T1 and T13, the two tasks with no dependencies. Nothing here changes behaviour: `single` tenancy is the default everywhere and the `default` AgentSpec reproduces Phase 2's argv byte for byte. shared/tenancy.py (T1) `userkey(issuer, userid)` is the only place an identity becomes a name. The slug is deliberately lossy so keys stay readable in `kubectl get kafkatopics`; that is safe only because 32 bits of SHA-256 over the canonical form are appended, so two identities that slug identically still differ. The issuer is inside the hashed bytes as well as the prefix — without it, configuring a second issuer is a way to impersonate an identity on the first. The digest input is joined with \x1f so the issuer/userid boundary is unambiguous. Canonicalisation is per-issuer and minimal: GitHub folds case (and must agree with `ghauth.is_allowed`, which a test pins), email lower-cases only the domain per RFC 5321, static is byte-exact. Erring toward two tenants for one human is wasteful; the other direction is a leak. `TopicSet` is the one place §3.1's layout is written down, and refuses to name a topic without a userkey in multi mode rather than falling back to a shared one. `ntfy_topic` is HMAC-derived so the name carries no identifier (§4.1). eventrunner/agentspec.py (T13) One TOML file per agent, carrying the tool policy that §7.4 depends on — which is why §9 puts this before triggers. Validation refuses rather than coerces: a policy that silently drops the clause it could not parse would hand a trigger-driven agent the tool the operator meant to remove. A missing non-default agent is an error event, never a silent fallback to `default`. `build_cmd` takes a spec. The request still wins on `max_turns`/`model` (Phase 0 wire parameters), but the tool policy, permission mode and system prompt are spec-only: a request that could widen the sandbox is a request that can escape it. Tests: 105 new, 656 total passing, no existing test modified. The argv-equality test compares against a literal transcription of Phase 2's `build_cmd` rather than a call into the current one, so it catches a regression instead of following one. The `claude --help` flag assertion covers only the flags Phase 3 adds. `--max-turns` is supported but undocumented in `--help` on 2.1.270, so asserting the pre-existing flags would fail against a working deployment. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventrunner/agentspec.py | 290 +++++++++++++++++++++++ eventing/eventrunner/config.py | 32 +++ eventing/eventrunner/runner.py | 69 +++++- eventing/shared/ce.py | 19 ++ eventing/shared/tenancy.py | 250 ++++++++++++++++++++ eventing/tests/test_agentspec.py | 376 ++++++++++++++++++++++++++++++ eventing/tests/test_tenancy.py | 282 ++++++++++++++++++++++ 7 files changed, 1313 insertions(+), 5 deletions(-) create mode 100644 eventing/eventrunner/agentspec.py create mode 100644 eventing/shared/tenancy.py create mode 100644 eventing/tests/test_agentspec.py create mode 100644 eventing/tests/test_tenancy.py diff --git a/eventing/eventrunner/agentspec.py b/eventing/eventrunner/agentspec.py new file mode 100644 index 00000000..05e103bf --- /dev/null +++ b/eventing/eventrunner/agentspec.py @@ -0,0 +1,290 @@ +"""Declarative agent definitions. DESIGN_PHASE3.md §5.1, §5.2, §5.5. + +Up to Phase 2 "the agent" was five flags in `build_cmd()`: there was no notion of +*which* agent, no system prompt, and no tool policy. A trigger (§7) has to answer "run +**what**", so that notion has to exist first — which is why §9 puts this before triggers +rather than after. + +One TOML file per agent, matching the `config.toml` precedent rather than introducing a +second config language: + + /etc/rossoctl/agents/ + default/agent.toml + triager/ + agent.toml + system.md + skills/issue-triage/SKILL.md + +**The tool policy IS the sandbox.** A trigger-supplied prompt is attacker-influenced +text (§7.4) and the tools are what turn text into consequences, so a spec written for +trigger use is default-deny. The loaded `default` spec is deliberately NOT: an +unspecified field inherits Phase 2's behaviour, so `build_cmd` emits byte-identical argv +and the existing e2e tests keep passing. That is what makes this additive, and +`tests/test_agentspec.py` pins the argv equality rather than trusting the claim. + +Stdlib only (`tomllib`, `pathlib`), per Phase 1 §1.1. +""" +from __future__ import annotations + +import pathlib +import re +import tomllib +from dataclasses import dataclass, field + +# An agent name becomes a path component under ER_AGENT_DIR, so it is validated as a +# DNS-1123 label rather than merely non-empty. That bounds it to characters that cannot +# traverse (`.` and `/` are both excluded) and keeps it usable as a Kubernetes object +# name later without a migration — the same reasoning §7.6 applies to trigger names. +NAME_RE = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +NAME_MAX = 63 + +# Phase 2's argv used `acceptEdits` unconditionally. It stays the default here so the +# `default` spec reproduces it; a spec intended for a trigger should say `plan`. +DEFAULT_PERMISSION_MODE = "acceptEdits" +PERMISSION_MODES = ("plan", "acceptEdits", "bypassPermissions") + +# Phase 0's wire default, applied when neither the request payload nor the spec says. +DEFAULT_MAX_TURNS = 3 + +SPEC_FILENAME = "agent.toml" + + +class SpecError(Exception): + """A spec that cannot be loaded or is not valid. + + Raised rather than returning a default, because §5.1 is explicit: a request naming + an agent the runner does not have is an error event, never a silent fallback to + `default`. Running a different agent than the one asked for is worse than not + running — the caller believes a policy applied that did not. + """ + + +@dataclass(frozen=True) +class Skill: + """One entry from `[[skills]]`. + + Phase 3 step 9 loads these but fetches nothing: only `image://` sources resolve, + because they are already present. `sha256` and `sig_kid` are carried so a spec + written today stays valid when T19 turns fetching on, and are validated here so a + bundle reference that could never verify fails at load rather than at fetch. + """ + name: str + source: str + sha256: str = "" + sig_kid: str = "" + + @property + def is_baked(self) -> bool: + return self.source.startswith("image://") + + +@dataclass(frozen=True) +class Limits: + timeout_s: float = 0.0 # 0 = no spec-imposed deadline + max_output_bytes: int = 0 # 0 = unbounded by the spec + max_events: int = 2000 # §5.6 response events per run + + +@dataclass(frozen=True) +class AgentSpec: + """A loaded agent definition. + + Every field defaults to the Phase 2 behaviour, so `AgentSpec(name="default")` with + no file is exactly what the runner did before this module existed. + """ + name: str = "default" + description: str = "" + model: str = "" + max_turns: int = DEFAULT_MAX_TURNS + permission_mode: str = DEFAULT_PERMISSION_MODE + allowed_tools: tuple[str, ...] = () + disallowed_tools: tuple[str, ...] = () + # Resolved to an absolute path at load time and read at build time, because + # `--append-system-prompt` takes the prompt text, not a path. + system_prompt: str = "" + settings_path: str = "" + mcp_config_path: str = "" + skills: tuple[Skill, ...] = () + limits: Limits = field(default_factory=Limits) + + @property + def is_default(self) -> bool: + """True when this spec imposes nothing beyond Phase 2's argv.""" + return (not self.model and not self.allowed_tools and not self.disallowed_tools + and not self.system_prompt and not self.settings_path + and not self.mcp_config_path + and self.permission_mode == DEFAULT_PERMISSION_MODE) + + +def _as_tuple(raw, field_name: str) -> tuple[str, ...]: + if raw is None: + return () + if not isinstance(raw, list) or not all(isinstance(x, str) for x in raw): + raise SpecError(f"{field_name} must be a list of strings") + return tuple(x.strip() for x in raw if x.strip()) + + +def _resolve_under(base: pathlib.Path, rel: str, field_name: str) -> str: + """Resolve a spec-relative path, refusing anything that escapes the agent dir. + + A spec is operator-supplied so this is not the primary trust boundary, but + `../../../etc/shadow` in `append_system_prompt_file` would read an arbitrary file + into the prompt — and once T19 allows a *fetched* bundle to carry an `agent.toml`, + that same field is attacker-supplied. Enforcing containment now means the gate + already exists when the threat model changes, which is cheaper than remembering to + add it then. + """ + base = base.resolve() + target = (base / rel).resolve() + if target != base and base not in target.parents: + raise SpecError(f"{field_name} {rel!r} escapes the agent directory") + if not target.is_file(): + raise SpecError(f"{field_name} {rel!r} does not exist") + return str(target) + + +def validate_name(name: str) -> str: + if not name or len(name) > NAME_MAX or not NAME_RE.match(name): + raise SpecError( + f"agent name {name!r} must be a DNS-1123 label of at most {NAME_MAX} chars") + return name + + +def parse(raw: str, *, name: str, base_dir: pathlib.Path | None = None) -> AgentSpec: + """Parse one `agent.toml`. `base_dir` resolves the file-valued fields. + + Validation is strict and refuses rather than coercing: a spec is a *policy*, and a + policy that silently drops the clause it could not understand is worse than one that + fails to load. An unparseable `disallowed_tools` would otherwise hand a + trigger-driven agent the Bash tool the operator meant to remove. + """ + validate_name(name) + try: + d = tomllib.loads(raw) + except tomllib.TOMLDecodeError as e: + raise SpecError(f"agent {name!r}: invalid TOML: {e}") from e + + # The file may name itself, but the directory wins: the directory is what the + # resolution order in `load` looks up, so a disagreement means a request for `x` + # would run a spec calling itself `y`. Refusing names both. + if (declared := d.get("name")) and declared != name: + raise SpecError(f"agent {name!r}: spec declares name={declared!r}; they must match") + + mode = str(d.get("permission_mode", DEFAULT_PERMISSION_MODE)) + if mode not in PERMISSION_MODES: + raise SpecError( + f"agent {name!r}: permission_mode {mode!r} not one of {PERMISSION_MODES}") + + try: + max_turns = int(d.get("max_turns", DEFAULT_MAX_TURNS)) + except (TypeError, ValueError) as e: + raise SpecError(f"agent {name!r}: max_turns must be an integer") from e + if max_turns < 1: + raise SpecError(f"agent {name!r}: max_turns must be >= 1, got {max_turns}") + + allowed = _as_tuple(d.get("allowed_tools"), "allowed_tools") + disallowed = _as_tuple(d.get("disallowed_tools"), "disallowed_tools") + # Both lists naming one tool is a contradiction the CLI resolves by its own + # precedence, which is not something a policy should depend on. The safe reading + # (deny wins) would also silently differ from what the operator wrote, so refuse. + if overlap := set(allowed) & set(disallowed): + raise SpecError( + f"agent {name!r}: {sorted(overlap)} in both allowed_tools and disallowed_tools") + + system_prompt = "" + if rel := d.get("append_system_prompt_file"): + if base_dir is None: + raise SpecError(f"agent {name!r}: append_system_prompt_file needs a base_dir") + system_prompt = pathlib.Path( + _resolve_under(base_dir, str(rel), "append_system_prompt_file") + ).read_text() + + settings_path = "" + if rel := d.get("settings_file"): + if base_dir is None: + raise SpecError(f"agent {name!r}: settings_file needs a base_dir") + settings_path = _resolve_under(base_dir, str(rel), "settings_file") + + mcp_config_path = "" + if rel := (d.get("mcp", {}) or {}).get("config_file"): + if base_dir is None: + raise SpecError(f"agent {name!r}: mcp.config_file needs a base_dir") + mcp_config_path = _resolve_under(base_dir, str(rel), "mcp.config_file") + + skills: list[Skill] = [] + for entry in d.get("skills", []) or []: + if not isinstance(entry, dict): + raise SpecError(f"agent {name!r}: each [[skills]] entry must be a table") + s_name = str(entry.get("name", "")).strip() + source = str(entry.get("source", "")).strip() + if not s_name or not source: + raise SpecError(f"agent {name!r}: a skill needs both name and source") + sha = str(entry.get("sha256", "")).strip().lower() + if sha and not re.fullmatch(r"[0-9a-f]{64}", sha): + raise SpecError(f"agent {name!r}: skill {s_name!r} sha256 must be 64 hex chars") + # §5.4 gate: a non-baked source without a digest can never be verified, so a + # spec that omits it is rejected at load rather than at fetch. Stated as a + # positive check on `image://` so a new scheme added later inherits the + # requirement instead of bypassing it. + if not source.startswith("image://") and not sha: + raise SpecError( + f"agent {name!r}: skill {s_name!r} has a fetched source and no sha256") + skills.append(Skill(name=s_name, source=source, sha256=sha, + sig_kid=str(entry.get("sig_kid", "")).strip())) + + lim = d.get("limits", {}) or {} + limits = Limits( + timeout_s=float(lim.get("timeout_s", 0.0)), + max_output_bytes=int(lim.get("max_output_bytes", 0)), + max_events=int(lim.get("max_events", Limits.max_events)), + ) + + return AgentSpec( + name=name, + description=str(d.get("description", "")), + model=str(d.get("model", "")).strip(), + max_turns=max_turns, + permission_mode=mode, + allowed_tools=allowed, + disallowed_tools=disallowed, + system_prompt=system_prompt, + settings_path=settings_path, + mcp_config_path=mcp_config_path, + skills=tuple(skills), + limits=limits, + ) + + +def load(agent_dir: str, name: str) -> AgentSpec: + """Load `{agent_dir}/{name}/agent.toml`. + + `default` with no file on disk returns the built-in spec — Phase 2's behaviour — + because the overwhelmingly common deployment bakes no agents at all and must keep + working. Any OTHER missing name raises: §5.1 requires an error event rather than a + silent fallback. + """ + validate_name(name) + base = pathlib.Path(agent_dir) / name + path = base / SPEC_FILENAME + if not path.is_file(): + if name == "default": + return AgentSpec() + raise SpecError(f"agent {name!r}: no {SPEC_FILENAME} under {base}") + try: + raw = path.read_text() + except OSError as e: + raise SpecError(f"agent {name!r}: cannot read {path}: {e}") from e + return parse(raw, name=name, base_dir=base) + + +def resolve_name(*candidates: str | None, fallback: str = "default") -> str: + """§5.1's resolution order, most specific first: the trigger's agent, then the + registry user's, then `ER_AGENT_NAME`, then `default`. + + Written as a helper so the order lives in one place rather than being re-expressed + as an `or` chain at each call site, where one site eventually gets it backwards. + """ + for c in candidates: + if c and c.strip(): + return c.strip() + return fallback diff --git a/eventing/eventrunner/config.py b/eventing/eventrunner/config.py index 4c8e2704..a6624916 100644 --- a/eventing/eventrunner/config.py +++ b/eventing/eventrunner/config.py @@ -105,6 +105,23 @@ class Cfg: # verifier holding exactly one key accepts it; name it as soon as there are two. signing_kid: str = "" + # ---- Phase 3 (DESIGN_PHASE3.md) ---- + + # §3.3 — which tenant this runner serves. Stamped on every response as + # `ce_userkey`, which is what lets EventBridge file the response in the right + # store. Empty is single-tenant mode (the default, = Phase 2). Required once + # REQUEST_TOPIC is not the default: a runner that stamps no userkey produces + # events EventBridge cannot attribute, and a silent default here would route one + # user's output into another user's store. `load` enforces that, loudly. + userkey: str = "" + + # §5.2 — where baked AgentSpecs live. A directory in the image, not a mount: + # an agent definition the agent itself could rewrite is not a policy. + agent_dir: str = "/etc/rossoctl/agents" + # §5.1 — the fallback agent when neither the request nor the registry names one. + # `default` with no file on disk is the built-in spec, i.e. Phase 2's argv. + agent_name: str = "default" + def load() -> Cfg: e = os.environ.get @@ -154,6 +171,21 @@ def load() -> Cfg: cfg.verify_keyset_path = e("ER_VERIFY_KEYSET_PATH", cfg.verify_keyset_path) cfg.signing_kid = e("ER_SIGNING_KID", cfg.signing_kid) + # §3.3, §5.1-§5.2. + cfg.userkey = (e("ER_USERKEY") or "").strip() + cfg.agent_dir = e("ER_AGENT_DIR", cfg.agent_dir) + cfg.agent_name = e("ER_AGENT_NAME", cfg.agent_name) + # A per-user runner is pointed at a per-user topic, so a non-default REQUEST_TOPIC + # with no ER_USERKEY means the deployment was rendered wrong. Refusing to start is + # the only safe direction: the alternative is responses EventBridge files into the + # `shared` store or drops, which looks exactly like an agent that never answered. + if cfg.request_topic != Cfg.request_topic and not cfg.userkey: + raise SystemExit( + f"ER_USERKEY is required when REQUEST_TOPIC is not {Cfg.request_topic!r} " + f"(got {cfg.request_topic!r}). A runner that stamps no userkey on its " + f"responses produces events EventBridge cannot attribute. " + f"See DESIGN_PHASE3.md §3.3.") + base = e("TMPDIR", "/tmp").rstrip("/") cfg.tmpdir = f"{base}/rossoctl-keda1" pathlib.Path(cfg.tmpdir, "eventrunner").mkdir(parents=True, exist_ok=True) diff --git a/eventing/eventrunner/runner.py b/eventing/eventrunner/runner.py index 17e9db60..6c9f6b6a 100644 --- a/eventing/eventrunner/runner.py +++ b/eventing/eventrunner/runner.py @@ -14,8 +14,11 @@ import threading import time +from eventrunner import agentspec +from eventrunner.agentspec import AgentSpec from eventrunner.config import API_KEY_VARS, Cfg from eventrunner.emit import Emitter +from shared import ce # Env vars forwarded to the `claude` subprocess. The child does NOT inherit # our full environment — only these keys pass through when they are set on @@ -213,7 +216,8 @@ def is_hook_frame(frame) -> bool: def build_cmd(cfg: Cfg, event, payload: dict, *, - resume_target: str | None = None) -> list[str]: + resume_target: str | None = None, + spec: AgentSpec | None = None) -> list[str]: """Assemble the claude argv for one turn. `resume_target` overrides what `--resume` gets. Phase 0 always passed the @@ -221,23 +225,57 @@ def build_cmd(cfg: Cfg, event, payload: dict, *, Phase 1 passes an absolute transcript path when it has one, which is what makes `/continue` survive a scale-to-zero (§16 Gap B, verified by scripts/verify_resume_by_path.py). + + Phase 3 §5.5: `spec` is an `AgentSpec` carrying the tool policy, system prompt and + model. `None` — and the built-in `default` spec — produce **byte-identical argv to + Phase 2**, which is what keeps the e2e path and the existing tests unchanged; + `tests/test_agentspec.py` asserts that equality rather than trusting it. + + Precedence between the request and the spec is deliberate and differs by field: + + * `max_turns` and `model` — the **request wins** when it names one. Both are + per-turn parameters a caller has always been able to send, and a spec silently + overriding them would break Phase 0's wire contract. + * the tool policy, permission mode and system prompt — **only the spec**. These are + the sandbox (§7.4), and a request that could widen them would be a request that + can escape them, which is the whole control inverted. + + The prompt goes through `argv`, never a shell: `Popen` with a list and no + `shell=True` is what makes §7.4's attacker-influenced prompt a prompt-injection + problem rather than also a command-injection one. The obvious "improvement" of + building a command string would silently reintroduce it. """ + spec = spec or AgentSpec() session = event["sessionuuid"] mode = event.get("mode", "start") prompt = payload.get("prompt", "") + max_turns = payload.get("max_turns") or spec.max_turns cmd = [ cfg.claude_bin, "-p", prompt, "--output-format", "stream-json", "--verbose", - "--max-turns", str(payload.get("max_turns", 3)), - "--permission-mode", "acceptEdits", + "--max-turns", str(max_turns), + "--permission-mode", spec.permission_mode, ] if mode == "start": cmd += ["--session-id", session] else: cmd += ["--resume", resume_target or session] - if model := payload.get("model"): + if model := (payload.get("model") or spec.model): cmd += ["--model", model] + # Appended after the Phase 2 flags, so the default spec's argv is a prefix-equal + # match rather than merely an equivalent set — easier to assert and to eyeball in a + # log line. + if spec.allowed_tools: + cmd += ["--allowedTools", ",".join(spec.allowed_tools)] + if spec.disallowed_tools: + cmd += ["--disallowedTools", ",".join(spec.disallowed_tools)] + if spec.system_prompt: + cmd += ["--append-system-prompt", spec.system_prompt] + if spec.settings_path: + cmd += ["--settings", spec.settings_path] + if spec.mcp_config_path: + cmd += ["--mcp-config", spec.mcp_config_path] return cmd @@ -291,6 +329,27 @@ def run_agent(cfg: Cfg, emitter: Emitter, event, *, transcripts=None) -> None: started = time.monotonic() + # §5.1 resolution order, most specific first. A request naming an agent this runner + # does not have is an ERROR EVENT, never a silent fallback to `default`: running a + # different agent than the one asked for means a policy the caller believes applied + # did not, which is worse than not running at all. + agent_name = agentspec.resolve_name(event.get(ce.EXT_AGENT), cfg.agent_name) + try: + spec = agentspec.load(cfg.agent_dir, agent_name) + except agentspec.SpecError as e: + print(f"[eventrunner] {corr}: {e}", file=sys.stderr, flush=True) + emitter.emit(correlationid=corr, sessionuuid=session, + sequence=emitter.next_seq(corr), phase="error", final=True, + data={"error": f"agent spec unavailable: {e}", + "agent": agent_name}, + causationid=causation, groupid=groupid) + return + if not spec.is_default: + print(f"[eventrunner] {corr}: agent={spec.name} " + f"permission_mode={spec.permission_mode} " + f"allowed={list(spec.allowed_tools)} " + f"disallowed={list(spec.disallowed_tools)}", flush=True) + if cfg.mock_claude: _run_mock(emitter, corr, session, payload, causation=causation, groupid=groupid, delay=_mock_delay(cfg)) @@ -308,7 +367,7 @@ def run_agent(cfg: Cfg, emitter: Emitter, event, *, transcripts=None) -> None: f"pod that is gone — DESIGN_PHASE1.md §16 Gap B)", file=sys.stderr, flush=True) - cmd = build_cmd(cfg, event, payload, resume_target=resume_target) + cmd = build_cmd(cfg, event, payload, resume_target=resume_target, spec=spec) try: proc = subprocess.Popen(cmd, cwd=str(workdir), stdin=subprocess.DEVNULL, diff --git a/eventing/shared/ce.py b/eventing/shared/ce.py index 13a9432f..19a0fb5a 100644 --- a/eventing/shared/ce.py +++ b/eventing/shared/ce.py @@ -61,6 +61,25 @@ # `ce_` prefix and validate nothing, so it round-trips here and is rejected or # silently dropped by a spec-compliant consumer a hop later. EXT_SUBMITTER_ISS = "submitteriss" +# Phase 3 §2.6 — the tenancy key, from `shared.tenancy.userkey(issuer, userid)`. +# +# Unlike `submitter`, which Phase 2 only ever compared and printed, this one is +# load-bearing: it decides which store a response is written into and which ntfy topic +# it is announced on. So it is in `signing.SIGNED_ATTRS` — an unsigned, mutable +# `userkey` would let anything with write access to a topic file an event into another +# user's history. +# +# Absent in single-tenant mode, which is the default and reproduces Phase 2 exactly. +EXT_USERKEY = "userkey" +# Phase 3 §5.1 — which agent definition should serve this request. Absent means the +# runner's own `ER_AGENT_NAME`, then `default`. +EXT_AGENT = "agent" +# Phase 3 §7.7 — hop count, incremented each time an event causes another event. The +# loop control the trigger API lives or dies by, which is why it is signed: a resettable +# depth is not a hop limit at all, it is a suggestion an attacker declines. +EXT_DEPTH = "depth" +# Phase 3 §7.2 — the trigger that caused this request, for attribution on the response. +EXT_TRIGGERID = "triggerid" CE_HEADER_PREFIX = "ce_" CORE_ATTRS = {"specversion", "type", "source", "id", "time", diff --git a/eventing/shared/tenancy.py b/eventing/shared/tenancy.py new file mode 100644 index 00000000..d9238702 --- /dev/null +++ b/eventing/shared/tenancy.py @@ -0,0 +1,250 @@ +"""Tenancy keys and the names derived from them. DESIGN_PHASE3.md §2.3, §3.1, §4.2. + +This is the ONLY place an identity becomes a name. Phase 2 put a login on the event +as an audit label (`ce_submitter`); Phase 3 turns it into a tenancy key that names +Kafka topics, Kubernetes objects and an ntfy topic. Those four jobs have four +different legality rules, and a raw login satisfies none of them reliably: + +| Job | Why a raw `submitter` fails | +|---|---| +| Kafka topic | `[a-zA-Z0-9._-]`, max 249. `alice@example.com` has an illegal `@`. | +| Kubernetes object | DNS-1123: lower-case `[a-z0-9-]`, 63 for a label. `MrSabath` has upper case. | +| ntfy topic | `[-_A-Za-z0-9]{1,64}`, and the name travels to a phone (§4.1). | +| Separate two tenants | The one that matters: any lossy mapping can merge two identities. | + +The fourth is a security bug rather than a cosmetic one, and it is easy to walk into: +replacing every illegal character with `-` maps `a.b@x.com`, `a_b@x.com` and +`a-b@x.com` onto one key, and then user B reads user A's events. `userkey` is built so +that cannot happen — see the digest note there. + +Pure stdlib (`hashlib`, `hmac`, `base64`), per Phase 1 §1.1, and every function here is +pure: no I/O, no clock, no config. That is what makes the whole module testable against +a corpus of adversarial identifiers without a cluster. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac + +# Short issuer tags. Two characters so the readable middle of the key keeps as much +# room as possible, and in the key at all so a human reading `kubectl get kafkatopics` +# can tell which issuer a topic belongs to. +ISS_PREFIX = {"github": "gh", "oidc": "oi", "static": "st"} + +# Mode names for TopicSet. `single` is the default everywhere: it reproduces Phase 2 +# byte for byte, which is what makes this phase additive. +SINGLE = "single" +MULTI = "multi" + +# How much of the canonical identity's digest is appended to the key. 32 bits, which +# is NOT collision-resistant in the cryptographic sense — a birthday collision is +# expected around 2**16 ~ 65,000 users. The registry's duplicate check (§2.5) is what +# covers that, by refusing a key already held by a different identity. Raise to 12 if +# a deployment ever approaches that scale. +_DIGEST_HEX = 8 + +# The readable middle, before the digest. Keeps the longest realistic topic name +# (`kev1-` + 2 + 1 + 24 + 1 + 8 + `-responses` = 51) comfortably inside Kafka's 249 +# and inside a 63-character DNS-1123 label. +_SLUG_MAX = 24 + + +def _canonicalise(issuer: str, userid: str) -> str: + """The ONLY lossy step, and deliberately minimal — per issuer, never guessed. + + Each branch is a claim about the issuer's own identifier semantics, and getting one + wrong in the permissive direction merges two humans into one tenant: + + * **GitHub** logins are case-insensitive, so lower-casing is *correct* here. It must + also match `ghauth.is_allowed`'s compare exactly — if the allow-list folds case + and this does not, `Alice` and `alice` are both approved and land in different + tenants, which reads to the user as half their history vanishing. + * **Email** normalisation is provider-specific and we must not guess: + `A.Lice@Gmail.com` and `alice@gmail.com` are one mailbox at Gmail and may be two + elsewhere. So only what RFC 5321 permits — lower-case the domain, leave the local + part byte-exact. One human with two spellings therefore gets two tenants, which + is wasteful, not unsafe. The unsafe direction (two humans, one tenant) is what the + digest in `userkey` makes impossible. + * **Static** identities are byte-exact: an operator typed them, and normalising + somebody's hand-written string on their behalf is how you merge two of them. + + `rpartition` rather than `partition` because the local part of an address may itself + contain `@` when quoted; the domain is what follows the LAST one. + """ + s = userid.strip() + if issuer == "github": + return s.lower() + if "@" in s: + local, _, domain = s.rpartition("@") + return f"{local}@{domain.lower()}" + return s + + +def _slug(canon: str) -> str: + """A readable, DNS-1123-safe rendering of the canonical identity. Lossy on purpose. + + Lossiness is the point: it is what keeps `gh-alice-a1b2c3d4` legible in a log line + and in `kubectl get pods`. It is survivable only because `userkey` appends a digest + over the canonical form, so two identities that slug identically still differ in the + key. + + Emits `-` as the only separator, never `.` or `_`. That is not stylistic: Kafka's + JMX metric names flatten both `.` and `_`, so a topic `a.b_c` and a topic `a_b.c` + collide and one silently overwrites the other's metrics. Keeping `-` as the sole + separator makes that unreachable — stated here because the next person to "improve" + this function needs to know. + """ + out: list[str] = [] + prev_dash = False + for ch in canon.lower(): + if ch.isascii() and (ch.isalnum()): + out.append(ch) + prev_dash = False + elif not prev_dash: + # Any run of illegal characters collapses to one dash, so `a..b` and `a@b` + # both render `a-b`. Safe only because of the digest; see `userkey`. + out.append("-") + prev_dash = True + return "".join(out) + + +def userkey(issuer: str, userid: str) -> str: + """Map (issuer, userid) to a short, legal, COLLISION-FREE tenancy key. + + Shape: `--<8 hex>` + + gh-alice-a1b2c3d4 + oi-alice-example-com-9e8f7a6b + st-ops-break-glass-00ff1122 + + Legal as a Kafka topic component, a DNS-1123 label, and an ntfy topic component. + + Three properties, each defending a decision: + + **The digest is the security control, not decoration.** `_slug` is lossy, which is + what makes the key readable. That is only safe because 32 bits of SHA-256 over the + canonical form are appended, so two distinct canonical identities cannot produce one + key no matter how the slug collapses them. + + **The issuer is in the key AND in the digest input.** GitHub user `alice` and OIDC + subject `alice` are different people. Without the issuer inside the hashed bytes, a + second configured issuer becomes a way to impersonate the first — the single worst + failure available here. It appears in the prefix so a human can see it, and in the + digest so it is load-bearing rather than cosmetic. + + **The separator inside the digest input is `\\x1f`**, a unit separator, which cannot + occur in an issuer name. Concatenating with a character that *can* occur in either + field would make the input ambiguous: with `-`, issuer `a` + user `b-c` and issuer + `a-b` + user `c` hash identically, which is exactly the cross-issuer collision the + issuer was added to prevent. + """ + iss = ISS_PREFIX.get(issuer, "xx") + canon = _canonicalise(issuer, userid) + digest = hashlib.sha256(f"{issuer}\x1f{canon}".encode()).hexdigest()[:_DIGEST_HEX] + # `strip("-")` after truncating, so a slug cut mid-run does not end in a dash — + # a DNS-1123 label may not start or end with one. `or "u"` covers an identifier + # with no alphanumerics at all, which would otherwise slug to "" and produce + # `gh--a1b2c3d4`: still unique thanks to the digest, but not a legal label. + slug = _slug(canon)[:_SLUG_MAX].strip("-") or "u" + return f"{iss}-{slug}-{digest}" + + +def ntfy_topic(prefix: str, userkey: str, secret: bytes) -> str: + """An unguessable, stable, per-user ntfy topic that contains no identifier. + + kev1-k7qf3mz2xa9pbw4nsdhe6tcyu5 (prefix + 26 chars of base32) + + §4.1 is why the obvious `{prefix}-{userkey}-ntfy` is wrong: the topic name travels + to a phone and appears in the notification UI, so embedding a login publishes who + the user is to anyone who sees the screen — and on a public server, anyone who + guesses the name reads the stream. + + The decisions: + + * **Keyed, not hashed.** A bare `sha256(userkey)` is computable by anyone, because + the userkey is public — it names topics and Kubernetes objects. The HMAC secret + (`EB_NTFY_TOPIC_SECRET_PATH`, a Secret mount) is what makes the topic unguessable + rather than merely ugly. + * **Derived, not stored.** No table and no row to drift from the Kafka topics. + Rotating the secret rotates every user's topic at once, which is the right + response to a leak. The cost is that in-flight notifications on the old topic are + orphaned, which is why nothing durable is keyed by the ntfy topic. + * **Base32, lower-cased.** ntfy allows `[-_A-Za-z0-9]{1,64}`; base32 is + case-insensitive-safe at 5 bits per character, so 26 characters carry 130 bits. + The truncation discards 126 of the 256 HMAC bits deliberately — the topic has to + be typeable, and 130 bits is not the weak link in any threat model here. + + The `b"ntfy|"` domain separator keeps this HMAC from colliding with any other use of + the same secret added later: without it, a second derivation over the bare userkey + would produce the same tag for a different purpose. + """ + mac = hmac.new(secret, b"ntfy|" + userkey.encode(), hashlib.sha256).digest() + tag = base64.b32encode(mac).decode().rstrip("=").lower()[:26] + return f"{prefix}-{tag}" + + +class TopicSet: + """The one place §3.1's topic layout is written down. + + | Topic | Name | Written by | Read by | + |---|---|---|---| + | requests | `{prefix}-{userkey}-requests` | EventBridge | that user's EventRunner | + | responses | `{prefix}-{userkey}-responses` | that user's EventRunner | EventBridge | + | inbox | `{prefix}-{userkey}-events` | EventBridge ingress | the trigger dispatcher | + | dead letter | `{prefix}-{userkey}-dead` | EventBridge | an operator with a console consumer | + + In `single` mode every method returns the configured request/response topic and + ignores `userkey` entirely. That is what makes the whole phase additive: a Phase + 0/1/2 deployment gets the same two topics it has now, through the same code path the + multi-tenant one uses — so the existing suite is the check on step 2 of §9's rollout. + + Not a dataclass because `single` mode has to carry the two explicit topic names + while `multi` mode derives all four from a prefix, and a constructor that validates + that pairing is clearer than four optional fields. + """ + + def __init__(self, prefix: str, *, tenancy: str = SINGLE, + request_topic: str = "requests", + response_topic: str = "responses") -> None: + if tenancy not in (SINGLE, MULTI): + raise ValueError(f"tenancy must be {SINGLE!r} or {MULTI!r}, got {tenancy!r}") + self.prefix = prefix + self.tenancy = tenancy + self._request_topic = request_topic + self._response_topic = response_topic + + @property + def multi(self) -> bool: + return self.tenancy == MULTI + + def _derived(self, userkey: str | None, suffix: str, fallback: str) -> str: + if not self.multi: + return fallback + if not userkey: + # A missing userkey in multi mode is a programming error, not a condition to + # paper over with a default: a request published to a fallback topic is one + # tenant's work executed by another tenant's runner, under that tenant's + # credential. Refusing loudly here is the only safe direction. + raise ValueError(f"userkey is required to name the {suffix!r} topic in multi mode") + return f"{self.prefix}-{userkey}-{suffix}" + + def requests(self, userkey: str | None = None) -> str: + return self._derived(userkey, "requests", self._request_topic) + + def responses(self, userkey: str | None = None) -> str: + return self._derived(userkey, "responses", self._response_topic) + + def events(self, userkey: str | None = None) -> str: + """The §7 trigger inbox. In single mode there is one, named off the prefix: + unlike requests/responses it has no Phase 2 predecessor to stay compatible + with, so it is derived rather than configured.""" + if not self.multi: + return f"{self.prefix}-events" + return self._derived(userkey, "events", "") + + def dead(self, userkey: str | None = None) -> str: + """The §7.5 dead letter topic. Same reasoning as `events`.""" + if not self.multi: + return f"{self.prefix}-dead" + return self._derived(userkey, "dead", "") diff --git a/eventing/tests/test_agentspec.py b/eventing/tests/test_agentspec.py new file mode 100644 index 00000000..e8555418 --- /dev/null +++ b/eventing/tests/test_agentspec.py @@ -0,0 +1,376 @@ +"""Declarative agent definitions and the argv they produce. DESIGN_PHASE3.md §5.1-§5.5. + +The load-bearing test here is `test_default_spec_reproduces_phase2_argv_exactly`. §5.1's +whole claim to being additive is that an unspecified field inherits Phase 2's behaviour, +and the only way to know that holds is to assert the argv against a literal copy of what +Phase 2 emitted rather than against the current implementation. +""" +from __future__ import annotations + +import pathlib +import shutil +import subprocess + +import pytest + +from eventrunner import agentspec +from eventrunner.agentspec import AgentSpec, SpecError +from eventrunner.config import Cfg as ErCfg +from eventrunner.runner import build_cmd +from shared import ce + + +def _event(mode="start", corr="brave-otter-4718", **over): + attrs = {"correlationid": corr, "sessionuuid": ce.session_uuid(corr), "mode": mode} + attrs.update(over) + return ce.CloudEvent(attrs=attrs) + + +def _spec_dir(tmp_path: pathlib.Path, name: str, toml: str, **files) -> pathlib.Path: + d = tmp_path / name + d.mkdir(parents=True, exist_ok=True) + (d / "agent.toml").write_text(toml) + for fname, content in files.items(): + (d / fname.replace("__", ".")).write_text(content) + return tmp_path + + +# ---- the additive guarantee ------------------------------------------------- + +def test_default_spec_reproduces_phase2_argv_exactly(): + """§5.1: the `default` agent must be byte-identical to Phase 2's `build_cmd`. + + The expected list is a literal transcription of the pre-Phase-3 implementation, NOT + a call into the current one — otherwise this test would follow a regression instead + of catching it. + """ + cfg = ErCfg() + corr = "brave-otter-4718" + session = ce.session_uuid(corr) + event = _event(corr=corr) + expected = [ + cfg.claude_bin, "-p", "hello", + "--output-format", "stream-json", + "--verbose", + "--max-turns", "3", + "--permission-mode", "acceptEdits", + "--session-id", session, + ] + assert build_cmd(cfg, event, {"prompt": "hello"}) == expected + # Explicitly passing the built-in spec must be the same as passing none. + assert build_cmd(cfg, event, {"prompt": "hello"}, spec=AgentSpec()) == expected + # And so must loading `default` from a directory that has no file in it. + loaded = agentspec.load("/nonexistent-agent-dir", "default") + assert build_cmd(cfg, event, {"prompt": "hello"}, spec=loaded) == expected + + +def test_default_spec_with_model_and_max_turns_matches_phase2(): + cfg = ErCfg() + corr = "brave-otter-4718" + event = _event(corr=corr) + cmd = build_cmd(cfg, event, {"prompt": "hi", "model": "sonnet", "max_turns": 9}) + assert cmd == [ + cfg.claude_bin, "-p", "hi", + "--output-format", "stream-json", + "--verbose", + "--max-turns", "9", + "--permission-mode", "acceptEdits", + "--session-id", ce.session_uuid(corr), + "--model", "sonnet", + ] + + +def test_builtin_default_is_recognised_as_imposing_nothing(): + assert AgentSpec().is_default + assert not AgentSpec(permission_mode="plan").is_default + assert not AgentSpec(allowed_tools=("Read",)).is_default + assert not AgentSpec(disallowed_tools=("Bash",)).is_default + assert not AgentSpec(model="sonnet").is_default + + +# ---- argv from a real spec -------------------------------------------------- + +def test_build_cmd_emits_the_tool_policy(): + cfg = ErCfg() + spec = AgentSpec(name="triager", model="sonnet", max_turns=8, + permission_mode="plan", + allowed_tools=("Read", "Grep", "Glob"), + disallowed_tools=("Bash", "Write")) + cmd = build_cmd(cfg, _event(), {"prompt": "triage"}, spec=spec) + assert cmd[cmd.index("--permission-mode") + 1] == "plan" + assert cmd[cmd.index("--allowedTools") + 1] == "Read,Grep,Glob" + assert cmd[cmd.index("--disallowedTools") + 1] == "Bash,Write" + assert cmd[cmd.index("--model") + 1] == "sonnet" + assert cmd[cmd.index("--max-turns") + 1] == "8" + + +def test_the_request_wins_on_max_turns_and_model_but_not_on_policy(): + """§5.5's precedence split. The per-turn parameters a caller has always been able to + send keep working; the sandbox is spec-only, because a request that could widen the + tool policy is a request that can escape it.""" + cfg = ErCfg() + spec = AgentSpec(name="triager", model="haiku", max_turns=8, + permission_mode="plan", disallowed_tools=("Bash",)) + cmd = build_cmd(cfg, _event(), {"prompt": "x", "model": "opus", "max_turns": 2}, + spec=spec) + assert cmd[cmd.index("--model") + 1] == "opus" + assert cmd[cmd.index("--max-turns") + 1] == "2" + # The policy is untouched by anything in the payload. + assert cmd[cmd.index("--permission-mode") + 1] == "plan" + assert cmd[cmd.index("--disallowedTools") + 1] == "Bash" + + +def test_a_request_cannot_widen_the_tool_policy(): + """The payload keys an attacker would try. None of them reach argv.""" + cfg = ErCfg() + spec = AgentSpec(name="t", permission_mode="plan", disallowed_tools=("Bash",)) + cmd = build_cmd(cfg, _event(), { + "prompt": "x", + "permission_mode": "bypassPermissions", + "allowed_tools": ["Bash"], + "allowedTools": ["Bash"], + "disallowed_tools": [], + }, spec=spec) + assert "bypassPermissions" not in cmd + assert cmd[cmd.index("--permission-mode") + 1] == "plan" + assert cmd[cmd.index("--disallowedTools") + 1] == "Bash" + assert "--allowedTools" not in cmd + + +def test_spec_max_turns_applies_when_the_request_omits_it(): + cfg = ErCfg() + cmd = build_cmd(cfg, _event(), {"prompt": "x"}, spec=AgentSpec(max_turns=12)) + assert cmd[cmd.index("--max-turns") + 1] == "12" + + +def test_build_cmd_emits_system_prompt_settings_and_mcp(): + cfg = ErCfg() + spec = AgentSpec(name="t", system_prompt="be terse", + settings_path="/etc/rossoctl/agents/t/settings.json", + mcp_config_path="/etc/rossoctl/agents/t/.mcp.json") + cmd = build_cmd(cfg, _event(), {"prompt": "x"}, spec=spec) + assert cmd[cmd.index("--append-system-prompt") + 1] == "be terse" + assert cmd[cmd.index("--settings") + 1] == "/etc/rossoctl/agents/t/settings.json" + assert cmd[cmd.index("--mcp-config") + 1] == "/etc/rossoctl/agents/t/.mcp.json" + + +def test_the_prompt_is_one_argv_element_however_hostile(): + """§5.5: the prompt goes through argv, never a shell. A prompt that would be a + command injection in a shell string must stay a single element here.""" + cfg = ErCfg() + nasty = "look; rm -rf / && echo $(whoami) `id` | tee /tmp/x" + cmd = build_cmd(cfg, _event(), {"prompt": nasty}, spec=AgentSpec()) + assert cmd.count(nasty) == 1 + assert cmd[cmd.index("-p") + 1] == nasty + + +def test_resume_still_works_with_a_spec(): + cfg = ErCfg() + spec = AgentSpec(name="t", permission_mode="plan") + cmd = build_cmd(cfg, _event(mode="continue"), {"prompt": "go on"}, + resume_target="/data/x.jsonl", spec=spec) + assert cmd[cmd.index("--resume") + 1] == "/data/x.jsonl" + assert "--session-id" not in cmd + + +# ---- parsing --------------------------------------------------------------- + +TRIAGER = """ +name = "triager" +description = "Triage an incoming issue." +model = "sonnet" +max_turns = 8 +permission_mode = "plan" +allowed_tools = ["Read", "Grep", "Glob", "Skill"] +disallowed_tools = ["Bash", "Write", "Edit"] +append_system_prompt_file = "system.md" + +[[skills]] +name = "eventbridge" +source = "image:///etc/rossoctl/agents/triager/skills/eventbridge" + +[limits] +timeout_s = 900 +max_output_bytes = 4000000 +max_events = 1000 +""" + + +def test_parse_a_full_spec(tmp_path): + root = _spec_dir(tmp_path, "triager", TRIAGER, system__md="Be terse.\n") + spec = agentspec.load(str(root), "triager") + assert spec.name == "triager" + assert spec.model == "sonnet" + assert spec.max_turns == 8 + assert spec.permission_mode == "plan" + assert spec.allowed_tools == ("Read", "Grep", "Glob", "Skill") + assert spec.disallowed_tools == ("Bash", "Write", "Edit") + assert spec.system_prompt == "Be terse.\n" + assert len(spec.skills) == 1 and spec.skills[0].is_baked + assert spec.limits.timeout_s == 900 + assert spec.limits.max_events == 1000 + assert not spec.is_default + + +def test_an_empty_spec_is_the_phase2_default(tmp_path): + root = _spec_dir(tmp_path, "bare", "") + spec = agentspec.load(str(root), "bare") + assert spec.is_default + assert spec.permission_mode == "acceptEdits" + assert spec.max_turns == 3 + + +def test_a_missing_non_default_agent_raises(tmp_path): + """§5.1: an error event, never a silent fallback to `default`.""" + with pytest.raises(SpecError, match="no agent.toml"): + agentspec.load(str(tmp_path), "nope") + + +def test_a_missing_default_agent_is_the_builtin(tmp_path): + """The common deployment bakes no agents at all and must keep working.""" + assert agentspec.load(str(tmp_path), "default").is_default + + +@pytest.mark.parametrize("name", [ + "", "Triager", "tri_ager", "-triager", "triager-", "tri.ager", + "../etc", "a/b", "x" * 64, +]) +def test_invalid_agent_names_are_refused(tmp_path, name): + with pytest.raises(SpecError): + agentspec.load(str(tmp_path), name) + + +def test_a_spec_whose_declared_name_disagrees_is_refused(tmp_path): + root = _spec_dir(tmp_path, "triager", 'name = "something-else"\n') + with pytest.raises(SpecError, match="must match"): + agentspec.load(str(root), "triager") + + +def test_invalid_toml_is_refused(tmp_path): + root = _spec_dir(tmp_path, "broken", "this is not = = toml\n") + with pytest.raises(SpecError, match="invalid TOML"): + agentspec.load(str(root), "broken") + + +@pytest.mark.parametrize("mode", ["", "yolo", "AcceptEdits", "accept-edits"]) +def test_an_unknown_permission_mode_is_refused(tmp_path, mode): + root = _spec_dir(tmp_path, "a", f'permission_mode = "{mode}"\n') + with pytest.raises(SpecError, match="permission_mode"): + agentspec.load(str(root), "a") + + +@pytest.mark.parametrize("value", ["0", "-1"]) +def test_a_nonsensical_max_turns_is_refused(tmp_path, value): + root = _spec_dir(tmp_path, "a", f"max_turns = {value}\n") + with pytest.raises(SpecError, match="max_turns"): + agentspec.load(str(root), "a") + + +def test_a_non_list_tool_policy_is_refused(tmp_path): + """A policy that silently drops the clause it could not parse would hand a + trigger-driven agent the tool the operator meant to remove.""" + root = _spec_dir(tmp_path, "a", 'disallowed_tools = "Bash"\n') + with pytest.raises(SpecError, match="list of strings"): + agentspec.load(str(root), "a") + + +def test_a_tool_in_both_lists_is_refused(tmp_path): + root = _spec_dir(tmp_path, "a", + 'allowed_tools = ["Read", "Bash"]\ndisallowed_tools = ["Bash"]\n') + with pytest.raises(SpecError, match="both allowed_tools and disallowed_tools"): + agentspec.load(str(root), "a") + + +def test_a_system_prompt_file_outside_the_agent_dir_is_refused(tmp_path): + (tmp_path / "secret.txt").write_text("sensitive") + root = _spec_dir(tmp_path, "a", 'append_system_prompt_file = "../secret.txt"\n') + with pytest.raises(SpecError, match="escapes the agent directory"): + agentspec.load(str(root), "a") + + +def test_an_absolute_system_prompt_path_is_refused(tmp_path): + root = _spec_dir(tmp_path, "a", 'append_system_prompt_file = "/etc/hosts"\n') + with pytest.raises(SpecError, match="escapes the agent directory"): + agentspec.load(str(root), "a") + + +def test_a_missing_system_prompt_file_is_refused(tmp_path): + root = _spec_dir(tmp_path, "a", 'append_system_prompt_file = "absent.md"\n') + with pytest.raises(SpecError, match="does not exist"): + agentspec.load(str(root), "a") + + +# ---- skills ---------------------------------------------------------------- + +def test_a_fetched_skill_without_a_digest_is_refused(tmp_path): + """§5.4: a bundle reference that could never verify fails at load, not at fetch.""" + root = _spec_dir(tmp_path, "a", """ +[[skills]] +name = "triage" +source = "github://rossoctl/skills@9f1c0ab3d2e4f5061728394a5b6c7d8e9f012345//triage" +""") + with pytest.raises(SpecError, match="fetched source and no sha256"): + agentspec.load(str(root), "a") + + +def test_a_baked_skill_needs_no_digest(tmp_path): + root = _spec_dir(tmp_path, "a", """ +[[skills]] +name = "eventbridge" +source = "image:///etc/rossoctl/agents/a/skills/eventbridge" +""") + spec = agentspec.load(str(root), "a") + assert spec.skills[0].is_baked and not spec.skills[0].sha256 + + +def test_a_malformed_digest_is_refused(tmp_path): + root = _spec_dir(tmp_path, "a", """ +[[skills]] +name = "t" +source = "https://example.com/b.tar.gz" +sha256 = "not-a-digest" +""") + with pytest.raises(SpecError, match="64 hex"): + agentspec.load(str(root), "a") + + +def test_a_skill_missing_name_or_source_is_refused(tmp_path): + root = _spec_dir(tmp_path, "a", '[[skills]]\nname = "t"\n') + with pytest.raises(SpecError, match="both name and source"): + agentspec.load(str(root), "a") + + +# ---- resolution order ------------------------------------------------------ + +def test_resolve_name_prefers_the_most_specific(): + assert agentspec.resolve_name("from-trigger", "from-env") == "from-trigger" + assert agentspec.resolve_name(None, "from-env") == "from-env" + assert agentspec.resolve_name(None, "") == "default" + assert agentspec.resolve_name(" ", None) == "default" + assert agentspec.resolve_name(" spaced ", None) == "spaced" + + +# ---- the pinned CLI -------------------------------------------------------- + +# Every NEW flag Phase 3 teaches `build_cmd` to emit. §5.5 wants a flag that vanishes in +# a CLI upgrade to fail a build rather than a production run, and this is that check. +# +# Deliberately excludes the flags Phase 2 already shipped (`-p`, `--output-format`, +# `--verbose`, `--max-turns`, `--permission-mode`, `--session-id`, `--resume`, +# `--model`). Those are proven by the e2e path on the pinned CLI, and at least one of +# them (`--max-turns`) is *supported but undocumented* in `--help` on 2.1.270 — so +# asserting them here would fail on a working deployment, which is a worse outcome than +# not asserting them. Scope the check to what Phase 3 adds, where a typo is a real risk. +EMITTED_FLAGS = [ + "--allowedTools", "--disallowedTools", + "--append-system-prompt", "--settings", "--mcp-config", +] + + +@pytest.mark.skipif(shutil.which("claude") is None, + reason="claude CLI not on PATH; the image build runs this") +def test_every_new_flag_build_cmd_emits_exists_in_the_pinned_cli(): + out = subprocess.run(["claude", "--help"], capture_output=True, text=True, + timeout=60).stdout + missing = [f for f in EMITTED_FLAGS if f not in out] + assert not missing, f"claude --help does not mention {missing}" diff --git a/eventing/tests/test_tenancy.py b/eventing/tests/test_tenancy.py new file mode 100644 index 00000000..a7202739 --- /dev/null +++ b/eventing/tests/test_tenancy.py @@ -0,0 +1,282 @@ +"""Tenancy keys, topic names and derived ntfy topics. DESIGN_PHASE3.md §2.3, §3.1, §4.2. + +The collision tests are the point of this file. `_slug` is deliberately lossy, so the +only thing standing between a readable key and a cross-tenant leak is the appended +digest — and the one way to know it is actually load-bearing is to feed it the +adversarial identifiers that slug identically and assert the keys still differ. +""" +from __future__ import annotations + +import re + +import pytest + +from eventbridge import ghauth +from shared import tenancy as T + +# DNS-1123 label: lower-case alphanumerics and dashes, no leading or trailing dash, +# 63 characters. A userkey has to satisfy this to name a Kubernetes object and to be +# a Kafka topic component at the same time. +DNS1123 = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +# Kafka's legal topic character set. A userkey is one *component* of a topic name, so +# it has to be legal here too. +KAFKA_TOPIC = re.compile(r"^[a-zA-Z0-9._-]+$") +# ntfy's topic rule, from the ntfy server source. +NTFY_TOPIC = re.compile(r"^[-_A-Za-z0-9]{1,64}$") + +# Identifiers chosen to break a naive implementation: characters illegal in every +# target namespace, runs of separators that collapse, upper case, leading and trailing +# punctuation, something that slugs to nothing at all, and an over-long local part. +# +# Every entry must map to a DISTINCT key (see test_all_adversarial_keys_are_distinct), +# so deliberately-equivalent spellings do NOT belong here: `MrSabath` and `mrsabath` are +# one GitHub identity by design, and live in the case-folding tests at the bottom. +ADVERSARIAL = [ + ("github", "mrsabath"), + ("github", "a"), + ("github", "a-b-c"), + ("oidc", "alice@example.com"), + ("oidc", "a.b@x.com"), + ("oidc", "a_b@x.com"), + ("oidc", "a-b@x.com"), + ("oidc", "A.Lice@Gmail.COM"), + ("oidc", "very.long.local.part.that.exceeds.the.slug.budget@example.com"), + ("static", "ops-break-glass"), + ("static", "ops break glass"), + ("static", "...."), + ("static", "@@@"), + ("static", "-leading-and-trailing-"), + ("static", "ünïcødé"), + ("static", "a/b\\c:d*e"), + ("unknown-issuer", "someone"), +] + + +@pytest.mark.parametrize("issuer,userid", ADVERSARIAL) +def test_userkey_is_legal_everywhere(issuer, userid): + """One key has to satisfy three different namespaces at once.""" + k = T.userkey(issuer, userid) + assert DNS1123.match(k), f"{k!r} is not a DNS-1123 label" + assert KAFKA_TOPIC.match(k), f"{k!r} is not legal in a Kafka topic" + assert NTFY_TOPIC.match(k), f"{k!r} is not a legal ntfy topic" + assert len(k) <= 63, f"{k!r} exceeds a 63-char DNS-1123 label" + # The separator rule from §3.1: `.` and `_` together collide in Kafka's JMX metric + # names, so the slug must emit neither. + assert "." not in k and "_" not in k + + +@pytest.mark.parametrize("issuer,userid", ADVERSARIAL) +def test_userkey_is_stable(issuer, userid): + assert T.userkey(issuer, userid) == T.userkey(issuer, userid) + + +def test_userkey_shape(): + k = T.userkey("github", "alice") + assert k == "gh-alice-" + k.rsplit("-", 1)[1] + assert re.fullmatch(r"gh-alice-[0-9a-f]{8}", k) + + +def test_all_adversarial_keys_are_distinct(): + """The whole corpus must map to distinct keys. + + Several of these slug identically — `a.b@x.com`, `a_b@x.com` and `a-b@x.com` all + render `a-b-x-com`, and `....`/`@@@` both render nothing. If the digest were + dropped or computed over the slug instead of the canonical form, this fails, and + the production symptom is one user reading another's events. + """ + keys = [T.userkey(i, u) for i, u in ADVERSARIAL] + dupes = {k for k in keys if keys.count(k) > 1} + assert not dupes, f"colliding userkeys: {dupes}" + + +@pytest.mark.parametrize("a,b", [ + ("a.b@x.com", "a_b@x.com"), + ("a.b@x.com", "a-b@x.com"), + ("a_b@x.com", "a-b@x.com"), +]) +def test_separator_variants_do_not_merge(a, b): + """The classic lossy-normalisation bug, asserted directly.""" + assert T._slug(T._canonicalise("oidc", a)) == T._slug(T._canonicalise("oidc", b)) + assert T.userkey("oidc", a) != T.userkey("oidc", b) + + +def test_slug_collapses_to_nothing_still_yields_a_legal_label(): + for userid in ("....", "@@@", " ", "///"): + k = T.userkey("static", userid) + assert DNS1123.match(k), k + assert "--" not in k, f"{k!r} has an empty slug segment" + + +def test_issuer_is_load_bearing_not_cosmetic(): + """Same userid under two issuers must be two tenants. + + Without the issuer inside the hashed input, configuring a second issuer becomes a + way to impersonate an identity on the first. + """ + gh = T.userkey("github", "alice") + oi = T.userkey("oidc", "alice") + assert gh != oi + # Not merely different prefixes — the digests must differ too, or stripping the + # prefix (as a future rename or a shorter tag might) would re-merge them. + assert gh.rsplit("-", 1)[1] != oi.rsplit("-", 1)[1] + + +def test_issuer_userid_boundary_is_unambiguous(): + """issuer `a` + userid `b-c` must not hash like issuer `a-b` + userid `c`. + + This is why the digest input is joined with `\\x1f` rather than a character that can + appear in either field. + """ + assert T.userkey("a", "b-c") != T.userkey("a-b", "c") + + +def test_unknown_issuer_gets_the_xx_tag_and_still_separates(): + k = T.userkey("whatever", "alice") + assert k.startswith("xx-") + assert k != T.userkey("github", "alice") + # Two unknown issuers must not merge just because they share the `xx` tag: the + # digest covers the real issuer string, not the tag. + assert T.userkey("iss-one", "alice") != T.userkey("iss-two", "alice") + + +# ---- the agreement with ghauth.is_allowed ----------------------------------- + +@pytest.mark.parametrize("login", ["Alice", "alice", "ALICE", "aLiCe"]) +def test_github_case_folding_agrees_with_is_allowed(login): + """§2.3: if the allow-list folds case and the key does not, every spelling is + approved and each lands in a different tenant — one human with their history split + in half, which reads as data loss. So the two functions have to agree, and this is + the cheap place to find out that they do not.""" + allowed = ghauth.parse_allowed_users("alice") + assert ghauth.is_allowed(login, allowed) + assert T.userkey("github", login) == T.userkey("github", "alice") + + +def test_github_whitespace_is_stripped_like_the_allow_list(): + allowed = ghauth.parse_allowed_users(" alice , bob ") + assert ghauth.is_allowed("alice", allowed) + assert T.userkey("github", " alice ") == T.userkey("github", "alice") + + +def test_email_local_part_stays_case_sensitive_but_domain_does_not(): + """Only what RFC 5321 permits. Erring toward two tenants for one human is + wasteful; erring the other way is a leak.""" + assert T.userkey("oidc", "Alice@example.com") != T.userkey("oidc", "alice@example.com") + assert T.userkey("oidc", "alice@EXAMPLE.com") == T.userkey("oidc", "alice@example.com") + + +def test_email_with_at_in_the_local_part_splits_on_the_last_at(): + assert T._canonicalise("oidc", '"a@b"@EXAMPLE.com') == '"a@b"@example.com' + + +def test_static_identities_are_byte_exact(): + """An operator typed these; normalising on their behalf is how two of them merge.""" + assert T.userkey("static", "Ops") != T.userkey("static", "ops") + + +# ---- TopicSet -------------------------------------------------------------- + +def test_single_mode_reproduces_phase2_topics_exactly(): + """Step 2 of §9's rollout: zero behaviour change. The existing suite is the real + check, but this pins the intent.""" + ts = T.TopicSet("kev1", tenancy=T.SINGLE, + request_topic="requests", response_topic="responses") + assert ts.requests() == "requests" + assert ts.responses() == "responses" + # A userkey is ignored entirely in single mode rather than changing the answer. + assert ts.requests("gh-alice-a1b2c3d4") == "requests" + assert ts.responses("gh-alice-a1b2c3d4") == "responses" + assert not ts.multi + + +def test_single_mode_honours_configured_topic_names(): + ts = T.TopicSet("kev1", request_topic="my-reqs", response_topic="my-resps") + assert ts.requests() == "my-reqs" + assert ts.responses() == "my-resps" + + +def test_multi_mode_names(): + ts = T.TopicSet("kev1", tenancy=T.MULTI) + uk = "gh-mrsabath-4c1d9e07" + assert ts.requests(uk) == "kev1-gh-mrsabath-4c1d9e07-requests" + assert ts.responses(uk) == "kev1-gh-mrsabath-4c1d9e07-responses" + assert ts.events(uk) == "kev1-gh-mrsabath-4c1d9e07-events" + assert ts.dead(uk) == "kev1-gh-mrsabath-4c1d9e07-dead" + assert ts.multi + + +def test_multi_mode_refuses_to_name_a_topic_without_a_userkey(): + """Falling back to a shared topic here would publish one tenant's work onto + another tenant's runner, under that tenant's credential.""" + ts = T.TopicSet("kev1", tenancy=T.MULTI) + for method in (ts.requests, ts.responses, ts.events, ts.dead): + with pytest.raises(ValueError, match="userkey is required"): + method(None) + with pytest.raises(ValueError, match="userkey is required"): + method("") + + +def test_longest_realistic_topic_name_fits_kafkas_limit(): + """§3.1 computes 51 against Kafka's 249. Worth checking rather than assuming.""" + uk = T.userkey("oidc", "a" * 64 + "@" + "b" * 60 + ".example.com") + longest = T.TopicSet("kev1", tenancy=T.MULTI).responses(uk) + assert len(longest) <= 249 + assert len(longest) < 60, f"unexpectedly long: {longest!r} ({len(longest)})" + assert KAFKA_TOPIC.match(longest) + + +def test_tenancy_mode_is_validated(): + with pytest.raises(ValueError): + T.TopicSet("kev1", tenancy="sort-of") + + +def test_single_mode_inbox_and_dead_are_derived_from_the_prefix(): + """Unlike requests/responses these have no Phase 2 predecessor to stay + compatible with, so there is nothing to configure them from.""" + ts = T.TopicSet("kev1") + assert ts.events() == "kev1-events" + assert ts.dead() == "kev1-dead" + + +# ---- ntfy_topic ------------------------------------------------------------ + +SECRET = b"a-test-ntfy-topic-secret" + + +def test_ntfy_topic_shape_and_legality(): + t = T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", SECRET) + assert NTFY_TOPIC.match(t), t + assert t.startswith("kev1-") + assert len(t) == len("kev1-") + 26 + assert t == t.lower() + + +def test_ntfy_topic_contains_no_identifier(): + """§4.1 is the whole design: the name travels to a phone.""" + t = T.ntfy_topic("kev1", T.userkey("github", "mrsabath"), SECRET) + assert "mrsabath" not in t + assert "gh" not in t.removeprefix("kev1-") + + +def test_ntfy_topic_is_stable_and_per_user(): + a = T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", SECRET) + b = T.ntfy_topic("kev1", "gh-bob-9e8f7a6b", SECRET) + assert a == T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", SECRET) + assert a != b + + +def test_ntfy_topic_is_keyed_not_merely_hashed(): + """The userkey is public, so an unkeyed digest would be computable by anyone. + Rotating the secret must rotate every topic.""" + a = T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", SECRET) + b = T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", b"a-different-secret") + assert a != b + + +def test_ntfy_topic_carries_enough_entropy(): + """26 base32 characters is 130 bits. Assert the tag is not accidentally short or + padded, which truncation bugs produce.""" + tag = T.ntfy_topic("kev1", "gh-alice-a1b2c3d4", SECRET).removeprefix("kev1-") + assert len(tag) == 26 + assert "=" not in tag + assert set(tag) <= set("abcdefghijklmnopqrstuvwxyz234567") From bf90bdae5b4ccd7f15a31c117685ea2c018d4aa9 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 17:47:03 -0400 Subject: [PATCH 02/14] =?UTF-8?q?feat(eventing):=20Phase=203=20steps=202-4?= =?UTF-8?q?=20=E2=80=94=20TopicSet=20wiring,=20user=20registry,=20signed?= =?UTF-8?q?=20tenancy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN_PHASE3.md T2, T3, T4 and T22. Still no behaviour change in the default configuration: `EB_TENANCY_MODE=single` is the default, and the 656 tests that existed before this branch all pass untouched. That is the property §9 asks steps 1-4 to preserve — refactors with tests, everything risky behind a flag that is off. T2 — TopicSet threaded through (§3.2) One set, built once in `__main__` and handed to the producer, the responses consumer, both mirrors and the config, so §3.1's layout appears exactly once. `publish_request` takes a `userkey` and resolves the topic from it; one producer can send to any topic, so there is no second producer and no connection per user. The explicit-topic-list option, not `subscribe(pattern=...)`. A pattern is picked up by metadata refresh (5 min by default), so a new user's first response can be published before anyone is subscribed and with `auto_offset_reset=latest` it is never read — the page stays empty and no error appears anywhere. The blocking `ensure_subscribed` that closes the remaining race is T6. GroupMirror takes the full topic list directly: it is one-shot at startup, so there is no new-user race to lose, and a tenant whose topic is not provisioned yet is skipped with a log line rather than costing everyone their group history. T3 — the registry and Caller (§2.4, §2.5) `Caller` carries the tenancy key so no caller derives it twice. `submitter_iss` is deliberately not the same as `issuer`: a static identity needs an issuer for its key to separate from a GitHub login of the same name, but Phase 2 made an absent `ce_submitteriss` mean "an operator typed this", and promoting it would upgrade an assertion into a verified claim. The registry refuses rather than skips — the opposite of `auth.parse_tokens`, because a skipped user is one whose topics exist and whose events have nowhere to go. A recorded userkey that disagrees with this build's derivation is a startup refusal naming both values, and two identities colliding on one key is refused, which is what covers §2.3's 32-bit birthday bound. `multi` mode refuses anonymous with 401 and an unregistered user with 403. Both disagree with the single-tenant defaults on purpose; `single` is byte-identical to Phase 2. T4 — SIGNED_ATTRS += userkey, depth (§8.3) One change, both attributes, because adding one changes canonicalisation and a signer and verifier on different versions disagree about every signature. Both have to be covered rather than merely present: a mutable `userkey` lets anything with topic write access file events into another user's history, and a resettable `depth` is not a hop limit. T22 — the Secret-vs-ConfigMap rule, pinned in test_manifests.py The secret-path variables are declared ahead of the tasks that read them (T8-T16) so the guard is in place when they land. The classification test only asserts the ConfigMap side against live code, and says so, rather than looking stronger than it is. Known gap, stated rather than hidden: `/continue` resolves the owning tenant from the recorded submitter so a resume reaches the right runner, but does not yet check that the caller IS the owner — owner-scoped reads and `?k=` keys are §6.2/§4.4 (T7/T9, step 5). Until those land, `multi` mode's `/continue` is as open as Phase 2's. The lookup also cannot distinguish a GitHub `alice` from a static `alice`; `multi` mode refuses both anonymous and static submissions, so GitHub is the only issuer that reaches it today, and this moves to T5's global index when that exists. Tests: 66 new (722 total), ruff clean. Verified both services start in every tenancy configuration and that all four refusals print readable messages rather than tracebacks. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventbridge/__main__.py | 54 ++- eventing/eventbridge/auth.py | 108 ++++++ eventing/eventbridge/config.py | 42 +++ eventing/eventbridge/group_service.py | 7 +- eventing/eventbridge/handlers.py | 80 ++++- eventing/eventbridge/kafka_group_mirror.py | 45 ++- eventing/eventbridge/kafka_in.py | 16 +- eventing/eventbridge/kafka_out.py | 33 +- eventing/eventbridge/kafka_requests_mirror.py | 13 +- eventing/eventbridge/registry.py | 221 ++++++++++++ eventing/eventrunner/__main__.py | 7 +- eventing/eventrunner/emit.py | 12 +- eventing/shared/signing.py | 16 +- eventing/tests/test_manifests.py | 77 +++++ eventing/tests/test_registry.py | 321 ++++++++++++++++++ eventing/tests/test_signed_tenancy_attrs.py | 117 +++++++ eventing/tests/test_topicset_wiring.py | 201 +++++++++++ 17 files changed, 1333 insertions(+), 37 deletions(-) create mode 100644 eventing/eventbridge/registry.py create mode 100644 eventing/tests/test_registry.py create mode 100644 eventing/tests/test_signed_tenancy_attrs.py create mode 100644 eventing/tests/test_topicset_wiring.py diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index 146f3ecf..4fe6d958 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -7,6 +7,7 @@ import sys import threading +from eventbridge import registry from eventbridge.config import load from eventbridge.correlation import Minter from eventbridge.group_service import GroupService @@ -76,11 +77,50 @@ def main() -> int: print("[eventbridge] response verification OFF " "(set EB_VERIFY_KEYSET_PATH to enable)") + # Phase 3 §3.2: one TopicSet, built once here and handed to everything that names a + # topic, so §3.1's layout appears exactly once in the codebase. In `single` mode + # every method returns the configured request/response topic, which is what makes + # this step a pure refactor — the existing suite is the check. + topics = cfg.topics + print(f"[eventbridge] tenancy={cfg.tenancy_mode}" + + (f" prefix={cfg.topic_prefix}" if topics.multi else + f" requests={cfg.request_topic} responses={cfg.response_topic}")) + + # §2.5 — a bad registry must stop startup, but as a readable refusal rather than a + # traceback: this is an operator-facing configuration error, and the message (which + # names both the recorded and the derived userkey) is the whole point. A traceback + # buries it under frames nobody reading `kubectl logs` needs. + try: + users = registry.load(cfg.user_registry_path) + except registry.RegistryError as e: + raise SystemExit(f"[eventbridge] user registry: {e}") from None + if topics.multi: + if not cfg.user_registry_path: + raise SystemExit( + "[eventbridge] EB_TENANCY_MODE=multi requires EB_USER_REGISTRY_PATH. " + "Per-user isolation has no meaning without a list of users to isolate. " + "See DESIGN_PHASE3.md §2.5.") + if not len(users): + # Phase 2 §2.4's reading, for Phase 2's reason: the other one turns a + # missing file into an open door. + raise SystemExit( + f"[eventbridge] the user registry at {cfg.user_registry_path} approves " + f"nobody. An empty registry denies everyone; add a user or run with " + f"EB_TENANCY_MODE=single.") + print(f"[eventbridge] registry: {len(users)} user(s) — " + f"{', '.join(users.userkeys)}") + elif len(users): + # Harmless, but worth saying: a registry that is loaded and ignored is usually + # somebody who set the path and forgot the mode. + print(f"[eventbridge] registry has {len(users)} user(s) but " + f"EB_TENANCY_MODE=single, so it is not consulted") + # The producer needs the RESPONSES topic too: group lifecycle events go there, # not on requests, because EventRunner would try to execute anything on requests. producer = Producer(cfg.kafka_bootstrap, cfg.request_topic, cfg.source_uri, response_topic=cfg.response_topic, - seed=seed, kid=cfg.signing_kid or None) + seed=seed, kid=cfg.signing_kid or None, + topics=topics) groups = GroupService(cfg, store, producer, minter) ntfy = NtfyPublisher(cfg.ntfy, cfg.public_base_url, store=store) @@ -93,22 +133,26 @@ def main() -> int: on_member_event=groups.on_member_event, keyset=ks, require_signature=cfg.require_response_signature, - bridge_kid=cfg.signing_kid or None) + bridge_kid=cfg.signing_kid or None, + topics=topics) consumer.start() # Back-fill prompts from the requests topic — also gives us prompt visibility # for correlations we didn't originate ourselves. - requests_mirror = RequestsMirror(cfg.kafka_bootstrap, cfg.request_topic, store) + requests_mirror = RequestsMirror(cfg.kafka_bootstrap, cfg.request_topic, store, + topics=topics) requests_mirror.start() # §21.2: rebuild group history from the responses topic. The live consumer above # commits offsets and so never re-reads, which meant a restarted pod (with an # emptyDir /data) served 404 for every earlier group even though its notifications # had already gone out. One-shot, never publishes, never notifies. - group_mirror = GroupMirror(cfg.kafka_bootstrap, cfg.response_topic, groups) + group_mirror = GroupMirror(cfg.kafka_bootstrap, cfg.response_topic, groups, + topics=topics) group_mirror.start() - h = Handlers(cfg, store, producer, minter, groups=groups) + h = Handlers(cfg, store, producer, minter, groups=groups, + registry=users if topics.multi else None) dsp = Dispatcher() dsp.add("POST", r"/v0/agents", h.start_agent) dsp.add("POST", r"/v0/agents/(?P[a-z0-9-]+)/continue", h.continue_agent) diff --git a/eventing/eventbridge/auth.py b/eventing/eventbridge/auth.py index da05d5ed..e979f8e6 100644 --- a/eventing/eventbridge/auth.py +++ b/eventing/eventbridge/auth.py @@ -37,12 +37,73 @@ from __future__ import annotations import hmac +from dataclasses import dataclass from typing import Any +from shared import tenancy + # Returned verbatim as the `WWW-Authenticate` value on a 401 so a client knows # which scheme to retry with. CHALLENGE = 'Bearer realm="eventbridge"' +# The issuer string recorded for a static `EB_AUTH_TOKENS` identity. `ce_submitteriss` +# stays ABSENT for these (Phase 2 §2.6: absent issuer = an operator typed this name), but +# the tenancy key still needs an issuer in its hashed input — otherwise a static `alice` +# and a GitHub `alice` would be one tenant, which is the §2.3 impersonation failure. +ISSUER_STATIC = "static" + + +@dataclass(frozen=True) +class Caller: + """Who is asking, and the tenancy key that follows from it. §2.4. + + `userkey` is carried here so no caller derives it twice: it is needed to pick a + topic, a store and an ntfy topic, and three independent derivations are three + chances to disagree. + + `None` for all of `userid`/`issuer`/`userkey` is the anonymous case, which is what + Phase 2's "no auth configured" default produces and is what keeps the demo working + out of the box. §2.4 is explicit that this is incompatible with per-user isolation — + there is no user to isolate — so `multi` mode refuses it with `401`. The two modes + disagree about this on purpose. + """ + userid: str | None = None + issuer: str | None = None + userkey: str | None = None + tier: str = "shared" + + @property + def anonymous(self) -> bool: + return self.userid is None + + @property + def submitter_iss(self) -> str | None: + """What rides on the event as `ce_submitteriss`. + + Deliberately NOT the same as `issuer`: a static identity has an issuer here (so + its tenancy key separates from a GitHub login of the same name) but must stay + absent on the wire, because Phase 2 made absence mean "an operator typed this". + Collapsing the two would quietly upgrade an operator's assertion into a verified + claim. + """ + return None if self.issuer in (None, ISSUER_STATIC) else self.issuer + + +def caller_for(userid: str | None, issuer: str | None, *, + tenancy_mode: str = tenancy.SINGLE, tier: str = "shared") -> Caller: + """Build a `Caller`, deriving the tenancy key exactly once. + + The key is derived only in `multi` mode. In `single` mode it stays `None`, which is + what keeps `ce_userkey` off the wire entirely for a Phase 2 deployment — the + attribute is additive, and an event that does not need it does not carry it. + """ + if userid is None: + return Caller() + key = None + if tenancy_mode == tenancy.MULTI: + key = tenancy.userkey(issuer or ISSUER_STATIC, userid) + return Caller(userid=userid, issuer=issuer, userkey=key, tier=tier) + def parse_tokens(raw: str) -> dict[str, str]: """`"alice:tok1,bob:tok2"` -> `{"tok1": "alice", "tok2": "bob"}`. @@ -205,3 +266,50 @@ def resolve(environ: dict[str, Any], cfg, *, cache=None, if why: return None, None, 401, why return name, None, None, None + + +def resolve_caller(environ: dict[str, Any], cfg, *, cache=None, fetch=None, + registry=None) -> tuple[Caller | None, int | None, str | None]: + """`resolve`, plus the tenancy key and the registry check. `(caller, status, reason)`. + + This is the Phase 3 entry point; `resolve` keeps its 4-tuple shape so Phase 2's + callers and tests are untouched. + + Two refusals exist only in `multi` mode, and both are deliberate disagreements with + the single-tenant defaults rather than oversights: + + * **Anonymous is `401`.** Phase 2's "no auth configured means allowed and anonymous" + is what makes the demo work out of the box, and it cannot coexist with per-user + isolation: there is no user to isolate, so there is no store to write to and no + topic to publish on. + * **Not in the registry is `403`.** The registry is the approved list in `multi` + mode, and an empty one approves nobody — Phase 2 §2.4's reading, for Phase 2's + reason. `403` rather than `404` follows Phase 2 §2.4 too: this is the "I know + exactly who you are and you are not approved" case, and naming the refused + identity is what makes it actionable for someone who already knows who they are. + (§6.2's `404`-not-`403` rule is a different question — hiding whether another + tenant's correlation exists — and does not apply here.) + """ + userid, issuer, status, reason = resolve(environ, cfg, cache=cache, fetch=fetch) + if status: + return None, status, reason + + mode = getattr(cfg, "tenancy_mode", tenancy.SINGLE) + if mode != tenancy.MULTI: + return caller_for(userid, issuer, tenancy_mode=mode), None, None + + if userid is None: + return None, 401, ("this deployment is multi-tenant; an authenticated identity " + "is required (EB_TENANCY_MODE=multi)") + + user = registry.lookup(issuer, userid) if registry is not None else None + if user is None: + return None, 403, f"{userid} is not in the user registry" + + caller = caller_for(userid, issuer, tenancy_mode=mode, tier=user.tier) + # The registry's recorded key is authoritative when present: `parse` has already + # refused any file whose recorded key disagrees with this build's derivation, so + # these are equal — using the registry's value documents which one wins if that + # check is ever relaxed. + return Caller(userid=caller.userid, issuer=caller.issuer, + userkey=user.userkey, tier=user.tier), None, None diff --git a/eventing/eventbridge/config.py b/eventing/eventbridge/config.py index e9dcad90..517d9f34 100644 --- a/eventing/eventbridge/config.py +++ b/eventing/eventbridge/config.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from eventbridge import auth, ghauth +from shared import tenancy @dataclass @@ -77,8 +78,38 @@ class Cfg: # Enforcement rewrites persisted rows and raises a priority-5 notification, so # there has to be a way to watch the reject rate before turning it on. require_response_signature: bool = False + + # ---- Phase 3 (DESIGN_PHASE3.md §8.1) ---- + + # §3.2 / §8.1 — `single` is Phase 2 behaviour: one topic pair, one store, no + # ownership checks. `multi` is everything in §3-§7. Default `single` so an existing + # deployment that upgrades and changes nothing behaves exactly as it did. + tenancy_mode: str = tenancy.SINGLE + # §3.1 — first component of every per-user topic name. Unused in `single` mode, + # where the configured request/response topics are returned verbatim. + topic_prefix: str = "kev1" + # §2.5 — the user registry ConfigMap. Empty in `multi` mode denies everyone, for + # the same reason Phase 2's empty allowed_users does: the other reading turns a + # missing file into an open door. + user_registry_path: str = "" + ntfy: NtfyCfg = field(default_factory=NtfyCfg) + @property + def topics(self) -> tenancy.TopicSet: + """§3.2's helper, built from this config. + + A property rather than a field so it cannot drift from `request_topic` / + `response_topic` after `load()` — the topic names and the set that derives from + them are one fact, and storing it twice is how they disagree. + """ + return tenancy.TopicSet( + self.topic_prefix, + tenancy=self.tenancy_mode, + request_topic=self.request_topic, + response_topic=self.response_topic, + ) + def _root_tmpdir() -> str: base = os.environ.get("TMPDIR", "/tmp").rstrip("/") @@ -149,6 +180,17 @@ def load() -> Cfg: e("EB_REQUIRE_RESPONSE_SIGNATURE", "true" if cfg.require_response_signature else "false").lower() == "true") + # §8.1. Env only for the mode and the registry path: tenancy is a deployment-shape + # decision, and a committed config.toml that could flip it on is a way to get + # per-user isolation half-enabled by accident. + cfg.tenancy_mode = (e("EB_TENANCY_MODE", cfg.tenancy_mode) or "").strip().lower() + if cfg.tenancy_mode not in (tenancy.SINGLE, tenancy.MULTI): + raise SystemExit( + f"EB_TENANCY_MODE must be {tenancy.SINGLE!r} or {tenancy.MULTI!r}, " + f"got {cfg.tenancy_mode!r}. See DESIGN_PHASE3.md §8.1.") + cfg.topic_prefix = e("EB_TOPIC_PREFIX", cfg.topic_prefix) + cfg.user_registry_path = e("EB_USER_REGISTRY_PATH", cfg.user_registry_path) + cfg.ntfy.enabled = (e("NTFY_ENABLED", "true" if cfg.ntfy.enabled else "false").lower() == "true") cfg.ntfy.base_url = e("NTFY_BASE_URL", cfg.ntfy.base_url) cfg.ntfy.topic = e("NTFY_TOPIC", cfg.ntfy.topic) diff --git a/eventing/eventbridge/group_service.py b/eventing/eventbridge/group_service.py index 9d4ed26b..066da4a7 100644 --- a/eventing/eventbridge/group_service.py +++ b/eventing/eventbridge/group_service.py @@ -56,7 +56,9 @@ def _publish_started(self, groupid, label, expected, min_success, deadline) -> N def submit_members(self, groupid: str, prompts: list[str], *, max_turns: int = 3, model: str | None = None, submitter: str | None = None, - submitter_iss: str | None = None) -> list[str]: + submitter_iss: str | None = None, + userkey: str | None = None, + agent: str | None = None) -> list[str]: """Publish every member's request, recording membership first. Membership is recorded BEFORE the request is published so a fast agent's @@ -74,7 +76,8 @@ def submit_members(self, groupid: str, prompts: list[str], *, self.producer.publish_request( prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", groupid=groupid, - submitter=submitter, submitter_iss=submitter_iss) + submitter=submitter, submitter_iss=submitter_iss, + userkey=userkey, agent=agent) corrs.append(corr) return corrs diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index 2bdc593e..72a16371 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -92,21 +92,85 @@ def _read_form(environ) -> dict: class Handlers: def __init__(self, cfg: Cfg, store: Store, producer: Producer, minter: Minter, - groups=None) -> None: + groups=None, registry=None) -> None: self.cfg = cfg self.store = store self.producer = producer self.minter = minter self.groups = groups + # §2.5 — the approved-user registry, or None in single-tenant mode, where there + # is nothing to look a caller up in. + self.registry = registry # One cache for the process, so a burst of requests from the same user # costs one GitHub call rather than one each. self.logins = ghauth.LoginCache(cfg.github_cache_ttl_s) + def _agent_for(self, caller, body) -> str | None: + """§5.1's resolution order, for the part EventBridge controls. + + The request may name an agent, otherwise the registry's default for this user. + Anything below that (`ER_AGENT_NAME`, then `default`) is the runner's decision + and is deliberately not second-guessed here — EventBridge does not know which + specs a given runner image has baked in. + """ + if named := (body or {}).get("agent"): + return str(named).strip() or None + if self.registry is not None and caller.userid: + user = self.registry.by_userkey(caller.userkey) if caller.userkey else None + if user and user.agent: + return user.agent + return None + + def _caller(self, environ): + """§2.4: resolve the caller once, carrying the tenancy key. + + In `single` mode this is Phase 2's `auth.resolve` with the key left `None`, so + `ce_userkey` never reaches the wire and behaviour is unchanged. + """ + return auth.resolve_caller(environ, self.cfg, cache=self.logins, + registry=self.registry) + + def _owner_userkey(self, correlationid: str) -> str | None: + """Which tenant owns this correlation, for a `/continue` turn. + + `None` in single-tenant mode, where topics do not depend on it. + + `/continue` is still unauthenticated here — Phase 2 §3.1's capability-URL + design — so this resolves the owner from what was *recorded at submit time* + rather than from the caller. The turn therefore reaches the right tenant's + runner, which is what stops a resume from being executed under another user's + credential. It does NOT yet check that the caller is that owner: owner-scoped + reads and `?k=` capability keys are §6.2/§4.4 (T7/T9, rollout step 5), and until + they land `multi` mode's `/continue` is as open as Phase 2's. + + The lookup goes through the recorded submitter rather than a + `correlationid -> userkey` index because that index is T5's global table, which + does not exist yet. When it does, this becomes one indexed `SELECT` and the + derivation here goes away — §2.6 is explicit that the index has to exist anyway, + for the uniqueness guarantee that keeps `sessionuuid` unsalted. + """ + if not self.cfg.topics.multi: + return None + prompts = self.store.get_prompts(correlationid) or [] + for p in prompts: + if submitter := p.get("submitter"): + # The issuer is not recorded alongside the submitter today, so this + # cannot distinguish a GitHub `alice` from a static `alice`. In `multi` + # mode anonymous and static submissions are already refused at the + # submit path (§2.4), so GitHub is the only issuer that reaches here — + # stated rather than assumed, because the day an OIDC issuer is wired + # up (§2.4's slot) this becomes wrong and has to move to the index. + if user := (self.registry.lookup("github", submitter) + if self.registry else None): + return user.userkey + return None + # ---- start ---- def start_agent(self, environ, start_response, **_): - submitter, sub_iss, status, why = auth.resolve(environ, self.cfg, cache=self.logins) + caller, status, why = self._caller(environ) if status: return _deny(start_response, why, status) + submitter, sub_iss = caller.userid, caller.submitter_iss body = _read_json(environ) prompt = body.get("prompt") if not prompt: @@ -131,10 +195,12 @@ def start_agent(self, environ, start_response, **_): prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", groupid=groupid, submitter=submitter, submitter_iss=sub_iss, + userkey=caller.userkey, agent=self._agent_for(caller, body), ) out = { "correlationid": corr, "sessionuuid": sess, - "event_id": event_id, "topic": self.cfg.request_topic, + "event_id": event_id, + "topic": self.cfg.topics.requests(caller.userkey), "html_url": f"{self.cfg.public_base_url}/v0/agents/{corr}", } if groupid: @@ -151,9 +217,10 @@ def create_group(self, environ, start_response, **_): the batch scale at all: every request is published before any is watched, so lag reaches N and KEDA scales past one pod. """ - submitter, sub_iss, status, why = auth.resolve(environ, self.cfg, cache=self.logins) + caller, status, why = self._caller(environ) if status: return _deny(start_response, why, status) + submitter, sub_iss = caller.userid, caller.submitter_iss if self.groups is None: return _json(start_response, "503 Service Unavailable", {"error": "group support not enabled"}) @@ -189,7 +256,8 @@ def create_group(self, environ, start_response, **_): corrs = self.groups.submit_members( groupid, [str(x) for x in prompts], max_turns=int(body.get("max_turns", 3)), model=body.get("model"), - submitter=submitter, submitter_iss=sub_iss) + submitter=submitter, submitter_iss=sub_iss, + userkey=caller.userkey, agent=self._agent_for(caller, body)) return _json(start_response, "202 Accepted", { "groupid": groupid, "created": True, "expected": expected or len(corrs), "members": corrs, @@ -306,6 +374,7 @@ def continue_agent(self, environ, start_response, correlationid: str, **_): event_id = self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", + userkey=self._owner_userkey(correlationid), ) next_seq = len(self.store.events_for(correlationid)) + 1 return _json(start_response, "202 Accepted", { @@ -328,6 +397,7 @@ def continue_agent_html(self, environ, start_response, correlationid: str, **_): self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", + userkey=self._owner_userkey(correlationid), ) start_response("303 See Other", [("Location", f"/v0/agents/{correlationid}")]) return [b""] diff --git a/eventing/eventbridge/kafka_group_mirror.py b/eventing/eventbridge/kafka_group_mirror.py index 0a3b53c7..21805ad7 100644 --- a/eventing/eventbridge/kafka_group_mirror.py +++ b/eventing/eventbridge/kafka_group_mirror.py @@ -35,7 +35,7 @@ from kafka import KafkaConsumer, TopicPartition -from shared import ce +from shared import ce, tenancy def _log(msg: str) -> None: @@ -46,10 +46,22 @@ class GroupMirror(threading.Thread): """One-shot catch-up scan of the responses topic, rebuilding group state.""" def __init__(self, bootstrap: str, response_topic: str, groups, - metadata_tries: int = 20, max_empty_polls: int = 15) -> None: + metadata_tries: int = 20, max_empty_polls: int = 15, + topics: tenancy.TopicSet | None = None, + userkeys: tuple[str, ...] = ()) -> None: super().__init__(daemon=True, name="kafka-group-mirror") self._bootstrap_servers = bootstrap - self._topic = response_topic + # §3.2: unlike the live consumers this one is one-shot at startup, so it "can + # take the full list directly" — there is no new-user race to lose, because a + # user who did not exist when it ran has no group history to rebuild. + self._topics = topics or tenancy.TopicSet( + "", response_topic=response_topic, request_topic="") + if self._topics.multi: + self._all_topics = tuple(self._topics.responses(uk) for uk in userkeys) + else: + self._all_topics = (self._topics.responses(),) + # Kept for the log lines and the single-topic metadata path below. + self._topic = self._all_topics[0] if self._all_topics else response_topic self._groups = groups # Partitions are assigned by hand and nothing is ever committed: the whole # point is to re-read from the beginning on every start, which the live @@ -67,14 +79,25 @@ def stop(self) -> None: def _partitions(self, c): """Topic metadata is fetched lazily, so the first call can legitimately return - None. Retry briefly rather than concluding the topic is empty.""" - for _ in range(self._metadata_tries): - parts = c.partitions_for_topic(self._topic) - if parts: - return [TopicPartition(self._topic, p) for p in sorted(parts)] - if self._stopping.wait(0.5): - return [] - return [] + None. Retry briefly rather than concluding the topic is empty. + + With several topics (multi mode) the retry is over the whole set, and a topic + that never resolves is skipped with a log line rather than failing the rebuild: + one tenant whose topic has not been provisioned yet must not cost every other + tenant their group history.""" + found: list[TopicPartition] = [] + for topic in self._all_topics: + for _ in range(self._metadata_tries): + parts = c.partitions_for_topic(topic) + if parts: + found += [TopicPartition(topic, p) for p in sorted(parts)] + break + if self._stopping.wait(0.5): + return found + else: + if len(self._all_topics) > 1: + _log(f"no metadata for {topic!r}; skipping it") + return found def run(self) -> None: # No consumer group: we assign every partition by hand. A group would make this diff --git a/eventing/eventbridge/kafka_in.py b/eventing/eventbridge/kafka_in.py index 73ffe880..28423238 100644 --- a/eventing/eventbridge/kafka_in.py +++ b/eventing/eventbridge/kafka_in.py @@ -25,7 +25,7 @@ from kafka import KafkaConsumer from eventbridge.store import Store -from shared import ce, signing +from shared import ce, signing, tenancy class Consumer(threading.Thread): @@ -41,10 +41,22 @@ def __init__( keyset=None, require_signature: bool = False, bridge_kid: str | None = None, + topics: tenancy.TopicSet | None = None, ) -> None: super().__init__(daemon=True, name="kafka-responses-consumer") self._bootstrap_servers = bootstrap - self._topic = response_topic + # §3.2: the chosen option is an EXPLICIT topic list, re-subscribed when the + # registry changes — not `subscribe(pattern=...)`. A pattern is picked up by + # metadata refresh (`metadata.max.age.ms`, 5 minutes by default), so a new + # user's first response can be published before anyone is subscribed, and with + # `auto_offset_reset=latest` it is simply never read: the page stays empty, the + # group counter never advances, and no error appears anywhere. + # + # In single mode the list is the one configured topic, which is today's + # behaviour. The blocking `ensure_subscribed` that makes multi mode safe is T6. + self._topics = topics or tenancy.TopicSet( + "", response_topic=response_topic, request_topic="") + self._topic = self._topics.responses() if not self._topics.multi else response_topic self._store = store self._on_event = on_event self._on_group_event = on_group_event diff --git a/eventing/eventbridge/kafka_out.py b/eventing/eventbridge/kafka_out.py index 2e0aa649..63d5b1a5 100644 --- a/eventing/eventbridge/kafka_out.py +++ b/eventing/eventbridge/kafka_out.py @@ -3,7 +3,7 @@ from kafka import KafkaProducer -from shared import ce, signing +from shared import ce, signing, tenancy class Producer: @@ -31,7 +31,8 @@ class Producer: def __init__(self, bootstrap: str, request_topic: str, source_uri: str, response_topic: str | None = None, - seed: bytes | None = None, kid: str | None = None) -> None: + seed: bytes | None = None, kid: str | None = None, + topics: tenancy.TopicSet | None = None) -> None: self._prod = KafkaProducer(bootstrap_servers=bootstrap, acks="all", linger_ms=5) self._topic = request_topic self._response_topic = response_topic @@ -39,6 +40,12 @@ def __init__(self, bootstrap: str, request_topic: str, source_uri: str, # Held on the instance so no call site has to know about signing. self._seed = seed self._kid = kid + # §3.2: one producer can send to any topic — the topic is an argument to + # `send()`, not to the constructor — so per-user topics need no second producer + # and no connection per user. `None` builds a single-mode set from the two + # explicit topics above, which is exactly today's behaviour. + self._topics = topics or tenancy.TopicSet( + "", request_topic=request_topic, response_topic=response_topic or "") def publish_request( self, @@ -53,7 +60,16 @@ def publish_request( groupid: str | None = None, submitter: str | None = None, submitter_iss: str | None = None, + userkey: str | None = None, + agent: str | None = None, ) -> str: + """Publish one request. `userkey` selects the topic in multi-tenant mode. + + §2.6: `userkey` also rides on the event, because EventRunner stamps it back onto + the response and that is what tells EventBridge which store to file the response + in. It is in `signing.SIGNED_ATTRS` (T4) so it cannot be rewritten in flight. + """ + topic = self._topics.requests(userkey) event = ce.new_event( type=ce.TYPE_REQUEST, source=self._source, @@ -66,17 +82,20 @@ def publish_request( **({"groupid": groupid} if groupid else {}), **({ce.EXT_SUBMITTER: submitter} if submitter else {}), **({ce.EXT_SUBMITTER_ISS: submitter_iss} if submitter_iss else {}), + **({ce.EXT_USERKEY: userkey} if userkey else {}), + **({ce.EXT_AGENT: agent} if agent else {}), ) # After new_event (which fills `id` and `time`, both signed) and before # serialisation, so the signature covers exactly what goes on the wire. signing.sign_into(event, self._seed, self._kid) headers, value = ce.to_kafka_binary(event) - future = self._prod.send(self._topic, key=correlationid.encode(), value=value, headers=headers) + future = self._prod.send(topic, key=correlationid.encode(), value=value, headers=headers) future.get(timeout=5) return event["id"] def publish_group_event(self, *, type_: str, groupid: str, - data: dict, subject: str = "group") -> str: + data: dict, subject: str = "group", + userkey: str | None = None) -> str: """Publish a group lifecycle event to the RESPONSES topic (§21.2). Not requests: EventRunner consumes that topic and would treat a group event as @@ -89,8 +108,9 @@ def publish_group_event(self, *, type_: str, groupid: str, worth closing: a forged `group.completed` ends a batch early and fires a "finished" notification for work that never ran. """ - if not self._response_topic: + if not self._topics.multi and not self._response_topic: raise RuntimeError("Producer has no response_topic; cannot publish group events") + topic = self._topics.responses(userkey) event = ce.new_event( type=type_, source=self._source, @@ -98,10 +118,11 @@ def publish_group_event(self, *, type_: str, groupid: str, datacontenttype="application/json", groupid=groupid, data=data, + **({ce.EXT_USERKEY: userkey} if userkey else {}), ) signing.sign_into(event, self._seed, self._kid) headers, value = ce.to_kafka_binary(event) - self._prod.send(self._response_topic, key=groupid.encode(), + self._prod.send(topic, key=groupid.encode(), value=value, headers=headers).get(timeout=5) return event["id"] diff --git a/eventing/eventbridge/kafka_requests_mirror.py b/eventing/eventbridge/kafka_requests_mirror.py index c5618e9c..a319f74f 100644 --- a/eventing/eventbridge/kafka_requests_mirror.py +++ b/eventing/eventbridge/kafka_requests_mirror.py @@ -20,15 +20,22 @@ from kafka import KafkaConsumer from eventbridge.store import Store -from shared import ce +from shared import ce, tenancy class RequestsMirror(threading.Thread): def __init__(self, bootstrap: str, request_topic: str, store: Store, - group_id: str | None = None) -> None: + group_id: str | None = None, + topics: tenancy.TopicSet | None = None) -> None: super().__init__(daemon=True, name="kafka-requests-mirror") self._bootstrap_servers = bootstrap - self._topic = request_topic + # §3.2: "RequestsMirror needs the same treatment" as the responses consumer — + # it is what back-fills prompts for correlations the bridge did not originate, + # so a topic it is not subscribed to is a correlation with no prompt. The + # per-user subscription management is T6; this threads the set through. + self._topics = topics or tenancy.TopicSet( + "", request_topic=request_topic, response_topic="") + self._topic = self._topics.requests() if not self._topics.multi else request_topic self._store = store # Per-process group id so every EB start re-scans the whole topic. # `backfill_prompt_if_missing()` makes re-inserts a no-op, so this diff --git a/eventing/eventbridge/registry.py b/eventing/eventbridge/registry.py new file mode 100644 index 00000000..3430ca0a --- /dev/null +++ b/eventing/eventbridge/registry.py @@ -0,0 +1,221 @@ +"""The approved-user registry. DESIGN_PHASE3.md §2.5. + +Phase 2's `EB_ALLOWED_USERS` answers one question — may this person submit? Per-user +topics, quotas, agent defaults and a runner Deployment need more, so the list gains a +structured sibling. It does not *replace* it: an operator with ten logins in an env var +should not have to write a file, so both are read and `EB_ALLOWED_USERS` keeps working. + +Four decisions, each defending itself: + +* **`userkey` is written in the file, not only computed.** It is derived, so it could be + recomputed at every startup — but it names Kafka topics, Kubernetes objects and an + ntfy topic that already exist. If a future change to `_slug` or `ISS_PREFIX` shifted + the derivation, a recomputing bridge would silently start publishing to new, empty + topics and every user's history would appear to vanish. So the file records it, + startup recomputes it, and a mismatch is a **startup refusal naming both values**. + Pinning the output of a derivation you intend to keep stable is cheap; discovering it + drifted is not. +* **An empty registry denies everyone**, exactly as Phase 2 §2.4's empty list does, and + for the same reason: the other reading turns a missing file into an open door. +* **It is a ConfigMap, not a Secret.** Nothing in it grants access; it records what an + operator approved. Per-user ntfy tokens and Kafka SCRAM passwords are the parts that + *do* grant access and live in Secrets provisioned alongside. +* **Provisioning is not automatic.** A user in the registry whose topics do not exist + gets a `503` naming the missing topic, never a request published into the void. + `scripts/k8s_tenant.py` (T12) renders the objects from this same file. + +Stdlib only (`json`, `pathlib`). +""" +from __future__ import annotations + +import json +import pathlib +from dataclasses import dataclass, field + +from shared import tenancy + +# §3.8. `isolated` gets its own topics, runner Deployment and store; `shared` rides the +# shared tier's topics and store, which is what single-tenant mode uses for everyone. +TIER_SHARED = "shared" +TIER_ISOLATED = "isolated" +TIERS = (TIER_SHARED, TIER_ISOLATED) + +SUPPORTED_VERSION = 1 + + +class RegistryError(Exception): + """A registry that cannot be loaded, or whose contents disagree with themselves. + + Raised at startup rather than per-request: a bridge that half-loaded its tenancy + configuration cannot be trusted to route events, and the failure mode of continuing + is one user's events in another user's store. + """ + + +@dataclass(frozen=True) +class Limits: + """Per-user ceilings. 0 means "no limit from the registry".""" + requests_per_hour: int = 0 + max_concurrent: int = 0 + transcript_bytes: int = 0 + + +@dataclass(frozen=True) +class User: + issuer: str + userid: str + userkey: str + tier: str = TIER_SHARED + # §5.1: the agent this user's requests default to, when the request does not name + # one. Empty falls through to the runner's own ER_AGENT_NAME, then `default`. + agent: str = "" + limits: Limits = field(default_factory=Limits) + + @property + def isolated(self) -> bool: + return self.tier == TIER_ISOLATED + + +@dataclass(frozen=True) +class Registry: + """An immutable snapshot, loaded once at startup. + + Live reload is deliberately absent, matching `keyset.py`'s reasoning: a mid-run edit + must not silently widen the set of tenants this bridge serves, and a registry change + needs `k8s_tenant.py` to provision topics anyway, so it is already not a + file-edit-only operation. + """ + users: tuple[User, ...] = () + + def __len__(self) -> int: + return len(self.users) + + @property + def userkeys(self) -> tuple[str, ...]: + return tuple(u.userkey for u in self.users) + + def lookup(self, issuer: str | None, userid: str | None) -> User | None: + """Find an approved user by (issuer, userid). + + Matching goes through `tenancy.userkey` rather than comparing `userid` strings, + so the comparison inherits the per-issuer canonicalisation for free: a GitHub + login differing only in case matches, an email differing only in domain case + matches, and a static identity must be byte-exact. Re-implementing that compare + here is how the two drift apart. + """ + if not userid: + return None + want = tenancy.userkey(issuer or "static", userid) + for u in self.users: + if u.userkey == want: + return u + return None + + def by_userkey(self, userkey: str) -> User | None: + for u in self.users: + if u.userkey == userkey: + return u + return None + + +def parse(raw: str) -> Registry: + """Parse registry JSON. Every inconsistency is a refusal, never a skipped entry. + + This differs from `auth.parse_tokens`, which skips malformed entries on purpose — a + typo in one token must not take the service down, and a skipped token simply fails + to authenticate, which is closed. Here the failure direction is the opposite: a + skipped user is a user whose topics exist and whose events have nowhere to go, or + worse, one whose `userkey` collides with somebody else's. So this refuses loudly. + """ + try: + d = json.loads(raw) + except json.JSONDecodeError as e: + raise RegistryError(f"registry is not valid JSON: {e}") from e + if not isinstance(d, dict): + raise RegistryError("registry must be a JSON object") + + version = d.get("version") + if version != SUPPORTED_VERSION: + # Refusing an unknown version rather than reading what it recognises: a v2 file + # may well mean something different by a field this code thinks it understands. + raise RegistryError( + f"registry version {version!r} is not supported (expected {SUPPORTED_VERSION})") + + entries = d.get("users") + if entries is None: + entries = [] + if not isinstance(entries, list): + raise RegistryError("registry `users` must be a list") + + users: list[User] = [] + seen_keys: dict[str, str] = {} + for i, e in enumerate(entries): + where = f"users[{i}]" + if not isinstance(e, dict): + raise RegistryError(f"{where} must be an object") + issuer = str(e.get("issuer", "")).strip() + userid = str(e.get("userid", "")).strip() + if not issuer or not userid: + raise RegistryError(f"{where} needs both issuer and userid") + + computed = tenancy.userkey(issuer, userid) + declared = str(e.get("userkey", "")).strip() + if declared and declared != computed: + # The startup refusal §2.5 asks for, naming BOTH values — the whole point is + # that an operator can see which one the live topics were named after. + raise RegistryError( + f"{where} ({issuer}/{userid}): registry records userkey {declared!r} but " + f"this build derives {computed!r}. The recorded value names topics and " + f"Kubernetes objects that already exist, so this build would publish to " + f"new, empty topics and the user's history would appear to vanish. " + f"Refusing to start. See DESIGN_PHASE3.md §2.5.") + + # The 32-bit birthday bound from §2.3, turned from a silent cross-tenant leak + # into a refused signup. Two DIFFERENT identities producing one key is the + # failure; the same identity listed twice is merely a duplicate, and is also + # refused because it makes the per-user limits ambiguous. + if computed in seen_keys: + raise RegistryError( + f"{where} ({issuer}/{userid}) collides with {seen_keys[computed]} on " + f"userkey {computed!r}. Two identities cannot share a tenancy key.") + seen_keys[computed] = f"{issuer}/{userid}" + + tier = str(e.get("tier", TIER_SHARED)).strip() or TIER_SHARED + if tier not in TIERS: + raise RegistryError(f"{where}: tier {tier!r} not one of {TIERS}") + + lim = e.get("limits") or {} + if not isinstance(lim, dict): + raise RegistryError(f"{where}: limits must be an object") + try: + limits = Limits( + requests_per_hour=int(lim.get("requests_per_hour", 0)), + max_concurrent=int(lim.get("max_concurrent", 0)), + transcript_bytes=int(lim.get("transcript_bytes", 0)), + ) + except (TypeError, ValueError) as exc: + raise RegistryError(f"{where}: limits must be integers") from exc + + users.append(User(issuer=issuer, userid=userid, userkey=computed, tier=tier, + agent=str(e.get("agent", "")).strip(), limits=limits)) + + return Registry(users=tuple(users)) + + +def load(path: str) -> Registry: + """Load the registry from `EB_USER_REGISTRY_PATH`. Empty path = empty registry. + + An empty registry is returned rather than raising, because whether "nobody is + approved" is an error depends on the tenancy mode: it is fatal in `multi` and + irrelevant in `single`. `__main__` decides that, where the mode is known. + """ + if not path: + return Registry() + p = pathlib.Path(path) + if not p.is_file(): + raise RegistryError(f"registry path {path!r} does not exist") + try: + raw = p.read_text() + except OSError as e: + raise RegistryError(f"cannot read registry {path!r}: {e}") from e + return parse(raw) diff --git a/eventing/eventrunner/__main__.py b/eventing/eventrunner/__main__.py index 3b70d159..6c31fd05 100644 --- a/eventing/eventrunner/__main__.py +++ b/eventing/eventrunner/__main__.py @@ -81,7 +81,12 @@ def main() -> int: "set ER_VERIFY_KEYSET_PATH for an approved-agent list") emitter = Emitter(cfg.kafka_bootstrap, cfg.response_topic, cfg.source_uri, - seed=seed, kid=cfg.signing_kid or None) + seed=seed, kid=cfg.signing_kid or None, + userkey=cfg.userkey or None) + # Phase 3 §3.3/§5.1. Printed because both are silent otherwise: a runner serving the + # wrong tenant and a runner running the wrong agent both look like working pods. + print(f"[eventrunner] tenancy={'userkey=' + cfg.userkey if cfg.userkey else 'single'} " + f"agent={cfg.agent_name} agent_dir={cfg.agent_dir}") def _run(event): run_agent(cfg, emitter, event, transcripts=transcripts) diff --git a/eventing/eventrunner/emit.py b/eventing/eventrunner/emit.py index f127981a..57735262 100644 --- a/eventing/eventrunner/emit.py +++ b/eventing/eventrunner/emit.py @@ -29,12 +29,20 @@ class Emitter: """ def __init__(self, bootstrap: str, response_topic: str, source_uri: str, - seed: bytes | None = None, kid: str | None = None) -> None: + seed: bytes | None = None, kid: str | None = None, + userkey: str | None = None) -> None: self._prod = KafkaProducer(bootstrap_servers=bootstrap, acks="all", linger_ms=5) self._topic = response_topic self._source = source_uri self._seed = seed self._kid = kid + # Phase 3 §3.3: stamped on every response so EventBridge knows which store to + # file it in. Held here rather than threaded through `emit`'s callers because + # it is a property of the *runner* — this pod serves exactly one tenant — and a + # per-call parameter would be one more thing a call site could forget, with a + # misfiled response as the symptom. Empty in single-tenant mode, where the + # attribute is simply absent and behaviour is Phase 2's. + self._userkey = userkey or None self._seq_lock = threading.Lock() self._seq_by_corr: dict[str, int] = {} @@ -65,6 +73,8 @@ def emit(self, *, correlationid: str, sessionuuid: str, sequence: int, attrs["causationid"] = causationid if groupid: attrs["groupid"] = groupid + if self._userkey: + attrs[ce.EXT_USERKEY] = self._userkey event = ce.new_event( type=ce.TYPE_RESPONSE, source=self._source, diff --git a/eventing/shared/signing.py b/eventing/shared/signing.py index bf939e82..763b1929 100644 --- a/eventing/shared/signing.py +++ b/eventing/shared/signing.py @@ -192,7 +192,21 @@ def verify(message: bytes, signature: bytes, pub: bytes) -> bool: # signature says nothing about which batch an event belongs to, so a # forged `groupid` could move a response into another batch and # corrupt its fan-in counts. - "submitter", "submitteriss", "groupid") + "submitter", "submitteriss", "groupid", + # Phase 3 §2.6 and §7.7, added together in ONE change — §8.3's rule. + # Adding an attribute changes canonicalisation, so a signer and a + # verifier on different versions disagree about every signature; + # splitting this into two commits would invalidate canonicalisation + # twice for no benefit. Deployments with signing already enabled must + # upgrade both services together. With signing off (the default) there + # is nothing to coordinate, which is most deployments. + # + # Both have to be here rather than merely present on the event: + # `userkey` decides which store a response is written into and which + # ntfy topic announces it, so a mutable one lets anything with topic + # write access file events into another user's history. `depth` is the + # hop limit, and a resettable hop limit does not limit hops. + "userkey", "depth") def data_bytes(data: Any) -> bytes: diff --git a/eventing/tests/test_manifests.py b/eventing/tests/test_manifests.py index 8695a829..6e649e8b 100644 --- a/eventing/tests/test_manifests.py +++ b/eventing/tests/test_manifests.py @@ -155,6 +155,83 @@ def test_no_credential_is_ever_rendered_into_a_manifest(): assert marker not in rendered, f"{overlay} may leak a credential ({marker!r})" +#: Phase 3 §8.3, continuing Phase 2 §5's rule. Every variable naming a path that +#: contains key material must come from a **Secret** mount, never from the committed +#: `config.toml` and never with a literal value in a ConfigMap. The rule is only worth +#: having if a test enforces it, which is T22. +#: +#: NOTE: none of these are read by `config.py` yet — they arrive with T8 (transcript +#: auth), T9 (capability keys), T10 (derived ntfy topics) and T16 (webhook ingress). They +#: are listed now on purpose, so the guard is already in place when the variable lands +#: rather than being remembered afterwards. `test_every_phase3_variable_is_classified` +#: deliberately does NOT require these to be read, which is why it only asserts the +#: ConfigMap side against live code. +SECRET_PATH_VARS = ( + "EB_NTFY_TOPIC_SECRET_PATH", + "EB_CAPABILITY_SECRET_PATH", + "EB_WEBHOOK_SECRETS_PATH", +) + +#: The other side of the same rule: these record what an operator approved and grant +#: nothing, so a ConfigMap (or an image path) is correct and a Secret would be cargo +#: cult. Listed explicitly so the next variable added lands on one side deliberately. +CONFIGMAP_VARS = ( + "EB_TENANCY_MODE", + "EB_TOPIC_PREFIX", + "EB_USER_REGISTRY_PATH", + "ER_AGENT_DIR", + "ER_AGENT_NAME", + "ER_USERKEY", +) + + +def test_secret_path_variables_are_never_set_from_the_committed_config(): + """A seed path in `config.toml` means the whole signing/notification block can be + half-configured from a file in git.""" + toml = (ROOT / "eventbridge" / "config.toml").read_text() + for var in SECRET_PATH_VARS: + assert var not in toml, f"{var} must be env-only (DESIGN_PHASE3.md §8.3)" + # The TOML spelling of the same thing — a lower-cased key without the prefix. + key = var.removeprefix("EB_").lower() + assert key not in toml, f"{key} in config.toml reintroduces {var} by another name" + + +@needs_kubectl +def test_no_phase3_secret_is_inlined_into_a_manifest(): + """The §8.3 rule, checked where it would actually leak: rendered output.""" + for overlay in OVERLAYS: + rendered = render(overlay) + for var in SECRET_PATH_VARS: + # Referencing the variable is fine; giving it a literal sibling value is + # not. A `secretKeyRef` has no `value:` on the same key. + for bad in (f"{var}:", f"name: {var}\n value:"): + assert bad not in rendered, \ + f"{overlay} appears to inline {var} rather than mounting a Secret" + + +def test_every_phase3_variable_is_classified(): + """A variable that is in neither list is one nobody decided about. + + §8.3's rule only works if adding a variable forces the Secret-vs-ConfigMap question, + so this asserts the two lists together cover what `config.py` actually reads. + """ + import re as _re + sources = [(ROOT / "eventbridge" / "config.py").read_text(), + (ROOT / "eventrunner" / "config.py").read_text()] + found = set() + for src in sources: + found |= set(_re.findall(r'e\("((?:EB|ER)_[A-Z0-9_]+)"', src)) + # Every ConfigMap-side variable must actually be read by the code — a stale entry + # here would silently stop checking anything. The Secret-side ones are deliberately + # exempt: they are declared ahead of the tasks that introduce them (see the note on + # SECRET_PATH_VARS), and requiring them now would just force a placeholder read. + for var in CONFIGMAP_VARS: + assert var in found, f"{var} is listed here but no config.py reads it" + # The sanity check that this test is looking at the right thing at all. + assert "EB_TENANCY_MODE" in found and len(found) > 10, \ + "config.py does not look like it was parsed; did the `e(\"VAR\")` idiom change?" + + @needs_kubectl def test_the_demo_overlay_requires_the_credential_secret(): doc = find_doc(render("demo"), "Deployment", "eventrunner") diff --git a/eventing/tests/test_registry.py b/eventing/tests/test_registry.py new file mode 100644 index 00000000..9224f9db --- /dev/null +++ b/eventing/tests/test_registry.py @@ -0,0 +1,321 @@ +"""The approved-user registry and the `Caller` it produces. DESIGN_PHASE3.md §2.4, §2.5. + +Two refusals carry most of the weight here, and both are startup refusals rather than +per-request errors: + +* a recorded `userkey` that disagrees with this build's derivation (§2.5) — the + alternative is a bridge that publishes into new, empty topics while every user's + history appears to vanish; +* two identities colliding on one key (§2.3's 32-bit birthday bound) — the alternative + is a silent cross-tenant leak. +""" +from __future__ import annotations + +import json + +import pytest + +from eventbridge import auth, registry +from eventbridge.config import Cfg +from shared import tenancy + + +def _reg(users, version=1) -> str: + return json.dumps({"version": version, "users": users}) + + +def _gh(userid, **over): + e = {"issuer": "github", "userid": userid, + "userkey": tenancy.userkey("github", userid)} + e.update(over) + return e + + +# ---- parsing --------------------------------------------------------------- + +def test_parse_a_full_registry(): + raw = _reg([ + {"issuer": "github", "userid": "mrsabath", + "userkey": tenancy.userkey("github", "mrsabath"), + "tier": "isolated", "agent": "triager", + "limits": {"requests_per_hour": 100, "max_concurrent": 3, + "transcript_bytes": 268435456}}, + _gh("aslom", tier="isolated"), + {"issuer": "static", "userid": "ops-break-glass", + "userkey": tenancy.userkey("static", "ops-break-glass"), "tier": "shared"}, + ]) + r = registry.parse(raw) + assert len(r) == 3 + u = r.lookup("github", "mrsabath") + assert u is not None + assert u.tier == "isolated" and u.isolated + assert u.agent == "triager" + assert u.limits.requests_per_hour == 100 + assert u.limits.max_concurrent == 3 + assert u.limits.transcript_bytes == 268435456 + assert not r.lookup("github", "ops-break-glass") + + +def test_an_empty_registry_denies_everyone(): + """Phase 2 §2.4's reading: the other one turns a missing file into an open door.""" + r = registry.parse(_reg([])) + assert len(r) == 0 + assert r.lookup("github", "anyone") is None + + +def test_a_missing_users_key_is_an_empty_registry(): + assert len(registry.parse(json.dumps({"version": 1}))) == 0 + + +def test_a_recorded_userkey_that_disagrees_is_a_startup_refusal(): + """§2.5's core check. The refusal must name BOTH values — an operator needs to know + which one the live topics were named after.""" + raw = _reg([{"issuer": "github", "userid": "mrsabath", + "userkey": "gh-mrsabath-deadbeef"}]) + with pytest.raises(registry.RegistryError) as e: + registry.parse(raw) + msg = str(e.value) + assert "gh-mrsabath-deadbeef" in msg + assert tenancy.userkey("github", "mrsabath") in msg + + +def test_a_recorded_userkey_may_be_omitted(): + """An operator with ten logins should not have to compute digests by hand.""" + r = registry.parse(_reg([{"issuer": "github", "userid": "mrsabath"}])) + assert r.users[0].userkey == tenancy.userkey("github", "mrsabath") + + +def test_two_identities_colliding_on_one_key_is_refused(): + """§2.3's 32-bit birthday bound, turned from a silent leak into a refused signup. + + Forced here by listing one identity twice, which produces the same collision the + bound describes — the registry cannot tell the two cases apart and must refuse both. + """ + raw = _reg([_gh("mrsabath"), _gh("MrSabath")]) + with pytest.raises(registry.RegistryError, match="collides with"): + registry.parse(raw) + + +def test_an_unsupported_version_is_refused(): + """A v2 file may mean something different by a field this code thinks it knows.""" + for v in (0, 2, "1", None): + with pytest.raises(registry.RegistryError, match="not supported"): + registry.parse(_reg([], version=v)) + + +def test_malformed_json_is_refused(): + with pytest.raises(registry.RegistryError, match="not valid JSON"): + registry.parse("{not json") + + +def test_a_non_object_registry_is_refused(): + with pytest.raises(registry.RegistryError, match="must be a JSON object"): + registry.parse("[]") + + +def test_an_entry_without_issuer_or_userid_is_refused(): + for e in ({"userid": "x"}, {"issuer": "github"}, {}): + with pytest.raises(registry.RegistryError, match="needs both"): + registry.parse(_reg([e])) + + +def test_an_unknown_tier_is_refused(): + with pytest.raises(registry.RegistryError, match="tier"): + registry.parse(_reg([_gh("a", tier="platinum")])) + + +def test_non_integer_limits_are_refused(): + with pytest.raises(registry.RegistryError, match="integers"): + registry.parse(_reg([_gh("a", limits={"max_concurrent": "lots"})])) + + +def test_a_non_object_entry_is_refused(): + with pytest.raises(registry.RegistryError, match="must be an object"): + registry.parse(_reg(["mrsabath"])) + + +def test_an_entry_is_refused_rather_than_skipped(): + """Unlike `auth.parse_tokens`, which skips malformed entries on purpose. Here a + skipped user is one whose topics exist and whose events have nowhere to go.""" + raw = _reg([_gh("good"), {"issuer": "github"}]) + with pytest.raises(registry.RegistryError): + registry.parse(raw) + + +# ---- lookup ---------------------------------------------------------------- + +def test_lookup_inherits_per_issuer_canonicalisation(): + """Matching goes through `tenancy.userkey`, so case folding is not reimplemented.""" + r = registry.parse(_reg([_gh("mrsabath")])) + for spelling in ("mrsabath", "MrSabath", "MRSABATH", " mrsabath "): + assert r.lookup("github", spelling) is not None, spelling + + +def test_lookup_separates_issuers(): + r = registry.parse(_reg([_gh("alice")])) + assert r.lookup("github", "alice") is not None + assert r.lookup("oidc", "alice") is None + assert r.lookup("static", "alice") is None + + +def test_lookup_of_a_static_identity_is_byte_exact(): + raw = _reg([{"issuer": "static", "userid": "Ops"}]) + r = registry.parse(raw) + assert r.lookup("static", "Ops") is not None + assert r.lookup("static", "ops") is None + + +def test_lookup_with_no_userid_is_none(): + r = registry.parse(_reg([_gh("alice")])) + assert r.lookup("github", None) is None + assert r.lookup(None, None) is None + + +def test_by_userkey(): + r = registry.parse(_reg([_gh("alice")])) + key = tenancy.userkey("github", "alice") + assert r.by_userkey(key).userid == "alice" + assert r.by_userkey("gh-nobody-00000000") is None + + +def test_userkeys_property(): + r = registry.parse(_reg([_gh("alice"), _gh("bob")])) + assert r.userkeys == (tenancy.userkey("github", "alice"), + tenancy.userkey("github", "bob")) + + +# ---- load ------------------------------------------------------------------ + +def test_load_an_empty_path_is_an_empty_registry(): + """Whether "nobody approved" is fatal depends on the tenancy mode, so `load` does + not decide it — `__main__` does, where the mode is known.""" + assert len(registry.load("")) == 0 + + +def test_load_a_missing_file_is_refused(): + with pytest.raises(registry.RegistryError, match="does not exist"): + registry.load("/nonexistent/registry.json") + + +def test_load_reads_a_real_file(tmp_path): + p = tmp_path / "registry.json" + p.write_text(_reg([_gh("alice")])) + assert len(registry.load(str(p))) == 1 + + +# ---- Caller ---------------------------------------------------------------- + +def test_caller_for_single_mode_leaves_the_key_unset(): + """`ce_userkey` must stay off the wire entirely for a Phase 2 deployment.""" + c = auth.caller_for("mrsabath", "github", tenancy_mode=tenancy.SINGLE) + assert c.userid == "mrsabath" + assert c.issuer == "github" + assert c.userkey is None + assert not c.anonymous + + +def test_caller_for_multi_mode_derives_the_key(): + c = auth.caller_for("mrsabath", "github", tenancy_mode=tenancy.MULTI) + assert c.userkey == tenancy.userkey("github", "mrsabath") + + +def test_an_anonymous_caller_carries_nothing(): + c = auth.caller_for(None, None, tenancy_mode=tenancy.MULTI) + assert c.anonymous + assert c.userid is None and c.issuer is None and c.userkey is None + + +def test_submitter_iss_stays_absent_for_a_static_identity(): + """Phase 2 §2.6 made an absent issuer mean "an operator typed this name". A static + identity needs an issuer for its tenancy key to separate from a GitHub login of the + same name, but promoting that onto the wire would upgrade an operator's assertion + into a verified claim.""" + c = auth.caller_for("ops", None, tenancy_mode=tenancy.MULTI) + assert c.submitter_iss is None + assert c.userkey == tenancy.userkey("static", "ops") + + gh = auth.caller_for("alice", "github", tenancy_mode=tenancy.MULTI) + assert gh.submitter_iss == "github" + + +def test_a_static_and_a_github_identity_of_one_name_are_two_tenants(): + """The §2.3 impersonation failure, asserted at the Caller level.""" + a = auth.caller_for("alice", "github", tenancy_mode=tenancy.MULTI) + b = auth.caller_for("alice", None, tenancy_mode=tenancy.MULTI) + assert a.userkey != b.userkey + + +# ---- resolve_caller -------------------------------------------------------- + +def _cfg(**over): + c = Cfg() + for k, v in over.items(): + setattr(c, k, v) + return c + + +def _environ(token=None): + return {"HTTP_AUTHORIZATION": f"Bearer {token}"} if token else {} + + +def test_resolve_caller_single_mode_is_phase2_anonymous(): + """With nothing configured: allowed and anonymous, which keeps the demo working.""" + caller, status, why = auth.resolve_caller(_environ(), _cfg()) + assert status is None and why is None + assert caller.anonymous and caller.userkey is None + + +def test_resolve_caller_single_mode_with_a_static_token(): + cfg = _cfg(auth_tokens={"tok": "alice"}) + caller, status, _ = auth.resolve_caller(_environ("tok"), cfg) + assert status is None + assert caller.userid == "alice" + assert caller.userkey is None # single mode + assert caller.submitter_iss is None # static + + +def test_resolve_caller_multi_mode_refuses_anonymous_with_401(): + """§2.4: Phase 2's open default cannot coexist with per-user isolation — there is + no user to isolate.""" + cfg = _cfg(tenancy_mode=tenancy.MULTI) + caller, status, why = auth.resolve_caller(_environ(), cfg, + registry=registry.Registry()) + assert caller is None + assert status == 401 + assert "multi-tenant" in why + + +def test_resolve_caller_multi_mode_refuses_an_unregistered_user_with_403(): + """`403`, not `404`: this is "I know who you are and you are not approved", and + naming the identity is what makes it actionable.""" + cfg = _cfg(tenancy_mode=tenancy.MULTI, auth_tokens={"tok": "bob"}) + reg = registry.parse(_reg([_gh("alice")])) + caller, status, why = auth.resolve_caller(_environ("tok"), cfg, registry=reg) + assert caller is None + assert status == 403 + assert "bob" in why + + +def test_resolve_caller_multi_mode_accepts_a_registered_user(): + cfg = _cfg(tenancy_mode=tenancy.MULTI, auth_tokens={"tok": "ops"}) + reg = registry.parse(_reg([{"issuer": "static", "userid": "ops", + "tier": "isolated"}])) + caller, status, _ = auth.resolve_caller(_environ("tok"), cfg, registry=reg) + assert status is None + assert caller.userid == "ops" + assert caller.userkey == tenancy.userkey("static", "ops") + assert caller.tier == "isolated" + + +def test_resolve_caller_multi_mode_uses_the_registrys_recorded_key(): + cfg = _cfg(tenancy_mode=tenancy.MULTI, auth_tokens={"tok": "ops"}) + reg = registry.parse(_reg([{"issuer": "static", "userid": "ops"}])) + caller, _, _ = auth.resolve_caller(_environ("tok"), cfg, registry=reg) + assert caller.userkey == reg.users[0].userkey + + +def test_resolve_caller_propagates_a_401_from_resolve(): + cfg = _cfg(tenancy_mode=tenancy.MULTI, auth_tokens={"tok": "ops"}) + caller, status, _ = auth.resolve_caller(_environ("wrong"), cfg, + registry=registry.Registry()) + assert caller is None and status == 401 diff --git a/eventing/tests/test_signed_tenancy_attrs.py b/eventing/tests/test_signed_tenancy_attrs.py new file mode 100644 index 00000000..eed55a1d --- /dev/null +++ b/eventing/tests/test_signed_tenancy_attrs.py @@ -0,0 +1,117 @@ +"""`userkey` and `depth` inside the signed attribute set. DESIGN_PHASE3.md §2.6, §7.7, §8.3. + +Why these two have to be signed rather than merely present on the event: + +* `userkey` decides which store a response is written into and which ntfy topic + announces it. An unsigned, mutable one lets anything with write access to a topic file + an event into another user's history. +* `depth` is the hop limit. A hop limit an attacker can reset does not limit hops, and + §7.7 is the control the whole trigger API lives or dies by. + +§8.3 requires both to land in ONE change, because adding an attribute changes +canonicalisation and a signer and verifier on different versions disagree about every +signature. +""" +from __future__ import annotations + +import binascii + +import pytest + +from shared import ce, keyset +from shared import signing as S + +SEED = binascii.unhexlify( + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") +PUB = S.public_key(SEED) + +CORR = "brave-otter-4718" +UK = "gh-mrsabath-4c1d9e07" + + +def _request(**over): + attrs = {"type": ce.TYPE_REQUEST, "source": "rossoctl://eventbridge/test", + "datacontenttype": "application/json", + ce.EXT_CORRELATIONID: CORR, + ce.EXT_SESSIONUUID: ce.session_uuid(CORR), + ce.EXT_MODE: "start"} + attrs.update(over) + return ce.new_event(**attrs, data={"prompt": "hello"}) + + +def test_both_attributes_are_in_the_signed_set(): + """§8.3: one change, both attributes.""" + assert ce.EXT_USERKEY in S.SIGNED_ATTRS + assert ce.EXT_DEPTH in S.SIGNED_ATTRS + + +def test_tampering_with_the_userkey_invalidates_the_signature(): + """The attack this exists to stop: filing an event into another user's history.""" + event = _request(**{ce.EXT_USERKEY: UK}) + S.sign_into(event, SEED, "eb-01") + ks = keyset.KeySet({"eb-01": PUB}) + assert S.verify_with_keyset(event, ks)[0] + + event.attrs[ce.EXT_USERKEY] = "gh-victim-0bad0bad" + ok, why = S.verify_with_keyset(event, ks) + assert not ok, why + + +def test_removing_the_userkey_invalidates_the_signature(): + """Stripping the attribute must not be a way around covering it — otherwise an + attacker downgrades a tenanted event to an untenanted one, which §6.1 files into + the `shared` store.""" + event = _request(**{ce.EXT_USERKEY: UK}) + S.sign_into(event, SEED, "eb-01") + ks = keyset.KeySet({"eb-01": PUB}) + del event.attrs[ce.EXT_USERKEY] + assert not S.verify_with_keyset(event, ks)[0] + + +def test_adding_a_userkey_to_an_unkeyed_signed_event_invalidates_it(): + """The reverse direction: a single-tenant event must not be promotable into + somebody's tenant.""" + event = _request() + S.sign_into(event, SEED, "eb-01") + ks = keyset.KeySet({"eb-01": PUB}) + assert S.verify_with_keyset(event, ks)[0] + event.attrs[ce.EXT_USERKEY] = UK + assert not S.verify_with_keyset(event, ks)[0] + + +def test_tampering_with_the_depth_invalidates_the_signature(): + """§7.7: a resettable hop count is not a hop limit.""" + event = _request(**{ce.EXT_USERKEY: UK, ce.EXT_DEPTH: "3"}) + S.sign_into(event, SEED, "eb-01") + ks = keyset.KeySet({"eb-01": PUB}) + assert S.verify_with_keyset(event, ks)[0] + event.attrs[ce.EXT_DEPTH] = "0" + assert not S.verify_with_keyset(event, ks)[0] + + +def test_a_signed_event_survives_the_kafka_wire_with_both_attributes(): + event = _request(**{ce.EXT_USERKEY: UK, ce.EXT_DEPTH: "2"}) + S.sign_into(event, SEED, "eb-01") + headers, value = ce.to_kafka_binary(event) + back = ce.from_kafka_binary(headers, value) + assert back.attrs[ce.EXT_USERKEY] == UK + assert back.attrs[ce.EXT_DEPTH] == "2" + assert S.verify_with_keyset(back, keyset.KeySet({"eb-01": PUB}))[0] + + +@pytest.mark.parametrize("attr", [ce.EXT_USERKEY, ce.EXT_DEPTH, ce.EXT_AGENT, + ce.EXT_TRIGGERID]) +def test_new_attribute_names_obey_the_cloudevents_character_rule(attr): + """CloudEvents v1.0 allows lower-case `[a-z0-9]` only in an attribute name. Phase 2 + §2.6 got this wrong once already with `submitter_iss`, and the codec would not have + caught it: `to_kafka_binary` just prefixes `ce_` and validates nothing, so a bad + name round-trips here and is dropped by a spec-compliant consumer a hop later.""" + assert attr.isascii() and attr.islower() and attr.isalnum() + + +def test_an_absent_attribute_is_omitted_not_signed_as_empty(): + """`canonical` omits absent attributes, which is what lets a single-tenant event + (no userkey, no depth) verify against a build that knows about both.""" + no_uk = S.canonical(_request().attrs, {"prompt": "hello"}) + assert b"userkey" not in no_uk + assert b"depth" not in no_uk diff --git a/eventing/tests/test_topicset_wiring.py b/eventing/tests/test_topicset_wiring.py new file mode 100644 index 00000000..ed3aec24 --- /dev/null +++ b/eventing/tests/test_topicset_wiring.py @@ -0,0 +1,201 @@ +"""`TopicSet` threaded through the producer and consumers. DESIGN_PHASE3.md §3.2, T2. + +§9's step 2 is "no behaviour change whatsoever; the existing suite is the check". That +suite is the real assertion; these tests pin the *intent* so a later change that breaks +single-mode compatibility fails with a message about compatibility rather than as a +mystery somewhere else. +""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from eventbridge.config import Cfg +from eventbridge.kafka_out import Producer +from shared import ce, tenancy + +UK = "gh-mrsabath-4c1d9e07" + + +def _producer(**over): + """A Producer with the KafkaProducer replaced, so `send` is observable.""" + with patch("eventbridge.kafka_out.KafkaProducer") as kp: + kp.return_value = MagicMock() + p = Producer("localhost:9092", "requests", "rossoctl://eventbridge/test", + response_topic="responses", **over) + return p, p._prod + + +def _sent_topic(prod): + return prod.send.call_args[0][0] + + +def _sent_event(prod): + kw = prod.send.call_args[1] + return ce.from_kafka_binary(kw["headers"], kw["value"]) + + +# ---- single mode: unchanged ------------------------------------------------ + +def test_single_mode_publishes_to_the_configured_request_topic(): + p, prod = _producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start") + assert _sent_topic(prod) == "requests" + + +def test_single_mode_ignores_a_userkey_for_topic_selection(): + """A userkey must not change the topic in single mode — that is what makes the + whole phase additive.""" + p, prod = _producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start", + userkey=UK) + assert _sent_topic(prod) == "requests" + + +def test_no_userkey_attribute_when_none_is_given(): + """`ce_userkey` must stay off the wire entirely for a Phase 2 deployment.""" + p, prod = _producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start") + assert ce.EXT_USERKEY not in _sent_event(prod).attrs + + +def test_group_events_still_go_to_the_responses_topic(): + p, prod = _producer() + p.publish_group_event(type_=ce.TYPE_GROUP_STARTED, groupid="brave-otter-0001", + data={"expected": 2}) + assert _sent_topic(prod) == "responses" + + +# ---- multi mode ----------------------------------------------------------- + +def _multi_producer(): + topics = tenancy.TopicSet("kev1", tenancy=tenancy.MULTI) + with patch("eventbridge.kafka_out.KafkaProducer") as kp: + kp.return_value = MagicMock() + p = Producer("localhost:9092", "requests", "rossoctl://eventbridge/test", + response_topic="responses", topics=topics) + return p, p._prod + + +def test_multi_mode_publishes_to_the_per_user_topic(): + p, prod = _multi_producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start", + userkey=UK) + assert _sent_topic(prod) == f"kev1-{UK}-requests" + + +def test_multi_mode_stamps_the_userkey_on_the_event(): + """§2.6: the runner stamps it back onto the response, which is how EventBridge + knows which store to file the response in.""" + p, prod = _multi_producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start", + userkey=UK) + assert _sent_event(prod).attrs[ce.EXT_USERKEY] == UK + + +def test_multi_mode_refuses_to_publish_without_a_userkey(): + """Falling back to a shared topic would run one tenant's work on another tenant's + runner, under that tenant's credential.""" + p, _ = _multi_producer() + with pytest.raises(ValueError, match="userkey is required"): + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), + mode="start") + + +def test_multi_mode_group_events_go_to_the_users_responses_topic(): + p, prod = _multi_producer() + p.publish_group_event(type_=ce.TYPE_GROUP_STARTED, groupid="brave-otter-0001", + data={"expected": 2}, userkey=UK) + assert _sent_topic(prod) == f"kev1-{UK}-responses" + + +def test_the_agent_attribute_rides_when_named(): + p, prod = _multi_producer() + p.publish_request(prompt="hi", correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), mode="start", + userkey=UK, agent="triager") + assert _sent_event(prod).attrs[ce.EXT_AGENT] == "triager" + + +# ---- the config property -------------------------------------------------- + +def test_cfg_topics_defaults_to_single_mode(): + cfg = Cfg() + assert cfg.tenancy_mode == tenancy.SINGLE + assert not cfg.topics.multi + assert cfg.topics.requests() == cfg.request_topic + assert cfg.topics.responses() == cfg.response_topic + + +def test_cfg_topics_tracks_a_renamed_topic(): + """The property exists so the set cannot drift from the configured names.""" + cfg = Cfg() + cfg.request_topic = "my-requests" + assert cfg.topics.requests() == "my-requests" + + +def test_cfg_topics_multi_mode_uses_the_prefix(): + cfg = Cfg() + cfg.tenancy_mode = tenancy.MULTI + cfg.topic_prefix = "kev9" + assert cfg.topics.requests(UK) == f"kev9-{UK}-requests" + + +# ---- the runner side ------------------------------------------------------ + +def test_the_emitter_stamps_its_userkey_on_every_response(): + """§3.3: held on the Emitter rather than passed per call, because this pod serves + exactly one tenant and a per-call parameter is one more thing to forget.""" + from eventrunner.emit import Emitter + with patch("eventrunner.emit.KafkaProducer") as kp: + kp.return_value = MagicMock() + em = Emitter("localhost:9092", "responses", "rossoctl://eventrunner/test", + userkey=UK) + em.emit(correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), + sequence=1, phase="result", final=True, data={"text": "hi"}) + assert _sent_event(em._prod).attrs[ce.EXT_USERKEY] == UK + + +def test_the_emitter_omits_the_userkey_in_single_mode(): + from eventrunner.emit import Emitter + with patch("eventrunner.emit.KafkaProducer") as kp: + kp.return_value = MagicMock() + em = Emitter("localhost:9092", "responses", "rossoctl://eventrunner/test") + em.emit(correlationid="brave-otter-4718", + sessionuuid=ce.session_uuid("brave-otter-4718"), + sequence=1, phase="result", final=True, data={"text": "hi"}) + assert ce.EXT_USERKEY not in _sent_event(em._prod).attrs + + +def test_the_runner_refuses_a_non_default_request_topic_without_a_userkey(monkeypatch): + """§3.3: a runner that stamps no userkey produces events EventBridge cannot + attribute, and a silent default would route one user's output into another's store.""" + from eventrunner import config as ercfg + monkeypatch.setenv("REQUEST_TOPIC", f"kev1-{UK}-requests") + monkeypatch.delenv("ER_USERKEY", raising=False) + with pytest.raises(SystemExit, match="ER_USERKEY is required"): + ercfg.load() + + +def test_the_runner_accepts_a_per_user_topic_with_a_userkey(monkeypatch): + from eventrunner import config as ercfg + monkeypatch.setenv("REQUEST_TOPIC", f"kev1-{UK}-requests") + monkeypatch.setenv("ER_USERKEY", UK) + cfg = ercfg.load() + assert cfg.userkey == UK + + +def test_the_runner_default_topic_needs_no_userkey(monkeypatch): + """Phase 2's deployment, unchanged.""" + from eventrunner import config as ercfg + monkeypatch.delenv("REQUEST_TOPIC", raising=False) + monkeypatch.delenv("ER_USERKEY", raising=False) + assert ercfg.load().userkey == "" From 7bf725d5ca22c2ab95099a4c05c074da47d290b8 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 17:50:37 -0400 Subject: [PATCH 03/14] docs(eventing): record which of Phase 3 is implemented and which is still design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Phase 3 design doc said "nothing in this document is implemented yet" and the agentdocs index said "Design only — not implemented". Both are now wrong in a way that matters: a reader who trusts them will either re-implement steps 1-4 or assume `EB_TENANCY_MODE=multi` gives them isolation it does not yet give. So the status lines name the tasks that landed (T1, T2, T3, T4, T13, T22 = §9 steps 1-4) and say plainly that everything from step 5 on — per-user stores, owner-scoped reads, transcript auth, `k8s_tenant.py`, ntfy isolation, triggers, fetched skills, Kafka ACLs — is still design only. The component README gains a Phase 3 configuration table covering only the six variables whose code exists, with two notes that are the point of including it at all: `multi` mode is not yet isolation anyone should present as such (the HTTP reads and `/continue` are still as open as Phase 2's, because steps 5-8 are not in), and adding a user is an operator action because EventBridge has no Kubernetes client. Verified against the code rather than assumed: only `POST /v0/agents` and `POST /v0/groups` call the auth path, so every GET, `/continue` and `/transcript` is open in both modes today. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/README.md | 23 +++++++++++++++++++++++ eventing/agentdocs/DESIGN_PHASE3.md | 10 +++++++++- eventing/agentdocs/README.md | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/eventing/README.md b/eventing/README.md index 705fa28e..6a282c46 100644 --- a/eventing/README.md +++ b/eventing/README.md @@ -183,6 +183,29 @@ the full list of what stays open: `agentdocs/README_PHASE1.md` § "Submit-path authentication". Two of these are load-bearing in ways that are easy to miss. +Added in Phase 3 (`agentdocs/DESIGN_PHASE3.md`). Only the variables whose code is +actually in the tree — §9's steps 1–4. Every one of them is off or single-tenant by +default, so an existing deployment that upgrades and changes nothing behaves exactly +as it did: + +| Env | Default | Effect | +|---|---|---| +| `EB_TENANCY_MODE` | `single` | `single` reproduces Phase 2: one topic pair, one store, no ownership checks. `multi` switches on per-user topics and requires an authenticated, registered caller | +| `EB_TOPIC_PREFIX` | `kev1` | First component of every per-user topic name, `{prefix}-{userkey}-requests`. Unused in `single` mode | +| `EB_USER_REGISTRY_PATH` | — | The approved-user registry (a ConfigMap, §2.5). **Required in `multi` mode, and an empty one denies everyone** — the other reading turns a missing file into an open door | +| `ER_USERKEY` | — | Which tenant this runner serves. Stamped on every response so EventBridge can file it. **Required once `REQUEST_TOPIC` is not `requests`**; the runner refuses to start otherwise | +| `ER_AGENT_DIR` | `/etc/rossoctl/agents` | Where baked `AgentSpec`s live (§5.2) | +| `ER_AGENT_NAME` | `default` | Fallback agent when neither the request nor the registry names one. `default` with no file on disk is Phase 2's argv exactly | + +Two notes on these. **`multi` mode is not yet isolation you should present as such** — +per-user *stores*, owner-scoped reads and transcript authentication are steps 5–8 and +are not implemented, so in `multi` mode the HTTP reads and `/continue` are still as open +as Phase 2's. And even once they land, separate topics without Kafka ACLs are +*organisation*, not isolation; `DESIGN_PHASE3.md` §3.6 and §10 are explicit about which +claim each configuration earns. **Adding a user is an operator action**: EventBridge has +no Kubernetes client and will not create topics, so a registry entry whose topics do not +exist is a refusal, not an auto-provision. + `ER_EVENTBRIDGE_URL` is what makes `/continue` survive a scale-to-zero *and* what keeps event `sequence` numbers unique across pods — without it a second pod restarts numbering at 1 and its rows overwrite the first turn's. diff --git a/eventing/agentdocs/DESIGN_PHASE3.md b/eventing/agentdocs/DESIGN_PHASE3.md index 7176c042..49c6ad9a 100644 --- a/eventing/agentdocs/DESIGN_PHASE3.md +++ b/eventing/agentdocs/DESIGN_PHASE3.md @@ -1,6 +1,14 @@ # DESIGN — Phase 3: per-user isolation, declarative agents, and event triggers -Status: draft (revision 1) — **design only, nothing in this document is implemented yet.** +Status: draft (revision 1) — **partly implemented.** §9's steps 1-4 are in the tree: +T1 (`shared/tenancy.py`), T2 (`TopicSet` threaded through both services), T3 (`Caller`, +the registry, `ce_userkey`), T4 (`SIGNED_ATTRS` += `userkey`, `depth`), T13 (`AgentSpec`) +and T22 (the Secret-vs-ConfigMap test). `EB_TENANCY_MODE` defaults to `single`, which +reproduces Phase 2 byte for byte, so nothing below is switched on anywhere yet. +Everything from step 5 on — per-user stores, owner-scoped reads, transcript auth, +`k8s_tenant.py`, ntfy isolation, triggers, fetched skills, Kafka ACLs — is still design +only. Where a section is implemented, the code is the authority on what it does; this +document remains the authority on *why*. Scope: **delta over `DESIGN_PHASE2.md`.** Read that first, and §2.1 of it before anything else here. diff --git a/eventing/agentdocs/README.md b/eventing/agentdocs/README.md index 9d2bcf44..3fd3145b 100644 --- a/eventing/agentdocs/README.md +++ b/eventing/agentdocs/README.md @@ -19,7 +19,7 @@ and what is *not* verified. | [`IMPLEMENTATION_REPORT1.md`](IMPLEMENTATION_REPORT1.md) | What Phase 1 built, the measured results on both clusters, 28 findings including two real bugs only a scale-to-zero deployment could expose, and an honest account of what is still blocked and why. | | [`README_PHASE1.md`](README_PHASE1.md) | The Phase 1 runbook in full detail, covering all four ways to run it: locally without containers, under Docker, on a Kind cluster, and on OpenShift. | | [`DESIGN_PHASE2.md`](DESIGN_PHASE2.md) | The Phase 2 design — identity on the event path. Why GitHub's opaque user token rules out local verification and forces a `GET /user` lookup plus a load-bearing cache; why `401` and `403` are kept distinct; what `ce_submitter` is and is not worth, and what a signature over it does and does not prove; how the `kid` and the approved-key set prove *which agent* answered, why group events are pinned to EventBridge's own key, and why a rejected response is stored as an error rather than dropped. **§8 records the thirteen claims across §§2.6, 3, 4.3 and 4.4 that a docs review found do not hold in the code, with the issue filed for each — and §8.5-§8.8 the procedure that came out of them: the five questions to ask of any control claim, in any phase, before it ships.** | -| [`DESIGN_PHASE3.md`](DESIGN_PHASE3.md) | The Phase 3 design — per-user isolation, declarative agents, and event triggers. Why a raw `submitter` cannot be a tenancy key and why the derived `userkey` ends in a hash; per-user Kafka topics (`{prefix}-{userkey}-requests`), one EventRunner Deployment and ScaledObject per user, and the honest difference between separately-named topics and ACL-enforced isolation; why a per-user ntfy topic must *not* contain the user's name, and the option of running ntfy as a service in the cluster; the `AgentSpec` — an agent baked into the image or assembled from skills fetched from GitHub, S3 or a URL, and the five gates a fetched bundle passes; authenticating and scoping the session transcript; and a Knative-inspired trigger API with the four loop controls that keep an event-driven agent system from becoming a cost incident. **Design only — not implemented.** | +| [`DESIGN_PHASE3.md`](DESIGN_PHASE3.md) | The Phase 3 design — per-user isolation, declarative agents, and event triggers. Why a raw `submitter` cannot be a tenancy key and why the derived `userkey` ends in a hash; per-user Kafka topics (`{prefix}-{userkey}-requests`), one EventRunner Deployment and ScaledObject per user, and the honest difference between separately-named topics and ACL-enforced isolation; why a per-user ntfy topic must *not* contain the user's name, and the option of running ntfy as a service in the cluster; the `AgentSpec` — an agent baked into the image or assembled from skills fetched from GitHub, S3 or a URL, and the five gates a fetched bundle passes; authenticating and scoping the session transcript; and a Knative-inspired trigger API with the four loop controls that keep an event-driven agent system from becoming a cost incident. **Partly implemented: §9's steps 1-4 (T1, T2, T3, T4, T13, T22) are in the tree; `EB_TENANCY_MODE` still defaults to `single`, which reproduces Phase 2 exactly. Steps 5-12 — per-user stores, owner-scoped reads, transcript auth, `k8s_tenant.py`, ntfy isolation, triggers, fetched skills, Kafka ACLs — are design only.** | ## Reading order From 80bdb92e1ea39650433ce8cd65de5fa34694e087 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 18:03:54 -0400 Subject: [PATCH 04/14] fix(eventing): refuse an unresolvable /continue with 503 instead of crashing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found reviewing the previous commit. In `multi` mode `_owner_userkey` returned `None` both when no key was needed (single mode) and when the owning tenant could not be determined. The second `None` flowed into `publish_request` -> `TopicSet.requests(None)`, which raises `ValueError` by design — publishing a resume to a shared topic would run it on another tenant's runner, under that tenant's credential — but nothing caught it, so the request died as a WSGI 500 with a stack trace instead of a status code. Three ways to reach it, none of them abuse: * a user removed from the registry; * a `prompts` row whose `submitter` is NULL — the column is nullable and pre-dates auth, so these rows exist in any store that outlived Phase 2; * a correlation back-filled from the topic by `RequestsMirror`, which records no submitter at all. `_owner_userkey` now returns `(userkey, unresolved_reason)` with exactly one set, so the two `None` cases are distinguishable, and both `/continue` paths check it before writing anything — no touched session row and no orphan prompt row claiming a turn that never ran. The JSON route answers `503` naming what could not be resolved, matching how §2.5 treats a user whose topics do not exist. The HTML form route answers `503` with a plain-text body rather than redirecting: a silent `303` back to the transcript page would look like the turn was accepted and then vanished, which is the Phase 2 §6.1 symptom class this phase keeps trying to avoid. Only reachable with `EB_TENANCY_MODE=multi`, so the default path was never affected. 13 new tests (735 total). Verified they are load-bearing by reintroducing the bug: the four refusal tests fail without the guard and pass with it. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventbridge/handlers.py | 69 ++++++-- eventing/tests/test_continue_tenancy.py | 208 ++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 17 deletions(-) create mode 100644 eventing/tests/test_continue_tenancy.py diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index 72a16371..e5923955 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -130,10 +130,25 @@ def _caller(self, environ): return auth.resolve_caller(environ, self.cfg, cache=self.logins, registry=self.registry) - def _owner_userkey(self, correlationid: str) -> str | None: + def _owner_userkey(self, correlationid: str) -> tuple[str | None, str | None]: """Which tenant owns this correlation, for a `/continue` turn. - `None` in single-tenant mode, where topics do not depend on it. + Returns `(userkey, unresolved_reason)`. Exactly one is ever set: + + * `(None, None)` — single-tenant mode; topics do not depend on a key. + * `(key, None)` — the owning tenant. + * `(None, reason)` — multi mode and the owner cannot be determined. + + The two-value return exists because the two `None` cases need opposite + handling and a bare `None` cannot tell them apart. Passing an unresolved + `None` into `TopicSet.requests()` raises `ValueError` — correctly, since + publishing to a shared topic would run this turn on another tenant's runner — + but that surfaces as a WSGI 500 with a stack trace instead of a status code. + A correlation with no resolvable owner is reachable in normal operation: a + user removed from the registry, a `prompts` row whose `submitter` is NULL + (the column is nullable and pre-dates auth), or a correlation back-filled + from the topic by `RequestsMirror`. So the caller refuses with a `503` naming + the problem, matching how §2.5 treats a user whose topics do not exist. `/continue` is still unauthenticated here — Phase 2 §3.1's capability-URL design — so this resolves the owner from what was *recorded at submit time* @@ -150,20 +165,24 @@ def _owner_userkey(self, correlationid: str) -> str | None: for the uniqueness guarantee that keeps `sessionuuid` unsalted. """ if not self.cfg.topics.multi: - return None + return None, None prompts = self.store.get_prompts(correlationid) or [] - for p in prompts: - if submitter := p.get("submitter"): - # The issuer is not recorded alongside the submitter today, so this - # cannot distinguish a GitHub `alice` from a static `alice`. In `multi` - # mode anonymous and static submissions are already refused at the - # submit path (§2.4), so GitHub is the only issuer that reaches here — - # stated rather than assumed, because the day an OIDC issuer is wired - # up (§2.4's slot) this becomes wrong and has to move to the index. - if user := (self.registry.lookup("github", submitter) - if self.registry else None): - return user.userkey - return None + submitters = [s for p in prompts if (s := p.get("submitter"))] + if not submitters: + return None, ("no submitter recorded for this correlation, so the owning " + "tenant cannot be determined") + for submitter in submitters: + # The issuer is not recorded alongside the submitter today, so this + # cannot distinguish a GitHub `alice` from a static `alice`. In `multi` + # mode anonymous and static submissions are already refused at the + # submit path (§2.4), so GitHub is the only issuer that reaches here — + # stated rather than assumed, because the day an OIDC issuer is wired + # up (§2.4's slot) this becomes wrong and has to move to the index. + if user := (self.registry.lookup("github", submitter) + if self.registry else None): + return user.userkey, None + return None, (f"{submitters[0]} is no longer in the user registry, so the " + f"owning tenant cannot be determined") # ---- start ---- def start_agent(self, environ, start_response, **_): @@ -369,12 +388,18 @@ def continue_agent(self, environ, start_response, correlationid: str, **_): if session is None: return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) sess = session["sessionuuid"] + owner, unresolved = self._owner_userkey(correlationid) + if unresolved: + # Checked before anything is written, so a refused turn leaves no touched + # session row and no orphan prompt row claiming a turn that never ran. + return _json(start_response, "503 Service Unavailable", + {"error": f"cannot resume: {unresolved}"}) self.store.upsert_session(correlationid, sess, session["workdir"], None) turn_index = self.store.insert_prompt(correlationid, "continue", prompt) event_id = self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", - userkey=self._owner_userkey(correlationid), + userkey=owner, ) next_seq = len(self.store.events_for(correlationid)) + 1 return _json(start_response, "202 Accepted", { @@ -392,12 +417,22 @@ def continue_agent_html(self, environ, start_response, correlationid: str, **_): session = self.store.get_session(correlationid) if session is not None: sess = session["sessionuuid"] + owner, unresolved = self._owner_userkey(correlationid) + if unresolved: + # The form posts from the transcript page, so the useful answer is + # the page with an error on it rather than a JSON body in the + # browser. Redirecting silently would look like the turn was + # accepted and then vanished, which is the Phase 2 §6.1 class of + # bug this phase keeps trying to avoid. + start_response("503 Service Unavailable", + [("Content-Type", "text/plain; charset=utf-8")]) + return [f"cannot resume {correlationid}: {unresolved}\n".encode()] self.store.upsert_session(correlationid, sess, session["workdir"], None) self.store.insert_prompt(correlationid, "continue", prompt) self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", - userkey=self._owner_userkey(correlationid), + userkey=owner, ) start_response("303 See Other", [("Location", f"/v0/agents/{correlationid}")]) return [b""] diff --git a/eventing/tests/test_continue_tenancy.py b/eventing/tests/test_continue_tenancy.py new file mode 100644 index 00000000..1fd4cdc4 --- /dev/null +++ b/eventing/tests/test_continue_tenancy.py @@ -0,0 +1,208 @@ +"""`/continue` in multi-tenant mode: resolving the owning tenant, and refusing cleanly. + +A `/continue` turn has to reach the tenant that *owns* the correlation, not the caller's +— the route is still unauthenticated (Phase 2 §3.1's capability URL), so there is no +caller identity to use, and publishing to a shared topic would run the turn on another +tenant's runner under that tenant's credential. + +When the owner cannot be determined, `TopicSet.requests(None)` raises by design. These +tests pin that the handler refuses with a status code first, because an uncaught +`ValueError` is a WSGI 500 with a stack trace, and three of the ways to reach it are +ordinary operation rather than abuse: + +* a user removed from the registry; +* a `prompts` row whose `submitter` is NULL (nullable, and pre-dates auth); +* a correlation back-filled from the topic by `RequestsMirror`, which records no + submitter at all. +""" +from __future__ import annotations + +import json +import pathlib +from unittest.mock import MagicMock + +import pytest + +from eventbridge import registry +from eventbridge.config import Cfg +from eventbridge.correlation import Minter +from eventbridge.handlers import Handlers +from eventbridge.store import Store +from shared import ce, tenancy + +CORR = "brave-otter-4718" +OWNER = "mrsabath" + + +def _reg(*userids): + return registry.parse(json.dumps({ + "version": 1, + "users": [{"issuer": "github", "userid": u, "tier": "isolated"} + for u in userids], + })) + + +def _handlers(tmp_path: pathlib.Path, *, mode=tenancy.MULTI, reg=None, submitter=OWNER): + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + cfg.tenancy_mode = mode + cfg.topic_prefix = "kev1" + store = Store(tmp_path) + store.upsert_session(CORR, ce.session_uuid(CORR), str(tmp_path / "w"), "hello") + # `submitter=None` is the pre-auth / mirror-backfilled shape: the column is + # nullable, so this is a real row, not a contrived one. + store.insert_prompt(CORR, "start", "hello", submitter=submitter) + producer = MagicMock() + producer.publish_request.return_value = "evt-1" + h = Handlers(cfg, store, producer, Minter(), + registry=reg if mode == tenancy.MULTI else None) + return h, producer + + +class _Start: + """Captures the WSGI status line.""" + + def __init__(self): + self.status = None + + def __call__(self, status, headers): + self.status = status + + +def _environ(prompt="go on"): + body = json.dumps({"prompt": prompt}).encode() + import io + return {"wsgi.input": io.BytesIO(body), "CONTENT_LENGTH": str(len(body)), + "CONTENT_TYPE": "application/json", "REQUEST_METHOD": "POST"} + + +def _form_environ(prompt="go on"): + import io + body = f"prompt={prompt}".encode() + return {"wsgi.input": io.BytesIO(body), "CONTENT_LENGTH": str(len(body)), + "CONTENT_TYPE": "application/x-www-form-urlencoded", + "REQUEST_METHOD": "POST"} + + +# ---- the happy path ------------------------------------------------------- + +def test_continue_publishes_to_the_owning_tenants_topic(tmp_path): + h, producer = _handlers(tmp_path, reg=_reg(OWNER)) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "202 Accepted" + assert producer.publish_request.call_args[1]["userkey"] == \ + tenancy.userkey("github", OWNER) + + +def test_single_mode_continue_is_unchanged(tmp_path): + """Phase 2's behaviour: no userkey, no ownership question.""" + h, producer = _handlers(tmp_path, mode=tenancy.SINGLE) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "202 Accepted" + assert producer.publish_request.call_args[1]["userkey"] is None + + +def test_single_mode_continue_works_with_no_submitter_recorded(tmp_path): + """The pre-auth row shape must not become a refusal in the default mode.""" + h, _ = _handlers(tmp_path, mode=tenancy.SINGLE, submitter=None) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "202 Accepted" + + +# ---- the three refusals --------------------------------------------------- + +def test_a_submitter_no_longer_in_the_registry_is_a_503_not_a_500(tmp_path): + h, producer = _handlers(tmp_path, reg=_reg("someone-else")) + sr = _Start() + out = h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "503 Service Unavailable" + assert OWNER in json.loads(b"".join(out))["error"] + producer.publish_request.assert_not_called() + + +def test_a_null_submitter_is_a_503_not_a_500(tmp_path): + """`prompts.submitter` is nullable and pre-dates auth, so this is a real row.""" + h, producer = _handlers(tmp_path, reg=_reg(OWNER), submitter=None) + sr = _Start() + out = h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "503 Service Unavailable" + assert "no submitter recorded" in json.loads(b"".join(out))["error"] + producer.publish_request.assert_not_called() + + +def test_an_empty_registry_is_a_503_not_a_500(tmp_path): + h, producer = _handlers(tmp_path, reg=registry.Registry()) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "503 Service Unavailable" + producer.publish_request.assert_not_called() + + +def test_a_refused_continue_writes_nothing(tmp_path): + """No orphan prompt row claiming a turn that never ran.""" + h, _ = _handlers(tmp_path, reg=_reg("someone-else")) + before = len(h.store.get_prompts(CORR)) + h.continue_agent(_environ(), _Start(), correlationid=CORR) + assert len(h.store.get_prompts(CORR)) == before + + +# ---- the HTML form path --------------------------------------------------- + +def test_the_html_form_refuses_with_503_rather_than_redirecting(tmp_path): + """Redirecting silently would look like the turn was accepted and then vanished, + which is exactly the symptom class this phase keeps trying to avoid.""" + h, producer = _handlers(tmp_path, reg=_reg("someone-else")) + sr = _Start() + out = h.continue_agent_html(_form_environ(), sr, correlationid=CORR) + assert sr.status == "503 Service Unavailable" + assert b"cannot resume" in b"".join(out) + producer.publish_request.assert_not_called() + + +def test_the_html_form_still_redirects_on_success(tmp_path): + h, producer = _handlers(tmp_path, reg=_reg(OWNER)) + sr = _Start() + h.continue_agent_html(_form_environ(), sr, correlationid=CORR) + assert sr.status == "303 See Other" + assert producer.publish_request.call_args[1]["userkey"] == \ + tenancy.userkey("github", OWNER) + + +# ---- the helper's contract ------------------------------------------------ + +def test_owner_userkey_returns_exactly_one_of_key_or_reason(tmp_path): + """The two-value return exists because a bare `None` cannot distinguish "single + mode, no key needed" from "multi mode, owner unknown".""" + h, _ = _handlers(tmp_path, mode=tenancy.SINGLE) + assert h._owner_userkey(CORR) == (None, None) + + h2, _ = _handlers(tmp_path / "b", reg=_reg(OWNER)) + key, reason = h2._owner_userkey(CORR) + assert key and reason is None + + h3, _ = _handlers(tmp_path / "c", reg=_reg("nobody")) + key, reason = h3._owner_userkey(CORR) + assert key is None and reason + + +def test_an_unknown_correlation_in_multi_mode_still_404s(tmp_path): + """The ownership check must not shadow the existing 404.""" + h, _ = _handlers(tmp_path, reg=_reg(OWNER)) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid="brave-otter-0000") + assert sr.status == "404 Not Found" + + +@pytest.mark.parametrize("spelling", ["MrSabath", "MRSABATH"]) +def test_owner_resolution_folds_github_case(tmp_path, spelling): + """A correlation submitted as `MrSabath` must resolve to the same tenant as + `mrsabath`, or a resume lands in a different tenant than the original turn.""" + h, producer = _handlers(tmp_path, reg=_reg(OWNER), submitter=spelling) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=CORR) + assert sr.status == "202 Accepted" + assert producer.publish_request.call_args[1]["userkey"] == \ + tenancy.userkey("github", OWNER) From 9df948e870f78ac3285e8fe75cdd5b51f34dffd8 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 18:59:40 -0400 Subject: [PATCH 05/14] =?UTF-8?q?feat(eventing):=20Phase=203=20T5=20?= =?UTF-8?q?=E2=80=94=20per-user=20stores=20and=20the=20global=20ownership?= =?UTF-8?q?=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN_PHASE3.md T5 (§6.1, §2.6), the keystone the rest of step 5+ depends on: T7 (owner-scoped reads), T8 (transcript auth), T15 (triggers) and T21 (deletion) are all blocked on it. `EB_TENANCY_MODE=single` remains the default and reproduces Phase 2. eventbridge/owner_index.py — the one deliberately cross-tenant table `correlationid -> userkey`, UNIQUE on `correlationid` **alone**. That is the part to get right: `(userkey, correlationid)` is the correct primary key inside a tenant's store and exactly the wrong constraint here, because two tenants minting one id produce two distinct tuples, never conflict, and the collision passes through silently — which is the failure the guarantee exists to prevent. It has to be cross-tenant: the lookup that decides which tenant owns a correlation must happen before a per-user store can be chosen. It is also what lets `sessionuuid = uuid5(NAMESPACE, correlationid)` stay UNSALTED (§2.6) — salting it would break every existing session for a collision this constraint removes. `owner_of` returns `(userkey, known)` because a bare `None` conflates "the shared tier owns this" with "never seen", and §6.2's 404 rule needs them distinguished. eventbridge/store_registry.py — per-user stores, LRU-bounded Separate FILES, not one database with a `userkey` column. `store.py` has 28 methods and every one touches tenant data; a `WHERE userkey = ?` remembered 28 times is one that will eventually be forgotten, with a user reading someone else's conversation as the symptom. Separate files make the mistake structurally unavailable — the connection IS the tenant. Two connections per tenant in WAL mode (main + -wal + -shm) bounds this near 150-200 tenants under a 1024 soft RLIMIT_NOFILE, hence `EB_MAX_OPEN_STORES=64`. A store with live SSE subscribers is PINNED: evicting one makes an open page stop updating with no error anywhere. If everything is pinned the cache exceeds its ceiling rather than breaking a viewer — over-budget degrades and is visible on /healthz; a dead stream does not recover. Minter now consults the index (§2.6) One indexed SELECT per mint replaces the startup seeding loop. With N per-user stores the old loop meant N SQLite opens before the socket binds — 100 tenants x 10,000 correlations for a guarantee one SELECT gives directly. The claim happens inside the retry loop, so a candidate lost to a concurrent worker retries instead of both callers believing they own it. `mint_for()` passes `userkey` only to minters that accept it, by signature inspection rather than by catching TypeError around the call — a working `mint()` can raise TypeError from its own body, and swallowing that would silently drop the ownership claim. Routing Responses are filed by the event's OWN signed `ce_userkey`. One with no userkey in multi mode lands in `shared/` and increments `unattributed` (now on /healthz): never guessed into a tenant's store, never dropped. `/continue`, the reads, the group routes and the deadline sweeper all route through the index — the sweeper across every tenant, because a stalled batch belonging to an untouched tenant is exactly what deadlines are for and also the least likely to be in the LRU. This retires the submitter-derived owner lookup from the previous commit, and with it the limitation that it could not tell a GitHub `alice` from a static `alice`. Single-tenant mode keeps its files exactly where Phase 2 left them — directly in the bridge root, not under `shared/` — so an upgraded deployment's sessions and transcripts keep showing up in the UI instead of appearing to vanish. Two bugs found and fixed while building this: * **Eviction could return a closed store.** With every older entry pinned, the LRU took the store it had just opened and handed the caller a closed one — surfacing later as `ProgrammingError: Cannot operate on a closed database` from somewhere that looks unrelated to caching. Fixed with `protect=`, with a regression test. * **`except Exception` hid a signature mismatch.** Adding `userkey=` to `publish_group_event` made every group event stop publishing, and the broad handler around it turned that into a batch that silently never completed. TypeError now re-raises at both call sites. `Store.close()` added (it had none), and it wakes subscribers first so a viewer blocked on `Event.wait()` notices shutdown instead of hanging to its own timeout. Tests: 49 new (785 total), ruff clean. Verified both modes start and that the on-disk layout is upgrade-safe. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventbridge/__main__.py | 36 ++- eventing/eventbridge/config.py | 6 + eventing/eventbridge/correlation.py | 103 ++++++++- eventing/eventbridge/group_service.py | 157 +++++++++---- eventing/eventbridge/handlers.py | 278 +++++++++++++++--------- eventing/eventbridge/kafka_in.py | 14 +- eventing/eventbridge/owner_index.py | 184 ++++++++++++++++ eventing/eventbridge/store.py | 23 ++ eventing/eventbridge/store_registry.py | 220 +++++++++++++++++++ eventing/tests/test_auth.py | 6 +- eventing/tests/test_continue_tenancy.py | 252 ++++++++++++--------- eventing/tests/test_groups.py | 6 +- eventing/tests/test_owner_index.py | 250 +++++++++++++++++++++ eventing/tests/test_store_registry.py | 257 ++++++++++++++++++++++ 14 files changed, 1527 insertions(+), 265 deletions(-) create mode 100644 eventing/eventbridge/owner_index.py create mode 100644 eventing/eventbridge/store_registry.py create mode 100644 eventing/tests/test_owner_index.py create mode 100644 eventing/tests/test_store_registry.py diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index 4fe6d958..7bd0abaf 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -20,8 +20,9 @@ from eventbridge.netcands import enumerate_candidates, format_startup_hint from eventbridge.ntfy import NtfyPublisher from eventbridge.openapi import spec +from eventbridge.owner_index import OwnerIndex from eventbridge.router import Dispatcher -from eventbridge.store import Store +from eventbridge.store_registry import StoreRegistry from shared import keyset, signing from shared.pidfile import PidFile @@ -43,10 +44,12 @@ def main() -> int: oas_path.parent.mkdir(parents=True, exist_ok=True) oas_path.write_text(json.dumps(spec(), indent=2)) - store = Store(pathlib.Path(cfg.tmpdir) / "eventbridge") - minter = Minter() - for corr in store.all_correlations(limit=10000): - minter.remember(corr) + eb_root = pathlib.Path(cfg.tmpdir) / "eventbridge" + # §2.6 — the global `correlationid -> userkey` index. Always built, including in + # single-tenant mode: the uniqueness guarantee it provides is worth having for every + # deployment, and it is what replaces the startup seeding loop below. + owners = OwnerIndex(eb_root) + minter = Minter(index=owners) # §11 — key material is loaded ONCE, here, and deliberately not caught. A bridge # that believes it is signing but is not fails silently; one that will not start @@ -115,13 +118,27 @@ def main() -> int: print(f"[eventbridge] registry has {len(users)} user(s) but " f"EB_TENANCY_MODE=single, so it is not consulted") + # §6.1 — per-user stores. Built after `topics` because the layout depends on the + # mode: single-tenant keeps `responses.sqlite`/`sessions.sqlite` directly in the + # bridge root, so a Phase 2 deployment's existing sessions and transcripts stay + # exactly where they are and keep showing up in the UI. + stores = StoreRegistry(eb_root, max_open=cfg.max_open_stores, + multi=topics.multi) + # The `shared` store: single-tenant mode's only store, and multi-tenant mode's home + # for anything that arrives without a `userkey` (§6.1's `unattributed` path). + store = stores.for_userkey(None) + if topics.multi: + print(f"[eventbridge] per-user stores under {eb_root}/users " + f"(max_open={cfg.max_open_stores}), " + f"{owners.count()} correlation(s) in the ownership index") + # The producer needs the RESPONSES topic too: group lifecycle events go there, # not on requests, because EventRunner would try to execute anything on requests. producer = Producer(cfg.kafka_bootstrap, cfg.request_topic, cfg.source_uri, response_topic=cfg.response_topic, seed=seed, kid=cfg.signing_kid or None, topics=topics) - groups = GroupService(cfg, store, producer, minter) + groups = GroupService(cfg, store, producer, minter, stores=stores) ntfy = NtfyPublisher(cfg.ntfy, cfg.public_base_url, store=store) if cfg.ntfy.enabled and cfg.ntfy.topic: @@ -134,7 +151,8 @@ def main() -> int: keyset=ks, require_signature=cfg.require_response_signature, bridge_kid=cfg.signing_kid or None, - topics=topics) + topics=topics, + stores=stores if topics.multi else None) consumer.start() # Back-fill prompts from the requests topic — also gives us prompt visibility @@ -152,7 +170,9 @@ def main() -> int: group_mirror.start() h = Handlers(cfg, store, producer, minter, groups=groups, - registry=users if topics.multi else None) + registry=users if topics.multi else None, + stores=stores if topics.multi else None, + owners=owners if topics.multi else None) dsp = Dispatcher() dsp.add("POST", r"/v0/agents", h.start_agent) dsp.add("POST", r"/v0/agents/(?P[a-z0-9-]+)/continue", h.continue_agent) diff --git a/eventing/eventbridge/config.py b/eventing/eventbridge/config.py index 517d9f34..5467955f 100644 --- a/eventing/eventbridge/config.py +++ b/eventing/eventbridge/config.py @@ -92,6 +92,11 @@ class Cfg: # the same reason Phase 2's empty allowed_users does: the other reading turns a # missing file into an open door. user_registry_path: str = "" + # §6.1 — LRU ceiling on open per-user SQLite stores. Two connections per tenant, + # each in WAL mode (main + `-wal` + `-shm`), so the default 1024 soft RLIMIT_NOFILE + # bounds this near 150-200 tenants; 64 leaves room for the Kafka sockets, the HTTP + # listener and the worker pool. + max_open_stores: int = 64 ntfy: NtfyCfg = field(default_factory=NtfyCfg) @@ -190,6 +195,7 @@ def load() -> Cfg: f"got {cfg.tenancy_mode!r}. See DESIGN_PHASE3.md §8.1.") cfg.topic_prefix = e("EB_TOPIC_PREFIX", cfg.topic_prefix) cfg.user_registry_path = e("EB_USER_REGISTRY_PATH", cfg.user_registry_path) + cfg.max_open_stores = int(e("EB_MAX_OPEN_STORES", str(cfg.max_open_stores))) cfg.ntfy.enabled = (e("NTFY_ENABLED", "true" if cfg.ntfy.enabled else "false").lower() == "true") cfg.ntfy.base_url = e("NTFY_BASE_URL", cfg.ntfy.base_url) diff --git a/eventing/eventbridge/correlation.py b/eventing/eventbridge/correlation.py index 1e0f027a..c1d95721 100644 --- a/eventing/eventbridge/correlation.py +++ b/eventing/eventbridge/correlation.py @@ -19,22 +19,113 @@ def _load(name: str) -> list[str]: class Minter: - def __init__(self, seed: int | None = None) -> None: + """Mints unguessable-ish, human-readable correlation ids that are globally unique. + + Phase 3 §2.6: with an `index` (the `OwnerIndex`), uniqueness is enforced by one + indexed `SELECT` against a `UNIQUE` column rather than by an in-memory `seen` set + seeded at startup. That matters for two reasons: + + * **Correctness across tenants.** Per-user stores (§6.1) mean the old seeding loop + would have to read `all_correlations()` from N stores — so N SQLite opens before the + socket binds, and at 100 tenants with 10,000 correlations each that is a measurable + startup delay for a guarantee one `SELECT` gives directly. + * **It is what keeps `sessionuuid` unsalted.** `uuid5(NAMESPACE, correlationid)` is + derived identically by both services with no mapping table. Salting it with the + userkey to avoid cross-tenant collisions would break every existing session, and + §2.6 rejects that because this uniqueness check removes the collision instead. + + Without an `index` it keeps the Phase 2 behaviour exactly — an in-memory set, seeded + by `remember()` — so existing callers and tests are unaffected. + """ + + def __init__(self, seed: int | None = None, index=None) -> None: self._rng = random.Random(seed) self._seen: set[str] = set() self._lock = threading.Lock() + self._index = index - def mint(self) -> str: + def mint(self, *, userkey: str | None = None) -> str: + """Mint an id nobody holds, and claim it for `userkey` in the same breath. + + `userkey` is keyword-only, and callers reach this through `mint_for()` rather + than passing it directly. Both choices exist for the same reason: `Minter` is + substituted by test fakes and by anything standing in for an id generator, and a + signature change here breaks every substitute at the *call site* rather than where + the substitute is defined. `mint_for` keeps `mint()` callable with no arguments, so + an older substitute keeps working untouched. + + The claim is inside the retry loop rather than after it, so a candidate that + another thread wins between the check and the insert simply fails and the loop + tries again. Claiming afterwards would reintroduce the race the index exists to + close — two HTTP workers can mint concurrently, since EventBridge is + single-replica but multi-threaded. + """ for _ in range(2000): candidate = f"{self._rng.choice(_ADJ)}-{self._rng.choice(_ANI)}-{self._rng.randint(0, 9999):04d}" if not REGEX.match(candidate): continue - with self._lock: - if candidate not in self._seen: - self._seen.add(candidate) - return candidate + if self._index is None: + with self._lock: + if candidate not in self._seen: + self._seen.add(candidate) + return candidate + continue + # The index is authoritative. `exists` is a fast negative filter; `claim` + # is the atomic one that actually decides, and its Collision is what makes a + # concurrent winner retry rather than both callers believing they own the id. + if self._index.exists(candidate): + continue + try: + self._index.claim(candidate, userkey) + except Exception: # noqa: BLE001 - a lost race is a retry, not a failure + continue + return candidate raise RuntimeError("correlation ID space exhausted") def remember(self, corr: str) -> None: + """Phase 2's seeding hook. A no-op when an index is configured — the index + already knows every claimed id, so there is nothing to remember.""" + if self._index is not None: + return with self._lock: self._seen.add(corr) + + +def mint_for(minter, userkey: str | None) -> str: + """Mint from `minter`, passing `userkey` only when it can accept it. + + A free function rather than a method because the object being called may not be a + `Minter` at all: the handlers and `GroupService` accept any object with `mint()`, and + several tests supply deterministic fakes that predate tenancy. Probing for support + keeps those working unchanged while real deployments get the ownership claim. + + The fallback is not silent about what it gives up — a substitute without `userkey` + support mints an id that is NOT claimed in the owner index, so the uniqueness + guarantee degrades to whatever that substitute implements. That is acceptable only + because it is reachable from test doubles, never from `__main__`, which always + constructs a real `Minter` with an index. + """ + if _accepts_userkey(minter.mint): + return minter.mint(userkey=userkey) + return minter.mint() + + +def _accepts_userkey(fn) -> bool: + """Whether `fn` can be called with a `userkey` keyword. + + Inspected rather than discovered by catching `TypeError` around the call: a working + `mint()` can itself raise `TypeError` from its own body, and swallowing that would + silently retry the mint without an ownership claim — turning a real bug into a + missing index entry, which is far harder to notice. Anything unintrospectable is + assumed to support it, so a `Mock` gets the keyword (and records it) rather than + being quietly downgraded. + """ + import inspect + try: + sig = inspect.signature(fn) + except (TypeError, ValueError): + return True + params = sig.parameters + if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()): + return True + return "userkey" in params diff --git a/eventing/eventbridge/group_service.py b/eventing/eventbridge/group_service.py index 066da4a7..77724512 100644 --- a/eventing/eventbridge/group_service.py +++ b/eventing/eventbridge/group_service.py @@ -9,6 +9,7 @@ import datetime as dt from typing import Any +from eventbridge.correlation import mint_for from eventbridge.groups import completion_reason, progress from shared import ce @@ -18,38 +19,75 @@ def _log(msg: str) -> None: class GroupService: - def __init__(self, cfg, store, producer, minter) -> None: + """Group lifecycle, between the HTTP handlers and the store. + + Phase 3 §6.1: `stores` is the `StoreRegistry` when per-user stores are in use. The + single `store` stays the default and is what every single-tenant deployment keeps + using, so `self.store` remains correct wherever a tenant is not in question — + `on_member_event` and the deadline sweeper both route by the event's own `userkey`. + """ + + def __init__(self, cfg, store, producer, minter, stores=None) -> None: self.cfg = cfg self.store = store self.producer = producer self.minter = minter + self.stores = stores + + def _store_for(self, userkey: str | None): + """The store one tenant's group data lives in. + + Falls back to the single store whenever there is no registry, which is both + single-tenant mode and every existing test — so this method is a no-op there + rather than a branch each caller has to remember. + """ + if self.stores is None: + return self.store + return self.stores.for_userkey(userkey) # ---- creation and fan-out --------------------------------------------- def create(self, *, label: str | None, expected: int | None, min_success: int | None = None, deadline_s: float | None = None, - idempotency_key: str | None = None) -> tuple[str, bool]: - groupid = self.minter.mint() + idempotency_key: str | None = None, + userkey: str | None = None) -> tuple[str, bool]: + # A group id comes from the same generator as a correlation id, so it is claimed + # in the same index: the two namespaces are separate in the URL space but share + # one id space, and letting a group id collide with a correlation id would make + # `owner_of` ambiguous for whichever arrived second. + groupid = mint_for(self.minter, userkey) deadline = None if deadline_s: deadline = (dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=deadline_s)).isoformat( timespec="milliseconds").replace("+00:00", "Z") - out = self.store.create_group(groupid, label=label, expected=expected, - min_success=min_success, deadline_utc=deadline, - idempotency_key=idempotency_key) + out = self._store_for(userkey).create_group( + groupid, label=label, expected=expected, + min_success=min_success, deadline_utc=deadline, + idempotency_key=idempotency_key) if not out["created"]: # An idempotency-key hit: the caller is retrying, not asking for a second # batch. Give back the original so a retry cannot double-launch 100 agents. return out["groupid"], False - self._publish_started(out["groupid"], label, expected, min_success, deadline) + self._publish_started(out["groupid"], label, expected, min_success, deadline, + userkey=userkey) return out["groupid"], True - def _publish_started(self, groupid, label, expected, min_success, deadline) -> None: + def _publish_started(self, groupid, label, expected, min_success, deadline, + userkey: str | None = None) -> None: try: self.producer.publish_group_event( type_=ce.TYPE_GROUP_STARTED, groupid=groupid, subject="group-started", data={"label": label, "expected": expected, "min_success": min_success, - "deadline_utc": deadline}) + "deadline_utc": deadline}, + userkey=userkey) + except TypeError: + # NOT swallowed. A TypeError here is a signature mismatch between this call + # and the producer — a programming error, not a broker being down — and the + # broad `except` below would turn it into a group that silently never + # announces itself. Phase 3 hit exactly that while adding `userkey`: every + # group event stopped being published and the only symptom was a batch that + # never completed. Loud is correct for this one. + raise except Exception as e: # noqa: BLE001 - the group still exists in the store _log(f"could not publish group.started for {groupid}: {e!r}") @@ -66,13 +104,17 @@ def submit_members(self, groupid: str, prompts: list[str], *, which a member's terminal event arrives for an unknown member. """ corrs: list[str] = [] + # Resolved once outside the loop: every member of a batch belongs to the caller + # who launched it, so re-resolving per member would be N LRU touches for one + # answer — and at 100 members that is 100 chances to evict something else. + store = self._store_for(userkey) for prompt in prompts: - corr = self.minter.mint() + corr = mint_for(self.minter, userkey) sess = ce.session_uuid(corr) workdir = f"{self.cfg.tmpdir}/eventrunner/work/{corr}" - self.store.upsert_session(corr, sess, workdir, prompt) - self.store.insert_prompt(corr, "start", prompt, submitter=submitter) - self.store.add_group_member(groupid, corr) + store.upsert_session(corr, sess, workdir, prompt) + store.insert_prompt(corr, "start", prompt, submitter=submitter) + store.add_group_member(groupid, corr) self.producer.publish_request( prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", groupid=groupid, @@ -94,15 +136,20 @@ def on_member_event(self, event: dict[str, Any], *, replay: bool = False) -> Non corr = event.get("correlationid") if not groupid or not corr: return + # §6.1: the store is chosen by the event's own `userkey`, which is signed (§2.6) + # and therefore not rewritable in flight. Routing by the event rather than by any + # ambient state is what keeps one tenant's batch progress out of another's store. + userkey = event.get(ce.EXT_USERKEY) + store = self._store_for(userkey) # Accepted even if the group row does not exist yet: a restart re-scans the # topic and a fast agent can finish before its group's started event lands. - self.store.add_group_member(groupid, corr) + store.add_group_member(groupid, corr) final = str(event.get("final", "false")).lower() == "true" if not final: - self.store.mark_member_running(groupid, corr) + store.mark_member_running(groupid, corr) return - first = self.store.mark_member_finished(groupid, corr, - event.get("phase") or "result") + first = store.mark_member_finished(groupid, corr, + event.get("phase") or "result") if replay: return if not first: @@ -110,7 +157,7 @@ def on_member_event(self, event: dict[str, Any], *, replay: bool = False) -> Non # exists to prevent, so say so once and stop. _log(f"duplicate terminal event for {groupid}/{corr} ignored") return - self.maybe_complete(groupid) + self.maybe_complete(groupid, userkey=userkey) def on_group_event(self, event: dict[str, Any], *, replay: bool = False) -> None: """Reconcile a group lifecycle event read back off the topic. @@ -124,30 +171,32 @@ def on_group_event(self, event: dict[str, Any], *, replay: bool = False) -> None if not groupid: return data = event.get("data") or {} + store = self._store_for(event.get(ce.EXT_USERKEY)) if event.get("type") == ce.TYPE_GROUP_STARTED: - self.store.create_group( + store.create_group( groupid, label=data.get("label"), expected=data.get("expected"), min_success=data.get("min_success"), deadline_utc=data.get("deadline_utc")) elif event.get("type") == ce.TYPE_GROUP_COMPLETED: # Marks completed_utc via the same guarded UPDATE, so a later member replay # cannot decide the group still needs completing. - self.store.complete_group(groupid, data.get("reason") or "all") + store.complete_group(groupid, data.get("reason") or "all") # ---- completion ------------------------------------------------------- - def maybe_complete(self, groupid: str) -> bool: - group = self.store.get_group(groupid) + def maybe_complete(self, groupid: str, *, userkey: str | None = None) -> bool: + store = self._store_for(userkey) + group = store.get_group(groupid) if group is None or group.get("completed_utc"): return False - counts = self.store.group_counts(groupid) + counts = store.group_counts(groupid) reason = completion_reason(group, counts) if reason is None: return False # The guard is the UPDATE, not this check: two threads can both get here. - if not self.store.complete_group(groupid, reason): + if not store.complete_group(groupid, reason): return False - group = self.store.get_group(groupid) - members = self.store.group_members(groupid) + group = store.get_group(groupid) + members = store.group_members(groupid) p = progress(group, members) _log(f"group {groupid} complete ({reason}): " f"{p['finished']} finished, {p['failed']} failed, {p['elapsed']}") @@ -159,42 +208,70 @@ def maybe_complete(self, groupid: str) -> bool: "finished": p["finished"], "failed": p["failed"], "expected": p["expected"], "members": p["members_seen"], "elapsed_s": p["elapsed_s"], "elapsed": p["elapsed"], - "slowest_member_s": _slowest(members)}) + "slowest_member_s": _slowest(members)}, + userkey=userkey) + except TypeError: + # See `_publish_started`: a signature mismatch is a programming error and + # must not be downgraded into "this batch never announced it finished". The + # guard is repeated at both call sites rather than factored into a helper, + # because what it protects is the argument list of *this* call, and a wrapper + # would move the mismatch one frame away from the arguments that caused it. + raise except Exception as e: # noqa: BLE001 _log(f"could not publish group.completed for {groupid}: {e!r}") return True - def close(self, groupid: str, expected: int | None = None) -> bool: - ok = self.store.close_group(groupid, expected) - self.maybe_complete(groupid) + def close(self, groupid: str, expected: int | None = None, *, + userkey: str | None = None) -> bool: + ok = self._store_for(userkey).close_group(groupid, expected) + self.maybe_complete(groupid, userkey=userkey) return ok - def cancel(self, groupid: str) -> bool: + def cancel(self, groupid: str, *, userkey: str | None = None) -> bool: """Stop queued members. Running agents are NOT stopped — see §21.9.5. Interrupting a live `claude` needs a mechanism EventRunner honours, which belongs with the Job-per-request work. Saying so plainly beats implying an abort that does not happen. """ - ok = self.store.cancel_group(groupid) - self.maybe_complete(groupid) + ok = self._store_for(userkey).cancel_group(groupid) + self.maybe_complete(groupid, userkey=userkey) return ok def sweep_deadlines(self) -> int: """Complete groups whose deadline passed. Without this a group that is short - one member never completes and therefore never notifies (§21.6).""" + one member never completes and therefore never notifies (§21.6). + + With per-user stores this has to sweep **every** tenant, not the one store: a + stalled batch belonging to a tenant nobody has touched recently is exactly the + case the deadline exists for, and it is also the tenant least likely to be in the + LRU. Sweeping only the default store would silently stop honouring deadlines for + everyone else — the failure mode being silence, which §21.6 calls the worst one. + """ n = 0 - for groupid in self.store.groups_past_deadline(): - if self.maybe_complete(groupid): - n += 1 + for userkey in self._tenants_to_sweep(): + for groupid in self._store_for(userkey).groups_past_deadline(): + if self.maybe_complete(groupid, userkey=userkey): + n += 1 return n + def _tenants_to_sweep(self) -> list[str | None]: + """Every tenant whose groups the sweeper must look at. + + `None` is always included: it is the single store in single-tenant mode and the + `shared` tier in multi-tenant mode, and in both cases it can hold groups. + """ + if self.stores is None: + return [None] + return [None, *self.stores.known_userkeys()] + # ---- reads ------------------------------------------------------------ - def snapshot(self, groupid: str) -> dict[str, Any] | None: - group = self.store.get_group(groupid) + def snapshot(self, groupid: str, *, userkey: str | None = None) -> dict[str, Any] | None: + store = self._store_for(userkey) + group = store.get_group(groupid) if group is None: return None - members = self.store.group_members(groupid) + members = store.group_members(groupid) p = progress(group, members) p["members"] = members return p diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index e5923955..9a37c344 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -10,7 +10,7 @@ from eventbridge import auth, ghauth from eventbridge.config import Cfg from eventbridge.correlation import REGEX as CORR_REGEX -from eventbridge.correlation import Minter +from eventbridge.correlation import Minter, mint_for from eventbridge.html_view import render from eventbridge.kafka_out import Producer from eventbridge.openapi import SWAGGER_HTML, spec @@ -92,8 +92,11 @@ def _read_form(environ) -> dict: class Handlers: def __init__(self, cfg: Cfg, store: Store, producer: Producer, minter: Minter, - groups=None, registry=None) -> None: + groups=None, registry=None, stores=None, owners=None) -> None: self.cfg = cfg + # The single store. In single-tenant mode this is the only one and every read + # and write goes through it, exactly as in Phase 2. In multi-tenant mode it is + # the `shared` tier's store and `_store_of` picks the per-tenant one instead. self.store = store self.producer = producer self.minter = minter @@ -101,6 +104,11 @@ def __init__(self, cfg: Cfg, store: Store, producer: Producer, minter: Minter, # §2.5 — the approved-user registry, or None in single-tenant mode, where there # is nothing to look a caller up in. self.registry = registry + # §6.1 — `userkey -> Store`. None in single-tenant mode. + self.stores = stores + # §2.6/§6.2 — the global `correlationid -> userkey` index. None in single-tenant + # mode, where there is only one tenant and nothing to scope to. + self.owners = owners # One cache for the process, so a burst of requests from the same user # costs one GitHub call rather than one each. self.logins = ghauth.LoginCache(cfg.github_cache_ttl_s) @@ -131,58 +139,60 @@ def _caller(self, environ): registry=self.registry) def _owner_userkey(self, correlationid: str) -> tuple[str | None, str | None]: - """Which tenant owns this correlation, for a `/continue` turn. + """Which tenant owns this correlation. §2.6, §6.2. Returns `(userkey, unresolved_reason)`. Exactly one is ever set: - * `(None, None)` — single-tenant mode; topics do not depend on a key. + * `(None, None)` — single-tenant mode, or the `shared` tier; no key needed. * `(key, None)` — the owning tenant. * `(None, reason)` — multi mode and the owner cannot be determined. - The two-value return exists because the two `None` cases need opposite - handling and a bare `None` cannot tell them apart. Passing an unresolved - `None` into `TopicSet.requests()` raises `ValueError` — correctly, since - publishing to a shared topic would run this turn on another tenant's runner — - but that surfaces as a WSGI 500 with a stack trace instead of a status code. - A correlation with no resolvable owner is reachable in normal operation: a - user removed from the registry, a `prompts` row whose `submitter` is NULL - (the column is nullable and pre-dates auth), or a correlation back-filled - from the topic by `RequestsMirror`. So the caller refuses with a `503` naming - the problem, matching how §2.5 treats a user whose topics do not exist. - - `/continue` is still unauthenticated here — Phase 2 §3.1's capability-URL - design — so this resolves the owner from what was *recorded at submit time* - rather than from the caller. The turn therefore reaches the right tenant's - runner, which is what stops a resume from being executed under another user's - credential. It does NOT yet check that the caller is that owner: owner-scoped - reads and `?k=` capability keys are §6.2/§4.4 (T7/T9, rollout step 5), and until - they land `multi` mode's `/continue` is as open as Phase 2's. - - The lookup goes through the recorded submitter rather than a - `correlationid -> userkey` index because that index is T5's global table, which - does not exist yet. When it does, this becomes one indexed `SELECT` and the - derivation here goes away — §2.6 is explicit that the index has to exist anyway, - for the uniqueness guarantee that keeps `sessionuuid` unsalted. + The two-value return exists because the two `None` cases need opposite handling + and a bare `None` cannot tell them apart. Passing an unresolved `None` into + `TopicSet.requests()` raises `ValueError` — correctly, since publishing to a + shared topic would run the turn on another tenant's runner — but that surfaces as + a WSGI 500 instead of a status code, so callers refuse with a `503` naming the + problem. + + This is now **one indexed lookup** against the global `correlationid -> userkey` + table. The previous implementation derived the owner from the recorded + `submitter`, which could not distinguish a GitHub `alice` from a static `alice` + and went wrong the moment a second issuer was configured; §2.6 always required + the index to exist for the uniqueness guarantee, so using it here costs nothing + extra and removes that limitation. + + `/continue` is still unauthenticated — Phase 2 §3.1's capability-URL design — so + this answers *who owns it*, not *who is asking*. The turn therefore reaches the + right tenant's runner, which is what stops a resume from executing under another + user's credential. Checking that the caller IS the owner is §6.2's read-scoping + (T7), still to come. """ - if not self.cfg.topics.multi: + if not self.cfg.topics.multi or self.owners is None: return None, None - prompts = self.store.get_prompts(correlationid) or [] - submitters = [s for p in prompts if (s := p.get("submitter"))] - if not submitters: - return None, ("no submitter recorded for this correlation, so the owning " - "tenant cannot be determined") - for submitter in submitters: - # The issuer is not recorded alongside the submitter today, so this - # cannot distinguish a GitHub `alice` from a static `alice`. In `multi` - # mode anonymous and static submissions are already refused at the - # submit path (§2.4), so GitHub is the only issuer that reaches here — - # stated rather than assumed, because the day an OIDC issuer is wired - # up (§2.4's slot) this becomes wrong and has to move to the index. - if user := (self.registry.lookup("github", submitter) - if self.registry else None): - return user.userkey, None - return None, (f"{submitters[0]} is no longer in the user registry, so the " - f"owning tenant cannot be determined") + userkey, known = self.owners.owner_of(correlationid) + if not known: + return None, (f"{correlationid} is not in the ownership index, so the " + f"owning tenant cannot be determined") + # A known correlation with a NULL owner is the `shared` tier, which is a real + # answer rather than a failure: those events live in `shared/` by design. + return userkey, None + + def _store_of(self, correlationid: str): + """The store holding one correlation's data, or `None` if it cannot be placed. + + Single-tenant mode always returns `self.store`, so every read path below is + byte-identical to Phase 2. In multi-tenant mode an unplaceable correlation + returns `None` and the caller answers `404` — which is also what §6.2 wants a + *foreign* correlation to look like, so the unknown and the not-yours cases are + indistinguishable to a client by construction rather than by remembering to make + them so. + """ + if self.stores is None: + return self.store + userkey, unresolved = self._owner_userkey(correlationid) + if unresolved: + return None + return self.stores.for_userkey(userkey) # ---- start ---- def start_agent(self, environ, start_response, **_): @@ -201,15 +211,19 @@ def start_agent(self, environ, start_response, **_): if groupid and not GROUP_REGEX.match(str(groupid)): return _json(start_response, "400 Bad Request", {"error": "bad groupid"}) - corr = self.minter.mint() + corr = mint_for(self.minter, caller.userkey) sess = ce.session_uuid(corr) workdir = str(pathlib.Path(self.cfg.tmpdir) / "eventrunner" / "work" / corr) - self.store.upsert_session(corr, sess, workdir, prompt) - self.store.insert_prompt(corr, "start", prompt, submitter=submitter) + # The submit path is the one place the tenant is known from the CALLER rather than + # looked up: `mint_for` just claimed this correlation for `caller.userkey`, so + # going back through the index would be a round trip for an answer already held. + store = self.stores.for_userkey(caller.userkey) if self.stores else self.store + store.upsert_session(corr, sess, workdir, prompt) + store.insert_prompt(corr, "start", prompt, submitter=submitter) if groupid: # Membership before publication: the reverse order leaves a window where a # fast agent's terminal event arrives for a member nobody has recorded. - self.store.add_group_member(groupid, corr) + store.add_group_member(groupid, corr) event_id = self.producer.publish_request( prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", @@ -266,7 +280,7 @@ def create_group(self, environ, start_response, **_): ) if not created: # A retried POST must not launch a second batch. - snap = self.groups.snapshot(groupid) or {} + snap = self.groups.snapshot(groupid, userkey=caller.userkey) or {} return _json(start_response, "200 OK", { "groupid": groupid, "created": False, "members": [m["correlationid"] for m in snap.get("members") or []], @@ -290,7 +304,16 @@ def list_groups(self, environ, start_response, **_): negotiation rather than a second path keeps /v0/groups the single name for "the list of batches". """ - groups = self.store.all_groups() + # §6.2: the list shows only the CALLER's groups in multi-tenant mode. Reading + # from their own store is what enforces that — not a filter applied afterwards, + # which is the kind of predicate §6.1 says will eventually be forgotten. + if self.stores is not None: + caller, status, why = self._caller(environ) + if status: + return _deny(start_response, why, status) + groups = self.stores.for_userkey(caller.userkey).all_groups() + else: + groups = self.store.all_groups() accept = environ.get("HTTP_ACCEPT", "") wants_html = ("text/html" in accept and "application/json" not in accept @@ -307,7 +330,9 @@ def group_status(self, environ, start_response, groupid: str, **_): """The JSON the CLI polls. Same arithmetic as the page, one source of truth.""" if not GROUP_REGEX.match(groupid): return _json(start_response, "400 Bad Request", {"error": "bad groupid"}) - snap = self.groups.snapshot(groupid) if self.groups else None + owner, unresolved = self._owner_userkey(groupid) + snap = (self.groups.snapshot(groupid, userkey=owner) + if self.groups and not unresolved else None) if snap is None: return _json(start_response, "404 Not Found", {"error": "unknown groupid"}) full = environ.get("QUERY_STRING", "").find("full=1") >= 0 @@ -322,7 +347,9 @@ def group_html(self, environ, start_response, groupid: str, **_): if not GROUP_REGEX.match(groupid): start_response("400 Bad Request", [("Content-Type", "text/plain")]) return [b"bad groupid"] - snap = self.groups.snapshot(groupid) if self.groups else None + owner, unresolved = self._owner_userkey(groupid) + snap = (self.groups.snapshot(groupid, userkey=owner) + if self.groups and not unresolved else None) if snap is None: start_response("404 Not Found", [("Content-Type", "text/plain")]) return [b"unknown groupid"] @@ -342,7 +369,8 @@ def _decorate_member(self, m: dict) -> dict: a minute instead of ten. """ out = dict(m) - events = self.store.events_for(m["correlationid"]) + member_store = self._store_of(m["correlationid"]) + events = member_store.events_for(m["correlationid"]) if member_store else [] text = None for e in reversed(events): t = (e.get("data") or {}).get("text") @@ -363,13 +391,17 @@ def close_group(self, environ, start_response, groupid: str, **_): if self.groups is None: return _json(start_response, "503 Service Unavailable", {"error": "disabled"}) body = _read_json(environ) - ok = self.groups.close(groupid, int(body["expected"]) if body.get("expected") else None) + owner, _ = self._owner_userkey(groupid) + ok = self.groups.close(groupid, + int(body["expected"]) if body.get("expected") else None, + userkey=owner) return _json(start_response, "200 OK", {"groupid": groupid, "closed": ok}) def cancel_group(self, environ, start_response, groupid: str, **_): if self.groups is None: return _json(start_response, "503 Service Unavailable", {"error": "disabled"}) - ok = self.groups.cancel(groupid) + owner, _ = self._owner_userkey(groupid) + ok = self.groups.cancel(groupid, userkey=owner) return _json(start_response, "200 OK", { "groupid": groupid, "cancelled": ok, "note": "queued members will not be submitted; agents already running " @@ -384,24 +416,24 @@ def continue_agent(self, environ, start_response, correlationid: str, **_): prompt = body.get("prompt") if not prompt: return _json(start_response, "400 Bad Request", {"error": "prompt required"}) - session = self.store.get_session(correlationid) + # One lookup decides both which store holds this conversation and which tenant + # the resume must be published for. An unplaceable correlation is a 404 — the same + # answer a foreign one gets (§6.2), so the two are indistinguishable to a client. + owner, unresolved = self._owner_userkey(correlationid) + store = None if unresolved else ( + self.stores.for_userkey(owner) if self.stores else self.store) + session = store.get_session(correlationid) if store else None if session is None: return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) sess = session["sessionuuid"] - owner, unresolved = self._owner_userkey(correlationid) - if unresolved: - # Checked before anything is written, so a refused turn leaves no touched - # session row and no orphan prompt row claiming a turn that never ran. - return _json(start_response, "503 Service Unavailable", - {"error": f"cannot resume: {unresolved}"}) - self.store.upsert_session(correlationid, sess, session["workdir"], None) - turn_index = self.store.insert_prompt(correlationid, "continue", prompt) + store.upsert_session(correlationid, sess, session["workdir"], None) + turn_index = store.insert_prompt(correlationid, "continue", prompt) event_id = self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", userkey=owner, ) - next_seq = len(self.store.events_for(correlationid)) + 1 + next_seq = len(store.events_for(correlationid)) + 1 return _json(start_response, "202 Accepted", { "correlationid": correlationid, "sequence": next_seq, "turn_index": turn_index, "event_id": event_id, @@ -414,21 +446,21 @@ def continue_agent_html(self, environ, start_response, correlationid: str, **_): form = _read_form(environ) prompt = form.get("prompt", "").strip() if prompt: - session = self.store.get_session(correlationid) + owner, unresolved = self._owner_userkey(correlationid) + if unresolved: + # The form posts from the transcript page, so the useful answer is the + # page with an error on it rather than a silent 303 — redirecting would + # look like the turn was accepted and then vanished, which is the Phase 2 + # §6.1 class of bug this phase keeps trying to avoid. + start_response("503 Service Unavailable", + [("Content-Type", "text/plain; charset=utf-8")]) + return [f"cannot resume {correlationid}: {unresolved}\n".encode()] + store = self.stores.for_userkey(owner) if self.stores else self.store + session = store.get_session(correlationid) if session is not None: sess = session["sessionuuid"] - owner, unresolved = self._owner_userkey(correlationid) - if unresolved: - # The form posts from the transcript page, so the useful answer is - # the page with an error on it rather than a JSON body in the - # browser. Redirecting silently would look like the turn was - # accepted and then vanished, which is the Phase 2 §6.1 class of - # bug this phase keeps trying to avoid. - start_response("503 Service Unavailable", - [("Content-Type", "text/plain; charset=utf-8")]) - return [f"cannot resume {correlationid}: {unresolved}\n".encode()] - self.store.upsert_session(correlationid, sess, session["workdir"], None) - self.store.insert_prompt(correlationid, "continue", prompt) + store.upsert_session(correlationid, sess, session["workdir"], None) + store.insert_prompt(correlationid, "continue", prompt) self.producer.publish_request( prompt=prompt, correlationid=correlationid, sessionuuid=sess, mode="continue", subject="resume", @@ -442,9 +474,10 @@ def get_html(self, environ, start_response, correlationid: str, **_): if not CORR_REGEX.match(correlationid): start_response("400 Bad Request", [("Content-Type", "text/plain")]) return [b"bad correlationid"] - session = self.store.get_session(correlationid) - events = self.store.events_for(correlationid) - prompts = self.store.get_prompts(correlationid) + store = self._store_of(correlationid) + session = store.get_session(correlationid) if store else None + events = store.events_for(correlationid) if store else [] + prompts = store.get_prompts(correlationid) if store else [] if session is None and not events and not prompts: start_response("404 Not Found", [("Content-Type", "text/plain")]) return [b"unknown correlationid"] @@ -454,10 +487,10 @@ def get_html(self, environ, start_response, correlationid: str, **_): show_continue = (qs.get("continue", ["0"])[0]).lower() in ("1", "true", "yes", "on") # A member reached from the group page needs a way back; without it the # operator has to hand-edit the URL to return to what they were watching. - groupid = self.store.group_of(correlationid) + groupid = store.group_of(correlationid) group_label = None if groupid: - g = self.store.get_group(groupid) or {} + g = store.get_group(groupid) or {} group_label = g.get("label") html = render(correlationid, session, events, prompts=prompts, show_continue=show_continue, @@ -468,9 +501,10 @@ def get_html(self, environ, start_response, correlationid: str, **_): # ---- JSON: turn-grouped view (chat) ---- def get_turns(self, environ, start_response, correlationid: str, **_): - session = self.store.get_session(correlationid) - events = self.store.events_for(correlationid) - prompts = self.store.get_prompts(correlationid) + store = self._store_of(correlationid) + session = store.get_session(correlationid) if store else None + events = store.events_for(correlationid) if store else [] + prompts = store.get_prompts(correlationid) if store else [] if session is None and not events and not prompts: return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) from eventbridge.html_view import group_by_turns @@ -496,9 +530,9 @@ def get_turns(self, environ, start_response, correlationid: str, **_): body = { "correlationid": correlationid, "turns": out, - "final": self.store.final_seen(correlationid), + "final": store.final_seen(correlationid), } - gid = self.store.group_of(correlationid) + gid = store.group_of(correlationid) if gid: body["groupid"] = gid body["group_url"] = f"{self.cfg.public_base_url}/v0/groups/{gid}" @@ -508,16 +542,24 @@ def get_turns(self, environ, start_response, correlationid: str, **_): def get_events(self, environ, start_response, correlationid: str, **_): qs = urllib.parse.parse_qs(environ.get("QUERY_STRING", "")) since = int((qs.get("since", ["0"])[0]) or 0) - events = self.store.events_for(correlationid, since_seq=since) + store = self._store_of(correlationid) + if store is None: + # §6.2: an unplaceable correlation reads as absent, never as "exists but not + # yours" — the latter would let anyone enumerate live ids across tenants. + return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) + events = store.events_for(correlationid, since_seq=since) return _json(start_response, "200 OK", { "correlationid": correlationid, "events": events, - "final": self.store.final_seen(correlationid), + "final": store.final_seen(correlationid), }) # ---- Raw CloudEvents (JSONL) ---- def get_events_jsonl(self, environ, start_response, correlationid: str, **_): - raws = self.store.raw_events_for(correlationid) + store = self._store_of(correlationid) + if store is None: + return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) + raws = store.raw_events_for(correlationid) body = "\n".join(json.dumps(e) for e in raws).encode() + b"\n" start_response("200 OK", [ ("Content-Type", "application/x-ndjson"), @@ -544,7 +586,17 @@ def _sse_generator(self, correlationid: str, since_seq: int = 0): # sequence already rendered server-side, this stream only delivers NEW # events — no duplicates, no reload loop. sent = since_seq - events = self.store.events_for(correlationid, since_seq=since_seq) + # Resolved ONCE and held for the life of the generator. Re-resolving per poll + # would be a correctness bug, not just a cost: `StoreRegistry` may return a + # *different* Store object after an eviction and reopen, and then `unsubscribe` + # in the `finally` would run against a store that never saw the `subscribe` — + # leaking the Event and leaving the old store pinned forever. Holding the + # reference is also what makes the registry's "pinned while subscribed" rule + # work, since the subscriber list lives on this object. + store = self._store_of(correlationid) + if store is None: + return + events = store.events_for(correlationid, since_seq=since_seq) seen_final = False for e in events: yield f"data: {json.dumps(e)}\n\n".encode() @@ -556,13 +608,13 @@ def _sse_generator(self, correlationid: str, since_seq: int = 0): # holding open and waiting for events that will never arrive. return - ev = self.store.subscribe(correlationid) + ev = store.subscribe(correlationid) try: deadline = time.monotonic() + 120.0 while time.monotonic() < deadline: if ev.wait(timeout=15.0): ev.clear() - new = self.store.events_for(correlationid, since_seq=sent) + new = store.events_for(correlationid, since_seq=sent) for e in new: yield f"data: {json.dumps(e)}\n\n".encode() sent = max(sent, e["sequence"]) @@ -571,7 +623,7 @@ def _sse_generator(self, correlationid: str, since_seq: int = 0): # keepalive comment yield b": keepalive\n\n" finally: - self.store.unsubscribe(correlationid, ev) + store.unsubscribe(correlationid, ev) # ---- transcripts (§16 Gap B) ---- def put_transcript(self, environ, start_response, correlationid: str, **_): @@ -588,7 +640,8 @@ def put_transcript(self, environ, start_response, correlationid: str, **_): return _json(start_response, "400 Bad Request", {"error": "bad correlationid"}) # Require a known correlation: accepting a transcript for an id we never # minted would let anyone seed arbitrary resume state. - if self.store.get_session(correlationid) is None: + store = self._store_of(correlationid) + if store is None or store.get_session(correlationid) is None: return _json(start_response, "404 Not Found", {"error": "unknown correlationid"}) body, err = _read_body(environ, self.cfg.transcript_max_bytes) @@ -596,18 +649,19 @@ def put_transcript(self, environ, start_response, correlationid: str, **_): status = ("413 Payload Too Large" if "over the" in err else "400 Bad Request") return _json(start_response, status, {"error": err}) - meta = self.store.put_transcript(correlationid, body) + meta = store.put_transcript(correlationid, body) return _json(start_response, "200 OK", meta) def get_transcript(self, environ, start_response, correlationid: str, **_): """Return the stored transcript verbatim, for `--resume `.""" if not CORR_REGEX.match(correlationid): return _json(start_response, "400 Bad Request", {"error": "bad correlationid"}) - body = self.store.get_transcript(correlationid) + store = self._store_of(correlationid) + body = store.get_transcript(correlationid) if store else None if body is None: return _json(start_response, "404 Not Found", {"error": "no transcript checkpointed for this correlationid"}) - meta = self.store.transcript_meta(correlationid) or {} + meta = store.transcript_meta(correlationid) or {} start_response("200 OK", [ ("Content-Type", "application/x-ndjson"), ("Content-Length", str(len(body))), @@ -638,4 +692,24 @@ def docs(self, environ, start_response, **_): return [body] def healthz(self, environ, start_response, **_): - return _json(start_response, "200 OK", {"ok": True}) + """Liveness, plus the tenancy counters §6.1 asks to be surfaced. + + `unattributed` is the one worth watching: it counts responses that arrived in + multi-tenant mode with no `userkey` and were filed under `shared/` rather than + guessed into a tenant's store. A non-zero and rising value means a runner is + misconfigured (no `ER_USERKEY`), which otherwise looks like a working pod whose + output lands somewhere nobody is reading. `ok` stays true regardless — this is + information for an operator, not a reason to fail a liveness probe and restart a + bridge that is working. + """ + out = {"ok": True} + if self.stores is not None: + out["tenancy"] = { + "mode": self.cfg.tenancy_mode, + "unattributed": self.stores.unattributed, + "open_stores": self.stores.open_count, + "evictions": self.stores.evictions, + } + if self.owners is not None: + out["tenancy"]["correlations"] = self.owners.count() + return _json(start_response, "200 OK", out) diff --git a/eventing/eventbridge/kafka_in.py b/eventing/eventbridge/kafka_in.py index 28423238..599da1b9 100644 --- a/eventing/eventbridge/kafka_in.py +++ b/eventing/eventbridge/kafka_in.py @@ -42,6 +42,7 @@ def __init__( require_signature: bool = False, bridge_kid: str | None = None, topics: tenancy.TopicSet | None = None, + stores=None, ) -> None: super().__init__(daemon=True, name="kafka-responses-consumer") self._bootstrap_servers = bootstrap @@ -58,6 +59,9 @@ def __init__( "", response_topic=response_topic, request_topic="") self._topic = self._topics.responses() if not self._topics.multi else response_topic self._store = store + # §6.1 — when set, each response is filed in the store its own `ce_userkey` + # names. `None` keeps the single-store behaviour, which is single-tenant mode. + self._stores = stores self._on_event = on_event self._on_group_event = on_group_event self._on_member_event = on_member_event @@ -150,7 +154,15 @@ def run(self) -> None: try: self._on_group_event(d) except Exception as e: print(f"[kafka_in] group event: {e!r}") else: - self._store.insert_response(d) + # §6.1: routed by the event's OWN userkey, which is signed + # (§2.6) so it cannot be rewritten in flight. An event with no + # userkey in multi-tenant mode lands in `shared/` and bumps the + # `unattributed` counter — never guessed into a tenant's store + # (that corrupts somebody's history) and never dropped (that is + # indistinguishable from an agent that never answered). + target = (self._stores.for_event(evt) if self._stores + else self._store) + target.insert_response(d) if self._on_member_event and d.get("groupid"): try: self._on_member_event(d) except Exception as e: print(f"[kafka_in] member event: {e!r}") diff --git a/eventing/eventbridge/owner_index.py b/eventing/eventbridge/owner_index.py new file mode 100644 index 00000000..605e5001 --- /dev/null +++ b/eventing/eventbridge/owner_index.py @@ -0,0 +1,184 @@ +"""The global `correlationid -> userkey` index. DESIGN_PHASE3.md §2.6, §6.2. + +**The one table that is deliberately cross-tenant**, and it has to be: the lookup that +decides *which* tenant owns a correlation must happen before a tenant — and therefore +before a per-user store (§6.1) — can be chosen. Everything else in Phase 3 pushes data +into per-user files precisely so a query cannot reach another tenant's rows; this is the +documented exception, and it holds nothing but a mapping from a correlation id to a +tenancy key. + +Two jobs, and the second is the reason the first is affordable: + +1. **Owner-scoped reads** (§6.2). `owner_of(corr)` answers "whose is this?" so a handler + can compare it to the caller and answer `404` when they differ. +2. **The global uniqueness guarantee** (§2.6). `UNIQUE` on `correlationid` makes two + tenants minting the same id impossible, which is what lets + `sessionuuid = uuid5(NAMESPACE, correlationid)` stay **unsalted**. Salting it with the + userkey is the tempting alternative and §2.6 rejects it: it would break every existing + session (`--resume` is keyed on that uuid, and Phase 1 §16 Gap B's checkpoint/restore + path is built on it) for a collision this constraint already prevents. + +**The constraint is on `correlationid` ALONE**, and this is the part to get right. +`(userkey, correlationid)` is the correct primary key *inside* a tenant's store and is +exactly the wrong constraint here: two tenants minting one `corr` produce two different +tuples, so it would never conflict and the collision would pass through silently — which +is the failure the whole guarantee exists to prevent. + +It also replaces the `Minter` seeding loop. Phase 2 seeded a `seen` set from +`all_correlations(limit=10000)` on one store; with N per-user stores that would mean N +SQLite opens before the socket binds. §2.6 is explicit that one indexed `SELECT` per mint +is both cheaper and simpler than seeding from every tenant. + +Stdlib only (`sqlite3`). +""" +from __future__ import annotations + +import pathlib +import sqlite3 +import threading + +# `userkey` is nullable on purpose: single-tenant mode and the `shared` tier both record +# ownership as NULL rather than inventing a key. That keeps the uniqueness guarantee on +# `correlationid` in force for every deployment — including the ones that never turn +# multi-tenancy on — so a single-tenant bridge gets the same protection against a +# duplicate mint that a multi-tenant one does. +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS correlation_owner ( + correlationid TEXT PRIMARY KEY, -- UNIQUE, and deliberately NOT (userkey, corr) + userkey TEXT, -- NULL = single-tenant or the `shared` tier + created_utc TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_owner_userkey ON correlation_owner(userkey); +""" + + +class Collision(Exception): + """A correlation id already claimed by a different tenant. + + Separate from the "already mine" case, which is idempotent and must not raise: the + submit path can legitimately re-record the same (corr, userkey) pair, and a retried + request should not fail. Only a genuine cross-tenant clash is an error. + """ + + +class OwnerIndex: + """`correlationid -> userkey`, with `correlationid` globally unique. + + One SQLite file, opened once for the process lifetime. Unlike the per-user stores + this is never evicted — it is small (three short columns per correlation), it is on + the hot path of every mint and every owner-scoped read, and it is the one table whose + absence would silently remove the uniqueness guarantee. + """ + + def __init__(self, root: str | pathlib.Path) -> None: + base = pathlib.Path(root) + base.mkdir(parents=True, exist_ok=True) + self._c = sqlite3.connect(base / "owners.sqlite", + check_same_thread=False, isolation_level=None) + self._c.execute("PRAGMA journal_mode=WAL") + self._c.execute("PRAGMA synchronous=NORMAL") + self._c.executescript(_SCHEMA) + # EventBridge is single-replica (Phase 1 §8.6) but multi-threaded: the HTTP + # worker pool and the responses consumer both reach this. `isolation_level=None` + # means autocommit, so the lock is what makes claim's check-then-insert atomic. + self._lock = threading.Lock() + + # ---- the mint-time guarantee ------------------------------------------- + + def claim(self, correlationid: str, userkey: str | None) -> None: + """Record ownership. Raises `Collision` if another tenant already owns it. + + Idempotent for the same owner, so a retried submit is not an error. The + comparison treats NULL and a key as different owners, which is correct: a + correlation minted in single-tenant mode and later claimed by a tenant is a + genuine ambiguity, not a no-op. + """ + with self._lock: + row = self._c.execute( + "SELECT userkey FROM correlation_owner WHERE correlationid=?", + (correlationid,)).fetchone() + if row is not None: + if row[0] == userkey: + return + raise Collision( + f"correlation {correlationid!r} is already owned by " + f"{row[0] or '(single-tenant)'}, cannot claim for " + f"{userkey or '(single-tenant)'}") + self._c.execute( + "INSERT INTO correlation_owner(correlationid,userkey,created_utc) " + "VALUES (?,?,strftime('%Y-%m-%dT%H:%M:%SZ','now'))", + (correlationid, userkey)) + + def exists(self, correlationid: str) -> bool: + """Whether this id is taken by anyone. The `Minter`'s uniqueness check. + + One indexed lookup against a PRIMARY KEY, which is what §2.6 trades the + N-store seeding loop for. + """ + return self._c.execute( + "SELECT 1 FROM correlation_owner WHERE correlationid=? LIMIT 1", + (correlationid,)).fetchone() is not None + + # ---- owner-scoped reads (§6.2) ----------------------------------------- + + def owner_of(self, correlationid: str) -> tuple[str | None, bool]: + """`(userkey, known)`. `known=False` means the correlation is not in the index. + + The two-value return distinguishes "owned by nobody in particular" (single-tenant + or the `shared` tier, a legitimate NULL) from "never seen". A bare `None` would + conflate them, and §6.2's `404` rule depends on telling them apart: an unknown + correlation and another tenant's correlation must look identical to the caller, + but a handler still needs to know which case it is in to decide whether to + consult a store at all. + """ + row = self._c.execute( + "SELECT userkey FROM correlation_owner WHERE correlationid=?", + (correlationid,)).fetchone() + if row is None: + return None, False + return row[0], True + + def correlations_for(self, userkey: str | None, limit: int = 1000) -> list[str]: + """Every correlation a tenant owns, newest first. Backs the group-list filter.""" + if userkey is None: + rows = self._c.execute( + "SELECT correlationid FROM correlation_owner WHERE userkey IS NULL " + "ORDER BY created_utc DESC LIMIT ?", (limit,)).fetchall() + else: + rows = self._c.execute( + "SELECT correlationid FROM correlation_owner WHERE userkey=? " + "ORDER BY created_utc DESC LIMIT ?", (userkey, limit)).fetchall() + return [r[0] for r in rows] + + def userkeys(self) -> list[str]: + """Tenants with at least one correlation. Used to enumerate stores to sweep.""" + rows = self._c.execute( + "SELECT DISTINCT userkey FROM correlation_owner " + "WHERE userkey IS NOT NULL").fetchall() + return [r[0] for r in rows] + + def forget(self, correlationid: str) -> None: + """Drop one mapping. For §6.5's deletion path. + + Note what this does NOT do: freeing the id for reuse is deliberate, since the + uniqueness guarantee only has to hold for correlations that still exist. The + caller is responsible for having removed the data first — this is the last step, + not the first. + """ + with self._lock: + self._c.execute("DELETE FROM correlation_owner WHERE correlationid=?", + (correlationid,)) + + def forget_tenant(self, userkey: str) -> int: + """Drop every mapping for one tenant, returning how many. §6.5's `rm -rf` half.""" + with self._lock: + cur = self._c.execute( + "DELETE FROM correlation_owner WHERE userkey=?", (userkey,)) + return cur.rowcount or 0 + + def count(self) -> int: + return int(self._c.execute( + "SELECT COUNT(*) FROM correlation_owner").fetchone()[0]) + + def close(self) -> None: + self._c.close() diff --git a/eventing/eventbridge/store.py b/eventing/eventbridge/store.py index db323e2e..27dbff9e 100644 --- a/eventing/eventbridge/store.py +++ b/eventing/eventbridge/store.py @@ -455,6 +455,29 @@ def _notify(self, corr: str) -> None: for ev in self._subscribers.get(corr, []): ev.set() + def close(self) -> None: + """Close both connections. Added for Phase 3 §6.1's LRU eviction. + + Safe to call on a store that may be reopened later: a `Store` holds no state + above SQLite — every method opens a transaction and commits — so reconstructing + it against the same directory yields an equivalent object. + + Wakes any SSE subscriber first. `StoreRegistry` will not evict a store that has + subscribers, so in that path this set is empty; it matters on shutdown, where a + viewer blocked on `Event.wait()` would otherwise hang until its own timeout + instead of noticing the process is going away. + """ + with self._sub_lock: + for evs in self._subscribers.values(): + for ev in evs: + ev.set() + with self._lock: + for c in (self._r, self._s): + try: + c.close() + except Exception: # noqa: BLE001 - shutdown must not raise + pass + def _now_iso() -> str: import datetime as dt diff --git a/eventing/eventbridge/store_registry.py b/eventing/eventbridge/store_registry.py new file mode 100644 index 00000000..3b20caec --- /dev/null +++ b/eventing/eventbridge/store_registry.py @@ -0,0 +1,220 @@ +"""Per-user stores, opened lazily and closed on an LRU. DESIGN_PHASE3.md §6.1. + +```text +{tmpdir}/eventbridge/ + users/ + gh-mrsabath-4c1d9e07/ + responses.sqlite # responses, keyed (correlationid, sequence) + sessions.sqlite # sessions, prompts, groups, group_members, transcripts + gh-aslom-7b2e55a1/ + ... + shared/ # tier `shared` (§3.8), and ALL of single-tenant mode +``` + +**Why separate files rather than one database with a `userkey` column.** The single-DB +version is cheaper — one connection pair, one startup, no file-descriptor arithmetic — +and §6.1 rejects it: `store.py` has 28 methods and every one reads or writes tenant data. +A `WHERE userkey = ?` that must be remembered 28 times, and in every method added later, +is a predicate that will eventually be forgotten, and the symptom is a user reading +someone else's conversation. Separate files make the mistake *structurally unavailable* — +the connection **is** the tenant, and a query cannot reach rows that are not in the file +it executes against. For a phase whose entire subject is isolation, buying that property +with file descriptors is the right trade. + +That is also why this class exposes `for_userkey()` returning a whole `Store` rather than +wrapping all 28 methods with a userkey argument: wrapping would reintroduce exactly the +per-call parameter the file split exists to eliminate. + +**The cost, as arithmetic.** Two SQLite connections per tenant, each in WAL mode — so the +main DB plus `-wal` and `-shm`. The default soft `RLIMIT_NOFILE` of 1024 bounds this +around 150-200 concurrently-open tenants before anything else the process holds, not 500, +because WAL multiplies the count. Hence the LRU. + +Stdlib only. +""" +from __future__ import annotations + +import pathlib +import threading + +from eventbridge.store import Store + +# §6.1 / §8.1 `EB_MAX_OPEN_STORES`. 64 pairs is ~192 file descriptors with WAL, which +# leaves comfortable room under a 1024 soft limit for the Kafka sockets, the HTTP +# listener and the worker pool. +DEFAULT_MAX_OPEN = 64 + +# The directory single-tenant mode and the `shared` tier both use. Named rather than +# derived so the on-disk layout does not change when tenancy is switched on: a bridge +# upgraded from Phase 2 keeps reading the store it already has. +SHARED = "shared" + + +class StoreRegistry: + """`userkey -> Store`, opened on demand, closed on an LRU. + + Closing is safe because a `Store` is stateless above SQLite — every method opens a + transaction and commits, so there is no in-memory state to lose. The ONE thing that + is not: `Store.subscribe()` holds `threading.Event`s for live SSE viewers, so a store + with subscribers is **pinned** and exempt from eviction. Evicting it would make an + open transcript page stop updating with no error anywhere — the Phase 2 §6.1 class of + bug, where the symptom looks like lost events rather than like a closed file. + """ + + def __init__(self, root: str | pathlib.Path, *, + max_open: int = DEFAULT_MAX_OPEN, + multi: bool = False) -> None: + self._root = pathlib.Path(root) + self._max_open = max(1, int(max_open)) + self._multi = multi + # Insertion-ordered, used as the LRU: a hit moves the key to the end, eviction + # takes from the front. `dict` has guaranteed insertion order, so this needs no + # OrderedDict and no linked list. + self._open: dict[str, Store] = {} + self._lock = threading.RLock() + self.evictions = 0 + # §6.1: an event whose `userkey` names nobody goes to `shared/` and bumps this, + # rather than being dropped or guessed into a tenant's store. Surfaced on + # /healthz so a misconfigured runner is visible instead of silent. + self.unattributed = 0 + + # ---- paths -------------------------------------------------------------- + + def _dir_for(self, userkey: str | None) -> pathlib.Path: + """Where one tenant's files live. + + In single-tenant mode this is the bridge root itself, NOT `shared/` — a Phase 2 + deployment already has `responses.sqlite` and `sessions.sqlite` sitting directly + in `{tmpdir}/eventbridge/`, and relocating them on upgrade would make every + existing session and transcript vanish from the UI. Multi-tenant mode is new, so + it is free to use the `users//` layout. + """ + if not self._multi: + return self._root + if not userkey: + return self._root / SHARED + return self._root / "users" / userkey + + # ---- the lookup --------------------------------------------------------- + + def for_userkey(self, userkey: str | None) -> Store: + """The `Store` for one tenant, opening it if necessary. + + In single-tenant mode every call returns the same store and `userkey` is ignored + entirely — the same shape `TopicSet` uses, and what keeps this additive. + """ + key = self._cache_key(userkey) + with self._lock: + store = self._open.get(key) + if store is not None: + # Mark as recently used. + del self._open[key] + self._open[key] = store + return store + store = Store(self._dir_for(userkey)) + self._open[key] = store + # `protect=key`: the store just opened is the one being returned, so it must + # never be the eviction victim. Without this, a cache whose older entries are + # all pinned evicts the newest instead — and then hands the caller a store + # whose connections are closed, which fails later as + # `ProgrammingError: Cannot operate on a closed database` from somewhere that + # looks unrelated to caching. + self._evict_if_needed(protect=key) + return store + + def _cache_key(self, userkey: str | None) -> str: + if not self._multi: + return SHARED + return userkey or SHARED + + def for_event(self, event) -> Store: + """The store an inbound response belongs in, from its `ce_userkey`. + + §6.1: an event with no `userkey` in multi-tenant mode goes to `shared/` and + increments `unattributed` — **never silently into a user's store, and never + dropped**. Both of those would be worse: filing it under a guess corrupts + somebody's history, and dropping it is indistinguishable from an agent that never + answered. + """ + from shared import ce + getter = getattr(event, "get", None) + userkey = getter(ce.EXT_USERKEY) if getter else None + if self._multi and not userkey: + with self._lock: + self.unattributed += 1 + return self.for_userkey(userkey) + + # ---- eviction ----------------------------------------------------------- + + def _evict_if_needed(self, protect: str | None = None) -> None: + """Close least-recently-used stores until the cache fits. Caller holds the lock. + + Two things are never evicted: `protect` (the store the current caller is about to + use) and any store with live SSE subscribers. If everything left is pinned the + cache is allowed to exceed `max_open` rather than breaking a live viewer — going + over a soft descriptor budget degrades and is visible in `/healthz`, while + evicting a subscribed store silently stops a page from updating. The first is + recoverable, the second is not. + """ + if len(self._open) <= self._max_open: + return + for key in list(self._open): + if len(self._open) <= self._max_open: + break + if key == protect: + continue + store = self._open[key] + if _has_subscribers(store): + continue + del self._open[key] + try: + store.close() + except Exception: # noqa: BLE001 - a failed close must not fail a request + pass + self.evictions += 1 + + # ---- lifecycle ---------------------------------------------------------- + + @property + def open_count(self) -> int: + with self._lock: + return len(self._open) + + def known_userkeys(self) -> list[str]: + """Tenants with a directory on disk, whether or not currently open. + + Read from the filesystem rather than from the open cache, because the cache is an + LRU and says nothing about what exists — a sweeper that trusted it would skip + every tenant that happens to be closed. + """ + base = self._root / "users" + if not base.is_dir(): + return [] + return sorted(p.name for p in base.iterdir() if p.is_dir()) + + def close(self) -> None: + with self._lock: + for store in self._open.values(): + try: + store.close() + except Exception: # noqa: BLE001 + pass + self._open.clear() + + +def _has_subscribers(store: Store) -> bool: + """Whether a store has live SSE viewers, and therefore must not be evicted. + + Reaches for a private attribute deliberately: adding a public accessor to `Store` + for this would suggest the subscriber map is part of its API, when the eviction rule + is the registry's concern. Defensive about the attribute's absence so an older or + stubbed Store cannot crash eviction — treating "cannot tell" as "pinned" is the safe + direction, since the cost is a file descriptor rather than a broken page. + """ + subs = getattr(store, "_subscribers", None) + if subs is None: + return True + try: + return any(subs.values()) + except Exception: # noqa: BLE001 + return True diff --git a/eventing/tests/test_auth.py b/eventing/tests/test_auth.py index 611ee1a3..2df15612 100644 --- a/eventing/tests/test_auth.py +++ b/eventing/tests/test_auth.py @@ -29,8 +29,10 @@ def publish_request(self, **kw): self.requests.append(kw) return f"evt-{len(self.requests)}" - def publish_group_event(self, *, type_, groupid, data, subject="group"): - self.group_events.append({"type": type_, "groupid": groupid, "data": data}) + def publish_group_event(self, *, type_, groupid, data, subject="group", + userkey=None): + self.group_events.append({"type": type_, "groupid": groupid, "data": data, + "userkey": userkey}) return f"gevt-{len(self.group_events)}" diff --git a/eventing/tests/test_continue_tenancy.py b/eventing/tests/test_continue_tenancy.py index 1fd4cdc4..31dd3825 100644 --- a/eventing/tests/test_continue_tenancy.py +++ b/eventing/tests/test_continue_tenancy.py @@ -5,32 +5,28 @@ caller identity to use, and publishing to a shared topic would run the turn on another tenant's runner under that tenant's credential. -When the owner cannot be determined, `TopicSet.requests(None)` raises by design. These -tests pin that the handler refuses with a status code first, because an uncaught -`ValueError` is a WSGI 500 with a stack trace, and three of the ways to reach it are -ordinary operation rather than abuse: - -* a user removed from the registry; -* a `prompts` row whose `submitter` is NULL (nullable, and pre-dates auth); -* a correlation back-filled from the topic by `RequestsMirror`, which records no - submitter at all. +T5 replaced the original submitter-derived lookup with the global +`correlationid -> userkey` index (§2.6). These tests cover the index-based behaviour and +keep the refusal guarantees that matter: when the owner cannot be determined, +`TopicSet.requests(None)` raises by design, and an uncaught `ValueError` is a WSGI 500 +with a stack trace rather than a status code. """ from __future__ import annotations +import io import json import pathlib from unittest.mock import MagicMock -import pytest - from eventbridge import registry from eventbridge.config import Cfg from eventbridge.correlation import Minter from eventbridge.handlers import Handlers +from eventbridge.owner_index import OwnerIndex from eventbridge.store import Store +from eventbridge.store_registry import StoreRegistry from shared import ce, tenancy -CORR = "brave-otter-4718" OWNER = "mrsabath" @@ -42,26 +38,25 @@ def _reg(*userids): })) -def _handlers(tmp_path: pathlib.Path, *, mode=tenancy.MULTI, reg=None, submitter=OWNER): +def _api(tmp_path: pathlib.Path, *, multi=True, reg=None): + """A Handlers wired the way `__main__` wires it, so the tests exercise real paths.""" cfg = Cfg() cfg.tmpdir = str(tmp_path) - cfg.tenancy_mode = mode + cfg.tenancy_mode = tenancy.MULTI if multi else tenancy.SINGLE cfg.topic_prefix = "kev1" - store = Store(tmp_path) - store.upsert_session(CORR, ce.session_uuid(CORR), str(tmp_path / "w"), "hello") - # `submitter=None` is the pre-auth / mirror-backfilled shape: the column is - # nullable, so this is a real row, not a contrived one. - store.insert_prompt(CORR, "start", "hello", submitter=submitter) + root = tmp_path / "eventbridge" + owners = OwnerIndex(root) if multi else None + stores = StoreRegistry(root, multi=multi) if multi else None + store = stores.for_userkey(None) if stores else Store(root) producer = MagicMock() producer.publish_request.return_value = "evt-1" - h = Handlers(cfg, store, producer, Minter(), - registry=reg if mode == tenancy.MULTI else None) - return h, producer + minter = Minter(index=owners) + h = Handlers(cfg, store, producer, minter, + registry=reg if multi else None, stores=stores, owners=owners) + return h, producer, owners, stores class _Start: - """Captures the WSGI status line.""" - def __init__(self): self.status = None @@ -71,138 +66,187 @@ def __call__(self, status, headers): def _environ(prompt="go on"): body = json.dumps({"prompt": prompt}).encode() - import io return {"wsgi.input": io.BytesIO(body), "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": "application/json", "REQUEST_METHOD": "POST"} def _form_environ(prompt="go on"): - import io body = f"prompt={prompt}".encode() return {"wsgi.input": io.BytesIO(body), "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": "application/x-www-form-urlencoded", "REQUEST_METHOD": "POST"} +def _seed(h, owners, stores, userkey): + """Create a correlation owned by `userkey`, the way the submit path would.""" + corr = h.minter.mint(userkey=userkey) + store = stores.for_userkey(userkey) if stores else h.store + store.upsert_session(corr, ce.session_uuid(corr), "/w", "hello") + store.insert_prompt(corr, "start", "hello", submitter=OWNER) + return corr + + # ---- the happy path ------------------------------------------------------- def test_continue_publishes_to_the_owning_tenants_topic(tmp_path): - h, producer = _handlers(tmp_path, reg=_reg(OWNER)) + h, producer, owners, stores = _api(tmp_path, reg=_reg(OWNER)) + uk = tenancy.userkey("github", OWNER) + corr = _seed(h, owners, stores, uk) sr = _Start() - h.continue_agent(_environ(), sr, correlationid=CORR) + h.continue_agent(_environ(), sr, correlationid=corr) assert sr.status == "202 Accepted" - assert producer.publish_request.call_args[1]["userkey"] == \ - tenancy.userkey("github", OWNER) + assert producer.publish_request.call_args[1]["userkey"] == uk -def test_single_mode_continue_is_unchanged(tmp_path): - """Phase 2's behaviour: no userkey, no ownership question.""" - h, producer = _handlers(tmp_path, mode=tenancy.SINGLE) +def test_the_owner_comes_from_the_index_not_the_submitter(tmp_path): + """T5's change: ownership is a recorded fact, not re-derived from a name. + + The submitter here is a login that is NOT in the registry, which the old + submitter-derived lookup would have failed on. The index knows the owner anyway. + """ + h, producer, owners, stores = _api(tmp_path, reg=registry.Registry()) + uk = tenancy.userkey("github", OWNER) + corr = h.minter.mint(userkey=uk) + store = stores.for_userkey(uk) + store.upsert_session(corr, ce.session_uuid(corr), "/w", "hello") + store.insert_prompt(corr, "start", "hello", submitter="someone-unregistered") sr = _Start() - h.continue_agent(_environ(), sr, correlationid=CORR) + h.continue_agent(_environ(), sr, correlationid=corr) assert sr.status == "202 Accepted" - assert producer.publish_request.call_args[1]["userkey"] is None + assert producer.publish_request.call_args[1]["userkey"] == uk + + +def test_a_correlation_with_no_submitter_at_all_still_resolves(tmp_path): + """A `prompts` row with a NULL submitter used to be unresolvable. The index does + not consult `submitter`, so a mirror-backfilled correlation works.""" + h, producer, owners, stores = _api(tmp_path, reg=_reg(OWNER)) + uk = tenancy.userkey("github", OWNER) + corr = h.minter.mint(userkey=uk) + store = stores.for_userkey(uk) + store.upsert_session(corr, ce.session_uuid(corr), "/w", "hello") + store.insert_prompt(corr, "start", "hello", submitter=None) + sr = _Start() + h.continue_agent(_environ(), sr, correlationid=corr) + assert sr.status == "202 Accepted" + assert producer.publish_request.call_args[1]["userkey"] == uk -def test_single_mode_continue_works_with_no_submitter_recorded(tmp_path): - """The pre-auth row shape must not become a refusal in the default mode.""" - h, _ = _handlers(tmp_path, mode=tenancy.SINGLE, submitter=None) +def test_single_mode_continue_is_unchanged(tmp_path): + """Phase 2's behaviour: no userkey, no ownership question.""" + h, producer, owners, stores = _api(tmp_path, multi=False) + corr = h.minter.mint() + h.store.upsert_session(corr, ce.session_uuid(corr), "/w", "hello") + h.store.insert_prompt(corr, "start", "hello", submitter=None) sr = _Start() - h.continue_agent(_environ(), sr, correlationid=CORR) + h.continue_agent(_environ(), sr, correlationid=corr) assert sr.status == "202 Accepted" + assert producer.publish_request.call_args[1]["userkey"] is None -# ---- the three refusals --------------------------------------------------- +# ---- refusals ------------------------------------------------------------- -def test_a_submitter_no_longer_in_the_registry_is_a_503_not_a_500(tmp_path): - h, producer = _handlers(tmp_path, reg=_reg("someone-else")) +def test_an_unknown_correlation_is_404_not_500(tmp_path): + """Not in the index at all: the store cannot be chosen, so there is nothing to + read and nothing to publish.""" + h, producer, _, _ = _api(tmp_path, reg=_reg(OWNER)) sr = _Start() - out = h.continue_agent(_environ(), sr, correlationid=CORR) - assert sr.status == "503 Service Unavailable" - assert OWNER in json.loads(b"".join(out))["error"] + h.continue_agent(_environ(), sr, correlationid="brave-otter-0000") + assert sr.status == "404 Not Found" producer.publish_request.assert_not_called() -def test_a_null_submitter_is_a_503_not_a_500(tmp_path): - """`prompts.submitter` is nullable and pre-dates auth, so this is a real row.""" - h, producer = _handlers(tmp_path, reg=_reg(OWNER), submitter=None) +def test_the_html_form_refuses_rather_than_redirecting(tmp_path): + """Redirecting silently would look like the turn was accepted and then vanished.""" + h, producer, _, _ = _api(tmp_path, reg=_reg(OWNER)) sr = _Start() - out = h.continue_agent(_environ(), sr, correlationid=CORR) + out = h.continue_agent_html(_form_environ(), sr, correlationid="brave-otter-0000") assert sr.status == "503 Service Unavailable" - assert "no submitter recorded" in json.loads(b"".join(out))["error"] + assert b"cannot resume" in b"".join(out) producer.publish_request.assert_not_called() -def test_an_empty_registry_is_a_503_not_a_500(tmp_path): - h, producer = _handlers(tmp_path, reg=registry.Registry()) +def test_the_html_form_still_redirects_on_success(tmp_path): + h, producer, owners, stores = _api(tmp_path, reg=_reg(OWNER)) + uk = tenancy.userkey("github", OWNER) + corr = _seed(h, owners, stores, uk) sr = _Start() - h.continue_agent(_environ(), sr, correlationid=CORR) - assert sr.status == "503 Service Unavailable" - producer.publish_request.assert_not_called() + h.continue_agent_html(_form_environ(), sr, correlationid=corr) + assert sr.status == "303 See Other" + assert producer.publish_request.call_args[1]["userkey"] == uk def test_a_refused_continue_writes_nothing(tmp_path): """No orphan prompt row claiming a turn that never ran.""" - h, _ = _handlers(tmp_path, reg=_reg("someone-else")) - before = len(h.store.get_prompts(CORR)) - h.continue_agent(_environ(), _Start(), correlationid=CORR) - assert len(h.store.get_prompts(CORR)) == before + h, _, owners, stores = _api(tmp_path, reg=_reg(OWNER)) + uk = tenancy.userkey("github", OWNER) + corr = _seed(h, owners, stores, uk) + before = len(stores.for_userkey(uk).get_prompts(corr)) + h.continue_agent(_environ(), _Start(), correlationid="brave-otter-0000") + assert len(stores.for_userkey(uk).get_prompts(corr)) == before -# ---- the HTML form path --------------------------------------------------- +# ---- the helper's contract ------------------------------------------------ -def test_the_html_form_refuses_with_503_rather_than_redirecting(tmp_path): - """Redirecting silently would look like the turn was accepted and then vanished, - which is exactly the symptom class this phase keeps trying to avoid.""" - h, producer = _handlers(tmp_path, reg=_reg("someone-else")) - sr = _Start() - out = h.continue_agent_html(_form_environ(), sr, correlationid=CORR) - assert sr.status == "503 Service Unavailable" - assert b"cannot resume" in b"".join(out) - producer.publish_request.assert_not_called() +def test_owner_userkey_returns_exactly_one_of_key_or_reason(tmp_path): + """The two-value return distinguishes "no key needed" from "owner unknown".""" + h1, _, _, _ = _api(tmp_path / "a", multi=False) + assert h1._owner_userkey("brave-otter-4718") == (None, None) + h2, _, owners2, stores2 = _api(tmp_path / "b", reg=_reg(OWNER)) + uk = tenancy.userkey("github", OWNER) + corr = _seed(h2, owners2, stores2, uk) + key, reason = h2._owner_userkey(corr) + assert key == uk and reason is None -def test_the_html_form_still_redirects_on_success(tmp_path): - h, producer = _handlers(tmp_path, reg=_reg(OWNER)) - sr = _Start() - h.continue_agent_html(_form_environ(), sr, correlationid=CORR) - assert sr.status == "303 See Other" - assert producer.publish_request.call_args[1]["userkey"] == \ - tenancy.userkey("github", OWNER) + key, reason = h2._owner_userkey("brave-otter-0000") + assert key is None and reason -# ---- the helper's contract ------------------------------------------------ +def test_a_shared_tier_correlation_resolves_to_a_null_owner(tmp_path): + """A known correlation with no tenant is a real answer, not a failure: those + events live in `shared/` by design.""" + h, _, owners, stores = _api(tmp_path, reg=_reg(OWNER)) + corr = h.minter.mint(userkey=None) + key, reason = h._owner_userkey(corr) + assert key is None and reason is None -def test_owner_userkey_returns_exactly_one_of_key_or_reason(tmp_path): - """The two-value return exists because a bare `None` cannot distinguish "single - mode, no key needed" from "multi mode, owner unknown".""" - h, _ = _handlers(tmp_path, mode=tenancy.SINGLE) - assert h._owner_userkey(CORR) == (None, None) - h2, _ = _handlers(tmp_path / "b", reg=_reg(OWNER)) - key, reason = h2._owner_userkey(CORR) - assert key and reason is None +def test_store_of_returns_none_for_an_unplaceable_correlation(tmp_path): + h, _, _, _ = _api(tmp_path, reg=_reg(OWNER)) + assert h._store_of("brave-otter-0000") is None - h3, _ = _handlers(tmp_path / "c", reg=_reg("nobody")) - key, reason = h3._owner_userkey(CORR) - assert key is None and reason +def test_store_of_is_the_single_store_in_single_mode(tmp_path): + h, _, _, _ = _api(tmp_path, multi=False) + assert h._store_of("brave-otter-0000") is h.store -def test_an_unknown_correlation_in_multi_mode_still_404s(tmp_path): - """The ownership check must not shadow the existing 404.""" - h, _ = _handlers(tmp_path, reg=_reg(OWNER)) - sr = _Start() - h.continue_agent(_environ(), sr, correlationid="brave-otter-0000") - assert sr.status == "404 Not Found" +# ---- cross-tenant isolation ---------------------------------------------- -@pytest.mark.parametrize("spelling", ["MrSabath", "MRSABATH"]) -def test_owner_resolution_folds_github_case(tmp_path, spelling): - """A correlation submitted as `MrSabath` must resolve to the same tenant as - `mrsabath`, or a resume lands in a different tenant than the original turn.""" - h, producer = _handlers(tmp_path, reg=_reg(OWNER), submitter=spelling) - sr = _Start() - h.continue_agent(_environ(), sr, correlationid=CORR) - assert sr.status == "202 Accepted" - assert producer.publish_request.call_args[1]["userkey"] == \ - tenancy.userkey("github", OWNER) +def test_two_tenants_get_different_stores(tmp_path): + """The property §6.1 buys with file descriptors: the connection IS the tenant.""" + h, _, owners, stores = _api(tmp_path, reg=_reg("alice", "bob")) + a = tenancy.userkey("github", "alice") + b = tenancy.userkey("github", "bob") + corr_a = _seed(h, owners, stores, a) + corr_b = _seed(h, owners, stores, b) + + store_a = stores.for_userkey(a) + store_b = stores.for_userkey(b) + assert store_a is not store_b + # Each tenant's store knows only its own correlation — not because of a WHERE + # clause, but because the other row is in a different file. + assert store_a.get_session(corr_a) is not None + assert store_a.get_session(corr_b) is None + assert store_b.get_session(corr_b) is not None + assert store_b.get_session(corr_a) is None + + +def test_reads_route_to_the_owning_tenants_store(tmp_path): + """A GET for alice's correlation must read alice's store, whoever asks.""" + h, _, owners, stores = _api(tmp_path, reg=_reg("alice", "bob")) + a = tenancy.userkey("github", "alice") + corr_a = _seed(h, owners, stores, a) + assert h._store_of(corr_a) is stores.for_userkey(a) + assert h._store_of(corr_a) is not stores.for_userkey( + tenancy.userkey("github", "bob")) diff --git a/eventing/tests/test_groups.py b/eventing/tests/test_groups.py index d519ffd0..9476497c 100644 --- a/eventing/tests/test_groups.py +++ b/eventing/tests/test_groups.py @@ -32,8 +32,10 @@ def publish_request(self, **kw): self.requests.append(kw) return f"evt-{len(self.requests)}" - def publish_group_event(self, *, type_, groupid, data, subject="group"): - self.group_events.append({"type": type_, "groupid": groupid, "data": data}) + def publish_group_event(self, *, type_, groupid, data, subject="group", + userkey=None): + self.group_events.append({"type": type_, "groupid": groupid, "data": data, + "userkey": userkey}) return f"gevt-{len(self.group_events)}" diff --git a/eventing/tests/test_owner_index.py b/eventing/tests/test_owner_index.py new file mode 100644 index 00000000..97c6a7e7 --- /dev/null +++ b/eventing/tests/test_owner_index.py @@ -0,0 +1,250 @@ +"""The global `correlationid -> userkey` index. DESIGN_PHASE3.md §2.6, §6.2. + +The uniqueness constraint is the load-bearing part, and the test that matters most is +`test_the_constraint_is_on_correlationid_alone`: `(userkey, correlationid)` is the right +primary key *inside* a tenant's store and exactly the wrong one here, because two tenants +minting one id would produce two distinct tuples, never conflict, and the collision would +pass through silently — which is the failure the whole guarantee exists to prevent. +""" +from __future__ import annotations + +import threading + +import pytest + +from eventbridge.correlation import Minter, mint_for +from eventbridge.owner_index import Collision, OwnerIndex + +A = "gh-alice-a1b2c3d4" +B = "gh-bob-9e8f7a6b" +CORR = "brave-otter-4718" + + +@pytest.fixture +def idx(tmp_path): + return OwnerIndex(tmp_path) + + +# ---- claim / exists ------------------------------------------------------- + +def test_claim_then_exists(idx): + assert not idx.exists(CORR) + idx.claim(CORR, A) + assert idx.exists(CORR) + + +def test_claim_is_idempotent_for_the_same_owner(idx): + """A retried submit must not fail.""" + idx.claim(CORR, A) + idx.claim(CORR, A) + assert idx.count() == 1 + + +def test_the_constraint_is_on_correlationid_alone(idx): + """Two tenants cannot share one correlation id. + + With `(userkey, correlationid)` as the key this would silently succeed and the two + tenants would each believe they owned it — and `owner_of` would return whichever + row the query happened to find. + """ + idx.claim(CORR, A) + with pytest.raises(Collision) as e: + idx.claim(CORR, B) + assert A in str(e.value) and B in str(e.value) + assert idx.count() == 1 + + +def test_a_null_owner_and_a_real_owner_are_different_owners(idx): + """A correlation minted in single-tenant mode is not free for a tenant to claim: + that is a genuine ambiguity, not a no-op.""" + idx.claim(CORR, None) + with pytest.raises(Collision): + idx.claim(CORR, A) + + +def test_a_null_owner_claim_is_idempotent(idx): + idx.claim(CORR, None) + idx.claim(CORR, None) + assert idx.count() == 1 + + +# ---- owner_of ------------------------------------------------------------ + +def test_owner_of_distinguishes_unknown_from_unowned(idx): + """The two-value return exists for exactly this: a bare `None` would conflate + "the shared tier owns it" with "never seen".""" + assert idx.owner_of(CORR) == (None, False) + idx.claim(CORR, None) + assert idx.owner_of(CORR) == (None, True) + idx.claim("brave-otter-0002", A) + assert idx.owner_of("brave-otter-0002") == (A, True) + + +# ---- listing ------------------------------------------------------------- + +def test_correlations_for_a_tenant(idx): + idx.claim("brave-otter-0001", A) + idx.claim("brave-otter-0002", A) + idx.claim("brave-otter-0003", B) + assert set(idx.correlations_for(A)) == {"brave-otter-0001", "brave-otter-0002"} + assert idx.correlations_for(B) == ["brave-otter-0003"] + + +def test_correlations_for_the_null_owner_does_not_leak_tenants(idx): + """`WHERE userkey IS NULL` rather than `= NULL`, which matches nothing in SQL.""" + idx.claim("brave-otter-0001", None) + idx.claim("brave-otter-0002", A) + assert idx.correlations_for(None) == ["brave-otter-0001"] + + +def test_userkeys_excludes_the_null_owner(idx): + idx.claim("brave-otter-0001", None) + idx.claim("brave-otter-0002", A) + idx.claim("brave-otter-0003", B) + assert set(idx.userkeys()) == {A, B} + + +# ---- deletion (§6.5 groundwork) ------------------------------------------ + +def test_forget_one(idx): + idx.claim(CORR, A) + idx.forget(CORR) + assert not idx.exists(CORR) + + +def test_forget_tenant_returns_a_count_and_leaves_others(idx): + idx.claim("brave-otter-0001", A) + idx.claim("brave-otter-0002", A) + idx.claim("brave-otter-0003", B) + assert idx.forget_tenant(A) == 2 + assert idx.correlations_for(B) == ["brave-otter-0003"] + + +# ---- persistence --------------------------------------------------------- + +def test_the_index_survives_a_restart(tmp_path): + """It names topics and stores that outlive the process; an in-memory set would + lose every ownership fact on restart.""" + one = OwnerIndex(tmp_path) + one.claim(CORR, A) + one.close() + two = OwnerIndex(tmp_path) + assert two.owner_of(CORR) == (A, True) + + +# ---- concurrency --------------------------------------------------------- + +def test_concurrent_claims_of_one_id_produce_exactly_one_winner(idx): + """EventBridge is single-replica but multi-threaded: the HTTP worker pool and the + responses consumer both reach this, so check-then-insert has to be atomic.""" + results: list[object] = [] + lock = threading.Lock() + + def claim(uk): + try: + idx.claim(CORR, uk) + with lock: + results.append(("ok", uk)) + except Collision: + with lock: + results.append(("collision", uk)) + + threads = [threading.Thread(target=claim, args=(f"gh-u{i}-0000000{i}",)) + for i in range(12)] + for t in threads: + t.start() + for t in threads: + t.join() + assert sum(1 for r, _ in results if r == "ok") == 1 + assert idx.count() == 1 + + +# ---- the Minter integration ---------------------------------------------- + +def test_the_minter_claims_what_it_mints(idx): + m = Minter(index=idx) + corr = m.mint(userkey=A) + assert idx.owner_of(corr) == (A, True) + + +def test_the_minter_never_returns_a_claimed_id(tmp_path): + """The uniqueness guarantee §2.6 relies on to keep `sessionuuid` unsalted. + + A 1-word corpus would be ideal but the real word lists are large, so instead mint + many and assert no repeats — with the index as the only thing preventing them. + """ + idx = OwnerIndex(tmp_path) + m = Minter(seed=1, index=idx) + minted = [m.mint(userkey=A) for _ in range(200)] + assert len(set(minted)) == 200 + assert idx.count() == 200 + + +def test_two_minters_sharing_an_index_do_not_collide(tmp_path): + """Two Minters with the SAME rng seed would produce identical sequences; the index + is what keeps them apart. This is the cross-tenant case in miniature.""" + idx = OwnerIndex(tmp_path) + a = Minter(seed=7, index=idx) + b = Minter(seed=7, index=idx) + first = [a.mint(userkey=A) for _ in range(20)] + second = [b.mint(userkey=B) for _ in range(20)] + assert not (set(first) & set(second)) + + +def test_the_minter_without_an_index_keeps_phase2_behaviour(tmp_path): + """No index configured: the in-memory `seen` set, exactly as before.""" + m = Minter(seed=3) + corr = m.mint() + assert corr + m.remember("brave-otter-0001") + # `remember` is a no-op with an index, and meaningful without one. Minting enough + # ids to hit the remembered one is impractical, so assert the set directly. + assert "brave-otter-0001" in m._seen + + +def test_remember_is_a_noop_once_an_index_is_configured(idx): + m = Minter(index=idx) + m.remember("brave-otter-0001") + assert m._seen == set() + + +# ---- mint_for's compatibility shim --------------------------------------- + +def test_mint_for_passes_the_userkey_when_supported(idx): + m = Minter(index=idx) + corr = mint_for(m, A) + assert idx.owner_of(corr) == (A, True) + + +def test_mint_for_falls_back_for_a_substitute_without_userkey(): + """Several tests supply deterministic fakes that predate tenancy; they must keep + working rather than failing at the call site.""" + class OldMinter: + def mint(self): + return "brave-otter-0001" + + assert mint_for(OldMinter(), A) == "brave-otter-0001" + + +def test_mint_for_does_not_swallow_a_typeerror_from_inside_mint(): + """A working `mint()` that itself raises TypeError must NOT be silently retried + without its ownership claim — that would turn a real bug into a missing index + entry, which is far harder to notice.""" + class Exploding: + def mint(self, *, userkey=None): + raise TypeError("something inside went wrong") + + with pytest.raises(TypeError, match="something inside"): + mint_for(Exploding(), A) + + +def test_mint_for_passes_the_userkey_to_a_kwargs_substitute(): + seen = {} + + class KwargsMinter: + def mint(self, **kw): + seen.update(kw) + return "brave-otter-0001" + + mint_for(KwargsMinter(), A) + assert seen == {"userkey": A} diff --git a/eventing/tests/test_store_registry.py b/eventing/tests/test_store_registry.py new file mode 100644 index 00000000..f5ee618c --- /dev/null +++ b/eventing/tests/test_store_registry.py @@ -0,0 +1,257 @@ +"""Per-user stores and the LRU that bounds them. DESIGN_PHASE3.md §6.1. + +Two properties carry the weight: + +* **Isolation is structural.** A query cannot reach rows that are not in the file it + executes against, so there is no `WHERE userkey = ?` to forget across `store.py`'s 28 + methods. `test_a_query_cannot_reach_another_tenants_rows` is that claim. +* **A subscribed store is pinned.** Evicting one would make an open SSE page stop + updating with no error anywhere — the Phase 2 §6.1 class of bug, where the symptom + looks like lost events rather than a closed file. +""" +from __future__ import annotations + +from eventbridge.store_registry import SHARED, StoreRegistry +from shared import ce + +A = "gh-alice-a1b2c3d4" +B = "gh-bob-9e8f7a6b" + + +def _multi(tmp_path, **kw): + return StoreRegistry(tmp_path, multi=True, **kw) + + +# ---- single-tenant mode: unchanged --------------------------------------- + +def test_single_mode_returns_one_store_for_everyone(tmp_path): + """Phase 2's behaviour: one store, and a userkey changes nothing.""" + r = StoreRegistry(tmp_path, multi=False) + assert r.for_userkey(None) is r.for_userkey(A) is r.for_userkey(B) + + +def test_single_mode_keeps_the_files_where_phase2_left_them(tmp_path): + """Relocating them on upgrade would make every existing session and transcript + vanish from the UI.""" + StoreRegistry(tmp_path, multi=False).for_userkey(None) + assert (tmp_path / "responses.sqlite").is_file() + assert (tmp_path / "sessions.sqlite").is_file() + assert not (tmp_path / "users").exists() + + +def test_single_mode_never_counts_anything_unattributed(tmp_path): + """There is no tenant to attribute to, so the counter must stay quiet.""" + r = StoreRegistry(tmp_path, multi=False) + r.for_event(ce.CloudEvent(attrs={"correlationid": "brave-otter-4718"})) + assert r.unattributed == 0 + + +# ---- multi-tenant layout ------------------------------------------------- + +def test_each_tenant_gets_its_own_directory(tmp_path): + r = _multi(tmp_path) + r.for_userkey(A) + r.for_userkey(B) + assert (tmp_path / "users" / A / "sessions.sqlite").is_file() + assert (tmp_path / "users" / B / "sessions.sqlite").is_file() + + +def test_the_shared_tier_has_its_own_directory(tmp_path): + r = _multi(tmp_path) + r.for_userkey(None) + assert (tmp_path / SHARED / "sessions.sqlite").is_file() + + +def test_the_same_tenant_gets_the_same_object(tmp_path): + r = _multi(tmp_path) + assert r.for_userkey(A) is r.for_userkey(A) + + +def test_different_tenants_get_different_objects(tmp_path): + r = _multi(tmp_path) + assert r.for_userkey(A) is not r.for_userkey(B) + + +def test_a_query_cannot_reach_another_tenants_rows(tmp_path): + """§6.1's whole argument, asserted directly: the connection IS the tenant.""" + r = _multi(tmp_path) + sa, sb = r.for_userkey(A), r.for_userkey(B) + sa.upsert_session("brave-otter-0001", "u1", "/w", "alice's prompt") + sb.upsert_session("brave-otter-0002", "u2", "/w", "bob's prompt") + assert sa.get_session("brave-otter-0001") is not None + assert sa.get_session("brave-otter-0002") is None + assert sb.get_session("brave-otter-0001") is None + + +def test_known_userkeys_reads_the_filesystem_not_the_cache(tmp_path): + """The cache is an LRU and says nothing about what exists — a sweeper that trusted + it would skip every tenant that happens to be closed.""" + r = _multi(tmp_path, max_open=1) + r.for_userkey(A) + r.for_userkey(B) # evicts A from the cache + assert r.open_count == 1 + assert set(r.known_userkeys()) == {A, B} + + +def test_known_userkeys_is_empty_before_any_tenant_exists(tmp_path): + assert _multi(tmp_path).known_userkeys() == [] + + +# ---- the LRU ------------------------------------------------------------- + +def test_the_lru_closes_the_least_recently_used(tmp_path): + r = _multi(tmp_path, max_open=2) + r.for_userkey("gh-a-00000001") + r.for_userkey("gh-b-00000002") + r.for_userkey("gh-c-00000003") + assert r.open_count == 2 + assert r.evictions == 1 + + +def test_a_hit_refreshes_recency(tmp_path): + r = _multi(tmp_path, max_open=2) + first = r.for_userkey("gh-a-00000001") + r.for_userkey("gh-b-00000002") + r.for_userkey("gh-a-00000001") # refresh A, so B is now oldest + r.for_userkey("gh-c-00000003") # evicts B + assert r.for_userkey("gh-a-00000001") is first + + +def test_an_evicted_store_reopens_with_its_data_intact(tmp_path): + """Closing is safe because a Store is stateless above SQLite.""" + r = _multi(tmp_path, max_open=1) + r.for_userkey(A).upsert_session("brave-otter-0001", "u1", "/w", "hello") + r.for_userkey(B) # evicts A + assert r.for_userkey(A).get_session("brave-otter-0001") is not None + + +def test_a_subscribed_store_is_never_evicted(tmp_path): + """Evicting it would make an open transcript page stop updating with no error.""" + r = _multi(tmp_path, max_open=1) + pinned = r.for_userkey(A) + pinned.subscribe("brave-otter-0001") + r.for_userkey(B) + r.for_userkey("gh-c-00000003") + # The pinned store is still the same object, and still open. + assert r.for_userkey(A) is pinned + assert r.evictions >= 1 # something else was evicted instead + + +def test_the_cache_may_exceed_max_open_when_everything_is_pinned(tmp_path): + """Going over a soft descriptor budget degrades; breaking a live viewer does not + recover. The first is the right direction to err.""" + r = _multi(tmp_path, max_open=1) + for uk in ("gh-a-00000001", "gh-b-00000002", "gh-c-00000003"): + r.for_userkey(uk).subscribe("brave-otter-0001") + assert r.open_count == 3 + assert r.evictions == 0 + + +def test_unsubscribing_makes_a_store_evictable_again(tmp_path): + r = _multi(tmp_path, max_open=1) + store = r.for_userkey(A) + ev = store.subscribe("brave-otter-0001") + r.for_userkey(B) + assert r.open_count == 2 + store.unsubscribe("brave-otter-0001", ev) + r.for_userkey("gh-c-00000003") + assert r.open_count <= 2 + + +def test_the_store_being_returned_is_never_the_eviction_victim(tmp_path): + """Regression: with every older entry pinned, eviction used to take the store it had + just opened and then hand the caller a CLOSED one — surfacing much later as + `ProgrammingError: Cannot operate on a closed database` from somewhere that looks + unrelated to caching.""" + r = _multi(tmp_path, max_open=1) + r.for_userkey(A).subscribe("brave-otter-0001") # pin A + fresh = r.for_userkey(B) + # The returned store must be usable, which is the property the bug broke. + fresh.upsert_session("brave-otter-0002", "u2", "/w", "hello") + assert fresh.get_session("brave-otter-0002") is not None + assert r.for_userkey(B) is fresh + + +def test_a_usable_store_is_returned_even_under_relentless_eviction(tmp_path): + """Every call opens a new tenant with max_open=1, so each one evicts the previous. + The store handed back must still work.""" + r = _multi(tmp_path, max_open=1) + for i in range(6): + uk = f"gh-u{i}-0000000{i}" + s = r.for_userkey(uk) + s.upsert_session(f"brave-otter-000{i}", f"u{i}", "/w", "hi") + assert s.get_session(f"brave-otter-000{i}") is not None + + +def test_max_open_is_floored_at_one(tmp_path): + """A zero or negative ceiling would evict every store immediately after opening + it, which is an infinite reopen loop rather than a small cache.""" + for bad in (0, -5): + r = StoreRegistry(tmp_path / f"m{bad}", multi=True, max_open=bad) + r.for_userkey(A) + assert r.open_count == 1 + + +# ---- for_event ----------------------------------------------------------- + +def test_for_event_routes_by_the_events_userkey(tmp_path): + r = _multi(tmp_path) + evt = ce.CloudEvent(attrs={"correlationid": "brave-otter-0001", + ce.EXT_USERKEY: A}) + assert r.for_event(evt) is r.for_userkey(A) + + +def test_an_event_without_a_userkey_goes_to_shared_and_is_counted(tmp_path): + """§6.1: never guessed into a tenant's store (that corrupts a history) and never + dropped (that looks like an agent which never answered).""" + r = _multi(tmp_path) + evt = ce.CloudEvent(attrs={"correlationid": "brave-otter-0001"}) + assert r.for_event(evt) is r.for_userkey(None) + assert r.unattributed == 1 + + +def test_the_unattributed_counter_accumulates(tmp_path): + r = _multi(tmp_path) + for _ in range(3): + r.for_event(ce.CloudEvent(attrs={"correlationid": "brave-otter-0001"})) + assert r.unattributed == 3 + + +def test_an_attributed_event_does_not_bump_the_counter(tmp_path): + r = _multi(tmp_path) + r.for_event(ce.CloudEvent(attrs={"correlationid": "x", ce.EXT_USERKEY: A})) + assert r.unattributed == 0 + + +def test_for_event_tolerates_a_plain_dict(tmp_path): + """The consumer hands around both CloudEvent objects and envelope dicts.""" + r = _multi(tmp_path) + assert r.for_event({ce.EXT_USERKEY: A}) is r.for_userkey(A) + + +# ---- lifecycle ---------------------------------------------------------- + +def test_close_closes_everything(tmp_path): + r = _multi(tmp_path) + r.for_userkey(A) + r.for_userkey(B) + r.close() + assert r.open_count == 0 + + +def test_store_close_wakes_subscribers(tmp_path): + """A viewer blocked on Event.wait() would otherwise hang until its own timeout + instead of noticing the process is going away.""" + r = _multi(tmp_path) + store = r.for_userkey(A) + ev = store.subscribe("brave-otter-0001") + assert not ev.is_set() + store.close() + assert ev.is_set() + + +def test_reopening_after_close_works(tmp_path): + r = _multi(tmp_path) + r.for_userkey(A).upsert_session("brave-otter-0001", "u1", "/w", "hi") + r.close() + assert r.for_userkey(A).get_session("brave-otter-0001") is not None From cee9eab822dcfa949cb26ae53dd87ced4b97457c Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 19:03:48 -0400 Subject: [PATCH 06/14] fix(eventing): evict stores by dropping the reference, not by closing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found reviewing T5. `StoreRegistry` eviction called `Store.close()`, which created a use-after-close race on every multi-call read path: a handler resolves a store, and while it is still making calls a *concurrent request for a different tenant* pushes the cache over `EB_MAX_OPEN_STORES` and closes it underneath. The next call fails with sqlite3.ProgrammingError: Cannot operate on a closed database from a line that looks nothing to do with caching. `get_html` alone makes five calls on one store, so the window is wide rather than theoretical, and reproducing it needs only `max_open=1` and two tenants. Eviction now drops the registry's reference and lets CPython close the connections when the last holder goes away — exactly the lifetime that is safe. File descriptors are reclaimed slightly later than an explicit close; that is the right trade, because the ceiling is a soft budget while a closed connection under an in-flight request is a 500. `_sse_generator` additionally now subscribes BEFORE the replay read. Pinning is what keeps a subscribed store in the cache so later readers get the same object and see the subscriber's notifications, and the old order left the store unpinned between `_store_of()` and `subscribe()`. Reordering is harmless: the Event only ever means "there may be something new", every read is `since_seq=sent`, and an already-set Event costs one extra query. Worth noting what the first fix attempt got wrong, since it is the more interesting mistake: pinning only the SSE path would have closed the case that was easiest to see while leaving every ordinary read exposed. The bug was in the eviction contract, not in one caller. Two regression tests, one asserting an evicted-but-held store stays usable for both reads and subscribe, one pinning the subscribe-before-replay order. 787 passing. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventbridge/handlers.py | 33 ++++++++++++++------- eventing/eventbridge/store_registry.py | 41 +++++++++++++++++--------- eventing/tests/test_store_registry.py | 32 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 25 deletions(-) diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index 9a37c344..b341987e 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -596,20 +596,31 @@ def _sse_generator(self, correlationid: str, since_seq: int = 0): store = self._store_of(correlationid) if store is None: return - events = store.events_for(correlationid, since_seq=since_seq) - seen_final = False - for e in events: - yield f"data: {json.dumps(e)}\n\n".encode() - sent = max(sent, e["sequence"]) - if e.get("final"): - seen_final = True - if seen_final: - # Nothing more to stream — connection closes cleanly instead of - # holding open and waiting for events that will never arrive. - return + # SUBSCRIBE FIRST, before the replay. The registry pins a store only while it has + # subscribers, so between `_store_of` and `subscribe` this store is evictable — + # and another request for a different tenant can close it underneath us, making + # the replay below fail with `sqlite3.ProgrammingError: Cannot operate on a closed + # database`. Subscribing first closes that window: from here until the `finally` + # the store cannot be evicted. + # + # Subscribing before the replay is also harmless for correctness: `ev` only ever + # means "there may be something new", every read is `since_seq=sent`, and a set + # Event simply costs one extra query. ev = store.subscribe(correlationid) try: + events = store.events_for(correlationid, since_seq=since_seq) + seen_final = False + for e in events: + yield f"data: {json.dumps(e)}\n\n".encode() + sent = max(sent, e["sequence"]) + if e.get("final"): + seen_final = True + if seen_final: + # Nothing more to stream — connection closes cleanly instead of + # holding open and waiting for events that will never arrive. + return + deadline = time.monotonic() + 120.0 while time.monotonic() < deadline: if ev.wait(timeout=15.0): diff --git a/eventing/eventbridge/store_registry.py b/eventing/eventbridge/store_registry.py index 3b20caec..6a0b6e3c 100644 --- a/eventing/eventbridge/store_registry.py +++ b/eventing/eventbridge/store_registry.py @@ -147,14 +147,31 @@ def for_event(self, event) -> Store: # ---- eviction ----------------------------------------------------------- def _evict_if_needed(self, protect: str | None = None) -> None: - """Close least-recently-used stores until the cache fits. Caller holds the lock. - - Two things are never evicted: `protect` (the store the current caller is about to - use) and any store with live SSE subscribers. If everything left is pinned the - cache is allowed to exceed `max_open` rather than breaking a live viewer — going - over a soft descriptor budget degrades and is visible in `/healthz`, while - evicting a subscribed store silently stops a page from updating. The first is - recoverable, the second is not. + """Evict least-recently-used stores until the cache fits. Caller holds the lock. + + **Eviction drops the registry's reference; it does NOT close the store.** That + distinction is the whole correctness argument, and closing here was a real + use-after-close bug: a handler resolves a store, and while it is still making + calls on it a *concurrent request for a different tenant* pushes the cache over + its ceiling and closes it underneath. The next call fails with + `sqlite3.ProgrammingError: Cannot operate on a closed database`, from a line that + looks nothing to do with caching. Every multi-call read path was exposed — + `get_html` alone makes five calls on one store — so pinning the SSE path only + would have fixed the case that was easiest to see, not the class. + + Dropping the reference instead means CPython closes the connections when the last + holder goes away, which is exactly the lifetime that is safe. The file descriptors + are reclaimed slightly later than an explicit close, and that is the right trade: + the ceiling is a soft budget, while a closed connection under an in-flight request + is a 500. + + Two things are never evicted even in this weaker sense: `protect` (the store the + current caller is about to use) and any store with live SSE subscribers. A + subscribed store is kept in the cache so a later reader gets the SAME object and + therefore sees the subscriber's notifications; evicting it would leave the viewer + watching an object nothing writes to any more — updates stop with no error, which + is the Phase 2 §6.1 symptom class. If everything left is pinned the cache exceeds + `max_open` rather than breaking a viewer. """ if len(self._open) <= self._max_open: return @@ -163,14 +180,10 @@ def _evict_if_needed(self, protect: str | None = None) -> None: break if key == protect: continue - store = self._open[key] - if _has_subscribers(store): + if _has_subscribers(self._open[key]): continue + # Reference dropped, not closed. See the docstring. del self._open[key] - try: - store.close() - except Exception: # noqa: BLE001 - a failed close must not fail a request - pass self.evictions += 1 # ---- lifecycle ---------------------------------------------------------- diff --git a/eventing/tests/test_store_registry.py b/eventing/tests/test_store_registry.py index f5ee618c..db225e4f 100644 --- a/eventing/tests/test_store_registry.py +++ b/eventing/tests/test_store_registry.py @@ -183,6 +183,38 @@ def test_a_usable_store_is_returned_even_under_relentless_eviction(tmp_path): assert s.get_session(f"brave-otter-000{i}") is not None +def test_an_evicted_store_stays_usable_while_someone_holds_it(tmp_path): + """Regression, and the reason eviction drops a reference rather than closing. + + A handler resolves a store and keeps making calls on it; meanwhile a concurrent + request for a different tenant pushes the cache over its ceiling. If eviction closed + the store, the holder's next call would fail with `ProgrammingError: Cannot operate + on a closed database` from a line that looks unrelated to caching — and EVERY + multi-call read path was exposed (`get_html` alone makes five calls). + """ + r = _multi(tmp_path, max_open=1) + held = r.for_userkey(A) + held.upsert_session("brave-otter-0001", "u1", "/w", "hello") + r.for_userkey(B) # evicts A from the cache + # The holder must still be able to use it, both for reads and for subscribing. + assert held.get_session("brave-otter-0001") is not None + ev = held.subscribe("brave-otter-0001") + assert held.events_for("brave-otter-0001") == [] + held.unsubscribe("brave-otter-0001", ev) + + +def test_the_sse_path_pins_its_store_before_reading(tmp_path): + """`_sse_generator` subscribes before the replay, so the store cannot be evicted + between resolving it and pinning it.""" + import inspect + + from eventbridge.handlers import Handlers + src = inspect.getsource(Handlers._sse_generator) + sub = src.index(".subscribe(") + replay = src.index("events_for(correlationid, since_seq=since_seq)") + assert sub < replay, "subscribe must come before the replay read" + + def test_max_open_is_floored_at_one(tmp_path): """A zero or negative ceiling would evict every store immediately after opening it, which is an infinite reopen loop rather than a small cache.""" From cb9fb83a05856852f82b59431e5e8c9efc395c90 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 19:13:38 -0400 Subject: [PATCH 07/14] fix(eventing): teach GroupMirror about tenancy, and stop hiding index failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs from an adversarial review of T5. All three were confirmed by driving the real code, and all three are multi-tenant-only — the default `single` path is unaffected. **1. Restart-orphaned groups were never settled (`_settle`).** It called `maybe_complete(gid)` with no `userkey`, so it looked in the SHARED store for a group that lives in a tenant's store, found nothing, and returned False. Every group a restart left unfinished stayed unfinished forever, and the failure mode is silence — which §21.6 calls the worst one. The mirror now resolves the owner per group from the ownership index. **2. Replayed member events landed in the shared store.** `on_member_event` reads `ce_userkey` off the event, and any response published *before* Phase 3 has none — so member rows went to `shared/` while the group row sat in the tenant's store, leaving the group's counts permanently wrong and `_decorate_member` reading from a third place. Pre-existing Kafka history is exactly what this mirror exists to replay, so that is the normal upgrade path, not an edge case. The mirror now back-fills a missing `userkey` from the index. It only ever FILLS: an event carrying its own `userkey` keeps it, because that attribute is signed (§2.6) and the index is not authoritative over a signed assertion. Both of these had one root cause — the mirror knew about per-user topics but not about which tenant a replayed group belonged to. **3. A broken index surfaced as "correlation ID space exhausted".** `mint()` caught `Exception` around `claim` to treat a lost race as a retry, so a genuinely broken index (disk full, database locked, schema missing) was retried 2000 times and then reported a message that sends the reader to the word lists rather than the database. Now catches `Collision` specifically; everything else belongs to the caller. Also closes a test-strength gap the review found: deleting the `_has_subscribers` pinning check entirely still passed every `test_store_registry.py` test. The existing tests asserted object identity and counters, not the notification path that is the actual reason for the rule — now that eviction only drops a reference, a held store keeps working, so what actually breaks is that the *writer* gets a new Store whose subscriber map is empty and the viewer is never notified. The new test asserts that end to end; removing the pinning check now fails 4 tests instead of 0. Tests: 8 new (795 total), ruff clean. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventbridge/__main__.py | 4 +- eventing/eventbridge/correlation.py | 12 +- eventing/eventbridge/kafka_group_mirror.py | 38 +++++- eventing/tests/test_mirror_tenancy.py | 149 +++++++++++++++++++++ eventing/tests/test_owner_index.py | 36 +++++ eventing/tests/test_store_registry.py | 27 ++++ 6 files changed, 261 insertions(+), 5 deletions(-) create mode 100644 eventing/tests/test_mirror_tenancy.py diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index 7bd0abaf..6a78308c 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -166,7 +166,9 @@ def main() -> int: # emptyDir /data) served 404 for every earlier group even though its notifications # had already gone out. One-shot, never publishes, never notifies. group_mirror = GroupMirror(cfg.kafka_bootstrap, cfg.response_topic, groups, - topics=topics) + topics=topics, + userkeys=users.userkeys if topics.multi else (), + owners=owners if topics.multi else None) group_mirror.start() h = Handlers(cfg, store, producer, minter, groups=groups, diff --git a/eventing/eventbridge/correlation.py b/eventing/eventbridge/correlation.py index c1d95721..ff1ecc8d 100644 --- a/eventing/eventbridge/correlation.py +++ b/eventing/eventbridge/correlation.py @@ -6,6 +6,8 @@ import re import threading +from eventbridge.owner_index import Collision + REGEX = re.compile(r"^[a-z]{3,10}-[a-z]{3,12}-\d{4}$") @@ -77,7 +79,15 @@ def mint(self, *, userkey: str | None = None) -> str: continue try: self._index.claim(candidate, userkey) - except Exception: # noqa: BLE001 - a lost race is a retry, not a failure + except Collision: + # Another worker won this id between `exists` and `claim`. That is a + # retry, not a failure. + # + # Deliberately NOT `except Exception`: a genuinely broken index (disk + # full, database locked, schema missing) would then be retried 2000 times + # and surface as `RuntimeError("correlation ID space exhausted")`, which + # sends the reader to the word lists instead of the database. A real + # failure belongs to the caller. continue return candidate raise RuntimeError("correlation ID space exhausted") diff --git a/eventing/eventbridge/kafka_group_mirror.py b/eventing/eventbridge/kafka_group_mirror.py index 21805ad7..2929d962 100644 --- a/eventing/eventbridge/kafka_group_mirror.py +++ b/eventing/eventbridge/kafka_group_mirror.py @@ -48,7 +48,8 @@ class GroupMirror(threading.Thread): def __init__(self, bootstrap: str, response_topic: str, groups, metadata_tries: int = 20, max_empty_polls: int = 15, topics: tenancy.TopicSet | None = None, - userkeys: tuple[str, ...] = ()) -> None: + userkeys: tuple[str, ...] = (), + owners=None) -> None: super().__init__(daemon=True, name="kafka-group-mirror") self._bootstrap_servers = bootstrap # §3.2: unlike the live consumers this one is one-shot at startup, so it "can @@ -63,6 +64,10 @@ def __init__(self, bootstrap: str, response_topic: str, groups, # Kept for the log lines and the single-topic metadata path below. self._topic = self._all_topics[0] if self._all_topics else response_topic self._groups = groups + # §2.6's ownership index. Used to resolve which tenant a replayed group belongs + # to, both for settling and for back-filling a missing `userkey`. `None` keeps + # the single-tenant behaviour, where there is one store and nothing to resolve. + self._owners = owners # Partitions are assigned by hand and nothing is ever committed: the whole # point is to re-read from the beginning on every start, which the live # committing consumer cannot do. See run() for why there is no group id. @@ -168,15 +173,28 @@ def run(self) -> None: def _settle(self, touched: set[str]) -> None: """Complete any group whose members all finished but whose `group.completed` is not on the topic — the crash window between the two. Publishing is correct here - and `maybe_complete` will not fire twice for one already completed.""" + and `maybe_complete` will not fire twice for one already completed. + + The owning tenant is resolved per group. Without that, `maybe_complete` looked in + the shared store for a group that lives in a tenant's store, found nothing, and + returned False — so in multi-tenant mode every group a restart left unfinished + stayed unfinished forever, and the failure mode is silence (§21.6's worst one). + """ for gid in sorted(touched): try: - if self._groups.maybe_complete(gid): + if self._groups.maybe_complete(gid, userkey=self._owner_of(gid)): self.settled += 1 _log(f"{gid} was left unfinished by a restart; completed it now") except Exception as e: # noqa: BLE001 _log(f"could not settle {gid}: {e!r}") + def _owner_of(self, correlationid: str) -> str | None: + """Which tenant owns this id, from the ownership index. `None` without one.""" + if self._owners is None: + return None + userkey, _known = self._owners.owner_of(correlationid) + return userkey + def _apply(self, rec) -> str | None: try: evt = ce.from_kafka_binary(rec.headers or [], rec.value) @@ -186,6 +204,20 @@ def _apply(self, rec) -> str | None: gid = d.get("groupid") if not gid: return None + # Back-fill the tenancy key for events published BEFORE Phase 3, which carry no + # `ce_userkey` at all. Without this, replayed member rows go to `shared/` while + # the group row sits in the tenant's store, so the group's counts are permanently + # wrong and `_decorate_member` reads from a third place. Pre-existing Kafka + # history is exactly what this mirror exists to replay, so that is the normal + # upgrade path rather than an edge case. + # + # Only ever FILLS a missing value — an event that carries its own `userkey` keeps + # it, because that one is signed (§2.6) and the index is not authoritative over a + # signed attribute. + if not d.get(ce.EXT_USERKEY): + owner = self._owner_of(d.get("correlationid") or gid) + if owner: + d[ce.EXT_USERKEY] = owner try: if ce.is_group_event(evt): self._groups.on_group_event(d, replay=True) diff --git a/eventing/tests/test_mirror_tenancy.py b/eventing/tests/test_mirror_tenancy.py new file mode 100644 index 00000000..d6be75d8 --- /dev/null +++ b/eventing/tests/test_mirror_tenancy.py @@ -0,0 +1,149 @@ +"""GroupMirror replay in multi-tenant mode. DESIGN_PHASE3.md §6.1, §21.2. + +The mirror is what makes §21.2's "replayable from Kafka alone" claim true, and it is the +normal upgrade path: the history it replays was published before Phase 3 existed, so none +of it carries `ce_userkey`. Both bugs covered here come from that — the mirror knew about +per-user topics but not about which tenant a replayed group belonged to. +""" +from __future__ import annotations + +import json +import pathlib +from unittest.mock import MagicMock + +from eventbridge import registry +from eventbridge.config import Cfg +from eventbridge.correlation import Minter +from eventbridge.group_service import GroupService +from eventbridge.kafka_group_mirror import GroupMirror +from eventbridge.owner_index import OwnerIndex +from eventbridge.store_registry import StoreRegistry +from shared import ce, tenancy + +OWNER = "alice" +UK = tenancy.userkey("github", OWNER) + + +def _svc(tmp_path: pathlib.Path): + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + cfg.tenancy_mode = tenancy.MULTI + root = tmp_path / "eventbridge" + owners = OwnerIndex(root) + stores = StoreRegistry(root, multi=True) + producer = MagicMock() + minter = Minter(index=owners) + svc = GroupService(cfg, stores.for_userkey(None), producer, minter, stores=stores) + return svc, owners, stores, producer + + +def _reg(): + return registry.parse(json.dumps({ + "version": 1, + "users": [{"issuer": "github", "userid": OWNER, "tier": "isolated"}]})) + + +class _Rec: + """A Kafka record carrying a CloudEvent, as the mirror's `_apply` expects.""" + + def __init__(self, event): + self.headers, self.value = ce.to_kafka_binary(event) + self.topic, self.partition, self.offset = "kev1-x-responses", 0, 0 + + +def _member_event(corr, groupid, *, userkey=None, final=True): + attrs = {"type": ce.TYPE_RESPONSE, "source": "rossoctl://eventrunner/test", + "datacontenttype": "application/json", + ce.EXT_CORRELATIONID: corr, ce.EXT_SESSIONUUID: ce.session_uuid(corr), + ce.EXT_SEQUENCE: "1", ce.EXT_PHASE: "result", + ce.EXT_FINAL: "true" if final else "false", + ce.EXT_GROUPID: groupid} + if userkey: + attrs[ce.EXT_USERKEY] = userkey + return ce.new_event(**attrs, data={"text": "done"}) + + +# ---- bug 1: settling completes against the owning tenant's store ---------- + +def test_a_group_left_unfinished_is_settled_in_the_owning_tenants_store(tmp_path): + """Regression. `_settle` used to call `maybe_complete(gid)` with no userkey, so it + looked in the SHARED store for a group that lives in a tenant's store, found nothing, + and returned False — leaving every restart-orphaned group unfinished forever, with + silence as the symptom (§21.6's worst failure mode).""" + svc, owners, stores, _ = _svc(tmp_path) + gid, _ = svc.create(label="b", expected=1, userkey=UK) + corrs = svc.submit_members(gid, ["p1"], userkey=UK) + # The member finished on the topic but `group.completed` never got written. + svc.on_member_event(_member_event(corrs[0], gid, userkey=UK), replay=True) + + mirror = GroupMirror("localhost:9092", "responses", svc, + topics=tenancy.TopicSet("kev1", tenancy=tenancy.MULTI), + userkeys=(UK,), owners=owners) + mirror._settle({gid}) + assert mirror.settled == 1 + assert stores.for_userkey(UK).get_group(gid)["completed_utc"] + + +def test_settling_without_an_index_still_works_single_tenant(tmp_path): + """Single-tenant mode has one store and nothing to resolve.""" + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + root = tmp_path / "eventbridge" + stores = StoreRegistry(root, multi=False) + store = stores.for_userkey(None) + svc = GroupService(cfg, store, MagicMock(), Minter()) + gid, _ = svc.create(label="b", expected=1) + corrs = svc.submit_members(gid, ["p1"]) + svc.on_member_event(_member_event(corrs[0], gid), replay=True) + + mirror = GroupMirror("localhost:9092", "responses", svc) + mirror._settle({gid}) + assert mirror.settled == 1 + assert store.get_group(gid)["completed_utc"] + + +# ---- bug 2: a pre-Phase-3 event gets its userkey back-filled -------------- + +def test_a_replayed_event_without_a_userkey_is_attributed_from_the_index(tmp_path): + """Regression. Events published before Phase 3 carry no `ce_userkey`, so replayed + member rows went to `shared/` while the group row sat in the tenant's store — the + group's counts then permanently wrong. Pre-existing history is exactly what the + mirror replays, so this is the upgrade path, not an edge case.""" + svc, owners, stores, _ = _svc(tmp_path) + gid, _ = svc.create(label="b", expected=1, userkey=UK) + corrs = svc.submit_members(gid, ["p1"], userkey=UK) + + mirror = GroupMirror("localhost:9092", "responses", svc, + topics=tenancy.TopicSet("kev1", tenancy=tenancy.MULTI), + userkeys=(UK,), owners=owners) + # No userkey on the wire, as a Phase 2 runner would have published it. + mirror._apply(_Rec(_member_event(corrs[0], gid, userkey=None))) + + assert stores.for_userkey(UK).group_members(gid), "member row missing from the tenant" + assert not stores.for_userkey(None).group_members(gid), "leaked into shared/" + + +def test_a_signed_userkey_on_the_event_is_never_overridden(tmp_path): + """The index only FILLS a missing value. `userkey` is signed (§2.6), so the index is + not authoritative over what the event itself asserts.""" + svc, owners, stores, _ = _svc(tmp_path) + gid, _ = svc.create(label="b", expected=1, userkey=UK) + corrs = svc.submit_members(gid, ["p1"], userkey=UK) + other = tenancy.userkey("github", "bob") + + mirror = GroupMirror("localhost:9092", "responses", svc, + topics=tenancy.TopicSet("kev1", tenancy=tenancy.MULTI), + userkeys=(UK, other), owners=owners) + mirror._apply(_Rec(_member_event(corrs[0], gid, userkey=other))) + # Followed the event, not the index. + assert stores.for_userkey(other).group_members(gid) + + +def test_an_unknown_correlation_is_left_unattributed(tmp_path): + """Nothing to back-fill from: it goes to `shared/` rather than being guessed.""" + svc, owners, stores, _ = _svc(tmp_path) + mirror = GroupMirror("localhost:9092", "responses", svc, + topics=tenancy.TopicSet("kev1", tenancy=tenancy.MULTI), + userkeys=(UK,), owners=owners) + mirror._apply(_Rec(_member_event("brave-otter-9999", "brave-otter-9998"))) + assert stores.for_userkey(None).group_members("brave-otter-9998") diff --git a/eventing/tests/test_owner_index.py b/eventing/tests/test_owner_index.py index 97c6a7e7..ac9df8b2 100644 --- a/eventing/tests/test_owner_index.py +++ b/eventing/tests/test_owner_index.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import sqlite3 import threading import pytest @@ -210,6 +211,41 @@ def test_remember_is_a_noop_once_an_index_is_configured(idx): # ---- mint_for's compatibility shim --------------------------------------- +def test_a_broken_index_surfaces_instead_of_exhausting_the_id_space(idx): + """Regression. `except Exception` around `claim` meant a genuinely broken index + (disk full, database locked, schema missing) was retried 2000 times and surfaced as + `RuntimeError: correlation ID space exhausted` — a message that sends the reader to + the word lists rather than the database.""" + class BrokenIndex: + def exists(self, corr): + return False + + def claim(self, corr, userkey): + raise sqlite3.OperationalError("database or disk is full") + + m = Minter(index=BrokenIndex()) + with pytest.raises(sqlite3.OperationalError, match="disk is full"): + m.mint(userkey=A) + + +def test_a_lost_race_is_still_retried_not_raised(idx): + """The Collision path must stay a retry: a concurrent winner is normal.""" + class RacyOnce: + def __init__(self): + self.calls = 0 + + def exists(self, corr): + return False + + def claim(self, corr, userkey): + self.calls += 1 + if self.calls == 1: + raise Collision("lost the race") + + m = Minter(index=RacyOnce()) + assert m.mint(userkey=A) # second candidate succeeds + + def test_mint_for_passes_the_userkey_when_supported(idx): m = Minter(index=idx) corr = mint_for(m, A) diff --git a/eventing/tests/test_store_registry.py b/eventing/tests/test_store_registry.py index db225e4f..d57693ec 100644 --- a/eventing/tests/test_store_registry.py +++ b/eventing/tests/test_store_registry.py @@ -137,6 +137,33 @@ def test_a_subscribed_store_is_never_evicted(tmp_path): assert r.evictions >= 1 # something else was evicted instead +def test_a_writer_reaches_a_live_viewer_across_an_eviction_wave(tmp_path): + """The notification path pinning actually protects, not just object identity. + + Without the pinning check the earlier tests still passed: eviction no longer closes a + store, so a held reference keeps working. What breaks is subtler — the *writer* calls + `for_userkey` and gets a NEW Store object, whose `_subscribers` map is empty, so + `insert_response` notifies nobody and the viewer's page stops updating with no error + anywhere. That is the Phase 2 §6.1 symptom class, and this is the assertion for it. + """ + r = _multi(tmp_path, max_open=1) + viewer_store = r.for_userkey(A) + ev = viewer_store.subscribe("brave-otter-0001") + + # A burst of other tenants, each of which would evict A if it were not pinned. + for i in range(5): + r.for_userkey(f"gh-u{i}-0000000{i}") + + # The writer resolves the store the way the responses consumer does. + writer_store = r.for_userkey(A) + assert writer_store is viewer_store, "writer and viewer must share one Store" + writer_store.insert_response({ + "correlationid": "brave-otter-0001", "sequence": 1, "phase": "result", + "final": "true", "id": "e1", "time": "2026-10-02T00:00:00Z", "data": {"text": "hi"}, + }) + assert ev.is_set(), "the viewer was never notified" + + def test_the_cache_may_exceed_max_open_when_everything_is_pinned(tmp_path): """Going over a soft descriptor budget degrades; breaking a live viewer does not recover. The first is the right direction to err.""" From ec4a07c612c8517270471bd65e586354e8a8ce7e Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Fri, 2 Oct 2026 19:16:05 -0400 Subject: [PATCH 08/14] docs(eventing): record T5 as implemented, and what it does not yet give you MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The status lines said steps 5-12 were design only; T5 (per-user stores and the global ownership index) is now in the tree, so they were wrong in the direction that matters — a reader would either re-implement it or, worse, read "per-user stores" as "isolation". The distinction the docs now draw explicitly: each tenant's sessions, responses and transcripts DO live in their own SQLite files, and a read routes to the owning tenant's store — but nothing yet checks that the *caller* is that owner, because owner-scoped reads (T7) and transcript auth (T8) are not implemented. So `multi` mode's reads, `/continue` and `PUT /transcript` remain as open as Phase 2's. Also adds `EB_MAX_OPEN_STORES` to the component README's Phase 3 table with the descriptor arithmetic that explains the default, and to `test_manifests.py`'s classification list so T22's "every Phase 3 variable is on one side of the Secret-vs-ConfigMap rule" check keeps covering it. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/README.md | 11 +++++++---- eventing/agentdocs/DESIGN_PHASE3.md | 21 +++++++++++++++------ eventing/agentdocs/README.md | 2 +- eventing/tests/test_manifests.py | 1 + 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/eventing/README.md b/eventing/README.md index 6a282c46..0f356920 100644 --- a/eventing/README.md +++ b/eventing/README.md @@ -193,14 +193,17 @@ as it did: | `EB_TENANCY_MODE` | `single` | `single` reproduces Phase 2: one topic pair, one store, no ownership checks. `multi` switches on per-user topics and requires an authenticated, registered caller | | `EB_TOPIC_PREFIX` | `kev1` | First component of every per-user topic name, `{prefix}-{userkey}-requests`. Unused in `single` mode | | `EB_USER_REGISTRY_PATH` | — | The approved-user registry (a ConfigMap, §2.5). **Required in `multi` mode, and an empty one denies everyone** — the other reading turns a missing file into an open door | +| `EB_MAX_OPEN_STORES` | `64` | LRU ceiling on open per-user SQLite stores. Two WAL connections per tenant (main + `-wal` + `-shm`), so a 1024 soft descriptor limit bounds this near 150–200 tenants | | `ER_USERKEY` | — | Which tenant this runner serves. Stamped on every response so EventBridge can file it. **Required once `REQUEST_TOPIC` is not `requests`**; the runner refuses to start otherwise | | `ER_AGENT_DIR` | `/etc/rossoctl/agents` | Where baked `AgentSpec`s live (§5.2) | | `ER_AGENT_NAME` | `default` | Fallback agent when neither the request nor the registry names one. `default` with no file on disk is Phase 2's argv exactly | -Two notes on these. **`multi` mode is not yet isolation you should present as such** — -per-user *stores*, owner-scoped reads and transcript authentication are steps 5–8 and -are not implemented, so in `multi` mode the HTTP reads and `/continue` are still as open -as Phase 2's. And even once they land, separate topics without Kafka ACLs are +Two notes on these. **`multi` mode is not yet isolation you should present as such.** +Per-user stores *are* implemented (T5), so each tenant's sessions, responses and +transcripts live in their own SQLite files and a read routes to the owning tenant's store +— but nothing yet checks that the *caller* is that owner, because owner-scoped reads (T7) +and transcript authentication (T8) are not in. So in `multi` mode the HTTP reads, +`/continue` and `PUT /transcript` are still as open as Phase 2's. And even once they land, separate topics without Kafka ACLs are *organisation*, not isolation; `DESIGN_PHASE3.md` §3.6 and §10 are explicit about which claim each configuration earns. **Adding a user is an operator action**: EventBridge has no Kubernetes client and will not create topics, so a registry entry whose topics do not diff --git a/eventing/agentdocs/DESIGN_PHASE3.md b/eventing/agentdocs/DESIGN_PHASE3.md index 49c6ad9a..113f2224 100644 --- a/eventing/agentdocs/DESIGN_PHASE3.md +++ b/eventing/agentdocs/DESIGN_PHASE3.md @@ -3,12 +3,21 @@ Status: draft (revision 1) — **partly implemented.** §9's steps 1-4 are in the tree: T1 (`shared/tenancy.py`), T2 (`TopicSet` threaded through both services), T3 (`Caller`, the registry, `ce_userkey`), T4 (`SIGNED_ATTRS` += `userkey`, `depth`), T13 (`AgentSpec`) -and T22 (the Secret-vs-ConfigMap test). `EB_TENANCY_MODE` defaults to `single`, which -reproduces Phase 2 byte for byte, so nothing below is switched on anywhere yet. -Everything from step 5 on — per-user stores, owner-scoped reads, transcript auth, -`k8s_tenant.py`, ntfy isolation, triggers, fetched skills, Kafka ACLs — is still design -only. Where a section is implemented, the code is the authority on what it does; this -document remains the authority on *why*. +and T22 (the Secret-vs-ConfigMap test). **T5 is also in** — per-user stores +(`store_registry.py`), the global `correlationid → userkey` index (`owner_index.py`), and +the `Minter` uniqueness check that replaces the startup seeding loop (§2.6). +`EB_TENANCY_MODE` defaults to `single`, which reproduces Phase 2 byte for byte, so nothing +below is switched on anywhere yet. + +Still design only: **T6** (`ensure_subscribed`), **T7** (owner-scoped reads — the per-user +stores exist and reads route to the owning tenant's store, but nothing yet checks that the +*caller* is the owner, so `multi` mode's reads are still as open as Phase 2's), **T8** +(transcript auth), **T9-T12** (capability keys, ntfy isolation, `k8s_tenant.py`), +**T14-T19** (triggers and fetched skills), **T20** (Kafka ACLs), **T21** (retention and +deletion). + +Where a section is implemented, the code is the authority on what it does; this document +remains the authority on *why*. Scope: **delta over `DESIGN_PHASE2.md`.** Read that first, and §2.1 of it before anything else here. diff --git a/eventing/agentdocs/README.md b/eventing/agentdocs/README.md index 3fd3145b..59fd489d 100644 --- a/eventing/agentdocs/README.md +++ b/eventing/agentdocs/README.md @@ -19,7 +19,7 @@ and what is *not* verified. | [`IMPLEMENTATION_REPORT1.md`](IMPLEMENTATION_REPORT1.md) | What Phase 1 built, the measured results on both clusters, 28 findings including two real bugs only a scale-to-zero deployment could expose, and an honest account of what is still blocked and why. | | [`README_PHASE1.md`](README_PHASE1.md) | The Phase 1 runbook in full detail, covering all four ways to run it: locally without containers, under Docker, on a Kind cluster, and on OpenShift. | | [`DESIGN_PHASE2.md`](DESIGN_PHASE2.md) | The Phase 2 design — identity on the event path. Why GitHub's opaque user token rules out local verification and forces a `GET /user` lookup plus a load-bearing cache; why `401` and `403` are kept distinct; what `ce_submitter` is and is not worth, and what a signature over it does and does not prove; how the `kid` and the approved-key set prove *which agent* answered, why group events are pinned to EventBridge's own key, and why a rejected response is stored as an error rather than dropped. **§8 records the thirteen claims across §§2.6, 3, 4.3 and 4.4 that a docs review found do not hold in the code, with the issue filed for each — and §8.5-§8.8 the procedure that came out of them: the five questions to ask of any control claim, in any phase, before it ships.** | -| [`DESIGN_PHASE3.md`](DESIGN_PHASE3.md) | The Phase 3 design — per-user isolation, declarative agents, and event triggers. Why a raw `submitter` cannot be a tenancy key and why the derived `userkey` ends in a hash; per-user Kafka topics (`{prefix}-{userkey}-requests`), one EventRunner Deployment and ScaledObject per user, and the honest difference between separately-named topics and ACL-enforced isolation; why a per-user ntfy topic must *not* contain the user's name, and the option of running ntfy as a service in the cluster; the `AgentSpec` — an agent baked into the image or assembled from skills fetched from GitHub, S3 or a URL, and the five gates a fetched bundle passes; authenticating and scoping the session transcript; and a Knative-inspired trigger API with the four loop controls that keep an event-driven agent system from becoming a cost incident. **Partly implemented: §9's steps 1-4 (T1, T2, T3, T4, T13, T22) are in the tree; `EB_TENANCY_MODE` still defaults to `single`, which reproduces Phase 2 exactly. Steps 5-12 — per-user stores, owner-scoped reads, transcript auth, `k8s_tenant.py`, ntfy isolation, triggers, fetched skills, Kafka ACLs — are design only.** | +| [`DESIGN_PHASE3.md`](DESIGN_PHASE3.md) | The Phase 3 design — per-user isolation, declarative agents, and event triggers. Why a raw `submitter` cannot be a tenancy key and why the derived `userkey` ends in a hash; per-user Kafka topics (`{prefix}-{userkey}-requests`), one EventRunner Deployment and ScaledObject per user, and the honest difference between separately-named topics and ACL-enforced isolation; why a per-user ntfy topic must *not* contain the user's name, and the option of running ntfy as a service in the cluster; the `AgentSpec` — an agent baked into the image or assembled from skills fetched from GitHub, S3 or a URL, and the five gates a fetched bundle passes; authenticating and scoping the session transcript; and a Knative-inspired trigger API with the four loop controls that keep an event-driven agent system from becoming a cost incident. **Partly implemented: §9's steps 1-4 (T1, T2, T3, T4, T13, T22) plus T5 (per-user stores and the global ownership index) are in the tree; `EB_TENANCY_MODE` still defaults to `single`, which reproduces Phase 2 exactly. Owner-scoped reads (T7), transcript auth (T8), `k8s_tenant.py` (T12), ntfy isolation, triggers, fetched skills and Kafka ACLs are design only — so `multi` mode routes each tenant to its own store and topics, but does not yet check that a reader is the owner.** | ## Reading order diff --git a/eventing/tests/test_manifests.py b/eventing/tests/test_manifests.py index 6e649e8b..9b24d314 100644 --- a/eventing/tests/test_manifests.py +++ b/eventing/tests/test_manifests.py @@ -179,6 +179,7 @@ def test_no_credential_is_ever_rendered_into_a_manifest(): "EB_TENANCY_MODE", "EB_TOPIC_PREFIX", "EB_USER_REGISTRY_PATH", + "EB_MAX_OPEN_STORES", "ER_AGENT_DIR", "ER_AGENT_NAME", "ER_USERKEY", From 4f769d7d2a676d6921d8d0b6e30b6b79ab0bc20c Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Sat, 3 Oct 2026 19:47:15 -0400 Subject: [PATCH 09/14] fix(eventing): address the five must-fixes from the #883 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every finding reproduced against 16158e5 before fixing, and every fix has a regression test in `tests/test_review_fixes.py` that fails without it (verified by mutation, one fix at a time). They are filed together because they share a cause worth naming: all five were invisible to 192 test functions, because the fixtures assembled the collaborators by hand with the arguments the production call sites fail to supply. **1. `single`-mode regression: the ownership index was never seeded.** The serious one, and against the exact guarantee this phase claims. Dropping Phase 2's `Minter` seeding loop left nothing to populate its replacement, so on the first start after an upgrade `owners.sqlite` is empty while `sessions.sqlite` is full and `exists()` reports live ids as free. With ~1,000 existing correlations a mint has roughly 1-in-25,000 odds of reissuing one, and then `upsert_session` overwrites a session and the new prompt appends to somebody's existing conversation. Phase 2's odds were zero. `OwnerIndex.seed_from()` plus a one-time backfill from every store at startup. §2.6 is right that per-start seeding is the wrong trade; it accidentally argued away the one-time backfill too, and §6.1a now records the distinction. **2. `create_group` did not pass the userkey.** `submit_members` got it and `groups.create()` did not, so in `multi` the group row and the groupid's owner claim landed in the shared store while every member row landed in the tenant's: 0 members forever, `maybe_complete` never fires, the batch never completes and never notifies. A retried POST with an Idempotency-Key read the tenant store `create` never wrote and returned `members: []` — worse than a second batch, because it looks like success. **3. `close_group`/`cancel_group` were unauthenticated cross-tenant mutations.** Neither called `_caller`; both resolved any tenant's group from the global index and acted on it. T7 covers *reads* staying as open as Phase 2's — Phase 2 had no tenants to cross and no cross-tenant mutation anywhere, so this was a capability being introduced, not an openness preserved. Now gated by `_authorize_mutation`, answering `404` for another tenant's group per §6.2's enumeration rule. Single-tenant mode is untouched. **4. `agent` was outside `SIGNED_ATTRS`** while `ce_agent` selects the AgentSpec that supplies `--permission-mode` and the tool allowlists — §5.1's "sandbox". A forged value widened the sandbox and the signature still verified, making a signed deployment worse than an unsigned one because an operator believes it is attested. Added in the same change as `userkey`/`depth`, per §8.3's one-canonicalisation-break rule. **5. `userkey` reached a filesystem path join unvalidated**, from an inbound Kafka header, and `Store.__init__` calls `mkdir(parents=True)` — so `..` created and wrote to a directory inside another tenant's store, or outside the tree. `tenancy.is_valid_userkey()` is the gate, exported from the module that is "the ONLY place an identity becomes a name" so a consumer can check the shape the producer guarantees. An invalid key is treated as a missing one — `shared/` plus `unattributed` — so a forgery is counted and visible rather than acted on. Note it is *truthy*, so the pre-existing `not userkey` check let it past. Also from the review: - **A known-but-unowned correlation 500'd on `/continue`.** `owner=None, unresolved=None` is falsy, so the handler proceeded, wrote a prompt row, then `TopicSet.requests(None)` raised — a traceback with an orphan row already committed, so the conversation showed a turn that was never submitted. Reading and publishing have different requirements, so `_publishable_owner` is now separate from `_owner_userkey`: shared-tier data stays readable, and a turn for it is refused before anything is written. - `claim()` uses `ON CONFLICT DO NOTHING` + a follow-up read, so `Collision` is the only way a conflict presents whether or not two writers race the in-process lock — the bare `INSERT` would have raised `IntegrityError`, which `Minter.mint` deliberately does not catch. - `created_utc` at millisecond precision (matching `ce.now_iso()`) with a `rowid` tiebreaker; at second granularity a 100-member batch had arbitrary order among its rows. - A bad `agent` name is a `400` on the call that made the mistake, not a `202` followed by an async `phase=error` the submitter may never read. - `max_turns: 0` is no longer silently rewritten to the spec's value while the spec path rejects zero. - `EB_TOPIC_PREFIX` validated (it is the one topic-name component `_slug` never sees, and `.`/`_` there reproduce the JMX metric collision) and `EB_MAX_OPEN_STORES` refuses readably instead of raising a bare `ValueError`. - Dropped a flaky assertion: `"gh" not in` 26 characters of base32 fails ~2.4% of the time by construction, and the `"mrsabath" not in` line above it is the real property. - `_canonicalise`'s docstring no longer claims static identities are byte-exact when the `@` branch catches address-shaped ones first. Design doc, where the code had disproved it (§9's "the code is the authority" rule): §6.1's layout corrected to put single-tenant stores in the bridge root; "Closing is safe" replaced with what commit 49aed05 established, including that the first fix attempt only pinned the SSE path and left the class open; new §6.1a documents `owner_index.py`; §2.6's "one new extension attribute" corrected to four, with which three are signed and why; §3.1 records the single-mode `events`/`dead` names. The status header now warns that `multi` does not work end to end until T6 — no response is consumed, which is more useful than "not yet isolated". Tests: 45 new (840 total), ruff clean. Still outstanding from the review and deliberately not in this commit: the T22 subset check, `RequestsMirror`/`NtfyPublisher` `stores=`, the handler-to-topic integration test, the `forget()` tombstone decision, and the four documents. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/agentdocs/DESIGN_PHASE3.md | 146 ++++++-- eventing/eventbridge/__main__.py | 23 ++ eventing/eventbridge/config.py | 21 +- eventing/eventbridge/handlers.py | 112 +++++- eventing/eventbridge/owner_index.py | 78 ++++- eventing/eventbridge/store_registry.py | 53 ++- eventing/eventrunner/runner.py | 8 +- eventing/shared/signing.py | 15 +- eventing/shared/tenancy.py | 46 ++- eventing/tests/test_review_fixes.py | 454 +++++++++++++++++++++++++ eventing/tests/test_tenancy.py | 4 +- 11 files changed, 900 insertions(+), 60 deletions(-) create mode 100644 eventing/tests/test_review_fixes.py diff --git a/eventing/agentdocs/DESIGN_PHASE3.md b/eventing/agentdocs/DESIGN_PHASE3.md index 113f2224..d8bfac1d 100644 --- a/eventing/agentdocs/DESIGN_PHASE3.md +++ b/eventing/agentdocs/DESIGN_PHASE3.md @@ -9,12 +9,20 @@ the `Minter` uniqueness check that replaces the startup seeding loop (§2.6). `EB_TENANCY_MODE` defaults to `single`, which reproduces Phase 2 byte for byte, so nothing below is switched on anywhere yet. -Still design only: **T6** (`ensure_subscribed`), **T7** (owner-scoped reads — the per-user -stores exist and reads route to the owning tenant's store, but nothing yet checks that the -*caller* is the owner, so `multi` mode's reads are still as open as Phase 2's), **T8** -(transcript auth), **T9-T12** (capability keys, ntfy isolation, `k8s_tenant.py`), -**T14-T19** (triggers and fetched skills), **T20** (Kafka ACLs), **T21** (retention and -deletion). +**`EB_TENANCY_MODE=multi` does not yet work end to end, and that is a stronger warning +than "not yet isolated".** Until **T6** lands, neither consumer subscribes to the per-user +topics — `kafka_in.Consumer` and `RequestsMirror` keep the configured `RESPONSE_TOPIC` / +`REQUEST_TOPIC` while the runner publishes to `{prefix}-{userkey}-responses` — so **no +response is ever consumed**: transcript pages stay empty, group counters never advance, +and nothing is logged. `multi` is currently for developing against, not for running. + +Still design only: **T6** (`ensure_subscribed`, and the end-to-end gap above), **T7** +(owner-scoped reads — the per-user stores exist and reads route to the owning tenant's +store, but nothing yet checks that the *caller* is the owner, so `multi` mode's reads are +still as open as Phase 2's; group *mutations* are authorized, because Phase 2 had no +cross-tenant mutation to preserve), **T8** (transcript auth), **T9-T12** (capability keys, +ntfy isolation, `k8s_tenant.py`), **T14-T19** (triggers and fetched skills), **T20** (Kafka +ACLs), **T21** (retention and deletion). Where a section is implemented, the code is the authority on what it does; this document remains the authority on *why*. @@ -307,14 +315,27 @@ Four decisions here: ### 2.6 What rides on the event, and the `sessionuuid` trap -One new extension attribute, and a deliberate non-change: +The tenancy key is the one this section is about; §1 lists all four attributes the phase +adds, and `shared/ce.py` defines all four: ```text ce_submitter: mrsabath # unchanged, Phase 2 §2.6 ce_submitteriss: github # unchanged -ce_userkey: gh-mrsabath-4c1d9e07 # NEW — the tenancy key +ce_userkey: gh-mrsabath-4c1d9e07 # NEW — the tenancy key (this section) +ce_agent: triager # NEW — which AgentSpec serves it (§5.1) +ce_depth: 1 # NEW — the hop limit (§7.7) +ce_triggerid: nightly-triage # NEW — attribution for a trigger (§7.2) ``` +**Three of the four are signed**: `userkey`, `depth` and `agent` all join +`signing.SIGNED_ATTRS` in one change, per §8.3. `agent` belongs there for the same reason +as the other two and arguably more urgently — it selects the `AgentSpec` that supplies +`--permission-mode` and the tool allowlists, which §5.1 calls "the sandbox". Outside the +signed set, anything with write access to a requests topic could rewrite `ce_agent` to +name a wider policy *and the signature would still verify*, which makes a signed +deployment worse than an unsigned one because an operator believes it is attested. +`triggerid` is attribution only and nothing routes or authorizes on it. + `userkey` joins `signing.SIGNED_ATTRS`. It must: it is what decides which store a response is written into and which ntfy topic it is announced on, so an unsigned, mutable `userkey` would let anything with write access to a topic file an event into @@ -343,6 +364,16 @@ before the socket binds, and at 100 tenants with 10,000 correlations each — th `__main__.py` already applies to the single-store seed today — that is a measurable startup delay for a guarantee one `SELECT` gives directly. +**But the index still has to be populated once, and this paragraph read as if it did +not.** Replacing a `seen` set *rebuilt on every start* with an index *consulted on every +mint* is the right trade; it does not remove the need to get the pre-existing ids into +that index the first time. An upgraded deployment starts with an empty `owners.sqlite` +and a full `sessions.sqlite`, so `exists()` answers "free" for live ids and a mint can +reissue one — overwriting a session and appending a new prompt to somebody else's +conversation, where Phase 2's odds of that were zero. §6.1a records the seeding that +closes it. The distinction worth keeping: *per-start seeding* is what §2.6 correctly +rejects, and *one-time backfill* is what it accidentally rejected too. + **The constraint has to be on `correlationid` alone**, and this is the part to get right. `(userkey, correlationid)` is the correct primary key *inside* a tenant's store (§6.1) and is exactly the wrong constraint for the global index: two tenants minting one @@ -364,8 +395,16 @@ The brief's shape, with `{userid}` replaced by `{userkey}` for the reasons in § | inbox (§7) | `{prefix}-{userkey}-events` | EventBridge ingress (`POST /v0/events`) | EventBridge trigger dispatcher | | dead letter (§7.5) | `{prefix}-{userkey}-dead` | EventBridge | nobody; an operator with `kafka-console-consumer` | +**In `single` mode** `requests` and `responses` are whatever `REQUEST_TOPIC` / +`RESPONSE_TOPIC` already name, verbatim — that is what keeps a Phase 0/1/2 deployment +unchanged. The two topics this table adds have no Phase 2 predecessor to stay compatible +with, so they are derived from the prefix instead: `{prefix}-events` and `{prefix}-dead`. + `{prefix}` defaults to the namespace (`kev1`), configurable as `EB_TOPIC_PREFIX`, -exactly as Phase 1 §5 prefixes the shared-broker topics today. Longest realistic +exactly as Phase 1 §5 prefixes the shared-broker topics today. It is validated as +`[a-zA-Z0-9-]{1,32}`: it is the one component of these names `_slug` never sees, and `.` +or `_` in it reproduces precisely the JMX metric collision §3.1's naming rules below warn +about. Longest realistic name: `kev1-` + 2 + 1 + 24 + 1 + 8 + `-responses` = 51 characters, against Kafka's 249 — comfortable, and worth having checked rather than assumed. @@ -1410,9 +1449,57 @@ state into another user's next turn, which means putting words in their agent's sessions.sqlite # sessions, prompts, groups, group_members, transcripts gh-aslom-7b2e55a1/ ... - shared/ # tier `shared` (§3.8), and all of single-tenant mode + shared/ # tier `shared` (§3.8), and any unattributable event + owners.sqlite # the global correlationid -> userkey index (§6.1a) + responses.sqlite # SINGLE-TENANT mode: the bridge root, NOT shared/ + sessions.sqlite ``` +**Where single-tenant mode's files live, corrected.** An earlier revision of this section +put them in `shared/`. The implementation deliberately leaves them in the bridge root, +and that is the right call: a Phase 2 deployment already has `responses.sqlite` and +`sessions.sqlite` directly in `{tmpdir}/eventbridge/`, so relocating them on upgrade would +make every existing session and transcript vanish from the UI. `shared/` is therefore used +only in multi-tenant mode, for the `shared` tier and for events that arrive with no usable +`userkey`. + +### 6.1a The ownership index + +`owners.sqlite` is a **single, deliberately cross-tenant table** — +`correlation_owner(correlationid PRIMARY KEY, userkey, created_utc)` — and it has to be: +the lookup that decides which tenant owns a correlation must happen *before* a per-user +store can be chosen. It holds nothing but the mapping. + +- **`correlationid` is the PRIMARY KEY, alone.** `(userkey, correlationid)` is correct + *inside* a tenant's store and exactly wrong here: two tenants minting one id produce two + distinct tuples, never conflict, and the collision passes silently. §2.6's guarantee that + `sessionuuid` can stay unsalted rests on this constraint. +- **`userkey` is nullable**, meaning single-tenant mode or the `shared` tier. So the + uniqueness guarantee is in force for *every* deployment, including those that never turn + multi-tenancy on. +- **`claim()` is idempotent for the same owner and raises `Collision` for a different + one.** A retried submit is not an error; two tenants claiming one id is. The `INSERT ... + ON CONFLICT DO NOTHING` plus follow-up read makes the *database* the authority, so a + conflict presents as `Collision` whether or not two writers race the in-process lock — + which matters because `Minter.mint` catches `Collision` specifically and would surface a + 500 on `sqlite3.IntegrityError`. +- **`owner_of()` returns `(userkey, known)`**, not a bare key. The two-value return is + load-bearing: a plain `None` cannot distinguish "the shared tier owns this" from "never + seen", and §6.2's `404` rule depends on telling them apart. +- **It must be seeded once, from each store, at startup.** This is a correctness + requirement rather than an optimisation, and omitting it is a regression in the + *default* configuration. On the first start after an upgrade the index is empty while + `sessions.sqlite` is full, so `exists()` answers "free" for ids that are in use and + `Minter` can reissue one — overwriting a session and appending a new prompt to somebody + else's conversation. Phase 2's `seen` set was seeded from the store for exactly this + reason. §2.6 is right that one `SELECT` per mint beats re-seeding on every start; it does + not follow that the index never needs seeding *at all*. +- **`mint_for()` degrades honestly.** A substitute `mint()` that cannot accept a `userkey` + mints an id that is never claimed, so the uniqueness guarantee falls back to whatever + that substitute implements. Reachable only from test doubles — `__main__` always builds a + real `Minter` with an index — but recorded here because §2.6 is where the guarantee is + claimed. + The alternative is one database with a `userkey` column and a `WHERE` clause on every query. It is cheaper — one connection pair, one startup, no file-descriptor arithmetic — and it is rejected: @@ -1433,22 +1520,43 @@ rather than a `GROUP BY`. the main DB plus `-wal` and `-shm`). The default soft `RLIMIT_NOFILE` of 1024 therefore bounds this somewhere around 150–200 concurrently-open tenants before anything else the process holds — not 500, because WAL mode multiplies the count. So `StoreRegistry` -opens lazily and closes on an LRU: +opens lazily and **evicts** on an LRU: ```python # eventbridge/store_registry.py class StoreRegistry: - """userkey -> Store, opened on demand, closed on an LRU (EB_MAX_OPEN_STORES=64). - - Closing is safe because a Store is stateless above SQLite — every method opens a - transaction and commits. The ONE thing that is not: `Store.subscribe()` holds - threading.Events for live SSE viewers, so a store with subscribers is pinned and - exempt from eviction. Evicting it would make an open transcript page stop updating - with no error anywhere, which is the Phase 2 §6.1 class of bug — symptoms that look - like lost events. + """userkey -> Store, opened on demand, evicted on an LRU (EB_MAX_OPEN_STORES=64). + + Eviction DROPS THE REFERENCE; it does not close the store. A store with live SSE + subscribers is additionally pinned — kept in the cache — so a later reader resolves + the same object and sees the subscriber's notifications. """ ``` +**"Closing is safe" was wrong, and the implementation corrected it.** An earlier revision +of this section said the LRU *closes* stores and that closing is safe "because a Store is +stateless above SQLite". The premise is true and the conclusion does not follow: a handler +resolves a store and then makes several calls on it — `get_html` makes five — so a +concurrent request for a different tenant could push the cache over its ceiling and close +that store mid-use. The next call then fails with `sqlite3.ProgrammingError: Cannot +operate on a closed database`, from a line with no visible connection to caching. Every +multi-call read path was exposed. + +Eviction therefore drops the registry's reference and lets the interpreter close the +connections when the last holder goes away, which is exactly the lifetime that is safe. +Two consequences worth recording: + +- **File descriptors are reclaimed slightly later than an explicit close.** That is the + right trade: the ceiling is a soft budget and over-running it degrades visibly on + `/healthz`, whereas a closed connection under an in-flight request is a 500. +- **The store being returned is never the eviction victim** (`protect=`). Without that, a + cache whose older entries are all pinned evicted the newest — and handed the caller the + store it had just closed. + +A note on the fix that is worth more than the fix: the first attempt pinned only the SSE +path, because that was the case easiest to see. It closed one instance and left the class +open. The bug was in the eviction contract, not in one caller. + The response consumer writes to whichever store `ce_userkey` names. An event with no `userkey` in multi-tenant mode goes to `shared/` and increments an `unattributed` counter surfaced on `/healthz` — never silently into a user's store, and never dropped. diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index 6a78308c..0ceae68c 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -127,6 +127,29 @@ def main() -> int: # The `shared` store: single-tenant mode's only store, and multi-tenant mode's home # for anything that arrives without a `userkey` (§6.1's `unattributed` path). store = stores.for_userkey(None) + + # Seed the ownership index from what is already on disk. REQUIRED for correctness on + # the first start after an upgrade, not an optimisation: the index is new and empty + # while `sessions.sqlite` still holds every pre-Phase-3 correlation, so without this + # `exists()` reports live ids as free and `Minter` can reissue one — overwriting a + # session and appending a new prompt to somebody's existing conversation. Phase 2 + # seeded its `seen` set from the store for exactly this reason, and dropping that loop + # without replacing it was a regression in the DEFAULT configuration. + # + # Cheap after the first run: `INSERT OR IGNORE` over ids the index already holds, and + # `all_correlations` is one indexed query per store. The `limit` matches the 10,000 + # Phase 2 applied to its own seed. + seeded = owners.seed_from(store.all_correlations(limit=10000)) + if topics.multi: + # Every tenant's store too, otherwise a tenant's existing ids stay invisible to + # the uniqueness check. `known_userkeys` reads the filesystem rather than the LRU, + # so a tenant that is merely closed is still seeded. + for uk in stores.known_userkeys(): + seeded += owners.seed_from( + stores.for_userkey(uk).all_correlations(limit=10000), uk) + if seeded: + print(f"[eventbridge] seeded {seeded} pre-existing correlation(s) into the " + f"ownership index") if topics.multi: print(f"[eventbridge] per-user stores under {eb_root}/users " f"(max_open={cfg.max_open_stores}), " diff --git a/eventing/eventbridge/config.py b/eventing/eventbridge/config.py index 5467955f..ad997470 100644 --- a/eventing/eventbridge/config.py +++ b/eventing/eventbridge/config.py @@ -3,6 +3,7 @@ import os import pathlib +import re import tomllib from dataclasses import dataclass, field @@ -194,8 +195,26 @@ def load() -> Cfg: f"EB_TENANCY_MODE must be {tenancy.SINGLE!r} or {tenancy.MULTI!r}, " f"got {cfg.tenancy_mode!r}. See DESIGN_PHASE3.md §8.1.") cfg.topic_prefix = e("EB_TOPIC_PREFIX", cfg.topic_prefix) + # The prefix is the one component of a topic name `tenancy._slug` never sees, so it + # is validated here instead. A prefix containing `_` or `.` produces exactly the JMX + # metric-name collision that `_slug`'s docstring warns about (`a.b_c` and `a_b.c` + # flatten to the same metric and one silently overwrites the other), and a space or + # `/` is simply an illegal topic name. + if not re.fullmatch(r"[a-zA-Z0-9-]{1,32}", cfg.topic_prefix or ""): + raise SystemExit( + f"EB_TOPIC_PREFIX must be 1-32 of [a-zA-Z0-9-], got " + f"{cfg.topic_prefix!r}. `.` and `_` are excluded on purpose: Kafka's JMX " + f"metric names flatten both, so two topics differing only in which one they " + f"use silently share one metric. See DESIGN_PHASE3.md §3.1.") cfg.user_registry_path = e("EB_USER_REGISTRY_PATH", cfg.user_registry_path) - cfg.max_open_stores = int(e("EB_MAX_OPEN_STORES", str(cfg.max_open_stores))) + # Readable refusal rather than a bare ValueError traceback, matching every other + # refusal on this startup path. + try: + cfg.max_open_stores = int(e("EB_MAX_OPEN_STORES", str(cfg.max_open_stores))) + except ValueError: + raise SystemExit( + f"EB_MAX_OPEN_STORES must be an integer, got " + f"{e('EB_MAX_OPEN_STORES')!r}") from None cfg.ntfy.enabled = (e("NTFY_ENABLED", "true" if cfg.ntfy.enabled else "false").lower() == "true") cfg.ntfy.base_url = e("NTFY_BASE_URL", cfg.ntfy.base_url) diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index b341987e..fd64d982 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -15,6 +15,7 @@ from eventbridge.kafka_out import Producer from eventbridge.openapi import SWAGGER_HTML, spec from eventbridge.store import Store +from eventrunner import agentspec from shared import ce @@ -90,6 +91,15 @@ def _read_form(environ) -> dict: GROUP_REGEX = CORR_REGEX +class _BadRequest(Exception): + """A 400 raised from a helper so the caller does not have to thread an error back. + + Only `_agent_for` uses it today. A helper that validates caller input needs some way + to refuse, and returning a sentinel would mean every call site remembering to check + one — which is the shape of mistake this phase keeps finding. + """ + + class Handlers: def __init__(self, cfg: Cfg, store: Store, producer: Producer, minter: Minter, groups=None, registry=None, stores=None, owners=None) -> None: @@ -122,7 +132,17 @@ def _agent_for(self, caller, body) -> str | None: specs a given runner image has baked in. """ if named := (body or {}).get("agent"): - return str(named).strip() or None + # Validated at the HTTP boundary, where the caller is still listening. + # `str(named)` would otherwise accept a dict, a list or `"../../etc"` and ship + # it as `ce_agent`; the runner then answers with an asynchronous `phase=error` + # event minutes later, which the submitter may never look at, instead of a + # `400` on the call that made the mistake. + candidate = str(named).strip() + try: + agentspec.validate_name(candidate) + except agentspec.SpecError as e: + raise _BadRequest(str(e)) from None + return candidate if self.registry is not None and caller.userid: user = self.registry.by_userkey(caller.userkey) if caller.userkey else None if user and user.agent: @@ -173,8 +193,31 @@ def _owner_userkey(self, correlationid: str) -> tuple[str | None, str | None]: if not known: return None, (f"{correlationid} is not in the ownership index, so the " f"owning tenant cannot be determined") - # A known correlation with a NULL owner is the `shared` tier, which is a real - # answer rather than a failure: those events live in `shared/` by design. + # A known correlation with a NULL owner is the `shared` tier (or one the mirror + # discovered). That is a real answer for READING — its data lives in `shared/` — + # and NOT a usable answer for PUBLISHING, because `TopicSet.requests(None)` raises + # in multi mode: there is no per-user topic to send a turn to. The two needs + # differ, so they are separate methods: `_store_of` resolves a store and accepts + # this case, `_publishable_owner` refuses it. Conflating them either makes + # shared-tier data unreadable or lets a `/continue` reach the ValueError. + return userkey, None + + def _publishable_owner(self, correlationid: str) -> tuple[str | None, str | None]: + """The owner to publish a turn for, or a reason it cannot be published. + + Stricter than `_owner_userkey`: in multi mode a correlation with no owning tenant + is unpublishable, because there is no per-user requests topic to send to. Refusing + here is what stops `TopicSet.requests(None)`'s `ValueError` becoming a WSGI 500 + with an orphan prompt row already committed — a conversation showing a turn that + was never submitted. + """ + userkey, unresolved = self._owner_userkey(correlationid) + if unresolved: + return None, unresolved + if self.cfg.topics.multi and userkey is None: + return None, (f"{correlationid} has no owning tenant (it predates tenancy or " + f"belongs to the shared tier), so there is no per-user topic to " + f"publish a turn to") return userkey, None def _store_of(self, correlationid: str): @@ -224,11 +267,15 @@ def start_agent(self, environ, start_response, **_): # Membership before publication: the reverse order leaves a window where a # fast agent's terminal event arrives for a member nobody has recorded. store.add_group_member(groupid, corr) + try: + agent = self._agent_for(caller, body) + except _BadRequest as e: + return _json(start_response, "400 Bad Request", {"error": str(e)}) event_id = self.producer.publish_request( prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", groupid=groupid, submitter=submitter, submitter_iss=sub_iss, - userkey=caller.userkey, agent=self._agent_for(caller, body), + userkey=caller.userkey, agent=agent, ) out = { "correlationid": corr, "sessionuuid": sess, @@ -277,6 +324,13 @@ def create_group(self, environ, start_response, **_): deadline_s=float(body["deadline_s"]) if body.get("deadline_s") else self.cfg.group_deadline_s, idempotency_key=idem, + # Without this the group ROW and the groupid's owner-index claim land in the + # shared store with a NULL owner while every member row lands in the caller's + # store: the batch reports 0 members forever, `maybe_complete` never fires, so + # it never completes and never notifies — and a retried POST carrying an + # Idempotency-Key reads the tenant store `create` never wrote and returns + # `members: []`, which looks like success. + userkey=caller.userkey, ) if not created: # A retried POST must not launch a second batch. @@ -286,11 +340,15 @@ def create_group(self, environ, start_response, **_): "members": [m["correlationid"] for m in snap.get("members") or []], "html_url": f"{self.cfg.public_base_url}/v0/groups/{groupid}", }) + try: + agent = self._agent_for(caller, body) + except _BadRequest as e: + return _json(start_response, "400 Bad Request", {"error": str(e)}) corrs = self.groups.submit_members( groupid, [str(x) for x in prompts], max_turns=int(body.get("max_turns", 3)), model=body.get("model"), submitter=submitter, submitter_iss=sub_iss, - userkey=caller.userkey, agent=self._agent_for(caller, body)) + userkey=caller.userkey, agent=agent) return _json(start_response, "202 Accepted", { "groupid": groupid, "created": True, "expected": expected or len(corrs), "members": corrs, @@ -387,11 +445,45 @@ def _decorate_member(self, m: dict) -> dict: out["duration"] = f"{ms / 1000:.1f}s" if ms else None return out + def _authorize_mutation(self, environ, groupid: str): + """Gate a route that CHANGES a group. Returns `(userkey, error)`. + + Mutating routes are authorized in `multi` mode even though reads are not yet + (§6.2/T7). The distinction matters: T7 is about reads *staying* as open as Phase + 2's, and Phase 2 had no tenants to cross and no cross-tenant mutation anywhere. + An unauthenticated `POST /v0/groups/{gid}/cancel` that resolves any tenant's group + from the global index and cancels their queued members is a capability this phase + would be *introducing*, not an openness it preserves — so it is refused here + rather than deferred. + + `404` for a groupid this caller does not own, matching §6.2: distinguishing + "exists but not yours" from "does not exist" lets anyone enumerate live ids + across tenants, and the caller learns nothing actionable from the difference. + """ + if self.stores is None: + # Single-tenant mode: Phase 2's behaviour, no ownership to check. + return None, None + caller, status, why = self._caller(environ) + if status: + return None, (why, status) + owner, unresolved = self._owner_userkey(groupid) + if unresolved or owner != caller.userkey: + return None, ("unknown groupid", 404) + return owner, None + + def _mutation_denied(self, start_response, err): + reason, status = err + if status == 404: + return _json(start_response, "404 Not Found", {"error": reason}) + return _deny(start_response, reason, status) + def close_group(self, environ, start_response, groupid: str, **_): if self.groups is None: return _json(start_response, "503 Service Unavailable", {"error": "disabled"}) + owner, err = self._authorize_mutation(environ, groupid) + if err: + return self._mutation_denied(start_response, err) body = _read_json(environ) - owner, _ = self._owner_userkey(groupid) ok = self.groups.close(groupid, int(body["expected"]) if body.get("expected") else None, userkey=owner) @@ -400,7 +492,9 @@ def close_group(self, environ, start_response, groupid: str, **_): def cancel_group(self, environ, start_response, groupid: str, **_): if self.groups is None: return _json(start_response, "503 Service Unavailable", {"error": "disabled"}) - owner, _ = self._owner_userkey(groupid) + owner, err = self._authorize_mutation(environ, groupid) + if err: + return self._mutation_denied(start_response, err) ok = self.groups.cancel(groupid, userkey=owner) return _json(start_response, "200 OK", { "groupid": groupid, "cancelled": ok, @@ -419,7 +513,7 @@ def continue_agent(self, environ, start_response, correlationid: str, **_): # One lookup decides both which store holds this conversation and which tenant # the resume must be published for. An unplaceable correlation is a 404 — the same # answer a foreign one gets (§6.2), so the two are indistinguishable to a client. - owner, unresolved = self._owner_userkey(correlationid) + owner, unresolved = self._publishable_owner(correlationid) store = None if unresolved else ( self.stores.for_userkey(owner) if self.stores else self.store) session = store.get_session(correlationid) if store else None @@ -446,7 +540,7 @@ def continue_agent_html(self, environ, start_response, correlationid: str, **_): form = _read_form(environ) prompt = form.get("prompt", "").strip() if prompt: - owner, unresolved = self._owner_userkey(correlationid) + owner, unresolved = self._publishable_owner(correlationid) if unresolved: # The form posts from the transcript page, so the useful answer is the # page with an error on it rather than a silent 303 — redirecting would diff --git a/eventing/eventbridge/owner_index.py b/eventing/eventbridge/owner_index.py index 605e5001..a7559d4e 100644 --- a/eventing/eventbridge/owner_index.py +++ b/eventing/eventbridge/owner_index.py @@ -46,6 +46,11 @@ CREATE TABLE IF NOT EXISTS correlation_owner ( correlationid TEXT PRIMARY KEY, -- UNIQUE, and deliberately NOT (userkey, corr) userkey TEXT, -- NULL = single-tenant or the `shared` tier + -- Millisecond precision, matching `ce.now_iso()`. At second granularity a batch + -- submitted inside one second had arbitrary order among its rows, so + -- `ORDER BY created_utc DESC LIMIT n` returned an arbitrary subset -- and + -- `submit_members` publishes a whole 100-member batch in a loop, which is exactly + -- that case. Ordered reads also tie-break on `rowid`, which is free and total. created_utc TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_owner_userkey ON correlation_owner(userkey); @@ -92,22 +97,35 @@ def claim(self, correlationid: str, userkey: str | None) -> None: comparison treats NULL and a key as different owners, which is correct: a correlation minted in single-tenant mode and later claimed by a tenant is a genuine ambiguity, not a no-op. + + **The database is the authority, not the lock.** `ON CONFLICT DO NOTHING` plus a + follow-up read means a conflict presents as `Collision` whether or not two writers + raced past the in-process lock. An earlier version did `SELECT` then a bare + `INSERT`: correct while the lock held, but if two writers ever did race the + `INSERT` raised `sqlite3.IntegrityError` rather than `Collision` — and + `Minter.mint` catches `Collision` specifically (deliberately, so a broken index is + not retried 2000 times), so that path would have surfaced a 500 instead of + retrying. §6.5's deletion path and any future sweeper are exactly the second + writer that makes this reachable. """ with self._lock: + cur = self._c.execute( + "INSERT INTO correlation_owner(correlationid,userkey,created_utc) " + "VALUES (?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now')) " + "ON CONFLICT(correlationid) DO NOTHING", + (correlationid, userkey)) + if cur.rowcount: + return + # The row already existed. Same owner is idempotent; anything else is a clash. row = self._c.execute( "SELECT userkey FROM correlation_owner WHERE correlationid=?", (correlationid,)).fetchone() - if row is not None: - if row[0] == userkey: - return - raise Collision( - f"correlation {correlationid!r} is already owned by " - f"{row[0] or '(single-tenant)'}, cannot claim for " - f"{userkey or '(single-tenant)'}") - self._c.execute( - "INSERT INTO correlation_owner(correlationid,userkey,created_utc) " - "VALUES (?,?,strftime('%Y-%m-%dT%H:%M:%SZ','now'))", - (correlationid, userkey)) + if row is not None and row[0] == userkey: + return + raise Collision( + f"correlation {correlationid!r} is already owned by " + f"{(row[0] if row else None) or '(single-tenant)'}, cannot claim for " + f"{userkey or '(single-tenant)'}") def exists(self, correlationid: str) -> bool: """Whether this id is taken by anyone. The `Minter`'s uniqueness check. @@ -119,6 +137,39 @@ def exists(self, correlationid: str) -> bool: "SELECT 1 FROM correlation_owner WHERE correlationid=? LIMIT 1", (correlationid,)).fetchone() is not None + def seed_from(self, correlationids, userkey: str | None = None) -> int: + """Record ids that already exist outside the index. Returns how many were new. + + **This is not an optimisation; it closes a correctness hole.** §2.6 is right that + one `SELECT` per mint beats seeding a `seen` set from N stores on every start — but + it does not follow that the index never needs seeding *once*. On the first start + after an upgrade, `owners.sqlite` is brand new and empty while `sessions.sqlite` + still holds every correlation from before, so `exists()` answers "free" for ids + that are in use. `Minter.mint` consults nothing else, so it can reissue a live id; + `upsert_session` then overwrites that session and the new prompt appends to + somebody's existing conversation. + + The id space is 50 adjectives x 50 animals x 10,000 = 25,000,000, so a deployment + with ~1,000 existing correlations has roughly a 1-in-25,000 chance per mint. + Phase 2's chance was zero, because its `seen` set was seeded from the store — so + without this, Phase 3 is a regression in the DEFAULT configuration, which is the + one thing the phase promises cannot happen. + + `INSERT OR IGNORE`, so re-seeding is free and a correlation already claimed by a + tenant keeps its owner. Seeding assigns `userkey=None` for ids recovered from a + single-tenant store, which is the correct owner for them: they predate tenancy. + """ + new = 0 + with self._lock: + for corr in correlationids: + cur = self._c.execute( + "INSERT OR IGNORE INTO correlation_owner" + "(correlationid,userkey,created_utc) " + "VALUES (?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))", + (corr, userkey)) + new += cur.rowcount or 0 + return new + # ---- owner-scoped reads (§6.2) ----------------------------------------- def owner_of(self, correlationid: str) -> tuple[str | None, bool]: @@ -143,11 +194,12 @@ def correlations_for(self, userkey: str | None, limit: int = 1000) -> list[str]: if userkey is None: rows = self._c.execute( "SELECT correlationid FROM correlation_owner WHERE userkey IS NULL " - "ORDER BY created_utc DESC LIMIT ?", (limit,)).fetchall() + "ORDER BY created_utc DESC, rowid DESC LIMIT ?", (limit,)).fetchall() else: rows = self._c.execute( "SELECT correlationid FROM correlation_owner WHERE userkey=? " - "ORDER BY created_utc DESC LIMIT ?", (userkey, limit)).fetchall() + "ORDER BY created_utc DESC, rowid DESC LIMIT ?", + (userkey, limit)).fetchall() return [r[0] for r in rows] def userkeys(self) -> list[str]: diff --git a/eventing/eventbridge/store_registry.py b/eventing/eventbridge/store_registry.py index 6a0b6e3c..15815da5 100644 --- a/eventing/eventbridge/store_registry.py +++ b/eventing/eventbridge/store_registry.py @@ -8,9 +8,18 @@ sessions.sqlite # sessions, prompts, groups, group_members, transcripts gh-aslom-7b2e55a1/ ... - shared/ # tier `shared` (§3.8), and ALL of single-tenant mode + shared/ # tier `shared` (§3.8), and any unattributable event + responses.sqlite # SINGLE-TENANT mode lives in the bridge root, not shared/ + sessions.sqlite + owners.sqlite # the global correlationid -> userkey index ``` +Note where single-tenant mode's files sit: **the bridge root, not `shared/`**. §6.1's +diagram puts them in `shared/`, and the code deliberately differs — a Phase 2 deployment +already has `responses.sqlite` and `sessions.sqlite` directly in `{tmpdir}/eventbridge/`, +so relocating them on upgrade would make every existing session and transcript vanish +from the UI. + **Why separate files rather than one database with a `userkey` column.** The single-DB version is cheaper — one connection pair, one startup, no file-descriptor arithmetic — and §6.1 rejects it: `store.py` has 28 methods and every one reads or writes tenant data. @@ -38,6 +47,7 @@ import threading from eventbridge.store import Store +from shared import tenancy # §6.1 / §8.1 `EB_MAX_OPEN_STORES`. 64 pairs is ~192 file descriptors with WAL, which # leaves comfortable room under a 1024 soft limit for the Kafka sockets, the HTTP @@ -51,14 +61,19 @@ class StoreRegistry: - """`userkey -> Store`, opened on demand, closed on an LRU. - - Closing is safe because a `Store` is stateless above SQLite — every method opens a - transaction and commits, so there is no in-memory state to lose. The ONE thing that - is not: `Store.subscribe()` holds `threading.Event`s for live SSE viewers, so a store - with subscribers is **pinned** and exempt from eviction. Evicting it would make an - open transcript page stop updating with no error anywhere — the Phase 2 §6.1 class of - bug, where the symptom looks like lost events rather than like a closed file. + """`userkey -> Store`, opened on demand, evicted on an LRU. + + **Eviction drops the registry's reference; it does not close the store.** §6.1 says + the LRU "closes" and that "closing is safe because a Store is stateless above SQLite". + The first half is no longer what the code does and the second was disproven: a handler + holds a store across several calls, so a concurrent request for another tenant could + close it mid-use and the next call failed with `ProgrammingError: Cannot operate on a + closed database`. See `_evict_if_needed` for the full argument. + + A store with live SSE subscribers is additionally **pinned** — kept in the cache, not + merely uncollected — so a later reader resolves the SAME object and therefore sees the + subscriber's notifications. Evicting it would leave an open page watching an object + nothing writes to any more: updates stop with no error, the Phase 2 §6.1 symptom class. """ def __init__(self, root: str | pathlib.Path, *, @@ -88,10 +103,17 @@ def _dir_for(self, userkey: str | None) -> pathlib.Path: in `{tmpdir}/eventbridge/`, and relocating them on upgrade would make every existing session and transcript vanish from the UI. Multi-tenant mode is new, so it is free to use the `users//` layout. + + **The key is validated before it reaches the join.** It can arrive from an inbound + Kafka header, and `Store.__init__` calls `mkdir(parents=True)`, so a key containing + `..` would create and then write to a directory inside another tenant's store or + outside the tree entirely. An invalid key is treated exactly like a missing one + rather than raising, so a forged value lands in `shared/`, bumps `unattributed` + and is visible — see `for_event`. """ if not self._multi: return self._root - if not userkey: + if not tenancy.is_valid_userkey(userkey): return self._root / SHARED return self._root / "users" / userkey @@ -123,9 +145,12 @@ def for_userkey(self, userkey: str | None) -> Store: return store def _cache_key(self, userkey: str | None) -> str: + """The LRU key. Must agree with `_dir_for` about where a key maps, or an invalid + one would get its own cache entry whose Store points at `shared/` — two entries, + one directory, and SQLite connections to the same files from both.""" if not self._multi: return SHARED - return userkey or SHARED + return userkey if tenancy.is_valid_userkey(userkey) else SHARED def for_event(self, event) -> Store: """The store an inbound response belongs in, from its `ce_userkey`. @@ -139,7 +164,11 @@ def for_event(self, event) -> Store: from shared import ce getter = getattr(event, "get", None) userkey = getter(ce.EXT_USERKEY) if getter else None - if self._multi and not userkey: + # An INVALID key counts as unattributed too, not just a missing one. It is + # truthy, so a bare `not userkey` check would let a forged `../..` sail through + # to the path join uncounted — the forgery has to be visible, which is the whole + # point of the counter. + if self._multi and not tenancy.is_valid_userkey(userkey): with self._lock: self.unattributed += 1 return self.for_userkey(userkey) diff --git a/eventing/eventrunner/runner.py b/eventing/eventrunner/runner.py index 6c9f6b6a..3d159d96 100644 --- a/eventing/eventrunner/runner.py +++ b/eventing/eventrunner/runner.py @@ -249,7 +249,13 @@ def build_cmd(cfg: Cfg, event, payload: dict, *, session = event["sessionuuid"] mode = event.get("mode", "start") prompt = payload.get("prompt", "") - max_turns = payload.get("max_turns") or spec.max_turns + # `is not None`, not `or`: `max_turns: 0` is nonsensical input, and with `or` it + # silently became the spec's value while the spec path REJECTS zero + # (`test_a_nonsensical_max_turns_is_refused`). Two paths disagreeing about the same + # bad input is worse than either answer — the HTTP boundary is where it should be + # refused, and until then both paths at least agree it is what the caller asked for. + req_turns = payload.get("max_turns") + max_turns = spec.max_turns if req_turns is None else req_turns cmd = [ cfg.claude_bin, "-p", prompt, "--output-format", "stream-json", diff --git a/eventing/shared/signing.py b/eventing/shared/signing.py index 763b1929..cfcf9103 100644 --- a/eventing/shared/signing.py +++ b/eventing/shared/signing.py @@ -201,12 +201,23 @@ def verify(message: bytes, signature: bytes, pub: bytes) -> bool: # upgrade both services together. With signing off (the default) there # is nothing to coordinate, which is most deployments. # - # Both have to be here rather than merely present on the event: + # All three have to be here rather than merely present on the event: # `userkey` decides which store a response is written into and which # ntfy topic announces it, so a mutable one lets anything with topic # write access file events into another user's history. `depth` is the # hop limit, and a resettable hop limit does not limit hops. - "userkey", "depth") + # + # `agent` selects the AgentSpec, and §5.1 calls the tool policy "the + # sandbox": the spec supplies `--permission-mode`, `--allowedTools`, + # `--disallowedTools`, `--settings` and `--mcp-config`. Outside this set, + # anything with write access to a requests topic could rewrite `ce_agent` + # to name a baked spec with `permission_mode = "bypassPermissions"` and no + # `disallowed_tools` — and THE SIGNATURE WOULD STILL VERIFY, so the runner + # executes the forged policy as an approved request. That makes the signed + # configuration worse than the unsigned one, because an operator believes + # it is attested. Added in the same change as the other two, per §8.3's + # one-canonicalisation-break rule. + "userkey", "depth", "agent") def data_bytes(data: Any) -> bytes: diff --git a/eventing/shared/tenancy.py b/eventing/shared/tenancy.py index d9238702..8d473f7b 100644 --- a/eventing/shared/tenancy.py +++ b/eventing/shared/tenancy.py @@ -26,6 +26,7 @@ import base64 import hashlib import hmac +import re # Short issuer tags. Two characters so the readable middle of the key keeps as much # room as possible, and in the key at all so a human reading `kubectl get kafkatopics` @@ -66,8 +67,13 @@ def _canonicalise(issuer: str, userid: str) -> str: part byte-exact. One human with two spellings therefore gets two tenants, which is wasteful, not unsafe. The unsafe direction (two humans, one tenant) is what the digest in `userkey` makes impossible. - * **Static** identities are byte-exact: an operator typed them, and normalising - somebody's hand-written string on their behalf is how you merge two of them. + * **Static** identities are byte-exact — *unless they are address-shaped*. The `@` + branch above is checked before the issuer, so a static `EB_AUTH_TOKENS` name of + `Ops@Example.COM` canonicalises to `Ops@example.com` rather than byte-exact. That + is deliberate: the email rule applies to anything containing `@` whatever the + issuer, because an operator who types an address means an address. Everything else + from a static issuer is left exactly as typed — normalising somebody's + hand-written string on their behalf is how you merge two of them. `rpartition` rather than `partition` because the local part of an address may itself contain `@` when quoted; the domain is what follows the LAST one. @@ -150,6 +156,42 @@ def userkey(issuer: str, userid: str) -> str: return f"{iss}-{slug}-{digest}" +# The exact shape `userkey()` produces: `<2-letter iss>--<8 hex>`. Exported as a +# validator because this module is "the ONLY place an identity becomes a name" and a +# producer's guarantee is worth nothing if no consumer can check it. +# +# The slug is at most _SLUG_MAX characters, starts with an alphanumeric (`_slug` strips +# leading dashes) and contains only `[a-z0-9-]`. +USERKEY_RE = re.compile( + rf"^[a-z]{{2}}-[a-z0-9][a-z0-9-]{{0,{_SLUG_MAX - 1}}}-[0-9a-f]{{{_DIGEST_HEX}}}$") + + +def is_valid_userkey(value: str | None) -> bool: + """Whether `value` is a key this module could have produced. + + **This is a security gate, not a type check.** A `userkey` reaches a filesystem path + join (`store_registry._dir_for`) and in multi-tenant mode it arrives from an inbound + Kafka header, so an unvalidated one containing `..` escapes `users/` into another + tenant's store — or out of the tree entirely, since `Store.__init__` calls + `mkdir(parents=True)` and therefore creates the directory rather than rejecting it. + + Two reasons that is reachable rather than theoretical: + + * `userkey` is in `signing.SIGNED_ATTRS`, but `EB_REQUIRE_RESPONSE_SIGNATURE` defaults + to `false` and audit mode stores the event unchanged — verification is what would + catch the forgery and it is off by default. + * §3.6 concedes that in Tier A anything with network access to the shared broker can + write to any topic. That is an accepted property for *reading* events; it must not + also be a filesystem write primitive. + + Callers treat an invalid key exactly like a missing one — the `shared` store plus the + `unattributed` counter — so a forged value is counted and visible rather than acted on. + Note that an invalid key is *truthy*, so it would otherwise sail past a `if not + userkey` check straight into the join. + """ + return bool(value) and bool(USERKEY_RE.fullmatch(value)) + + def ntfy_topic(prefix: str, userkey: str, secret: bytes) -> str: """An unguessable, stable, per-user ntfy topic that contains no identifier. diff --git a/eventing/tests/test_review_fixes.py b/eventing/tests/test_review_fixes.py new file mode 100644 index 00000000..2ec49842 --- /dev/null +++ b/eventing/tests/test_review_fixes.py @@ -0,0 +1,454 @@ +"""Regressions for the five must-fixes in the #883 review. + +Each test fails without its fix. They live together because they share a theme worth +naming: **every one was invisible to a suite of 192 test functions**, and for the same +reason — the fixtures assembled the collaborators by hand with the arguments the +production call sites fail to supply. So each test here drives the real entry point +(`Handlers`, `__main__`'s wiring order, `SIGNED_ATTRS`) rather than the piece underneath. +""" +from __future__ import annotations + +import io +import json +import pathlib +import sqlite3 +from unittest.mock import MagicMock, patch + +import pytest + +from eventbridge import auth, registry +from eventbridge.config import Cfg +from eventbridge.correlation import Minter +from eventbridge.group_service import GroupService +from eventbridge.handlers import Handlers +from eventbridge.kafka_out import Producer +from eventbridge.owner_index import OwnerIndex +from eventbridge.store import Store +from eventbridge.store_registry import StoreRegistry +from shared import ce, keyset, tenancy +from shared import signing as S + +# These tests authenticate with static bearer tokens (`EB_AUTH_TOKENS`), which resolve +# to issuer `static` — so the registry lists the users under that issuer. Using `github` +# here would be a 403, correctly: §2.3 keeps a static `alice` and a GitHub `alice` as two +# different tenants on purpose. +OWNER = "alice" +UK = tenancy.userkey(auth.ISSUER_STATIC, OWNER) +OTHER = tenancy.userkey(auth.ISSUER_STATIC, "bob") + + +def _api(tmp_path: pathlib.Path, *, multi=True, real_producer=False): + """Handlers wired as `__main__` wires it, so the tests exercise the real paths.""" + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + cfg.tenancy_mode = tenancy.MULTI if multi else tenancy.SINGLE + root = tmp_path / "eventbridge" + owners = OwnerIndex(root) if multi else None + stores = StoreRegistry(root, multi=multi) + store = stores.for_userkey(None) + minter = Minter(index=owners) + if real_producer: + with patch("eventbridge.kafka_out.KafkaProducer") as kp: + kp.return_value = MagicMock() + producer = Producer("b", "requests", "src", + response_topic="responses", topics=cfg.topics) + else: + producer = MagicMock() + producer.publish_request.return_value = "evt-1" + groups = GroupService(cfg, store, producer, minter, + stores=stores if multi else None) + reg = registry.parse(json.dumps({ + "version": 1, + "users": [{"issuer": auth.ISSUER_STATIC, "userid": u, "tier": "isolated"} + for u in (OWNER, "bob")]})) if multi else None + h = Handlers(cfg, store, producer, minter, groups=groups, registry=reg, + stores=stores if multi else None, owners=owners) + return h, producer, owners, stores + + +class _Start: + def __init__(self): + self.status = None + + def __call__(self, status, headers): + self.status = status + + +def _env(body=None, *, token=None, form=False): + if form: + raw = "&".join(f"{k}={v}" for k, v in (body or {}).items()).encode() + ctype = "application/x-www-form-urlencoded" + else: + raw = json.dumps(body or {}).encode() + ctype = "application/json" + e = {"wsgi.input": io.BytesIO(raw), "CONTENT_LENGTH": str(len(raw)), + "CONTENT_TYPE": ctype, "REQUEST_METHOD": "POST", "QUERY_STRING": ""} + if token: + e["HTTP_AUTHORIZATION"] = f"Bearer {token}" + return e + + +def _as(h, userid): + """Authenticate as a registry user, via a static token.""" + h.cfg.auth_tokens = {f"tok-{userid}": userid} + return f"tok-{userid}" + + +# ---- 1. the single-mode index seeding regression -------------------------- + +def test_the_index_is_seeded_from_an_existing_store(tmp_path): + """Regression, and the serious one: a `single`-mode regression against the exact + guarantee the phase claims. + + On the first start after an upgrade `owners.sqlite` is new and empty while + `sessions.sqlite` is full, so `exists()` reported live ids as free and `Minter` could + reissue one — `upsert_session` then overwrites that session and the new prompt + appends to somebody's existing conversation. Phase 2's odds were zero, because it + seeded its `seen` set from the store. + """ + root = tmp_path / "eventbridge" + store = Store(root) + pre = Minter(seed=1) + live = [] + for _ in range(40): + c = pre.mint() + store.upsert_session(c, "u", "/w", "an existing conversation") + live.append(c) + + owners = OwnerIndex(root) + assert owners.count() == 0, "a fresh index starts empty — the precondition" + seeded = owners.seed_from(store.all_correlations(limit=10000)) + assert seeded == 40 + assert not [c for c in live if not owners.exists(c)], \ + "a live correlation is still reported free, so Minter can reissue it" + + +def test_seeding_is_idempotent_across_restarts(tmp_path): + root = tmp_path / "eventbridge" + store = Store(root) + store.upsert_session("brave-otter-0001", "u", "/w", "hi") + owners = OwnerIndex(root) + assert owners.seed_from(store.all_correlations()) == 1 + assert owners.seed_from(store.all_correlations()) == 0 + + +def test_seeding_does_not_steal_an_existing_owner(tmp_path): + """A correlation a tenant already claimed keeps its owner when the store is seeded.""" + root = tmp_path / "eventbridge" + owners = OwnerIndex(root) + owners.claim("brave-otter-0001", UK) + owners.seed_from(["brave-otter-0001"], None) + assert owners.owner_of("brave-otter-0001") == (UK, True) + + +def test_claim_raises_collision_not_integrityerror(tmp_path): + """`Minter.mint` catches `Collision` specifically, so a conflict presenting as + `sqlite3.IntegrityError` would surface a 500 rather than retrying. The database is + the authority now, not the in-process lock.""" + from eventbridge.owner_index import Collision + owners = OwnerIndex(tmp_path) + owners.claim("brave-otter-0001", UK) + with pytest.raises(Collision): + owners.claim("brave-otter-0001", OTHER) + + +# ---- 2. create_group did not pass the userkey ---------------------------- + +def test_create_group_puts_the_group_in_the_callers_store(tmp_path): + """Regression. `submit_members` got the userkey and `groups.create` did not, so the + group row landed in the shared store while members landed in the tenant's: the batch + reported 0 members forever and never completed. + + This drives `Handlers.create_group`, which is what no existing test did — every + multi-mode group test called `GroupService.create` directly WITH the userkey. + """ + h, _, owners, stores = _api(tmp_path) + tok = _as(h, OWNER) + sr = _Start() + out = h.create_group(_env({"prompts": ["p1", "p2"]}, token=tok), sr) + assert sr.status.startswith("202"), sr.status + gid = json.loads(b"".join(out))["groupid"] + + assert stores.for_userkey(UK).get_group(gid) is not None, \ + "group row is not in the caller's store" + assert stores.for_userkey(None).get_group(gid) is None, \ + "group row leaked into the shared store" + assert owners.owner_of(gid) == (UK, True) + assert len(stores.for_userkey(UK).group_members(gid)) == 2 + + +def test_a_retried_create_group_returns_its_members(tmp_path): + """The mirror image: `not created` read the tenant store `create` never wrote, so an + Idempotency-Key retry answered `members: []` — worse than a second batch, because it + looks like success.""" + h, _, _, _ = _api(tmp_path) + tok = _as(h, OWNER) + e1 = _env({"prompts": ["p1", "p2"]}, token=tok) + e1["HTTP_IDEMPOTENCY_KEY"] = "key-1" + first = json.loads(b"".join(h.create_group(e1, _Start()))) + e2 = _env({"prompts": ["p1", "p2"]}, token=tok) + e2["HTTP_IDEMPOTENCY_KEY"] = "key-1" + second = json.loads(b"".join(h.create_group(e2, _Start()))) + assert second["created"] is False + assert second["groupid"] == first["groupid"] + # Compared as sets: `snapshot` does not promise member order, and the bug was that + # the retry returned NOTHING — an empty list where the original had two members. + assert set(second["members"]) == set(first["members"]) + assert second["members"], "a retry returned no members at all" + + +# ---- 3. close/cancel were unauthenticated cross-tenant mutations --------- + +@pytest.mark.parametrize("route", ["close_group", "cancel_group"]) +def test_group_mutation_requires_authentication(tmp_path, route): + """Regression. An unauthenticated POST resolved any tenant's group from the global + index and mutated it. T7 covers reads staying open; Phase 2 had no cross-tenant + mutation anywhere, so this was a capability being introduced.""" + h, _, _, stores = _api(tmp_path) + tok = _as(h, OWNER) + gid = json.loads(b"".join( + h.create_group(_env({"prompts": ["p1"]}, token=tok), _Start())))["groupid"] + + sr = _Start() + getattr(h, route)(_env(), sr, groupid=gid) # no credential + assert sr.status.startswith("401"), sr.status + assert not stores.for_userkey(UK).get_group(gid)["completed_utc"] + + +@pytest.mark.parametrize("route", ["close_group", "cancel_group"]) +def test_group_mutation_is_404_for_another_tenants_group(tmp_path, route): + """§6.2's rule: "exists but not yours" must be indistinguishable from "does not + exist", or anyone can enumerate live ids across tenants.""" + h, _, _, _ = _api(tmp_path) + gid = json.loads(b"".join(h.create_group( + _env({"prompts": ["p1"]}, token=_as(h, OWNER)), _Start())))["groupid"] + + h.cfg.auth_tokens = {"tok-alice": OWNER, "tok-bob": "bob"} + sr = _Start() + getattr(h, route)(_env(token="tok-bob"), sr, groupid=gid) + assert sr.status.startswith("404"), sr.status + + +@pytest.mark.parametrize("route", ["close_group", "cancel_group"]) +def test_group_mutation_works_for_the_owner(tmp_path, route): + h, _, _, _ = _api(tmp_path) + tok = _as(h, OWNER) + gid = json.loads(b"".join( + h.create_group(_env({"prompts": ["p1"]}, token=tok), _Start())))["groupid"] + sr = _Start() + getattr(h, route)(_env(token=tok), sr, groupid=gid) + assert sr.status.startswith("200"), sr.status + + +@pytest.mark.parametrize("route", ["close_group", "cancel_group"]) +def test_single_mode_group_mutation_stays_open(tmp_path, route): + """Phase 2's behaviour is unchanged: no tenants, nothing to authorize.""" + h, _, _, _ = _api(tmp_path, multi=False) + gid, _created = h.groups.create(label="b", expected=1) + sr = _Start() + getattr(h, route)(_env(), sr, groupid=gid) + assert sr.status.startswith("200"), sr.status + + +# ---- 4. `agent` was outside SIGNED_ATTRS --------------------------------- + +SEED = bytes.fromhex( + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") + + +def test_agent_is_signed(tmp_path): + """Regression. `ce_agent` selects the AgentSpec that supplies `--permission-mode` + and the tool allowlists — §5.1 calls that "the sandbox". Outside `SIGNED_ATTRS`, a + forged value widened the sandbox AND THE SIGNATURE STILL VERIFIED, which makes the + signed configuration worse than the unsigned one.""" + assert ce.EXT_AGENT in S.SIGNED_ATTRS + + corr = "brave-otter-4718" + event = ce.new_event(type=ce.TYPE_REQUEST, source="rossoctl://eventbridge/test", + datacontenttype="application/json", + correlationid=corr, sessionuuid=ce.session_uuid(corr), + mode="start", agent="triager", + data={"prompt": "hi"}) + S.sign_into(event, SEED, "eb-01") + ks = keyset.KeySet({"eb-01": S.public_key(SEED)}) + assert S.verify_with_keyset(event, ks)[0] + + # The attack: name a spec with a wider policy. + event.attrs[ce.EXT_AGENT] = "unsandboxed" + assert not S.verify_with_keyset(event, ks)[0] + + +def test_removing_the_agent_attribute_also_invalidates(tmp_path): + """Stripping it must not be a way around covering it — that would fall back to + `ER_AGENT_NAME`, which is a different policy than the caller asked for.""" + corr = "brave-otter-4718" + event = ce.new_event(type=ce.TYPE_REQUEST, source="s", + datacontenttype="application/json", + correlationid=corr, sessionuuid=ce.session_uuid(corr), + mode="start", agent="triager", data={"prompt": "hi"}) + S.sign_into(event, SEED, "eb-01") + del event.attrs[ce.EXT_AGENT] + assert not S.verify_with_keyset( + event, keyset.KeySet({"eb-01": S.public_key(SEED)}))[0] + + +# ---- 5. userkey reached a path join unvalidated --------------------------- + +@pytest.mark.parametrize("hostile", [ + "../users/gh-victim-0bad0bad", + "../../../../tmp/pwned", + "..", + "gh-alice-a1b2c3d4/../../etc", + "/absolute/path", + "gh-alice-A1B2C3D4", # upper-case hex: not a key this module produces + "GH-alice-a1b2c3d4", +]) +def test_a_hostile_userkey_cannot_escape_the_users_directory(tmp_path, hostile): + """Regression. The key arrives from an inbound Kafka header and + `Store.__init__` calls `mkdir(parents=True)`, so `..` created and wrote to a + directory in another tenant's store — or outside the tree entirely.""" + root = tmp_path / "eventbridge" + r = StoreRegistry(root, multi=True) + resolved = r._dir_for(hostile).resolve() + assert resolved == (root / "shared").resolve(), \ + f"{hostile!r} resolved outside shared/: {resolved}" + + +def test_a_hostile_userkey_is_counted_as_unattributed(tmp_path): + """It is truthy, so a bare `not userkey` check would let it past uncounted. The + forgery has to be visible, which is the counter's whole purpose.""" + r = StoreRegistry(tmp_path, multi=True) + r.for_event(ce.CloudEvent(attrs={ce.EXT_USERKEY: "../../tmp/pwned"})) + assert r.unattributed == 1 + + +def test_a_hostile_userkey_shares_the_shared_cache_entry(tmp_path): + """`_cache_key` must agree with `_dir_for`, or an invalid key gets its own entry + whose Store points at `shared/` — two entries, one directory, two connections.""" + r = StoreRegistry(tmp_path, multi=True) + assert r.for_userkey("../../tmp/pwned") is r.for_userkey(None) + assert r.open_count == 1 + + +def test_the_validator_round_trips_over_the_adversarial_corpus(): + """Anything `userkey()` produces must pass `is_valid_userkey`, or a legitimate + tenant would be silently demoted to `shared/`.""" + corpus = [("github", "MrSabath"), ("github", "a"), ("oidc", "a.b@x.com"), + ("oidc", "very.long.local.part.that.exceeds.the.budget@example.com"), + ("static", "...."), ("static", "@@@"), ("static", "ünïcødé"), + ("static", "-leading-and-trailing-"), ("unknown", "someone")] + for issuer, userid in corpus: + key = tenancy.userkey(issuer, userid) + assert tenancy.is_valid_userkey(key), f"{issuer}/{userid} -> {key}" + + +# ---- the "known but unowned" /continue 500 ------------------------------- + +def test_a_shared_tier_continue_is_refused_not_a_500(tmp_path): + """Regression. A correlation KNOWN to the index with a NULL owner gave + `owner=None, unresolved=None`, so the handler proceeded, wrote a prompt row, and then + `TopicSet.requests(None)` raised — a WSGI 500 with an orphan row already committed, + so the conversation showed a turn that was never submitted.""" + h, producer, owners, stores = _api(tmp_path, real_producer=True) + corr = h.minter.mint(userkey=None) # shared tier + stores.for_userkey(None).upsert_session(corr, ce.session_uuid(corr), "/w", "hi") + + sr = _Start() + h.continue_agent(_env({"prompt": "go on"}), sr, correlationid=corr) + assert sr.status.startswith(("404", "503")), sr.status + assert stores.for_userkey(None).get_prompts(corr) == [], "orphan prompt row written" + + +def test_a_shared_tier_correlation_is_still_readable(tmp_path): + """Reading and publishing have different requirements: its data lives in `shared/` + and must stay readable, even though a turn cannot be published for it.""" + h, _, _, stores = _api(tmp_path) + corr = h.minter.mint(userkey=None) + stores.for_userkey(None).upsert_session(corr, ce.session_uuid(corr), "/w", "hi") + assert h._store_of(corr) is not None + assert h._publishable_owner(corr)[1] is not None + + +# ---- the agent-name boundary check --------------------------------------- + +@pytest.mark.parametrize("bad", ["../../etc", "Triager", "a/b", {"x": 1}, ["a"], + "x" * 64]) +def test_a_bad_agent_name_is_400_not_an_async_error_event(tmp_path, bad): + """`202 Accepted` followed minutes later by a `phase=error` event the submitter may + never look at is the wrong answer to a typo in the call that made it.""" + h, producer, _, _ = _api(tmp_path) + sr = _Start() + h.start_agent(_env({"prompt": "hi", "agent": bad}, token=_as(h, OWNER)), sr) + assert sr.status.startswith("400"), sr.status + producer.publish_request.assert_not_called() + + +def test_a_good_agent_name_still_rides(tmp_path): + h, producer, _, _ = _api(tmp_path) + sr = _Start() + h.start_agent(_env({"prompt": "hi", "agent": "triager"}, + token=_as(h, OWNER)), sr) + assert sr.status.startswith("202"), sr.status + assert producer.publish_request.call_args[1]["agent"] == "triager" + + +# ---- max_turns: 0 -------------------------------------------------------- + +def test_max_turns_zero_is_not_silently_rewritten(): + """`or` made `max_turns: 0` become the spec's value, while the spec path REJECTS + zero — two paths disagreeing about the same nonsensical input.""" + from eventrunner.agentspec import AgentSpec + from eventrunner.config import Cfg as ErCfg + from eventrunner.runner import build_cmd + corr = "brave-otter-4718" + event = ce.CloudEvent(attrs={"correlationid": corr, + "sessionuuid": ce.session_uuid(corr), + "mode": "start"}) + cmd = build_cmd(ErCfg(), event, {"prompt": "x", "max_turns": 0}, + spec=AgentSpec(max_turns=8)) + assert cmd[cmd.index("--max-turns") + 1] == "0" + + +def test_an_absent_max_turns_still_takes_the_spec_value(): + from eventrunner.agentspec import AgentSpec + from eventrunner.config import Cfg as ErCfg + from eventrunner.runner import build_cmd + corr = "brave-otter-4718" + event = ce.CloudEvent(attrs={"correlationid": corr, + "sessionuuid": ce.session_uuid(corr), + "mode": "start"}) + cmd = build_cmd(ErCfg(), event, {"prompt": "x"}, spec=AgentSpec(max_turns=8)) + assert cmd[cmd.index("--max-turns") + 1] == "8" + + +# ---- config validation --------------------------------------------------- + +@pytest.mark.parametrize("prefix", ["a.b", "a_b", "has space", "a/b", "", "x" * 33]) +def test_a_bad_topic_prefix_is_refused_readably(monkeypatch, prefix): + """The prefix is the one topic-name component `_slug` never sees, and `.`/`_` + together collide in Kafka's JMX metric names.""" + from eventbridge import config as ebcfg + monkeypatch.setenv("EB_TOPIC_PREFIX", prefix) + with pytest.raises(SystemExit, match="EB_TOPIC_PREFIX"): + ebcfg.load() + + +def test_a_non_integer_max_open_stores_is_refused_readably(monkeypatch): + from eventbridge import config as ebcfg + monkeypatch.setenv("EB_MAX_OPEN_STORES", "lots") + with pytest.raises(SystemExit, match="EB_MAX_OPEN_STORES"): + ebcfg.load() + + +def test_a_broken_index_is_not_reported_as_an_exhausted_id_space(): + """Already covered in test_owner_index.py; asserted here too because it is one of + the review's findings and belongs in the same place as its siblings.""" + class Broken: + def exists(self, corr): + return False + + def claim(self, corr, userkey): + raise sqlite3.OperationalError("database or disk is full") + + with pytest.raises(sqlite3.OperationalError): + Minter(index=Broken()).mint(userkey=UK) diff --git a/eventing/tests/test_tenancy.py b/eventing/tests/test_tenancy.py index a7202739..947db03b 100644 --- a/eventing/tests/test_tenancy.py +++ b/eventing/tests/test_tenancy.py @@ -255,7 +255,9 @@ def test_ntfy_topic_contains_no_identifier(): """§4.1 is the whole design: the name travels to a phone.""" t = T.ntfy_topic("kev1", T.userkey("github", "mrsabath"), SECRET) assert "mrsabath" not in t - assert "gh" not in t.removeprefix("kev1-") + # No assertion about the issuer tag: base32's alphabet is `a-z2-7`, so "gh" appears + # in 26 random characters about 2.4% of the time. Testing that a 130-bit digest does + # not happen to contain a common bigram is a flaky test, not a property. def test_ntfy_topic_is_stable_and_per_user(): From 25ecafa8d8d703c6c3b8bbd34d499f26e47e0280 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Sat, 3 Oct 2026 20:30:11 -0400 Subject: [PATCH 10/14] fix(eventing): the four remaining #883 suggestions, and tombstone deletion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round two of the review, covering what was deferred from 6ebdd80. **T22 now checks the direction that enforces the rule.** `CONFIGMAP_VARS ⊆ found` catches a stale entry and nothing else; the direction that catches a *new* variable classified as neither is `found ⊆ CONFIGMAP_VARS ∪ SECRET_PATH_VARS`, which is the case the test exists for. Scoped to Phase 3's variables — the 36 that pre-date §8.3 are listed explicitly, with `test_the_baseline_list_is_still_accurate` keeping that list honest rather than letting it silently widen the exemption. Verified by deliberate failure, as asked: injecting an unclassified `e("EB_UNCLASSIFIED_X")` read into `config.py` turns the test red with the variable named. Also replaced the hardcoded ten-space indentation in the inlined-secret check with `\s*`, matching the precedent elsewhere in the same file — the old pattern would miss the same variable inlined in an initContainer, a sidecar or a deeper patch. **`RequestsMirror` and `NtfyPublisher` now take `stores=`.** Both knew the per-tenant topic names and still wrote to the shared store, which does look like the sweep in 77807b6 not reaching them. The mirror back-filled every prompt into `shared/`, so a correlation whose request the bridge did not originate showed a BLANK PROMPT on its owner's page — the exact gap the mirror exists to close. ntfy's `_last_prompt` and `_last_assistant_text` read only the shared store, so a tenant's notification arrived without the prompt or the reply, which §4.1 says is the content that lets a phone that cannot reach EventBridge still see the result. Its group-existence check is routed too. **A handler→topic integration test**, which the review argued was a better investment than more unit tests on the pieces. It is: faking at `KafkaProducer` rather than at `Producer` means the real handler, the real `Producer` and the real `TopicSet` all run, and the assertion is *which topic the bytes went to*. Confirmed by reverting each fix that it catches both defects it was written for — the `create_group` missing `userkey` and the `/continue` 500 on an unowned correlation. That test immediately earned itself twice over: the mutation run surfaced that `_publish_started`/`maybe_complete` swallow `ValueError` from `TopicSet` into a log line, so a caller forgetting a `userkey` produced a group that silently never announced itself — the same shape as the `TypeError` already guarded there. Both guards now cover both. **Deletion tombstones rather than freeing the id.** This is the one I said needed thought rather than a quick answer, and the answer is that reuse is unsafe. Verified: `forget()` used to `DELETE` the row, `exists()` is the `Minter`'s only uniqueness check, and `sessionuuid = uuid5(NAMESPACE, correlationid)` is unsalted by design — so a reissued `correlationid` derives *the same session uuid*, and a `claude` transcript left on a runner's volume becomes resumable by the new correlation. That is one user's conversation continuing inside somebody else's agent. So `forget()`/`forget_tenant()` keep the row, set `deleted_utc` and clear `userkey` — the id is reserved forever, and the tombstone discloses nothing about whose it was. The alternative is guaranteeing every transcript keyed on that uuid is purged everywhere, including volumes this process does not own, which is not a guarantee EventBridge can make. Cost is one short row per deleted correlation. Migrated with an `ALTER TABLE`, the same idiom `store.py` uses for `prompts.submitter`, and a test covers an index created before the column existed. The reviewer was also right that the docstring contradicted the code: it claimed "freeing the id for reuse is deliberate" about a `DELETE` nobody had reasoned about. Design doc: §6.1a records the tombstone decision and why, since the index is what enforces it. Tests: 855 passing (+13), ruff clean. Still outstanding: the four phase documents (`IMPLEMENTATION_REPORT3.md`, `README_PHASE3.md`, and the two owed Phase 2 documents). Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/agentdocs/DESIGN_PHASE3.md | 11 + eventing/eventbridge/__main__.py | 6 +- eventing/eventbridge/group_service.py | 34 ++- eventing/eventbridge/kafka_requests_mirror.py | 13 +- eventing/eventbridge/ntfy.py | 25 +- eventing/eventbridge/owner_index.py | 64 ++++- eventing/tests/test_manifests.py | 103 ++++++- eventing/tests/test_owner_index.py | 43 ++- eventing/tests/test_topic_routing_e2e.py | 258 ++++++++++++++++++ 9 files changed, 506 insertions(+), 51 deletions(-) create mode 100644 eventing/tests/test_topic_routing_e2e.py diff --git a/eventing/agentdocs/DESIGN_PHASE3.md b/eventing/agentdocs/DESIGN_PHASE3.md index d8bfac1d..c7d048e1 100644 --- a/eventing/agentdocs/DESIGN_PHASE3.md +++ b/eventing/agentdocs/DESIGN_PHASE3.md @@ -1499,6 +1499,17 @@ store can be chosen. It holds nothing but the mapping. that substitute implements. Reachable only from test doubles — `__main__` always builds a real `Minter` with an index — but recorded here because §2.6 is where the guarantee is claimed. +- **Deletion tombstones; it never frees an id for reuse.** This is a §6.5 decision that + belongs here because the index is what enforces it. `forget()` clears the owner and sets + `deleted_utc`, keeping the row, because `exists()` is the `Minter`'s only uniqueness + check and **`sessionuuid` is unsalted**: a reissued `correlationid` derives *the same + session uuid* as the deleted one. A `claude` transcript left on a runner's volume, or a + checkpoint that outlived the delete, would then be resumable by the new correlation — + one user's conversation continuing inside somebody else's agent. The alternative is + guaranteeing every transcript keyed on that uuid is purged everywhere, including volumes + EventBridge does not own, which is not a guarantee this component can make. The cost is + one short row per deleted correlation. The owner is cleared so a tombstone discloses + nothing about whose the correlation was, which matters on a deletion path. The alternative is one database with a `userkey` column and a `WHERE` clause on every query. It is cheaper — one connection pair, one startup, no file-descriptor arithmetic — diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index 0ceae68c..ec97d6f2 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -163,7 +163,8 @@ def main() -> int: topics=topics) groups = GroupService(cfg, store, producer, minter, stores=stores) - ntfy = NtfyPublisher(cfg.ntfy, cfg.public_base_url, store=store) + ntfy = NtfyPublisher(cfg.ntfy, cfg.public_base_url, store=store, + stores=stores if topics.multi else None) if cfg.ntfy.enabled and cfg.ntfy.topic: ntfy.start() @@ -181,7 +182,8 @@ def main() -> int: # Back-fill prompts from the requests topic — also gives us prompt visibility # for correlations we didn't originate ourselves. requests_mirror = RequestsMirror(cfg.kafka_bootstrap, cfg.request_topic, store, - topics=topics) + topics=topics, + stores=stores if topics.multi else None) requests_mirror.start() # §21.2: rebuild group history from the responses topic. The live consumer above diff --git a/eventing/eventbridge/group_service.py b/eventing/eventbridge/group_service.py index 77724512..1c235bb0 100644 --- a/eventing/eventbridge/group_service.py +++ b/eventing/eventbridge/group_service.py @@ -80,13 +80,20 @@ def _publish_started(self, groupid, label, expected, min_success, deadline, data={"label": label, "expected": expected, "min_success": min_success, "deadline_utc": deadline}, userkey=userkey) - except TypeError: - # NOT swallowed. A TypeError here is a signature mismatch between this call - # and the producer — a programming error, not a broker being down — and the - # broad `except` below would turn it into a group that silently never - # announces itself. Phase 3 hit exactly that while adding `userkey`: every - # group event stopped being published and the only symptom was a batch that - # never completed. Loud is correct for this one. + except (TypeError, ValueError): + # NOT swallowed. Both are programming errors rather than a broker being down, + # and the broad `except` below would turn either into a group that silently + # never announces itself: + # + # * TypeError — a signature mismatch between this call and the producer. + # Phase 3 hit exactly that while adding `userkey`: every group event + # stopped publishing and the only symptom was a batch that never completed. + # * ValueError — `TopicSet` refusing to name a topic without a userkey in + # multi mode. That means a caller failed to pass one, which is the + # `create_group` defect the #883 review found; logging it would hide the + # cause and leave the same silent never-completing batch. + # + # Loud is correct for both. A genuine broker failure is still handled below. raise except Exception as e: # noqa: BLE001 - the group still exists in the store _log(f"could not publish group.started for {groupid}: {e!r}") @@ -210,12 +217,13 @@ def maybe_complete(self, groupid: str, *, userkey: str | None = None) -> bool: "elapsed_s": p["elapsed_s"], "elapsed": p["elapsed"], "slowest_member_s": _slowest(members)}, userkey=userkey) - except TypeError: - # See `_publish_started`: a signature mismatch is a programming error and - # must not be downgraded into "this batch never announced it finished". The - # guard is repeated at both call sites rather than factored into a helper, - # because what it protects is the argument list of *this* call, and a wrapper - # would move the mismatch one frame away from the arguments that caused it. + except (TypeError, ValueError): + # See `_publish_started`: a signature mismatch or a missing userkey is a + # programming error and must not be downgraded into "this batch never + # announced it finished". The guard is repeated at both call sites rather than + # factored into a helper, because what it protects is the argument list of + # *this* call, and a wrapper would move the mismatch one frame away from the + # arguments that caused it. raise except Exception as e: # noqa: BLE001 _log(f"could not publish group.completed for {groupid}: {e!r}") diff --git a/eventing/eventbridge/kafka_requests_mirror.py b/eventing/eventbridge/kafka_requests_mirror.py index a319f74f..984a37ab 100644 --- a/eventing/eventbridge/kafka_requests_mirror.py +++ b/eventing/eventbridge/kafka_requests_mirror.py @@ -26,7 +26,8 @@ class RequestsMirror(threading.Thread): def __init__(self, bootstrap: str, request_topic: str, store: Store, group_id: str | None = None, - topics: tenancy.TopicSet | None = None) -> None: + topics: tenancy.TopicSet | None = None, + stores=None) -> None: super().__init__(daemon=True, name="kafka-requests-mirror") self._bootstrap_servers = bootstrap # §3.2: "RequestsMirror needs the same treatment" as the responses consumer — @@ -37,6 +38,12 @@ def __init__(self, bootstrap: str, request_topic: str, store: Store, "", request_topic=request_topic, response_topic="") self._topic = self._topics.requests() if not self._topics.multi else request_topic self._store = store + # §6.1 — route each back-filled prompt into the store its own `ce_userkey` names. + # Without this the mirror knows the per-tenant TOPIC names and still writes every + # prompt into the shared store, so a correlation whose request the bridge did not + # originate shows a BLANK PROMPT on its owner's page — which is precisely the gap + # this mirror exists to close. + self._stores = stores # Per-process group id so every EB start re-scans the whole topic. # `backfill_prompt_if_missing()` makes re-inserts a no-op, so this # is idempotent and cheap. A fixed group id would commit offsets @@ -81,7 +88,9 @@ def run(self) -> None: if not prompt: continue submitted = evt.attrs.get("time") - self._store.backfill_prompt_if_missing( + target = (self._stores.for_event(evt) if self._stores + else self._store) + target.backfill_prompt_if_missing( correlationid=corr, mode=mode, prompt=prompt, submitted_utc=submitted, ) diff --git a/eventing/eventbridge/ntfy.py b/eventing/eventbridge/ntfy.py index c74d8ab5..3768ad63 100644 --- a/eventing/eventbridge/ntfy.py +++ b/eventing/eventbridge/ntfy.py @@ -145,14 +145,27 @@ def compose_title(event: dict[str, Any]) -> str: # --- Publisher thread ------------------------------------------------------ class NtfyPublisher(threading.Thread): - def __init__(self, cfg: NtfyCfg, public_base_url: str, store=None) -> None: + def __init__(self, cfg: NtfyCfg, public_base_url: str, store=None, + stores=None) -> None: super().__init__(daemon=True, name="ntfy-publisher") self._cfg = cfg self._base = public_base_url.rstrip("/") self._store = store + # §6.1 — resolve the store per event, from its own `ce_userkey`. Without this, + # `_last_prompt` and `_last_assistant_text` read only the shared store, so in + # multi-tenant mode a notification for a tenant's correlation arrives with no + # prompt and no reply — exactly the content §4.1 says the body carries so a phone + # that cannot reach EventBridge still sees the result. + self._stores = stores self._q: queue.Queue[dict[str, Any] | None] = queue.Queue() self._stopping = threading.Event() + def _store_for(self, event: dict[str, Any]): + """The store holding this event's prompt and replies.""" + if self._stores is None: + return self._store + return self._stores.for_event(event) + def submit(self, event: dict[str, Any]) -> None: if not self._cfg.enabled or not self._cfg.topic: return @@ -186,15 +199,17 @@ def _suppressed_member(self, event: dict[str, Any]) -> bool: for the whole life of the group. """ groupid = event.get("groupid") - if not groupid or self._store is None: + store = self._store_for(event) + if not groupid or store is None: return False if self._cfg.group_notify_errors and event.get("phase") == "error": return False try: # An unknown groupid is treated as ungrouped: if we have no group row we # will never send group notifications either, so suppressing would mean - # sending nothing at all. - return self._store.get_group(groupid) is not None + # sending nothing at all. Routed by the event's own userkey for the same + # reason as the body lookup — the group row is in its owner's store. + return store.get_group(groupid) is not None except Exception: # noqa: BLE001 return False @@ -265,7 +280,7 @@ def _publish(self, event: dict[str, Any]) -> None: corr = event.get("correlationid", "?") phase = event.get("phase", "?") - body_msg = compose_body(event, self._store) + body_msg = compose_body(event, self._store_for(event)) title = compose_title(event) tags = ["robot", phase] if phase == "error": diff --git a/eventing/eventbridge/owner_index.py b/eventing/eventbridge/owner_index.py index a7559d4e..7374f8ff 100644 --- a/eventing/eventbridge/owner_index.py +++ b/eventing/eventbridge/owner_index.py @@ -51,7 +51,10 @@ -- `ORDER BY created_utc DESC LIMIT n` returned an arbitrary subset -- and -- `submit_members` publishes a whole 100-member batch in a loop, which is exactly -- that case. Ordered reads also tie-break on `rowid`, which is free and total. - created_utc TEXT NOT NULL + created_utc TEXT NOT NULL, + -- A TOMBSTONE marks a correlation whose data is gone but whose id must never be + -- reissued. See `forget`: reuse is unsafe because `sessionuuid` is unsalted. + deleted_utc TEXT ); CREATE INDEX IF NOT EXISTS idx_owner_userkey ON correlation_owner(userkey); """ @@ -83,6 +86,13 @@ def __init__(self, root: str | pathlib.Path) -> None: self._c.execute("PRAGMA journal_mode=WAL") self._c.execute("PRAGMA synchronous=NORMAL") self._c.executescript(_SCHEMA) + # `CREATE TABLE IF NOT EXISTS` is a no-op on an existing file, so a column added + # after one was created needs an explicit ALTER — the same idiom `store.py` uses + # for `prompts.submitter`. Idempotent: SQLite raises when it already exists. + try: + self._c.execute("ALTER TABLE correlation_owner ADD COLUMN deleted_utc TEXT") + except sqlite3.OperationalError: + pass # EventBridge is single-replica (Phase 1 §8.6) but multi-threaded: the HTTP # worker pool and the responses consumer both reach this. `isolation_level=None` # means autocommit, so the lock is what makes claim's check-then-insert atomic. @@ -193,7 +203,8 @@ def correlations_for(self, userkey: str | None, limit: int = 1000) -> list[str]: """Every correlation a tenant owns, newest first. Backs the group-list filter.""" if userkey is None: rows = self._c.execute( - "SELECT correlationid FROM correlation_owner WHERE userkey IS NULL " + "SELECT correlationid FROM correlation_owner " + "WHERE userkey IS NULL AND deleted_utc IS NULL " "ORDER BY created_utc DESC, rowid DESC LIMIT ?", (limit,)).fetchall() else: rows = self._c.execute( @@ -210,24 +221,53 @@ def userkeys(self) -> list[str]: return [r[0] for r in rows] def forget(self, correlationid: str) -> None: - """Drop one mapping. For §6.5's deletion path. - - Note what this does NOT do: freeing the id for reuse is deliberate, since the - uniqueness guarantee only has to hold for correlations that still exist. The - caller is responsible for having removed the data first — this is the last step, - not the first. + """Tombstone one mapping: clear the owner, keep the id reserved. §6.5. + + **The id is never reissued, and that is the whole point.** An earlier version + `DELETE`d the row and its docstring claimed "freeing the id for reuse is + deliberate" — which was both wrong about the code's intent and unsafe. `exists()` + is the `Minter`'s only uniqueness check, so a deleted row frees the id; and + because `sessionuuid = uuid5(NAMESPACE, correlationid)` is **unsalted** by design + (§2.6), a reissued `correlationid` derives *the same session uuid* as the deleted + one. A `claude` transcript left on a runner's volume, or a checkpoint that + outlived the delete, is then resumable by the new correlation — one user's + conversation continuing inside somebody else's agent. + + Tombstoning costs one short row per deleted correlation and removes the whole + class. The alternative — guaranteeing every transcript keyed on that uuid is + purged everywhere, including volumes this process does not own — is not a + guarantee this component can make. + + The owner is cleared so a tombstone leaks nothing about who the correlation + belonged to, which matters because §6.5 is a deletion path. """ with self._lock: - self._c.execute("DELETE FROM correlation_owner WHERE correlationid=?", - (correlationid,)) + self._c.execute( + "UPDATE correlation_owner " + "SET userkey=NULL, deleted_utc=strftime('%Y-%m-%dT%H:%M:%fZ','now') " + "WHERE correlationid=?", (correlationid,)) def forget_tenant(self, userkey: str) -> int: - """Drop every mapping for one tenant, returning how many. §6.5's `rm -rf` half.""" + """Tombstone every mapping for one tenant, returning how many. §6.5's `rm -rf`. + + Same reasoning as `forget`: the rows stay as tombstones so none of the tenant's + ids can be reissued, and the owner is cleared so the tombstones say nothing about + whose they were. + """ with self._lock: cur = self._c.execute( - "DELETE FROM correlation_owner WHERE userkey=?", (userkey,)) + "UPDATE correlation_owner " + "SET userkey=NULL, deleted_utc=strftime('%Y-%m-%dT%H:%M:%fZ','now') " + "WHERE userkey=?", (userkey,)) return cur.rowcount or 0 + def is_tombstoned(self, correlationid: str) -> bool: + """Whether this id is reserved by a deleted correlation.""" + row = self._c.execute( + "SELECT deleted_utc FROM correlation_owner WHERE correlationid=?", + (correlationid,)).fetchone() + return bool(row and row[0]) + def count(self) -> int: return int(self._c.execute( "SELECT COUNT(*) FROM correlation_owner").fetchone()[0]) diff --git a/eventing/tests/test_manifests.py b/eventing/tests/test_manifests.py index 9b24d314..75046fd2 100644 --- a/eventing/tests/test_manifests.py +++ b/eventing/tests/test_manifests.py @@ -205,32 +205,105 @@ def test_no_phase3_secret_is_inlined_into_a_manifest(): for var in SECRET_PATH_VARS: # Referencing the variable is fine; giving it a literal sibling value is # not. A `secretKeyRef` has no `value:` on the same key. - for bad in (f"{var}:", f"name: {var}\n value:"): - assert bad not in rendered, \ - f"{overlay} appears to inline {var} rather than mounting a Secret" + # + # `\s*` rather than hardcoded indentation: ten spaces matches the demo + # overlay's container `env:` today, but the same variable inlined in an + # initContainer, a sidecar or a deeper patch would pass silently. + # test_manifests.py's own earlier tests use `\s*` for this reason. + inlined = re.search(rf"name:\s*{re.escape(var)}\s*\n\s*value:", rendered) + assert f"{var}:" not in rendered and not inlined, \ + f"{overlay} appears to inline {var} rather than mounting a Secret" + + +#: Variables that pre-date §8.3 and are therefore out of its scope. Retro-classifying all +#: 36 is a separate change from making the rule hold for anything new, so they are listed +#: here explicitly — and `test_the_baseline_list_is_still_accurate` keeps the list honest +#: by checking it against the config files rather than trusting it. +PRE_PHASE3_VARS = ( + "EB_ALLOWED_USERS", "EB_AUTH_TOKENS", "EB_GITHUB_CACHE_TTL_S", + "EB_GITHUB_CLIENT_ID", "EB_GROUP_DEADLINE_S", "EB_HTTP_ADDR", + "EB_REQUIRE_RESPONSE_SIGNATURE", "EB_SIGNING_KEY_PATH", "EB_SIGNING_KID", + "EB_TRANSCRIPT_MAX_BYTES", "EB_VERIFY_KEYSET_PATH", "EB_WORKERS", + "ER_COMMIT_AFTER_TERMINAL", "ER_CONSUMER_GROUP", "ER_DEDUPE_FINAL_TEXT", + "ER_DRAIN_TIMEOUT_S", "ER_EMIT_SYSTEM_HOOKS", "ER_EVENTBRIDGE_URL", + "ER_FINAL_COMMIT_WAIT_S", "ER_HEARTBEAT_MAX_AGE_S", "ER_HEARTBEAT_PATH", + "ER_INCLUDE_RAW", "ER_KAFKA_RETRY_INITIAL_S", "ER_KAFKA_RETRY_MAX_S", + "ER_MAX_CONCURRENT", "ER_MAX_REQUEST_AGE_S", "ER_MOCK_CLAUDE", + "ER_MOCK_DELAY_MAX_S", "ER_MOCK_DELAY_MIN_S", "ER_REQUIRE_SIGNATURE", + "ER_SIGNING_KEY_PATH", "ER_SIGNING_KID", "ER_TRANSCRIPT_MAX_BYTES", + "ER_TRANSCRIPT_SYNC", "ER_VERIFY_KEYSET_PATH", "ER_VERIFY_KEY_PATH", +) + + +def _vars_read(path: pathlib.Path) -> set[str]: + """Every `EB_*`/`ER_*` name a config file reads via the `e("VAR")` idiom.""" + return set(re.findall(r'e\("((?:EB|ER)_[A-Z0-9_]+)"', path.read_text())) + + +def test_the_baseline_list_is_still_accurate(): + """A stale `PRE_PHASE3_VARS` entry silently widens the exemption above. + + If a pre-Phase-3 variable is removed from `config.py` and left here, it keeps + exempting a name nobody reads — harmless — but if one is *renamed*, the new name + becomes unclassified and the check above catches it. This asserts the simpler + property: every name here is still read somewhere. + """ + found = (_vars_read(ROOT / "eventbridge" / "config.py") + | _vars_read(ROOT / "eventrunner" / "config.py")) + stale = set(PRE_PHASE3_VARS) - found + assert not stale, ( + f"{sorted(stale)} are in PRE_PHASE3_VARS but no config.py reads them; " + f"remove them so the §8.3 exemption stays as narrow as it should be") def test_every_phase3_variable_is_classified(): - """A variable that is in neither list is one nobody decided about. + """A variable in neither list is one nobody answered the §8.3 question about. - §8.3's rule only works if adding a variable forces the Secret-vs-ConfigMap question, - so this asserts the two lists together cover what `config.py` actually reads. + **Both directions matter, and only one of them enforces the rule.** + `CONFIGMAP_VARS ⊆ found` catches a stale entry here. The direction that catches a NEW + variable classified as neither is `found ⊆ CONFIGMAP_VARS ∪ SECRET_PATH_VARS` — which + is the case this test exists for, and which an earlier version of it did not check. """ - import re as _re - sources = [(ROOT / "eventbridge" / "config.py").read_text(), - (ROOT / "eventrunner" / "config.py").read_text()] - found = set() - for src in sources: - found |= set(_re.findall(r'e\("((?:EB|ER)_[A-Z0-9_]+)"', src)) + eb = ROOT / "eventbridge" / "config.py" + er = ROOT / "eventrunner" / "config.py" + found = _vars_read(eb) | _vars_read(er) + + # The sanity check that this test is looking at the right thing at all. If the + # `e("VAR")` idiom ever changes, `found` goes empty and every assertion below passes + # vacuously — so fail loudly here instead. + assert "EB_TENANCY_MODE" in found and len(found) > 10, \ + 'config.py does not look like it was parsed; did the `e("VAR")` idiom change?' + # Every ConfigMap-side variable must actually be read by the code — a stale entry # here would silently stop checking anything. The Secret-side ones are deliberately # exempt: they are declared ahead of the tasks that introduce them (see the note on # SECRET_PATH_VARS), and requiring them now would just force a placeholder read. for var in CONFIGMAP_VARS: assert var in found, f"{var} is listed here but no config.py reads it" - # The sanity check that this test is looking at the right thing at all. - assert "EB_TENANCY_MODE" in found and len(found) > 10, \ - "config.py does not look like it was parsed; did the `e(\"VAR\")` idiom change?" + + # THE DIRECTION THAT ENFORCES §8.3. Scoped to variables Phase 3 introduced: the + # pre-Phase-3 ones pre-date the rule, and retro-classifying 36 of them is a separate + # change from making the rule hold for new ones. + baseline = set(PRE_PHASE3_VARS) + unclassified = found - set(CONFIGMAP_VARS) - set(SECRET_PATH_VARS) - baseline + assert not unclassified, ( + f"{sorted(unclassified)} are read by config.py but classified as neither " + f"ConfigMap nor Secret. DESIGN_PHASE3.md §8.3 requires the question be answered: " + f"add each to CONFIGMAP_VARS or SECRET_PATH_VARS in this file.") + + +def test_the_classification_check_can_actually_fail(): + """The deliberate-failure check the review asked for. + + A test whose failure mode has never been observed is a test nobody should trust — + `test_every_phase3_variable_is_classified` replaced an assertion that could not fail + for its stated reason, so this one proves the replacement does. + """ + baseline = set(PRE_PHASE3_VARS) + pretend_found = {"EB_TENANCY_MODE", "EB_SOMETHING_NEW_X"} | baseline + unclassified = pretend_found - set(CONFIGMAP_VARS) - set(SECRET_PATH_VARS) - baseline + assert unclassified == {"EB_SOMETHING_NEW_X"}, \ + "an unclassified new variable must be detected" @needs_kubectl diff --git a/eventing/tests/test_owner_index.py b/eventing/tests/test_owner_index.py index ac9df8b2..89712031 100644 --- a/eventing/tests/test_owner_index.py +++ b/eventing/tests/test_owner_index.py @@ -107,10 +107,28 @@ def test_userkeys_excludes_the_null_owner(idx): # ---- deletion (§6.5 groundwork) ------------------------------------------ -def test_forget_one(idx): +def test_forget_tombstones_rather_than_freeing_the_id(idx): + """The id must NEVER be reissued. `sessionuuid` is unsalted (§2.6), so a reused + `correlationid` derives the same session uuid as the deleted one — and a transcript + left on a runner's volume is then resumable by the new correlation, continuing one + user's conversation inside somebody else's agent.""" idx.claim(CORR, A) idx.forget(CORR) - assert not idx.exists(CORR) + assert idx.exists(CORR), "the id was freed for reuse" + assert idx.is_tombstoned(CORR) + # The owner is cleared: a tombstone must not say whose the correlation was. + assert idx.owner_of(CORR) == (None, True) + assert CORR not in idx.correlations_for(A) + assert CORR not in idx.correlations_for(None), "a tombstone is not a live shared row" + + +def test_the_minter_will_not_reissue_a_tombstoned_id(idx): + """The property the tombstone exists for, through the Minter rather than the index.""" + from eventbridge.owner_index import Collision + idx.claim(CORR, A) + idx.forget(CORR) + with pytest.raises(Collision): + idx.claim(CORR, B) def test_forget_tenant_returns_a_count_and_leaves_others(idx): @@ -119,6 +137,27 @@ def test_forget_tenant_returns_a_count_and_leaves_others(idx): idx.claim("brave-otter-0003", B) assert idx.forget_tenant(A) == 2 assert idx.correlations_for(B) == ["brave-otter-0003"] + # A's ids stay reserved, and stop being attributed to A. + assert idx.correlations_for(A) == [] + for c in ("brave-otter-0001", "brave-otter-0002"): + assert idx.exists(c) and idx.is_tombstoned(c) + + +def test_the_deleted_column_is_added_to_an_existing_index(tmp_path): + """An index created before tombstoning gains the column by ALTER, as `store.py` + does for `prompts.submitter`.""" + import sqlite3 + (tmp_path / "owners.sqlite").unlink(missing_ok=True) + c = sqlite3.connect(tmp_path / "owners.sqlite") + c.execute("CREATE TABLE correlation_owner (correlationid TEXT PRIMARY KEY, " + "userkey TEXT, created_utc TEXT NOT NULL)") + c.execute("INSERT INTO correlation_owner VALUES ('brave-otter-0001', 'gh-a-1', 'x')") + c.commit() + c.close() + idx = OwnerIndex(tmp_path) + assert idx.owner_of("brave-otter-0001") == ("gh-a-1", True) + idx.forget("brave-otter-0001") + assert idx.is_tombstoned("brave-otter-0001") # ---- persistence --------------------------------------------------------- diff --git a/eventing/tests/test_topic_routing_e2e.py b/eventing/tests/test_topic_routing_e2e.py new file mode 100644 index 00000000..23e8a67b --- /dev/null +++ b/eventing/tests/test_topic_routing_e2e.py @@ -0,0 +1,258 @@ +"""Handler → Producer → **topic string**, with a fake Kafka. DESIGN_PHASE3.md §3.1, §6.1. + +The #883 review identified this as the suite's structural blind spot, and it was right: +`create_group` shipping without a `userkey` and `/continue` 500-ing on an unowned +correlation both survived 217 new test functions, because every multi-mode test either +called `GroupService` directly with the argument the handler fails to pass, or replaced +`Producer` with a mock and asserted on its keyword arguments. + +So this file fakes **one layer lower** — at `KafkaProducer`, so the real `Producer`, the +real `TopicSet` and the real handler all run — and asserts the thing that actually +matters: *which topic did the bytes go to*. `test_topicset_wiring.py` covers `TopicSet` and +`Producer` in isolation; nothing covered the chain. + +The stated hazard in `test_continue_tenancy.py` is that "publishing to a shared topic would +run the turn on another tenant's runner, under that tenant's credential". These are the +assertions for that sentence. +""" +from __future__ import annotations + +import io +import json + +import pytest + +from eventbridge import auth, registry +from eventbridge.config import Cfg +from eventbridge.correlation import Minter +from eventbridge.group_service import GroupService +from eventbridge.handlers import Handlers +from eventbridge.kafka_out import Producer +from eventbridge.owner_index import OwnerIndex +from eventbridge.store_registry import StoreRegistry +from shared import ce, tenancy + +ALICE = "alice" +BOB = "bob" +UK_A = tenancy.userkey(auth.ISSUER_STATIC, ALICE) +UK_B = tenancy.userkey(auth.ISSUER_STATIC, BOB) + + +class FakeKafkaProducer: + """Records `(topic, headers, value)` for every send. The seam the suite was missing. + + Deliberately NOT a `Producer` stand-in: the point is that `Producer.publish_request` + and `TopicSet.requests` both execute for real, so a missing `userkey` at the handler + shows up here as the wrong topic rather than as an absent keyword argument. + """ + + def __init__(self, *a, **kw): + self.sent: list[dict] = [] + + def send(self, topic, *, key=None, value=None, headers=None): + self.sent.append({"topic": topic, "key": key, "value": value, + "headers": dict(headers or [])}) + + class _F: + def get(self, timeout=None): + return None + return _F() + + def flush(self, timeout=None): + pass + + def close(self, timeout=None): + pass + + # ---- helpers the tests read ---- + @property + def topics(self) -> list[str]: + return [s["topic"] for s in self.sent] + + def events(self): + return [ce.from_kafka_binary(list(s["headers"].items()), s["value"]) + for s in self.sent] + + +@pytest.fixture +def api(tmp_path, monkeypatch): + """A real Handlers + real Producer over a fake Kafka, in multi-tenant mode.""" + monkeypatch.setattr("eventbridge.kafka_out.KafkaProducer", FakeKafkaProducer) + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + cfg.tenancy_mode = tenancy.MULTI + cfg.topic_prefix = "kev1" + cfg.auth_tokens = {f"tok-{ALICE}": ALICE, f"tok-{BOB}": BOB} + root = tmp_path / "eventbridge" + owners = OwnerIndex(root) + stores = StoreRegistry(root, multi=True) + store = stores.for_userkey(None) + producer = Producer("b:9092", "requests", "rossoctl://eventbridge/test", + response_topic="responses", topics=cfg.topics) + minter = Minter(index=owners) + groups = GroupService(cfg, store, producer, minter, stores=stores) + reg = registry.parse(json.dumps({ + "version": 1, + "users": [{"issuer": auth.ISSUER_STATIC, "userid": u, "tier": "isolated"} + for u in (ALICE, BOB)]})) + h = Handlers(cfg, store, producer, minter, groups=groups, registry=reg, + stores=stores, owners=owners) + return h, producer._prod, owners, stores + + +class _Start: + def __init__(self): + self.status = None + + def __call__(self, status, headers): + self.status = status + + +def _post(body, token): + raw = json.dumps(body).encode() + return {"wsgi.input": io.BytesIO(raw), "CONTENT_LENGTH": str(len(raw)), + "CONTENT_TYPE": "application/json", "REQUEST_METHOD": "POST", + "QUERY_STRING": "", "HTTP_AUTHORIZATION": f"Bearer {token}"} + + +# ---- POST /v0/agents ----------------------------------------------------- + +def test_start_agent_publishes_to_the_callers_own_requests_topic(api): + h, kafka, _, _ = api + sr = _Start() + h.start_agent(_post({"prompt": "hi"}, f"tok-{ALICE}"), sr) + assert sr.status.startswith("202"), sr.status + assert kafka.topics == [f"kev1-{UK_A}-requests"] + + +def test_two_tenants_never_share_a_requests_topic(api): + """The property the whole phase is for, asserted on the wire.""" + h, kafka, _, _ = api + h.start_agent(_post({"prompt": "a"}, f"tok-{ALICE}"), _Start()) + h.start_agent(_post({"prompt": "b"}, f"tok-{BOB}"), _Start()) + assert kafka.topics == [f"kev1-{UK_A}-requests", f"kev1-{UK_B}-requests"] + assert len(set(kafka.topics)) == 2 + + +def test_the_published_event_carries_the_matching_userkey(api): + """The topic and the attribute have to agree, or the runner's response comes back + stamped for a different tenant than the one whose topic served it.""" + h, kafka, _, _ = api + h.start_agent(_post({"prompt": "hi"}, f"tok-{ALICE}"), _Start()) + evt = kafka.events()[0] + assert evt.attrs[ce.EXT_USERKEY] == UK_A + assert kafka.topics[0] == f"kev1-{UK_A}-requests" + + +def test_no_request_is_ever_published_to_the_single_mode_topic(api): + """`requests` is the single-tenant name. Seeing it in multi mode means a userkey was + dropped somewhere between the handler and `TopicSet`.""" + h, kafka, _, _ = api + h.start_agent(_post({"prompt": "a"}, f"tok-{ALICE}"), _Start()) + h.create_group(_post({"prompts": ["x", "y"]}, f"tok-{BOB}"), _Start()) + assert "requests" not in kafka.topics + assert "responses" not in kafka.topics + + +# ---- POST /v0/groups ----------------------------------------------------- + +def test_create_group_publishes_members_and_the_started_event_to_one_tenant(api): + """The `create_group` defect, caught at the layer that would have caught it. + + The group lifecycle event goes to the tenant's RESPONSES topic (§21.2: EventRunner + would try to execute anything on requests), and every member to their requests topic. + All of them belong to the same tenant. + """ + h, kafka, owners, stores = api + sr = _Start() + out = h.create_group(_post({"prompts": ["p1", "p2"]}, f"tok-{ALICE}"), sr) + assert sr.status.startswith("202"), sr.status + gid = json.loads(b"".join(out))["groupid"] + + assert kafka.topics.count(f"kev1-{UK_A}-requests") == 2 + assert f"kev1-{UK_A}-responses" in kafka.topics + assert all(UK_A in t for t in kafka.topics), kafka.topics + # And the store side agrees with the wire side. + assert stores.for_userkey(UK_A).get_group(gid) is not None + assert owners.owner_of(gid) == (UK_A, True) + + +def test_a_group_event_is_never_published_to_another_tenants_topic(api): + h, kafka, _, _ = api + h.create_group(_post({"prompts": ["p1"]}, f"tok-{ALICE}"), _Start()) + assert not [t for t in kafka.topics if UK_B in t] + + +# ---- POST /continue ------------------------------------------------------ + +def test_continue_publishes_to_the_owning_tenants_topic(api): + h, kafka, _, _ = api + out = h.start_agent(_post({"prompt": "hi"}, f"tok-{ALICE}"), _Start()) + corr = json.loads(b"".join(out))["correlationid"] + kafka.sent.clear() + + sr = _Start() + # Note the credential: BOB's. `/continue` is unauthenticated (Phase 2 §3.1's + # capability URL), so what decides the topic is the correlation's OWNER, not the + # caller — which is what stops a resume running under another tenant's credential. + h.continue_agent(_post({"prompt": "go on"}, f"tok-{BOB}"), sr, correlationid=corr) + assert sr.status.startswith("202"), sr.status + assert kafka.topics == [f"kev1-{UK_A}-requests"] + + +def test_continue_for_an_unowned_correlation_publishes_nothing(api): + """The `/continue` 500: it used to write a prompt row and then raise. Nothing may + reach Kafka, and nothing may be written.""" + h, kafka, owners, stores = api + corr = h.minter.mint(userkey=None) # shared tier: known, unowned + stores.for_userkey(None).upsert_session(corr, ce.session_uuid(corr), "/w", "hi") + kafka.sent.clear() + + sr = _Start() + h.continue_agent(_post({"prompt": "go on"}, f"tok-{ALICE}"), sr, correlationid=corr) + assert sr.status.startswith(("404", "503")), sr.status + assert kafka.sent == [] + assert stores.for_userkey(None).get_prompts(corr) == [] + + +def test_continue_for_an_unknown_correlation_publishes_nothing(api): + h, kafka, _, _ = api + sr = _Start() + h.continue_agent(_post({"prompt": "go on"}, f"tok-{ALICE}"), sr, + correlationid="brave-otter-0000") + assert sr.status.startswith("404"), sr.status + assert kafka.sent == [] + + +# ---- single-tenant mode -------------------------------------------------- + +@pytest.fixture +def single(tmp_path, monkeypatch): + monkeypatch.setattr("eventbridge.kafka_out.KafkaProducer", FakeKafkaProducer) + cfg = Cfg() + cfg.tmpdir = str(tmp_path) + root = tmp_path / "eventbridge" + stores = StoreRegistry(root, multi=False) + store = stores.for_userkey(None) + producer = Producer("b:9092", "requests", "rossoctl://eventbridge/test", + response_topic="responses", topics=cfg.topics) + minter = Minter(index=OwnerIndex(root)) + groups = GroupService(cfg, store, producer, minter) + return (Handlers(cfg, store, producer, minter, groups=groups), + producer._prod) + + +def test_single_mode_publishes_to_the_configured_topics(single): + """Phase 2's wire behaviour, asserted on the wire rather than inferred.""" + h, kafka = single + out = h.start_agent(_post({"prompt": "hi"}, "unused"), _Start()) + corr = json.loads(b"".join(out))["correlationid"] + h.continue_agent(_post({"prompt": "more"}, "unused"), _Start(), correlationid=corr) + h.create_group(_post({"prompts": ["p1"]}, "unused"), _Start()) + assert set(kafka.topics) == {"requests", "responses"} + + +def test_single_mode_puts_no_userkey_on_the_wire(single): + h, kafka = single + h.start_agent(_post({"prompt": "hi"}, "unused"), _Start()) + assert ce.EXT_USERKEY not in kafka.events()[0].attrs From 3a328ff4c60bcefd90b648e72d10a796fbe2ec4e Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Sat, 3 Oct 2026 21:04:00 -0400 Subject: [PATCH 11/14] =?UTF-8?q?docs(eventing):=20IMPLEMENTATION=5FREPORT?= =?UTF-8?q?3.md=20=E2=80=94=20what=20Phase=203=20built,=20measured?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asked for in the #883 review, and the argument for writing it now rather than later was the most persuasive one there: the findings are attributable to specific commits today and will not be in a month. 361 lines, following `IMPLEMENTATION_REPORT1.md`'s contract — measured figures rather than estimates, failures with root causes, and an explicit account of what is NOT verified. Phase 0 and Phase 1 each ship three documents; Phase 2 shipped none, and the gap shows in that there is nowhere to put the §6.1 split-consumer trap except inside a design document. This starts closing that for Phase 3. **Measured rather than asserted.** §6.1 estimates the per-tenant descriptor cost and derives a ceiling; measured it is 6.0 descriptors per open store (10 tenants, 60 fds), which puts ~680 tenants under this host's 4096 soft limit and ~170 under the 1024 the design assumed — so squarely inside its "150-200" once the limit is held constant. The ownership index costs 27.4 µs per `claim`, 1.7 µs per `exists`, and 270 ms to seed 10,000 ids. That last figure is the one that matters: it settles §2.6's objection to seeding, because the *one-time* backfill it appeared to rule out costs 270 ms once, while the *per-start* seeding it correctly rejects is what the index exists to avoid. **Seven deltas from the design, recorded as decisions rather than drift** — the single-tenant store location, eviction by reference-drop, tombstoned deletion, the single-mode `events`/`dead` names, `agent` joining `SIGNED_ATTRS`, the exported userkey validator, and prefix validation. **The findings, ordered by what they cost to find.** Three are worth the document on their own: - The `single`-mode regression shipped behind a headline claiming it could not happen, with the reproduction (50 of 50 live ids reported free), the 1-in-25,000 arithmetic over a verified 50x50x10,000 id space, and the three reasons it is instructive: it was in the default configuration, the suite could not see it because no fixture simulated an upgrade, and §2.6's wording contributed. - The eviction contract, where the lesson is the *first* fix: pinning only the SSE path addressed the case that was easy to picture and left the class open. - `except Exception` converting three separate programming errors into log lines, on three separate occasions, which is why it is filed as one finding with the pattern extracted. Also records the suite's structural blind spot as a finding in its own right — a fake placed at the boundary under test cannot test that boundary — since that is what let two defects survive 200-odd new test functions. **§5 is the longest section, deliberately.** `multi` does not work end to end until T6 (no response is consumed); no cluster, no two-tenant deployment, no Kafka; reads not owner-scoped; Tier A is not isolation; the per-tenant throughput ceiling §10 asks for is still unmeasured; the `claude --help` flag assertion does not run in CI; `seed_from` is capped at 10,000 per store; and `is_valid_userkey` is shape validation, not authorisation. Every figure in the report was measured on this branch, every cited commit verified to be on it, and every cited design section verified to exist — §21.6 turned out to be `DESIGN_PHASE1.md`'s and is now qualified as such. Two numbers in my own PR description were wrong and are corrected here: the test delta is +232 unique functions (525 to 757), not the 139 or 217 claimed earlier. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/agentdocs/IMPLEMENTATION_REPORT3.md | 361 +++++++++++++++++++ eventing/agentdocs/README.md | 4 + 2 files changed, 365 insertions(+) create mode 100644 eventing/agentdocs/IMPLEMENTATION_REPORT3.md diff --git a/eventing/agentdocs/IMPLEMENTATION_REPORT3.md b/eventing/agentdocs/IMPLEMENTATION_REPORT3.md new file mode 100644 index 00000000..bf070dfb --- /dev/null +++ b/eventing/agentdocs/IMPLEMENTATION_REPORT3.md @@ -0,0 +1,361 @@ +# IMPLEMENTATION REPORT — Phase 3 (partial: §9 steps 1–4, plus T5) + +Status: **in progress.** This report covers what landed on +`feat/eventing-phase3-tenancy-foundation` (PR #883): §9's rollout steps 1–4, plus T5, T13 +and T22. It will be extended as later steps land rather than rewritten. + +Written to the same contract as `IMPLEMENTATION_REPORT1.md`: measured figures rather than +estimates, failures with their root causes, and an explicit list of what is **not** +verified — which for this phase is longer and more useful than the list of what is. + +One framing note, because it shapes everything below. Phase 3 is a 12-step, 22-task phase +and this is the first slice of it. §9 singles out steps 1–4 as safe to merge while `multi` +has never been switched on anywhere: *"the first four steps are refactors with tests, and +everything risky is behind a flag that defaults off."* That property held, and it is the +reason the largest diff in the phase is also the least risky. **It did not hold on the +first attempt**, and §4.1 is about why. + +--- + +## 1. What was built + +| Task | Module | Lines | +|---|---|---| +| T1 | `shared/tenancy.py` — `userkey()`, `TopicSet`, `ntfy_topic()`, `is_valid_userkey()` | 292 | +| T2 | `TopicSet` threaded through `kafka_out`, `kafka_in`, both mirrors, `eventrunner/consume` | — | +| T3 | `eventbridge/registry.py` — the approved-user registry; `auth.Caller` | 221 | +| T4 | `SIGNED_ATTRS` += `userkey`, `depth`, `agent` | — | +| T5 | `eventbridge/store_registry.py` — per-user stores, LRU | 262 | +| T5 | `eventbridge/owner_index.py` — the global `correlationid → userkey` index | 276 | +| T13 | `eventrunner/agentspec.py` — declarative agent definitions | 290 | +| T22 | `test_manifests.py` — the Secret-vs-ConfigMap rule, both directions | — | + +Ten commits, 40 files, **+5933/−173**. New configuration: `EB_TENANCY_MODE` (default +`single`), `EB_TOPIC_PREFIX`, `EB_USER_REGISTRY_PATH`, `EB_MAX_OPEN_STORES`, `ER_USERKEY`, +`ER_AGENT_DIR`, `ER_AGENT_NAME`. + +### The additive guarantee, and how it was checked + +`EB_TENANCY_MODE=single` is the default and reproduces Phase 2. The check is that the +**525 test functions that existed before this branch pass untouched** — three test *fakes* +were widened to match a producer signature, and no assertion was weakened or removed. + +Two specific claims were pinned rather than asserted in prose: + +- `test_default_spec_reproduces_phase2_argv_exactly` compares against a **literal + transcription** of the pre-Phase-3 `build_cmd`, not a call into the current one. A test + that calls the implementation to decide what the implementation should produce follows a + regression rather than catching it. +- Single-tenant mode keeps its SQLite files in the bridge root, not under `shared/`, so an + upgraded deployment's sessions and transcripts stay where the UI looks for them. §6.1's + diagram says `shared/`; the code deliberately differs and §6.1 now records why. + +--- + +## 2. Test results + +### Local unit tests + +``` +855 passed, 5 skipped in 56s (Python 3.14.2 free-threaded, macOS arm64) +860 collected (the 5 skips are the difference) +``` + +**525 → 757 unique test functions** (+232); 860 collected with parametrisation. All five +skips are `test_manifests.py`'s `needs_kubectl` tests, reported as "no reachable cluster" +— disclosed, and §5 is what they would have covered. + +One skip that did *not* fire locally is worth naming because it fires in CI: +`test_agentspec.py`'s `skipif(shutil.which("claude") is None)`. That is the test pinning +§5.5's CLI flag set, and the runner has no `claude`, so **the assertion that a flag +vanishing in a CLI upgrade fails a build does not actually run in CI** — it ran here. + +`ruff check` and `ruff format --check` clean against the pinned 0.11.4. + +### What was NOT run + +No cluster. No two-tenant deployment. No Kafka. Every result above is from unit tests and +in-process fakes. §5 is the full list; stating it here because the table above otherwise +reads like more assurance than it is. + +### Measurements against the design's assumptions + +§6.1 estimates the per-tenant file-descriptor cost and derives a ceiling from it. Measured: + +| Quantity | §6.1's estimate | Measured | +|---|---|---| +| Descriptors per open tenant store | "two connections … main plus `-wal` and `-shm`" | **6.0** (10 tenants, 60 fds) | +| Concurrently-open tenants under the soft limit | "around 150–200" | **~680** at `RLIMIT_NOFILE=4096` | + +The design's arithmetic was right per connection and its *conclusion* was anchored to a +1024 soft limit. The measurement is on a host with 4096, so the two numbers do not +disagree — 1024/6 ≈ 170, which is squarely inside "150–200". Recorded because the ceiling +is deployment-specific and `EB_MAX_OPEN_STORES=64` is well under either. + +Ownership index, 5,000 rows, same host: + +| Operation | Measured | +|---|---| +| `claim()` | 27.4 µs | +| `exists()` | 1.7 µs | +| `seed_from()`, 10,000 ids | 270 ms | + +That last figure settles the question §2.6 raises. §2.6 rejects seeding on the grounds +that reading `all_correlations()` from N stores is "a measurable startup delay" — true for +*per-start* seeding at 100 tenants, and the reason the index exists. But the **one-time** +backfill it also appeared to rule out costs 270 ms for the 10,000-id cap Phase 2 already +applied, once, on the first start after an upgrade. §4.1 is why that distinction was not +academic. + +--- + +## 3. Deltas from the design, recorded as decisions + +Not drift. Each of these is a place the implementation chose differently and the design has +been corrected to match (§9's "the code is the authority where implemented" rule). + +1. **Single-tenant stores live in the bridge root, not `shared/`.** Upgrade safety: a + Phase 2 deployment already has `responses.sqlite` and `sessions.sqlite` in + `{tmpdir}/eventbridge/`, and relocating them makes every existing session vanish from + the UI. `shared/` is used only in multi-tenant mode. +2. **The LRU evicts by dropping a reference; it does not close.** §6.1 said "closes on an + LRU" and that "closing is safe". The premise was true and the conclusion false — §4.1. +3. **Deletion tombstones; an id is never reissued.** §6.5 implied `forget()` frees the id. + It cannot: `sessionuuid` is unsalted, so a reissued `correlationid` derives the same + session uuid and a leftover transcript becomes resumable by a different user's run. +4. **`events`/`dead` have single-mode names** (`{prefix}-events`, `{prefix}-dead`). §3.1 + defined only the per-user forms; neither topic has a Phase 2 predecessor to stay + compatible with, so deriving them from the prefix is free. +5. **`agent` joins `SIGNED_ATTRS`.** §2.6 named only `userkey`, and §8.3 only `userkey` and + `depth`. `ce_agent` selects the `AgentSpec` that supplies the tool policy — §5.1's + "sandbox" — so leaving it unsigned made a signed deployment *worse* than an unsigned one. +6. **`tenancy.py` exports a validator.** §2.3 defines the key's shape in one place and gave + consumers no way to check it, which is what made §4.5 possible. +7. **`EB_TOPIC_PREFIX` is validated** `[a-zA-Z0-9-]{1,32}`. It is the one component of a + topic name `_slug` never sees, so `.` or `_` there reproduces exactly the JMX + metric-name collision §3.1 warns about. + +--- + +## 4. Findings + +Ordered by what they cost to find, not by severity. Every one was reproduced by executing +the code before being fixed; the commit is named so the diff is reviewable alongside. + +### 4.1 The `single`-mode regression that shipped behind a claim it could not happen + +**Found by:** review (#883). **Fixed in:** `6ebdd80`. + +Dropping Phase 2's `Minter` seeding loop left nothing to populate its replacement. On the +first start after an upgrade, `owners.sqlite` is new and empty while `sessions.sqlite` +holds every prior correlation, so `exists()` answers "free" for ids that are in use. +Reproduced directly: + +``` +sessions in store: 50 +index count on first start after upgrade: 0 +live ids the index reports as FREE: 50 of 50 +``` + +The id space is 50 adjectives × 50 animals × 10,000 = 25,000,000, so a deployment with +~1,000 live correlations has roughly a **1-in-25,000 chance per mint** of reissuing one. +When it hits, `upsert_session` overwrites that session and `insert_prompt` appends the new +prompt to the old conversation. Phase 2's chance was zero. + +Three things make this the most instructive finding in the phase: + +- **It is a regression in the DEFAULT configuration**, which is the one thing §9's + merge-safety property promises cannot happen — and the PR claiming that property was the + PR that broke it. +- **The suite could not see it.** No test covered an empty index with a populated store, + because every fixture built the two together. The state that causes it is only reachable + by *upgrading*, which no test simulated. +- **The design contributed.** §2.6 argues convincingly that one `SELECT` per mint beats + re-seeding a `seen` set on every start, and in arguing that it reads as though the index + needs no seeding at all. Those are different claims. The distinction — *per-start + seeding* rejected, *one-time backfill* required — is now explicit in §2.6 and §6.1a. + +### 4.2 Eviction closed stores that callers still held + +**Found by:** self-review before merge. **Fixed in:** `49aed05`. + +`StoreRegistry` eviction called `Store.close()`. A handler resolves a store and then makes +several calls on it — `get_html` makes five — so a concurrent request for a *different* +tenant could push the cache over `EB_MAX_OPEN_STORES` and close it mid-use. Reproduced with +`max_open=1` and two tenants: + +``` +sqlite3.ProgrammingError: Cannot operate on a closed database. +``` + +Every multi-call read path was exposed. Eviction now drops the registry's reference and +lets the interpreter close at the last holder, which is exactly the lifetime that is safe. + +**The lesson is the first fix, not the second.** The first attempt pinned only the SSE +path, because that was the case easiest to see — a generator holding a store across a +two-minute stream. It closed one instance and left the class open. The bug was in the +**eviction contract**, not in one caller. A fix aimed at the symptom you can picture is a +fix that will be re-found later. + +Secondary: `protect=` was then needed because a cache whose older entries are all pinned +evicted the newest — and handed the caller the store it had just closed. + +### 4.3 `except Exception` hid three different programming errors + +**Found by:** three separate occasions. **Fixed in:** `77807b6`, `3278d75`. + +The same mistake three times, which is why it is one finding: + +1. Adding `userkey=` to `publish_group_event` made every group event stop publishing. The + broad handler around the call turned a `TypeError` signature mismatch into a log line, + and the only visible symptom was a batch that never completed. +2. `Minter.mint` caught `Exception` around `claim` to treat a lost race as a retry, so a + genuinely broken index (disk full, locked database) was retried 2,000 times and reported + as `RuntimeError("correlation ID space exhausted")` — a message that sends the reader to + the word lists instead of the database. +3. The same group-publish handler swallowed `ValueError` from `TopicSet`, so a caller + forgetting a `userkey` also produced a silently never-announcing group. Found by the new + integration test (§4.6) while mutating an unrelated fix. + +All three now re-raise the programming error and still handle a genuine broker failure. The +pattern worth extracting: **a broad `except` on a publish path converts "this code is +wrong" into "the broker had a bad moment", and the two need opposite responses.** + +### 4.4 `GroupMirror` never learned about tenancy + +**Found by:** self-review. **Fixed in:** `77807b6`. + +Two bugs, one cause — the mirror knew the per-user *topic* names and not which tenant a +replayed group belonged to: + +- `_settle` called `maybe_complete(gid)` with no `userkey`, so it looked in the shared + store for a group living in a tenant's store, found nothing, returned `False`. Every + group a restart left unfinished stayed unfinished **forever**, and the failure mode is + silence — which `DESIGN_PHASE1.md` §21.6 calls the worst one. +- Replayed events published *before* Phase 3 carry no `ce_userkey`, so member rows went to + `shared/` while the group row sat in the tenant's store, leaving the counts permanently + wrong. Pre-existing Kafka history is precisely what the mirror replays, so this is the + **normal upgrade path**, not an edge case. + +The mirror now resolves the owner from the index and back-fills a missing `userkey` — only +ever *filling*, never overriding, since `userkey` is signed and the index is not +authoritative over a signed assertion. + +### 4.5 A `userkey` reached a filesystem path join unvalidated + +**Found by:** review (#883). **Fixed in:** `6ebdd80`. + +In multi mode the key arrives from an inbound Kafka header, and `Store.__init__` calls +`mkdir(parents=True)` — so it is created, not rejected: + +``` +'gh-alice-a1b2c3d4' -> …/users/gh-alice-a1b2c3d4 +'../users/gh-victim-0bad0bad' -> …/users/gh-victim-0bad0bad # another tenant +'../../../../tmp/pwned' -> /tmp/pwned # outside the tree +'..' -> …/eventbridge # the bridge root +``` + +Reachable because `EB_REQUIRE_RESPONSE_SIGNATURE` defaults to `false` and §3.6 already +concedes that in Tier A anything reaching the shared broker can write to any topic — an +accepted property for *reading* events, which must not also be a filesystem write +primitive. + +Two details worth keeping: + +- **An invalid key is truthy**, so it sailed past the existing `if not userkey` branch + straight into the join. The guard that looked like it covered this did not. +- `agentspec.py` got the same problem right 150 lines away — `_resolve_under` refuses a + path that escapes the agent directory and reasons explicitly about `../../../etc/shadow`. + The gate was present on operator-supplied input and missing on the one input that is + genuinely attacker-supplied. + +An invalid key is now treated exactly like a missing one — `shared/` plus the +`unattributed` counter — so a forgery is *counted and visible* rather than rejected +silently. + +### 4.6 The suite's structural blind spot + +**Found by:** review (#883). **Fixed in:** `3278d75`. + +Two of the defects above survived 200-odd new test functions, for one reason: every multi-mode +test either called `GroupService` directly **with** the `userkey` the handler fails to +pass, or replaced `Producer` with a mock and asserted on its keyword arguments. Nothing +asserted a **topic string**, which is the thing that decides whose runner executes a turn. + +`tests/test_topic_routing_e2e.py` fakes one layer lower — at `KafkaProducer`, so the real +handler, the real `Producer` and the real `TopicSet` all run. Confirmed by reverting each +fix that it catches both. It then found §4.3's third instance unprompted. + +Generalisable: **a fake placed at the boundary you are testing cannot test that +boundary.** The fakes were at `Producer`, and the bug was in what the handler passed to +`Producer`. + +### 4.7 Smaller findings + +- **`/continue` 500 on an unresolvable owner** (`7590ec8`), and again on a *known but + unowned* correlation (`6ebdd80`). The second taught something: reading and publishing + need different answers. Shared-tier data must stay readable while a turn for it is + unpublishable, so `_store_of` and `_publishable_owner` are separate. +- **Unauthenticated cross-tenant group mutation** (`6ebdd80`). `close_group`/`cancel_group` + called no auth and resolved any tenant's group from the global index. "T7 is not in yet" + does not cover it: T7 is about reads *staying* as open as Phase 2's, and Phase 2 had no + cross-tenant mutation to preserve. +- **A test that could not fail for its stated reason** (`3278d75`). T22's classification + check asserted `CONFIGMAP_VARS ⊆ found`, catching a stale entry; the direction that + catches a *new* unclassified variable is the reverse. Now verified by deliberate failure. +- **A test flaky by construction** (`6ebdd80`). `"gh" not in` 26 characters of base32 fails + ~2.4% of the time — base32's alphabet contains `g` and `h`. It also tested nothing: the + `"mrsabath" not in` assertion above it is the property §4.1 cares about. +- **`max_turns: 0` silently rewritten** (`6ebdd80`), because `payload.get(...) or + spec.max_turns` treats `0` as absent while the spec path *rejects* zero. Two paths + disagreeing about the same nonsensical input. + +--- + +## 5. Not verified, and honest limits + +- **`EB_TENANCY_MODE=multi` does not work end to end.** Until T6, neither consumer + subscribes to the per-user topics, so **no response is ever consumed**: transcript pages + stay empty, group counters never advance, nothing is logged. `multi` is for developing + against, not running. This is a more useful warning than "not yet isolated", which + implies a working-but-unisolated mode. +- **No cluster run, and no two-tenant deployment.** Everything measured here is unit tests + and in-process fakes on one laptop. `scripts/k8s_tenant.py` (T12) does not exist, so + there is no way to provision a second tenant's topics, Deployment or ScaledObject. +- **Reads are not owner-scoped.** Per-user stores exist and a read routes to the owning + tenant's store, but nothing checks the *caller* is that owner (T7). `PUT /transcript` is + still unauthenticated (T8) — §6's opening paragraph is explicit that this must land + before a second tenant exists, since an overwritable transcript injects resume state into + another user's next turn. +- **Tier A is not isolation**, and no part of this changes that. Separately-named topics + without Kafka ACLs are organisation; §3.6's dedicated-broker option (T20) is what earns + the word. +- **The per-tenant throughput ceiling is still unmeasured**, as §10 asks. EventBridge is + single-replica and now holds N stores, the ownership index and the Ed25519 signing. The + figures in §2 are per-operation costs, not a throughput model. +- **The `claude --help` flag assertion does not run in CI** (no `claude` on the runner), so + §5.5's promise that a flag vanishing in a CLI upgrade fails a build is currently a + promise about a local run. +- **`seed_from` is bounded at 10,000 ids per store**, matching Phase 2's cap. A store with + more than that keeps the §4.1 hazard for the remainder. The cap should become a config + value, or the seed should page. +- **`is_valid_userkey` is shape validation, not authorisation.** It stops a path escape; it + does not establish that the named tenant exists or that the sender may act for it. With + signature verification off — the default — a forged-but-well-formed key still files events + into `shared/`. +- **No measurement of the LRU under real contention.** The pinning rule is unit-tested; + what has not been observed is a cache thrashing at `max_open` with live SSE viewers. + +--- + +## 6. Next actions + +1. **T6** — `ensure_subscribed`, which is what makes `multi` function at all. The race must + be written as a genuinely failing test first: publish before subscribe must fail. +2. **T7 + T8** — owner-scoped reads and transcript auth, in that order, and both before a + second tenant exists anywhere. +3. **T12** — `scripts/k8s_tenant.py`, which is the prerequisite for any of the above being + verified on a cluster rather than in fakes. +4. **Measure the per-tenant ceiling** (§10) before this is deployed for more than a handful + of users. It is the one open item that could invalidate a design decision rather than + just leave a gap. diff --git a/eventing/agentdocs/README.md b/eventing/agentdocs/README.md index 59fd489d..34d02306 100644 --- a/eventing/agentdocs/README.md +++ b/eventing/agentdocs/README.md @@ -20,6 +20,7 @@ and what is *not* verified. | [`README_PHASE1.md`](README_PHASE1.md) | The Phase 1 runbook in full detail, covering all four ways to run it: locally without containers, under Docker, on a Kind cluster, and on OpenShift. | | [`DESIGN_PHASE2.md`](DESIGN_PHASE2.md) | The Phase 2 design — identity on the event path. Why GitHub's opaque user token rules out local verification and forces a `GET /user` lookup plus a load-bearing cache; why `401` and `403` are kept distinct; what `ce_submitter` is and is not worth, and what a signature over it does and does not prove; how the `kid` and the approved-key set prove *which agent* answered, why group events are pinned to EventBridge's own key, and why a rejected response is stored as an error rather than dropped. **§8 records the thirteen claims across §§2.6, 3, 4.3 and 4.4 that a docs review found do not hold in the code, with the issue filed for each — and §8.5-§8.8 the procedure that came out of them: the five questions to ask of any control claim, in any phase, before it ships.** | | [`DESIGN_PHASE3.md`](DESIGN_PHASE3.md) | The Phase 3 design — per-user isolation, declarative agents, and event triggers. Why a raw `submitter` cannot be a tenancy key and why the derived `userkey` ends in a hash; per-user Kafka topics (`{prefix}-{userkey}-requests`), one EventRunner Deployment and ScaledObject per user, and the honest difference between separately-named topics and ACL-enforced isolation; why a per-user ntfy topic must *not* contain the user's name, and the option of running ntfy as a service in the cluster; the `AgentSpec` — an agent baked into the image or assembled from skills fetched from GitHub, S3 or a URL, and the five gates a fetched bundle passes; authenticating and scoping the session transcript; and a Knative-inspired trigger API with the four loop controls that keep an event-driven agent system from becoming a cost incident. **Partly implemented: §9's steps 1-4 (T1, T2, T3, T4, T13, T22) plus T5 (per-user stores and the global ownership index) are in the tree; `EB_TENANCY_MODE` still defaults to `single`, which reproduces Phase 2 exactly. Owner-scoped reads (T7), transcript auth (T8), `k8s_tenant.py` (T12), ntfy isolation, triggers, fetched skills and Kafka ACLs are design only — so `multi` mode routes each tenant to its own store and topics, but does not yet check that a reader is the owner.** | +| [`IMPLEMENTATION_REPORT3.md`](IMPLEMENTATION_REPORT3.md) | What Phase 3 has built so far (§9 steps 1–4 plus T5), measured rather than estimated: the per-tenant file-descriptor cost against §6.1's estimate, the ownership index's per-operation cost, and the 270 ms that settles §2.6's objection to seeding. Seven deltas from the design recorded as decisions, and the findings with their root causes — including a `single`-mode regression that shipped behind a headline claiming it could not happen, an eviction contract whose first fix addressed only the visible case, and `except Exception` converting three separate programming errors into log lines. **Partial: it will be extended as later steps land rather than rewritten.** | ## Reading order @@ -31,6 +32,9 @@ and what is *not* verified. model. - **Working on auth or identity?** [`DESIGN_PHASE2.md`](DESIGN_PHASE2.md) §2.1 first: the opaque-token constraint is what rules out the design most people reach for. +- **Debugging something odd in the tenancy code?** + [`IMPLEMENTATION_REPORT3.md`](IMPLEMENTATION_REPORT3.md) §4 — the findings are there with + the symptom that hid each one, and §5 lists what is not verified at all. - **Debugging something odd?** [`IMPLEMENTATION_REPORT1.md`](IMPLEMENTATION_REPORT1.md) §4 — most surprises are already recorded there with their cause. - **Writing a claim about a control, in any phase?** From 2c8090056bc49b4edafe514cc5f738c2d37410fd Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Mon, 5 Oct 2026 11:42:36 -0400 Subject: [PATCH 12/14] fix(eventing): two orphan-write paths, and three design/code gaps (#883 round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both must-fixes reproduced against c25a0c77 before fixing, and both are my own round-1 fix introducing a new bug class: `6ebdd80` added validation of the request's `agent` name at the HTTP boundary — right place, right reason — and placed it AFTER the writes. **create_group** had already committed the group row and published `group.started` when the 400 returned, and because the row is keyed on the Idempotency-Key the state was permanent: every retry read back `created: false, members: []`, which looks like success. That is verbatim the failure the `userkey=` comment added in the same round describes, reached by a route that fix did not cover. POST /v0/groups {"prompts":["a","b"],"agent":"../../etc"} Idempotency-Key: retry-me -> 400, topics: ['kev1-st-alice-c8ff0431-responses'], groups: ['wild-weasel-1995'] retry, same key, valid name -> 200 {created: false, members: []} **start_agent** had written the session row, the prompt row and claimed the correlation id — a transcript showing a turn that was never submitted, plus an id nothing ever un-claims, since tombstoning is for deletion rather than abandonment. Both resolutions now happen before the first write. `_agent_for` needs only `caller` and `body`, so there was never anything to gain by deferring it. After the fix both paths publish nothing, write nothing, and the group retry launches the batch properly. The reviewer's observation about the tests is the more useful half: the assertion that would have caught both — `get_prompts(corr) == []`, *a refusal writes nothing* — already existed 25 lines earlier in the same file, on a different route. The new tests assert it on both paths, with the Idempotency-Key retry for the group case, and were verified by reverting each ordering (7 failures). ### Three design/code gaps §3 of the report had missed **The backfill was per-start, not one-time.** §6.1a says "seeded once" and the report said "once, on the first start after an upgrade"; the loop re-opened every tenant store and re-read `all_correlations` on every boot — exactly the N-store startup seeding §2.6 rejects, reintroduced by the fix for the missing seed. `INSERT OR IGNORE` kept it correct; what repeated was the cost §2.6's argument is entirely about. A `seed_state(scope, done_utc)` table now records each scope, and `needs_seeding` is checked before the store is opened. Measured at 100 tenants x 200 correlations: before: start 1 702 ms / 101 opens start 2 210 ms / 101 opens after: start 1 702 ms / 101 opens start 2 0.8 ms / 1 open Sound because nothing can add an unindexed id to a seeded store: every mint claims as it mints. **`AgentSpec.limits` was parsed and enforced nowhere, silently.** `Skill` states plainly that it carries `sha256` for T19; `Limits` made no such disclaimer, so `timeout_s = 900` loaded clean and did nothing. Now says so, with which field belongs to which later task. Its three conversions also escaped the module's own `SpecError` contract — `timeout_s = "900s"` raised a bare `ValueError` past `run_agent`'s handler instead of becoming the `phase=error` event §5.1 promises. **§8.3 disagreed with §2.6 and the code.** It still said the signed set "changes twice" and named two attributes, while §2.6, `signing.py` and `ce.py` all said three — and `signing.py` cited §8.3 as the authority for a rule §8.3 stated over the wrong count. Two smaller ones from the same pass: a tombstoned correlation could be re-claimed by the NULL owner (`row[0] == userkey` reads as "already mine" when both are NULL), contradicting what `forget` guarantees — unreachable from `Minter.mint`, which checks `exists()` first, but reachable as soon as a correlationid arrives from outside, which §7's triggers will bring. And `registry.parse` accepted `agent` unchecked, the only field it did not validate, in a function whose docstring promises every inconsistency is a refusal; a typo there meant an asynchronous `phase=error` on every request from that user rather than a startup refusal. Verified: the registry now refuses with the bad value named. ### Documentation - The component README's Phase 3 block no longer orphans the "Two of these are load-bearing" sentence from its payoff 25 lines later — the block moved below it under its own `#### Phase 3` heading, and the note count is now right (three, not two). Added the T6 "does not work end to end" warning here too. - `IMPLEMENTATION_REPORT3.md` §1 now counts from a named point (`3278d75`) instead of "currently", since a report cannot be right about the diff of the commit containing it; both figures are given. New §4.7 and §4.8 record this round's findings, including that fixing one orphan-write instance did not teach me to look for the class — the same lesson §4.2 records about the eviction contract, learned a second time. Tests: 860 passing (+5), ruff clean, both multi-mode startup paths smoke-tested. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/README.md | 34 +++++--- eventing/agentdocs/DESIGN_PHASE3.md | 25 ++++-- eventing/agentdocs/IMPLEMENTATION_REPORT3.md | 83 +++++++++++++++++++- eventing/eventbridge/__main__.py | 16 +++- eventing/eventbridge/handlers.py | 34 ++++++-- eventing/eventbridge/owner_index.py | 54 ++++++++++++- eventing/eventbridge/registry.py | 18 ++++- eventing/eventrunner/agentspec.py | 31 ++++++-- eventing/tests/test_review_fixes.py | 82 ++++++++++++++++++- 9 files changed, 336 insertions(+), 41 deletions(-) diff --git a/eventing/README.md b/eventing/README.md index 0f356920..c49445be 100644 --- a/eventing/README.md +++ b/eventing/README.md @@ -183,6 +183,14 @@ the full list of what stays open: `agentdocs/README_PHASE1.md` § "Submit-path authentication". Two of these are load-bearing in ways that are easy to miss. +`ER_EVENTBRIDGE_URL` is what makes `/continue` survive a scale-to-zero *and* what +keeps event `sequence` numbers unique across pods — without it a second pod +restarts numbering at 1 and its rows overwrite the first turn's. +`ER_CONSUMER_GROUP` must match the ScaledObject exactly or KEDA measures a group +nobody joins. + +#### Phase 3 — per-user isolation + Added in Phase 3 (`agentdocs/DESIGN_PHASE3.md`). Only the variables whose code is actually in the tree — §9's steps 1–4. Every one of them is off or single-tenant by default, so an existing deployment that upgrades and changes nothing behaves exactly @@ -198,22 +206,24 @@ as it did: | `ER_AGENT_DIR` | `/etc/rossoctl/agents` | Where baked `AgentSpec`s live (§5.2) | | `ER_AGENT_NAME` | `default` | Fallback agent when neither the request nor the registry names one. `default` with no file on disk is Phase 2's argv exactly | -Two notes on these. **`multi` mode is not yet isolation you should present as such.** +Three notes on these. **`multi` mode is not yet isolation you should present as +such.** Per-user stores *are* implemented (T5), so each tenant's sessions, responses and transcripts live in their own SQLite files and a read routes to the owning tenant's store — but nothing yet checks that the *caller* is that owner, because owner-scoped reads (T7) and transcript authentication (T8) are not in. So in `multi` mode the HTTP reads, -`/continue` and `PUT /transcript` are still as open as Phase 2's. And even once they land, separate topics without Kafka ACLs are -*organisation*, not isolation; `DESIGN_PHASE3.md` §3.6 and §10 are explicit about which -claim each configuration earns. **Adding a user is an operator action**: EventBridge has -no Kubernetes client and will not create topics, so a registry entry whose topics do not -exist is a refusal, not an auto-provision. - -`ER_EVENTBRIDGE_URL` is what makes `/continue` survive a scale-to-zero *and* what -keeps event `sequence` numbers unique across pods — without it a second pod -restarts numbering at 1 and its rows overwrite the first turn's. -`ER_CONSUMER_GROUP` must match the ScaledObject exactly or KEDA measures a group -nobody joins. +`/continue` and `PUT /transcript` are still as open as Phase 2's. And even once they +land, separate topics without Kafka ACLs are *organisation*, not isolation; +`DESIGN_PHASE3.md` §3.6 and §10 are explicit about which claim each configuration earns. + +**`multi` also does not work end to end yet.** Until T6 neither consumer subscribes to +the per-user topics, so no response is ever consumed: transcript pages stay empty, group +counters never advance, and nothing is logged. Treat it as something to develop against, +not to run. + +**Adding a user is an operator action.** EventBridge has no Kubernetes client and will +not create topics, so a registry entry whose topics do not exist is a refusal, not an +auto-provision. EventRunner does **not** hand its whole environment to the child process. It forwards a curated set — `ANTHROPIC_BASE_URL`, `ANTHROPIC_MODEL`, diff --git a/eventing/agentdocs/DESIGN_PHASE3.md b/eventing/agentdocs/DESIGN_PHASE3.md index c7d048e1..c958fd7b 100644 --- a/eventing/agentdocs/DESIGN_PHASE3.md +++ b/eventing/agentdocs/DESIGN_PHASE3.md @@ -2053,14 +2053,23 @@ upgrades and changes nothing behaves exactly as it did in Phase 2. ### 8.3 Two notes that are not table rows -**The signed-attribute set changes twice, and both changes must land together.** -`userkey` (§2.6) and `depth` (§7.7) both join `signing.SIGNED_ATTRS`. Phase 2 §4.2 -already set the precedent and the rule: adding an attribute changes canonicalisation, so -a signer and a verifier on different versions disagree about every signature. Doing it -in two commits would invalidate canonicalisation twice for no benefit, so it is **one -change**, and deployments with signing already enabled must upgrade both services -together. Deployments with signing off (the default) are unaffected, which is most of -them. +**The signed-attribute set changes once, for three attributes, and that change must +land in one commit.** `userkey` (§2.6), `depth` (§7.7) and `agent` (§5.1) all join +`signing.SIGNED_ATTRS`. Phase 2 §4.2 already set the precedent and the rule: adding an +attribute changes canonicalisation, so a signer and a verifier on different versions +disagree about every signature. Doing it in three commits would invalidate +canonicalisation three times for no benefit, so it is **one change**, and deployments +with signing already enabled must upgrade both services together. Deployments with +signing off (the default) are unaffected, which is most of them. + +An earlier revision of this paragraph said "changes twice" and named only `userkey` and +`depth`. `agent` belongs in the set for the same reason as the other two and arguably +more urgently: it selects the `AgentSpec` that supplies `--permission-mode` and the tool +allowlists, which §5.1 calls "the sandbox". Outside the signed set, anything with write +access to a requests topic could rewrite `ce_agent` to name a spec with a wider policy +and **the signature would still verify** — making a signed deployment worse than an +unsigned one, because an operator believes it is attested. §2.6 has the full list of what +rides on the event and which three are signed. **Secret-vs-ConfigMap, continuing Phase 2 §5's rule.** Seed and HMAC-secret paths (`EB_NTFY_TOPIC_SECRET_PATH`, `EB_CAPABILITY_SECRET_PATH`, `EB_WEBHOOK_SECRETS_PATH`) diff --git a/eventing/agentdocs/IMPLEMENTATION_REPORT3.md b/eventing/agentdocs/IMPLEMENTATION_REPORT3.md index bf070dfb..dddb3e48 100644 --- a/eventing/agentdocs/IMPLEMENTATION_REPORT3.md +++ b/eventing/agentdocs/IMPLEMENTATION_REPORT3.md @@ -30,7 +30,14 @@ first attempt**, and §4.1 is about why. | T13 | `eventrunner/agentspec.py` — declarative agent definitions | 290 | | T22 | `test_manifests.py` — the Secret-vs-ConfigMap rule, both directions | — | -Ten commits, 40 files, **+5933/−173**. New configuration: `EB_TENANCY_MODE` (default +**As of `3278d75`, the commit before this report:** ten commits, 40 files, +**+5933/−173**. Counted from a named point rather than "currently", because a report that +counts the branch it is committed on can never be right about its own diff — at +`c25a0c77` the figures are 11 commits, 41 files, +6298/−173, the difference being this +file and its index entry. A reader checking either number should get the one the text +claims. + +New configuration: `EB_TENANCY_MODE` (default `single`), `EB_TOPIC_PREFIX`, `EB_USER_REGISTRY_PATH`, `EB_MAX_OPEN_STORES`, `ER_USERKEY`, `ER_AGENT_DIR`, `ER_AGENT_NAME`. @@ -290,7 +297,79 @@ Generalisable: **a fake placed at the boundary you are testing cannot test that boundary.** The fakes were at `Producer`, and the bug was in what the handler passed to `Producer`. -### 4.7 Smaller findings +### 4.7 A fix that introduced a new bug class: validation after the writes + +**Found by:** review, round 2. **Fixed in:** this round. + +Round 1 added validation of the request's `agent` name at the HTTP boundary — the right +place, for the right reason — and placed it **after** the writes in both submit paths. So +a `400` left committed state behind: + +``` +POST /v0/groups {"prompts":["a","b"],"agent":"../../etc"} Idempotency-Key: retry-me + -> 400 Bad Request + topics published: ['kev1-st-alice-c8ff0431-responses'] # group.started went out + groups in alice's store: ['wild-weasel-1995'] # 0 members, forever + retry, same key, valid name -> 200 {created: false, members: []} + +POST /v0/agents {"prompt":"hi","agent":"Not A Label"} + -> 400 Bad Request + sessions committed: ['fancy-puma-7571'] # a turn never submitted + ownership index count: 1 # an id nothing un-claims +``` + +The group case is the worse of the two and the more interesting: the state is permanent +*because the Idempotency-Key worked*. Every retry reads back the member-less row, which +is verbatim the failure the `userkey=` comment added in the same round describes — reached +through a route that fix did not cover. + +Three things make this worth its own section rather than a line in §4.9: + +- **It is the same orphan-write shape as §4.9's `/continue` 500**, which round 1 fixed. + Fixing an instance did not teach me to look for the class — the same lesson §4.2 records + about the eviction contract, learned again. +- **The test for the new validation asserted the status and that nothing was published**, + both of which held. The assertion that would have caught it — + `get_prompts(corr) == []`, *a refusal writes nothing* — already existed 25 lines + earlier in the same file, on a different route. +- **`test_topic_routing_e2e.py` did not catch it either**, because it asserts topics and + the `start_agent` case publishes nothing. A test aimed at one invariant does not + incidentally cover another. + +Both resolutions now happen before the first write; `_agent_for` needs only `caller` and +`body`, so there was never anything to gain by deferring it. + +### 4.8 Three places the design still disagreed with the code + +**Found by:** review, round 2. **Fixed in:** this round. Grouped because the cause is one +thing: §3 of this report claimed the design/code gap was closed, and checking it found +three items it had missed. + +- **The backfill was per-start, not one-time.** §6.1a says "seeded once" and §2 of this + report said "once, on the first start after an upgrade"; the loop re-opened every tenant + store and re-read `all_correlations` on every boot. `seed_from` is `INSERT OR IGNORE` so + correctness held — what repeated was exactly the cost §2.6's argument is about. Measured + at 100 tenants × 200 correlations: **210 ms per restart, 101 store opens**, and at + §2.6's own worked example (100 × 10,000) it extrapolates to ~6 s on every boot. A + `seed_state` table now records each scope, so an already-seeded store is never opened: + **0.8 ms and 1 store open** per restart. The claim is now true. +- **`AgentSpec.limits` was parsed and enforced nowhere, silently.** `Skill` states plainly + that it carries `sha256` for T19; `Limits` made no such disclaimer, so `timeout_s = 900` + loaded clean and did nothing. Now says so. Its three conversions also escaped the + module's `SpecError` contract — `timeout_s = "900s"` raised a bare `ValueError` past + `run_agent`'s handler instead of becoming the `phase=error` event §5.1 promises. +- **§8.3 was not updated with §2.6.** It still said the signed set "changes twice" and + named two attributes while §2.6, `signing.py` and `ce.py` all said three — and + `signing.py` cited §8.3 as the authority for a rule §8.3 stated over the wrong count. + +Two smaller ones from the same pass: a tombstoned correlation could be re-claimed by the +NULL owner (`row[0] == userkey` reads as "already mine" when both are `NULL`), +contradicting `forget`'s guarantee — unreachable from `Minter.mint`, which checks +`exists()` first, but reachable as soon as a `correlationid` arrives from outside. And +`registry.parse` accepted `agent` unchecked, the only field it did not validate, in a +function whose docstring promises every inconsistency is a refusal. + +### 4.9 Smaller findings - **`/continue` 500 on an unresolvable owner** (`7590ec8`), and again on a *known but unowned* correlation (`6ebdd80`). The second taught something: reading and publishing diff --git a/eventing/eventbridge/__main__.py b/eventing/eventbridge/__main__.py index ec97d6f2..844402bf 100644 --- a/eventing/eventbridge/__main__.py +++ b/eventing/eventbridge/__main__.py @@ -139,12 +139,26 @@ def main() -> int: # Cheap after the first run: `INSERT OR IGNORE` over ids the index already holds, and # `all_correlations` is one indexed query per store. The `limit` matches the 10,000 # Phase 2 applied to its own seed. - seeded = owners.seed_from(store.all_correlations(limit=10000)) + # `needs_seeding` is checked FIRST so an already-seeded store is never opened and + # never scanned. Without that gate this loop is the per-start N-store seeding §2.6 + # rejects: `seed_from` is `INSERT OR IGNORE` so repeating it is harmless to + # correctness, but the cost §2.6's whole argument is about — two SQLite connections + # and an indexed scan per tenant, before the socket binds — was being paid on every + # boot. Measured at ~210 ms per restart for 100 tenants x 200 correlations, and §2.6's + # own worked example (100 x 10,000) extrapolates to ~6 s added to every restart. + # + # Skipping is sound because nothing can add an UNINDEXED id to a store after it is + # seeded: every mint claims as it mints, so a seeded store cannot later acquire one. + seeded = 0 + if owners.needs_seeding(): + seeded += owners.seed_from(store.all_correlations(limit=10000)) if topics.multi: # Every tenant's store too, otherwise a tenant's existing ids stay invisible to # the uniqueness check. `known_userkeys` reads the filesystem rather than the LRU, # so a tenant that is merely closed is still seeded. for uk in stores.known_userkeys(): + if not owners.needs_seeding(uk): + continue seeded += owners.seed_from( stores.for_userkey(uk).all_correlations(limit=10000), uk) if seeded: diff --git a/eventing/eventbridge/handlers.py b/eventing/eventbridge/handlers.py index fd64d982..3f2a79f3 100644 --- a/eventing/eventbridge/handlers.py +++ b/eventing/eventbridge/handlers.py @@ -254,6 +254,20 @@ def start_agent(self, environ, start_response, **_): if groupid and not GROUP_REGEX.match(str(groupid)): return _json(start_response, "400 Bad Request", {"error": "bad groupid"}) + # EVERY refusal happens before the first write. `_agent_for` can reject, and + # validating after the session and prompt rows are committed leaves a transcript + # page showing a turn that was never submitted — plus a correlation id claimed in + # the ownership index that nothing ever un-claims, since tombstoning is for + # deletion rather than for abandonment. That is the same orphan-write shape as the + # `/continue` 500 fixed in 7590ec8, reached by a route that fix did not cover. + # + # It depends only on `caller` and `body`, so there is nothing to gain by deferring + # it: resolve first, mint and write second. + try: + agent = self._agent_for(caller, body) + except _BadRequest as e: + return _json(start_response, "400 Bad Request", {"error": str(e)}) + corr = mint_for(self.minter, caller.userkey) sess = ce.session_uuid(corr) workdir = str(pathlib.Path(self.cfg.tmpdir) / "eventrunner" / "work" / corr) @@ -267,10 +281,6 @@ def start_agent(self, environ, start_response, **_): # Membership before publication: the reverse order leaves a window where a # fast agent's terminal event arrives for a member nobody has recorded. store.add_group_member(groupid, corr) - try: - agent = self._agent_for(caller, body) - except _BadRequest as e: - return _json(start_response, "400 Bad Request", {"error": str(e)}) event_id = self.producer.publish_request( prompt=prompt, correlationid=corr, sessionuuid=sess, mode="start", model=model, max_turns=max_turns, subject="start", @@ -317,6 +327,18 @@ def create_group(self, environ, start_response, **_): {"error": f"expected={expected} contradicts {len(prompts)} prompts"}) idem = environ.get("HTTP_IDEMPOTENCY_KEY") or None + + # Before `groups.create`, which commits the group row AND publishes + # `group.started`. A `400` after that leaves a group with no members that can + # never gain any — and because the row is keyed on the Idempotency-Key, every + # retry reads back `created: false, members: []`, which looks like success. That + # is verbatim the failure the `userkey=` comment below describes, reached through + # a different route: the state is permanent precisely because the key worked. + try: + agent = self._agent_for(caller, body) + except _BadRequest as e: + return _json(start_response, "400 Bad Request", {"error": str(e)}) + groupid, created = self.groups.create( label=body.get("label"), expected=int(expected) if expected else (len(prompts) or None), @@ -340,10 +362,6 @@ def create_group(self, environ, start_response, **_): "members": [m["correlationid"] for m in snap.get("members") or []], "html_url": f"{self.cfg.public_base_url}/v0/groups/{groupid}", }) - try: - agent = self._agent_for(caller, body) - except _BadRequest as e: - return _json(start_response, "400 Bad Request", {"error": str(e)}) corrs = self.groups.submit_members( groupid, [str(x) for x in prompts], max_turns=int(body.get("max_turns", 3)), model=body.get("model"), diff --git a/eventing/eventbridge/owner_index.py b/eventing/eventbridge/owner_index.py index 7374f8ff..5143a34d 100644 --- a/eventing/eventbridge/owner_index.py +++ b/eventing/eventbridge/owner_index.py @@ -57,6 +57,16 @@ deleted_utc TEXT ); CREATE INDEX IF NOT EXISTS idx_owner_userkey ON correlation_owner(userkey); + +-- Which stores have already been seeded (`seed_from`). Without this the startup backfill +-- re-opens every tenant store and re-reads `all_correlations` on EVERY boot, which is the +-- per-start N-store seeding §2.6 rejects — reintroduced by the fix for the missing seed. +-- Sound because nothing can add an UNINDEXED id to a store after it is seeded: every mint +-- claims as it mints, so a seeded store cannot later acquire one. +CREATE TABLE IF NOT EXISTS seed_state ( + scope TEXT PRIMARY KEY, -- "" for the shared/single store, else the userkey + done_utc TEXT NOT NULL +); """ @@ -126,10 +136,24 @@ def claim(self, correlationid: str, userkey: str | None) -> None: (correlationid, userkey)) if cur.rowcount: return - # The row already existed. Same owner is idempotent; anything else is a clash. + # The row already existed. Same owner is idempotent; anything else is a + # clash. A TOMBSTONE is never a re-claim, whoever asks: `forget` clears + # `userkey` to NULL, so for `claim(corr, None)` the `row[0] == userkey` test + # below would read as "already mine" and succeed — contradicting the + # guarantee `forget` makes, that the id is reserved forever because a reissue + # derives the same unsalted `sessionuuid`. + # + # Not reachable from `Minter.mint`, which checks `exists()` first and a + # tombstone is `True`. It becomes reachable the moment a `correlationid` + # arrives from outside — which §7's triggers and T6/T7 plausibly bring — so it + # is closed here, where the invariant is written down. row = self._c.execute( - "SELECT userkey FROM correlation_owner WHERE correlationid=?", + "SELECT userkey, deleted_utc FROM correlation_owner WHERE correlationid=?", (correlationid,)).fetchone() + if row is not None and row[1]: + raise Collision( + f"correlation {correlationid!r} is tombstoned and can never be " + f"reissued; a reissue would derive the same sessionuuid") if row is not None and row[0] == userkey: return raise Collision( @@ -147,6 +171,25 @@ def exists(self, correlationid: str) -> bool: "SELECT 1 FROM correlation_owner WHERE correlationid=? LIMIT 1", (correlationid,)).fetchone() is not None + def needs_seeding(self, userkey: str | None = None) -> bool: + """Whether this store's ids still have to be backfilled. + + Lets the caller skip the `Store` open and the `all_correlations` query entirely, + which is the whole cost §2.6 objects to. `seed_from` is `INSERT OR IGNORE` so + repeating it was harmless to *correctness* — what repeated was two SQLite + connections and an indexed scan per tenant before the socket binds, on every boot. + """ + return self._c.execute( + "SELECT 1 FROM seed_state WHERE scope=? LIMIT 1", + (userkey or "",)).fetchone() is None + + def mark_seeded(self, userkey: str | None = None) -> None: + """Record that this store has been backfilled. Idempotent.""" + with self._lock: + self._c.execute( + "INSERT OR REPLACE INTO seed_state(scope,done_utc) " + "VALUES (?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))", (userkey or "",)) + def seed_from(self, correlationids, userkey: str | None = None) -> int: """Record ids that already exist outside the index. Returns how many were new. @@ -165,6 +208,10 @@ def seed_from(self, correlationids, userkey: str | None = None) -> int: without this, Phase 3 is a regression in the DEFAULT configuration, which is the one thing the phase promises cannot happen. + Marks the scope seeded on completion, so `needs_seeding` returns `False` + afterwards and the caller can skip the store open next boot. Callers that bypass + `needs_seeding` still get a correct (if wasted) re-seed. + `INSERT OR IGNORE`, so re-seeding is free and a correlation already claimed by a tenant keeps its owner. Seeding assigns `userkey=None` for ids recovered from a single-tenant store, which is the correct owner for them: they predate tenancy. @@ -178,6 +225,9 @@ def seed_from(self, correlationids, userkey: str | None = None) -> int: "VALUES (?,?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))", (corr, userkey)) new += cur.rowcount or 0 + self._c.execute( + "INSERT OR REPLACE INTO seed_state(scope,done_utc) " + "VALUES (?,strftime('%Y-%m-%dT%H:%M:%fZ','now'))", (userkey or "",)) return new # ---- owner-scoped reads (§6.2) ----------------------------------------- diff --git a/eventing/eventbridge/registry.py b/eventing/eventbridge/registry.py index 3430ca0a..32d0d6fc 100644 --- a/eventing/eventbridge/registry.py +++ b/eventing/eventbridge/registry.py @@ -196,8 +196,24 @@ def parse(raw: str) -> Registry: except (TypeError, ValueError) as exc: raise RegistryError(f"{where}: limits must be integers") from exc + agent = str(e.get("agent", "")).strip() + if agent: + # The same gate the HTTP boundary applies (`handlers._agent_for`), and the + # last field in this function that was accepted unchecked — against a + # docstring promising "every inconsistency is a refusal". A startup refusal + # rather than a 400, because this value reaches `ce_agent` via + # `by_userkey(...).agent` on a path that never sees `validate_name`: the + # runner then rejects it, so a registry typo would mean an asynchronous + # `phase=error` on EVERY request from that user — the failure mode the + # request path was moved away from. + from eventrunner import agentspec + try: + agentspec.validate_name(agent) + except agentspec.SpecError as exc: + raise RegistryError(f"{where} ({issuer}/{userid}): {exc}") from None + users.append(User(issuer=issuer, userid=userid, userkey=computed, tier=tier, - agent=str(e.get("agent", "")).strip(), limits=limits)) + agent=agent, limits=limits)) return Registry(users=tuple(users)) diff --git a/eventing/eventrunner/agentspec.py b/eventing/eventrunner/agentspec.py index 05e103bf..d9ad1bf7 100644 --- a/eventing/eventrunner/agentspec.py +++ b/eventing/eventrunner/agentspec.py @@ -80,6 +80,17 @@ def is_baked(self) -> bool: @dataclass(frozen=True) class Limits: + """Per-run ceilings from `[limits]`. + + **Carried, not yet enforced** — the same arrangement as `Skill.sha256`: parsed and + validated here so a spec written today stays valid, and applied when the code that + applies it lands. `max_events` is T14's (§5.6's emit socket). `timeout_s` and + `max_output_bytes` need a wrapper around `run_agent`'s `Popen` and have none, so a + spec that sets them loads clean and bounds nothing. + + Said plainly because the alternative is an operator writing `timeout_s = 900`, + getting no error, and believing there is a deadline. + """ timeout_s: float = 0.0 # 0 = no spec-imposed deadline max_output_bytes: int = 0 # 0 = unbounded by the spec max_events: int = 2000 # §5.6 response events per run @@ -233,11 +244,21 @@ def parse(raw: str, *, name: str, base_dir: pathlib.Path | None = None) -> Agent sig_kid=str(entry.get("sig_kid", "")).strip())) lim = d.get("limits", {}) or {} - limits = Limits( - timeout_s=float(lim.get("timeout_s", 0.0)), - max_output_bytes=int(lim.get("max_output_bytes", 0)), - max_events=int(lim.get("max_events", Limits.max_events)), - ) + if not isinstance(lim, dict): + raise SpecError(f"agent {name!r}: limits must be a table") + # Wrapped for the same reason `max_turns` is, 50 lines up: these three conversions + # raise ValueError/TypeError, and `run_agent` catches only `SpecError` — so + # `timeout_s = "900s"` propagated out of `run_agent` instead of becoming the + # `phase=error` event §5.1 promises. This block was the one place in `parse` that + # escaped the module's own contract. + try: + limits = Limits( + timeout_s=float(lim.get("timeout_s", 0.0)), + max_output_bytes=int(lim.get("max_output_bytes", 0)), + max_events=int(lim.get("max_events", Limits.max_events)), + ) + except (TypeError, ValueError) as e: + raise SpecError(f"agent {name!r}: limits must be numbers: {e}") from None return AgentSpec( name=name, diff --git a/eventing/tests/test_review_fixes.py b/eventing/tests/test_review_fixes.py index 2ec49842..c6c50ad9 100644 --- a/eventing/tests/test_review_fixes.py +++ b/eventing/tests/test_review_fixes.py @@ -123,6 +123,45 @@ def test_the_index_is_seeded_from_an_existing_store(tmp_path): "a live correlation is still reported free, so Minter can reissue it" +def test_a_seeded_scope_is_not_seeded_again(tmp_path): + """Round 2. §6.1a claims the backfill is one-time; the first implementation re-opened + every tenant store and re-scanned `all_correlations` on EVERY boot — the per-start + N-store seeding §2.6 rejects, reintroduced by the fix for the missing seed. + + `needs_seeding` is what lets `__main__` skip the store open entirely. Measured effect + at 100 tenants x 200 correlations: 210 ms -> 0.8 ms per restart, 101 store opens -> 1. + """ + owners = OwnerIndex(tmp_path / "eventbridge") + assert owners.needs_seeding() + owners.seed_from(["brave-otter-0001"]) + assert not owners.needs_seeding(), "the shared scope was not marked seeded" + + +def test_seed_scopes_are_tracked_independently(tmp_path): + """A seeded tenant must not mark another, or a new tenant's existing ids would stay + invisible to the uniqueness check.""" + owners = OwnerIndex(tmp_path / "eventbridge") + owners.seed_from(["brave-otter-0001"], UK) + assert not owners.needs_seeding(UK) + assert owners.needs_seeding(OTHER) + assert owners.needs_seeding() # the shared scope is its own + + +def test_an_empty_store_is_still_marked_seeded(tmp_path): + """Otherwise it is re-opened and re-scanned on every boot forever.""" + owners = OwnerIndex(tmp_path / "eventbridge") + assert owners.seed_from([]) == 0 + assert not owners.needs_seeding() + + +def test_the_seed_mark_survives_a_restart(tmp_path): + root = tmp_path / "eventbridge" + one = OwnerIndex(root) + one.seed_from(["brave-otter-0001"], UK) + one.close() + assert not OwnerIndex(root).needs_seeding(UK) + + def test_seeding_is_idempotent_across_restarts(tmp_path): root = tmp_path / "eventbridge" store = Store(root) @@ -375,12 +414,51 @@ def test_a_shared_tier_correlation_is_still_readable(tmp_path): "x" * 64]) def test_a_bad_agent_name_is_400_not_an_async_error_event(tmp_path, bad): """`202 Accepted` followed minutes later by a `phase=error` event the submitter may - never look at is the wrong answer to a typo in the call that made it.""" - h, producer, _, _ = _api(tmp_path) + never look at is the wrong answer to a typo in the call that made it. + + **And a `400` must write nothing** — the same property + `test_a_shared_tier_continue_is_refused_not_a_500` pins for `/continue`, which this + test originally failed to assert 25 lines away. Round 1 added the validation but put + it AFTER the writes, so a `400` left a session row, a prompt row and a claimed + correlation id behind: a transcript page showing a turn that was never submitted, + plus an id nothing will ever un-claim, since tombstoning is for deletion rather than + abandonment. + """ + h, producer, owners, stores = _api(tmp_path) sr = _Start() h.start_agent(_env({"prompt": "hi", "agent": bad}, token=_as(h, OWNER)), sr) assert sr.status.startswith("400"), sr.status producer.publish_request.assert_not_called() + assert stores.for_userkey(UK).all_correlations(limit=10) == [], \ + "a refused submit committed a session row" + assert owners.count() == 0, "a refused submit claimed a correlation id" + + +def test_a_bad_agent_name_on_create_group_writes_nothing(tmp_path): + """The same ordering bug on the group path, where the state becomes PERMANENT. + + `groups.create` commits the group row and publishes `group.started`, so a `400` after + it left a group with no members that could never gain any — and because the row is + keyed on the Idempotency-Key, every retry read back `created: false, members: []`, + which looks like success. The retry is the point of this test. + """ + h, producer, owners, stores = _api(tmp_path) + tok = _as(h, OWNER) + env = _env({"prompts": ["a", "b"], "agent": "../../etc"}, token=tok) + env["HTTP_IDEMPOTENCY_KEY"] = "retry-me" + sr = _Start() + h.create_group(env, sr) + assert sr.status.startswith("400"), sr.status + assert stores.for_userkey(UK).all_groups() == [], "a refused batch created a group" + assert owners.count() == 0, "a refused batch claimed a groupid" + + # The retry, with the same key and a valid name, must actually launch the batch. + good = _env({"prompts": ["a", "b"], "agent": "triager"}, token=tok) + good["HTTP_IDEMPOTENCY_KEY"] = "retry-me" + sr2 = _Start() + out = json.loads(b"".join(h.create_group(good, sr2))) + assert sr2.status.startswith("202"), sr2.status + assert out["created"] is True and len(out["members"]) == 2 def test_a_good_agent_name_still_rides(tmp_path): From fdef56d26dc1a59b36eece10a91cff72732bbc6b Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Mon, 5 Oct 2026 15:07:19 -0400 Subject: [PATCH 13/14] =?UTF-8?q?test(eventing):=20Phase=203=20=E2=80=94?= =?UTF-8?q?=20regression=20tests=20for=20the=20three=20untested=20fixes=20?= =?UTF-8?q?(#883)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the #883 approval: reverting `agentspec`'s `try/except`, `claim`'s tombstone check or `registry`'s `validate_name` left all 858 tests passing, so three of the six fixes in `258da7a` were invisible to the suite. The standard this PR has applied since `6ebdd80` is "every fix has a regression test, verified by reverting its fix", and it had been applied to half that commit — and, as the review put it, the three without one are the three whose absence is invisible. Fourteen tests, each verified by reverting its fix independently: | Fix | Failures when reverted | |---|---| | `[limits]` conversions wrapped in `SpecError` | 4 | | tombstone never re-claimable | 1 | | `registry.parse` validates `agent` | 5 | The tombstone test covers the case that was actually wrong — `claim(corr, None)` against a tombstoned row, where `row[0] == userkey` is `None == None` and read as "already mine" — and also asserts a *tenant* re-claim now fails with "tombstoned" rather than "already owned by (single-tenant)", so the message points at the real reason. The registry tests cover the shapes `validate_name` rejects and, separately, that a valid name loads and an absent one stays empty so `ER_AGENT_NAME` still applies. A gate that rejects what it should allow is the other way to get this wrong. **Also closes the range-check gap the review flagged as on-the-record.** `timeout_s = -5` and `max_events = 0` loaded clean: timeout_s = -5.0 | max_events = 0 Defensible while `Limits` is documented inert, but a negative deadline is nonsensical input whether or not anything reads it yet, the module's own standard is that a policy which cannot be understood fails to load, and the validation was already right there. Rejecting now also means whoever wires up the `Popen` wrapper inherits a value they can trust instead of re-validating. `0` keeps its documented meaning for `timeout_s` and `max_output_bytes` ("no deadline" / "unbounded"), with a test pinning that the range check does not reject the sentinels; `max_events = 0` is refused because a run that may emit no events cannot report its own result. Tests: 875 passing (+14), ruff clean. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/eventrunner/agentspec.py | 19 +++++ eventing/tests/test_review_fixes.py | 121 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/eventing/eventrunner/agentspec.py b/eventing/eventrunner/agentspec.py index d9ad1bf7..308ddb7f 100644 --- a/eventing/eventrunner/agentspec.py +++ b/eventing/eventrunner/agentspec.py @@ -90,6 +90,9 @@ class Limits: Said plainly because the alternative is an operator writing `timeout_s = 900`, getting no error, and believing there is a deadline. + + Types AND ranges are validated at load even though nothing reads the values yet, so + the code that eventually applies them inherits a value it can trust. """ timeout_s: float = 0.0 # 0 = no spec-imposed deadline max_output_bytes: int = 0 # 0 = unbounded by the spec @@ -259,6 +262,22 @@ def parse(raw: str, *, name: str, base_dir: pathlib.Path | None = None) -> Agent ) except (TypeError, ValueError) as e: raise SpecError(f"agent {name!r}: limits must be numbers: {e}") from None + # Ranges, not just types. The review left this as a noted gap on the grounds that + # `Limits` is documented inert — but a negative deadline or a zero event budget is + # nonsensical input whether or not anything reads it yet, and the module's own + # standard is that a policy which cannot be understood fails to load. Rejecting now + # also means the code that eventually wraps `Popen` inherits a value it can trust + # rather than having to re-validate. + if limits.timeout_s < 0: + raise SpecError(f"agent {name!r}: limits.timeout_s must be >= 0 " + f"(0 means no spec-imposed deadline), got {limits.timeout_s}") + if limits.max_output_bytes < 0: + raise SpecError(f"agent {name!r}: limits.max_output_bytes must be >= 0 " + f"(0 means unbounded), got {limits.max_output_bytes}") + if limits.max_events < 1: + raise SpecError(f"agent {name!r}: limits.max_events must be >= 1 — a run that " + f"may emit no events cannot report its own result, got " + f"{limits.max_events}") return AgentSpec( name=name, diff --git a/eventing/tests/test_review_fixes.py b/eventing/tests/test_review_fixes.py index c6c50ad9..5b6dea9a 100644 --- a/eventing/tests/test_review_fixes.py +++ b/eventing/tests/test_review_fixes.py @@ -530,3 +530,124 @@ def claim(self, corr, userkey): with pytest.raises(sqlite3.OperationalError): Minter(index=Broken()).mint(userkey=UK) + + +# ---- round 3: the three fixes that shipped without a regression test ------ +# +# The #883 approval noted that reverting `agentspec`'s try/except, `claim`'s tombstone +# check or `registry`'s `validate_name` left all 858 tests passing — so three of the six +# fixes in 258da7a were invisible to the suite. The standard this PR has applied since +# 6ebdd80 is "every fix has a regression test, verified by reverting its fix", and it +# had been applied to half that commit. These close the gap. + + +@pytest.mark.parametrize("field,value", [ + ("timeout_s", '"900s"'), + ("max_output_bytes", '"4MB"'), + ("max_events", '"lots"'), +]) +def test_a_non_numeric_limit_raises_specerror_not_valueerror(tmp_path, field, value): + """`run_agent` catches only `SpecError`, so a bare `ValueError` from these three + conversions propagated out of it instead of becoming the `phase=error` event §5.1 + promises. `max_turns` is wrapped for exactly this reason; `[limits]` was the one + block in `parse` that escaped the module's own stated contract.""" + from eventrunner import agentspec + d = tmp_path / "a" + d.mkdir() + (d / "agent.toml").write_text(f"[limits]\n{field} = {value}\n") + with pytest.raises(agentspec.SpecError, match="limits must be numbers"): + agentspec.load(str(tmp_path), "a") + + +@pytest.mark.parametrize("block,match", [ + ("[limits]\ntimeout_s = -5\n", "timeout_s must be >= 0"), + ("[limits]\nmax_output_bytes = -1\n", "max_output_bytes must be >= 0"), + ("[limits]\nmax_events = 0\n", "max_events must be >= 1"), +]) +def test_a_nonsensical_limit_range_is_refused(tmp_path, block, match): + """The review left this as a noted gap, since `Limits` is documented inert. Closed + anyway: a negative deadline is nonsensical whether or not anything reads it yet, and + whoever wires up the `Popen` wrapper should inherit a value they can trust rather + than re-validate. `max_events = 0` is the interesting one — a run that may emit no + events cannot report its own result.""" + from eventrunner import agentspec + d = tmp_path / "a" + d.mkdir() + (d / "agent.toml").write_text(block) + with pytest.raises(agentspec.SpecError, match=match): + agentspec.load(str(tmp_path), "a") + + +def test_the_documented_sentinels_still_load(tmp_path): + """`0` means "no spec-imposed deadline"/"unbounded" per the docstring, so the range + check must not reject the documented defaults.""" + from eventrunner import agentspec + d = tmp_path / "a" + d.mkdir() + (d / "agent.toml").write_text("[limits]\ntimeout_s = 0\nmax_output_bytes = 0\n") + spec = agentspec.load(str(tmp_path), "a") + assert spec.limits.timeout_s == 0.0 + assert spec.limits.max_output_bytes == 0 + assert spec.limits.max_events == 2000 + + +def test_a_non_table_limits_block_is_refused(tmp_path): + from eventrunner import agentspec + d = tmp_path / "a" + d.mkdir() + (d / "agent.toml").write_text('limits = "nope"\n') + with pytest.raises(agentspec.SpecError, match="limits must be a table"): + agentspec.load(str(tmp_path), "a") + + +def test_a_tombstoned_correlation_cannot_be_reclaimed_by_anyone(tmp_path): + """`forget` clears `userkey` to NULL, so `claim(corr, None)` hit `None == None` and + returned as an idempotent re-claim — contradicting the guarantee `forget` makes, that + the id is reserved forever because a reissue derives the same unsalted `sessionuuid`. + + Unreachable from `Minter.mint`, which checks `exists()` first; reachable as soon as a + `correlationid` arrives from outside, which §7's triggers bring. + """ + from eventbridge.owner_index import Collision, OwnerIndex + owners = OwnerIndex(tmp_path / "eventbridge") + owners.claim("brave-otter-0001", UK) + owners.forget("brave-otter-0001") + + # The NULL owner is the case `row[0] == userkey` got wrong. + with pytest.raises(Collision, match="tombstoned"): + owners.claim("brave-otter-0001", None) + # A tenant failed correctly before, but must keep failing for the RIGHT reason: the + # message should say "tombstoned", not "already owned by (single-tenant)". + with pytest.raises(Collision, match="tombstoned"): + owners.claim("brave-otter-0001", OTHER) + assert owners.is_tombstoned("brave-otter-0001") + + +def test_a_registry_agent_name_is_validated_at_startup(): + """`agent` was the only field `parse` accepted unchecked, in a function whose + docstring promises every inconsistency is a refusal. It reaches `ce_agent` via + `by_userkey(...).agent` on a path that never sees `validate_name`, so a typo meant an + asynchronous `phase=error` on EVERY request from that user — the failure mode the + request path was moved away from in `6ebdd80`.""" + raw = json.dumps({"version": 1, "users": [ + {"issuer": "github", "userid": "alice", "agent": "Not A Label"}]}) + with pytest.raises(registry.RegistryError, match="DNS-1123"): + registry.parse(raw) + + +@pytest.mark.parametrize("bad", ["../../etc", "a/b", "x" * 64, "UPPER"]) +def test_every_invalid_registry_agent_shape_is_refused(bad): + raw = json.dumps({"version": 1, "users": [ + {"issuer": "github", "userid": "alice", "agent": bad}]}) + with pytest.raises(registry.RegistryError): + registry.parse(raw) + + +def test_a_valid_registry_agent_is_accepted_and_an_absent_one_falls_through(): + """The gate must not reject what it should allow: a good name loads, and an omitted + `agent` stays empty so the runner's own `ER_AGENT_NAME` applies.""" + r = registry.parse(json.dumps({"version": 1, "users": [ + {"issuer": "github", "userid": "alice", "agent": "triager"}, + {"issuer": "github", "userid": "bob"}]})) + assert r.lookup("github", "alice").agent == "triager" + assert r.lookup("github", "bob").agent == "" From f6c9f6406d71a495a074a610fe3d2f94619d37cd Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Mon, 5 Oct 2026 18:19:01 -0400 Subject: [PATCH 14/14] =?UTF-8?q?test(eventing):=20extend=20the=20=C2=A78.?= =?UTF-8?q?3=20rule=20to=20#884's=20seed=20paths,=20and=20cover=20kind-sig?= =?UTF-8?q?ned?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to rebasing onto `main` now that #884 and #890 have merged. Both predicted conflicts were in `agentdocs/README.md` and resolved as planned — `main`'s Phase 2 row (which gained §8 from #890) plus this branch's Phase 3 row. `test_manifests.py` auto-merged, keeping #884's `allowed_secrets` fix beside this branch's Phase 3 classification. That left two things the merge did not resolve by itself. **Extending `SECRET_PATH_VARS` to #884's seed paths needed the rule split, not widened.** #884's review predicted this tuple would be extended to `ER_/EB_SIGNING_KEY_PATH` and checked that doing so stays green. It does for the ConfigMap half and would NOT for the literal-value half, because those two variables hold a **mount point, not key material** — and #884 deliberately sets them as container `env` entries so the path sits beside the volume that supplies the seed, which is what `configmap.yaml` asks for. So the tuple is now two: - `SECRET_VALUE_VARS` — the three whose *value* is key material. No literal value anywhere, which is the original assertion. - `SEED_PATH_VARS` — #884's two. A literal value is correct; a ConfigMap key is not. Both still obey "never from `config.toml`", so that check covers all five now where it covered three. Treating a path naming a secret as if it were the secret would have forced #884 to either revert its own correct fix or carve itself out of the rule — the distinction is the thing worth encoding. **`kind-signed` was not in `OVERLAYS`.** #884 added the overlay and every assertion in this file silently skipped it — including the seed-path rule the PR exists to respect, the `replicas` invariants, and the no-credential checks. Added, which is also what makes the new assertion meaningful: with `kind-signed` absent, injecting a seed path back into `eventing-config` passed; with it present, that regression fails as it should. Verified both ways. Tests: 899 passing (875 here + #884's 24 picked up in the rebase), ruff clean, all four overlays build. Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- eventing/tests/test_manifests.py | 51 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/eventing/tests/test_manifests.py b/eventing/tests/test_manifests.py index 75046fd2..99a75627 100644 --- a/eventing/tests/test_manifests.py +++ b/eventing/tests/test_manifests.py @@ -20,9 +20,13 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] K8S = ROOT / "k8s" +# `kind-signed` (#884) is included deliberately: it was absent when that PR added the +# overlay, so every assertion in this file silently skipped the one overlay that turns +# signing on — including the seed-path rule the PR exists to respect. OVERLAYS = {"test": K8S / "overlays" / "test", "kind": K8S / "overlays" / "kind", - "demo": K8S / "overlays" / "demo"} + "demo": K8S / "overlays" / "demo", + "kind-signed": K8S / "overlays" / "kind-signed"} _HAVE_KUBECTL = shutil.which("kubectl") is not None @@ -166,12 +170,28 @@ def test_no_credential_is_ever_rendered_into_a_manifest(): #: rather than being remembered afterwards. `test_every_phase3_variable_is_classified` #: deliberately does NOT require these to be read, which is why it only asserts the #: ConfigMap side against live code. -SECRET_PATH_VARS = ( +SECRET_VALUE_VARS = ( "EB_NTFY_TOPIC_SECRET_PATH", "EB_CAPABILITY_SECRET_PATH", "EB_WEBHOOK_SECRETS_PATH", ) +#: The signing seed paths, from #884. Same §5 rule, different obligation — and the +#: distinction is the one that PR had to settle: these hold a **mount point**, not key +#: material, so a literal value is correct. What they must not do is come from the +#: committed ConfigMap: `configmap.yaml` says seed paths "name Secret mounts and belong +#: in the Deployment beside the volume", and #884 moved them to a container `env` entry +#: for exactly that reason after an earlier revision put them in `eventing-config` and +#: quoted the rule immediately before breaking it. +SEED_PATH_VARS = ( + "ER_SIGNING_KEY_PATH", + "EB_SIGNING_KEY_PATH", +) + +#: Both groups obey "never from `config.toml`", which is what +#: `test_secret_path_variables_are_never_set_from_the_committed_config` checks. +SECRET_PATH_VARS = SECRET_VALUE_VARS + SEED_PATH_VARS + #: The other side of the same rule: these record what an operator approved and grant #: nothing, so a ConfigMap (or an image path) is correct and a Secret would be cargo #: cult. Listed explicitly so the next variable added lands on one side deliberately. @@ -199,10 +219,21 @@ def test_secret_path_variables_are_never_set_from_the_committed_config(): @needs_kubectl def test_no_phase3_secret_is_inlined_into_a_manifest(): - """The §8.3 rule, checked where it would actually leak: rendered output.""" + """The §8.3 rule, checked where it would actually leak: rendered output. + + **A path naming a secret is not a secret.** `*_SIGNING_KEY_PATH` holds + `/etc/ce-signing/seed.hex` — a mount point, not key material — and #884 deliberately + sets it as a container `env` entry so it sits *beside the volume that supplies the + seed*, which is what `configmap.yaml` asks for. So a literal value is correct for + those, and the rule they must obey is a different one: the **seed itself** must never + appear, and the path must not be set from the committed ConfigMap. + + The variables whose *value* is key material — none of them read yet, see the note on + SECRET_PATH_VARS — must have no literal value anywhere. + """ for overlay in OVERLAYS: rendered = render(overlay) - for var in SECRET_PATH_VARS: + for var in SECRET_VALUE_VARS: # Referencing the variable is fine; giving it a literal sibling value is # not. A `secretKeyRef` has no `value:` on the same key. # @@ -213,6 +244,18 @@ def test_no_phase3_secret_is_inlined_into_a_manifest(): inlined = re.search(rf"name:\s*{re.escape(var)}\s*\n\s*value:", rendered) assert f"{var}:" not in rendered and not inlined, \ f"{overlay} appears to inline {var} rather than mounting a Secret" + # The seed-path variables: a literal value is fine, a ConfigMap key is not. + # `f"{var}:"` matches the ConfigMap `data:` spelling (`VAR: value`) but not the + # env-entry spelling (`- name: VAR` / `value: …`), which is the distinction. + for var in SEED_PATH_VARS: + assert f"{var}:" not in rendered, \ + (f"{overlay} sets {var} as a ConfigMap key; `configmap.yaml` says seed " + f"paths name Secret mounts and belong in the Deployment beside the " + f"volume (see #884)") + # And no seed material, by any route, in any overlay. + for overlay in OVERLAYS: + rendered = render(overlay) + assert "seed.hex:" not in rendered, f"{overlay} may inline a seed" #: Variables that pre-date §8.3 and are therefore out of its scope. Retro-classifying all