From 28787d21dc6050ce8756cef5e0cc1c073bac925a Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Tue, 9 Jun 2026 13:26:25 -0700 Subject: [PATCH 1/7] Add innovation plan files --- doc/plans/Innovation-1.1-pop-tokens.md | 172 ++++++ doc/plans/Innovation-1.2-vtpm-sealing.md | 159 ++++++ doc/plans/Innovation-1.3-measured-identity.md | 157 ++++++ doc/plans/Innovation-1.4-capability-scopes.md | 144 +++++ doc/plans/Innovation-2.1-canonical-request.md | 506 ++++++++++++++++++ .../Innovation-2.2-typed-policy-cedar.md | 137 +++++ .../Innovation-2.3-versioned-snapshots.md | 123 +++++ .../Innovation-2.4-differential-testing.md | 119 ++++ doc/plans/Innovation-3.1-hash-chained-log.md | 114 ++++ doc/plans/Innovation-3.2-otel-export.md | 99 ++++ doc/plans/Innovation-3.3-self-attestation.md | 108 ++++ doc/plans/Innovation-3.4-supply-chain.md | 91 ++++ doc/plans/Innovation-4.1-sk-lookup-bpf-lsm.md | 117 ++++ doc/plans/Innovation-4.2-core-unify-ebpf.md | 103 ++++ doc/plans/Innovation-4.3-ipv6-dual-stack.md | 110 ++++ .../Innovation-4.4-ebpf-throttling-lru.md | 111 ++++ .../Innovation-5.1-aks-container-native.md | 177 ++++++ .../Innovation-5.2-gate-more-endpoints.md | 104 ++++ doc/plans/Innovation-5.3-cross-cloud-port.md | 100 ++++ doc/plans/Innovation-6.1-policy-simulator.md | 110 ++++ doc/plans/Innovation-6.2-gpa-doctor.md | 88 +++ doc/plans/Innovation-6.3-rule-authoring-ux.md | 101 ++++ doc/plans/Innovation-6.4-wasm-rule-sandbox.md | 106 ++++ doc/plans/Innovation-7.1-io-uring-hot-path.md | 84 +++ doc/plans/Innovation-7.2-zero-copy-splice.md | 75 +++ .../Innovation-7.3-crate-consolidation.md | 90 ++++ doc/plans/Innovation-Directions.md | 450 ++++++++++++++++ doc/plans/Innovation-Plans-Milestones.md | 436 +++++++++++++++ 28 files changed, 4291 insertions(+) create mode 100644 doc/plans/Innovation-1.1-pop-tokens.md create mode 100644 doc/plans/Innovation-1.2-vtpm-sealing.md create mode 100644 doc/plans/Innovation-1.3-measured-identity.md create mode 100644 doc/plans/Innovation-1.4-capability-scopes.md create mode 100644 doc/plans/Innovation-2.1-canonical-request.md create mode 100644 doc/plans/Innovation-2.2-typed-policy-cedar.md create mode 100644 doc/plans/Innovation-2.3-versioned-snapshots.md create mode 100644 doc/plans/Innovation-2.4-differential-testing.md create mode 100644 doc/plans/Innovation-3.1-hash-chained-log.md create mode 100644 doc/plans/Innovation-3.2-otel-export.md create mode 100644 doc/plans/Innovation-3.3-self-attestation.md create mode 100644 doc/plans/Innovation-3.4-supply-chain.md create mode 100644 doc/plans/Innovation-4.1-sk-lookup-bpf-lsm.md create mode 100644 doc/plans/Innovation-4.2-core-unify-ebpf.md create mode 100644 doc/plans/Innovation-4.3-ipv6-dual-stack.md create mode 100644 doc/plans/Innovation-4.4-ebpf-throttling-lru.md create mode 100644 doc/plans/Innovation-5.1-aks-container-native.md create mode 100644 doc/plans/Innovation-5.2-gate-more-endpoints.md create mode 100644 doc/plans/Innovation-5.3-cross-cloud-port.md create mode 100644 doc/plans/Innovation-6.1-policy-simulator.md create mode 100644 doc/plans/Innovation-6.2-gpa-doctor.md create mode 100644 doc/plans/Innovation-6.3-rule-authoring-ux.md create mode 100644 doc/plans/Innovation-6.4-wasm-rule-sandbox.md create mode 100644 doc/plans/Innovation-7.1-io-uring-hot-path.md create mode 100644 doc/plans/Innovation-7.2-zero-copy-splice.md create mode 100644 doc/plans/Innovation-7.3-crate-consolidation.md create mode 100644 doc/plans/Innovation-Directions.md create mode 100644 doc/plans/Innovation-Plans-Milestones.md diff --git a/doc/plans/Innovation-1.1-pop-tokens.md b/doc/plans/Innovation-1.1-pop-tokens.md new file mode 100644 index 00000000..1c4f4a03 --- /dev/null +++ b/doc/plans/Innovation-1.1-pop-tokens.md @@ -0,0 +1,172 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Token format](#token) +4. [4. Mint & verify](#mint) +5. [5. Wire protocol](#wire) +6. [6. Integration](#integration) +7. [7. Rollout](#rollout) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 1.1** · **AuthN** + +# Detailed Design — Short-lived Proof-of-Possession Tokens + +Replace the long-lived HMAC signature header with a compact, audience-scoped, time-bound token derived from the latched key, so a leaked key file cannot be used to sign arbitrary requests offline or to replay captured ones. + +**Files affected:** `proxy_agent_shared/src/` (new `pop_token` module), `proxy_agent/src/key_keeper/key.rs`, `proxy_agent/src/proxy/proxy_server.rs`. + +> **Prerequisites:** None — foundational identity-layer change. Strengthened by [1.2 vTPM sealing](Innovation-1.2-vtpm-sealing.md) (key-source hardening) and [1.3 Measured identity](Innovation-1.3-measured-identity.md) (binding the signer). + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------------|------------|---------------------|-----------------------------| +| **High** kills replay + key-leak | **Medium** | **Fabric coupling** | **agent + WireServer/IMDS** | + +### 1.1 Goals + +- Each request bears a token valid for ≤ 30 s, bound to *caller*, *destination*, and *URL*. +- The latched key never appears on the wire and never signs raw HTTP — it signs a derived session key. +- Replay (pentest `B2`) becomes structurally impossible. +- A leaked key file (pentest `B3`) is still useless without live caller-fingerprint material (cgroup, pid-starttime, vTPM PCRs from direction 1.2). + +### 1.2 Non-goals + +- Replacing the underlying primitive (still HMAC-SHA256 in v1; PQ migration is out of scope). +- Asymmetric tokens — would force a fabric crypto change we want to defer. + +## 2. Today's Behavior + +GPA signs each authorized request with an HMAC over a static string derived from method, URL, and a coarse "time tick"; the signature is placed in `x-ms-azure-signature` with a sibling `x-ms-azure-time-tick` header. The HMAC key is the latched key written at provisioning. Once disclosed, it can sign anything indefinitely. + +The fabric checks the signature against the latched key it holds for this VM. There is no per-request nonce, no audience binding, and no caller binding. + +## 3. Token Format + +Compact, JWS-like, three base64url segments joined by `.`: + + HEADER = { "alg":"HS256", "kid":, "v":2 } + PAYLOAD = { + "iss": "gpa", // issuer + "aud": "wireserver" | "imds" | "hostga", + "sub": , // see §3.1 + "iat": , + "exp": , + "nbf": , + "jti": <128-bit random>, // nonce (anti-replay) + "url": , + "dip": , + "src": + } + SIG = HMAC-SHA256( derive_session_key(latched, jti), HEADER || "." || PAYLOAD ) + +### 3.1 Caller fingerprint (`sub`) + +- `sub = sha256( cgroup_id || pid_starttime_ns || exe_hash )`. +- `exe_hash` is the IMA / fs-verity hash from direction 1.3 when available; falls back to `processFullPath` bytes. +- The fabric does not interpret `sub`; it only ensures the same `sub` isn't reused after expiry. + +### 3.2 Session key derivation + + session_key = HKDF-SHA256( + ikm = latched_key, + salt = jti, + info = "gpa-pop-v2" || aud || dip + ) + +This means the latched key never directly produces a tag visible on the wire; recovering the latched key requires inverting HKDF, not HMAC. + +## 4. Mint & Verify + +### 4.1 Rust API + + pub struct PopToken(String); + + pub struct MintParams<'a> { + pub aud: Audience, + pub canonical_url_method_hash: [u8; 32], + pub destination: SocketAddr, + pub caller: &'a CallerFingerprint, + pub ttl: Duration, // clamp to <=30s + } + + impl PopToken { + pub fn mint(key: &LatchedKey, p: &MintParams) -> Result; + pub fn verify(token: &str, key: &LatchedKey, now: SystemTime, + expected_aud: Audience) -> Result; + } + +### 4.2 Constant-time comparison + +- Use `subtle::ConstantTimeEq` for the signature compare in `verify`. +- HMAC computation uses `hmac` crate with `sha2::Sha256`; both are constant-time and already in the dependency tree. + +### 4.3 Anti-replay storage + +- Agent side: nothing (tokens are stateless on the way out). +- Fabric side: bloom filter or LRU of recently-seen `jti` values keyed by `(aud, sub)` with TTL ≥ 2× max `exp - iat`. Detail belongs to the fabric design but the agent must pick `jti` from a CSPRNG with at least 128 bits. + +## 5. Wire Protocol + +### 5.1 Headers (v2) + +| Header | Direction | Notes | +|---------------------------|----------------|-----------------------------------------------------------------| +| `x-ms-azure-pop` | agent → fabric | The compact token from §3. | +| `x-ms-azure-pop-aud` | agent → fabric | Redundant audience hint to allow fast rejection before parsing. | +| `x-ms-azure-signature` | agent → fabric | Legacy header, still emitted during dual-emit phase. | +| `x-ms-azure-pop-rejected` | fabric → agent | Reason code on 401; consumed by GPA telemetry only. | + +### 5.2 Header stripping + +GPA **always** strips any inbound `x-ms-azure-pop*` and `x-ms-azure-signature*` headers from the client request before forwarding (pentest `B4`); never propagates client-supplied values. + +## 6. Integration Points + +| File | Change | +|---------------------------------------------|----------------------------------------------------------------------------------------------| +| `proxy_agent_shared/src/pop_token/` | New module: types, mint, verify, fuzz target. | +| `proxy_agent/src/key_keeper/key.rs` | Add `derive_session_key`; expose `kid()`. | +| `proxy_agent/src/proxy/proxy_server.rs` | Replace HMAC mint with `PopToken::mint`; keep legacy header behind `pop_v2.mode != enforce`. | +| `proxy_agent/src/proxy/proxy_authorizer.rs` | Compute canonical URL hash (reuse `CanonicalRequest` from 2.1) and caller fingerprint. | +| `config/GuestProxyAgent.*.json` | New `popToken.mode` = `off|dual|enforce`. | + +## 7. Rollout + +1. **Phase A — Off:** ship code, dormant. Unit / fuzz tests run in CI only. +2. **Phase B — Dual-emit:** emit both headers; fabric ignores PoP. Telemetry only. +3. **Phase C — Dual-verify:** fabric verifies PoP if present, accepts either. Agent telemetry tracks fabric verdicts via the `x-ms-azure-pop-rejected` header. +4. **Phase D — PoP-only:** fabric rejects requests without PoP. Legacy header removed in the next release. + +A region only advances to the next phase when error rate \< 0.001 % for 14 days. + +## 8. Test Strategy + +- Golden vectors signed by a reference implementation; agent must verify identical bytes. +- Property test: round-trip `mint → verify` always succeeds for fresh tokens; modifying any byte fails. +- Property test: skewing the clock by \> 60 s rejects; within ±60 s accepts (configurable skew). +- `cargo fuzz` on `verify`: must never panic. +- Pentest reruns: `B2` replay → REJECT; `B3` stolen-key replay on a different VM → REJECT once fabric checks `sub` stickiness. +- Soak: 1 million mint/verify pairs/sec on a single core baseline; track regressions. + +## 9. Risks & Mitigations + +- **Clock drift:** ±60 s tolerance; if NTP is broken GPA can request fabric time via WireServer health endpoint. +- **Header bloat:** typical token ≈ 380 bytes b64u; bounded. +- **Fabric rollout coupling:** dual-emit/dual-verify phases decouple agent and fabric releases. +- **HSM-bound key future:** session-key derivation already isolates the latched key, easing later migration to a vTPM-resident key (direction 1.2). + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|----------------------------------------|----------------------------------------| +| M1 | `pop_token` crate + 200 golden vectors | Fuzz clean for 1 CPU-day | +| M2 | Dual-emit behind flag in canary region | Zero correctness regressions vs legacy | +| M3 | Fabric dual-verify enabled | Pentest B2/B3 PASS | +| M4 | PoP-only enforcement | Legacy header deletion PR merged | + +Detail design for direction 1.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-1.2-vtpm-sealing.md b/doc/plans/Innovation-1.2-vtpm-sealing.md new file mode 100644 index 00000000..2a820fbf --- /dev/null +++ b/doc/plans/Innovation-1.2-vtpm-sealing.md @@ -0,0 +1,159 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Sealing design](#design) +4. [4. Backends](#backends) +5. [5. PCR / report bindings](#pcrs) +6. [6. Provisioning flow](#provision) +7. [7. Unseal flow](#unseal) +8. [8. Integration](#integration) +9. [9. Tests](#tests) +10. [10. Risks](#risks) +11. [11. Milestones](#milestones) + +**GPA** · **Direction 1.2** · **Hardware root of trust** + +# Detailed Design — vTPM / CVM Attestation Binding for the Latched Key + +Seal the latched key to a hardware root of trust so that copying `/var/lib/azure-proxy-agent/keys/*` to another VM, restoring an older snapshot, or booting a tampered image yields an unrecoverable key blob. + +**Files affected:** new `proxy_agent/src/key_keeper/sealing/` module, `proxy_agent/src/key_keeper/key.rs`, `proxy_agent/src/provision.rs`. + +> **Prerequisites:** None — foundational TPM/sealing layer. Enables [1.3 Measured identity](Innovation-1.3-measured-identity.md) and [3.3 Self-attestation](Innovation-3.3-self-attestation.md). + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-------------------------------------|------------|---------------------|---------------------------------| +| **High** kills key-theft & rollback | **Medium** | **Hardware matrix** | **agent + fabric KID registry** | + +### Goals + +- Stolen key blob on disk cannot be used on a different VM (pentest `B3`). +- Older sealed blob cannot be replayed after rotation (pentest `E5`). +- Booting a tampered kernel/agent invalidates the seal and forces re-provisioning. +- CVM (SEV-SNP / TDX) deployments get cryptographic guest-identity binding. + +### Non-goals + +- Generic vTPM management or attestation service implementation — we consume Azure's MAA (Microsoft Azure Attestation) where applicable. +- Migrating the HMAC algorithm itself (PoP work is direction 1.1). + +## 2. Today's Behavior + +The latched key is written as a plain file under `/var/lib/azure-proxy-agent/keys/` with mode 0600. Any root-level compromise reads it; a snapshot of the directory survives migration to a different VM; a previous file restored after rotation works against the fabric until rotation logic catches up. + +## 3. Sealing Design + +### 3.1 On-disk format + + // .sealed file format (binary, versioned) + struct SealedBlob { + magic: [u8;4] = b"GSP1", + version: u8 = 1, + backend: u8 = TPM2 | SNP | TDX | NOOP, + kid: [u8;16], // key id, same as PoP header kid + counter: u64, // monotonic, signed by backend (anti-rollback) + attestation_ref: [u8;32], // sha256 of attestation report or PCR digest + ciphertext_len: u32, + ciphertext: [u8], // AES-256-GCM-SIV under a backend-managed KEK + tag_len: u32, + tag: [u8], // backend-specific attestation/seal proof + } + +### 3.2 Layered keys + +- **LatchedKey** (random, 32 bytes) — what the rest of GPA already uses. +- **KEK** (key-encryption key) — derived inside the backend (vTPM sealed object, SNP-derived key, or TDX MRTD-bound key). +- Plaintext LatchedKey is unwrapped only into protected memory (`mlock` + `zeroize::Zeroizing`) and never reaches disk. + +## 4. Backends + +| Backend | Crate | Detection | Notes | +|---------|----------------|----------------------------------------|----------------------------------------------------------------------------------------------------------| +| `tpm2` | `tss-esapi` | `/dev/tpmrm0` on Linux; TBS on Windows | Uses TPM 2.0 sealed object + PolicyPCR. | +| `snp` | `sev` + custom | `SEV_STATUS` MSR / `/dev/sev-guest` | Derives KEK from SNP `VLEK`/`VCEK`; attestation report embeds VM measurement. | +| `tdx` | `tdx-attest` | TDX guest module device | KEK from TDX RTMR; attestation report from TD QUOTE. | +| `noop` | — | fallback | Encrypts with a host-stored DPAPI / Linux kernel keyring entry; explicitly weaker, only for legacy SKUs. | + +### 4.1 Backend trait + + pub trait SealingBackend: Send + Sync { + fn id(&self) -> BackendId; + fn seal(&self, plaintext: &[u8], policy: &SealPolicy) + -> Result; + fn unseal(&self, blob: &SealedBlob) + -> Result>, SealError>; + fn attest(&self, nonce: &[u8]) -> Result; + fn monotonic_counter_get(&self) -> Result; + fn monotonic_counter_increment(&self) -> Result; + } + +## 5. PCR / Report Bindings + +### 5.1 TPM2 (PCR selection) + +| PCR | Measures | Why | +|--------|----------------------------------------|--------------------------------| +| 0 | Firmware code | Detect firmware swap. | +| 4 | Bootloader | Detect bootloader swap. | +| 7 | Secure Boot policy | Detect SB disable / new keys. | +| 8 | Kernel + initrd (via grub measurement) | Detect kernel swap. | +| 9 / 14 | IMA log root | Detect agent binary tampering. | + +### 5.2 SNP / TDX + +- Bind the seal to the launch measurement (`MEASUREMENT` field in SNP report, `MRTD`+`RTMR` in TDX QUOTE). +- Include the agent binary digest in `REPORT_DATA`/`RTMR3` so post-launch upgrades trigger a controlled re-seal rather than failing open. + +### 5.3 Anti-rollback counter + +- TPM: NV index with `NVCounter`; agent reads and compares against blob counter on every unseal. +- SNP/TDX: use the agent's own VM-persistent virtual counter file *plus* a hash of the latest signed counter embedded in the next attestation request to the fabric (anchor to fabric monotonicity). + +## 6. Provisioning Flow + +agent fabric │ probe backend ───────────────────► │ ◄──── selected: tpm2 │ │ attest(nonce_from_fabric) ─────► │ ◄──── ack + bound_kid │ │ generate LatchedKey (CSPRNG) │ seal(LatchedKey, policy{PCR set, counter+1}) → SealedBlob │ persist SealedBlob to disk │ register(bound_kid, attestation_doc) ─► │ ◄──── 200 OK + +## 7. Unseal Flow on Service Start + +1. Read `.sealed` blob; reject if magic/version mismatch (fail-closed). +2. Call `backend.unseal`; on policy mismatch (PCR changed), enter **reprovision** state, request a fresh latch from fabric, do not serve traffic until success. +3. Verify `blob.counter ≥ backend.monotonic_counter_get()`; equal allowed once per boot, lesser rejected as rollback. +4. Place plaintext in `Zeroizing>`, `mlock` the buffer. +5. Erase plaintext on shutdown / SIGTERM (already happens with `zeroize` drop). + +## 8. Integration Points + +- `proxy_agent/src/key_keeper/key.rs` — add `load_sealed` / `store_sealed`, gated on a config flag. Existing plain-file path is the `noop` backend. +- `proxy_agent/src/provision.rs` — attestation handshake; expose `Reprovisioning` state to the status endpoint. +- `proxy_agent/src/service/` — surface backend id and `kid` in startup log + status JSON. +- Build: feature flags `seal-tpm2`, `seal-snp`, `seal-tdx` so distros without those crates still build. + +## 9. Tests + +- **Hermetic backend simulator** for CI — implements the trait with in-memory PCRs to validate flows without real hardware. +- Modify a simulated PCR after seal → unseal fails → agent enters reprovision; verify it serves nothing in the interim (closes a fail-open window). +- Rollback test: write `counter-1` blob → unseal rejected with `SealError::Rollback`. +- Cross-VM test in staging: snapshot `/var/lib/azure-proxy-agent` from VM-A, place on VM-B → unseal fails with policy mismatch. +- Tamper kernel cmdline → next boot PCR9 differs → reprovision triggered. +- Pentest `E5`: rollback rejected; `B3`: stolen blob useless on new VM. + +## 10. Risks & Mitigations + +- **Routine kernel updates trigger reprovision storms.** Mitigation: ride the OS update pipeline; pre-stage a new seal during update window before old kernel reboots. +- **Backend crate maturity.** Mitigation: ship behind feature flags; default to `noop` on legacy SKUs. +- **Latency at start.** TPM unseal ≈ 20–50 ms; acceptable because it's once per boot. +- **NV counter exhaustion (TPM).** Mitigation: increment only on rotation, not on each boot. + +## 11. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------------|------------------------------------------------| +| M1 | Trait + `noop` + simulator backend | All current tests still pass | +| M2 | TPM2 backend behind `seal-tpm2` | Tamper/rollback tests PASS on a TPM-enabled VM | +| M3 | SNP + TDX backends | Cross-VM pentest `B3` PASS on CVM SKUs | +| M4 | Default-on for CVM SKUs | Field error rate \< 0.001 % for 14 days | + +Detail design for direction 1.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-1.3-measured-identity.md b/doc/plans/Innovation-1.3-measured-identity.md new file mode 100644 index 00000000..0b8f20e7 --- /dev/null +++ b/doc/plans/Innovation-1.3-measured-identity.md @@ -0,0 +1,157 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Hash sources](#sources) +4. [4. eBPF event shape](#ebpf) +5. [5. Rule schema](#rule) +6. [6. Matcher](#matcher) +7. [7. Hash enrollment tool](#enroll) +8. [8. Rollout](#rollout) +9. [9. Tests](#tests) +10. [10. Risks](#risks) +11. [11. Milestones](#milestones) + +**GPA** · **Direction 1.3** · **Identity** + +# Detailed Design — Measured Caller Identity + +Replace path-string identity matching with a kernel-measured binary hash (IMA / fs-verity on Linux; code-integrity hash on Windows) so that bind-mount tricks, symlinks, and renamed exploits cannot impersonate allow-listed binaries. + +**Files affected:** `linux-ebpf/ebpf_cgroup.c`, `ebpf/redirect.bpf.c`, `proxy_agent/src/redirector/`, `proxy_agent/src/key_keeper/key.rs`, `proxy_agent/src/proxy/authorization_rules.rs`. + +> **Prerequisites:** [1.2 vTPM sealing](Innovation-1.2-vtpm-sealing.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|------------------------|------------|---------------------------|------------------| +| **High** kills C3 / D2 | **Medium** | **Kernel feature matrix** | **agent + eBPF** | + +### Goals + +- Identity rules match the binary that ran, not a filesystem path the caller can control. +- Bind-mount `/proc/self/exe` (pentest `C3`) and symlink-as-allowed-binary (pentest `D2`) both fail. +- Path rules continue to work for back-compat; hash rules are opt-in per identity. + +### Non-goals + +- Full TCB attestation of the executing process — that needs IPE/eBPF-LSM and is broader scope. +- Hash-based identity for scripts (the interpreter is what matters; document this). + +## 2. Today's Behavior + +The redirector reads the caller's executable via `/proc//exe` (Linux) or `NtQueryInformationProcess` (Windows) and reports the textual path as `processFullPath`. The rule engine compares this path string against `Identity::exePath`. Both ends can be spoofed in a user namespace by a non-root attacker with mount privileges in their own ns. + +## 3. Hash Sources by Platform + +| Platform | Source | Notes | +|---------------------|----------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| Linux (modern) | **fs-verity** root hash via `FS_IOC_MEASURE_VERITY` or `ima_file_hash` kfunc | Available on ext4/btrfs/f2fs with kernel ≥ 5.4; root hash is signed and cannot be modified. | +| Linux (fallback) | **IMA-Measurement** from `/sys/kernel/security/ima/ascii_runtime_measurements` | Requires `ima_policy=tcb`; agent reads at process exec via kprobe. | +| Linux (last resort) | SHA-256 of mmap'd file from kernel side via bpf helper `bpf_d_path` + read | More CPU; mark as "advisory" in rule. | +| Windows | **Code Integrity** Authenticode hash from `NtQuerySystemInformation(SystemModuleInformationEx)` or WDAC policy cache | Already computed by CI; reuse. | + +## 4. eBPF Event Shape + +Extend the audit map value to carry the measurement: + + // linux-ebpf/audit_event.h (shared with userspace via libbpf) + struct gpa_audit_event { + __u64 cgroup_id; + __u32 pid; + __u64 pid_starttime_ns; + __u32 uid; + __u32 gid; + __u32 measurement_kind; // 0=none, 1=fs-verity, 2=ima, 3=fallback-sha256 + __u8 measurement[32]; // sha256 truncated to 32 bytes (fs-verity uses root hash) + char exe_path[256]; // kept for diagnostics, NEVER used for matching when measurement_kind != 0 + }; + +The collector populates `measurement` in-kernel for fs-verity (single ioctl-equivalent), and lazily for IMA paths (cache by cgroup+pid_starttime). + +## 5. Rule Schema + + // JSON + "identities": [ + { + "name": "walinuxagent", + "userName": "root", + "exePath": "/usr/sbin/walinuxagent", // legacy, advisory + "exeMeasurement": { + "kind": "fs-verity|ima|sha256", + "value": "0x9f8a...", // hex sha256 + "enforce": true // when true, path mismatch -> reject + } + } + ] + +### 5.1 Compatibility + +- Rules without `exeMeasurement` behave as today. +- If `exeMeasurement.enforce == true` and the caller has no measurement available, identity does *not* match (fail-closed). +- `enforce=false` is "audit only": agent logs measurement mismatch but still applies path-based decision. + +## 6. Matcher Changes + + impl Identity { + pub fn is_match(&self, logger: &mut ConnectionLogger, claims: &Claims) -> bool { + // existing user/group/processName checks ... + + match (&self.exeMeasurement, &claims.exe_measurement) { + (Some(rule_m), Some(claim_m)) if rule_m.kind == claim_m.kind => { + if !constant_time_eq(&rule_m.value, &claim_m.value) { + logger.warn("measurement mismatch"); + return false; + } + } + (Some(rule_m), _) if rule_m.enforce => { + logger.warn("measurement required but unavailable"); + return false; // fail-closed + } + _ => {} // advisory mode or no measurement rule + } + + // existing exePath fallback ... + } + } + +## 7. Hash Enrollment Tool + +New CLI: `gpa identity hash `. + +- Detects available measurement source (fs-verity enabled? IMA active? otherwise sha256). +- Prints a ready-to-paste JSON snippet for the rules file. +- `--enable-verity` flag enables fs-verity on the file (`FS_IOC_ENABLE_VERITY`) if FS supports it. +- Batch mode for enumerating allow-listed extension handlers during package build. + +## 8. Rollout + +1. Ship eBPF + claims plumbing, no rule changes. Audit-log measurement values only. +2. Customers add `exeMeasurement.enforce=false` entries; observe divergence in logs. +3. Flip to `enforce=true` per rule when customers are ready. +4. Document fs-verity prerequisites and provide enable scripts for standard images. + +## 9. Tests + +- Bind-mount a copy of `/bin/cat` over `/usr/sbin/walinuxagent` path → measurement mismatch → deny (pentest `C3`). +- Symlink an allowed binary → fs-verity / IMA root hash differs → deny (pentest `D2`). +- Disable IMA, no fs-verity → with `enforce=true`, deny; with `enforce=false`, allow + audit warning. +- Property test: scriptable interpreter (`python /opt/foo.py`) reports interpreter hash, not script — assert documentation explicit. + +## 10. Risks + +- **fs-verity coverage is partial.** Mitigation: tooling to enable per file; document IMA fallback. +- **Updates flip the hash.** Mitigation: rule schema accepts `"value": ["hash_a", "hash_b"]` list during rolling upgrade windows. +- **Interpreter scripts.** Document — hash is over the interpreter; identify scripts by additional `cmdline` match if needed. + +## 11. Milestones + +| M | Deliverable | Exit | +|-----|-----------------------------------------------------------------------|------------------------------------------------------------------| +| M1 | Extend eBPF audit event + claims | Measurements visible in connection log; no rule semantics change | +| M2 | Rule schema + matcher | Round-trip tests for advisory mode | +| M3 | `gpa identity hash` CLI | Shipped in setup package | +| M4 | Enforce-mode for first-party rules (walinuxagent, host-side handlers) | Pentest C3, D2 PASS | + +Detail design for direction 1.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-1.4-capability-scopes.md b/doc/plans/Innovation-1.4-capability-scopes.md new file mode 100644 index 00000000..f07df5f2 --- /dev/null +++ b/doc/plans/Innovation-1.4-capability-scopes.md @@ -0,0 +1,144 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Scope model](#model) +3. [3. URL classifier](#classifier) +4. [4. Rule schema](#schema) +5. [5. Evaluation](#eval) +6. [6. Static analysis](#analysis) +7. [7. Migration](#migration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 1.4** · **AuthZ** + +# Detailed Design — Capability-style Scoped Grants + +Move from "path X is allowed for identity Y" to verifiable, typed scopes (e.g. `imds:identity:read`). A classifier maps each request to a typed `(Action, Resource)` pair; the matcher just checks scope containment. + +**Files affected:** new `proxy_agent/src/proxy/scope/` module, integrates with the canonical request model (2.1). + +> **Prerequisites:** [2.1 Canonical request](Innovation-2.1-canonical-request.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|--------------------------------|------------|------------------------|----------------| +| **High** enables analyzability | **Medium** | **Low** additive layer | **agent only** | + +### Goals + +- Decouple "what the customer wants to allow" from "how the URL happens to be spelled." +- Make rules statically analyzable: "does any rule grant unauthenticated WireServer write?" +- Eliminate URL-encoding bypass categories because the classifier normalizes once per endpoint. + +## 2. Scope Model + + // proxy_agent/src/proxy/scope/mod.rs + pub struct Scope { + pub service: ServiceId, // imds | wireserver | hostga + pub resource: ResourceId, // instance | identity | goalstate | extensions | ... + pub action: ActionId, // read | write | invoke | enumerate + pub qualifier: Option, // e.g. tenant id, extension name + } + + impl Scope { + pub fn satisfies(&self, required: &Scope) -> bool; + // exact match, or wildcard semantics: read < write < admin; * matches all. + } + +Wire form: `service:resource:action[:qualifier]` e.g. `imds:identity:read`, `wireserver:goalstate:read`, `hostga:extensions:status:write:GuestProxyAgent`. + +## 3. URL Classifier + +A single table maps `(Destination, CanonicalRequest)` → required `Scope`. Built from the canonical model (2.1) so the matcher never re-parses strings. + + // proxy_agent/src/proxy/scope/classifier.rs + pub fn required_scope(req: &CanonicalRequest) -> Result; + + // Backing table (compile-time built): + const IMDS_TABLE: &[(&[&str], Method, Scope)] = &[ + (&["metadata","instance"], Method::GET, scope!("imds:instance:read")), + (&["metadata","identity","oauth2","token"], Method::GET, scope!("imds:identity:read")), + (&["metadata","attested","document"], Method::GET, scope!("imds:attested:read")), + // ... + ]; + +### 3.1 Unknown URLs + +- Anything not in the table maps to a synthetic `imds:unknown:read` scope. +- Default rules deny unknown scopes; explicit allow-listing per scope keeps rules small. + +## 4. Rule Schema + + { + "version": 2, + "grants": [ + { + "identity": "walinuxagent", + "scopes": ["wireserver:goalstate:read", "wireserver:extensions:status:write"] + }, + { + "identity": "*", + "scopes": ["imds:instance:read"] + } + ] + } + +Legacy `privileges + roles + assignments` shape is compiled down to capability grants at load time. + +## 5. Evaluation + + fn authorize(req: &CanonicalRequest, caller: &ResolvedIdentity) -> Decision { + let required = classifier::required_scope(req)?; + let granted = caller.scopes(); // pre-computed at rule-load time + if granted.iter().any(|g| g.satisfies(&required)) { + Decision::Allow { matched_scope: required } + } else { + Decision::Deny { required } + } + } + +- O(N_scopes) per request where N_scopes is typically ≤ 10 — much smaller than today's privilege list. +- Match metadata captured in the decision so audit and divergence telemetry can attribute precisely. + +## 6. Static Analysis + +Because scopes are typed and finite, a separate `gpa policy analyze` command can answer: + +- "Which identities can write to WireServer?" +- "Are there grants for the synthetic `*:unknown:*` scope?" +- "Which scopes are unreachable given the URL classifier?" +- "Diff between current and proposed rule files in scope-space, not text-space." + +## 7. Migration + +1. **Phase A:** ship classifier + scope evaluator behind feature flag; dual-evaluate (legacy + scope), log divergences (re-use the same shadow-mode plumbing as 2.1). +2. **Phase B:** tool to auto-convert legacy `privileges` to scope grants; require human review of conversions. +3. **Phase C:** flip enforcement to scopes; keep legacy adapter for one release. +4. **Phase D:** delete legacy path. + +## 8. Tests + +- Property test: every canonical request produces exactly one required scope (totality). +- Golden vectors: every documented IMDS / WireServer endpoint has a stable scope mapping pinned in tests. +- Differential test: scope-evaluator decision == legacy decision for every request in a captured production trace. +- Pentest re-runs: `D1`/`C7` bypasses produce the same scope as the canonical form, so they cannot escape via spelling tricks. + +## 9. Risks + +- **Classifier table drift** when IMDS adds endpoints. Mitigation: scope mapping ships with the agent; an unknown URL falls into `:unknown:` and is denied by default — fail-closed. +- **Customer rules written in old style.** Mitigation: dual-run for one release; provide auto-converter. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|-------------------------------------------|----------------------------------------------| +| M1 | Scope + classifier types + table for IMDS | Unit tests green for IMDS endpoints | +| M2 | Table for WireServer + HostGAPlugin | Full endpoint coverage doc reviewed | +| M3 | Dual-eval in shadow mode | Zero divergence vs legacy | +| M4 | Auto-converter + enforce | One release in enforce mode without rollback | +| M5 | Legacy removal + `policy analyze` | Codebase reduction recorded | + +Detail design for direction 1.4. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-2.1-canonical-request.md b/doc/plans/Innovation-2.1-canonical-request.md new file mode 100644 index 00000000..7cfb50d0 --- /dev/null +++ b/doc/plans/Innovation-2.1-canonical-request.md @@ -0,0 +1,506 @@ +## Sections + +1. [1. Overview & Goals](#overview) +2. [2. Today's Behavior](#today) +3. [3. Threats & Bypass Patterns](#threats) +4. [4. CanonicalRequest Model](#model) +5. [5. Normalization Pipeline](#pipeline) +6. [6. Public API & Rust Sketch](#api) +7. [7. Error Taxonomy & Fail-Closed](#errors) +8. [8. Integration Points](#integration) +9. [9. Shadow-Mode Rollout](#shadow) +10. [10. Test Strategy](#tests) +11. [11. Performance Budget](#perf) +12. [12. Telemetry & Observability](#telemetry) +13. [13. Risks & Open Questions](#risks) +14. [14. Milestones](#milestones) +15. [Appendix A — Vector Table](#appendix) + +**GPA** · **Direction 2.1** · **Security-critical refactor** + +# Detailed Design — Canonical Request Model + +A single, total, well-tested normalization step shared by rule loading and request matching, designed to eliminate the rule/request asymmetry that produces SSRF-style AuthZ bypasses. + +**Primary files affected:** `proxy_agent/src/proxy/authorization_rules.rs`, `proxy_agent/src/key_keeper/key.rs`, new module `proxy_agent/src/proxy/canonical/`. + +> **Prerequisites:** None — foundational request-normalization layer. Required by [1.4](Innovation-1.4-capability-scopes.md), [2.2](Innovation-2.2-typed-policy-cedar.md), [2.4](Innovation-2.4-differential-testing.md), [5.2](Innovation-5.2-gate-more-endpoints.md), [5.3](Innovation-5.3-cross-cloud-port.md), [6.1](Innovation-6.1-policy-simulator.md). + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-------------------------------|-------------------------|-----------------------------|----------------| +| **High** closes a vuln family | **Medium** ~2–3 sprints | **Low** shadow-mode rollout | **agent only** | + +### 1.1 Problem statement + +Today the rule-matching pipeline performs ad-hoc, partial normalization in *two different places* — once when rules are loaded and once when requests are matched. The two normalizations are not byte-identical, which creates a class of bypass where the attacker crafts a URL that the agent considers different from the rule pattern but that the upstream metadata service treats as semantically equivalent. + +### 1.2 Goals + +- **One normalizer, one type.** Both rules and requests are reduced to a single canonical form (`CanonicalRequest`) before they ever meet the matcher. +- **Total function with explicit failure.** The normalizer either returns a fully-canonical value or a typed error; the matcher never sees ambiguous input. +- **Fail-closed semantics.** Any normalization error denies the request and logs a structured event. +- **Byte-stable output.** Round-tripping a canonical form through the normalizer yields the same bytes (idempotent). This is the property property-tests will enforce. +- **Zero behavior change at cutover.** Shadow-mode dual-evaluation must show 0 divergences for N days before flipping enforcement. + +### 1.3 Non-goals + +- Replacing the policy language itself (that is Direction 2.2 — Cedar). +- Identity normalization for users/processes (Direction 1.3 — measured identity). +- Changing the on-wire request format sent upstream. We canonicalize for *matching*; the request forwarded to IMDS / WireServer is the original. + +## 2. Today's Behavior (and why it's fragile) + +### 2.1 Normalization in `authorization_rules.rs` + +At rule load time (`ComputedAuthorizationItem::from_authorization_item`), each privilege's path and query parameters are lowercased: + + let normalized = Privilege { + name: privilege.name, + path: privilege.path.to_lowercase(), + queryParameters: privilege.queryParameters.map(|qp| { + qp.into_iter() + .map(|(k, v)| (k.to_lowercase(), v.to_lowercase())) + .collect() + }), + }; + +At request time (`ComputedAuthorizationItem::is_allowed`), the request URL is percent-decoded once then lowercased: + + let decoded_path = percent_encoding::percent_decode_str(request_url.path()) + .decode_utf8_lossy(); + let lowered_request_path = decoded_path.to_lowercase(); + +The actual match (`Privilege::is_match` in `key.rs`) does: + +- `actual_path.starts_with(&self.path)` +- splits on `?` in the *decoded* path to harvest extra query pairs (handles `%3F` trick) +- compares query parameters case-insensitively with one more percent-decode on the key + +### 2.2 The asymmetries + +| Step | Rule side | Request side | Risk | +|-------------------------------------|--------------------------|---------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| Percent decoding | Not applied to rule path | Applied once to request path | A rule containing `%2F` by accident becomes unreachable; a request can introduce decoded characters the rule author did not anticipate. | +| Path segment collapsing (`..`, `.`) | None | None | `/metadata/identity/../identity/oauth2/token` bypasses a deny on `/metadata/identity/oauth2`. | +| Trailing slash | Author-controlled | Author-controlled | Prefix `starts_with` means rule `/metadata` matches `/metadata-attacker`. | +| Matrix params `;foo=bar` | Not handled | Not handled | Some HTTP stacks strip them, some don't. | +| Host normalization | N/A (no host in rule) | N/A | Pentest C7: `0xa9fea9fe`, `2852039166`, `[::ffff:169.254.169.254]` reach the same IMDS. | +| UTF-8 validity | Assumed | `decode_utf8_lossy` silently replaces | Lossy substitution may yield matches the rule author didn't intend. | +| Query key/value decoding | Lowercased only | Decoded again at match time | Double-encoding (`%2525`) yields different views. | + +## 3. Threats & Bypass Patterns + +The canonical model targets, at minimum, every pattern in pentest scenarios `D1` and `C7`. + +### 3.1 URL-encoding differentials (pentest D1) + +- `%2F` vs `/`, mixed case `%2f`. +- Double-encoding: `%252e%252e`. +- Overlong UTF-8 for `/`. +- Semicolon matrix params on path segments. +- Trailing dot or whitespace in path. +- Embedded `?` via `%3F` that re-introduces query parameters into the path string. + +### 3.2 Host-form differentials (pentest C7) + +- IPv4 dotted: `169.254.169.254` +- IPv4 decimal: `2852039166` +- IPv4 hex: `0xa9fea9fe` +- IPv4 octal: `0251.0376.0251.0376` +- IPv4-mapped IPv6: `[::ffff:169.254.169.254]`, `[::ffff:a9fe:a9fe]` +- Uppercased hostnames, trailing dots: `METADATA.azure.internal.` +- Userinfo smuggling: `http://attacker@169.254.169.254/` +- Port-form smuggling: `http://169.254.169.254:80@evil/` + +### 3.3 Header / line smuggling (out of scope, but related) + +Request smuggling at the HTTP framing layer is handled separately by Hyper config + pentest A3/A4. The canonical model assumes Hyper produced a well-formed `hyper::Uri`. + +## 4. The CanonicalRequest Model + +### 4.1 Type + + // proxy_agent/src/proxy/canonical/mod.rs + #[derive(Clone, Debug, PartialEq, Eq, Hash)] + pub struct CanonicalRequest { + /// HTTP method, uppercased ASCII (GET, POST, ...). + pub method: Method, + + /// Canonical destination (already classified as one of GPA's known endpoints). + pub destination: Destination, + + /// Canonical path segments: percent-decoded, NFC-normalized, lowercased, + /// with `.` collapsed, `..` resolved against earlier segments, matrix params stripped. + /// Always begins with the empty root segment; never contains empty segments + /// except the final one when the original ended with `/`. + pub path_segments: Vec, + + /// Whether the original path had a trailing slash (preserved as a single bit so + /// rules can opt to be slash-sensitive without re-introducing string-level asymmetry). + pub trailing_slash: bool, + + /// Query parameters in a canonical multi-map form: keys lowercased + decoded once, + /// values decoded once, preserved order is not significant (BTreeMap of Vec). + pub query: BTreeMap>, + } + + #[derive(Clone, Debug, PartialEq, Eq, Hash)] + pub enum Destination { + Imds, // 169.254.169.254 in any encoding, port 80 + WireServer, // 168.63.129.16:80 + HostGaPlugin, // 168.63.129.16:32526 + Unknown { // anything else; matcher will deny unless an explicit rule allows + family: AddrFamily, + ip: IpAddr, + port: u16, + host_text: Option, // original host text for audit, never used for matching + }, + } + +### 4.2 Invariants (checked by debug asserts and property tests) + +- **Idempotent:** `canonicalize(canonicalize(x)) == canonicalize(x)`. +- **Total:** for every `hyper::Uri`, the function returns either `Ok(CanonicalRequest)` or a typed `CanonError`; it never panics. +- **Round-trip-stable rendering:** a debug `Display` impl produces a string that, when re-parsed and canonicalized, yields the same value. +- **UTF-8 strict:** invalid UTF-8 in path or query is an error, not a lossy replacement. +- **No host text in matching:** the matcher only sees the typed `Destination` enum, never the raw host string. + +## 5. The Normalization Pipeline + +Each step is a small pure function with its own unit tests. + +hyper::Uri │ ▼ parse_scheme_method (must be http; reject https/ws/...; method allow-list) │ ▼ classify_destination (IP/host -\> Destination enum; covers numeric forms) │ ▼ validate_userinfo (must be empty; reject \`user@host\`) │ ▼ decode_path_once (single percent-decode; reject malformed %XY; reject overlong UTF-8) │ ▼ reject_control_chars (no CR/LF/NUL/HTAB after decode) │ ▼ nfc_normalize (Unicode NFC) │ ▼ ascii_lowercase_path (path is matched case-insensitively) │ ▼ split_segments (split on '/'; collapse \`.\`; resolve \`..\`; error on underflow) │ ▼ strip_matrix_params (drop \`;k=v\` suffix on each segment, preserve segment text only) │ ▼ decode_query_once (k/v percent-decode once; error on malformed; lowercase keys) │ ▼ reject_embedded_query (if decoded path now contains '?' -\> error: ambiguous) │ ▼ fold_into_btreemap (group by key; values preserve insertion order within a key) │ ▼ CanonicalRequest + +### 5.1 Step details + +#### 5.1.1 `classify_destination` + +- If host is a bracketed IPv6, parse with `std::net::Ipv6Addr`; if it is IPv4-mapped, project to IPv4 and continue. +- If host parses as `Ipv4Addr` directly, use it. +- Else attempt the historic numeric forms manually: dotted-quad with any base per octet (octal-leading-zero, hex-leading-`0x`, plain decimal), and 32-bit packed forms. A small dedicated parser, not `inet_aton`, because `inet_aton` behavior is libc-dependent. +- Map the resolved IP+port to `Destination` via a constant table; unknown destinations land in `Destination::Unknown`. +- Hostnames that are not IPs (e.g. `metadata.azure.internal`) are *not* resolved at this layer — DNS is a confused-deputy surface. They are returned as `Unknown { host_text: Some(...) }` and require an explicit allow rule keyed on host text. + +#### 5.1.2 `decode_path_once` + +- One pass of percent decoding. A second pass is never attempted — that is exactly the asymmetry we want to remove. +- Malformed sequences (`%2`, `%ZZ`) → `CanonError::MalformedPercent`. +- Detect overlong UTF-8 encodings of ASCII (e.g. `%C0%AF` for `/`) → `CanonError::OverlongUtf8`. + +#### 5.1.3 `split_segments` + dot-segment resolution (RFC 3986 §5.2.4) + +- Empty segments collapsed (treat `//` as `/`). +- `.` dropped. +- `..` pops the previous segment; popping past root is an error (`CanonError::PathUnderflow`) rather than a no-op, because a real client would never produce it. + +#### 5.1.4 `strip_matrix_params` + +- For each segment, drop everything after the first `;`. +- Document this clearly: matrix params are **never** used in authorization decisions. + +#### 5.1.5 `reject_embedded_query` + +- If the decoded-and-rebuilt path contains a literal `?`, the request is ambiguous: an attacker may have used `%3F` to smuggle query into the path. Today's matcher tries to rescue this; the new model rejects it as an error and logs. + +## 6. Public API & Rust Sketch + +### 6.1 Module layout + + proxy_agent/src/proxy/canonical/ + ├── mod.rs // CanonicalRequest, Destination, CanonError, canonicalize() + ├── destination.rs // IP/host classification + numeric-form parser + ├── path.rs // decode + dot-segment + matrix-strip + ├── query.rs // decode + fold into BTreeMap + ├── rule.rs // canonicalize a rule pattern into CanonicalPattern + └── tests/ + ├── vectors.rs // 300+ golden vectors from pentest D1 / C7 + ├── proptests.rs // proptest invariants + └── differential.rs // dual-evaluate against legacy matcher in shadow mode + +### 6.2 Public surface + + pub fn canonicalize(uri: &hyper::Uri, method: &hyper::Method) + -> Result; + + /// Canonical form of a rule pattern. Same pipeline, but path segments may end + /// in a "*" sentinel to mark prefix match, and `Destination` may be `Any` for + /// rules that intentionally span endpoints. + pub struct CanonicalPattern { /* ... */ } + + pub fn canonicalize_pattern(raw: &RawPrivilege) -> Result; + + /// Matching is now a pure structural comparison on canonical forms. + impl CanonicalPattern { + pub fn matches(&self, req: &CanonicalRequest) -> bool; + } + +### 6.3 Error type + + #[derive(Debug, thiserror::Error)] + pub enum CanonError { + #[error("scheme not http")] SchemeNotHttp, + #[error("method not allowed")] MethodNotAllowed, + #[error("userinfo present in URL")] UserinfoPresent, + #[error("malformed percent-encoding")] MalformedPercent, + #[error("overlong UTF-8 in path/query")] OverlongUtf8, + #[error("invalid UTF-8 in path/query")] InvalidUtf8, + #[error("control character in path/query")]ControlChar, + #[error("path traversal past root")] PathUnderflow, + #[error("embedded '?' after decoding")] EmbeddedQuery, + #[error("unparseable host")] BadHost, + #[error("unparseable port")] BadPort, + } + + impl CanonError { + /// All variants are fail-closed; this is here so callers can record a + /// stable string for telemetry / pentest assertions. + pub fn code(&self) -> &'static str { /* ... */ } + } + +### 6.4 Matcher call site (after the change) + + // Replaces ComputedAuthorizationItem::is_allowed's URL handling. + let canon = match canonical::canonicalize(&request.uri(), request.method()) { + Ok(c) => c, + Err(e) => { + logger.write(LoggerLevel::Warn, + format!("Canonicalization failed: {} ({})", e, e.code())); + return false; // fail-closed + } + }; + for pattern in self.compiled_patterns.iter() { + if pattern.matches(&canon) { /* identity check ... */ } + } + +## 7. Error Taxonomy & Fail-Closed Semantics + +| Error | Likely cause | Action | Audit event code | +|--------------------|---------------------------------------|-----------|-------------------| +| `SchemeNotHttp` | WS upgrade probe (pentest A4) | Deny; 405 | `CANON_SCHEME` | +| `MethodNotAllowed` | CONNECT / TRACE | Deny; 405 | `CANON_METHOD` | +| `UserinfoPresent` | Host smuggling attempt | Deny; 400 | `CANON_USERINFO` | +| `MalformedPercent` | Truncated / non-hex `%XX` | Deny; 400 | `CANON_PCT` | +| `OverlongUtf8` | Classic IDS-bypass payload | Deny; 400 | `CANON_OVERLONG` | +| `InvalidUtf8` | Random bytes or wrong codec | Deny; 400 | `CANON_UTF8` | +| `ControlChar` | CRLF injection attempt | Deny; 400 | `CANON_CTRL` | +| `PathUnderflow` | Too many `..` | Deny; 400 | `CANON_UNDERFLOW` | +| `EmbeddedQuery` | `%3F` smuggling | Deny; 400 | `CANON_EMBQ` | +| `BadHost` | Mixed numeric forms that fail parsing | Deny; 400 | `CANON_HOST` | +| `BadPort` | Out-of-range port | Deny; 400 | `CANON_PORT` | + +**Fail-closed rule:** every `CanonError` path returns `false` from the matcher and emits a structured audit entry that includes the error code, the original (redacted) URL, the caller cgroup id, and the active `policy_epoch`. There is no "best effort" branch. + +## 8. Integration Points + +| File | Today | After change | +|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `proxy_agent/src/proxy/authorization_rules.rs` | Lowercases privilege path/query during `from_authorization_item`; percent-decodes request path during `is_allowed`. | Calls `canonical::canonicalize_pattern` at load time, `canonical::canonicalize` at request time; *no* ad-hoc string ops. | +| `proxy_agent/src/key_keeper/key.rs` | `Privilege::is_match` does `starts_with` + embedded-query rescue + per-key percent-decode. | Method removed (or wraps `CanonicalPattern::matches` for back-compat callers). | +| `proxy_agent/src/proxy/proxy_authorizer.rs` | Calls `is_allowed(uri, claims)`. | Calls `canonicalize` once, then `is_allowed(canon, claims)`; logs `CanonError` on failure. | +| `proxy_agent/src/proxy/proxy_server.rs` | Hands raw `Uri` down. | Unchanged; canonical form is computed once inside the authorizer and cached on the connection context. | +| `proxy_agent/src/key_keeper/local_rules.rs` | Lowercases rule fields during merge. | Calls `canonicalize_pattern`; rejects rules that fail canonicalization (fail-closed). | + +## 9. Shadow-Mode Rollout + +The canonicalizer ships before the matcher cuts over. + +### 9.1 Mode flag + + // In GuestProxyAgent.linux.json / .windows.json + "canonicalRequest": { + "mode": "shadow" // "off" | "shadow" | "enforce" + } + +- **off** — legacy path only (default in first release). +- **shadow** — legacy decides; canonical runs in parallel; divergences logged. +- **enforce** — canonical decides; legacy still computes for divergence telemetry. + +### 9.2 Divergence record + + { + "ts": "2026-06-01T12:34:56Z", + "policy_epoch": 174, + "request_uri_redacted": "/metadata/identity/oauth2/token?api-version=2018-02-01", + "legacy_decision": "allow", + "canon_decision": "deny", + "canon_error": null, + "matched_rule_id": "imds.identity.read", + "caller_cgroup": "/sys/fs/cgroup/system.slice/walinuxagent.service", + "delta_reason": "trailing_slash_difference" + } + +### 9.3 Cutover criteria + +- ≥ 14 consecutive days with zero divergences across the production fleet sample. +- All pentest D1 and C7 vectors PASS in enforcement mode in the CI canary. +- p99 added latency \< 100 µs (measured during shadow mode). +- One full release in **shadow** behind a feature flag before any region flips to **enforce**. + +## 10. Test Strategy + +### 10.1 Golden vectors + +A frozen table of `(input_uri, expected_canonical | expected_error)`. The seed set comes from: + +- Every `D1` and `C7` case in `pentest/linux/DESIGN.md`. +- OWASP URL-canonicalization corpus. +- Hand-curated IMDS / WireServer real-world URLs harvested from production logs (redacted). + +### 10.2 Property tests (`proptest`) + + proptest! { + #[test] + fn idempotent(uri in any_uri()) { + if let Ok(c1) = canonicalize_uri(&uri) { + let c2 = canonicalize_uri(&c1.render()).unwrap(); + prop_assert_eq!(c1, c2); + } + } + + #[test] + fn no_panics(uri_bytes in any::>()) { + let _ = std::panic::catch_unwind(|| { + if let Ok(uri) = hyper::Uri::try_from(uri_bytes) { + let _ = canonicalize(&uri, &hyper::Method::GET); + } + }).unwrap(); + } + + #[test] + fn host_form_equivalence(ip in any::()) { + let dotted = format!("http://{}/x", ip); + let decimal = format!("http://{}/x", u32::from(ip)); + let hex = format!("http://0x{:x}/x", u32::from(ip)); + prop_assert_eq!( + canonicalize_str(&dotted).map(|c| c.destination), + canonicalize_str(&decimal).map(|c| c.destination), + ); + prop_assert_eq!( + canonicalize_str(&dotted).map(|c| c.destination), + canonicalize_str(&hex).map(|c| c.destination), + ); + } + } + +### 10.3 Differential test against legacy matcher + +Bound the legacy matcher and the new canonical matcher to the same rule set and the same request stream (harvested from production logs). Any divergence is a CI failure during the enforce-prep window. + +### 10.4 Fuzzing + +- `cargo fuzz` target on `canonicalize(bytes)` — must never panic. +- Second target on `CanonicalPattern::matches` — must never panic; pattern produced from random rule JSON. +- Run for ≥ 1 CPU-day before each release; record corpora in `proxy_agent/src/proxy/canonical/tests/corpus/`. + +### 10.5 Pentest re-runs + +Add a new pentest phase in `pentest/linux/phase4_rules_fuzz/`: + +- **S20** — every D1 vector must return identical decisions in legacy and canonical modes (or canonical-strictly-stricter). +- **S21** — every C7 host form must resolve to the same `Destination` as the dotted form. +- **S22** — invalid UTF-8, overlong UTF-8, and embedded `?` must produce `CanonError` and a deny. + +## 11. Performance Budget + +| Operation | Target p50 | Target p99 | Notes | +|-----------------------------------|------------|------------|------------------------------------------------------------------------------| +| `canonicalize` (typical IMDS GET) | ≤ 5 µs | ≤ 30 µs | One pass each over path and query; no allocations beyond small Vec/BTreeMap. | +| `CanonicalPattern::matches` | ≤ 1 µs | ≤ 5 µs | Slice-equality over pre-sized segment vec. | +| Total added latency vs legacy | — | ≤ 100 µs | Measured end-to-end during shadow mode. | + +### 11.1 Allocation strategy + +- Use `SmallVec<[Cow<'a, str>; 8]>` for path segments; most IMDS paths are ≤ 6 segments. +- Borrow from the source `Uri` wherever the decode is a no-op (no `%` in the segment). +- BTreeMap is acceptable here because query maps are tiny (typical: 1–3 keys); benchmark before optimizing. + +### 11.2 Hot path caching + +- Compiled patterns are stored once at rule load, swapped via `arc_swap::ArcSwap>` (this also satisfies the TOCTOU concern from pentest `D5`). + +## 12. Telemetry & Observability + +### 12.1 Metrics + +- `gpa_canon_calls_total{result="ok|error"}` +- `gpa_canon_errors_total{code="CANON_PCT|CANON_OVERLONG|..."}` +- `gpa_canon_divergence_total{reason="trailing_slash|embedded_query|host_form|..."}` (shadow mode only) +- `gpa_canon_latency_microseconds` (histogram) + +### 12.2 Audit log fields + +New fields appended to each entry in `ProxyAgent.Connection.log`: + +- `canon_path` — rendered canonical path (redacted: identifiers replaced with placeholder). +- `canon_dest` — `imds|wireserver|hostga|unknown`. +- `canon_error` — error code or null. +- `policy_epoch` — snapshot id used for this request. + +### 12.3 Operator-visible signal + +A non-zero divergence rate after the first week of shadow mode is the single most important signal: it directly identifies rules whose authors implicitly relied on the legacy normalization quirks. Surface this in `gpa-doctor` (Direction 6.2) so operators can fix their rules *before* enforce mode is enabled. + +## 13. Risks & Open Questions + +### 13.1 Risks + +- **Existing rules may rely on quirks.** Mitigation: shadow mode + divergence reporting + a one-release overlap period. +- **Hostname rules.** If a customer has a rule keyed on a hostname rather than an IP, our refusal to DNS-resolve at the matcher means the rule will only match if the client also uses that exact hostname text. Document clearly; provide a migration tool. +- **IPv6 link-local zone IDs** (`fe80::1%eth0`) — decide whether to strip or reject; current proposal is to reject (fail-closed). +- **Performance regression** on tiny VMs with high IMDS QPS. Mitigation: benchmark suite gated in CI; SmallVec; borrow-when-possible decoding. + +### 13.2 Open questions + +1. Should `CanonicalPattern` support glob/regex on segments, or only exact + prefix? Recommendation: exact + prefix only; richer matching is the policy-language work in Direction 2.2. +2. Do we expose the canonical form on `/.well-known/gpa/attestation` (Direction 3.3) for diagnostic use? Recommendation: yes, but redact identifiers. +3. For unknown destinations, do we ever forward, or strictly deny? Current proposal: strictly deny. + +## 14. Milestones + +| M | Deliverable | Exit criteria | +|-----|--------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| M1 | Module skeleton + types + error taxonomy | Compiles; unit tests for each helper at \> 90% line coverage | +| M2 | Golden vectors + property tests + fuzz target | Zero panics in 1 CPU-day of fuzzing; all D1/C7 vectors pass | +| M3 | Shadow-mode integration in `proxy_authorizer.rs` | Divergence telemetry visible in dev/test; behavior unchanged for production traffic | +| M4 | Rule-loader uses canonical patterns; legacy `Privilege` deprecated | All existing rule files still load; old API marked `#[deprecated]` | +| M5 | Region-by-region cutover to enforce mode | Zero divergence for 14 days per region; pentest S20–S22 pass | +| M6 | Removal of legacy matcher | All call sites migrated; legacy code deleted; codebase reduction recorded | + +## AAppendix — Representative Vector Table + +A sample of the golden vectors. The full table lives in `proxy_agent/src/proxy/canonical/tests/vectors.rs`. + +### A.1 Path vectors + +| Input path | Canonical | Or error | +|-----------------------------------------|------------------------|------------------------| +| `/metadata/identity` | `/metadata/identity` | — | +| `/Metadata/Identity` | `/metadata/identity` | — | +| `/metadata//identity` | `/metadata/identity` | — | +| `/metadata/./identity` | `/metadata/identity` | — | +| `/metadata/x/../identity` | `/metadata/identity` | — | +| `/metadata%2Fidentity` | `/metadata/identity` | — | +| `/metadata%252Fidentity` | `/metadata%2fidentity` | — (single decode only) | +| `/metadata/%C0%AFidentity` | — | `OverlongUtf8` | +| `/metadata/identity/../../..` | — | `PathUnderflow` | +| `/metadata/identity;jsessionid=abc` | `/metadata/identity` | — | +| `/metadata/identity%3Fapi-version=2018` | — | `EmbeddedQuery` | +| `/metadata/identity%0A` | — | `ControlChar` | + +### A.2 Host vectors (all should map to `Destination::Imds`) + +| Host text | Result | +|-------------------------------|-------------------------------------------------| +| `169.254.169.254` | `Imds` | +| `2852039166` | `Imds` | +| `0xa9fea9fe` | `Imds` | +| `0251.0376.0251.0376` | `Imds` | +| `[::ffff:169.254.169.254]` | `Imds` | +| `[::ffff:a9fe:a9fe]` | `Imds` | +| `user@169.254.169.254` | `CanonError::UserinfoPresent` | +| `169.254.169.254:80@evil.com` | `CanonError::BadHost` | +| `metadata.azure.internal` | `Destination::Unknown { host_text: Some(...) }` | + +Detailed design for direction 2.1 of the GPA innovation plan. Parent doc: [Innovation-Directions.md](Innovation-Directions.md). Source-of-truth files: `proxy_agent/src/proxy/authorization_rules.rs`, `proxy_agent/src/key_keeper/key.rs`, `pentest/linux/DESIGN.md`. diff --git a/doc/plans/Innovation-2.2-typed-policy-cedar.md b/doc/plans/Innovation-2.2-typed-policy-cedar.md new file mode 100644 index 00000000..58fc1a50 --- /dev/null +++ b/doc/plans/Innovation-2.2-typed-policy-cedar.md @@ -0,0 +1,137 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Why Cedar](#why) +3. [3. Entity model](#model) +4. [4. Policy form](#policy) +5. [5. Compile pipeline](#compile) +6. [6. Integration](#integration) +7. [7. Dual-eval](#dual) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 2.2** · **Policy** + +# Detailed Design — Typed Policy Language (Cedar) + +Replace the ad-hoc JSON rule shape with Cedar, a typed, analyzable, verified-evaluator policy language. Existing rules compile down to Cedar at load time; the matcher becomes a thin call into the Cedar evaluator over `CanonicalRequest`-derived entities. + +**Files affected:** new `proxy_agent/src/proxy/policy/` module (Cedar adapter), integrates with 1.4 scopes and 2.1 canonical model. + +> **Prerequisites:** [2.1 Canonical request](Innovation-2.1-canonical-request.md)[1.4 Capability scopes](Innovation-1.4-capability-scopes.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|------------|------------------------|----------------| +| **High** analyzable policy | **Medium** | **Low** shadow rollout | **agent only** | + +### Goals + +- Typed actions and entities — no more string-prefix matching surface. +- Formal analysis: "is policy P at least as strict as policy Q?" (Cedar's policy analyzer). +- Stable, versioned grammar; precise diagnostics on bad rules. +- Drop-in path: existing JSON rules continue to load via a compiler. + +## 2. Why Cedar (vs Rego / OPA / Custom DSL) + +| Option | Pro | Con | +|------------|------------------------------------------------------------------------------------------------------|------------------------------------------------------------| +| **Cedar** | Rust-native crate, verified evaluator (Lean), built-in policy analyzer, deterministic, low footprint | Smaller community than OPA | +| OPA / Rego | Huge ecosystem | Heavyweight runtime, Go dependency, less formal guarantees | +| Custom DSL | Tailored | Reinvents analyzer, parser, fuzz suite — long tail of bugs | + +**Choice:** Cedar (`cedar-policy` crate). It runs in-process, has stable serialized form, and supports schema-based static type checking — directly enabling the analyses described in 1.4. + +## 3. Entity Model + + // Cedar schema (simplified) + entity Identity in [Role] = { + name: String, + userName?: String, + groupName?: String, + exeMeasurement?: { kind: String, value: String, enforce: Bool }, + }; + entity Role = { name: String }; + entity Service { }; // Imds, WireServer, HostGa + entity Resource in [Service] = { service: Service, name: String }; + action read,write,invoke,enumerate appliesTo { + principal: [Identity], + resource: [Resource], + context: { + url: String, // canonical URL hash, not raw + scope: String, // from 1.4 capability classifier + canon: { dest: String, segments: [String], query_keys: [String] } + } + }; + +## 4. Policy Form + + // Allow waagent to read goalstate + permit ( + principal == Identity::"walinuxagent", + action == Action::"read", + resource == Resource::"WireServer::goalstate" + ); + + // Anyone may read instance metadata + permit ( + principal, + action == Action::"read", + resource == Resource::"Imds::instance" + ); + + // Forbid identity matching that requires measurement when measurement missing + forbid (principal, action, resource) + when { + principal.exeMeasurement has "enforce" && + principal.exeMeasurement.enforce && + context.canon.dest != "imds" // example forbid condition + }; + +## 5. Compile Pipeline + +JSON rules (legacy v1) │ ▼ legacy_to_cedar // small translator; 1:1 mapping for permit shapes │ ▼ cedar::PolicySet // typed AST │ ▼ schema_validate // reject if a policy references unknown entities │ ▼ ArcSwap\ + +- Compilation happens off the hot path (rule reload thread). +- Validation errors abort the swap; fail-closed: previous policy stays active. + +## 6. Integration with 1.4 and 2.1 + +- **2.1 Canonical model** provides `CanonicalRequest`. The classifier (1.4) reduces it to a `Scope`. +- Cedar context = `{ url: hash, scope, canon: { dest, segments, query_keys } }`. Policies primarily key on `scope`; the rest is available for advanced rules. +- The evaluator returns `Decision::Allow | Deny` plus the matched policy id (for audit). + +## 7. Dual-Evaluation Rollout + +Same mechanism as direction 2.1: + +- `policy.mode = off | shadow | enforce`. +- In shadow, legacy decides; Cedar's verdict is logged with policy-id reasoning. +- Cutover gate: ≥ 14 days zero divergence in production sample. + +## 8. Tests + +- Round-trip: any legacy rule file → Cedar policy set → produced JSON → same decisions on the request corpus. +- Cedar's own policy analyzer used in CI to verify invariants (e.g. no `permit` for `wireserver:*:write` by `principal: any`). +- Fuzz: random Cedar policies + random canonical requests — evaluator must not panic. +- Property test: enforce decisions monotone in policy strictness ("more strict policy never allows more"). + +## 9. Risks + +- **Crate version churn.** Pin to a Cedar release line; vendor source if needed. +- **Customer-authored Cedar (future).** Out of scope for v1 — only auto-translated policies are accepted; advanced customers go through review. +- **Evaluator cost.** Cedar is fast (microseconds), but bench against the legacy matcher and gate p99. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|--------------------------------------------|----------------------------------------------------------------| +| M1 | Cedar schema + translator for legacy rules | Lossless round-trip on test fixtures | +| M2 | Evaluator integrated behind feature flag | Shadow-mode running in CI | +| M3 | Dual-eval in production canary | Zero divergence 14 days | +| M4 | Enforce mode | Legacy matcher marked `#[deprecated]` | +| M5 | Remove legacy | Codebase reduction recorded; analyzer hooked into `gpa policy` | + +Detail design for direction 2.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-2.3-versioned-snapshots.md b/doc/plans/Innovation-2.3-versioned-snapshots.md new file mode 100644 index 00000000..d3d0939f --- /dev/null +++ b/doc/plans/Innovation-2.3-versioned-snapshots.md @@ -0,0 +1,123 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Design](#design) +4. [4. API](#api) +5. [5. Reload protocol](#reload) +6. [6. Audit emission](#audit) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 2.3** · **Concurrency** + +# Detailed Design — Versioned, Per-Request Policy Snapshots + +Wrap the active policy in `ArcSwap` with a monotonic `epoch`. Each incoming request captures the policy snapshot at accept time and uses it for the whole forwarding decision. Closes pentest `D5` (TOCTOU between rule reload and in-flight request) and gives operators a precise audit trail. + +**Files affected:** `proxy_agent/src/key_keeper/key.rs`, `proxy_agent/src/proxy/proxy_authorizer.rs`, `proxy_agent/src/proxy/proxy_server.rs`, `proxy_agent/src/key_keeper/local_rules.rs`. + +> **Prerequisites:** [2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------|-----------|---------|----------------| +| **Medium** closes D5 | **Small** | **Low** | **agent only** | + +### Goals + +- A request that started under policy *P_n* finishes under *P_n*, even if reload occurs mid-request. +- Audit log records the exact policy `epoch` that authorized each request. +- Reloads are wait-free for readers; no mutex on the hot path. + +## 2. Today's Behavior + +Rules are stored in shared mutable state. A reload can race with an in-flight authorize call — different parts of the decision can read different versions, and there is no *per-request* identifier of which policy version applied. + +## 3. Design + +### 3.1 Types + + pub struct PolicyEpoch(pub u64); + + pub struct PolicySnapshot { + pub epoch: PolicyEpoch, + pub computed: ComputedAuthorizationRules, + pub source_hash: [u8;32], + pub loaded_at: SystemTime, + } + + pub struct PolicyStore { + inner: arc_swap::ArcSwap, + next_epoch: AtomicU64, + } + + impl PolicyStore { + pub fn current(&self) -> Arc; + pub fn install(&self, computed: ComputedAuthorizationRules, source_hash: [u8;32]) + -> PolicyEpoch; + } + +### 3.2 Invariants + +- `epoch` is monotonically increasing across the agent process lifetime; persists across restart by reading the last `epoch` stamped in the `AuthorizationRules_*.json` file and incrementing. +- Installation is fail-closed: if validation fails, no install occurs and previous snapshot stays active. A counter `gpa_policy_install_failed_total` increments. +- Readers never block writers; writers never block readers. + +## 4. Usage + + // Accept site (proxy_server.rs) + let snap = policy_store.current(); // cheap Arc clone + ctx.policy_snapshot = snap; + ctx.policy_epoch = snap.epoch; + + // Authorizer (proxy_authorizer.rs) + let decision = ctx.policy_snapshot.is_allowed(&canon_req, &claims); + logger.attach_field("policy_epoch", ctx.policy_epoch.0); + +## 5. Reload Protocol + +1. Reload thread fetches new rules (remote + local merge per `local_rules.rs`). +2. Compile to `ComputedAuthorizationRules` (and Cedar policy set when 2.2 lands). +3. Validate (schema + structural). On failure: log + telemetry + leave previous in place. +4. `PolicyStore::install` assigns the next epoch and swaps the Arc. +5. Emit a structured event `PolicyInstalled{epoch, source_hash, loaded_at}`. + +## 6. Audit Emission + +- Every connection log entry gains `policy_epoch`. +- `status.json` reports `active_policy_epoch`, `last_failed_install_at`, `last_failed_install_reason`. +- Telemetry: histogram of `request_age_vs_policy_age_seconds` to detect long-lived connections still bound to ancient snapshots (potentially a sign of upstream hang). + +## 7. Integration Points + +- `proxy_agent/src/key_keeper/key.rs` — replace direct sharing with `Arc`. +- `proxy_agent/src/proxy/proxy_server.rs` — capture snapshot on accept and stash on connection context. +- `proxy_agent/src/proxy/proxy_authorizer.rs` — read from context, not global. +- `proxy_agent/src/proxy/proxy_summary.rs` — propagate `policy_epoch` into log entries. +- `proxy_agent/src/key_keeper/local_rules.rs` — fail-closed merge already exists; just route the install through the store. + +## 8. Tests + +- Concurrent test: 64 worker threads issue authorize calls; reload thread installs new policies at random intervals. Assert every decision is consistent (Allow/Deny matches the snapshot's rules) and no thread observes a half-installed state. +- Pentest `D5`: with an in-flight request held by a slow upstream, install a deny policy; the in-flight request still completes with the prior epoch (documented behavior) but no new connections see the old policy. +- Fail-closed: corrupt the rules file → install fails → previous epoch remains active and is reflected in `status.json`. + +## 9. Risks + +- **Long-lived requests retain old policies.** Mitigation: documented behavior; emit warning when request_age \> threshold. +- **Epoch wraparound:** `u64`, no practical issue. +- **Stale snapshot retained by Arc.** Memory only; small (one struct per request in flight). + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|-----------------------------------------|-----------------------------------------------------| +| M1 | Introduce `PolicyStore` + plumb context | All unit tests pass; `policy_epoch` visible in logs | +| M2 | Status + telemetry fields | Operator dashboards updated | +| M3 | Pentest D5 regression test added | Test green; runs in CI | + +Detail design for direction 2.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-2.4-differential-testing.md b/doc/plans/Innovation-2.4-differential-testing.md new file mode 100644 index 00000000..4bcd9855 --- /dev/null +++ b/doc/plans/Innovation-2.4-differential-testing.md @@ -0,0 +1,119 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Design](#design) +3. [3. Mutator catalog](#mutators) +4. [4. Runner](#runner) +5. [5. Integration](#integration) +6. [6. Failure handling](#failmode) +7. [7. Perf budget](#perf) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 2.4** · **Self-test** + +# Detailed Design — Differential & Property Testing of Rules + +For each rule loaded, auto-generate "evil twin" requests (case toggles, percent-encoded slashes, IPv6 forms, Unicode confusables). Run the matcher on each variant; any mismatch with the canonical request indicates a latent bypass and blocks the rule from going live. + +**Files affected:** new `proxy_agent/src/proxy/policy/selftest/` module, hooked into `local_rules.rs` reload path and CI. + +> **Prerequisites:** [2.1 Canonical request](Innovation-2.1-canonical-request.md)[2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|---------------------------------------|-----------|---------|----------------| +| **Medium** proactive bypass detection | **Small** | **Low** | **agent + CI** | + +### Goals + +- Every rule reload runs a self-test that proves the rule is robust to known bypass patterns. +- Same self-test runs in CI on the repository's bundled rule files. +- Fail-closed: a rule that fails self-test is rejected during reload, and the previous policy remains active. + +## 2. Design + +For every rule, the runner derives a small set of synthetic requests: + +1. **Canonical request** matching the rule's intent exactly. +2. **Evil twins** — produced by mutators that should normalize to the same canonical form per 2.1. +3. **Negative twins** — close-but-not-matching requests that should be rejected by the rule. + +The runner asserts: *canonical and evil twins produce the same decision; negative twins produce a different decision.* + +## 3. Mutator Catalog + +| Mutator | Example | Pentest mapping | +|--------------------------------------|-----------------------------------------|------------------------------------| +| Case-toggle | `/Metadata/Identity` | D1 | +| Percent-encoded slash | `/metadata%2Fidentity` | D1 | +| Double-encoding | `%252e%252e` | D1 | +| Trailing dot / whitespace | `/metadata./` | D1 | +| Matrix params | `/metadata;jsessionid=x/identity` | D1 | +| Embedded query via `%3F` | `/metadata/identity%3Fapi-version=2018` | D1 (now rejected by canonicalizer) | +| IPv4 numeric | `http://2852039166/...` | C7 | +| IPv4 hex | `http://0xa9fea9fe/...` | C7 | +| IPv4 octal | `http://0251.0376.0251.0376/...` | C7 | +| IPv4-mapped IPv6 | `http://[::ffff:169.254.169.254]/...` | C7 | +| Unicode confusables in identity name | `"r\u00f6ot"` vs `"root"` | D3 | + +## 4. Runner + + pub struct SelfTestReport { + pub rule_id: String, + pub passed: bool, + pub failures: Vec, + } + + pub struct SelfTestFailure { + pub mutator: &'static str, + pub canonical_decision: Decision, + pub mutated_decision: Decision, + pub mutated_uri: String, + pub reason: &'static str, + } + + pub fn selftest(policy: &CompiledPolicy) -> Vec; + +- Runs over the compiled policy, not the raw JSON, so it tests the actual evaluator path. +- Per-rule budget: ≤ 1 ms for 30 mutators; total budget ≤ 100 ms per reload for typical rule counts. + +## 5. Integration + +- **Reload path** (`local_rules.rs`): selftest runs before `PolicyStore::install`. Failure → install aborted, previous snapshot stays. +- **CI**: `cargo test --features selftest -- --include-ignored` runs against every fixture rule file in `config/`. +- **Operator visibility**: `status.json` exposes `last_selftest_failures`; `gpa-doctor` (direction 6.2) surfaces this prominently. + +## 6. Failure Handling + +- Selftest failures during reload are non-fatal for serving (we keep the prior policy) but block the new one. +- Selftest failures in CI are fatal — block PR merges. Authors fix either the rule or the mutator. +- Each failure includes a *minimum-reproducer URI* for fast triage. + +## 7. Performance + +- Selftest runs off the hot path (rule-reload thread). +- Time-bounded; if budget exceeded, emit a warning and continue (don't deadlock on a pathological rule set). + +## 8. Tests for the selftest itself + +- Inject an intentionally-buggy matcher (e.g. case-sensitive substring) and confirm selftest catches the bypass. +- Inject a "perfect" matcher and confirm selftest reports zero failures. +- Property test: for any rule, mutated canonical request and original canonical request canonicalize to equal forms. + +## 9. Risks + +- **False positives** if a mutator generates a request that legitimately differs semantically. Mitigation: maintain mutators in a curated list, not generic fuzz. +- **Rule writers blocked** by unfamiliar failures. Mitigation: the report includes the reproducer URI and the mutator name; `gpa policy simulate` (6.1) provides a single-step debugger. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------------------------|---------------------------------------------| +| M1 | 10 mutators + report type | Runs in CI on bundled rules | +| M2 | Reload-path integration | Install aborted on failure; covered by test | +| M3 | Operator surface (`status.json`, `gpa-doctor`) | Documented in operator guide | + +Detail design for direction 2.4. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-3.1-hash-chained-log.md b/doc/plans/Innovation-3.1-hash-chained-log.md new file mode 100644 index 00000000..582e3588 --- /dev/null +++ b/doc/plans/Innovation-3.1-hash-chained-log.md @@ -0,0 +1,114 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Record format](#format) +4. [4. Chain semantics](#chain) +5. [5. Anchoring](#anchor) +6. [6. Rotation](#rotation) +7. [7. Verifier tool](#verify) +8. [8. Integration](#integration) +9. [9. Tests](#tests) +10. [10. Risks](#risks) +11. [11. Milestones](#milestones) + +**GPA** · **Direction 3.1** · **Audit integrity** + +# Detailed Design — Hash-chained, Append-only Audit Log + +Wrap the connection log with a Merkle/hash chain so any post-hoc tampering (deletion, edit, injection) breaks the chain and is detectable. Optional anchoring to an external transparency log makes the chain non-repudiable. + +**Files affected:** `proxy_agent/src/common/logger` (Sink abstraction), `proxy_agent/src/proxy/proxy_summary.rs`, new `proxy_agent/src/audit/chain.rs`. + +> **Prerequisites:** None — foundational audit layer. Required by [3.2 OTel export](Innovation-3.2-otel-export.md), [3.3 Self-attestation](Innovation-3.3-self-attestation.md), [6.2 GPA doctor](Innovation-6.2-gpa-doctor.md). + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|-----------|---------|-----------| +| **Medium** compliance + IR | **Small** | **Low** | **agent** | + +### Goals + +- Tampering with the audit log is detectable; addresses pentest `F2` (log injection) and `F3` (rotation race). +- No central service required for tamper-evidence; anchoring optional. +- Negligible runtime overhead (≤ 1 µs / record). + +## 2. Today + +Connection log is a plain newline-delimited JSON file. A root attacker can edit or delete entries; an attacker who can inject newlines into a process name or URL can forge entries that pass downstream parsers. + +## 3. Record Format + + // One line per record (NDJSON) + { + "seq": 175201, + "ts": "2026-06-01T12:34:56.789Z", + "kind": "decision|policy_install|service|...", + "payload": { ... arbitrary ... }, + "prev_hash": "b3:9f8a...", // hash of record seq-1 (full line bytes) + "hash": "b3:2c1d..." // sha256 of (prev_hash || canonical_json(payload) || ts || seq) + } + +- **Canonical JSON** for the payload to ensure deterministic hashing across runtimes. +- **Length-prefix** embedded in the line (start of line: `16-hex-len `) to defeat newline-injection — parsers ignore content past `len`. +- Every record carries its sequence number; gaps are immediately detected. + +## 4. Chain Semantics + +- The chain is a forward hash list — minimal compute, sufficient for tamper-evidence. +- Periodic "checkpoint" records (every N records or T seconds) include a Merkle root over the latest segment to allow O(log N) inclusion proofs later. +- `prev_hash` at sequence 0 is a fixed sentinel containing service start time, binary hash, and policy epoch at startup. + +## 5. Anchoring (optional) + +- Periodic checkpoint hashes can be: + - Submitted to a Rekor-compatible transparency log. + - Sent to Azure Monitor as a signed custom-log entry, signed by the VM's vTPM AIK if available. + - Mirrored locally to a read-only directory under `/var/log/azure-proxy-agent/checkpoints/` for offline forensic use. +- Anchoring failures do not block log writes — fail-open for availability, log telemetry instead. + +## 6. Rotation (closes F3) + +- Rotation creates a new file `ProxyAgent.Connection.NNN.log`; the first record in NNN+1 is a "rotation" record that includes the final `hash` of NNN. +- Open file via `O_NOFOLLOW | O_CREAT | O_EXCL` to defeat symlink swap. +- Permissions enforced as 0600 root:root via `fchmod` after creation and verified before each append. +- Rotation never overwrites; old files are renamed atomically. + +## 7. Verifier Tool + +`gpa audit verify [--from N] [--to M] [--anchor file]` + +- Walks the chain, validates each record's hash; reports first divergence with sequence numbers and byte offsets. +- `--anchor` validates against a fetched checkpoint file (or rekor inclusion proof). +- Exits non-zero on tamper-detection; suitable for SIEM integration. + +## 8. Integration + +- Logger refactored to a `trait Sink` with implementations: `PlainFileSink` (legacy), `ChainedFileSink`, `SyslogSink`, `TestSink`. +- Choice via config `audit.sink = "chained"`; default off in v1. +- Connection log writer + service log writer share the sink trait so both gain integrity. + +## 9. Tests + +- Append, verify → pass. Edit one byte of payload → verifier reports failure at that sequence. +- Delete a record line → next record's `prev_hash` mismatch detected. +- Newline injection in process name → verifier still parses correctly because length-prefix bounds the record. +- Symlink swap pre-rotation → `O_NOFOLLOW` open fails; alert event written to previous file before exit. +- Pentest `F2`/`F3` reruns: PASS. + +## 10. Risks + +- **Performance:** hashing every record adds ~1 µs; bounded. +- **Recovery after crash:** last partial line skipped on restart; checkpoint emitted noting the gap. +- **Disk full:** chain still self-consistent; rotation policy must avoid pruning without a verified anchor. + +## 11. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------------|--------------------------------------------| +| M1 | Sink trait + plain + chained impls | Logger refactor merged behind feature flag | +| M2 | Verifier CLI | F2/F3 pentest PASS | +| M3 | Optional Rekor anchoring | Anchor docs published | + +Detail design for direction 3.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-3.2-otel-export.md b/doc/plans/Innovation-3.2-otel-export.md new file mode 100644 index 00000000..e4ea6d0e --- /dev/null +++ b/doc/plans/Innovation-3.2-otel-export.md @@ -0,0 +1,99 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Schema](#schema) +3. [3. Metrics](#metrics) +4. [4. Traces](#traces) +5. [5. Exporter](#exporter) +6. [6. Perf](#perf) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 3.2** · **Observability** + +# Detailed Design — OpenTelemetry Export + +Emit standards-based metrics and traces so GPA can be observed by any modern monitoring stack (Azure Monitor, Prometheus, OTel collectors). Closes the gap where today's only signal is a text log. + +**Files affected:** new `proxy_agent/src/telemetry/` module, light hooks in proxy/redirector. + +> **Prerequisites:** [3.1 Hash-chained log](Innovation-3.1-hash-chained-log.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|------------------------------------|-----------|---------|-----------| +| **Medium** SLO + incident response | **Small** | **Low** | **agent** | + +### Goals + +- Production teams can graph allow/deny by rule id and chase regressions without parsing logs. +- Optional OTLP exporter; default off keeps footprint minimal. +- No PII / secrets in metric labels. + +## 2. Resource Attributes + + service.name = "gpa" + service.version = + host.id = + gpa.binary.hash = + gpa.policy.epoch = // updated on reload + gpa.seal.backend = "tpm2|snp|tdx|noop" + +## 3. Metrics + +| Name | Type | Labels | Notes | +|------------------------------|-----------|--------------------|-------------------------------| +| `gpa_requests_total` | counter | `dest`, `decision` | Decision = allow\|deny\|error | +| `gpa_request_latency_us` | histogram | `dest`, `decision` | End-to-end through the agent | +| `gpa_canon_errors_total` | counter | `code` | From direction 2.1 | +| `gpa_policy_install_total` | counter | `result` | success\|failed | +| `gpa_policy_epoch` | gauge | — | Currently active epoch | +| `gpa_ebpf_audit_map_entries` | gauge | — | From direction 4.4 | +| `gpa_pop_verify_total` | counter | `result` | From direction 1.1 | +| `gpa_restart_total` | counter | `reason` | graceful\|crash\|sigterm | + +## 4. Traces + +- One span per inbound request: `gpa.serve_request` with attributes `http.method`, `gpa.dest`, `gpa.decision`, `gpa.policy_epoch`, `gpa.matched_scope`. +- Child span: `gpa.upstream_request` with `net.peer.ip`, `http.status_code`. +- W3C trace context propagation: *do not* propagate inbound trace context to upstream metadata services (avoid leaking client trace ids into fabric). GPA spans share its own trace id rooted at the connection. + +## 5. Exporter + +- Default: **Prometheus exposition over Unix socket** at `/run/azure-proxy-agent/metrics.sock` (so it never goes over TCP). +- Optional: OTLP/gRPC to a configurable endpoint (used by AKS observability pipelines). +- Sampling: traces sampled at 1/100 by default; head-based; configurable. + +## 6. Performance Budget + +- Recording overhead per request ≤ 1 µs in default mode; ≤ 5 µs with trace sampled. +- Exporter flush is asynchronous, bounded queue; backpressure drops oldest with a counter. + +## 7. Integration + +- Use `opentelemetry` + `opentelemetry_sdk` crates; `prometheus` crate for exposition. +- Init in `main.rs` behind `--features otel`; no-op otherwise. +- Hook points: accept, authorize result, upstream call boundary, reload, eBPF map size sampler (every 30 s). + +## 8. Tests + +- Unit: counters/histograms record expected values. +- Integration: spawn agent with metrics socket, hit it with sample requests, assert metrics endpoint output matches. +- Soak: 24 h with metrics on; memory and CPU within budget. + +## 9. Risks + +- **Cardinality explosion** if URL or identity goes into labels. Mitigation: only typed enums in labels; never raw strings from requests. +- **Dependency weight.** Mitigation: feature flag default off. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|-------------------------------|---------------------------------| +| M1 | Metric registry + Prom socket | Local dashboards working | +| M2 | Traces + OTLP | Pilot in one production cluster | + +Detail design for direction 3.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-3.3-self-attestation.md b/doc/plans/Innovation-3.3-self-attestation.md new file mode 100644 index 00000000..4f063e15 --- /dev/null +++ b/doc/plans/Innovation-3.3-self-attestation.md @@ -0,0 +1,108 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Endpoint](#endpoint) +3. [3. Payload](#payload) +4. [4. Access control](#access) +5. [5. Signing](#sign) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 3.3** · **Attestation** + +# Detailed Design — Self-Attestation Endpoint + +Expose a read-only endpoint that returns GPA's own measurements: binary hash, loaded eBPF program ids and bytecode hashes, attached cgroup, active policy epoch, sealed-key id, attestation backend. Consumable by Defender for Cloud, Azure Policy, and operator tooling. + +**Files affected:** `proxy_agent/src/proxy/proxy_server.rs` (new route), new `proxy_agent/src/attestation/`. + +> **Prerequisites:** [1.2 vTPM sealing](Innovation-1.2-vtpm-sealing.md)[3.1 Hash-chained log](Innovation-3.1-hash-chained-log.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-----------------------------------------|-----------|---------|-----------| +| **Medium** compliance + drift detection | **Small** | **Low** | **agent** | + +### Goals + +- Externally verifiable proof of which GPA is running, with which policy, and which kernel-side components are attached. +- Cheap probe with no secrets in the payload. + +## 2. Endpoint + +- HTTP GET `/.well-known/gpa/attestation` on the local listener (`127.0.0.1:3080`). +- Optional `?nonce=BASE64URL(32 bytes)` for freshness. +- Response: `application/json` with a signed `jws` field when an attestation backend is present. + +## 3. Payload Shape + + { + "version": 1, + "service": { + "name": "gpa", + "version": "1.X.Y", + "binary_hash": "sha256:...", + "uptime_s": 12345 + }, + "policy": { + "epoch": 175201, + "source_hash": "sha256:...", + "loaded_at": "2026-06-01T12:30:00Z", + "selftest_passed": true + }, + "ebpf": [ + { "name": "cgroup_connect", "id": 42, "bytecode_hash": "sha256:...", "attach_cgroup": "/sys/fs/cgroup" }, + { "name": "audit_event", "id": 43, "bytecode_hash": "sha256:..." } + ], + "seal": { + "backend": "tpm2|snp|tdx|noop", + "kid": "...", + "counter": 7 + }, + "nonce": "...", + "ts": "2026-06-01T12:34:56Z", + "jws": "eyJhbGc..." // optional, present when sealed + } + +## 4. Access Control + +- Reachable only via the localhost listener; no external exposure. +- Subject to standard GPA AuthZ: by default, any in-VM caller may read; rules can restrict if desired. +- Rate-limited (1 req/s per caller cgroup) to prevent measurement-driven side channels. + +## 5. Signing + +- When a hardware backend (1.2) is present, sign the canonical JSON payload + nonce with the attestation key (TPM AIK, SNP report, TDX QUOTE) and place the proof in `jws`. +- When `noop` backend, omit `jws`; tooling treats the response as unauthenticated diagnostic. +- Nonce is reflected verbatim; binding nonce + measurements prevents replay across calls. + +## 6. Integration + +- `proxy_agent/src/proxy/proxy_server.rs` — new handler before the generic forwarding path. +- Pulls fields from: build-time const (version), `policy_store.current()`, `redirector::loaded_programs()`, `key_keeper::sealing`. +- `gpa-doctor` (direction 6.2) and Azure Policy hook can both consume this. + +## 7. Tests + +- Probe returns expected fields; binary_hash matches actual file hash. +- With `tpm2` backend simulator, JWS validates with the AIK. +- Rate-limit test: 100 rapid requests → some 429s; payload counter stable. +- Mutate binary (in test harness) → next call returns the new hash; external monitor detects drift. + +## 8. Risks + +- **Information disclosure** of internal program ids. Mitigation: ids are not secrets; documented as public attestation surface. +- **Hot-loop callers** can heat up the agent. Mitigation: rate limit + cheap payload caching with nonce-only signing per request. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|--------------------------|-----------------------------| +| M1 | Unsigned endpoint | Surfaces in `gpa-doctor` | +| M2 | Signed via vTPM backend | JWS verification documented | +| M3 | Defender for Cloud probe | Drift alerts configured | + +Detail design for direction 3.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-3.4-supply-chain.md b/doc/plans/Innovation-3.4-supply-chain.md new file mode 100644 index 00000000..8244bb16 --- /dev/null +++ b/doc/plans/Innovation-3.4-supply-chain.md @@ -0,0 +1,91 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. SBOM](#sbom) +3. [3. Reproducible build](#repro) +4. [4. Signing (Sigstore)](#signing) +5. [5. Setup verification](#verify) +6. [6. CI pipeline](#pipeline) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 3.4** · **Supply chain** + +# Detailed Design — Supply-chain Hardening (SBOM, Reproducible Builds, Sigstore) + +Produce an SBOM, make the build bit-reproducible, sign artifacts with Sigstore, and have `proxy_agent_setup` verify the signature before installing. Closes pentest `H1` (rollback to malicious previous-version archive). + +**Files affected:** CI pipeline, `proxy_agent_setup/`, package build for Linux/Windows. + +> **Prerequisites:** None — build-time / release-pipeline change, independent of agent runtime work. + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-----------------------------------|------------|---------|-------------------| +| **Medium** compliance + integrity | **Medium** | **Low** | **build + setup** | + +### Goals + +- Auditable list of every transitive crate / system library shipped. +- Builds bit-reproducible across two builders; output hash matches what is signed. +- `proxy_agent_setup` refuses to install an unsigned, downgraded, or tampered archive. + +## 2. SBOM + +- Generate CycloneDX with `cargo-cyclonedx --format json` for each crate in the workspace, merged into a single document. +- Include the eBPF object files and their `clang` + `BTF` versions. +- Ship SBOM alongside the package (`gpa-.sbom.json`); attach to GitHub Release. + +## 3. Reproducible Build + +- Pin toolchain via `rust-toolchain.toml`. +- Vendor dependencies (`cargo vendor`); commit checksum. +- Strip build paths: `RUSTFLAGS="--remap-path-prefix $(pwd)=. -C link-arg=-Wl,--build-id=none"`. +- Pin clang version for eBPF objects. +- CI runs two independent builders; compares sha256 of all output artifacts; fail if not equal. + +## 4. Signing (Sigstore / cosign) + +- **Keyless signing** via `cosign sign --certificate-identity ...` backed by GitHub Actions OIDC token. +- Signed artifacts: the agent binaries, the extension `HandlerManifest`, eBPF objects, SBOM. +- Signatures + Rekor transparency entries are published in the same release. +- Optional in-toto attestation describing the build (commit, builder, dependencies). + +## 5. Setup-side Verification + +- `proxy_agent_setup` ships with the Sigstore root + the expected identity (the GitHub repo / workflow). +- Before any install / replace operation: + 1. Verify cosign signature on the new archive. + 2. Verify the Rekor inclusion proof (offline-verifiable bundle ships alongside the artifact). + 3. Verify the new version is ≥ the previously-installed version recorded in `/var/lib/azure-proxy-agent/installed_version`; refuse downgrades unless an explicit operator override flag is provided. +- Any failure: leave the prior installation in place, write a structured audit event, exit non-zero. + +## 6. CI Pipeline + +PR build: cargo build / test / clippy / fmt cargo-cyclonedx (SBOM) cargo audit (advisory DB) Release build (two builders in parallel): build → diff hashes → must match cosign sign --keyless (each artifact) publish: artifacts + signatures + Rekor entries + SBOM + +## 7. Tests + +- Tamper a release archive → setup verification fails with explicit reason. +- Downgrade attempt → setup refuses; override flag accepted only via documented path. +- Reproducible-build diff job catches an intentionally inserted timestamp. +- Pentest `H1`: malicious rollback archive → REJECTED. + +## 8. Risks + +- **Reproducibility on Windows** is harder due to PDB embedding. Mitigation: strip PDBs from the verified path; archive separately for symbol servers. +- **Sigstore root rotation** requires periodic updates. Mitigation: ship root bundle; refresh on agent update. +- **Operator downgrade workflows** need a documented override; design it so the override itself is signed and audited. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------|-------------------------------------------------| +| M1 | SBOM in CI artifacts | Published with releases | +| M2 | Reproducible build job | Two builders match for two consecutive releases | +| M3 | Cosign signing + Rekor | Setup verifies on install in staging | +| M4 | Downgrade refusal default-on | Pentest H1 PASS | + +Detail design for direction 3.4. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-4.1-sk-lookup-bpf-lsm.md b/doc/plans/Innovation-4.1-sk-lookup-bpf-lsm.md new file mode 100644 index 00000000..eea985c9 --- /dev/null +++ b/doc/plans/Innovation-4.1-sk-lookup-bpf-lsm.md @@ -0,0 +1,117 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Design](#design) +4. [4. sk_lookup program](#sklookup) +5. [5. bpf_lsm hook](#lsm) +6. [6. Loader strategy](#loader) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 4.1** · **eBPF** + +# Detailed Design — sk_lookup + bpf_lsm Redirect + +Move from `cgroup/connect4` SNAT-style redirect to `sk_lookup` (listener-side steering, preserves original destination) augmented with `bpf_lsm` socket hooks that close netns/cgroup escape paths (pentest `C5`, `C6`, `C7`). + +**Files affected:** `linux-ebpf/` (split into `cgroup_connect.bpf.c`, `sk_lookup.bpf.c`, `lsm.bpf.c`), `proxy_agent/src/redirector/linux/`. + +> **Prerequisites:** [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------|------------|-------------------|-----------------------------| +| **High** kills C5–C7 | **Medium** | **Kernel matrix** | **linux-ebpf + redirector** | + +### Goals + +- Original destination IP is preserved end-to-end (no SNAT to localhost) so the agent can authoritatively match on it after netns shenanigans. +- An LSM hook denies connect attempts to fabric IPs from sockets the redirect cannot capture (different netns, unshared cgroup). +- Fallback path retains today's `cgroup/connect4` for older kernels. + +## 2. Today + +A `cgroup/connect4` program rewrites the destination of outbound TCP connect calls from fabric IPs to `127.0.0.1:3080`. Caveats: + +- It's a destination rewrite — the agent has to recover the original IP from a side channel. +- It's attached to the root cgroup, so a workload that escapes to a sibling cgroup hierarchy (pentest C5) or new netns (C6) escapes the redirect. +- Address-encoding bypasses (C7) are mitigated only by string parsing in user space. + +## 3. Design + +client connect(168.63.129.16:80) │ ▼ (1) cgroup/connect4 — kept for back-compat; sets sk_storage.original_dest │ ▼ (2) bpf_lsm socket_connect — DENY if dest is fabric AND sk is outside the agent's view │ kernel routing │ ▼ (3) sk_lookup at agent's listener — selects the agent's accept socket │ and preserves SO_ORIGINAL_DST for the agent to read │ agent accept() + +## 4. sk_lookup Program + + // linux-ebpf/sk_lookup.bpf.c + SEC("sk_lookup") + int gpa_sk_lookup(struct bpf_sk_lookup *ctx) { + __u32 dip = ctx->remote_ip4 ? ctx->local_ip4 : 0; + __u16 dport = ctx->local_port; + if (!is_fabric_dest(dip, dport)) return SK_PASS; + bpf_sk_assign(ctx, &agent_listener_sk, 0); + return SK_PASS; + } + +- The agent registers its listener socket via `BPF_MAP_TYPE_SK_LOOKUP` map; the kernel preserves the original destination tuple, available to the agent via `SO_ORIGINAL_DST`. +- No payload mutation, no SNAT, so the agent reads the real destination IP for AuthZ. +- IPv6 variant attached as separate program. + +## 5. bpf_lsm Hook + + // linux-ebpf/lsm.bpf.c + SEC("lsm/socket_connect") + int BPF_PROG(gpa_block, struct socket *sock, struct sockaddr *addr, int addrlen, int ret) { + if (ret) return ret; + if (!is_fabric_dest_sockaddr(addr)) return 0; + // If this socket's cgroup is not under the GPA-attached cgroup root, + // or sk_lookup is not present for this netns, deny. + if (!cgroup_is_under_root(sock->sk) || !sk_lookup_present_in_netns(sock->sk)) return -EPERM; + return 0; + } + +- This is the "you didn't go through GPA → you can't reach the fabric" rule, enforced at the kernel boundary. +- Compiled as `bpf_lsm`; requires kernel with `CONFIG_BPF_LSM=y` and `lsm=bpf` on kernel cmdline. + +## 6. Loader Strategy + +| Kernel feature | Programs loaded | Behavior | +|------------------------------|-----------------------------------------------------|---------------------------------------------| +| bpf_lsm + sk_lookup (≥ 5.13) | LSM + sk_lookup + cgroup_connect (defense-in-depth) | Best case | +| sk_lookup only | sk_lookup + cgroup_connect | No LSM deny; sk_lookup still preserves dest | +| cgroup_connect only | cgroup_connect (legacy) | Today's behavior | + +Loader probes feature availability at startup using `libbpf` feature probes; logs the chosen mode and exposes it via the attestation endpoint (3.3). + +## 7. Integration + +- `proxy_agent/src/redirector/linux/` — refactor to manage three programs with per-program lifecycle (load, attach, detach, pin under `/sys/fs/bpf/gpa/`). +- `proxy_agent/src/proxy/proxy_server.rs` — read original destination via `SO_ORIGINAL_DST` (IPv4) and `IPV6_RECVORIGDSTADDR` (IPv6). +- `linux-ebpf/` — adopt CO-RE (see 4.2) so we ship one object per program for all supported kernels. + +## 8. Tests + +- Pentest `C5` (unshare cgroup): LSM denies; without LSM, sk_lookup still captures. +- Pentest `C6` (new netns): LSM denies because sk_lookup is not present in the new netns. +- Pentest `C7` (address-encoding): all encodings reach `sk_lookup` at the same destination tuple; canonical model (2.1) then handles host normalization for AuthZ. +- IPv6 path: same checks on link-local fabric equivalents. + +## 9. Risks + +- **Kernel feature matrix** — older distros lack `bpf_lsm`. Mitigation: tiered loader, telemetry exposes which mode is in use. +- **BTF availability** on stripped kernels — vendor BTF in the package as a fallback. +- **Performance** of an extra LSM hook per connect — micro-benchmarked; expected ≤ 200 ns. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|-------------------------------------------|----------------------------------------------| +| M1 | sk_lookup program + loader tier detection | Original dest preserved on supported kernels | +| M2 | bpf_lsm deny hook | C5, C6 pentest PASS on lsm-enabled kernels | +| M3 | IPv6 variants (ties to direction 4.3) | Dual-stack VMs covered | + +Detail design for direction 4.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-4.2-core-unify-ebpf.md b/doc/plans/Innovation-4.2-core-unify-ebpf.md new file mode 100644 index 00000000..a2fd30de --- /dev/null +++ b/doc/plans/Innovation-4.2-core-unify-ebpf.md @@ -0,0 +1,103 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. CO-RE design](#design) +4. [4. Shared headers](#shared) +5. [5. Build system](#build) +6. [6. BTF strategy](#btf) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 4.2** · **eBPF** + +# Detailed Design — Unify Linux/Windows eBPF on CO-RE + +Replace today's per-kernel-version eBPF source duplication with CO-RE (Compile Once, Run Everywhere). Share data structures between Linux (`linux-ebpf/`) and Windows (`ebpf/`), reduce build matrix to one object per program per platform. + +**Files affected:** `linux-ebpf/`, `ebpf/`, `proxy_agent/build.rs`, `proxy_agent/src/redirector/`. + +> **Prerequisites:** None — foundational eBPF layer. Unblocks [4.1](Innovation-4.1-sk-lookup-bpf-lsm.md), [4.3](Innovation-4.3-ipv6-dual-stack.md), [4.4](Innovation-4.4-ebpf-throttling-lru.md), [5.1](Innovation-5.1-aks-container-native.md), [5.3](Innovation-5.3-cross-cloud-port.md). + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|------------------------------------|------------|---------|------------------| +| **Medium** maintainability + reach | **Medium** | **Low** | **eBPF + build** | + +### Goals + +- One eBPF object per program, portable across kernel versions via CO-RE relocations. +- Shared C header for the audit event struct, consumed by both Linux and Windows. +- Lower QA burden — no need to rebuild per kernel. + +## 2. Today + +Linux and Windows have two source trees with similar logic but separate definitions of the audit event struct, depending on platform tooling. Some Linux fields are hardcoded to specific kernel offsets, requiring per-distro builds for older targets. + +## 3. CO-RE Design + +- Adopt **libbpf** and **libbpf-rs** for the Linux side; use `BPF_CORE_READ()` for any kernel-struct dereference so the verifier relocates at load time. +- Use `vmlinux.h` generated from the build host's BTF as the canonical source for kernel-side types; relocations adapt to the target. +- For Windows, eBPF-for-Windows already uses BTF; mirror the same approach so user-space struct layouts align byte-for-byte. + +## 4. Shared Headers + + // shared-ebpf/include/gpa_audit_event.h (consumed by both linux-ebpf and ebpf) + #pragma once + #define GPA_AUDIT_PATH_MAX 256 + struct gpa_audit_event { + __u64 cgroup_id; + __u32 pid; + __u64 pid_starttime_ns; + __u32 uid; + __u32 gid; + __u32 measurement_kind; + __u8 measurement[32]; + char exe_path[GPA_AUDIT_PATH_MAX]; + }; + _Static_assert(sizeof(struct gpa_audit_event) == 320, "stable layout"); + +- Rust bindings generated with `bindgen` in `build.rs`; the struct layout is asserted in CI on both platforms. + +## 5. Build System + +- `proxy_agent/build.rs` invokes `clang -target bpf -O2 -g -c` with `-emit-llvm`+`llc` for each program file. +- Pin `clang` version (15+); pin `llvm`. +- Generated `.bpf.o` files are checked into the artifact pipeline (not the repo). +- One output per program: `cgroup_connect.bpf.o`, `sk_lookup.bpf.o`, `lsm.bpf.o`, `audit_event.bpf.o`. + +## 6. BTF Strategy + +- Prefer kernel-provided `/sys/kernel/btf/vmlinux` at load time. +- For kernels without BTF, ship a small BTF stub generated by `pahole` for the structs we actually use (`struct sock`, `struct task_struct`'s relevant fields). +- Document a minimum kernel of 5.4 (per current README compatibility) with CO-RE relocations rather than per-kernel builds. + +## 7. Integration + +- `proxy_agent/src/redirector/linux/` uses `libbpf-rs` Skel objects generated by `libbpf-cargo`. +- Windows side switches to consume the same header for the user-space event ring buffer. +- Removed: per-distro compile-time forks; replaced with feature probes at load. + +## 8. Tests + +- Cross-kernel matrix CI (5.4, 5.15, 6.1, 6.8) loads each program and runs a smoke test that triggers the audit event. +- Layout assertion test on both Linux and Windows; deliberate field add must update version number. +- Performance: load time \< 100 ms across all kernels. + +## 9. Risks + +- **clang/LLVM bugs** in CO-RE relocation. Mitigation: pin known-good toolchain; fallback to per-kernel build path retained for one release. +- **vmlinux.h size** — large but only at build time; not shipped. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|----------------------------------------------|------------------------------------------------------| +| M1 | libbpf-rs adoption | Existing program runs unchanged on supported kernels | +| M2 | Shared header + cross-platform Rust bindings | Layout assertions green on both OSes | +| M3 | Drop per-kernel builds | CI matrix \< 1/3 of previous count | + +Detail design for direction 4.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-4.3-ipv6-dual-stack.md b/doc/plans/Innovation-4.3-ipv6-dual-stack.md new file mode 100644 index 00000000..9d30dd23 --- /dev/null +++ b/doc/plans/Innovation-4.3-ipv6-dual-stack.md @@ -0,0 +1,110 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Design](#design) +4. [4. eBPF v6 programs](#ebpf) +5. [5. Listener](#listener) +6. [6. Canonical Destination v6](#canon) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 4.3** · **Network** + +# Detailed Design — IPv6 / Dual-stack Support + +Extend the redirect, listener, canonical model, and rule engine to handle IPv6 fabric endpoints uniformly with IPv4. Closes the gap on dual-stack VMs. + +**Files affected:** `linux-ebpf/sk_lookup.bpf.c`, `ebpf/redirect.bpf.c`, `proxy_agent/src/proxy/proxy_server.rs`, `proxy_agent/src/proxy/canonical/destination.rs`. + +> **Prerequisites:** [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|------------|---------|------------------| +| **Medium** future-proofing | **Medium** | **Low** | **eBPF + agent** | + +### Goals + +- IPv6 fabric link-local addresses (e.g. `fe80::a9fe:a9fe`) caught by eBPF and routed through agent. +- Canonical destination enum unified across families; rule engine sees one `Destination::Imds` regardless of family. +- Defeat IPv4-mapped IPv6 bypasses (pentest C7) at the kernel layer. + +## 2. Today + +Redirect handles IPv4 only. Dual-stack VMs that route fabric over v6 (uncommon today but increasing) bypass the agent. The canonical model in direction 2.1 already plans for v6 typed destinations; this direction wires it through the kernel. + +## 3. Design + +- Add v6 sibling programs: `cgroup_connect6`, `sk_lookup_v6`. +- Listener binds IPv6 socket with `IPV6_V6ONLY=0` dual-stack on Linux, or two sockets where dual-stack is unavailable. +- Canonical `Destination` resolves IPv4-mapped IPv6 (`::ffff:a.b.c.d`) to the v4 destination — there is exactly one `Destination::Imds` regardless of family. +- Per-destination address tables published to BPF programs via a `BPF_MAP_TYPE_HASH` keyed on a 16-byte normalized address. + +## 4. eBPF v6 Programs + + SEC("cgroup/connect6") + int gpa_connect6(struct bpf_sock_addr *ctx) { + struct in6_addr dst; + __builtin_memcpy(&dst, ctx->user_ip6, sizeof(dst)); + if (!is_fabric_dest6(&dst, bpf_ntohs(ctx->user_port))) return 1; + // Redirect: rewrite to agent's v6 listener + set_user_dest_v6(ctx, &agent_v6, agent_port); + return 1; + } + +- `is_fabric_dest6` recognizes the v6 link-local equivalent (typically `fe80::a9fe:a9fe` if used) and IPv4-mapped forms. +- SO_ORIGINAL_DST equivalent for v6 via `IP6T_SO_ORIGINAL_DST`; reachable from user space. + +## 5. Listener + +- Bind `[::1]:3080` in addition to `127.0.0.1:3080` (or single dual-stack socket). +- Original destination read on accept via family-appropriate `getsockopt`. +- Listener exposed via attestation endpoint (3.3) with all bound addresses. + +## 6. Canonical Destination v6 + + impl Destination { + pub fn from_ip(ip: IpAddr, port: u16) -> Destination { + let v4 = match ip { + IpAddr::V4(v) => Some(v), + IpAddr::V6(v) => v.to_ipv4_mapped(), + }; + match (v4, port) { + (Some(Ipv4Addr::new(169,254,169,254)), 80) => Destination::Imds, + (Some(Ipv4Addr::new(168,63,129,16)), 80) => Destination::WireServer, + (Some(Ipv4Addr::new(168,63,129,16)), 32526) => Destination::HostGaPlugin, + _ => Destination::Unknown { /* ... */ }, + } + } + } + +## 7. Integration + +- Loader (4.2) detects v6 enablement on the host and loads v6 programs only when needed. +- PoP token (1.1) `dip` claim is always a 16-byte normalized form so signatures cover both families. +- Telemetry: per-family labels on `gpa_requests_total`. + +## 8. Tests + +- Dual-stack pod test: v4 and v6 requests both reach the agent and produce identical `Destination`. +- Pentest C7 v6 variants: all map to `Destination::Imds` after canonicalization. +- Linkup test for hosts without v6 — programs not loaded; no warnings. + +## 9. Risks + +- **Fabric v6 endpoints not finalized** in some regions — make destinations data-driven via the BPF map so production can update without redeploying eBPF. +- **Dual-stack socket semantics** vary on Windows — keep two sockets there. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------|---------------------------------------------| +| M1 | v6 listener + canonical fold | v4 behavior unchanged | +| M2 | connect6 + sk_lookup_v6 | v6 fabric traffic captured in dual-stack VM | +| M3 | Data-driven dest table | Region rollout without rebuild | + +Detail design for direction 4.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-4.4-ebpf-throttling-lru.md b/doc/plans/Innovation-4.4-ebpf-throttling-lru.md new file mode 100644 index 00000000..9c1da887 --- /dev/null +++ b/doc/plans/Innovation-4.4-ebpf-throttling-lru.md @@ -0,0 +1,111 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. LRU map](#lru) +4. [4. Token bucket](#bucket) +5. [5. Sizing](#sizing) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 4.4** · **DoS hardening** + +# Detailed Design — Kernel-side Throttling & Audit-map LRU + +Replace the audit hash map with an LRU-evicting map, and add a per-cgroup token bucket in BPF so connection floods are dropped early. Mitigates pentest `G1` (connection flood) and `G3` (audit-map exhaustion). + +**Files affected:** `linux-ebpf/cgroup_connect.bpf.c`, `linux-ebpf/audit_event.bpf.c`, `proxy_agent/src/redirector/`. + +> **Prerequisites:** [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-------------------------|-----------|---------|----------| +| **Medium** availability | **Small** | **Low** | **eBPF** | + +### Goals + +- Audit map cannot be exhausted to evict legitimate entries. +- A noisy cgroup cannot DoS the agent into a fail-open window. +- Throttling visible via the metrics in 3.2. + +## 2. Today + +The audit map is a fixed-size hash map. Once full, new identities cannot be recorded and a legitimate caller may be missing an audit entry at decision time. There is no kernel-side rate limit. + +## 3. LRU Map + + struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, __u64); // cgroup_id + __type(value, struct gpa_audit_event); + __uint(max_entries, AUDIT_MAP_MAX); + } gpa_audit_map SEC(".maps"); + +- Eviction is least-recently-used; a stale cgroup ages out, an active one stays. +- `AUDIT_MAP_MAX` sized from `(expected unique cgroups) * 2`; default 16,384. +- Map size sampled by the agent every 30 s and exported (`gpa_ebpf_audit_map_entries`). + +## 4. Token Bucket + + struct token_bucket { __u64 tokens; __u64 last_refill_ns; }; + struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, __u64); // cgroup_id + __type(value, struct token_bucket); + __uint(max_entries, AUDIT_MAP_MAX); + } gpa_rl SEC(".maps"); + + SEC("cgroup/connect4") + int gpa_rate_limit(struct bpf_sock_addr *ctx) { + __u64 cg = bpf_get_current_cgroup_id(); + struct token_bucket *b = bpf_map_lookup_elem(&gpa_rl, &cg); + if (!b) { /* lazy init */ } + refill(b, bpf_ktime_get_ns()); + if (b->tokens == 0) { + increment_counter(METRIC_RL_DROPPED, cg); + return 0; // deny connect + } + b->tokens--; + return 1; + } + +- Bucket capacity: 100 connects, refill 50/s per cgroup (tunable). +- Dropped connect returns `EACCES` to the caller; not a silent black-hole. +- Per-cgroup, so a single noisy container cannot starve neighbors. + +## 5. Sizing & Tuning + +| Param | Default | Source | +|--------------------------|---------|--------------------------------------------| +| `AUDIT_MAP_MAX` | 16384 | Config; overridable via `--ebpf-audit-max` | +| Token bucket capacity | 100 | Config | +| Token bucket refill rate | 50/s | Config | +| Map sampler interval | 30 s | Config | + +## 6. Integration + +- Redirector loads the new map types and exposes counters to OTel (3.2). +- Telemetry: `gpa_ebpf_audit_map_evictions_total`, `gpa_ebpf_rate_limited_total{cgroup}`. +- `gpa-doctor` warns when eviction rate or RL drop rate is non-zero over 5 minutes. + +## 7. Tests + +- Spawn many short-lived cgroups → map grows but never exceeds `AUDIT_MAP_MAX`; eviction counter increases; legitimate cgroup retains entry due to recency. +- Connect-flood single cgroup → RL drops counter increments; agent CPU stays bounded; service does not crash (pentest `G1`). +- G3 audit-map exhaustion attempt → legitimate caller retains decision attribution. + +## 8. Risks + +- **Bursty workloads** hit RL ceiling. Mitigation: per-config bucket; observable in metrics; doc tuning guidance. +- **LRU not strict FIFO** — acceptable for this use case. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|-------------------|----------------------------------------| +| M1 | LRU map + sampler | G3 pentest PASS | +| M2 | Token bucket | G1 pentest PASS; agent steady at flood | + +Detail design for direction 4.4. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-5.1-aks-container-native.md b/doc/plans/Innovation-5.1-aks-container-native.md new file mode 100644 index 00000000..4ca6ba4d --- /dev/null +++ b/doc/plans/Innovation-5.1-aks-container-native.md @@ -0,0 +1,177 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Problem](#problem) +3. [3. Pod identity](#identity) +4. [4. Azure Workload Identity primer](#workload-identity) +5. [5. Token issuance](#tokens) +6. [6. Deployment](#deploy) +7. [7. Rule shape](#rules) +8. [8. Integration](#integration) +9. [9. Tests](#tests) +10. [10. Risks](#risks) +11. [11. Milestones](#milestones) + +**GPA** · **Direction 5.1** · **AKS** + +# Detailed Design — Kubernetes / AKS-native Mode + +Run GPA per node as a DaemonSet; map the eBPF-captured `cgroup_id` for each connect to a Kubernetes pod identity. Issue pod-scoped tokens via Azure Workload Identity instead of handing back the node-MI token — closes the "pod steals node MI" class of attack. + +**Prerequisite eBPF change:** the audit map entry must include `cgroup_id` (see [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md)). Today's `sock_addr_audit_entry` only carries `process_id` — see [linux-ebpf/socket.h](../linux-ebpf/socket.h). + +**Files affected:** new `proxy_agent/src/k8s/` module, deployment manifests, integrates with PoP (1.1). + +> **Prerequisites:** [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md)[1.4 Capability scopes](Innovation-1.4-capability-scopes.md)[1.1 PoP tokens](Innovation-1.1-pop-tokens.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------------|-----------|---------------------------|-----------------------| +| **High** new surface, big market | **Large** | **Coordination with AKS** | **agent + ecosystem** | + +### Goals + +- A pod that calls IMDS gets a token scoped to *its* ServiceAccount, not the node identity. +- Operators write rules using familiar K8s identifiers (namespace, ServiceAccount, label selectors). +- Zero application code change for pods that already use Azure Workload Identity SDKs. + +## 2. Problem + +On an AKS node, any pod with hostNetwork or a permissive NetworkPolicy can `curl 169.254.169.254/metadata/identity/oauth2/token` and obtain the node managed identity. This is documented as the most common cluster-credential escalation. Existing mitigations are network-policy and Workload Identity but they are easily misconfigured. + +## 3. Pod Identity Resolution + +**Prerequisite (not yet implemented):** step 1 below requires extending the eBPF `sock_addr_audit_entry` to carry `cgroup_id` populated via `bpf_get_current_cgroup_id()`. Today the struct in [linux-ebpf/socket.h](../linux-ebpf/socket.h) (and the Windows counterpart in [ebpf/socket.h](../ebpf/socket.h)) only stores `process_id`. The field is proposed in the unified schema of [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md). Until then, the agent must derive cgroup from `/proc//cgroup` via the audit entry's `process_id` (slower, racy for short-lived PIDs). + +1. eBPF audit map provides `cgroup_id` for each connect *(post-4.2)*; pre-4.2 fallback uses `process_id` + `/proc//cgroup`. +2. Agent reads `/proc//cgroup` + CRI socket (`containerd` / `cri-o`) to map cgroup → container ID → pod. +3. Pod metadata (namespace, name, ServiceAccount, labels) is cached from the local kubelet pod-resources API and the `--pod-manifest-path` watch. +4. Cache invalidated when pod sandbox is recreated; cached entries hold for ≤ 60 s after pod deletion to handle in-flight requests. + +## 4. Azure Workload Identity — Primer + +Azure Workload Identity (AWI) is the upstream Kubernetes-native way for pods to authenticate to Entra ID (Azure AD) **without storing secrets**. GPA's AKS mode consumes AWI as the identity source — the projected ServiceAccount token (or the resulting AAD token) becomes the caller identity bound to a pod, replacing host-level IMDS interception as the trust anchor. + +### 4.1 How it works + +1. The AKS cluster publishes a public OIDC discovery document (`/.well-known/openid-configuration` + JWKS). +2. Each pod gets a **projected ServiceAccount JWT** mounted by kubelet, signed by the cluster issuer. +3. The pod (via Azure SDK `DefaultAzureCredential` / `WorkloadIdentityCredential`) exchanges that JWT at Entra ID's `/oauth2/v2.0/token` endpoint using the *federated credential* flow (`client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer`). +4. Entra ID validates `iss` (cluster issuer URL) and `sub` (`system:serviceaccount::`) against a **Federated Identity Credential** configured on the target App Registration / User-Assigned Managed Identity, and returns an AAD access token scoped to that workload. + +End-to-end token exchange (baseline AWI, no GPA in the path): + +sequenceDiagram autonumber participant Pod as Pod (Azure SDK) participant Kubelet participant Entra as Entra ID (AAD) participant Azure as Azure Resource +(Key Vault / Storage / ARM) Kubelet-\>\>Pod: Mount projected SA JWT +(iss = cluster OIDC, sub = system:serviceaccount:ns:sa) Pod-\>\>Pod: Read AZURE_FEDERATED_TOKEN_FILE + +AZURE_CLIENT_ID / TENANT_ID (injected env) Pod-\>\>Entra: POST /oauth2/v2.0/token +grant_type=client_credentials +client_assertion_type=jwt-bearer +client_assertion=\ Entra-\>\>Entra: Validate iss + sub against +Federated Identity Credential Entra--\>\>Pod: AAD access token (pod-scoped) Pod-\>\>Azure: API call with Bearer \ Azure--\>\>Pod: 200 OK + +With GPA in AKS-mode (§5) synthesizing the IMDS response for legacy callers: + +sequenceDiagram autonumber participant App as Legacy Pod +(curl 169.254.169.254) participant eBPF as eBPF redirect +(on node) participant GPA as GPA DaemonSet +(node-local) participant CRI as containerd / CRI participant Entra as Entra ID (AAD) App-\>\>eBPF: GET /metadata/identity/oauth2/token eBPF-\>\>GPA: Redirect + cgroup_id, pid GPA-\>\>CRI: Resolve cgroup -\> container -\> pod CRI--\>\>GPA: namespace, ServiceAccount, labels GPA-\>\>GPA: Policy check (§7 grants) +match k8s_pod principal alt allowed GPA-\>\>GPA: Read pod's projected SA JWT +from kubelet token volume GPA-\>\>Entra: jwt-bearer exchange +(SA JWT -\> AAD token) Entra--\>\>GPA: Pod-scoped AAD token GPA--\>\>App: IMDS-shaped JSON +{access_token, expires_in, ...} else denied GPA--\>\>App: 403 + structured audit event end + +### 4.2 Key pieces + +| Component | Role | +|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **AKS OIDC issuer** | Cluster publishes JWKS that Entra ID trusts. | +| **ServiceAccount annotation** | `azure.workload.identity/client-id: ` links SA → Entra app/UAMI. | +| **Federated Identity Credential** | On the Entra app/UAMI: trusts tokens where `iss=` and `sub=system:serviceaccount::`. | +| **Mutating webhook** | Injects `AZURE_*` env vars (`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_FEDERATED_TOKEN_FILE`, `AZURE_AUTHORITY_HOST`) and the projected token volume into pods labeled `azure.workload.identity/use: "true"`. | +| **Azure SDK credential** | Performs the JWT-bearer exchange transparently. | + +### 4.3 Why it matters (vs. predecessors) + +- **No secrets** — replaces client-secret-based service principals. +- **Supersedes AAD Pod Identity v1** (deprecated May 2024), which used the NMI/MIC DaemonSets to intercept IMDS — brittle, racy at pod startup, and the very pattern this innovation hardens. +- **Per-pod identity** — each ServiceAccount maps to a distinct Entra identity, enabling least privilege. +- **Cloud-agnostic shape** — same OIDC federation model used by GitHub Actions, GKE, EKS. + +### 4.4 Relevance to GPA + +- GPA *does not replace* AWI — it complements it: pods that already use the AWI SDK keep working unchanged, and GPA additionally enforces a node-side policy so that **even compromised or misconfigured pods cannot fall back to the node MI** via raw IMDS. +- For pods that bypass the SDK and hit `169.254.169.254` directly, GPA can synthesize an IMDS-shaped response backed by the pod's AWI-derived AAD token (see §5) — making AWI the default, transparently. +- The `sub` claim from the projected SA JWT (`system:serviceaccount::`) is the same string GPA's rule engine matches in §7 grants — one identity model end-to-end. + +### 4.5 References + +- [AKS Workload Identity overview](https://learn.microsoft.com/azure/aks/workload-identity-overview) +- [azure-workload-identity project site](https://azure.github.io/azure-workload-identity/) +- [Entra ID workload identity federation](https://learn.microsoft.com/entra/workload-id/workload-identity-federation) + +## 5. Token Issuance + +- Pod's projected ServiceAccount token (OIDC, signed by cluster) is sent to AAD's federated credential endpoint; AAD returns a pod-scoped access token. +- The IMDS-shaped response is synthesized by GPA from this token: same JSON contract as IMDS `/identity/oauth2/token`. +- For non-Workload-Identity pods, behavior is policy-driven: *deny* (default), *node identity* (explicit opt-in), or a fixed allow-list. + +## 6. Deployment + +- **DaemonSet**: one privileged GPA pod per node; mounts host paths for cgroup, kubelet sockets, and the BPF FS. +- **NetworkPolicy** shipped as a sample: blocks all egress to 169.254/16 and 168.63/29 except from the GPA pod's host network. +- **Helm chart** + reference Azure Policy that pins the configuration. + +## 7. Rule Shape + + { + "version": 2, + "grants": [ + { + "principal": { "kind": "k8s_pod", + "namespace": "billing", + "serviceAccount": "frontend" }, + "scopes": ["imds:identity:read"] + }, + { + "principal": { "kind": "k8s_pod", + "namespace": "*", + "labelSelector": "tier=batch" }, + "scopes": ["imds:instance:read"] + } + ] + } + +- Builds on direction 1.4 capability scopes; principal types are pluggable. + +## 8. Integration + +- `proxy_agent/src/k8s/identity_resolver.rs` — cgroup → pod mapping with caching. +- `proxy_agent/src/k8s/token_issuer.rs` — exchanges projected SA token for AAD access token. +- Hooks into the authorizer after canonicalization to enrich `Claims` with pod identity. +- Telemetry: per-pod allow/deny counters with namespace+SA labels (bounded cardinality). + +## 9. Tests + +- Two pods in different namespaces — each gets only its own scoped token. +- Pod with no matching grant → deny + structured audit event. +- Pod sandbox restart → cache reflects new pod identity within a bounded window. +- NetworkPolicy fail-open test: even if NP misconfigured, GPA still authoritatively decides. + +## 10. Risks + +- **CRI sockets vary** across runtimes. Mitigation: pluggable resolver, support containerd + cri-o. +- **Race between connect and pod-metadata cache fill**. Mitigation: brief synchronous lookup on cache miss; bounded by timeout. +- **AKS coordination** on default install and policy library. + +## 11. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------------------|------------------------------------------------| +| M1 | Identity resolver (cgroup → pod) + tests | Demo: per-pod identity in audit log | +| M2 | Token issuer + IMDS contract | Workload Identity SDK passes integration tests | +| M3 | Helm + NetworkPolicy + Azure Policy | Pilot on internal cluster | +| M4 | GA in AKS as opt-in | SLA met for 1 month | + +Detail design for direction 5.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-5.2-gate-more-endpoints.md b/doc/plans/Innovation-5.2-gate-more-endpoints.md new file mode 100644 index 00000000..da7053e2 --- /dev/null +++ b/doc/plans/Innovation-5.2-gate-more-endpoints.md @@ -0,0 +1,104 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Destination drivers](#design) +4. [4. KeyVault example](#kv) +5. [5. ARM token example](#arm) +6. [6. Config](#config) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 5.2** · **New endpoints** + +# Detailed Design — Gate Additional Cloud-Credential Endpoints + +Generalize GPA from "IMDS + WireServer + HostGAPlugin" to a pluggable framework where additional cloud-credential endpoints (KeyVault MSI, ARM token, Storage MI) are governed by the same rule engine. + +**Files affected:** new `proxy_agent/src/destinations/` module, canonical model (2.1), classifier (1.4). + +> **Prerequisites:** [2.1 Canonical request](Innovation-2.1-canonical-request.md)[2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|---------------------------------|------------|------------------|-----------| +| **High** bigger product surface | **Medium** | **Low** additive | **agent** | + +### Goals + +- One rule language to govern all cloud-credential egress. +- New endpoints add a driver module; no core changes. +- Authoring stays declarative. + +## 2. Today + +Destination IPs are hard-coded; classifier knows only IMDS/WireServer/HostGAPlugin URL shapes. Customers wanting to gate KeyVault calls have no path through GPA. + +## 3. Destination Drivers + + pub trait DestinationDriver: Send + Sync { + fn id(&self) -> &'static str; + fn addresses(&self) -> &[SocketAddrSpec]; // IPs/ports for eBPF redirect map + fn classify(&self, req: &CanonicalRequest) -> Option; + fn signer(&self) -> &dyn RequestSigner; // adds creds before forwarding + fn upstream(&self, req: &CanonicalRequest) -> Upstream; // resolved URL/host + } + +- Built-in drivers: `imds`, `wireserver`, `hostga`. +- New drivers: `keyvault_msi`, `arm_token`, `storage_mi`. +- Drivers register their address specs at startup; eBPF redirect map is populated dynamically. + +## 4. KeyVault MSI Example + +- Destination spec: `*.vault.azure.net:443` (TLS-terminated; SNI used for routing). +- Classifier maps `GET /secrets/?api-version=...` to scope `keyvault:secret:read:`. +- Signer fetches an AAD token using PoP-bound identity (no static client secrets in the agent). +- Rules: `{ identity: "billing-pod", scopes: ["keyvault:secret:read:billing-vault"] }`. + +## 5. ARM Token Example + +- Destination: `management.azure.com:443`. +- Classifier maps verb + resource provider to typed scopes (`arm:Microsoft.Compute/virtualMachines:read`). +- Scopes intentionally align with Azure RBAC action names so policy is reviewable side-by-side with RBAC role definitions. + +## 6. Config + + { + "destinations": { + "enabled": ["imds","wireserver","hostga","keyvault_msi"], + "keyvault_msi": { "vaults": ["billing-vault","app-vault"] } + } + } + +- Enabled set drives which BPF redirect entries are loaded. +- Disabling a driver removes its redirect entries safely. + +## 7. Integration + +- Canonical model (2.1) needs to handle TLS-fronted destinations; for those, the agent terminates TLS using a fabric-provisioned cert (acceptable for the localhost hop). +- PoP (1.1) signing applies uniformly because tokens are minted with audience = driver id. +- OTel (3.2) labels include driver id. + +## 8. Tests + +- Per-driver classification golden vectors. +- Disable driver → no redirect entry → connection bypasses agent → fabric-side AAD rejects (defense-in-depth). +- End-to-end: pod calls KeyVault → agent enforces scope → upstream succeeds. + +## 9. Risks + +- **TLS termination at localhost** requires careful cert handling. Mitigation: certs generated per-boot, pinned by SPKI; never exposed off-host. +- **Endpoint churn** — APIs evolve. Mitigation: driver tables updateable independently of agent core. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|--------------------------------------------|-------------------------------| +| M1 | Driver trait + refactor existing endpoints | No behavior change | +| M2 | KeyVault MSI driver | Pilot customer | +| M3 | ARM token driver | Mapped scopes align with RBAC | + +Detail design for direction 5.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-5.3-cross-cloud-port.md b/doc/plans/Innovation-5.3-cross-cloud-port.md new file mode 100644 index 00000000..8737c1be --- /dev/null +++ b/doc/plans/Innovation-5.3-cross-cloud-port.md @@ -0,0 +1,100 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Why cross-cloud](#why) +3. [3. Abstraction](#abstraction) +4. [4. AWS driver](#aws) +5. [5. GCP driver](#gcp) +6. [6. Packaging](#packaging) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones](#milestones) + +**GPA** · **Direction 5.3** · **Multi-cloud** + +# Detailed Design — Cross-Cloud Port + +Refactor signer + destinations into traits so community-supported drivers can govern AWS IMDSv2 and GCP metadata server traffic with the same eBPF chokepoint and rule engine. Positioning: a *metadata firewall for any cloud*. + +**Files affected:** trait surfaces in `proxy_agent/src/destinations/` and `proxy_agent/src/key_keeper/`. + +> **Prerequisites:** [4.2 Core eBPF unification](Innovation-4.2-core-unify-ebpf.md)[2.1 Canonical request](Innovation-2.1-canonical-request.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|------------|------------------|---------------------| +| **Medium** ecosystem reach | **Medium** | **Low** additive | **agent + drivers** | + +### Goals + +- AWS / GCP metadata services can be governed by the same agent. +- Driver authors implement two traits; core remains untouched. +- Existing Azure behavior unchanged. + +## 2. Why Cross-Cloud + +- AWS IMDSv2 and GCP metadata both have the confused-deputy class of bugs that motivated GPA. +- Multi-cloud security teams want one mental model for "what can read instance credentials." +- Architecture (cgroup eBPF + identity-aware proxy) is cloud-neutral; only the signer and destination set are Azure-specific. + +## 3. Abstraction Lines + + pub trait CloudPlatform: Send + Sync { + fn name(&self) -> &'static str; + fn destinations(&self) -> Vec>; + fn local_identity_source(&self) -> Box; // node identity / instance role + } + + pub trait RequestSigner: Send + Sync { + fn sign(&self, req: &mut http::Request, dest: &Destination, caller: &ResolvedIdentity) + -> Result<(), SignError>; + } + +- Azure platform implementation reuses today's logic. +- AWS / GCP implementations provided as separate crates so they can iterate independently. + +## 4. AWS Driver + +- Destinations: `169.254.169.254:80` (IMDS) and instance-profile endpoints. +- Signer: re-mints IMDSv2 session tokens with TTL bound by the caller's policy; rejects IMDSv1 PUT-less requests entirely. +- Caller-scoped tokens optionally bound to pod identity in EKS. + +## 5. GCP Driver + +- Destinations: `metadata.google.internal` (169.254.169.254) and `metadata.google.internal:80`. +- Mandates `Metadata-Flavor: Google` header presence (the standard GCP SSRF guard) and rejects requests without it. +- Authorization scopes derived from URL path (`/instance/service-accounts/default/token` → `gcp:identity:read`). + +## 6. Packaging + +- `gpa-azure`, `gpa-aws`, `gpa-gcp` binaries built from the same workspace with feature flags. +- Default release is `gpa-azure`; AWS/GCP builds maintained by community + signed via the same Sigstore process (3.4). + +## 7. Integration + +- Canonical model (2.1) cloud-neutral; `Destination::Unknown` becomes `Destination::CloudSpecific(&'static str)` for non-fabric endpoints. +- Capability scopes (1.4) namespaced per platform: `aws:sts:assume_role`, `gcp:storage:read`. +- Telemetry (3.2) labels include `cloud`. + +## 8. Tests + +- Conformance tests per driver against documented metadata API shapes. +- Regression: Azure default build behaves byte-identically to today's release. +- SSRF regression: cross-cloud known bypasses (e.g. `Metadata-Flavor` missing on GCP) blocked. + +## 9. Risks + +- **Community ownership** for non-Azure drivers — must clearly mark non-Azure builds as community-maintained. +- **License compatibility** for any cloud-specific SDKs — prefer plain HTTP clients. + +## 10. Milestones + +| M | Deliverable | Exit | +|-----|---------------------------------|-------------------------------------------| +| M1 | Trait refactor; Azure unchanged | All existing tests pass | +| M2 | AWS driver MVP | EC2 instance with IMDSv2 enforcement demo | +| M3 | GCP driver MVP | GCE instance demo | + +Detail design for direction 5.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-6.1-policy-simulator.md b/doc/plans/Innovation-6.1-policy-simulator.md new file mode 100644 index 00000000..165ca4c5 --- /dev/null +++ b/doc/plans/Innovation-6.1-policy-simulator.md @@ -0,0 +1,110 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. CLI design](#cli) +4. [4. Library mode](#lib) +5. [5. Output format](#output) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 6.1** · **DX** + +# Detailed Design — Policy Simulator CLI + +A `gpa policy simulate` command that takes a rule file, a request, and a caller identity, and reports the exact decision plus the matched rule and canonicalization trace. Used by operators in CI before rolling rules and by support engineers to reproduce production decisions. + +**Files affected:** new `proxy_agent/src/bin/gpa_policy.rs`, library reuse from `proxy_agent/src/authorization/`. + +> **Prerequisites:** [2.1 Canonical request](Innovation-2.1-canonical-request.md)[2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md)[2.3 Versioned snapshots](Innovation-2.3-versioned-snapshots.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|------------------------------|-----------|---------|---------| +| **Medium** author confidence | **Small** | **Low** | **CLI** | + +### Goals + +- Same engine, same answer — the CLI uses the production authorizer (no reimplementation). +- Round-tripable input: stdin / file / args. +- Reproduces decisions captured in the audit log byte-for-byte. + +## 2. Today + +Rule authors have no offline way to test rule changes against canonical request inputs short of standing up a VM. Production debugging requires reading source. There is no "explain" output. + +## 3. CLI Design + + gpa policy simulate \ + --rules ./rules.json \ + --request ./req.json \ + --caller ./caller.json \ + [--explain] [--json] + +| Flag | Meaning | +|-------------|------------------------------------------------------| +| `--rules` | Rules JSON, same schema as production | +| `--request` | HTTP request shape: method, url, headers, body | +| `--caller` | Caller identity: pid + measurement + cgroup + claims | +| `--explain` | Emit canonicalization steps and matched rule | +| `--json` | Machine-readable output for CI | + +Exit codes: `0` allow, `1` deny, `2` error (malformed input). + +## 4. Library Mode + + // proxy_agent_shared/src/policy_eval.rs + pub fn simulate(rules: &RuleSet, req: &CanonicalRequest, caller: &ResolvedIdentity) + -> SimulationResult { /* ... */ } + + pub struct SimulationResult { + pub decision: Decision, // Allow / Deny + pub matched_rule: Option, + pub trace: Vec, // each canonicalization step + } + +- Same crate consumed by unit tests across the workspace and by the future VS Code extension (6.3). + +## 5. Output Format + + $ gpa policy simulate --rules r.json --request req.json --caller c.json --explain + DECISION: deny + RULE: none matched + TRACE: + url.raw = http://169.254.169.254/metadata/instance?api-version=2021-12-13 + url.host_norm = 169.254.169.254 + destination = imds + path.norm = /metadata/instance + scope = imds:instance:read + caller.id = sha256:abcd... (binary unmeasured) + caller.scopes = [] + reason = caller.scopes did not include imds:instance:read + +## 6. Integration + +- Same canonical types from direction 2.1. +- Reads remote rule format and the local override (`proxy_agent/src/authorization/local_rules.rs`) so simulation mirrors production resolution order. +- Importable from CI: `gpa policy simulate --rules ./new.json --suite ./golden/` applies a directory of test cases. + +## 7. Tests + +- Round-trip: every entry in `doc/audit-samples/` reproduces exactly under simulation. +- CI corpus: golden test cases checked in; PR-blocking if rules diverge from expectations. +- Fuzz: random rules + random requests; cross-check engine vs. simulator. + +## 8. Risks + +- **Drift** between simulator and production. Mitigation: same library; CI gates on equality of decisions. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|----------------------------|--------------------------------| +| M1 | `simulate()` library + CLI | Reproduces 20 audit samples | +| M2 | Golden test runner | Used in CI | +| M3 | VS Code integration (6.3) | Inline decisions while editing | + +Detail design for direction 6.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-6.2-gpa-doctor.md b/doc/plans/Innovation-6.2-gpa-doctor.md new file mode 100644 index 00000000..ebadff07 --- /dev/null +++ b/doc/plans/Innovation-6.2-gpa-doctor.md @@ -0,0 +1,88 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Checks](#checks) +3. [3. Report](#report) +4. [4. Production safety](#safety) +5. [5. Integration](#integration) +6. [6. Tests](#tests) +7. [7. Risks](#risks) +8. [8. Milestones](#milestones) + +**GPA** · **Direction 6.2** · **Operability** + +# Detailed Design — gpa-doctor + +One-command hardening check derived from the pentest suite. Runs read-only probes that mirror specific pentest cases (A1, E1, E4, G4, D4) and emits a coloured green/yellow/red report. Safe to run on production. + +**Files affected:** new `proxy_agent/src/bin/gpa_doctor.rs`; reuses telemetry endpoints. + +> **Prerequisites:** [3.1 Hash-chained log](Innovation-3.1-hash-chained-log.md)[3.2 OTel export](Innovation-3.2-otel-export.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-----------------------------|-----------|-------------------|---------| +| **High** support deflection | **Small** | **Low** read-only | **CLI** | + +### Goals + +- One command. One report. Zero side effects. +- Each finding cites the pentest case it derives from. +- Run by support engineers and by customers themselves. + +## 2. Checks + +| Check | Maps to | What it does | +|---------------------------------|---------|---------------------------------------------------------------------------------| +| eBPF programs loaded & attached | A1 | Verifies the redirect program is live; tests connect-to-fabric path is captured | +| Loopback bypass | E1 | Probes whether non-agent local processes can reach upstream fabric directly | +| Header smuggling | E4 | Confirms agent strips/rejects suspicious headers known to bypass authorization | +| State file integrity | G4 | Checks owner/mode/SELinux label on `/var/lib/azure/proxyagent/*` | +| Time/clock sanity | D4 | Verifies system clock skew vs WireServer; PoP token expirations | +| Audit log shape | — | Last 1000 entries parse cleanly; no broken hash chain (ties to 3.1) | + +## 3. Report + + $ gpa-doctor + [ OK ] eBPF programs loaded (cgroup_connect4, sk_lookup) [A1] + [ WARN ] Loopback bypass possible: nginx on :8080 has direct route [E1] + [ OK ] Header strip configured [E4] + [ FAIL ] /var/lib/azure/proxyagent/state.json mode is 0644 (want 0640) [G4] + [ OK ] Clock skew 12 ms [D4] + [ OK ] Audit log chain intact (last 1000 entries) + 2 OK · 1 WARN · 1 FAIL · suggested actions printed below + +- Each line ends with the pentest case ID so customers can find the public write-up. +- `--json` mode for monitoring integration. +- Suggestions reference exact `chmod` / config edits. + +## 4. Production Safety + +- All probes are read or self-targeted (we only test our own listener). +- No upstream traffic generated except a single benign IMDS "instance" GET (idempotent). +- Runtime \< 1 second; CPU bounded. +- Refuses to run as a non-root user with a clear message — except for the read-only checks that don't need root. + +## 5. Integration + +- Optional cron/systemd timer publishes the report to Geneva/OTel daily. +- Linked from the README, troubleshooting docs, and CES/CSS playbooks. + +## 6. Tests + +- Each check has a positive and a negative fixture. +- Idempotency: 100 runs produce identical reports given a static system. + +## 7. Risks + +- **False positives** on heterogeneous host configurations. Mitigation: WARN (not FAIL) for ambiguous findings; "more info" link. + +## 8. Milestones + +| M | Deliverable | Exit | +|-----|------------------------------|-------------------------------| +| M1 | Six built-in checks + report | Used by CSS in one ticket | +| M2 | JSON mode + nightly reporter | Telemetry shows fleet posture | + +Detail design for direction 6.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-6.3-rule-authoring-ux.md b/doc/plans/Innovation-6.3-rule-authoring-ux.md new file mode 100644 index 00000000..ff5dd648 --- /dev/null +++ b/doc/plans/Innovation-6.3-rule-authoring-ux.md @@ -0,0 +1,101 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. JSON Schema](#schema) +3. [3. VS Code extension](#ext) +4. [4. Diff view](#diff) +5. [5. Integration](#integration) +6. [6. Tests](#tests) +7. [7. Risks](#risks) +8. [8. Milestones](#milestones) + +**GPA** · **Direction 6.3** · **DX** + +# Detailed Design — Rule Authoring UX + +A JSON Schema for the rule format and a VS Code extension that gives autocomplete, validation, hover docs, and a diff view between the remote rule set and the local override (`local_rules.rs`). + +**Files affected:** new `schema/rules.schema.json`; new `tools/vscode-gpa-rules/` extension; uses the simulator from 6.1. + +> **Prerequisites:** [2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md)[2.3 Versioned snapshots](Innovation-2.3-versioned-snapshots.md)[6.1 Policy simulator](Innovation-6.1-policy-simulator.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|------------|---------|-------------| +| **Medium** author velocity | **Medium** | **Low** | **tooling** | + +### Goals + +- Schema-driven authoring; no need to memorize field names. +- Real-time decision preview using simulator (6.1). +- Make local-override drift obvious. + +## 2. JSON Schema + + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gpa.azure.com/schema/rules.json", + "type": "object", + "required": ["version","grants"], + "properties": { + "version": { "const": 2 }, + "grants": { + "type": "array", + "items": { "$ref": "#/$defs/grant" } + } + }, + "$defs": { + "grant": { + "type": "object", + "required": ["principal","scopes"], + "properties": { + "principal": { "$ref": "#/$defs/principal" }, + "scopes": { "type": "array", "items": { "pattern": + "^(imds|wireserver|hostga|keyvault|arm):" } }, + "conditions": { "$ref": "#/$defs/conditions" } + } + } + } + } + +- Schema is the source of truth; agent parser is generated from it (or tested against it). +- Published at a stable URL so editors auto-fetch. + +## 3. VS Code Extension + +- Activates on files matching `**/gpa.rules.json` or with the JSON schema header. +- Autocomplete for principal kinds, scope strings, conditions. +- Hover docs explain each scope (e.g. "imds:identity:read — allows the token endpoint, response includes the access token"). +- Status-bar shows simulator verdict for a focused request (loaded from a sibling `.req.json`). +- Quick fix: convert `identity: "*"` to scoped rules using cluster/fleet audit data. + +## 4. Diff View + +- "GPA: Compare Local Override to Remote" command opens a diff between the remote rule set and the compiled-in overrides from `proxy_agent/src/authorization/local_rules.rs`. +- Renders both as canonical JSON for clean diff regardless of source format. +- Flags any local rule that is broader than the remote rule. + +## 5. Integration + +- Extension invokes `gpa policy simulate` via a sidecar process (no remote dependencies). +- Schema versioned alongside the agent; CI generates updated schema on each release. + +## 6. Tests + +- Schema validation matches agent parser on a corpus of known-good + known-bad rule files. +- Extension contract tests using `vscode-test`. + +## 7. Risks + +- **Schema drift** vs. parser. Mitigation: generate parser tests from schema; CI fails on drift. + +## 8. Milestones + +| M | Deliverable | Exit | +|-----|---------------------------------------------|------------------------------------------| +| M1 | JSON Schema + drift CI | Schema used in agent tests | +| M2 | VS Code extension (autocomplete + simulate) | Used by ≥ 3 rule authors | +| M3 | Diff view | Local override drift visible at a glance | + +Detail design for direction 6.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-6.4-wasm-rule-sandbox.md b/doc/plans/Innovation-6.4-wasm-rule-sandbox.md new file mode 100644 index 00000000..93c0054f --- /dev/null +++ b/doc/plans/Innovation-6.4-wasm-rule-sandbox.md @@ -0,0 +1,106 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Host ABI](#abi) +3. [3. Limits](#limits) +4. [4. Example](#example) +5. [5. Opt-in](#optin) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 6.4** · **Extensibility** + +# Detailed Design — WASM Rule Sandbox + +An opt-in WebAssembly extension point for organizations that need conditions richer than the declarative rule language. WASM modules are deny-by-default sandboxed: no syscalls, no clocks, no network, no I/O — just compute over the canonical request and claims. + +**Files affected:** new `proxy_agent/src/authorization/wasm/` module; integrates with rule engine (after Cedar from 2.2). + +> **Prerequisites:** [2.2 Typed policy (Cedar)](Innovation-2.2-typed-policy-cedar.md) + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|-------------------------------|------------|-------------------------|-----------| +| **Medium** niche but powerful | **Medium** | **Sandbox correctness** | **agent** | + +### Goals + +- Custom conditions without exposing the agent to arbitrary code. +- Hard, enforced CPU + memory limits. +- Modules are content-addressed; auditable via 3.4 supply-chain pipeline. + +## 2. Host ABI + + // Exported by the WASM module: + // fn decide(req_ptr: i32, req_len: i32, + // claims_ptr: i32, claims_len: i32) -> i32 // 0 = allow, 1 = deny + + // Imported from host (the only imports allowed): + // fn log(ptr: i32, len: i32); // append to a per-decision diag string + // fn abort(); // immediate deny + + // No clocks. No filesystem. No network. No randomness. + +- Inputs and outputs are JSON-encoded canonical request + resolved claims. +- Runtime: [wasmtime](https://github.com/bytecodealliance/wasmtime) with all WASI features disabled. +- Memory cap: 16 MB; fuel cap: 100,000 instructions per call. + +## 3. Limits + +| Limit | Default | Reason | +|----------------------|---------------------|-------------------------| +| Memory | 16 MB | Bounded request size | +| Fuel | 100k instructions | ~1 ms on typical hosts | +| Imports | `log`, `abort` only | Deny-by-default surface | +| Modules per rule set | 16 | Bounded compile time | +| Wall-clock per call | 5 ms hard kill | Tail-latency guard | + +## 4. Example + + // rust crate compiled to wasm32-unknown-unknown + #[no_mangle] + pub extern "C" fn decide(req_ptr: i32, req_len: i32, + claims_ptr: i32, claims_len: i32) -> i32 { + let req: CanonicalRequest = json_read(req_ptr, req_len); + let claims: ResolvedIdentity = json_read(claims_ptr, claims_len); + // Custom rule: only allow IMDS identity reads from pods that exist for > 5 min + if req.destination == "imds" && claims.pod_age_secs.unwrap_or(0) >= 300 { 0 } else { 1 } + } + +The module is referenced from a normal grant: `"condition": { "wasm": "sha256:..."}`. + +## 5. Opt-in + +- Disabled by default. Enabled per-deployment via `--enable-wasm-rules`. +- Module hash must be present in the trust list (Sigstore-signed; see 3.4). +- Per-call telemetry tagged with module hash so noisy modules are findable. + +## 6. Integration + +- After Cedar (2.2) returns "allow with condition C", the engine invokes the WASM condition module. +- Result combined with Cedar's verdict; deny always wins. +- Module compiled once at load, cached in memory. + +## 7. Tests + +- Resource-exhaustion modules (infinite loop, oversize allocation) are bounded as expected. +- Malformed input handling: module returns deny; agent treats abort as deny. +- Sandbox escape attempts (using removed WASI imports) fail to link. + +## 8. Risks + +- **wasmtime CVEs** — pin to LTS; track advisories. +- **Performance regression** if customers push hot paths into WASM. Mitigation: opt-in + telemetry visible. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|--------------------------------------|----------------------------| +| M1 | Host ABI + limits + reference module | Internal demo | +| M2 | Sigstore-signed module store | Trust list enforced | +| M3 | Pilot | One real customer use-case | + +Detail design for direction 6.4. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-7.1-io-uring-hot-path.md b/doc/plans/Innovation-7.1-io-uring-hot-path.md new file mode 100644 index 00000000..5e91ec03 --- /dev/null +++ b/doc/plans/Innovation-7.1-io-uring-hot-path.md @@ -0,0 +1,84 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Design](#design) +4. [4. Feature flag](#features) +5. [5. Benchmark plan](#bench) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 7.1** · **Perf** + +# Detailed Design — io_uring Hot Path + +Switch the IMDS GET hot path to `tokio-uring` (or `monoio`) behind a feature flag. Cuts syscall overhead for the highest-volume request shape. + +**Files affected:** `proxy_agent/src/proxy/proxy_server.rs`; new runtime selection module. + +> **Prerequisites:** None — performance-only change, independent of identity / policy / audit work. + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|--------------------------|------------|------------------|-----------------| +| **Medium** latency + CPU | **Medium** | **Runtime swap** | **proxy_agent** | + +### Goals + +- p50 latency reduction ≥ 30% on IMDS GET path on modern kernels. +- CPU per million requests reduced ≥ 20%. +- No regression on legacy kernels (feature flag off). + +## 2. Today + +Tokio + epoll on Linux. Every request: `accept`, `read`, `write`, `read`, `write`. For a fast localhost GET this is dominated by syscall overhead and scheduler wake-ups. + +## 3. Design + +- Use `tokio-uring` for accept + read + write on the proxy hot loop; the rest of the agent stays on stock Tokio. +- Single-threaded reactor per CPU; SO_REUSEPORT to spread accept. +- Buffer pool: reusable 4 KB buffers registered with io_uring (zero allocations on hot path). +- For unauthorized requests the agent still falls back to the standard path (cold). + +## 4. Feature Flag + +- Cargo feature `io_uring`; off by default. +- Runtime probe: kernel ≥ 5.15 and unrestricted `io_uring_setup`; otherwise the proxy falls back to the Tokio path with one INFO log line. +- Attestation endpoint (3.3) advertises which path is in use. + +## 5. Benchmark Plan + +| Workload | Metric | Target | +|---------------------|-------------------|------------------------------| +| 1 client × 10k req | p50 / p99 latency | ≥ 30% / ≥ 20% lower | +| 50 clients × 10 min | RPS / CPU | ≥ 30% higher RPS at same CPU | +| 500 clients | Tail latency | p99.9 stable | + +## 6. Integration + +- Authorizer stays unchanged; lives behind an async boundary. +- Telemetry (3.2) emits `gpa_runtime_kind` label. + +## 7. Tests + +- Functional parity test: same input → same response on both runtimes. +- Kernel matrix CI (5.4, 5.15, 6.1, 6.8): flag-on tests skipped on unsupported kernels. +- Fuzz: malformed HTTP requests handled identically. + +## 8. Risks + +- **io_uring CVEs** on older kernels. Mitigation: explicit kernel-version gate; documented minimum. +- **Code split** between two runtimes. Mitigation: keep the hot path tiny; AuthZ remains shared. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|----------------------------------|-----------------------| +| M1 | Hot path prototype (flag off) | Benchmark numbers | +| M2 | Feature flag + probe + telemetry | Internal opt-in | +| M3 | Default-on for supported kernels | SLOs hold for 1 month | + +Detail design for direction 7.1. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-7.2-zero-copy-splice.md b/doc/plans/Innovation-7.2-zero-copy-splice.md new file mode 100644 index 00000000..1b922932 --- /dev/null +++ b/doc/plans/Innovation-7.2-zero-copy-splice.md @@ -0,0 +1,75 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Design](#design) +4. [4. Header handling](#headers) +5. [5. Integration](#integration) +6. [6. Tests](#tests) +7. [7. Risks](#risks) +8. [8. Milestones](#milestones) + +**GPA** · **Direction 7.2** · **Perf** + +# Detailed Design — Zero-Copy splice(2) after AuthZ Pass + +Once authorization succeeds, splice the client socket directly to the upstream socket via `splice(2)`. The agent stops touching the payload; bytes flow through a kernel pipe without user-space copy. + +**Files affected:** `proxy_agent/src/proxy/proxy_server.rs`, `proxy_agent/src/proxy/upstream.rs`. + +> **Prerequisites:** None — performance-only change, independent of identity / policy / audit work. + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|----------------------------|-----------|---------|-----------------| +| **Medium** bandwidth + CPU | **Small** | **Low** | **proxy_agent** | + +### Goals + +- Zero user-space copies for the response body. +- Reduce CPU by ≥ 15% on bodies \> 16 KB. +- Preserve TLS-terminated paths (5.2) by skipping splice when content must be inspected/transformed. + +## 2. Today + +Each response goes `recv → userland buffer → send`. For large IMDS goal-state pulls or WireServer extensions data this dominates CPU. + +## 3. Design + +client ──accept──\> agent │ ▼ AuthZ pass (headers parsed) │ ▼ open upstream socket │ ▼ splice(client_fd, upstream_fd) for request body │ ▼ splice(upstream_fd, client_fd) for response body │ └── still record byte counts via tee(2) for audit + +- Two kernel pipes per direction; `splice(2)` with `SPLICE_F_MOVE | SPLICE_F_MORE`. +- `tee(2)` teaches an audit ring of message lengths without copying payload. +- Skip splice when: TLS-terminated, payload transformation required, or body \< 4 KB. + +## 4. Header Handling + +- Agent still parses request line + headers in user space (needed for AuthZ + canonical model). +- Once the boundary is found and the verdict is allow, the remaining body and response are spliced. +- Response headers from upstream are parsed and re-emitted under agent control; only the body is spliced. + +## 5. Integration + +- Falls back to copy-loop on non-Linux and when splice is unsupported. +- Telemetry: `gpa_splice_bytes_total` vs `gpa_copy_bytes_total`. + +## 6. Tests + +- Functional parity: identical byte-for-byte response under both paths. +- Large body benchmark: ≥ 15% CPU reduction at 1 MB bodies. +- Short body verification: short paths unaffected (still copy). + +## 7. Risks + +- **Connection lifecycle** bugs are subtle (half-close, RST during splice). Mitigation: explicit unit + integration tests for terminations. +- **Bypassing future hooks** that would want to inspect payload — keep splice optional per destination driver. + +## 8. Milestones + +| M | Deliverable | Exit | +|-----|---------------------------|----------------------------------| +| M1 | Splice for IMDS large GET | CPU target met | +| M2 | Per-driver opt-in | TLS-terminated paths skip splice | + +Detail design for direction 7.2. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-7.3-crate-consolidation.md b/doc/plans/Innovation-7.3-crate-consolidation.md new file mode 100644 index 00000000..c1e82d58 --- /dev/null +++ b/doc/plans/Innovation-7.3-crate-consolidation.md @@ -0,0 +1,90 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Today](#today) +3. [3. Code moves](#moves) +4. [4. musl static](#musl) +5. [5. Bloat budget](#bloat) +6. [6. Integration](#integration) +7. [7. Tests](#tests) +8. [8. Risks](#risks) +9. [9. Milestones](#milestones) + +**GPA** · **Direction 7.3** · **Footprint** + +# Detailed Design — Crate Consolidation, musl Static, Bloat Budget + +Move duplicated helpers (logging init, config loader, version probe) into `proxy_agent_shared`. Ship a single static `musl` binary per role with a hard cargo-bloat budget enforced in CI. + +**Files affected:** `proxy_agent_shared/`, `proxy_agent/`, `proxy_agent_extension/`, `proxy_agent_setup/`, CI pipeline. + +> **Prerequisites:** None — internal refactor, independent of feature work. + +## 1. Overview & Goals + +| Impact | Effort | Risk | Scope | +|------------------------------|-----------|---------|---------------| +| **Internal** maintainability | **Small** | **Low** | **workspace** | + +### Goals + +- Single source of truth for cross-cutting helpers. +- One static binary per role: `azure-proxy-agent`, `azure-proxy-agent-ext`, `azure-proxy-agent-setup`. +- Binary size capped in CI — surprise growth blocks merge. + +## 2. Today + +Logger init, config loader, and version probe exist in slightly different shapes across `proxy_agent`, `proxy_agent_extension`, and `proxy_agent_setup`. `proxy_agent_shared` already exists but is under-used. + +## 3. Code Moves + +| From | To | Notes | +|----------------------------|---------------------------------|---------------------------------------| +| per-crate `logger` setup | `proxy_agent_shared::logging` | One `init(role, level)` entry point | +| per-crate `config` loaders | `proxy_agent_shared::config` | Layered: file → env → defaults; serde | +| OS / version probe | `proxy_agent_shared::host_info` | One function returns full struct | +| HTTP client helpers | `proxy_agent_shared::http` | One client, with the right TLS roots | + +## 4. musl Static Build + +- Add CI target `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`. +- Drop dynamic libc dependency for the agent role; setup tool optionally remains glibc. +- Cross-distro install: same binary on RHEL 8/9, Ubuntu 22.04/24.04, Azure Linux. +- eBPF object files shipped separately (loaded by libbpf at runtime), independent of libc. + +## 5. Bloat Budget + + # CI step + cargo install cargo-bloat --locked + cargo bloat --release --crates --message-format json > bloat.json + python3 ci/check_bloat.py --max-binary-bytes 20000000 --max-crate-share 0.10 + +- Hard ceiling on total binary size (e.g. 20 MB stripped). +- No single non-first-party crate may exceed 10% of total text. +- Drift detector: PR comment shows top contributors and delta from main. + +## 6. Integration + +- SBOM (3.4) generated from the static binary's compiled crate graph. +- Attestation endpoint (3.3) reports binary hash and bloat report URL. +- Pkg builds (deb/rpm) pick up the static binary unchanged. + +## 7. Tests + +- Smoke test on each supported distro using the static binary. +- Bloat budget test runs on every PR; documented override path with mandatory two-reviewer approval. + +## 8. Risks + +- **musl perf** for DNS / network can differ. Mitigation: micro-benchmark before/after. +- **Bloat budget** false alarms on benign upgrades. Mitigation: weekly main-branch baseline refresh. + +## 9. Milestones + +| M | Deliverable | Exit | +|-----|--------------------------------------|--------------------------------| +| M1 | Helper moves to `proxy_agent_shared` | No duplicate code; tests green | +| M2 | musl static binary in CI | Cross-distro smoke green | +| M3 | Bloat budget enforced | PRs blocked over ceiling | + +Detail design for direction 7.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-Directions.md b/doc/plans/Innovation-Directions.md new file mode 100644 index 00000000..88df6264 --- /dev/null +++ b/doc/plans/Innovation-Directions.md @@ -0,0 +1,450 @@ +## Innovation Directions + +1. [1. AuthN/AuthZ Model](#d1) + - [Short-lived PoP tokens](#d1-tokens) + - [vTPM / CVM sealing](#d1-vtpm) + - [Measured identity](#d1-identity) + - [Capability grants](#d1-cap) +2. [2. Rule Engine Modernization](#d2) +3. [3. Observability & Supply Chain](#d3) +4. [4. eBPF / Kernel Hardening](#d4) +5. [5. Threat Coverage Expansion](#d5) +6. [6. Developer & Operator UX](#d6) +7. [7. Performance & Footprint](#d7) +8. [Cross-cutting Roadmap](#roadmap) +9. [**★ Plans & Milestones**](Innovation-Plans-Milestones.md) + +**Azure Guest Proxy Agent** · **Rust + eBPF** · **Security** + +# Innovation Directions — Detailed Designs & Implementation Plans + +Companion document to the repo analysis. Each direction includes goals, design, code-level touch points in the GPA codebase, an incremental implementation plan, test strategy, risks, and success metrics. + +→ See also: [**Consolidated Plans & Milestones**](Innovation-Plans-Milestones.md) — cross-track schedule, dependency map, per-innovation M0–M4 milestones, RACI, risks, and program exit criteria. + +**Reference areas:** `proxy_agent/src/proxy/authorization_rules.rs`, `proxy_agent/src/key_keeper/`, `proxy_agent/src/redirector/`, `proxy_agent_extension/`, `pentest/linux/`. + +## 1. Strengthen the AuthN/AuthZ Model + +| Impact | Effort | Risk | Scope | +|------------------------------------|------------------|-----------------------------|--------------------------| +| **High** closes whole vuln classes | **Medium–Large** | **Wire-compat with fabric** | **agent + fabric coord** | + +Today GPA authenticates with a long-lived HMAC key latched at provisioning time (`proxy_agent/src/key_keeper/key.rs`) and adds `x-ms-azure-signature` to every authorized request. Identity is taken from cgroup + `processFullPath` reported by eBPF audit. Four sub-initiatives raise the floor. + +### 1.1 Short-lived Proof-of-Possession (PoP) tokens + +[Detailed design](Innovation-1.1-pop-tokens.md) + +#### Goal + +- Eliminate the "steal key file → sign forever" path (pentest `B3`, `E5`). +- Bind each token to caller, destination, and time, so replay (`B2`) becomes structurally impossible. + +#### Design + +- Replace the single HMAC over `METHOD || URL || time-tick` with a JWS-like compact token: `header.payload.sig` where payload includes `{aud, sub (caller fingerprint), iat, exp ≤ 30s, nonce, dest_ip, url_hash}`. +- Signature stays HMAC-SHA256 initially (no fabric crypto change), but the *signed claims* shape changes, so the fabric can reject any token without an `exp`. +- Token is minted per request in `proxy_server.rs` right before forwarding; replaces direct header injection. +- Add a *session key* derived from latched key + nonce so the latched key never appears on the wire and never signs raw HTTP. + +#### Code touch points + +- `proxy_agent/src/key_keeper/key.rs` — add `derive_session_key(nonce) -> SessionKey`. +- `proxy_agent/src/proxy/proxy_server.rs` — replace `x-ms-azure-signature` mint path with `mint_pop_token(req, caller, dest)`. +- `proxy_agent_shared` — new module `pop_token` with serde structs + constant-time compare. +- Wire-compat shim: emit both legacy and new headers behind a feature flag `pop_v2` until fabric is ready. + +#### Plan + +1. RFC + threat model doc; align with WireServer/IMDS team on header name and accepted claim set. +2. Implement `pop_token` crate with golden-vector tests. +3. Ship dual-emit behind config flag; telemetry-only (fabric ignores new header). +4. Fabric flips acceptance; deprecate legacy header after one release cycle. + +#### Tests + +- Unit: claim canonicalization, clock-skew tolerance, constant-time verify. +- Pentest re-runs: `B2` replay must fail; `B3` stolen-key + cross-VM must still fail because `sub` binds caller identity verified by fabric+vTPM (see 1.2). +- Fuzz the token parser (`cargo-fuzz`). + +#### Risks + +- Clock drift; mitigate by accepting ±60s and refreshing via fabric time. +- Fabric rollout coupling; mitigate with dual-emit flag. + +### 1.2 vTPM / Confidential-VM attestation binding + +[Detailed design](Innovation-1.2-vtpm-sealing.md) + +#### Goal + +Make a stolen key file useless on another VM, and make key rollback (`E5`) cryptographically infeasible. + +#### Design + +- At provisioning, seal the latched key to vTPM PCRs covering: firmware, bootloader, kernel cmdline, and `azure-proxy-agent` binary measurement (IMA). +- Under CVM (SEV-SNP / TDX), include the attestation report hash. The fabric stores the bound report and only honors signatures whose KID matches. +- On unseal failure (rebooted into a tampered image), GPA enters fail-closed and requests a re-provisioning. + +#### Code touch points + +- New crate `proxy_agent/src/key_keeper/sealing/` with backends: `tpm2.rs` (uses `tss-esapi`), `snp.rs`, `tdx.rs`, `noop.rs`. +- `key.rs` — add `load_sealed()` / `store_sealed()` wrapping current on-disk reads. +- Provisioning flow in `provision.rs` — request fresh attestation, hand to fabric, persist sealed blob. + +#### Plan + +1. Backend detection helper (probe `/dev/tpmrm0`, SEV-SNP MSRs, TDX guest module). +2. Implement `noop` + `tpm2` backends behind feature flag; keep current plain-file path as default. +3. Pilot in selected SKUs; collect attestation latency telemetry. +4. Promote to default for CVM SKUs; keep plain-file as fallback for legacy SKUs. + +#### Tests + +- Reboot with modified kernel cmdline must produce unseal failure → fail-closed. +- Snapshot+restore of `/var/lib/azure-proxy-agent` to a different VM must fail unseal. +- Pentest `E5` rollback: re-introducing an older sealed blob fails monotonic counter check. + +### 1.3 Measured caller identity (replace path-string matching) + +[Detailed design](Innovation-1.3-measured-identity.md) + +#### Goal + +Defeat pentest scenarios `C3` (bind-mount over `/proc/self/exe`) and `D2` (symlink to allowed binary) by matching on *what the code is*, not *where it lives*. + +#### Design + +- Capture binary measurement in-kernel via **IMA** (`ima_file_hash()`) or **fs-verity** root hash; on Windows, use code-integrity / WDAC hash. +- Emit measurement in the eBPF audit event consumed by `redirector::lookup_audit`. +- Extend `Privilege` in `key.rs` with optional `exeHash: Option`; matcher prefers hash over path when present, and rejects when both diverge. + +#### Code touch points + +- `linux-ebpf/ebpf_cgroup.c` — augment audit map value with hash bytes. +- `proxy_agent/src/redirector/linux/` — surface hash field. +- `proxy_agent/src/proxy/authorization_rules.rs` — new matcher predicate; back-compat: if rule lacks hash, fall back to path matching. + +#### Plan + +1. Add hash plumbing end-to-end, audit-only (log mismatches). +2. Author tooling to generate hashes for allow-listed binaries (extension handlers, customer agents). +3. Enable enforcement per rule via `enforceMeasurement: true`. + +### 1.4 Capability-style scoped grants + +[Detailed design](Innovation-1.4-capability-scopes.md) + +#### Goal + +Move from "this path is allowed for this identity" to verifiable scopes (`imds:instance:read`, `wireserver:goalstate:read`, `hostga:extensions:status:write`). + +#### Design + +- Introduce a Cedar/CEL-style policy compiled to an evaluation IR at rule-load time. +- Each request is mapped to a typed `Action` + `Resource` by a URL classifier (one canonical mapping table per endpoint). +- Identity carries a set of granted scopes; decision is `scopes ⊇ required(action)`. + +#### Why it matters + +- Enables static analysis ("does any rule grant unauthenticated WireServer write?") — see direction 2. +- Eliminates SSRF-style URL-encoding bypasses because the classifier normalizes once and operates on the typed action. + +## 2. Modernize the Rule Engine + +| Impact | Effort | Risk | Scope | +|-----------------------------------|------------|--------------------|----------------| +| **High** kills SSRF-bypass family | **Medium** | **Self-contained** | **agent only** | + +Current matcher: lowercased `request.path().starts_with(rule.path)` plus a query-param map (authorization_rules.rs:194, key.rs:223). This is the largest single source of latent AuthZ bypass surface. + +### 2.1 Canonical request model + +[Detailed design](Innovation-2.1-canonical-request.md) + +- Build a single `CanonicalRequest` type produced by *one* normalizer: percent-decode → collapse `.`/`..` → strip `;params` → lowercase host → resolve numeric IP forms (decimal, hex, IPv4-mapped IPv6 — pentest `C7`). +- Use the same normalizer for rule loading and request matching — eliminates rule/request asymmetry. +- Reject requests whose normalization is ambiguous (e.g. invalid UTF-8 in path) — fail-closed. + +### 2.2 Typed policy language + +[Detailed design](Innovation-2.2-typed-policy-cedar.md) + +- Pick one: **Cedar** (Rust-native, fast, analyzable) or **OPA/Rego** (familiar). Recommendation: **Cedar** — has a verified evaluator and supports static analysis. +- Compile JSON rules at load time into a Cedar policy set; keep a legacy adapter so existing rules continue to work. + +  + + // proxy_agent/src/proxy/policy/mod.rs + pub struct CompiledPolicy { /* Cedar policy set + entity store */ } + + impl CompiledPolicy { + pub fn from_legacy(rules: &AuthorizationItem) -> Result { /* ... */ } + pub fn evaluate(&self, req: &CanonicalRequest, caller: &CallerIdentity) -> Decision { /* ... */ } + } + +### 2.3 Versioned snapshots (TOCTOU fix — pentest D5) + +[Detailed design](Innovation-2.3-versioned-snapshots.md) + +- Wrap the active policy in `arc_swap::ArcSwap`. +- Each request captures `(arc, epoch)` at accept time and uses it for the whole forwarding decision. +- Surface `policy_epoch` in the connection log. + +### 2.4 Differential / property testing + +[Detailed design](Innovation-2.4-differential-testing.md) + +- For each rule loaded, auto-generate "evil twins": case toggles, percent-encoded slashes, trailing-slash variants, IPv6 forms, Unicode confusables. +- Run the matcher on each variant; mismatch with the original ⇒ block the rule and alert. +- Integrate into `local_rules.rs` reload path and into CI. + +### Plan + +1. Introduce `CanonicalRequest` + normalizer; add property-tests (`proptest`) and run against the current matcher in shadow mode. +2. Land Cedar compilation behind a feature flag; dual-evaluate (legacy + Cedar), log divergences. +3. Flip enforcement to Cedar once divergence rate is zero for N days in production telemetry. +4. Remove legacy matcher. + +### Metrics + +- Divergence rate (legacy vs Cedar) → must reach 0 before cutover. +- Pentest `D1`/`C7` scenarios reach 100% pass. +- Rule load time \< 50 ms for 1k rules. + +## 3. Observability & Supply-Chain Trust + +| Impact | Effort | Risk | Scope | +|-------------------------------|------------|---------|----------------------------| +| **Medium** + audit/compliance | **Medium** | **Low** | **agent + build pipeline** | + +### 3.1 Hash-chained, append-only audit log + +[Detailed design](Innovation-3.1-hash-chained-log.md) + +- Wrap the existing connection log with a Merkle chain: `entry_n.hash = SHA256(entry_n.payload || entry_{n-1}.hash)`. +- Periodically anchor the tip to a transparency log (rekor-compatible) or to Azure Monitor as a signed sentinel. +- Closes pentest `F2` (log injection) and `F3` (rotation race) — tampering breaks the chain and is detectable. + +### 3.2 OpenTelemetry export + +[Detailed design](Innovation-3.2-otel-export.md) + +- Emit metrics: allow/deny counts by rule id, signer latency, eBPF map occupancy, restart count. +- Emit traces for the proxy hop (accept → authz → upstream → response) with W3C trace context. +- Optional OTLP endpoint; defaults to local Prometheus exposition on a UDS only. + +### 3.3 Self-attestation endpoint + +[Detailed design](Innovation-3.3-self-attestation.md) + +- New read-only endpoint on the local listener: `GET /.well-known/gpa/attestation`. +- Returns: agent version, binary measurement, loaded eBPF prog ids and bytecode hash, attached cgroup, active `policy_epoch`, sealed-key KID, attestation backend in use. +- Consumable by Defender for Cloud, Azure Policy, or operator scripts. + +### 3.4 Supply chain + +[Detailed design](Innovation-3.4-supply-chain.md) + +- **SBOM**: generate CycloneDX during the cargo build (`cargo-cyclonedx`). +- **Reproducible builds**: pin `rust-toolchain.toml`, vendor deps, use `-Clink-arg=-Wl,--build-id=none`; verify via two-builder diff in CI. +- **in-toto / Sigstore**: sign release artifacts; `proxy_agent_setup` verifies signature before installing — closes pentest `H1`. + +### Plan + +1. Refactor logger into a `trait Sink` with a chained-Merkle implementation. +2. Wire OTel behind `--features otel`; default off to keep footprint. +3. Add attestation endpoint (no secrets in payload; just measurements). +4. Pipeline work: SBOM, reproducible build, Sigstore signing, verification in setup binary. + +## 4. eBPF / Kernel Hardening + +| Impact | Effort | Risk | Scope | +|----------------------------------|------------|---------------------------|-----------------------------| +| **High** kernel-layer chokepoint | **Medium** | **Kernel-version matrix** | **linux-ebpf + redirector** | + +### 4.1 Move from cgroup/`connect4` to `bpf_lsm` + `sk_lookup` + +[Detailed design](Innovation-4.1-sk-lookup-bpf-lsm.md) + +- `sk_lookup` redirects on the listening side; the original destination IP is preserved (no SNAT-to-localhost), so we can match on real destination after netns shenanigans (pentest `C5`, `C6`). +- `bpf_lsm` hooks (`socket_connect`) provide a deny path even when a hostile program tries to construct sockets in alternate namespaces. +- Fallback to existing cgroup hook for kernels \< 5.13 (no `sk_lookup`). + +### 4.2 Unify Linux + Windows eBPF on CO-RE + +[Detailed design](Innovation-4.2-core-unify-ebpf.md) + +- Today: separate sources in `ebpf/` (Windows) and `linux-ebpf/` (Linux). +- Adopt libbpf-rs with CO-RE relocations; share the audit-event struct via a common header. +- Ship a single BTF-portable object per platform; drop kernel-version-specific builds. + +### 4.3 IPv6 / dual-stack + +[Detailed design](Innovation-4.3-ipv6-dual-stack.md) + +- Add v6 redirect for IMDS/WireServer link-local equivalents. +- Normalize address family at the audit-event boundary so the rule engine sees a unified `Destination` enum, not raw bytes. + +### 4.4 Kernel-side throttling and audit-map LRU + +[Detailed design](Innovation-4.4-ebpf-throttling-lru.md) + +- Replace the audit hash map with `BPF_MAP_TYPE_LRU_HASH` sized by cgroup count (pentest `G3`). +- Add a token bucket per cgroup in BPF; over-rate connections get an early reject before reaching user space (mitigates `G1`). + +### Code touch points + +- `linux-ebpf/ebpf_cgroup.c` → split into `cgroup_connect.bpf.c`, `sk_lookup.bpf.c`, `lsm.bpf.c`. +- `proxy_agent/src/redirector/linux/` → loader picks the best available program set. +- Build system: introduce `build.rs` step invoking `clang -target bpf` with BTF. + +### Plan + +1. Add CO-RE build, keep behavior identical (no semantic change). +2. Land `sk_lookup` as optional, gated by kernel probe; A/B in pentest harness. +3. Add `bpf_lsm` deny hook; verify with `C5`/`C6` scenarios. +4. Switch audit map to LRU and add token bucket; verify with `G1`/`G3`. +5. IPv6 path last (depends on fabric v6 readiness). + +## 5. Expand the Threat Coverage + +| Impact | Effort | Risk | Scope | +|------------------------------|-----------|-----------------------------|-----------------------| +| **High** new product surface | **Large** | **Cross-team coordination** | **agent + ecosystem** | + +### 5.1 Container-native / AKS mode + +[Detailed design](Innovation-5.1-aks-container-native.md) + +#### Problem + +On AKS nodes, any pod with hostNetwork or a permissive NetworkPolicy can reach node IMDS and steal the node managed identity. This is the well-known "pod-steals-node-credentials" class. GPA already runs as the IMDS chokepoint — the missing piece is *per-pod identity*. + +#### Design + +- Map cgroup id (already captured by eBPF audit) → Kubernetes pod via the kubelet pod-resources API or by reading `/proc//cgroup` + the CRI socket. +- Project pod ServiceAccount → SPIFFE ID; use Azure Workload Identity federation to mint a pod-scoped token instead of handing back the node MI token. +- Rule shape: `{ namespace: "app-*", serviceAccount: "billing", allow: ["imds:identity:read"] }`. + +#### Plan + +1. Ship a `--mode kubernetes` flag and a sidecar/DaemonSet manifest. +2. Integrate with Azure Workload Identity issuer; reuse OIDC trust to AKS cluster. +3. Pilot on internal clusters; publish reference NetworkPolicy that forces all IMDS traffic through GPA. + +### 5.2 Gate other cloud endpoints + +[Detailed design](Innovation-5.2-gate-more-endpoints.md) + +- Generalize destination handling so KeyVault MSI, ARM token endpoint, and Storage MI flows can be authorized through the same rule engine. +- Add pluggable *destination drivers* with: address set, request classifier (URL → typed action), signer. +- Customer-visible: one rule language to govern all cloud-credential egress. + +### 5.3 Cross-cloud port + +[Detailed design](Innovation-5.3-cross-cloud-port.md) + +- Architecture (cgroup eBPF + identity-aware proxy) is cloud-neutral. The signer and destination set are Azure-specific. +- Factor `signer` and `destinations` into traits; ship community drivers for AWS IMDSv2 and GCP metadata. +- Positioning: *a metadata firewall for any cloud*. + +## 6. Developer & Operator Experience + +| Impact | Effort | Risk | Scope | +|-----------------------------------------|------------------|---------|-------------| +| **Medium** adoption + incident response | **Small–Medium** | **Low** | **tooling** | + +### 6.1 Policy simulator / dry-run + +[Detailed design](Innovation-6.1-policy-simulator.md) + +- CLI: `gpa policy simulate --rules rules.json --request 'GET http://169.254.169.254/metadata/identity?api-version=2021-08-01' --caller pid=1234` +- Output: decision, which rule matched, canonicalization steps applied, divergence vs current production rules. +- Library-mode for unit tests so customers can lock down expected behavior. + +### 6.2 `gpa-doctor` + +[Detailed design](Innovation-6.2-gpa-doctor.md) + +- One command runs hardening checks derived from `pentest/linux/`: port exposure (A1), file modes (E1), restart safety (E4), orphan eBPF programs (G4), rule loader sanity (D4). +- Green/yellow/red report + remediation links; safe to run on production VMs. + +### 6.3 Rule authoring UX + +[Detailed design](Innovation-6.3-rule-authoring-ux.md) + +- JSON Schema for the rules file; ship in repo. +- VS Code extension providing schema-driven autocomplete, live validation, and a "rule diff" view for remote vs `local_rules.rs` overrides. +- Portal-side experience reuses the same schema. + +### 6.4 WASM rule sandbox (optional) + +[Detailed design](Innovation-6.4-wasm-rule-sandbox.md) + +- Allow customer-supplied AuthZ in WebAssembly with a tightly scoped host ABI (read normalized request, read claims, return decision; no syscalls, no clocks). +- Useful for advanced customers needing logic that doesn't fit a declarative rule. +- Guardrails: hard CPU/memory limits per invocation; disabled by default. + +## 7. Performance & Footprint + +| Impact | Effort | Risk | Scope | +|---------------------------|------------|---------|----------------| +| **Medium** latency + cost | **Medium** | **Low** | **agent only** | + +### 7.1 io_uring hot path + +[Detailed design](Innovation-7.1-io-uring-hot-path.md) + +- Behind feature flag, replace tokio-default reactor for the listener with `tokio-uring` or `monoio`. +- Targets the IMDS read path (the most frequent request shape). + +### 7.2 Zero-copy forwarding + +[Detailed design](Innovation-7.2-zero-copy-splice.md) + +- After AuthN/AuthZ pass, splice the body between accept socket and upstream socket using `splice(2)` when the request has no body transformation. +- Avoids two user-space copies on the common GET path. + +### 7.3 Crate consolidation + +[Detailed design](Innovation-7.3-crate-consolidation.md) + +- Today: `proxy_agent`, `proxy_agent_extension`, `proxy_agent_setup`, `proxy_agent_shared`. +- Move duplicated helpers (logging, config loading, version probing) into `proxy_agent_shared`. +- Build with musl for a single static binary per role; run `cargo-bloat` in CI with a budget. + +### Metrics + +- p99 added latency per IMDS request ≤ 1 ms (target). +- RSS at idle ≤ 20 MB (target). +- Binary size ≤ 8 MB stripped (target). + +## ★. Cross-Cutting Roadmap + +| Phase | Focus | Items | Exit criteria | +|---------------------------|----------------------------------------|-------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------| +| P1 — Foundations | Safe refactors, no behavior change | 2.1 canonicalizer (shadow), 2.3 ArcSwap epochs, 3.2 OTel skeleton, 7.3 crate consolidation, 4.2 CO-RE build | Zero shadow-mode divergence; CI green on all targets | +| P2 — Hardening | Close pentest-known gaps | 2.2 Cedar dual-eval, 4.1 sk_lookup + bpf_lsm, 4.4 LRU + token bucket, 3.1 hash-chained log, 3.4 Sigstore-verified setup | Pentest categories C5–C7, D1, D4–D5, F2–F3, G1, G3, H1 all PASS | +| P3 — Identity step-change | Defeat key-theft and identity spoofing | 1.1 PoP tokens (dual-emit), 1.2 vTPM/CVM sealing, 1.3 measured identity | Fabric accepts PoP; CVM SKUs default to sealed keys | +| P4 — Surface expansion | New customers, new endpoints | 1.4 capability scopes, 5.1 AKS mode, 5.2 more endpoints, 6.1 simulator, 6.2 `gpa-doctor`, 6.3 schema/VS Code | Pilot AKS customer; first non-IMDS endpoint governed by GPA | +| P5 — Reach | Ecosystem & perf polish | 5.3 cross-cloud drivers, 6.4 WASM sandbox, 7.1 io_uring, 7.2 splice | Latency & size budgets met; community-maintained AWS/GCP drivers | + +**Suggested first PR sequence** + +1. Introduce `CanonicalRequest` + property tests in shadow mode (direction 2.1). +2. Wrap policy in `ArcSwap` with per-request epoch logging (direction 2.3). +3. Add CO-RE build for the eBPF objects without behavior change (direction 4.2). +4. Add hash-chained log sink behind a feature flag (direction 3.1). +5. Begin Cedar policy compilation in dual-evaluation mode (direction 2.2). + +These five are low-risk, independently shippable, and unlock most of the later work. + +**Coordination required** + +- Direction 1.1 (PoP tokens) and 1.2 (vTPM sealing) require WireServer/IMDS fabric-side acceptance changes. +- Direction 5.1 (AKS mode) requires alignment with Azure Workload Identity team. +- Direction 4.1 needs a kernel-version matrix decision (CO-RE fallback path). + +Generated companion to the GPA repo analysis. Cross-references: `proxy_agent/src/proxy/authorization_rules.rs`, `proxy_agent/src/key_keeper/key.rs`, `proxy_agent/src/redirector/`, `pentest/linux/DESIGN.md`. diff --git a/doc/plans/Innovation-Plans-Milestones.md b/doc/plans/Innovation-Plans-Milestones.md new file mode 100644 index 00000000..2cfe919e --- /dev/null +++ b/doc/plans/Innovation-Plans-Milestones.md @@ -0,0 +1,436 @@ +## Sections + +1. [1. Overview](#overview) +2. [2. Phases & KPIs](#phases) +3. [3. 12-Quarter Roadmap](#roadmap) +4. [4. Dependency Map](#deps) +5. [5. Per-Innovation Milestones](#milestones) + - [D1 — AuthN/AuthZ](#d1-plans) + - [D2 — Rule Engine](#d2-plans) + - [D3 — Observability & Supply Chain](#d3-plans) + - [D4 — eBPF / Kernel](#d4-plans) + - [D5 — Threat Coverage](#d5-plans) + - [D6 — Dev/Operator UX](#d6-plans) + - [D7 — Performance](#d7-plans) +6. [6. RACI / Coordination](#raci) +7. [7. Risk Register](#risks) +8. [8. Program Exit Criteria](#exit) + +**GPA** · **Program Plan** · **P1 → P5** · **~12 quarters** + +# Innovation Program — Consolidated Plans & Milestones + +One operational view across all 25 innovations: phase placement, dependencies, per-track milestones (M0–M4), exit gates, owners, and risks. Each detailed design page remains the source of truth for its own scope; this page is the scheduling and coordination contract. + +**Conventions:** milestone numbering is uniform — `M0` design lock, `M1` prototype/shadow, `M2` dual-mode behind flag, `M3` default-on for target SKU, `M4` legacy removed. Week numbers (`W1..W48`) are relative to program start; quarters Q1–Q12 mirror the swimlane. + +## 1. Program Overview + +The roadmap groups the 25 innovations into five sequenced phases that respect prerequisite chains and the cross-team coupling already called out in [Innovation Directions § Roadmap](Innovation-Directions.md#roadmap). Each phase ends with a quantitative gate; later phases cannot start their *default-on* step until earlier phases reach `M3` on the same SKU class. + +| Innovations | Directions | Phases (P1–P5) | Quarters end-to-end | +|-------------|------------|----------------|---------------------| +| **25** | **7** | **5** | **≈12** | + +## 2. Phases & KPIs + +| Phase | Window | Theme | Items | Phase exit KPI | +|-------------------------------|---------|-------------------------------------|-----------------------------------|--------------------------------------------------------------------------------------------------------| +| **P1 — Foundations** | Q1–Q3 | Safe refactors, no behavior change | 2.1, 2.3, 2.4, 3.2, 4.2, 7.3 | Zero shadow-mode divergence across 7 days of prod traffic; CI green on win+linux; binary ≤ 8 MB | +| **P2 — Hardening** | Q3–Q6 | Close pentest-known gaps | 2.2, 3.1, 3.3, 3.4, 4.1, 4.3, 4.4 | Pentest categories C5–C7, D1, D4–D5, F2–F3, G1, G3, H1 → 100% PASS in CI harness | +| **P3 — Identity step-change** | Q5–Q9 | Defeat key-theft and identity spoof | 1.1, 1.2, 1.3 | Fabric accepts PoP-v2 in 100% of regions; CVM SKUs default to sealed keys; pentest B2/B3/C3/D2/E5 PASS | +| **P4 — Surface expansion** | Q7–Q11 | New customers, new endpoints | 1.4, 5.1, 5.2, 6.1, 6.2, 6.3 | ≥ 1 AKS pilot in prod; ≥ 1 non-IMDS endpoint governed; `gpa-doctor` shipped in agent package | +| **P5 — Reach** | Q10–Q12 | Ecosystem & perf polish | 5.3, 6.4, 7.1, 7.2 | p99 added latency ≤ 1 ms; RSS ≤ 20 MB idle; community drivers building in CI | + +## 3. 12-Quarter Swimlane + +| Track | Q1 | Q2 | Q3 | Q4 | Q5 | Q6 | Q7 | Q8 | Q9 | Q10 | Q11 | Q12 | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| **D1 AuthN/Z** | — | — | — | — | 1.2 M0–M1 | 1.1 M0–M1 | 1.1 M2 | 1.2 M2 | 1.3 M2 | 1.4 M1 | 1.4 M2 | _M3/4_ | +| **D2 Rules** | 2.1 M1 | 2.3 M2 | 2.4 M2 | 2.2 M1 | 2.2 M2 | 2.2 M3 | _M4 remove_ | — | — | — | — | — | +| **D3 Obs/SC** | — | 3.2 M1 | 3.2 M2 | 3.1 M1 | 3.4 M2 | 3.3 M2 | 3.1 M3 | _M4_ | — | — | — | — | +| **D4 eBPF** | — | 4.2 M1 | 4.2 M2 | 4.1 M1 | 4.4 M2 | 4.1 M2 | 4.3 M2 | 4.1 M3 | — | — | — | — | +| **D5 Threats** | — | — | — | — | — | — | 5.1 M1 | 5.2 M1 | 5.1 M2 | 5.2 M2 | 5.3 M1 | 5.3 M2 | +| **D6 UX** | — | — | 6.3 M1 | — | — | — | 6.1 M2 | 6.2 M2 | 6.3 M2 | 6.4 M1 | 6.4 M2 | — | +| **D7 Perf** | 7.3 M1 | 7.3 M2 | — | — | — | — | — | — | — | 7.1 M2 | 7.2 M2 | _tune_ | + +Legend: **P1** Foundations · **P2** Hardening · **P3** Identity · **P4** Surface · **P5** Reach. +Milestones — `M0` design lock · `M1` prototype/shadow · `M2` dual-mode behind flag · `M3` default-on · `M4` legacy removed. + +Bars use uniform milestone names: `M0` design lock · `M1` prototype/shadow · `M2` dual-mode behind flag · `M3` default-on · `M4` legacy removed. Cells without a bar are intentionally idle for that track that quarter; idle quarters are buffer for risk. + +## 4. Dependency Map + +Hard prerequisites (must reach `M2` on the upstream item before the downstream item can begin `M1`): + +- **1.1 PoP tokens** ← 2.1 CanonicalRequest (`url_hash` uses canonical form) +- **1.2 vTPM sealing** ← 3.3 Self-attestation (KID surfaced for fabric pinning) +- **1.3 Measured identity** ← 4.2 CO-RE eBPF (audit event carries hash bytes) +- **1.4 Capability scopes** ← 2.2 Cedar (scopes encoded as Cedar actions) +- **2.2 Cedar** ← 2.1 CanonicalRequest + 2.3 ArcSwap +- **2.4 Diff testing** ← 2.1 CanonicalRequest +- **3.1 Hash-chained log** ← 3.2 OTel skeleton (shared sink trait) +- **3.3 Self-attestation** ← 4.2 CO-RE (prog id + bytecode hash exposure) +- **3.4 Supply chain** ← 7.3 Crate consolidation (single artifact to sign) +- **4.1 sk_lookup + bpf_lsm** ← 4.2 CO-RE +- **4.3 IPv6 dual-stack** ← 4.1 (unified destination enum) +- **4.4 LRU + throttling** ← 4.2 CO-RE +- **5.1 AKS mode** ← 1.4 Capability scopes + 4.2 CO-RE +- **5.2 More endpoints** ← 2.2 Cedar + 1.4 scopes +- **5.3 Cross-cloud** ← 5.2 (destination driver trait) +- **6.1 Simulator** ← 2.1 + 2.2 (uses compiled policy in library mode) +- **6.2 gpa-doctor** ← 3.3 Self-attestation +- **6.3 Authoring UX** ← 2.2 (schema generated from Cedar) +- **6.4 WASM sandbox** ← 2.2 + 6.1 +- **7.1 io_uring** ← 7.3 Crate consolidation +- **7.2 splice forwarding** ← 1.1 (decision before splice means token mint stays in user-space) + +## 5. Per-Innovation Milestones + +Each entry below uses the same template: `M0 design lock` · `M1 prototype/shadow` · `M2 dual-mode behind flag` · `M3 default-on` · `M4 legacy removed`. Week numbers are relative to program start. + +### Direction 1 — AuthN/AuthZ Model + +#### 1.1 Short-lived PoP tokens — [design](Innovation-1.1-pop-tokens.md) + +##### Deliverables + +- `proxy_agent_shared/src/pop_token/` module with serde + constant-time verify +- `derive_session_key()` in `key_keeper/key.rs` +- Dual-emit behind `--feature pop_v2` in `proxy_server.rs` +- Fabric-side acceptance change (WireServer/IMDS) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-------------------------------------------------------------------|-------------------------------------------| +| M0 | W17 | RFC + threat-model signed off with fabric team | Header name + claim set frozen | +| M1 | W20 | pop_token crate + golden vectors; agent mints into local log only | Fuzz 60 min clean | +| M2 | W26 | Dual-emit in production; fabric ignores, telemetry tracks parity | ≥ 99.99% mint success; clock-skew \< 0.1% | +| M3 | W34 | Fabric flips acceptance; legacy header kept for 1 release | B2 replay pentest FAIL ⇒ blocked | +| M4 | W42 | Legacy `x-ms-azure-signature` removed | 0 legacy headers in 14-day telemetry | + +#### 1.2 vTPM / CVM sealing — [design](Innovation-1.2-vtpm-sealing.md) + +##### Deliverables + +- `key_keeper/sealing/` with backends `noop`, `tpm2`, `snp`, `tdx` +- `load_sealed()` / `store_sealed()` in `key.rs` +- Backend probe in `provision.rs`; fail-closed on unseal failure + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------------------|---------------------------------------| +| M0 | W14 | PCR set + monotonic-counter design | Sec review pass | +| M1 | W22 | tpm2 backend on a single SKU; noop fallback | Reboot survives; unseal \< 200 ms p99 | +| M2 | W30 | Pilot CVM SKUs (SEV-SNP first), feature-flagged | E5 rollback pentest blocked | +| M3 | W36 | Default-on for CVM SKUs; legacy plain-file for non-CVM | Snapshot-restore-other-VM FAIL | +| M4 | W48 | Plain-file path deleted on CVM build | 0 plain-file reads in CVM telemetry | + +#### 1.3 Measured caller identity — [design](Innovation-1.3-measured-identity.md) + +##### Deliverables + +- Augment audit map value in `linux-ebpf/ebpf_cgroup.c` with IMA / fs-verity hash +- Surface `exeHash` in `redirector::lookup_audit` +- `enforceMeasurement` matcher in `authorization_rules.rs` +- Hash-generation CLI for allow-listed binaries + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-----------------------------------------------------------------|--------------------------------| +| M0 | W26 | Choose IMA vs fs-verity per distro; pick Windows CI hash source | Spec lock | +| M1 | W30 | End-to-end hash plumbing, audit-only logging | Mismatch rate observable | +| M2 | W36 | Per-rule `enforceMeasurement` opt-in | C3 + D2 pentest FAIL ⇒ blocked | +| M3 | W44 | Default-on for fabric-shipped extension handlers | No false-positives in 14 days | + +#### 1.4 Capability-style scoped grants — [design](Innovation-1.4-capability-scopes.md) + +##### Deliverables + +- URL classifier (one canonical action/resource table per endpoint) +- Scope type carried in `CallerIdentity` +- Cedar action set generated from classifier + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-------------------------------------------------------------------|-------------------------------| +| M0 | W30 | Classifier table reviewed with IMDS + WireServer owners | Action set frozen v1 | +| M1 | W34 | Library mode + simulator integration (6.1) | All current rules round-trip | +| M2 | W40 | Scoped grants accepted in rules, dual-evaluated with path matcher | 0 divergences for 7 days | +| M3 | W46 | Path matcher disabled for scoped rules | Encoding-bypass class blocked | + +### Direction 2 — Rule Engine Modernization + +#### 2.1 CanonicalRequest — [design](Innovation-2.1-canonical-request.md) + +##### Deliverables + +- Single normalizer module used by rule loader and request path +- `proptest` suite (case, %-encoding, dots, IPv4/IPv6, Unicode confusables) +- Shadow comparison harness in `proxy_server.rs` + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|----------------------------------------------------------|----------------------------------| +| M0 | W1 | Normalizer spec frozen with fail-closed cases enumerated | Sec review pass | +| M1 | W3 | Normalizer + property tests; shadow log in prod | ≥ 99.9% match with legacy | +| M2 | W8 | Used by Cedar evaluator (2.2) | 0 ambiguous requests in 7 days | +| M3 | W14 | Legacy normalizer paths removed | CI green; one normalizer in tree | + +#### 2.2 Typed policy (Cedar) — [design](Innovation-2.2-typed-policy-cedar.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-------------------------------------------------------|-------------------------------------| +| M0 | W8 | Cedar adopted; legacy adapter spec | Policy-set schema v1 | +| M1 | W12 | Compile-on-load + dual-eval shadow | Rule load ≤ 50 ms / 1k rules | +| M2 | W18 | Enforcement gated by config flag | Divergence = 0 for 7 prod days | +| M3 | W22 | Cedar is default; legacy adapter still loads old JSON | D1 / C7 pentest 100% PASS | +| M4 | W28 | Legacy matcher code removed | No `starts_with` on path in matcher | + +#### 2.3 Versioned snapshots (ArcSwap) — [design](Innovation-2.3-versioned-snapshots.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-------------------------------------------------------|----------------------------------| +| M1 | W4 | `ArcSwap`, epoch captured per request | D5 TOCTOU pentest FAIL ⇒ blocked | +| M2 | W6 | `policy_epoch` in connection log + OTel metric | Reload latency \< 5 ms p99 | + +#### 2.4 Differential / property testing — [design](Innovation-2.4-differential-testing.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|----------------------------------------|-------------------------------------| +| M1 | W6 | Evil-twin generator + CI gate | 100 variants/rule, 0 false-mismatch | +| M2 | W9 | Plug into `local_rules.rs` reload path | Bad rule = reject + telemetry alert | + +### Direction 3 — Observability & Supply Chain + +#### 3.1 Hash-chained audit log — [design](Innovation-3.1-hash-chained-log.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------|----------------------------------| +| M0 | W10 | Sink trait + Merkle chain spec | Recovery from torn write defined | +| M1 | W14 | Chain implementation; tip anchored locally | Tamper detect 100% | +| M2 | W18 | Rekor / Monitor sentinel publisher | F2 + F3 pentest blocked | +| M3 | W26 | Default sink on all SKUs | \< 2% log throughput overhead | + +#### 3.2 OpenTelemetry export — [design](Innovation-3.2-otel-export.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-------------------------------------------|-----------------------------------| +| M1 | W5 | Metrics skeleton on UDS (Prom exposition) | No new heap on hot path | +| M2 | W9 | OTLP exporter behind `--features otel` | Trace W3C ctx propagates upstream | + +#### 3.3 Self-attestation endpoint — [design](Innovation-3.3-self-attestation.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-----------------------------------------|-------------------------| +| M0 | W14 | Payload schema, no-secret contract | Sec review pass | +| M2 | W22 | Endpoint live, consumed by `gpa-doctor` | Defender pulls in pilot | + +#### 3.4 Supply chain (SBOM + repro + Sigstore) — [design](Innovation-3.4-supply-chain.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|---------------------------------------------------------------|---------------------------------| +| M1 | W14 | CycloneDX SBOM emitted; reproducible flags set | Two-builder diff bit-identical | +| M2 | W20 | Sigstore signing; `proxy_agent_setup` verifies before install | H1 supply-chain pentest blocked | + +### Direction 4 — eBPF / Kernel Hardening + +#### 4.1 sk_lookup + bpf_lsm — [design](Innovation-4.1-sk-lookup-bpf-lsm.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|---------------------------------------|--------------------------------| +| M0 | W14 | Kernel-version matrix + fallback plan | Min kernel set frozen | +| M1 | W18 | sk_lookup probe + gated load | A/B match cgroup-connect path | +| M2 | W24 | bpf_lsm deny hook | C5 + C6 pentest FAIL ⇒ blocked | +| M3 | W30 | Default-on for supported kernels | No regression vs cgroup-only | + +#### 4.2 CO-RE unification — [design](Innovation-4.2-core-unify-ebpf.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|----------------------------------------------------|-----------------------------| +| M1 | W5 | libbpf-rs build, shared header, no behavior change | All current tests pass | +| M2 | W9 | Single BTF-portable object per platform | Loads on 3+ kernel versions | + +#### 4.3 IPv6 / dual-stack — [design](Innovation-4.3-ipv6-dual-stack.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------------------|----------------------| +| M1 | W22 | Unified `Destination` enum in user-space | v4 path unaffected | +| M2 | W26 | v6 redirect for IMDS/WireServer link-local equivalents | Fabric v6 acceptance | + +#### 4.4 LRU + throttling — [design](Innovation-4.4-ebpf-throttling-lru.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|------------------------------|--------------------| +| M1 | W18 | `BPF_MAP_TYPE_LRU_HASH` swap | G3 pentest blocked | +| M2 | W22 | Per-cgroup token bucket | G1 pentest blocked | + +### Direction 5 — Threat Coverage Expansion + +#### 5.1 AKS / container-native — [design](Innovation-5.1-aks-container-native.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|---------------------------------------------------------------|------------------------------------| +| M0 | W26 | Alignment with Azure Workload Identity team | OIDC trust path agreed | +| M1 | W34 | `--mode kubernetes` DaemonSet manifest, cgroup→pod resolution | SPIFFE ID minted per pod | +| M2 | W40 | Pilot on internal AKS cluster | Pod-steals-node-cred class blocked | + +#### 5.2 Gate more endpoints — [design](Innovation-5.2-gate-more-endpoints.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|------------------------------------------------|-----------------------------------| +| M1 | W34 | Destination-driver trait + KeyVault MSI driver | Same rule lang governs both | +| M2 | W42 | ARM token + Storage MI drivers | ≥ 1 customer enabling beyond IMDS | + +#### 5.3 Cross-cloud port — [design](Innovation-5.3-cross-cloud-port.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------------|----------------------------------| +| M1 | W42 | Signer + destination traits factored | AWS IMDSv2 driver compiles in CI | +| M2 | W46 | GCP metadata driver; positioned as cloud-neutral | External contributor PR landed | + +### Direction 6 — Developer & Operator UX + +#### 6.1 Policy simulator — [design](Innovation-6.1-policy-simulator.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|------------------------------------------|---------------------------| +| M1 | W30 | `gpa policy simulate` CLI + library mode | Reproduces prod decisions | +| M2 | W34 | Diff vs production rules; CI helper | Used by ≥ 1 customer test | + +#### 6.2 gpa-doctor — [design](Innovation-6.2-gpa-doctor.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-----------------------------------|------------------------------------------| +| M1 | W32 | Checks A1/E1/E4/G4/D4 implemented | Green-yellow-red report | +| M2 | W38 | Shipped in agent package | Safe on prod; no privileged side-effects | + +#### 6.3 Rule authoring UX — [design](Innovation-6.3-rule-authoring-ux.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|------------------------------------|--------------------------| +| M1 | W9 | JSON Schema in repo | Portal reuses schema | +| M2 | W40 | VS Code extension + rule-diff view | Marketplace listing live | + +#### 6.4 WASM rule sandbox — [design](Innovation-6.4-wasm-rule-sandbox.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|-----------------------------------------|---------------------------| +| M1 | W42 | Tightly scoped host ABI, wasmtime embed | CPU/mem caps enforced | +| M2 | W46 | Opt-in for one preview customer | No syscall escape in fuzz | + +### Direction 7 — Performance & Footprint + +#### 7.1 io_uring hot path — [design](Innovation-7.1-io-uring-hot-path.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|------------------------------------------------|----------------------------| +| M1 | W40 | `tokio-uring` behind feature flag for listener | Bench shows ≥ 20% p99 win | +| M2 | W44 | Default-on for Linux ≥ 5.15 | No regression on small VMs | + +#### 7.2 Zero-copy splice — [design](Innovation-7.2-zero-copy-splice.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------------|------------------------------| +| M1 | W42 | Splice path for body-unchanged GET | 2 fewer copies on perf trace | +| M2 | W46 | Default-on; fallback for HTTPS-terminating paths | p99 added latency ≤ 1 ms | + +#### 7.3 Crate consolidation — [design](Innovation-7.3-crate-consolidation.md) + +##### Milestones + +| Milestone | Week | Description | Exit criteria | +|-----------|------|--------------------------------------------------------|------------------------------------| +| M1 | W2 | Shared helpers moved to `proxy_agent_shared` | No duplicated logger / config code | +| M2 | W6 | musl static build per role; `cargo-bloat` budget in CI | Binary ≤ 8 MB stripped | + +## 6. RACI & Coordination + +| Item | Owner (R) | Approver (A) | Consulted (C) | Informed (I) | +|-------------------------|-------------|--------------|-----------------------|--------------------| +| 1.1 PoP tokens | GPA core | GPA TL | WireServer, IMDS | Defender, AzPolicy | +| 1.2 vTPM sealing | GPA core | GPA TL | CVM team, Host OS | Compliance | +| 1.3 Measured identity | GPA core | GPA TL | Linux IMA, Windows CI | Extension teams | +| 1.4 Capability scopes | GPA core | Security PM | IMDS, WireServer | Customers | +| 2.x Rule engine | GPA core | GPA TL | Cedar SIG | Portal | +| 3.1 Hash-chained log | GPA core | Sec review | Rekor / Sigstore | Compliance | +| 3.4 Supply chain | Build owner | Sec review | 1ES pipelines | Release mgmt | +| 4.1 sk_lookup / bpf_lsm | Kernel SIG | GPA TL | Distro vendors | Customers | +| 5.1 AKS mode | GPA core | AKS PM | Workload Identity | Customers | +| 5.3 Cross-cloud | Community | GPA TL | — | External | +| 7.x Performance | GPA core | GPA TL | — | Customers | + +## 7. Risk Register + +| \# | Risk | Phase | Severity | Mitigation | +|-----|------------------------------------------------------|-------|----------|----------------------------------------------------------------------------| +| R1 | Fabric acceptance of PoP-v2 slips a quarter | P3 | High | Dual-emit indefinitely; legacy header gated by remote feature switch | +| R2 | CO-RE breaks on a niche kernel | P1/P2 | Med | Keep classic-BPF path; runtime probe + automatic fallback | +| R3 | Cedar evaluator divergence non-zero at cutover | P2 | Med | Hold M3; auto-revert to legacy via ArcSwap epoch | +| R4 | vTPM unavailable on legacy SKUs | P3 | Med | noop backend remains supported; sealing is opt-in by SKU class | +| R5 | AKS pod-resources API instability | P4 | Low | Fall back to CRI socket; pin tested k8s versions | +| R6 | splice(2) path mis-applied to body-rewriting request | P5 | Med | Strict precondition matrix + property test; default off until matrix green | +| R7 | Sigstore outage blocks installs | P2 | Low | Cache signed bundle locally; verify-or-warn during outage window | +| R8 | WASM rule escape | P5 | High | Default-off; hard limits; opt-in single customer; differential fuzz | + +## 8. Program Exit Criteria + +**The program is "done" when all of the following hold simultaneously:** + +1. Every pentest scenario currently tracked in `pentest/linux/DESIGN.md` reports PASS in CI for two consecutive releases. +2. Fabric (WireServer + IMDS) accepts only PoP-v2 tokens; legacy signature path removed from agent. +3. Cedar is the sole authorization evaluator; legacy `starts_with` matcher deleted. +4. CO-RE eBPF objects load on all supported kernels; `sk_lookup` + `bpf_lsm` default-on for kernel ≥ 5.13. +5. Default-on hash-chained audit log + Sigstore-verified installer on all SKUs. +6. ≥ 1 AKS production customer; ≥ 1 non-IMDS endpoint governed; `gpa-doctor` shipped. +7. Performance budgets met: p99 added latency ≤ 1 ms, RSS ≤ 20 MB idle, stripped binary ≤ 8 MB. +8. Community drivers for AWS / GCP build green in CI. + +**Hard gates between phases.** P2 cannot start `M3` on any item until P1 `M3` is universal. P3 cannot start `M3` until P2 pentest exit KPI is green. P4 surface expansion is blocked from `M3` until P3 identity step-change reaches `M2` in dual-emit. + +Companion to [Innovation-Directions.md](Innovation-Directions.md) and the 25 per-innovation detailed designs. Source-of-truth for scheduling, dependencies, and exit gates across the GPA innovation program. From 834a4a44f5e2c69adcaff3360f44d2c556ff5aec Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Tue, 9 Jun 2026 13:42:25 -0700 Subject: [PATCH 2/7] Update 7.3 md files --- .../Innovation-7.3-crate-consolidation.md | 162 ++++++++++++++---- doc/plans/Innovation-Directions.md | 4 +- 2 files changed, 135 insertions(+), 31 deletions(-) diff --git a/doc/plans/Innovation-7.3-crate-consolidation.md b/doc/plans/Innovation-7.3-crate-consolidation.md index c1e82d58..5ec483f5 100644 --- a/doc/plans/Innovation-7.3-crate-consolidation.md +++ b/doc/plans/Innovation-7.3-crate-consolidation.md @@ -5,12 +5,13 @@ 3. [3. Code moves](#moves) 4. [4. musl static](#musl) 5. [5. Bloat budget](#bloat) -6. [6. Integration](#integration) -7. [7. Tests](#tests) -8. [8. Risks](#risks) -9. [9. Milestones](#milestones) +6. [6. `signing` feature gate](#signing) +7. [7. Integration](#integration) +8. [8. Tests](#tests) +9. [9. Risks](#risks) +10. [10. Milestones & status](#milestones) -**GPA** · **Direction 7.3** · **Footprint** +**GPA** · **Direction 7.3** · **Footprint** · **Status: ✅ done** # Detailed Design — Crate Consolidation, musl Static, Bloat Budget @@ -26,6 +27,13 @@ Move duplicated helpers (logging init, config loader, version probe) into `proxy |------------------------------|-----------|---------|---------------| | **Internal** maintainability | **Small** | **Low** | **workspace** | +### Status snapshot — ✅ done + +- **Done** — musl static binaries (`x86_64-unknown-linux-musl`, `aarch64-unknown-linux-musl`) are already produced by [`build-linux.sh`](../../build-linux.sh) driven from [`reusable-build.yml`](../../.github/workflows/reusable-build.yml); Windows MSVC builds go through [`build.cmd`](../../build.cmd) in the same pipeline. +- **Done** — PR 353: logger init + GPA service name moved to `proxy_agent_shared`. +- **Done** — PR 352: `cargo-bloat` budget enforced per-(target, role) on top of those builds; `signing` Cargo feature lets non-signing binaries drop OpenSSL. +- **Out of scope** — *config loader* and *OS/version probe* consolidation. See [§3](#moves) for why: only `proxy_agent` has a JSON config loader (the other two binaries don't load JSON config at all), and the OS/version probe already lives in `proxy_agent_shared`. + ### Goals - Single source of truth for cross-cutting helpers. @@ -34,57 +42,151 @@ Move duplicated helpers (logging init, config loader, version probe) into `proxy ## 2. Today -Logger init, config loader, and version probe exist in slightly different shapes across `proxy_agent`, `proxy_agent_extension`, and `proxy_agent_setup`. `proxy_agent_shared` already exists but is under-used. +Logger init used to be duplicated across `proxy_agent`, `proxy_agent_extension`, and `proxy_agent_setup` (PR 353 removed that), and the OS service-name constant had silently drifted across `proxy_agent` / `proxy_agent_extension` / `proxy_agent_setup` (PR 353 unified that too). The OS / version probe already lives in `proxy_agent_shared::{current_info, linux, windows}`, and a workspace-wide JSON config loader exists only in `proxy_agent` — the other two binaries don't read a JSON config, so there is no "third copy" to fold in. ## 3. Code Moves -| From | To | Notes | -|----------------------------|---------------------------------|---------------------------------------| -| per-crate `logger` setup | `proxy_agent_shared::logging` | One `init(role, level)` entry point | -| per-crate `config` loaders | `proxy_agent_shared::config` | Layered: file → env → defaults; serde | -| OS / version probe | `proxy_agent_shared::host_info` | One function returns full struct | -| HTTP client helpers | `proxy_agent_shared::http` | One client, with the right TLS roots | +| From | To | Status | Notes | +|----------------------------|---------------------------------------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| per-crate logger setup | `proxy_agent_shared::logger::init_loggers` | **done** (PR 353) | One helper takes `(log_folder, &[(key, file)], primary_key, max_size, max_count, level)` and constructs the `RollingLogger` map in one place. | +| GPA service / display name | `proxy_agent_shared::constants` | **done** (PR 353) | `PROXY_AGENT_SERVICE_NAME` (`GuestProxyAgent` on Windows, `azure-proxy-agent` on Linux) + `PROXY_AGENT_SERVICE_DISPLAY_NAME` live in one module. | +| OS / version probe | `proxy_agent_shared::{current_info, linux, windows}` | **already done** | Pre-existing. `proxy_agent_extension/src/handler_main.rs` already calls into `proxy_agent_shared::{linux, windows}::get_os_version`; no second copy to fold in. | +| HTTP client helpers | `proxy_agent_shared::hyper_client` | **already done** | `hyper_client` is the only HTTP client; the three binaries all call into it directly. The per-binary "wrapper" assumption from the original plan turned out not to exist. | +| ~~per-crate config loaders~~ | ~~`proxy_agent_shared::config`~~ | **dropped** (not needed) | Only `proxy_agent/src/common/config.rs` reads a JSON config file. `proxy_agent_extension` is driven by the VM-extension HandlerEnvironment + `*.settings` sequence files (a different shape, owned by the extension framework), and `proxy_agent_setup` has no runtime config at all. Nothing to consolidate. | + +### What the logger consolidation looks like + +Before (duplicated across `proxy_agent`, `proxy_agent_extension`, `proxy_agent_setup`): + + let agent_logger = RollingLogger::create_new(log_folder.clone(), "ProxyAgent.log".to_string(), MAX_SIZE, MAX_COUNT); + let connection_logger = RollingLogger::create_new(log_folder.clone(), "ProxyAgent.Connection.log".to_string(), MAX_SIZE, MAX_COUNT); + let mut loggers = HashMap::new(); + loggers.insert(AGENT_LOGGER_KEY.to_string(), agent_logger); + loggers.insert(CONNECTION_LOGGER_KEY.to_string(), connection_logger); + logger_manager::set_loggers(loggers, AGENT_LOGGER_KEY.to_string(), level); + +After (one call from each binary, including the in-tree `logger_manager` tests): + + proxy_agent_shared::logger::init_loggers( + log_folder, + &[ + (logger::AGENT_LOGGER_KEY, "ProxyAgent.log"), + (ConnectionLogger::CONNECTION_LOGGER_KEY, "ProxyAgent.Connection.log"), + ], + logger::AGENT_LOGGER_KEY, + constants::MAX_LOG_FILE_SIZE, + constants::MAX_LOG_FILE_COUNT as u16, + config::get_file_log_level(), + ); + +Net: ~60 lines of boilerplate removed across the three binaries, and the rolling-logger contract (panic if `primary_key` isn't registered) is enforced in one place. ## 4. musl Static Build -- Add CI target `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`. -- Drop dynamic libc dependency for the agent role; setup tool optionally remains glibc. +**Status: done** (pre-existing; not part of PR 352 / 353). Both Linux targets are built by [`build-linux.sh`](../../build-linux.sh) and invoked from the shared [`reusable-build.yml`](../../.github/workflows/reusable-build.yml) workflow (`build-linux-amd64` / `build-linux-arm64`); Windows MSVC builds go through [`build.cmd`](../../build.cmd) in the same file (`build-windows-amd64` / `build-windows-arm64`). The bloat workflow piggy-backs on these and only adds the regression gate on top of them. + +- CI targets: `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl` (selected by the `-Target amd64|arm64` argument in `build-linux.sh`). +- No dynamic libc dependency for the agent role; the setup tool ships from the same musl target. - Cross-distro install: same binary on RHEL 8/9, Ubuntu 22.04/24.04, Azure Linux. - eBPF object files shipped separately (loaded by libbpf at runtime), independent of libc. ## 5. Bloat Budget - # CI step +**Status: done (PR 352).** Enforced on every PR via [`.github/workflows/bloat.yml`](../../.github/workflows/bloat.yml) and [`ci/check_bloat.py`](../../ci/check_bloat.py). See [`ci/README.md`](../../ci/README.md) for the full design and override path. + +The gate has two ceilings on purpose: + +- **Absolute ceiling** (`--max-binary-bytes`) catches "total growth" no matter where it came from. +- **Per-crate share ceiling** (`--max-crate-share`, default `0.10` = 10% of `.text`) catches "one bad dependency dominates the binary" even when the total is under the absolute ceiling — the failure message names the offending crate, turning a vague size regression into a specific code-review conversation. + +First-party workspace crates (`azure-proxy-agent`, `ProxyAgentExt`, `proxy_agent_setup`, `proxy_agent_shared`) are exempt from the share gate; the absolute ceiling still bounds them. Per-(target, crate, dependency) ceiling overrides go through `--crate-share-override clap_builder=0.35` so the policy stays narrow: raising the ceiling for `clap` in `proxy_agent_setup` does not also raise it for every other binary. + +### Per-(target, role) ceilings + +A Linux musl binary and a Windows MSVC binary (with `static_vcruntime` + `windows-sys`) have very different baselines. One shared ceiling would either let Windows regress silently or false-flag every Linux PR, so the workflow runs as a matrix: + +| Target | Role binary | Max stripped size | +|------------------------------|-----------------------|-------------------| +| `x86_64-unknown-linux-musl` | `azure-proxy-agent` | 20 MB | +| `x86_64-unknown-linux-musl` | `ProxyAgentExt` | 9 MB | +| `x86_64-unknown-linux-musl` | `proxy_agent_setup` | 6 MB | +| `aarch64-unknown-linux-musl` | `azure-proxy-agent` | 20 MB | +| `aarch64-unknown-linux-musl` | `ProxyAgentExt` | 16 MB | +| `aarch64-unknown-linux-musl` | `proxy_agent_setup` | 11 MB | +| `x86_64-pc-windows-msvc` | `azure-proxy-agent` | 10 MB | +| `x86_64-pc-windows-msvc` | `ProxyAgentExt` | 5 MB | +| `x86_64-pc-windows-msvc` | `proxy_agent_setup` | 4 MB | +| `aarch64-pc-windows-msvc` | `azure-proxy-agent` | 8 MB | +| `aarch64-pc-windows-msvc` | `ProxyAgentExt` | 5 MB | +| `aarch64-pc-windows-msvc` | `proxy_agent_setup` | 4 MB | + +### Running locally + + rustup target add x86_64-unknown-linux-musl + sudo apt-get install -y musl-tools cargo install cargo-bloat --locked - cargo bloat --release --crates --message-format json > bloat.json - python3 ci/check_bloat.py --max-binary-bytes 20000000 --max-crate-share 0.10 -- Hard ceiling on total binary size (e.g. 20 MB stripped). -- No single non-first-party crate may exceed 10% of total text. -- Drift detector: PR comment shows top contributors and delta from main. + cargo bloat --release --crates \ + --target x86_64-unknown-linux-musl \ + -p azure-proxy-agent \ + --message-format json > bloat.json + + python3 ci/check_bloat.py \ + --bloat-json bloat.json \ + --max-binary-bytes 20000000 \ + --max-crate-share 0.10 + +Exit `0` = within budget, `1` = ceiling tripped (prints top contributors and which ceiling), `2` = bad input. + +### Override path + +Bloat regressions are intentional sometimes (new feature, security-driven dep upgrade). Procedure: + +1. Run the commands above locally and copy the report into the PR. +2. Bump the relevant `max_binary_bytes` entry (or add a `crate_share_overrides` line) in `.github/workflows/bloat.yml` **and** update the table in `ci/README.md` in the same PR. Only the regressing row(s). +3. Get two reviewer approvals specifically acknowledging the budget change (`LGTM-bloat` review tag convention). +4. After merge, the new ceiling becomes the baseline. + +Unauthorized bypasses (`--no-verify`, removing the workflow) are not permitted. + +## 6. `signing` feature gate + +**Status: done (PR 352).** `proxy_agent_shared` now exposes an opt-in `signing` Cargo feature: + + # proxy_agent_shared/Cargo.toml + [features] + default = [] + # Enables compute_signature (HMAC-SHA256 via OpenSSL on Linux). + # Binaries that don't sign anything (e.g. proxy_agent_setup) should leave this off. + signing = ["dep:openssl"] + +`openssl` is now an *optional* dep on both musl and gnu Linux targets and is `#[cfg(feature = "signing")]`-gated everywhere it is touched (`linux.rs`, `hyper_client.rs`, `misc_helpers.rs`, `error.rs`). Only `proxy_agent` opts in (`proxy_agent_shared = { path = "...", features = ["signing"] }`); `proxy_agent_setup` and `ProxyAgentExt` drop vendored OpenSSL entirely, which is a multi-MB win on musl (and lets the bloat ceilings for those two binaries land where they did). -## 6. Integration +## 7. Integration - SBOM (3.4) generated from the static binary's compiled crate graph. - Attestation endpoint (3.3) reports binary hash and bloat report URL. - Pkg builds (deb/rpm) pick up the static binary unchanged. -## 7. Tests +## 8. Tests - Smoke test on each supported distro using the static binary. - Bloat budget test runs on every PR; documented override path with mandatory two-reviewer approval. +- The in-tree `logger_manager` unit tests now exercise `init_loggers` directly, so the consolidated helper is covered by the existing test suite. -## 8. Risks +## 9. Risks - **musl perf** for DNS / network can differ. Mitigation: micro-benchmark before/after. -- **Bloat budget** false alarms on benign upgrades. Mitigation: weekly main-branch baseline refresh. +- **Bloat budget** false alarms on benign upgrades. Mitigation: per-(target, crate) overrides in `bloat.yml` + weekly main-branch baseline refresh. +- **`signing` feature drift**: adding a new caller of `compute_signature` from a binary that doesn't opt into `signing` is a compile-time error rather than a runtime surprise — the `#[cfg(feature = "signing")]` gates make the contract explicit. -## 9. Milestones +## 10. Milestones & status -| M | Deliverable | Exit | -|-----|--------------------------------------|--------------------------------| -| M1 | Helper moves to `proxy_agent_shared` | No duplicate code; tests green | -| M2 | musl static binary in CI | Cross-distro smoke green | -| M3 | Bloat budget enforced | PRs blocked over ceiling | +| M | Deliverable | Status | Exit | +|-----|--------------------------------------|-------------------------------------------------|-------------------------------------------------------------------| +| M1 | Helper moves to `proxy_agent_shared` | **done** (PR 353 + pre-existing) | Logger setup + GPA service name unified in PR 353; OS/version probe and HTTP client already lived in `proxy_agent_shared`. Config loader intentionally not moved — only one binary has one. | +| M2 | musl static binary in CI | **done** (pre-existing) | Built by `build-linux.sh` via `reusable-build.yml` (amd64 + arm64) | +| M3 | Bloat budget enforced | **done** (PR 352) | Per-(target, role) ceilings active; PRs blocked over ceiling | +| M4 | OpenSSL gated off non-signing roles | **done** (PR 352) | `proxy_agent_shared` `signing` feature; setup + ext build without | Detail design for direction 7.3. Parent: [Innovation-Directions.md](Innovation-Directions.md). diff --git a/doc/plans/Innovation-Directions.md b/doc/plans/Innovation-Directions.md index 88df6264..871dfc44 100644 --- a/doc/plans/Innovation-Directions.md +++ b/doc/plans/Innovation-Directions.md @@ -407,7 +407,7 @@ On AKS nodes, any pod with hostNetwork or a permissive NetworkPolicy can reach n - After AuthN/AuthZ pass, splice the body between accept socket and upstream socket using `splice(2)` when the request has no body transformation. - Avoids two user-space copies on the common GET path. -### 7.3 Crate consolidation +### 7.3 Crate consolidation — ✅ done [Detailed design](Innovation-7.3-crate-consolidation.md) @@ -415,6 +415,8 @@ On AKS nodes, any pod with hostNetwork or a permissive NetworkPolicy can reach n - Move duplicated helpers (logging, config loading, version probing) into `proxy_agent_shared`. - Build with musl for a single static binary per role; run `cargo-bloat` in CI with a budget. +**Status: done.** Logger setup and the GPA service-name constants have moved into `proxy_agent_shared` (PR 353), removing ~60 lines of boilerplate from the three binaries. The OS/version probe and the HTTP client already live in `proxy_agent_shared`; the JSON config loader is intentionally not moved — only `proxy_agent` reads a JSON config (the extension is driven by HandlerEnvironment / sequence files, and the setup tool has no runtime config), so there is no second copy to consolidate. The musl static-binary builds for both `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl` are already produced by `build-linux.sh` from the shared `reusable-build.yml` workflow (Windows MSVC builds go through `build.cmd` in the same file). On top of those, the `cargo-bloat` regression gate is now live in CI as a per-(target, role) matrix with absolute + per-crate-share ceilings (PR 352, see [`ci/README.md`](../../ci/README.md)). A new opt-in `signing` Cargo feature on `proxy_agent_shared` lets `proxy_agent_setup` and `ProxyAgentExt` drop the vendored OpenSSL dep entirely. + ### Metrics - p99 added latency per IMDS request ≤ 1 ms (target). From b30401d8be827f9d8618d92128d676c4c1b00ecf Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Tue, 9 Jun 2026 14:06:08 -0700 Subject: [PATCH 3/7] fix spelling --- .github/actions/spelling/expect.txt | 107 +++++++++++++++++++++++++++- proxy_agent/src/provision.rs | 2 +- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 929d8ec1..5fbce87a 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -1,16 +1,25 @@ aab AAFFBB +AAppendix aarch +abcd abcdefghijkmnopqrstuvwxyz ABCDEFGHJKLMNPQRSTUVWXYZ abe +addrlen addrpair advapi +AFidentity +aks almalinux ATL ATLMFC +aton +Authenticode autobuild +autonumber autocrlf +AWI aya AZUREPUBLICCLOUD azuretools @@ -19,6 +28,7 @@ backcompat backdoored bak bierner +bindgen binpath binskim bitflag @@ -26,6 +36,7 @@ boofuzz bpf bpftool btf +btreemap btrfs bufptr Bufs @@ -33,11 +44,15 @@ BUILDIN buildroot buildshell byos +bytecodealliance cacheline callouts +canonicalizer cbl ccbdee ccbf +CEL +CES cgroups cgroupv cgtop @@ -54,12 +69,15 @@ collectguestlogs commandline COMPUTERNAME comspec +confusables consoleloggerparameters +containerd coredumpctl covrec CPlat cplusplus cpptools +cri crpteste CRYPTOAPI CSPRNG @@ -68,6 +86,8 @@ customout customoutput cvd CVEs +CVM +cyclonedx czf DABC DACL @@ -104,6 +124,7 @@ dotnet doxygen DPAPI dport +DSL dtolnay dumps'd Dvm @@ -116,9 +137,12 @@ EBPFCORE eef egor ele +EMBQ ent +entra entriesread esac +esapi EStorage etest etestoutputs @@ -138,6 +162,7 @@ EXTCONFIG extconfig exthandlers fafbfc +failmode Fapi Fbar fde @@ -150,6 +175,7 @@ FFFF FFFFFFFF fffi ffi +fidentity FIXEDFILEINFO Fmanagement FOF @@ -164,12 +190,14 @@ fuzzer fwlink Fzpeng gaplugin +gcp getifaddrs goalstate gpa gpalinuxdev gpapen gpawindev +GSP guestproxyagentmsis guiddef handleapi @@ -180,18 +208,24 @@ HMACs homoglyph hostga hostingenvironmentconfig +HTAB httpwg httpx Iaa +iat ICredentials idstepsrun IEnumerable ieq +ietf iex ifaddrs ifindex IList +ikm +ima imds +IMDSv imm immediateruncommandservice intellectualproperty @@ -199,19 +233,29 @@ Intelli intellij INVALIDMETHOD INVM -Ioctl +ioctl +iss iusr jetbrains +Jhb jqlang JOBOBJECT jobsjob journalctl joutvhu JScript +jsessionid +jti +JWKS +jws +KEK keyonly +keyvault +kfunc kinvolk kmemleak kotlin +KPI kprobe ktime kusto @@ -221,12 +265,14 @@ libbpf libbpfcc libloading linkid +llc llvmorg logdir Loggerhas logon Lrs lsa +lsm ltsc luid macikgo @@ -235,20 +281,28 @@ mcr MEMORYSTATUSEX metabuild MFC +mgmt microsoftcblmariner microsoftlosangeles microsoftwindowsdesktop misconfig +mlock +mmap'd MMdd mmm mnt +monoio monomorphization +MRTD msasn msc msp +MSR msrc multilib +nbf ncurl +NDJSON netapi netcoreapp netebpfext @@ -256,11 +310,13 @@ nethook netns netsh Newtonsoft +nfc nftables NICs nifs nmake nmap +NNN nocapture NOCONFIRMATION nodet @@ -276,16 +332,24 @@ nprintf NSG nsudo ntdll +ntohs NTSTATUS ntimeout OICI +OIDC onscreen onebranch openprocess +opentelemetry oneshot opencode opensource +optin osinfo +otel +OTLP +OWASP +pahole PAI parseable passwordless @@ -294,6 +358,8 @@ pcap PCAP pcapng pcaps +pcr +PDB PEERCRED pentest PENTEST @@ -302,6 +368,7 @@ pgpkey pgrep pidof PIDs +PII pipefail pkgversion pktmon @@ -322,6 +389,7 @@ printk PROCESSINFOCLASS procdump processthreadsapi +proptest proxyagent proxyagentextensionvalidation proxyagentvalidation @@ -329,21 +397,27 @@ pscustomobject ptrace pwstr radamsa +raci Razr rcv RDFE +RECVORIGDSTADDR redhat Redist refcnt registereventsourcew registrykey +reimplementation +rekor relativeurl +Reprovisioning resf reuseport rgr rgs rhel RINGBUF +roadmap Roboto rockylinux rolename @@ -351,23 +425,27 @@ rootdir rpmbuild RPMS rstr +RTMR rul ruleset runas runthis -Runtimes +runtimes +RUSTFLAGS rustfmt rustup saddr sandboxing sas +sbom scapy schtasks scm SDDL secauthz Segoe -serice +selftest +serviceaccount SETFCAP SETPCAP sev @@ -376,19 +454,27 @@ shellcheck sids SIEM sigid +Sigstore SIO +SIV skc +sklookup sku sles sln +SLO smp +SNAT +snp sourced spellright +SPIFFE splitn SRPMS SSRF SSZ stackoverflow +starttime stdbool stdint stdoutput @@ -402,14 +488,17 @@ Swatinem SWbem SYD SYG +syscalls sysinfoapi sysinit SYSLIB SYSTEMDRIVE taiki TASKKILL +tcb tcpdump TCPDUMP +tdx telemetrydata tensin testcasesetting @@ -431,13 +520,16 @@ TOCTOU tokio topdir totalentries +tpmrm transitioning +tripable trustlevel trustyuser tshark tsv Tsv TSV +UAMI UBR UBRSTRING udev @@ -449,8 +541,14 @@ unparseable Unregistering unregisters unspec +updateable +uppercased +uring +userinfo uzers valu +VCEK +VLEK VCpus vcruntime vendored @@ -472,7 +570,9 @@ waagent waappagent walinuxagent wasecagentprov +wasmtime Wbem +WDAC wdk wdksetup Werror @@ -516,6 +616,7 @@ xxxx XXXXXX xxxxxxxx xxxxxxxxxxx +Zeroizing zipsas zureuser zypper \ No newline at end of file diff --git a/proxy_agent/src/provision.rs b/proxy_agent/src/provision.rs index 827af8f3..f3a600c7 100644 --- a/proxy_agent/src/provision.rs +++ b/proxy_agent/src/provision.rs @@ -557,7 +557,7 @@ async fn get_provision_failed_state_message( } /// Get provision state -/// It returns the current GPA serice provision state (from shared_state) for GPA service +/// It returns the current GPA service provision state (from shared_state) for GPA service /// This function is designed and invoked in GPA service pub async fn get_provision_state_internal( provision_shared_state: ProvisionSharedState, From dfe16cef0acbcb8e8f30cbe6e4753da96fadff54 Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Tue, 9 Jun 2026 14:13:36 -0700 Subject: [PATCH 4/7] fix --- .github/actions/spelling/expect.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 5fbce87a..a6e0b49c 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -255,7 +255,7 @@ kfunc kinvolk kmemleak kotlin -KPI +kpi kprobe ktime kusto @@ -297,7 +297,7 @@ MRTD msasn msc msp -MSR +msr msrc multilib nbf @@ -359,7 +359,7 @@ PCAP pcapng pcaps pcr -PDB +pdb PEERCRED pentest PENTEST @@ -462,7 +462,7 @@ sklookup sku sles sln -SLO +slo smp SNAT snp From ae0e6f7bbaa865d03ae43f76fe9244ea4ccc91d4 Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Tue, 9 Jun 2026 14:26:40 -0700 Subject: [PATCH 5/7] update spelling --- .github/actions/spelling/expect.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index a6e0b49c..8619463d 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -299,6 +299,7 @@ msc msp msr msrc +MSRs multilib nbf ncurl @@ -360,6 +361,7 @@ pcapng pcaps pcr pdb +PDBs PEERCRED pentest PENTEST From 25da91c83187c166d8c4cce26469809d95bfe335 Mon Sep 17 00:00:00 2001 From: "Zhidong Peng (HE/HIM)" Date: Wed, 10 Jun 2026 11:10:30 -0700 Subject: [PATCH 6/7] Update spelling --- .github/actions/spelling/expect.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 8619463d..a8622c20 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -105,7 +105,6 @@ defattr dentity deploymentid desync -desyncs detbox detlbl dettitle @@ -158,7 +157,6 @@ exampleosdiskname examplevmname exepath exfil -EXTCONFIG extconfig exthandlers fafbfc @@ -171,7 +169,6 @@ ffcecb ffebe fff ffff -FFFF FFFFFFFF fffi ffi @@ -299,7 +296,6 @@ msc msp msr msrc -MSRs multilib nbf ncurl @@ -356,12 +352,9 @@ parseable passwordless peekable pcap -PCAP pcapng -pcaps pcr pdb -PDBs PEERCRED pentest PENTEST @@ -621,4 +614,4 @@ xxxxxxxxxxx Zeroizing zipsas zureuser -zypper \ No newline at end of file +zypper From ef5fd342ce778e01fd8e12e5a8b818f0a1e9aa70 Mon Sep 17 00:00:00 2001 From: Zhidong Peng Date: Wed, 10 Jun 2026 18:18:13 +0000 Subject: [PATCH 7/7] Update check-spelling metadata --- .github/actions/spelling/excludes.txt | 4 +- .github/actions/spelling/expect.txt | 97 +++++++-------------------- 2 files changed, 26 insertions(+), 75 deletions(-) diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt index 3fcc16a7..aa4efc67 100644 --- a/.github/actions/spelling/excludes.txt +++ b/.github/actions/spelling/excludes.txt @@ -1,6 +1,8 @@ .github\actions\spelling\expect.txt +^\Q.github/actions/spelling/expect.txt\E$ +^pkg_debian/rules$ Cargo.lock doc/GPA Arch Diagram.vsdx doc/GuestProxyAgent.png e2etest/GuestProxyAgentTest/Resources/GuestProxyAgentLoadedModulesBaseline.txt -proxy_agent_shared/src/secrets_redactor.rs \ No newline at end of file +proxy_agent_shared/src/secrets_redactor.rs diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index a8622c20..113389e7 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -17,19 +17,17 @@ ATLMFC aton Authenticode autobuild -autonumber autocrlf +autonumber AWI aya AZUREPUBLICCLOUD azuretools -azureuser backcompat backdoored bak bierner bindgen -binpath binskim bitflag boofuzz @@ -45,7 +43,6 @@ buildroot buildshell byos bytecodealliance -cacheline callouts canonicalizer cbl @@ -59,29 +56,21 @@ cgtop chokepoint cicd Cim -cimv cla closehandle -cmds codeofconduct codeql -collectguestlogs commandline COMPUTERNAME -comspec confusables consoleloggerparameters containerd coredumpctl -covrec CPlat -cplusplus cpptools cri crpteste -CRYPTOAPI CSPRNG -csum customout customoutput cvd @@ -116,11 +105,8 @@ distros dllmain dministrator dnf -dockerenv -dodce dodce dotnet -doxygen DPAPI dport DSL @@ -132,7 +118,6 @@ eaeef EAF ebpf ebpfapi -EBPFCORE eef egor ele @@ -158,7 +143,6 @@ examplevmname exepath exfil extconfig -exthandlers fafbfc failmode Fapi @@ -175,10 +159,8 @@ ffi fidentity FIXEDFILEINFO Fmanagement -FOF Fresource FSETID -FSO fsprogs fstorage fstype @@ -186,9 +168,7 @@ ftoken fuzzer fwlink Fzpeng -gaplugin gcp -getifaddrs goalstate gpa gpalinuxdev @@ -216,15 +196,12 @@ IEnumerable ieq ietf iex -ifaddrs -ifindex -IList ikm +IList ima imds IMDSv imm -immediateruncommandservice intellectualproperty Intelli intellij @@ -235,12 +212,11 @@ iss iusr jetbrains Jhb -jqlang JOBOBJECT jobsjob journalctl joutvhu -JScript +jqlang jsessionid jti JWKS @@ -265,7 +241,6 @@ linkid llc llvmorg logdir -Loggerhas logon Lrs lsa @@ -279,7 +254,6 @@ MEMORYSTATUSEX metabuild MFC mgmt -microsoftcblmariner microsoftlosangeles microsoftwindowsdesktop misconfig @@ -293,17 +267,14 @@ monomorphization MRTD msasn msc -msp -msr msrc +MSRs multilib nbf ncurl NDJSON netapi netcoreapp -netebpfext -nethook netns netsh Newtonsoft @@ -315,32 +286,30 @@ nmake nmap NNN nocapture -NOCONFIRMATION nodet -NOERRORUI NONINFRINGEMENT +norestart nosuchuser notcontains notjson notlike -norestart npidof nprintf NSG nsudo ntdll +ntimeout ntohs NTSTATUS -ntimeout OICI OIDC -onscreen onebranch -openprocess -opentelemetry oneshot +onscreen opencode +openprocess opensource +opentelemetry optin osinfo otel @@ -348,16 +317,15 @@ OTLP OWASP pahole PAI -parseable passwordless -peekable pcap pcapng pcr -pdb +PDBs +peekable PEERCRED -pentest PENTEST +pentest PERCPU pgpkey pgrep @@ -372,17 +340,16 @@ portaddr portpair postinst pprev -prandom prctl predef -prefixer Prefixer +prefixer prefmaxlen preprovisioned Prereqs printk -PROCESSINFOCLASS procdump +PROCESSINFOCLASS processthreadsapi proptest proxyagent @@ -391,8 +358,8 @@ proxyagentvalidation pscustomobject ptrace pwstr -radamsa raci +radamsa Razr rcv RDFE @@ -434,7 +401,6 @@ sandboxing sas sbom scapy -schtasks scm SDDL secauthz @@ -458,7 +424,6 @@ sku sles sln slo -smp SNAT snp sourced @@ -480,19 +445,17 @@ Substatuses SUIDSGID suse Swatinem -SWbem SYD SYG syscalls sysinfoapi -sysinit SYSLIB SYSTEMDRIVE taiki TASKKILL tcb -tcpdump TCPDUMP +tcpdump tdx telemetrydata tensin @@ -504,7 +467,6 @@ THH thiserror Thu timedout -timeup tlsv tmpfs tnc @@ -521,21 +483,16 @@ tripable trustlevel trustyuser tshark -tsv -Tsv TSV +Tsv +tsv UAMI UBR UBRSTRING udev -uers -uninstalls unistd unmark unparseable -Unregistering -unregisters -unspec updateable uppercased uring @@ -543,12 +500,12 @@ userinfo uzers valu VCEK -VLEK VCpus vcruntime vendored vflji vhd +VLEK vmagentlog VMGA vmhwm @@ -562,24 +519,19 @@ vns VTeam vtpm waagent -waappagent walinuxagent -wasecagentprov wasmtime -Wbem WDAC wdk wdksetup Werror westus wevtapi -wfp WFP +wfp winapi winbase -windowsazureguestagent winget -winmgmts winnt winres wireserver @@ -587,14 +539,11 @@ wireserverand wireserverandimds WMI workarounds -WORKINGSET -workdir WORKDIR +workdir +WORKINGSET wrk wrongvalue -WScript -wsf -Wsh WSL wstr wsum @@ -602,9 +551,9 @@ wyy xamarin xbb xbf -xef xcopy XDP +xef xfsprogs xsi xxxx