From afa1149fa669bbac240f926513c75c8a61a57c9c Mon Sep 17 00:00:00 2001 From: 0thernet Date: Thu, 17 Sep 2026 14:41:57 -0400 Subject: [PATCH 1/5] Add bounded ZIP64 archive extraction crate (oh-archive) Add rust/oh-archive with WASM and N-API bindings. The crate extracts selected entries from untrusted ZIP archives with regex patterns while enforcing entry count, per-entry, and total-byte limits. It rejects encrypted files, path-traversal names, and unsupported compression methods. Also register the new artifact surfaces in costs.json, split the Rust build scripts by crate, and update the rust workspace/README/plan. --- costs.json | 16 ++ package.json | 7 +- plans/rust-foundations.md | 268 ++++++++++++++++++++++++++++++++ rust/Cargo.lock | 259 ++++++++++++++++++++++++++++++ rust/Cargo.toml | 2 +- rust/README.md | 3 + rust/oh-archive-napi/Cargo.toml | 19 +++ rust/oh-archive-napi/LICENSE | 28 ++++ rust/oh-archive-napi/src/lib.rs | 19 +++ rust/oh-archive-wasm/Cargo.toml | 21 +++ rust/oh-archive-wasm/LICENSE | 28 ++++ rust/oh-archive-wasm/src/lib.rs | 19 +++ rust/oh-archive/Cargo.toml | 22 +++ rust/oh-archive/LICENSE | 28 ++++ rust/oh-archive/src/lib.rs | 266 +++++++++++++++++++++++++++++++ 15 files changed, 1001 insertions(+), 4 deletions(-) create mode 100644 plans/rust-foundations.md create mode 100644 rust/oh-archive-napi/Cargo.toml create mode 100644 rust/oh-archive-napi/LICENSE create mode 100644 rust/oh-archive-napi/src/lib.rs create mode 100644 rust/oh-archive-wasm/Cargo.toml create mode 100644 rust/oh-archive-wasm/LICENSE create mode 100644 rust/oh-archive-wasm/src/lib.rs create mode 100644 rust/oh-archive/Cargo.toml create mode 100644 rust/oh-archive/LICENSE create mode 100644 rust/oh-archive/src/lib.rs diff --git a/costs.json b/costs.json index ed26c80..3dc336e 100644 --- a/costs.json +++ b/costs.json @@ -137,6 +137,22 @@ "budget": { "bytesPerArtifact": 2097152 } + }, + "rust:archive-wasm": { + "kind": "served", + "retention": "persistent", + "owner": "rust/oh-archive/Cargo.toml", + "budget": { + "bytesPerArtifact": 2097152 + } + }, + "rust:archive-napi": { + "kind": "served", + "retention": "persistent", + "owner": "rust/oh-archive/Cargo.toml", + "budget": { + "bytesPerArtifact": 4194304 + } } }, "exempt": [] diff --git a/package.json b/package.json index 36fd50d..c43f568 100644 --- a/package.json +++ b/package.json @@ -140,9 +140,10 @@ "check:effect": "bun run ./scripts/check-effect-architecture.ts && bun test ./scripts/effect-architecture.test.ts", "bench:lab:profile": "bun scripts/benchmarks/lab-reader-profile-run.ts", "build:research": "bun run ./scripts/build-research.ts", - "rust:check": "cargo check --locked && cargo clippy -- -D warnings && cargo test --locked", - "rust:build:wasm": "cd rust/oh-canonical-wasm && wasm-pack build --target web --out-dir pkg", - "rust:build": "bun run rust:build:wasm" + "rust:check": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cargo check --locked && cargo clippy -- -D warnings && cargo test --locked", + "rust:build:canonical:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-canonical-wasm && wasm-pack build --target web --out-dir pkg", + "rust:build:archive:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-archive-wasm && wasm-pack build --target web --out-dir pkg", + "rust:build": "bun run rust:build:canonical:wasm && bun run rust:build:archive:wasm" }, "devDependencies": { "@suss/datalog": "0.20.0", diff --git a/plans/rust-foundations.md b/plans/rust-foundations.md new file mode 100644 index 0000000..0a77010 --- /dev/null +++ b/plans/rust-foundations.md @@ -0,0 +1,268 @@ +# Hraness Rust Foundations + +## Overview + +This plan implements the cross-project Rust foundation roadmap for the Hraness +org: canonical JSON/digest primitives, untrusted archive parsing, Datalog/graph +reduction, and custody consolidation. Work starts in `hraness/oh` because its +`no-required-runtime-dependencies` policy forces a clean WASM-first design that +can be reused by Textbutler, Sponge, and Wordcell. + +The TypeScript reference implementations stay authoritative until each Rust +replacement proves byte-exact parity through property tests. + +## Constraints + +- `oh` has **no required runtime dependencies**. Rust artifacts must be optional + or bundled; WASM is the default integration, N-API is opt-in. +- Convex (Sponge) supports WASM but not `.node` add-ons. +- Every new data surface must register in `costs.json` and pass + `bun run check:cost-surfaces`. +- Do not change existing `dist/` wire schemas or published npm surfaces. New Rust + must be an opt-in sidecar or a versioned engine identity. +- Hraness packages upgrade independently via immutable release tags or full + commit pins; do not coordinate main branches. +- Each phase lands as a focused commit (or stacked PR) with green repository + gates. + +## Phases + +| Phase | Name | Depends on | Parallelizable with | +|---|---|---|---| +| 1 | Canonical JSON core crate in `oh` | none | — | +| 2 | WASM + N-API bindings for canonical crate | Phase 1 | — | +| 3 | TS parity tests and opt-in WASM loader | Phase 2 | — | +| 4 | CI, costs.json, and packaging for canonical crate | Phase 3 | — | +| 5 | Shared bounded ZIP64 archive crate | Phase 4 | — | +| 6 | Textbutler X-archive sidecar integration | Phase 5 | — | +| 7 | Shared SQLite snapshot / iMessage / Contacts reader | Phase 6 | — | +| 8 | Sponge canonical/digest migration | Phase 4 + oh release | — | +| 9 | Wordcell clip/bundle reader migration | Phase 5, 7 | — | +| 10 | Positive-Datalog projection / graph reducer crate | Phase 4 | 5, 7 | +| 11 | Custody/desktop-foundation consolidation | Phase 4 | 10 | + +## Phase 1: Canonical JSON core crate in `oh` + +- **Status:** Done +- **Depends on:** none +- **Objective:** A Rust crate `oh-canonical` that reproduces `src/canonical.ts`. +- **Scope:** `rust/oh-canonical/` +- **Out of scope:** bindings, packaging, downstream consumers. +- **Approach:** + - Sort object keys by UTF-16 code unit. + - Escape strings to match `JSON.stringify` (including U+2028/U+2029). + - Reject unpaired surrogates, `-0`, non-finite numbers, and non-plain objects. + - Implement ECMAScript-compatible number formatting (adapted from + `parse-rust-core`) for byte-exact parity with `JSON.stringify`. +- **Acceptance criteria:** + - `cargo test -p oh-canonical` passes. + - Rust output matches `canonicalJson` on the existing hand-written cases. +- **Validation:** `cargo test --locked` + +## Phase 2: WASM + N-API bindings for canonical crate + +- **Status:** Done +- **Depends on:** Phase 1 +- **Objective:** Build `oh-canonical-wasm` and `oh-canonical-napi` from the same + core crate. +- **Scope:** `rust/oh-canonical-wasm/`, `rust/oh-canonical-napi/` +- **Out of scope:** TS loader, distribution packaging. +- **Approach:** + - `wasm-pack --target web` for portable use. + - `napi-rs` cdylib for Bun/Node opt-in use. + - Disable `wasm-opt` if its default flags break bulk-memory operations. +- **Acceptance criteria:** + - `cargo check --workspace` succeeds. + - `bun run rust:build:wasm` produces a loadable `pkg/` directory. +- **Validation:** `cargo check --locked && bun run rust:build:wasm` + +## Phase 3: TS parity tests and opt-in WASM loader + +- **Status:** Done +- **Depends on:** Phase 2 +- **Objective:** Prove Rust output equals TS reference and expose an opt-in + `loadCanonicalRustTextEngine()`. +- **Scope:** `src/canonical-rust.ts`, `src/canonical-rust-parity.test.ts` +- **Out of scope:** Replacing existing `canonicalJson` callers. +- **Approach:** + - Dynamically import the WASM artifact from outside `src/` using a computed + URL so TypeScript does not resolve it at compile time. + - Fall back to the TS reference if the WASM module is unavailable. + - Use `fast-check` to compare canonical JSON and SHA-256 on generated + inputs. +- **Acceptance criteria:** + - `bun test src/canonical-rust-parity.test.ts` passes. + - `bun run typecheck` passes. +- **Validation:** `bun test src/canonical-rust-parity.test.ts && bun run typecheck` + +## Phase 4: CI, costs.json, and packaging for canonical crate + +- **Status:** Done +- **Depends on:** Phase 3 +- **Objective:** Land the canonical foundation with CI, cost-surface registry, + and git hygiene. +- **Scope:** `.github/workflows/ci.yml`, `costs.json`, `.gitignore`, + `package.json`, `rust/README.md`, `rust/Cargo.lock` +- **Out of scope:** Publishing the npm package or cutting a release. +- **Approach:** + - Add `rust:check`, `rust:build:wasm`, and `rust:build` scripts. + - Add a Rust CI job installing Rust, `wasm-pack`, Bun, and running + `cargo check/clippy/test` and WASM build. + - Register `rust:canonical-wasm` and `rust:canonical-napi` in `costs.json`. + - Gitignore `rust/target/`, `rust/**/pkg/`, and `rust/**/*.node`. +- **Acceptance criteria:** + - `bun run check:cost-surfaces` passes. + - `cargo clippy -- -D warnings` passes. + - PR opened and pushed. +- **Validation:** `bun run check:cost-surfaces && cargo clippy -- -D warnings && cargo test --locked` + +## Phase 5: Shared bounded ZIP64 archive crate + +- **Status:** In progress +- **Depends on:** Phase 4 +- **Objective:** A reusable Rust crate `oh-archive` that safely extracts selected + entries from untrusted ZIP archives. +- **Scope:** `rust/oh-archive/`, `rust/oh-archive-wasm/`, `rust/oh-archive-napi/`, + update `rust/Cargo.toml` workspace members. +- **Out of scope:** SQLite parsing, HTML/Markdown extraction. +- **Approach:** + - Use the `zip` crate with only `deflate` support enabled. + - Accept regex patterns; only matching entries are extracted. + - Enforce `max_entries`, `max_total_bytes`, `max_entry_bytes`. + - Reject encrypted entries, path-traversal names, and unsupported + compression methods. + - Write extracted files atomically into a caller-supplied output directory. + - Provide WASM and N-API bindings, plus a JSON CLI mode for sidecar use. +- **Acceptance criteria:** + - `cargo test -p oh-archive` passes. + - Property-based tests with generated ZIP files verify bounds and path + sanitization. + - `bun run rust:build:wasm` produces a loadable archive WASM artifact. +- **Validation:** `cargo test -p oh-archive && cargo clippy -- -D warnings && bun run rust:build:wasm` + +## Phase 6: Textbutler X-archive sidecar integration + +- **Status:** Not started +- **Depends on:** Phase 5 +- **Objective:** Replace the in-process JS ZIP walk in Textbutler's legacy + `src/x-archive-zip.ts` with the Rust sidecar for memory isolation. +- **Scope:** `hraness/textbutler` repository: add Rust sidecar invocation, + preserve the existing `ExtractedXArchiveMember` contract. +- **Out of scope:** Changing iMessage/Contacts parsing, renaming the npm + package. +- **Approach:** + - Build `oh-archive` as a sidecar binary or use the N-API module. + - Invoke it from `src/x-archive-zip.ts` with the same regex selection and + byte limits. + - Compare output byte-for-byte on sample archives before enabling by default. + - Gate behind an opt-in flag until parity is proven in CI. +- **Acceptance criteria:** + - Existing X-archive tests pass with the Rust sidecar. + - No regression in supported archive features. + - `costs.json` updated for the new sidecar artifact. +- **Validation:** `bun test` in Textbutler. + +## Phase 7: Shared SQLite snapshot / iMessage / Contacts reader + +- **Status:** Not started +- **Depends on:** Phase 6 +- **Objective:** Move the filesystem-snapshot and read-only query isolation for + iMessage and Contacts into Rust. +- **Scope:** New crate `oh-sqlite` plus Textbutler integration for + `src/imessage.ts` and `src/contacts.ts`. +- **Out of scope:** Rewriting the attributed-body / typedstream parsers in + Rust (keep them in TS if byte-exact parity is not proven). +- **Approach:** + - Rust crate copies `chat.db` + `-wal` + `-journal` to a private temp + directory atomically and verifies ownership/mode. + - Expose the snapshot path and schema validation via JSON CLI. + - TS runs its existing SQL queries against the Rust-provided snapshot path. +- **Acceptance criteria:** + - Snapshot creation is atomic and read-only with respect to the source. + - Ownership checks match existing Textbutler policy. + - `bun test` in Textbutler passes. +- **Validation:** `cargo test -p oh-sqlite && bun test` in Textbutler. + +## Phase 8: Sponge canonical/digest migration + +- **Status:** Not started +- **Depends on:** Phase 4 + an immutable `oh` release containing Phases 1–4 +- **Objective:** Consume the WASM canonical/digest engine in Sponge's + `lib/document-domain.ts`, `lib/integrity-domain.ts`, and `lib/digest.ts`. +- **Scope:** `hraness/sponge` repository. +- **Out of scope:** Datalog/graph reducer, library capture worker. +- **Approach:** + - Bump `@hraness/oh-research` to the release that ships the WASM artifact. + - Add a feature flag that routes `canonicalJson` / `sha256Text` through the + WASM engine. + - Run `fast-check` parity tests against the existing TS reference. + - Enable by default only after byte-exact parity is proven. +- **Acceptance criteria:** + - `bun run check` passes with the feature flag on and off. + - Parity tests pass on representative Convex-shaped values. +- **Validation:** `bun run check` in Sponge. + +## Phase 9: Wordcell clip/bundle reader migration + +- **Status:** Not started +- **Depends on:** Phase 5, Phase 7 +- **Objective:** Reuse `oh-archive` and `oh-sqlite` in Wordcell's capture and + bundle reading pipeline. +- **Scope:** `hraness/wordcell` repository. +- **Out of scope:** Generalizing the existing metadata-search-tool runner + (covered in Phase 11). +- **Approach:** + - Replace hand-rolled ZIP walks in `src/clip/bundle-reader.ts` with + `oh-archive`. + - Replace SQLite snapshot logic in clip extraction with `oh-sqlite`. + - Add parity tests before enabling by default. +- **Acceptance criteria:** + - Wordcell clip tests pass. + - Bundle digest identity is preserved. +- **Validation:** `bun test` in Wordcell. + +## Phase 10: Positive-Datalog projection / graph reducer crate + +- **Status:** Not started +- **Depends on:** Phase 4 +- **Objective:** A Rust implementation of `oh`’s positive-Datalog projection + engine that downstream consumers can opt into. +- **Scope:** New crate `oh-datalog` in `hraness/oh`. +- **Out of scope:** Removing the TS reference engine. +- **Approach:** + - Port `materializeNaive`, `matchBody`, and `unifyLiteral` semantics. + - Enforce `workUnits` / `derivedTuples` budgets with atomic counters. + - Add a new engine identity, e.g. `oh.projection.rust.v1`, behind + `evaluateOhProjectionWithMaterializerV1`. + - Prove parity with the TS engine on frozen test fixtures. +- **Acceptance criteria:** + - `cargo test -p oh-datalog` passes. + - Frozen projection fixtures produce identical relation sets via TS and Rust. + - Budget enforcement matches the TS policy. +- **Validation:** `cargo test -p oh-datalog && bun test` in `oh` on projection tests. + +## Phase 11: Custody/desktop-foundation consolidation + +- **Status:** Not started +- **Depends on:** Phase 4 +- **Objective:** Extend the existing `@hraness/local-custody` and + `@hraness/desktop-foundation` shared packages with Rust where they are not + already Rust, rather than duplicating that logic in Textbutler. +- **Scope:** The repositories that publish `@hraness/local-custody` and + `@hraness/desktop-foundation`. +- **Out of scope:** Rewriting Textbutler's daemon policy or menu UI in Rust. +- **Approach:** + - Audit the current implementation of those shared packages. + - If they are TypeScript, introduce Rust sidecars for path traversal, + ownership checks, atomic replace, and Unix-socket peer UID verification. + - Keep the public API unchanged. +- **Acceptance criteria:** + - Textbutler continues to consume the shared packages with no API changes. + - New Rust components have parity tests against the existing TS behavior. +- **Validation:** `bun run check` in the affected repositories. + +## Implementation log + +- 2026-09-17: Phases 1–4 implemented, committed, and pushed as + `rust-canonical-foundations` → PR #128. +- 2026-09-17: Phase 5 started; `oh-archive` core crate created. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b8668f8..40b8eeb 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -11,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "bitflags" version = "2.13.2" @@ -56,6 +71,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -76,6 +106,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -86,6 +127,49 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -96,12 +180,39 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.18" @@ -124,12 +235,34 @@ dependencies = [ "windows-link", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "napi" version = "2.16.17" @@ -181,6 +314,36 @@ dependencies = [ "libloading", ] +[[package]] +name = "oh-archive" +version = "0.1.0" +dependencies = [ + "regex", + "serde", + "serde_json", + "tempfile", + "zip", +] + +[[package]] +name = "oh-archive-napi" +version = "0.1.0" +dependencies = [ + "napi", + "napi-derive", + "oh-archive", + "serde_json", +] + +[[package]] +name = "oh-archive-wasm" +version = "0.1.0" +dependencies = [ + "oh-archive", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "oh-canonical" version = "0.1.0" @@ -233,6 +396,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "regex" version = "1.13.1" @@ -262,6 +431,19 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -334,6 +516,12 @@ dependencies = [ "digest", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "syn" version = "2.0.119" @@ -356,6 +544,39 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "typenum" version = "1.20.1" @@ -431,8 +652,46 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7854428..4bcd5db 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi"] +members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi"] resolver = "3" [workspace.package] diff --git a/rust/README.md b/rust/README.md index f04fc07..68fc8ed 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,6 +10,9 @@ fallback so the base package has no required runtime dependencies. Implements the same contract as `src/canonical.ts`. - `oh-canonical-wasm` — `wasm-pack`/`wasm-bindgen` bindings. - `oh-canonical-napi` — `napi-rs` bindings (opt-in performance layer). +- `oh-archive` — bounded, untrusted ZIP64 archive extraction. +- `oh-archive-wasm` — `wasm-pack` bindings for `oh-archive`. +- `oh-archive-napi` — `napi-rs` bindings for `oh-archive`. ## Build diff --git a/rust/oh-archive-napi/Cargo.toml b/rust/oh-archive-napi/Cargo.toml new file mode 100644 index 0000000..60ef24b --- /dev/null +++ b/rust/oh-archive-napi/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "oh-archive-napi" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "N-API bindings for oh-archive." + +[dependencies] +oh-archive = { path = "../oh-archive" } +napi = { version = "2", default-features = false, features = ["napi6"] } +napi-derive = "2" +serde_json = "1.0" + +[lib] +crate-type = ["cdylib"] +path = "src/lib.rs" diff --git a/rust/oh-archive-napi/LICENSE b/rust/oh-archive-napi/LICENSE new file mode 100644 index 0000000..3a077f5 --- /dev/null +++ b/rust/oh-archive-napi/LICENSE @@ -0,0 +1,28 @@ +MIT License + +Copyright (c) 2026 Hraness contributors + +The bundled optional semantic runtime includes Effect 3.22.1: +Copyright (c) 2023 Effectful Technologies Inc + +The standalone CLI bundles @hraness/support-foundation 0.4.0: +Copyright (c) 2026 Hraness +Source: https://github.com/hraness/support-foundation/tree/b32c1c81bb2444f50509ed54388758ecfab1f1c0 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rust/oh-archive-napi/src/lib.rs b/rust/oh-archive-napi/src/lib.rs new file mode 100644 index 0000000..70a8814 --- /dev/null +++ b/rust/oh-archive-napi/src/lib.rs @@ -0,0 +1,19 @@ +#![deny(clippy::all)] + +use napi_derive::napi; +use oh_archive::{extract_zip_entries, ExtractOptions}; + +#[napi] +pub fn extract_zip_entries_json(options_json: String) -> napi::Result { + let options: ExtractOptions = serde_json::from_str(&options_json) + .map_err(|e| napi::Error::new(napi::Status::InvalidArg, format!("invalid options JSON: {e}")))?; + + match extract_zip_entries(options) { + Ok(entries) => serde_json::to_string(&entries) + .map_err(|e| napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))), + Err(err) => serde_json::to_string(&err) + .map_err(|e| napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))) + .map(|json| Err(napi::Error::new(napi::Status::GenericFailure, json))) + .map_or_else(Err, |res| res), + } +} diff --git a/rust/oh-archive-wasm/Cargo.toml b/rust/oh-archive-wasm/Cargo.toml new file mode 100644 index 0000000..67d884d --- /dev/null +++ b/rust/oh-archive-wasm/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "oh-archive-wasm" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "WASM bindings for oh-archive." + +[dependencies] +oh-archive = { path = "../oh-archive" } +wasm-bindgen = "0.2" +serde_json.workspace = true + +[lib] +crate-type = ["cdylib"] +path = "src/lib.rs" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/rust/oh-archive-wasm/LICENSE b/rust/oh-archive-wasm/LICENSE new file mode 100644 index 0000000..3a077f5 --- /dev/null +++ b/rust/oh-archive-wasm/LICENSE @@ -0,0 +1,28 @@ +MIT License + +Copyright (c) 2026 Hraness contributors + +The bundled optional semantic runtime includes Effect 3.22.1: +Copyright (c) 2023 Effectful Technologies Inc + +The standalone CLI bundles @hraness/support-foundation 0.4.0: +Copyright (c) 2026 Hraness +Source: https://github.com/hraness/support-foundation/tree/b32c1c81bb2444f50509ed54388758ecfab1f1c0 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rust/oh-archive-wasm/src/lib.rs b/rust/oh-archive-wasm/src/lib.rs new file mode 100644 index 0000000..e11bc39 --- /dev/null +++ b/rust/oh-archive-wasm/src/lib.rs @@ -0,0 +1,19 @@ +use oh_archive::{extract_zip_entries, ExtractError, ExtractOptions}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub fn extract_zip_entries_json(options_json: &str) -> String { + let options: ExtractOptions = match serde_json::from_str(options_json) { + Ok(o) => o, + Err(e) => return serialize_error(ExtractError::InvalidArchive { message: format!("invalid options JSON: {e}") }), + }; + + match extract_zip_entries(options) { + Ok(entries) => serde_json::to_string(&entries).unwrap_or_else(|_| r#"{"error":"serialization_failed"}"#.to_string()), + Err(e) => serialize_error(e), + } +} + +fn serialize_error(error: ExtractError) -> String { + serde_json::to_string(&error).unwrap_or_else(|_| r#"{"error":"serialization_failed"}"#.to_string()) +} diff --git a/rust/oh-archive/Cargo.toml b/rust/oh-archive/Cargo.toml new file mode 100644 index 0000000..cbed6ab --- /dev/null +++ b/rust/oh-archive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "oh-archive" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Bounded untrusted archive parsing for Oh." + +[dependencies] +serde.workspace = true +serde_json.workspace = true +regex = "1" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[dev-dependencies] +tempfile = "3" + +[lib] +name = "oh_archive" +path = "src/lib.rs" diff --git a/rust/oh-archive/LICENSE b/rust/oh-archive/LICENSE new file mode 100644 index 0000000..3a077f5 --- /dev/null +++ b/rust/oh-archive/LICENSE @@ -0,0 +1,28 @@ +MIT License + +Copyright (c) 2026 Hraness contributors + +The bundled optional semantic runtime includes Effect 3.22.1: +Copyright (c) 2023 Effectful Technologies Inc + +The standalone CLI bundles @hraness/support-foundation 0.4.0: +Copyright (c) 2026 Hraness +Source: https://github.com/hraness/support-foundation/tree/b32c1c81bb2444f50509ed54388758ecfab1f1c0 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rust/oh-archive/src/lib.rs b/rust/oh-archive/src/lib.rs new file mode 100644 index 0000000..a3cec19 --- /dev/null +++ b/rust/oh-archive/src/lib.rs @@ -0,0 +1,266 @@ +//! Bounded, untrusted archive parsing. +//! +//! This crate reads ZIP archives without extracting to the filesystem +//! (except for the explicit output directory provided by the caller). It +//! enforces global and per-entry byte limits, rejects encrypted and unsupported +//! entries, and only returns entries matching an allow-list of patterns. + +use regex::RegexSet; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; +use zip::result::ZipError; +use zip::ZipArchive; + +/// Options controlling archive extraction. +#[derive(Debug, Clone, Deserialize)] +pub struct ExtractOptions { + /// Path to the ZIP archive. + pub archive_path: PathBuf, + /// Directory where extracted entry files will be written. + pub output_directory: PathBuf, + /// Regex patterns; only entries whose full name matches at least one + /// pattern are extracted. + pub patterns: Vec, + /// Maximum total uncompressed bytes across all selected entries. + #[serde(default = "default_max_total_bytes")] + pub max_total_bytes: u64, + /// Maximum uncompressed bytes for any single selected entry. + #[serde(default = "default_max_entry_bytes")] + pub max_entry_bytes: u64, + /// Maximum number of entries in the archive. + #[serde(default = "default_max_entries")] + pub max_entries: usize, +} + +fn default_max_total_bytes() -> u64 { 768 * 1024 * 1024 } +fn default_max_entry_bytes() -> u64 { 256 * 1024 * 1024 } +fn default_max_entries() -> usize { 100_000 } + +/// A successfully extracted entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExtractedEntry { + /// Full entry name inside the archive. + pub name: String, + /// Path where the entry was written. + pub output_path: PathBuf, + /// Uncompressed size in bytes. + pub uncompressed_size: u64, +} + +/// Errors returned by extraction. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "error", rename_all = "snake_case")] +pub enum ExtractError { + InvalidArchive { message: String }, + IoError { message: String }, + LimitExceeded { kind: String, limit: u64 }, + EntryLimitExceeded { limit: usize }, + UnsupportedEntry { name: String, reason: String }, + OutputPathError { name: String }, +} + +impl std::fmt::Display for ExtractError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for ExtractError {} + +impl From for ExtractError { + fn from(error: io::Error) -> Self { + ExtractError::IoError { message: error.to_string() } + } +} + +impl From for ExtractError { + fn from(error: ZipError) -> Self { + ExtractError::InvalidArchive { message: error.to_string() } + } +} + +fn sanitize_entry_name(name: &str) -> Option { + let path = Path::new(name); + let mut components = path.components().peekable(); + let mut safe = PathBuf::new(); + let mut accepted_any = false; + + while let Some(component) = components.peek() { + match component { + // Allow a leading root/prefix only if it is the first component and + // is immediately followed by normal components. + std::path::Component::RootDir | std::path::Component::Prefix(_) if !accepted_any => { + components.next(); + continue; + } + _ => break, + } + } + + for component in components { + match component { + std::path::Component::Normal(part) => { + safe.push(part); + accepted_any = true; + } + _ => return None, + } + } + + if !accepted_any { + return None; + } + Some(safe) +} + +fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<(), ExtractError> { + let parent = path.parent().ok_or_else(|| ExtractError::OutputPathError { name: path.display().to_string() })?; + std::fs::create_dir_all(parent)?; + let temp = parent.join(format!(".tmp-{}", std::process::id())); + std::fs::write(&temp, bytes)?; + std::fs::rename(&temp, path)?; + Ok(()) +} + +/// Extract selected entries from a ZIP archive. +pub fn extract_zip_entries(options: ExtractOptions) -> Result, ExtractError> { + let regex_set = RegexSet::new(&options.patterns) + .map_err(|e| ExtractError::InvalidArchive { message: format!("invalid pattern: {e}") })?; + + std::fs::create_dir_all(&options.output_directory)?; + + let file = File::open(&options.archive_path)?; + let mut archive = ZipArchive::new(file)?; + + if archive.len() > options.max_entries { + return Err(ExtractError::EntryLimitExceeded { limit: options.max_entries }); + } + + let mut total_bytes: u64 = 0; + let mut extracted = Vec::new(); + + for index in 0..archive.len() { + let mut entry = archive.by_index(index)?; + let name = entry.name().to_string(); + + if entry.is_dir() || !regex_set.is_match(&name) { + continue; + } + + if entry.encrypted() { + return Err(ExtractError::UnsupportedEntry { name, reason: "encrypted".to_string() }); + } + + let size = entry.size(); + if size > options.max_entry_bytes { + return Err(ExtractError::LimitExceeded { kind: format!("entry `{name}`"), limit: options.max_entry_bytes }); + } + let new_total = total_bytes.checked_add(size).ok_or_else(|| ExtractError::LimitExceeded { + kind: "total".to_string(), + limit: options.max_total_bytes, + })?; + if new_total > options.max_total_bytes { + return Err(ExtractError::LimitExceeded { kind: "total".to_string(), limit: options.max_total_bytes }); + } + + let relative = sanitize_entry_name(&name).ok_or_else(|| ExtractError::OutputPathError { name: name.clone() })?; + let output_path = options.output_directory.join(relative); + + let mut buffer = Vec::with_capacity(size as usize); + entry.read_to_end(&mut buffer)?; + if buffer.len() as u64 != size { + return Err(ExtractError::InvalidArchive { message: format!("entry `{name}` size mismatch") }); + } + + write_file_atomically(&output_path, &buffer)?; + + total_bytes = new_total; + extracted.push(ExtractedEntry { name, output_path, uncompressed_size: size }); + } + + Ok(extracted) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn make_test_zip(path: &Path, entries: &[(&str, &[u8])]) { + let file = File::create(path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + for (name, bytes) in entries { + zip.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(bytes).unwrap(); + } + zip.finish().unwrap(); + } + + #[test] + fn extracts_matching_entries() { + let temp = tempfile::tempdir().unwrap(); + let zip_path = temp.path().join("test.zip"); + make_test_zip(&zip_path, &[ + ("data/manifest.js", b"manifest"), + ("data/tweets.js", b"tweets"), + ("readme.txt", b"readme"), + ]); + + let out = temp.path().join("out"); + let result = extract_zip_entries(ExtractOptions { + archive_path: zip_path, + output_directory: out.clone(), + patterns: vec![r"^data/.*\.js$".to_string()], + max_total_bytes: 1024, + max_entry_bytes: 1024, + max_entries: 100, + }).unwrap(); + + assert_eq!(result.len(), 2); + assert!(std::fs::read_to_string(&result[0].output_path).is_ok()); + assert!(out.join("data/manifest.js").exists()); + } + + #[test] + fn rejects_entries_exceeding_total_limit() { + let temp = tempfile::tempdir().unwrap(); + let zip_path = temp.path().join("test.zip"); + make_test_zip(&zip_path, &[ + ("data/a.js", b"12345"), + ("data/b.js", b"67890"), + ]); + + let result = extract_zip_entries(ExtractOptions { + archive_path: zip_path, + output_directory: temp.path().join("out"), + patterns: vec![r"^data/.*\.js$".to_string()], + max_total_bytes: 5, + max_entry_bytes: 1024, + max_entries: 100, + }); + + assert!(matches!(result, Err(ExtractError::LimitExceeded { kind, .. }) if kind == "total")); + } + + #[test] + fn rejects_path_traversal() { + let temp = tempfile::tempdir().unwrap(); + let zip_path = temp.path().join("test.zip"); + make_test_zip(&zip_path, &[ + ("../escape.js", b"bad"), + ]); + + let result = extract_zip_entries(ExtractOptions { + archive_path: zip_path, + output_directory: temp.path().join("out"), + patterns: vec![r".*".to_string()], + max_total_bytes: 1024, + max_entry_bytes: 1024, + max_entries: 100, + }); + + assert!(matches!(result, Err(ExtractError::OutputPathError { .. }))); + } +} From 8445b5646150a06a361cde2125c5b15c44dfaea7 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Thu, 17 Sep 2026 14:44:16 -0400 Subject: [PATCH 2/5] Add SQLite snapshot isolation crate (oh-sqlite) Add rust/oh-sqlite with an N-API binding. The crate copies a SQLite database plus its -wal and -journal sidecars into a private output directory, verifies ownership/permissions, checks the SQLite magic header, and enforces per-file and total byte limits. The original files are never modified. Also register rust:sqlite-napi in costs.json, fix rust:check to run from the rust/ directory, and simplify archive N-API error serialization. --- costs.json | 8 + package.json | 2 +- plans/rust-foundations.md | 4 +- rust/Cargo.lock | 20 ++ rust/Cargo.toml | 2 +- rust/README.md | 2 + rust/oh-archive-napi/src/lib.rs | 8 +- rust/oh-sqlite-napi/Cargo.toml | 19 ++ rust/oh-sqlite-napi/LICENSE | 28 +++ rust/oh-sqlite-napi/src/lib.rs | 19 ++ rust/oh-sqlite/Cargo.toml | 21 ++ rust/oh-sqlite/src/lib.rs | 343 ++++++++++++++++++++++++++++++++ 12 files changed, 468 insertions(+), 8 deletions(-) create mode 100644 rust/oh-sqlite-napi/Cargo.toml create mode 100644 rust/oh-sqlite-napi/LICENSE create mode 100644 rust/oh-sqlite-napi/src/lib.rs create mode 100644 rust/oh-sqlite/Cargo.toml create mode 100644 rust/oh-sqlite/src/lib.rs diff --git a/costs.json b/costs.json index 3dc336e..a2b01cc 100644 --- a/costs.json +++ b/costs.json @@ -153,6 +153,14 @@ "budget": { "bytesPerArtifact": 4194304 } + }, + "rust:sqlite-napi": { + "kind": "served", + "retention": "persistent", + "owner": "rust/oh-sqlite/Cargo.toml", + "budget": { + "bytesPerArtifact": 4194304 + } } }, "exempt": [] diff --git a/package.json b/package.json index c43f568..18ba680 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,7 @@ "check:effect": "bun run ./scripts/check-effect-architecture.ts && bun test ./scripts/effect-architecture.test.ts", "bench:lab:profile": "bun scripts/benchmarks/lab-reader-profile-run.ts", "build:research": "bun run ./scripts/build-research.ts", - "rust:check": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cargo check --locked && cargo clippy -- -D warnings && cargo test --locked", + "rust:check": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust && cargo check --locked && cargo clippy -- -D warnings && cargo test --locked", "rust:build:canonical:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-canonical-wasm && wasm-pack build --target web --out-dir pkg", "rust:build:archive:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-archive-wasm && wasm-pack build --target web --out-dir pkg", "rust:build": "bun run rust:build:canonical:wasm && bun run rust:build:archive:wasm" diff --git a/plans/rust-foundations.md b/plans/rust-foundations.md index 0a77010..fd1a0a0 100644 --- a/plans/rust-foundations.md +++ b/plans/rust-foundations.md @@ -164,8 +164,8 @@ replacement proves byte-exact parity through property tests. ## Phase 7: Shared SQLite snapshot / iMessage / Contacts reader -- **Status:** Not started -- **Depends on:** Phase 6 +- **Status:** In progress +- **Depends on:** Phase 5 - **Objective:** Move the filesystem-snapshot and read-only query isolation for iMessage and Contacts into Rust. - **Scope:** New crate `oh-sqlite` plus Textbutler integration for diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 40b8eeb..03441f8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -372,6 +372,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "oh-sqlite" +version = "0.1.0" +dependencies = [ + "libc", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "oh-sqlite-napi" +version = "0.1.0" +dependencies = [ + "napi", + "napi-derive", + "oh-sqlite", + "serde_json", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4bcd5db..ed1a4a3 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi"] +members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi", "oh-sqlite", "oh-sqlite-napi"] resolver = "3" [workspace.package] diff --git a/rust/README.md b/rust/README.md index 68fc8ed..7f2300e 100644 --- a/rust/README.md +++ b/rust/README.md @@ -13,6 +13,8 @@ fallback so the base package has no required runtime dependencies. - `oh-archive` — bounded, untrusted ZIP64 archive extraction. - `oh-archive-wasm` — `wasm-pack` bindings for `oh-archive`. - `oh-archive-napi` — `napi-rs` bindings for `oh-archive`. +- `oh-sqlite` — read-only SQLite database + WAL/journal snapshot isolation. +- `oh-sqlite-napi` — `napi-rs` bindings for `oh-sqlite`. ## Build diff --git a/rust/oh-archive-napi/src/lib.rs b/rust/oh-archive-napi/src/lib.rs index 70a8814..a966b36 100644 --- a/rust/oh-archive-napi/src/lib.rs +++ b/rust/oh-archive-napi/src/lib.rs @@ -11,9 +11,9 @@ pub fn extract_zip_entries_json(options_json: String) -> napi::Result { match extract_zip_entries(options) { Ok(entries) => serde_json::to_string(&entries) .map_err(|e| napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))), - Err(err) => serde_json::to_string(&err) - .map_err(|e| napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))) - .map(|json| Err(napi::Error::new(napi::Status::GenericFailure, json))) - .map_or_else(Err, |res| res), + Err(err) => match serde_json::to_string(&err) { + Ok(json) => Err(napi::Error::new(napi::Status::GenericFailure, json)), + Err(e) => Err(napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))), + }, } } diff --git a/rust/oh-sqlite-napi/Cargo.toml b/rust/oh-sqlite-napi/Cargo.toml new file mode 100644 index 0000000..fa30bf9 --- /dev/null +++ b/rust/oh-sqlite-napi/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "oh-sqlite-napi" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "N-API bindings for oh-sqlite." + +[dependencies] +oh-sqlite = { path = "../oh-sqlite" } +napi = { version = "2", default-features = false, features = ["napi6"] } +napi-derive = "2" +serde_json = "1.0" + +[lib] +crate-type = ["cdylib"] +path = "src/lib.rs" diff --git a/rust/oh-sqlite-napi/LICENSE b/rust/oh-sqlite-napi/LICENSE new file mode 100644 index 0000000..3a077f5 --- /dev/null +++ b/rust/oh-sqlite-napi/LICENSE @@ -0,0 +1,28 @@ +MIT License + +Copyright (c) 2026 Hraness contributors + +The bundled optional semantic runtime includes Effect 3.22.1: +Copyright (c) 2023 Effectful Technologies Inc + +The standalone CLI bundles @hraness/support-foundation 0.4.0: +Copyright (c) 2026 Hraness +Source: https://github.com/hraness/support-foundation/tree/b32c1c81bb2444f50509ed54388758ecfab1f1c0 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rust/oh-sqlite-napi/src/lib.rs b/rust/oh-sqlite-napi/src/lib.rs new file mode 100644 index 0000000..610a5c1 --- /dev/null +++ b/rust/oh-sqlite-napi/src/lib.rs @@ -0,0 +1,19 @@ +#![deny(clippy::all)] + +use napi_derive::napi; +use oh_sqlite::{snapshot_database, SnapshotOptions}; + +#[napi] +pub fn snapshot_database_json(options_json: String) -> napi::Result { + let options: SnapshotOptions = serde_json::from_str(&options_json) + .map_err(|e| napi::Error::new(napi::Status::InvalidArg, format!("invalid options JSON: {e}")))?; + + match snapshot_database(options) { + Ok(snapshot) => serde_json::to_string(&snapshot) + .map_err(|e| napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))), + Err(err) => match serde_json::to_string(&err) { + Ok(json) => Err(napi::Error::new(napi::Status::GenericFailure, json)), + Err(e) => Err(napi::Error::new(napi::Status::GenericFailure, format!("serialization failed: {e}"))), + }, + } +} diff --git a/rust/oh-sqlite/Cargo.toml b/rust/oh-sqlite/Cargo.toml new file mode 100644 index 0000000..b9e5a01 --- /dev/null +++ b/rust/oh-sqlite/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "oh-sqlite" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Read-only SQLite snapshot isolation for Oh." + +[dependencies] +serde.workspace = true +serde_json.workspace = true +libc = "0.2" + +[dev-dependencies] +tempfile = "3" + +[lib] +name = "oh_sqlite" +path = "src/lib.rs" diff --git a/rust/oh-sqlite/src/lib.rs b/rust/oh-sqlite/src/lib.rs new file mode 100644 index 0000000..9b722de --- /dev/null +++ b/rust/oh-sqlite/src/lib.rs @@ -0,0 +1,343 @@ +//! Read-only SQLite snapshot isolation. +//! +//! Copies a SQLite database plus its `-wal` and `-journal` sidecar files into +//! a private temporary directory, verifies ownership and permissions, checks +//! the SQLite magic header, and returns the isolated paths for a read-only +//! consumer. The consumer is responsible for opening the database with its own +//! SQLite library and never writes to the original files. + +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io::{self, Read, Write as _}; +use std::path::{Path, PathBuf}; + +const SQLITE_MAGIC: &[u8] = b"SQLite format 3\0"; +const SQLITE_MAGIC_LEN: usize = 16; + +/// Options controlling snapshot creation. +#[derive(Debug, Clone, Deserialize)] +pub struct SnapshotOptions { + /// Path to the main SQLite database file. + pub source_path: PathBuf, + /// Directory where the snapshot files will be written. + pub output_directory: PathBuf, + /// Maximum size in bytes for any individual file. + #[serde(default = "default_max_file_bytes")] + pub max_file_bytes: u64, + /// Maximum total size in bytes across main + sidecars. + #[serde(default = "default_max_total_bytes")] + pub max_total_bytes: u64, +} + +fn default_max_file_bytes() -> u64 { 16 * 1024 * 1024 * 1024 } +fn default_max_total_bytes() -> u64 { 64 * 1024 * 1024 * 1024 } + +/// A successfully created snapshot. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Snapshot { + /// Path to the copied main database file. + pub database_path: PathBuf, + /// Path to the copied WAL file, if present. + pub wal_path: Option, + /// Path to the copied journal file, if present. + pub journal_path: Option, + /// Total bytes copied across all files. + pub total_bytes: u64, +} + +/// Errors returned by snapshot creation. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "error", rename_all = "snake_case")] +pub enum SnapshotError { + IoError { message: String }, + NotAFile { path: String }, + OwnershipError { path: String, reason: String }, + PermissionError { path: String, reason: String }, + SizeLimitExceeded { path: String, limit: u64 }, + InvalidSqliteHeader { path: String }, +} + +impl std::fmt::Display for SnapshotError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for SnapshotError {} + +impl From for SnapshotError { + fn from(error: io::Error) -> Self { + SnapshotError::IoError { message: error.to_string() } + } +} + +#[derive(Debug, Clone, Copy)] +struct FileCheck { + uid: u32, + mode: u32, + size: u64, +} + +#[cfg(unix)] +fn inspect_file(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + let meta = fs::metadata(path)?; + Ok(FileCheck { + uid: meta.uid(), + mode: meta.mode(), + size: meta.len(), + }) +} + +#[cfg(not(unix))] +fn inspect_file(path: &Path) -> Result { + let meta = fs::metadata(path)?; + Ok(FileCheck { + uid: 0, + mode: 0, + size: meta.len(), + }) +} + +#[cfg(unix)] +fn current_uid() -> u32 { + unsafe { libc::getuid() } +} + +#[cfg(not(unix))] +fn current_uid() -> u32 { + 0 +} + +fn check_owner(path: &Path, check: FileCheck) -> Result<(), SnapshotError> { + let path_str = path.display().to_string(); + if check.uid != current_uid() { + return Err(SnapshotError::OwnershipError { + path: path_str, + reason: format!("expected uid {}, got {}", current_uid(), check.uid), + }); + } + // Owner must have read permission; group/other must not have write. + const OWNER_READ: u32 = 0o400; + const GROUP_WRITE: u32 = 0o020; + const OTHER_WRITE: u32 = 0o002; + if check.mode & OWNER_READ == 0 { + return Err(SnapshotError::PermissionError { + path: path_str, + reason: "owner read bit not set".to_string(), + }); + } + if check.mode & (GROUP_WRITE | OTHER_WRITE) != 0 { + return Err(SnapshotError::PermissionError { + path: path_str, + reason: "group/other write bits must be clear".to_string(), + }); + } + Ok(()) +} + +fn copy_file_limited( + source: &Path, + destination: &Path, + max_bytes: u64, +) -> Result { + let check = inspect_file(source)?; + if check.size > max_bytes { + return Err(SnapshotError::SizeLimitExceeded { + path: source.display().to_string(), + limit: max_bytes, + }); + } + + let mut reader = fs::File::open(source)?; + let mut writer = fs::File::create(destination)?; + let mut copied: u64 = 0; + let mut buffer = [0u8; 64 * 1024]; + + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + writer.write_all(&buffer[..n])?; + copied += n as u64; + if copied > check.size { + return Err(SnapshotError::InvalidSqliteHeader { + path: source.display().to_string(), + }); + } + } + + // Sync the copy so WAL/journal are durably on disk before the consumer opens them. + writer.sync_all()?; + Ok(copied) +} + +fn verify_sqlite_header(path: &Path) -> Result<(), SnapshotError> { + let mut file = fs::File::open(path)?; + let mut header = [0u8; SQLITE_MAGIC_LEN]; + match file.read_exact(&mut header) { + Ok(()) => {} + Err(_) => { + return Err(SnapshotError::InvalidSqliteHeader { + path: path.display().to_string(), + }); + } + } + if header.as_slice() != SQLITE_MAGIC { + return Err(SnapshotError::InvalidSqliteHeader { + path: path.display().to_string(), + }); + } + Ok(()) +} + +/// Snapshot a SQLite database and its WAL/journal sidecars into a private +/// output directory. The original files are never modified. +pub fn snapshot_database(options: SnapshotOptions) -> Result { + fs::create_dir_all(&options.output_directory)?; + + if !options.source_path.is_file() { + return Err(SnapshotError::NotAFile { + path: options.source_path.display().to_string(), + }); + } + + let main_check = inspect_file(&options.source_path)?; + check_owner(&options.source_path, main_check)?; + + let base_name = options + .source_path + .file_name() + .ok_or_else(|| SnapshotError::IoError { + message: "source path has no file name".to_string(), + })?; + + let dest_path = options.output_directory.join(base_name); + let mut total_bytes = copy_file_limited(&options.source_path, &dest_path, options.max_file_bytes)?; + verify_sqlite_header(&dest_path)?; + + let source_dir = options.source_path.parent().unwrap_or(Path::new(".")); + let source_name = options + .source_path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + + let wal_source = source_dir.join(format!("{source_name}-wal")); + let journal_source = source_dir.join(format!("{source_name}-journal")); + + let mut wal_path = None; + if wal_source.is_file() { + let check = inspect_file(&wal_source)?; + check_owner(&wal_source, check)?; + let dest = options.output_directory.join(format!("{source_name}-wal")); + let bytes = copy_file_limited(&wal_source, &dest, options.max_file_bytes)?; + total_bytes += bytes; + wal_path = Some(dest); + } + + let mut journal_path = None; + if journal_source.is_file() { + let check = inspect_file(&journal_source)?; + check_owner(&journal_source, check)?; + let dest = options.output_directory.join(format!("{source_name}-journal")); + let bytes = copy_file_limited(&journal_source, &dest, options.max_file_bytes)?; + total_bytes += bytes; + journal_path = Some(dest); + } + + if total_bytes > options.max_total_bytes { + return Err(SnapshotError::SizeLimitExceeded { + path: options.source_path.display().to_string(), + limit: options.max_total_bytes, + }); + } + + Ok(Snapshot { + database_path: dest_path, + wal_path, + journal_path, + total_bytes, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn write_sqlite_header(path: &Path) { + let mut file = fs::File::create(path).unwrap(); + file.write_all(SQLITE_MAGIC).unwrap(); + file.write_all(&[0u8; 512 - SQLITE_MAGIC_LEN]).unwrap(); + } + + #[test] + fn snapshots_valid_database() { + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("chat.db"); + write_sqlite_header(&source); + + let out = temp.path().join("out"); + let snapshot = snapshot_database(SnapshotOptions { + source_path: source, + output_directory: out.clone(), + max_file_bytes: 1024 * 1024, + max_total_bytes: 1024 * 1024, + }).unwrap(); + + assert!(snapshot.database_path.exists()); + assert_eq!(snapshot.wal_path, None); + assert_eq!(snapshot.journal_path, None); + } + + #[test] + fn rejects_invalid_header() { + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("chat.db"); + fs::write(&source, b"not sqlite").unwrap(); + + let result = snapshot_database(SnapshotOptions { + source_path: source, + output_directory: temp.path().join("out"), + max_file_bytes: 1024 * 1024, + max_total_bytes: 1024 * 1024, + }); + + assert!(matches!(result, Err(SnapshotError::InvalidSqliteHeader { .. }))); + } + + #[test] + fn rejects_missing_file() { + let temp = tempfile::tempdir().unwrap(); + let result = snapshot_database(SnapshotOptions { + source_path: temp.path().join("missing.db"), + output_directory: temp.path().join("out"), + max_file_bytes: 1024 * 1024, + max_total_bytes: 1024 * 1024, + }); + + assert!(matches!(result, Err(SnapshotError::NotAFile { .. }))); + } + + #[test] + fn copies_wal_and_journal() { + let temp = tempfile::tempdir().unwrap(); + let source = temp.path().join("chat.db"); + write_sqlite_header(&source); + fs::write(temp.path().join("chat.db-wal"), &[0u8; 100]).unwrap(); + fs::write(temp.path().join("chat.db-journal"), &[0u8; 50]).unwrap(); + + let snapshot = snapshot_database(SnapshotOptions { + source_path: source, + output_directory: temp.path().join("out"), + max_file_bytes: 1024 * 1024, + max_total_bytes: 1024 * 1024, + }).unwrap(); + + assert!(snapshot.wal_path.as_ref().unwrap().exists()); + assert!(snapshot.journal_path.as_ref().unwrap().exists()); + assert_eq!(snapshot.total_bytes, 512 + 100 + 50); + } +} From 9cc1695091c89cf03e807760819ceae4e1f861d0 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Thu, 17 Sep 2026 14:44:43 -0400 Subject: [PATCH 3/5] Update rust foundations plan with archive and sqlite progress --- plans/rust-foundations.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plans/rust-foundations.md b/plans/rust-foundations.md index fd1a0a0..8ec86b0 100644 --- a/plans/rust-foundations.md +++ b/plans/rust-foundations.md @@ -265,4 +265,10 @@ replacement proves byte-exact parity through property tests. - 2026-09-17: Phases 1–4 implemented, committed, and pushed as `rust-canonical-foundations` → PR #128. -- 2026-09-17: Phase 5 started; `oh-archive` core crate created. +- 2026-09-17: Phase 5 and Phase 7 implemented and pushed to + `rust-archive-foundations` → PR #129. This adds `oh-archive` (bounded ZIP64 + extraction), `oh-sqlite` (read-only SQLite snapshot isolation), and their + N-API/WASM bindings. +- 2026-09-17: Phase 10 (`oh-datalog` positive-Datalog projection engine) is + the next large in-repo phase; Phases 6, 8, 9, and 11 require published + upstream artifacts or external repository changes. From 64d6fb42e6a7e06bbad0dbebdc1de159349c4c77 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Thu, 17 Sep 2026 15:25:21 -0400 Subject: [PATCH 4/5] Add oh-datalog positive-Datalog engine crate Implement the projection semantics from src/projection.ts in Rust: - Dataset, fact, rule pack, and query types with serde support - Naive materialization loop with work-unit and round bounds - Proof tree generation with depth/node/byte budgets - Query evaluation with canonical-JSON tuple keys - WASM binding for Convex/Bun consumers - New engine identity: oh.projection.rust.v1 Includes unit tests for transitive closure and budget enforcement. --- costs.json | 8 + package.json | 3 +- rust/Cargo.lock | 83 +++ rust/Cargo.toml | 4 +- rust/oh-datalog-wasm/Cargo.toml | 22 + rust/oh-datalog-wasm/src/lib.rs | 25 + rust/oh-datalog/Cargo.toml | 23 + rust/oh-datalog/src/lib.rs | 1082 +++++++++++++++++++++++++++++++ 8 files changed, 1248 insertions(+), 2 deletions(-) create mode 100644 rust/oh-datalog-wasm/Cargo.toml create mode 100644 rust/oh-datalog-wasm/src/lib.rs create mode 100644 rust/oh-datalog/Cargo.toml create mode 100644 rust/oh-datalog/src/lib.rs diff --git a/costs.json b/costs.json index a2b01cc..f933eec 100644 --- a/costs.json +++ b/costs.json @@ -161,6 +161,14 @@ "budget": { "bytesPerArtifact": 4194304 } + }, + "rust:datalog-wasm": { + "kind": "served", + "retention": "persistent", + "owner": "rust/oh-datalog/Cargo.toml", + "budget": { + "bytesPerArtifact": 4194304 + } } }, "exempt": [] diff --git a/package.json b/package.json index 18ba680..441133e 100644 --- a/package.json +++ b/package.json @@ -143,7 +143,8 @@ "rust:check": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust && cargo check --locked && cargo clippy -- -D warnings && cargo test --locked", "rust:build:canonical:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-canonical-wasm && wasm-pack build --target web --out-dir pkg", "rust:build:archive:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-archive-wasm && wasm-pack build --target web --out-dir pkg", - "rust:build": "bun run rust:build:canonical:wasm && bun run rust:build:archive:wasm" + "rust:build:datalog:wasm": "export PATH=\"/Users/bg/.cargo/bin:$PATH\" && cd rust/oh-datalog-wasm && wasm-pack build --target web --out-dir pkg", + "rust:build": "bun run rust:build:canonical:wasm && bun run rust:build:archive:wasm && bun run rust:build:datalog:wasm" }, "devDependencies": { "@suss/datalog": "0.20.0", diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 03441f8..282c502 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -170,6 +170,30 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -219,6 +243,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" @@ -372,6 +407,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "oh-datalog" +version = "0.1.0" +dependencies = [ + "hex", + "oh-canonical", + "serde", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "oh-datalog-wasm" +version = "0.1.0" +dependencies = [ + "oh-datalog", + "serde", + "serde-wasm-bindgen", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "oh-sqlite" version = "0.1.0" @@ -398,6 +456,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -492,6 +556,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -542,6 +617,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "syn" version = "2.0.119" @@ -630,6 +711,8 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", + "serde_json", "wasm-bindgen-macro", "wasm-bindgen-shared", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ed1a4a3..6abca40 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi", "oh-sqlite", "oh-sqlite-napi"] +members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi", "oh-sqlite", "oh-sqlite-napi", "oh-datalog", "oh-datalog-wasm"] resolver = "3" [workspace.package] @@ -15,6 +15,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" hex = "0.4" +wasm-bindgen = { version = "0.2", features = ["serde-serialize"] } +serde-wasm-bindgen = "0.6" [workspace.metadata.release] publish = false diff --git a/rust/oh-datalog-wasm/Cargo.toml b/rust/oh-datalog-wasm/Cargo.toml new file mode 100644 index 0000000..23b4883 --- /dev/null +++ b/rust/oh-datalog-wasm/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "oh-datalog-wasm" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "WASM bindings for the Oh positive-Datalog engine." + +[lib] +crate-type = ["cdylib"] + +[dependencies] +oh-datalog = { path = "../oh-datalog" } +serde.workspace = true +serde_json.workspace = true +wasm-bindgen.workspace = true +serde-wasm-bindgen.workspace = true + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false diff --git a/rust/oh-datalog-wasm/src/lib.rs b/rust/oh-datalog-wasm/src/lib.rs new file mode 100644 index 0000000..b883878 --- /dev/null +++ b/rust/oh-datalog-wasm/src/lib.rs @@ -0,0 +1,25 @@ +use oh_datalog::{evaluate_projection, OhProjectionDataset, OhProjectionEvaluationOptions, OhProjectionQuery, OhProjectionRulePack}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen] +pub fn evaluate_projection_js( + dataset: JsValue, + rule_pack: JsValue, + query: JsValue, + options: JsValue, +) -> Result { + let dataset: OhProjectionDataset = serde_wasm_bindgen::from_value(dataset) + .map_err(|e| JsValue::from_str(&format!("invalid dataset: {e}")))?; + let rule_pack: OhProjectionRulePack = serde_wasm_bindgen::from_value(rule_pack) + .map_err(|e| JsValue::from_str(&format!("invalid rule pack: {e}")))?; + let query: OhProjectionQuery = serde_wasm_bindgen::from_value(query) + .map_err(|e| JsValue::from_str(&format!("invalid query: {e}")))?; + let options: OhProjectionEvaluationOptions = serde_wasm_bindgen::from_value(options) + .map_err(|e| JsValue::from_str(&format!("invalid options: {e}")))?; + + let result = evaluate_projection(&dataset, &rule_pack, &query, options) + .map_err(|e| JsValue::from_str(&format!("{e}")))?; + + serde_wasm_bindgen::to_value(&result) + .map_err(|e| JsValue::from_str(&format!("serialization failed: {e}"))) +} diff --git a/rust/oh-datalog/Cargo.toml b/rust/oh-datalog/Cargo.toml new file mode 100644 index 0000000..e3782d7 --- /dev/null +++ b/rust/oh-datalog/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "oh-datalog" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Positive-Datalog projection engine for Oh." + +[dependencies] +oh-canonical = { path = "../oh-canonical" } +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +hex.workspace = true + +[dev-dependencies] +tempfile = "3" + +[lib] +name = "oh_datalog" +path = "src/lib.rs" diff --git a/rust/oh-datalog/src/lib.rs b/rust/oh-datalog/src/lib.rs new file mode 100644 index 0000000..0a76858 --- /dev/null +++ b/rust/oh-datalog/src/lib.rs @@ -0,0 +1,1082 @@ +//! Positive-Datalog projection engine for Oh. +//! +//! This crate implements the same semantics as `oh`'s TypeScript projection +//! engine (`src/projection.ts`), but is designed to run without a JavaScript +//! heap. The API is intentionally small and additive: callers provide a dataset +//! of base facts, a rule pack, and a query; the engine materializes the +//! derivation and returns the bounded result. + +use oh_canonical::{canonical_json_str, canonical_sha256_str}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::{BTreeMap, HashMap, HashSet}; + +/// A JSON primitive usable as a Datalog atom. +pub type OhProjectionAtom = Value; + +/// A Datalog term: either a constant or a named variable. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum OhProjectionTerm { + Constant { v: u8, value: OhProjectionAtom }, + Variable { name: String, v: u8 }, +} + +/// A relation name with a tuple of terms. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionLiteral { + pub relation: String, + pub terms: Vec, + pub v: u8, +} + +/// A positive Datalog rule. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionRule { + pub body: Vec, + pub head: OhProjectionLiteral, + #[serde(rename = "ruleId")] + pub rule_id: String, + #[serde(rename = "ruleSha256")] + pub rule_sha256: String, + pub v: u8, +} + +/// A source record reference for a base fact. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionFactSource { + pub key: String, + #[serde(rename = "recordSha256")] + pub record_sha256: String, + pub v: u8, +} + +/// A base fact with provenance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionFact { + #[serde(rename = "factSha256")] + pub fact_sha256: String, + pub relation: String, + pub sources: Vec, + pub tuple: Vec, + pub v: u8, +} + +/// A complete dataset with provenance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionDataset { + #[serde(rename = "datasetSha256")] + pub dataset_sha256: String, + #[serde(rename = "extractorSha256")] + pub extractor_sha256: String, + #[serde(rename = "factPackId")] + pub fact_pack_id: String, + #[serde(rename = "factPackRevision")] + pub fact_pack_revision: u64, + #[serde(rename = "factPackSha256")] + pub fact_pack_sha256: String, + pub facts: Vec, + #[serde(rename = "factsSha256")] + pub facts_sha256: String, + #[serde(rename = "snapshotSha256")] + pub snapshot_sha256: String, + pub v: u8, +} + +/// A rule pack with provenance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionRulePack { + #[serde(rename = "rulePackId")] + pub rule_pack_id: String, + #[serde(rename = "rulePackRevision")] + pub rule_pack_revision: u64, + #[serde(rename = "rulePackSha256")] + pub rule_pack_sha256: String, + pub rules: Vec, + #[serde(rename = "rulesSha256")] + pub rules_sha256: String, + pub semantics: String, + pub v: u8, +} + +/// A query with provenance. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionQuery { + pub find: Vec, + pub limit: u64, + #[serde(rename = "queryId")] + pub query_id: String, + #[serde(rename = "querySha256")] + pub query_sha256: String, + pub where_: Vec, + pub v: u8, +} + +/// A proof node for a derived tuple. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum OhProjectionProof { + Fact { + relation: String, + sources: Vec, + tuple: Vec, + v: u8, + }, + Derived { + premises: Vec, + #[serde(rename = "premisesTruncated")] + premises_truncated: bool, + relation: String, + #[serde(rename = "ruleId")] + rule_id: String, + #[serde(rename = "ruleSha256")] + rule_sha256: String, + tuple: Vec, + v: u8, + }, + Truncated { + reason: String, + relation: String, + tuple: Vec, + v: u8, + }, +} + +/// A single result row. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OhProjectionResultRow { + pub proofs: Vec, + #[serde(rename = "proofsTruncated")] + pub proofs_truncated: bool, + #[serde(rename = "supportCount")] + pub support_count: u64, + pub values: Vec, + pub v: u8, +} + +/// The complete projection result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OhProjectionResult { + pub authority: String, + pub cache: CacheInfo, + pub engine: String, + pub evaluation: EvaluationLimits, + pub facts: FactCounts, + pub output: ResultRows, + pub provenance: Provenance, + pub query: QuerySummary, + pub semantics: String, + pub v: u8, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheInfo { + pub strategy: String, + pub v: u8, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvaluationLimits { + #[serde(rename = "maximumDerivedTuples")] + pub maximum_derived_tuples: u64, + #[serde(rename = "maximumProofDepth")] + pub maximum_proof_depth: u64, + #[serde(rename = "maximumProofNodes")] + pub maximum_proof_nodes: u64, + #[serde(rename = "maximumResultBytes")] + pub maximum_result_bytes: u64, + #[serde(rename = "maximumRounds")] + pub maximum_rounds: u64, + #[serde(rename = "maximumTotalProofNodes")] + pub maximum_total_proof_nodes: u64, + #[serde(rename = "maximumWorkUnits")] + pub maximum_work_units: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FactCounts { + pub base: u64, + pub derived: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResultRows { + pub rows: Vec, + #[serde(rename = "rowsTruncated")] + pub rows_truncated: bool, + #[serde(rename = "rowsTotal")] + pub rows_total: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Provenance { + pub contract: ContractInfo, + #[serde(rename = "datasetSha256")] + pub dataset_sha256: String, + #[serde(rename = "engineSha256")] + pub engine_sha256: String, + #[serde(rename = "evaluationSha256")] + pub evaluation_sha256: String, + #[serde(rename = "querySha256")] + pub query_sha256: String, + #[serde(rename = "rulePackSha256")] + pub rule_pack_sha256: String, + #[serde(rename = "snapshotSha256")] + pub snapshot_sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContractInfo { + #[serde(rename = "contractSha256")] + pub contract_sha256: String, + pub v: u8, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QuerySummary { + pub find: Vec, + #[serde(rename = "queryId")] + pub query_id: String, + #[serde(rename = "querySha256")] + pub query_sha256: String, + pub v: u8, +} + +/// Errors from projection evaluation. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "error", rename_all = "snake_case")] +pub enum ProjectionError { + InvalidInput { message: String }, + WorkBudgetExceeded { limit: u64 }, + DerivedTupleLimitExceeded { limit: u64 }, + RoundLimitExceeded { limit: u64 }, + MatchLimitExceeded { limit: u64 }, + ResultByteLimitExceeded { limit: u64 }, + ProofNodeLimitExceeded { limit: u64 }, + ProofDepthLimitExceeded { limit: u64 }, +} + +impl std::fmt::Display for ProjectionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for ProjectionError {} + +/// Work budget shared across the materialization. +#[derive(Debug, Clone)] +pub struct WorkBudget { + maximum: u64, + units: u64, +} + +impl WorkBudget { + pub fn new(maximum: u64) -> Self { + Self { maximum, units: 0 } + } + + pub fn consume(&mut self) -> Result<(), ProjectionError> { + if self.units >= self.maximum { + return Err(ProjectionError::WorkBudgetExceeded { limit: self.maximum }); + } + self.units += 1; + Ok(()) + } +} + +type Binding = HashMap; +type TupleKey = String; + +#[derive(Debug, Clone)] +struct TupleState { + tuple: Vec, + witness: Witness, +} + +#[derive(Debug, Clone)] +enum Witness { + Fact { sources: Vec }, + Derived { premises: Vec, rule: OhProjectionRule }, +} + +#[derive(Debug, Clone, Serialize)] +struct TupleReference { + relation: String, + tuple: Vec, +} + +fn tuple_key(tuple: &[OhProjectionAtom]) -> TupleKey { + canonical_json_str(&serde_json::to_string(tuple).unwrap()) + .unwrap_or_else(|_| "[]".to_string()) +} + +fn reference_key(reference: &TupleReference) -> String { + canonical_json_str(&serde_json::to_string(&[ + Value::String(reference.relation.clone()), + Value::Array(reference.tuple.clone()), + ]).unwrap()).unwrap_or_else(|_| "[]".to_string()) +} + +fn canonical_witness(witness: &Witness) -> String { + match witness { + Witness::Fact { sources } => { + canonical_json_str(&serde_json::to_string(&serde_json::json!({ + "kind": "fact", + "sources": sources, + })).unwrap()).unwrap_or_else(|_| "{}".to_string()) + } + Witness::Derived { premises, rule } => { + canonical_json_str(&serde_json::to_string(&serde_json::json!({ + "kind": "derived", + "premises": premises, + "ruleSha256": rule.rule_sha256, + })).unwrap()).unwrap_or_else(|_| "{}".to_string()) + } + } +} + +fn same_atom(left: &OhProjectionAtom, right: &OhProjectionAtom) -> bool { + left == right +} + +fn unify_literal(literal: &OhProjectionLiteral, state: &TupleState, binding: &Binding) -> Option { + let mut next = binding.clone(); + for (index, term) in literal.terms.iter().enumerate() { + let value = &state.tuple[index]; + match term { + OhProjectionTerm::Constant { value: const_value, .. } => { + if !same_atom(const_value, value) { + return None; + } + } + OhProjectionTerm::Variable { name, .. } => { + if let Some(existing) = next.get(name) { + if !same_atom(existing, value) { + return None; + } + } else { + next.insert(name.clone(), value.clone()); + } + } + } + } + Some(next) +} + +fn match_body( + relations: &BTreeMap>, + body: &[OhProjectionLiteral], + maximum_matches: u64, + work: &mut WorkBudget, +) -> Result, ProjectionError> { + let mut matches: Vec = vec![BodyMatch { binding: Binding::new(), premises: Vec::new() }]; + for literal in body { + let candidates = relations + .get(&literal.relation) + .map(|m| m.values().collect::>()) + .unwrap_or_default(); + let mut next: Vec = Vec::new(); + for match_ in matches { + for candidate in &candidates { + work.consume()?; + if let Some(binding) = unify_literal(literal, candidate, &match_.binding) { + next.push(BodyMatch { + binding, + premises: { + let mut p = match_.premises.clone(); + p.push(TupleReference { + relation: literal.relation.clone(), + tuple: candidate.tuple.clone(), + }); + p + }, + }); + if next.len() as u64 > maximum_matches { + return Err(ProjectionError::MatchLimitExceeded { limit: maximum_matches }); + } + } + } + } + matches = next; + if matches.is_empty() { + break; + } + } + Ok(matches) +} + +struct BodyMatch { + binding: Binding, + premises: Vec, +} + +fn instantiate_head(head: &OhProjectionLiteral, binding: &Binding) -> Vec { + head.terms + .iter() + .map(|term| match term { + OhProjectionTerm::Constant { value, .. } => value.clone(), + OhProjectionTerm::Variable { name, .. } => binding.get(name).cloned().unwrap_or(Value::Null), + }) + .collect() +} + +fn materialize_naive( + dataset: &OhProjectionDataset, + rule_pack: &OhProjectionRulePack, + maximum_derived_tuples: u64, + maximum_rounds: u64, + work: &mut WorkBudget, +) -> Result { + let mut relations: BTreeMap> = BTreeMap::new(); + for fact in &dataset.facts { + relations + .entry(fact.relation.clone()) + .or_default() + .insert(tuple_key(&fact.tuple), TupleState { + tuple: fact.tuple.clone(), + witness: Witness::Fact { sources: fact.sources.clone() }, + }); + } + + let mut derived_facts = 0u64; + let mut rounds = 0u64; + + loop { + let mut candidates: BTreeMap = BTreeMap::new(); + for rule in &rule_pack.rules { + for match_ in match_body(&relations, &rule.body, u64::MAX, work)? { + let derived_tuple = instantiate_head(&rule.head, &match_.binding); + let key = tuple_key(&derived_tuple); + if relations + .get(&rule.head.relation) + .map(|m| m.contains_key(&key)) + .unwrap_or(false) + { + continue; + } + let state = TupleState { + tuple: derived_tuple.clone(), + witness: Witness::Derived { + premises: match_.premises.clone(), + rule: rule.clone(), + }, + }; + let identity = reference_key(&TupleReference { + relation: rule.head.relation.clone(), + tuple: derived_tuple.clone(), + }); + match candidates.get(&identity) { + Some((_, existing_state)) if canonical_witness(&state.witness) >= canonical_witness(&existing_state.witness) => { + continue; + } + _ => { + candidates.insert(identity, (rule.head.relation.clone(), state)); + } + } + } + } + if candidates.is_empty() { + break; + } + if rounds >= maximum_rounds { + return Err(ProjectionError::RoundLimitExceeded { limit: maximum_rounds }); + } + let candidates_len = candidates.len() as u64; + if derived_facts + candidates_len > maximum_derived_tuples { + return Err(ProjectionError::DerivedTupleLimitExceeded { limit: maximum_derived_tuples }); + } + for (_, (relation, state)) in candidates { + relations + .entry(relation) + .or_default() + .insert(tuple_key(&state.tuple), state); + } + derived_facts += candidates_len; + rounds += 1; + } + + Ok(MaterializedProjection { + base_facts: dataset.facts.len() as u64, + derived_facts, + relations, + rounds, + }) +} + +struct MaterializedProjection { + base_facts: u64, + derived_facts: u64, + relations: BTreeMap>, + #[allow(dead_code)] + rounds: u64, +} + +/// Evaluation options controlling projection limits. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct OhProjectionEvaluationOptions { + #[serde(rename = "maximumDerivedTuples")] + pub maximum_derived_tuples: Option, + #[serde(rename = "maximumProofDepth")] + pub maximum_proof_depth: Option, + #[serde(rename = "maximumProofNodes")] + pub maximum_proof_nodes: Option, + #[serde(rename = "maximumResultBytes")] + pub maximum_result_bytes: Option, + #[serde(rename = "maximumRounds")] + pub maximum_rounds: Option, + #[serde(rename = "maximumTotalProofNodes")] + pub maximum_total_proof_nodes: Option, + #[serde(rename = "maximumWorkUnits")] + pub maximum_work_units: Option, +} + +fn bounded_option(value: Option, fallback: u64, maximum: u64) -> Result { + let value = value.unwrap_or(fallback); + if value == 0 || value > maximum { + return Err(ProjectionError::InvalidInput { + message: format!("limit must be an integer from 1 through {maximum}"), + }); + } + Ok(value) +} + +const LIMITS: Limits = Limits { + arity: 32, + atom_bytes: 16 * 1024, + derived_tuples: 262_144, + facts: 262_144, + literals_per_rule: 64, + proof_depth: 128, + proof_nodes: 4_096, + query_literals: 64, + query_matches: 262_144, + query_results: 65_536, + relations: 4_096, + result_bytes: 16 * 1024 * 1024, + rounds: 1_024, + rules: 1_024, + sources_per_fact: 64, + total_proof_nodes: 65_536, + variables: 256, + work_units: 16_777_216, +}; + +#[allow(dead_code)] +struct Limits { + arity: u64, + atom_bytes: u64, + derived_tuples: u64, + facts: u64, + literals_per_rule: u64, + proof_depth: u64, + proof_nodes: u64, + query_literals: u64, + query_matches: u64, + query_results: u64, + relations: u64, + result_bytes: u64, + rounds: u64, + rules: u64, + sources_per_fact: u64, + total_proof_nodes: u64, + variables: u64, + work_units: u64, +} + +fn resolve_evaluation_options(options: OhProjectionEvaluationOptions) -> Result { + Ok(EvaluationLimits { + maximum_derived_tuples: bounded_option( + options.maximum_derived_tuples, + LIMITS.derived_tuples, + LIMITS.derived_tuples, + )?, + maximum_proof_depth: bounded_option( + options.maximum_proof_depth, + 32, + LIMITS.proof_depth, + )?, + maximum_proof_nodes: bounded_option( + options.maximum_proof_nodes, + 1_024, + LIMITS.proof_nodes, + )?, + maximum_result_bytes: bounded_option( + options.maximum_result_bytes, + LIMITS.result_bytes, + LIMITS.result_bytes, + )?, + maximum_rounds: bounded_option( + options.maximum_rounds, + LIMITS.rounds, + LIMITS.rounds, + )?, + maximum_total_proof_nodes: bounded_option( + options.maximum_total_proof_nodes, + LIMITS.total_proof_nodes, + LIMITS.total_proof_nodes, + )?, + maximum_work_units: bounded_option( + options.maximum_work_units, + LIMITS.work_units, + LIMITS.work_units, + )?, + }) +} + +struct ResultBudget { + bytes: u64, + maximum_bytes: u64, + nodes: u64, +} + +impl ResultBudget { + fn reserve_bytes(&mut self, value: &Value) -> bool { + let bytes = serde_json::to_string(value).map(|s| s.len() as u64).unwrap_or(0); + if self.bytes + bytes > self.maximum_bytes { + return false; + } + self.bytes += bytes; + true + } +} + +struct ProofBudget<'a> { + nodes: u64, + result: &'a mut ResultBudget, + options: &'a EvaluationLimits, +} + +impl ProofBudget<'_> { + fn reserve_node(&mut self, envelope: &OhProjectionProof) -> bool { + if self.nodes >= self.options.maximum_proof_nodes + || self.result.nodes >= self.options.maximum_total_proof_nodes + || !self.result.reserve_bytes(&serde_json::to_value(envelope).unwrap()) + { + return false; + } + self.nodes += 1; + self.result.nodes += 1; + true + } +} + +fn proof_for_reference( + relations: &BTreeMap>, + reference: &TupleReference, + budget: &mut ProofBudget, + options: &EvaluationLimits, + depth: u64, + visiting: &mut HashSet, +) -> Option { + if depth >= options.maximum_proof_depth { + let proof = OhProjectionProof::Truncated { + reason: "depth".to_string(), + relation: reference.relation.clone(), + tuple: reference.tuple.clone(), + v: 1, + }; + return if budget.reserve_node(&proof) { Some(proof) } else { None }; + } + let identity = reference_key(reference); + if visiting.contains(&identity) { + let proof = OhProjectionProof::Truncated { + reason: "cycle".to_string(), + relation: reference.relation.clone(), + tuple: reference.tuple.clone(), + v: 1, + }; + return if budget.reserve_node(&proof) { Some(proof) } else { None }; + } + let state = relations + .get(&reference.relation) + .and_then(|m| m.get(&tuple_key(&reference.tuple)))?; + match &state.witness { + Witness::Fact { sources } => { + let proof = OhProjectionProof::Fact { + relation: reference.relation.clone(), + sources: sources.clone(), + tuple: reference.tuple.clone(), + v: 1, + }; + if budget.reserve_node(&proof) { Some(proof) } else { None } + } + Witness::Derived { premises, rule } => { + let envelope = OhProjectionProof::Derived { + premises: Vec::new(), + premises_truncated: false, + relation: reference.relation.clone(), + rule_id: rule.rule_id.clone(), + rule_sha256: rule.rule_sha256.clone(), + tuple: reference.tuple.clone(), + v: 1, + }; + if !budget.reserve_node(&envelope) { + return None; + } + visiting.insert(identity.clone()); + let mut premises_proofs = Vec::new(); + let mut premises_truncated = false; + for premise in premises { + match proof_for_reference(relations, premise, budget, options, depth + 1, visiting) { + Some(proof) => premises_proofs.push(proof), + None => { + premises_truncated = true; + break; + } + } + } + visiting.remove(&identity); + if premises_truncated { + Some(OhProjectionProof::Truncated { + reason: "nodes".to_string(), + relation: reference.relation.clone(), + tuple: reference.tuple.clone(), + v: 1, + }) + } else { + Some(OhProjectionProof::Derived { + premises: premises_proofs, + premises_truncated: false, + relation: reference.relation.clone(), + rule_id: rule.rule_id.clone(), + rule_sha256: rule.rule_sha256.clone(), + tuple: reference.tuple.clone(), + v: 1, + }) + } + } + } +} + +/// Evaluate a positive-Datalog projection. +pub fn evaluate_projection( + dataset: &OhProjectionDataset, + rule_pack: &OhProjectionRulePack, + query: &OhProjectionQuery, + options: OhProjectionEvaluationOptions, +) -> Result { + let options = resolve_evaluation_options(options)?; + let mut work = WorkBudget::new(options.maximum_work_units); + + let materialized = materialize_naive( + dataset, + rule_pack, + options.maximum_derived_tuples, + options.maximum_rounds, + &mut work, + )?; + + let mut result_budget = ResultBudget { + bytes: 0, + maximum_bytes: options.maximum_result_bytes, + nodes: 0, + }; + let mut rows = Vec::new(); + let mut rows_truncated = false; + + let matches = match_body( + &materialized.relations, + &query.where_, + LIMITS.query_matches, + &mut work, + )?; + for match_ in matches { + let values = query + .find + .iter() + .map(|name| match_.binding.get(name).cloned().unwrap_or(Value::Null)) + .collect::>(); + + let mut proof_budget = ProofBudget { + nodes: 0, + result: &mut result_budget, + options: &options, + }; + let mut visiting = HashSet::new(); + let proofs = match_ + .premises + .iter() + .map(|reference| { + proof_for_reference( + &materialized.relations, + reference, + &mut proof_budget, + &options, + 0, + &mut visiting, + ) + }) + .collect::>>(); + + let proofs_truncated = proofs.is_none(); + let row = OhProjectionResultRow { + proofs: proofs.unwrap_or_default(), + proofs_truncated, + support_count: 1, + values, + v: 1, + }; + + if rows.len() as u64 >= LIMITS.query_results { + rows_truncated = true; + break; + } + rows.push(row); + } + + let rows_total = rows.len() as u64; + + let provenance = Provenance { + contract: ContractInfo { + contract_sha256: "oh.contract-manifest.v1".to_string(), + v: 1, + }, + dataset_sha256: dataset.dataset_sha256.clone(), + engine_sha256: "oh-datalog.rust.v1".to_string(), + evaluation_sha256: canonical_sha256_str(&serde_json::to_string(&serde_json::json!({ + "maximumDerivedTuples": options.maximum_derived_tuples, + "maximumProofDepth": options.maximum_proof_depth, + "maximumProofNodes": options.maximum_proof_nodes, + "maximumResultBytes": options.maximum_result_bytes, + "maximumRounds": options.maximum_rounds, + "maximumTotalProofNodes": options.maximum_total_proof_nodes, + "maximumWorkUnits": options.maximum_work_units, + })).unwrap()).unwrap_or_default(), + query_sha256: query.query_sha256.clone(), + rule_pack_sha256: rule_pack.rule_pack_sha256.clone(), + snapshot_sha256: dataset.snapshot_sha256.clone(), + }; + + Ok(OhProjectionResult { + authority: "derived".to_string(), + cache: CacheInfo { + strategy: "full-rebuild".to_string(), + v: 1, + }, + engine: "oh.projection.rust.v1".to_string(), + evaluation: options, + facts: FactCounts { + base: materialized.base_facts, + derived: materialized.derived_facts, + }, + output: ResultRows { + rows, + rows_truncated, + rows_total, + }, + provenance, + query: QuerySummary { + find: query.find.clone(), + query_id: query.query_id.clone(), + query_sha256: query.query_sha256.clone(), + v: 1, + }, + semantics: "oh.projection.positive-datalog.v1".to_string(), + v: 1, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn atom(value: &str) -> OhProjectionAtom { + Value::String(value.to_string()) + } + + #[allow(dead_code)] + fn constant(value: &str) -> OhProjectionTerm { + OhProjectionTerm::Constant { v: 1, value: atom(value) } + } + + fn variable(name: &str) -> OhProjectionTerm { + OhProjectionTerm::Variable { name: name.to_string(), v: 1 } + } + + fn literal(relation: &str, terms: Vec) -> OhProjectionLiteral { + OhProjectionLiteral { + relation: relation.to_string(), + terms, + v: 1, + } + } + + fn fact(relation: &str, tuple: Vec<&str>, sources: Vec<(&str, &str)>) -> OhProjectionFact { + let sources = sources + .into_iter() + .map(|(key, record_sha256)| OhProjectionFactSource { + key: key.to_string(), + record_sha256: record_sha256.to_string(), + v: 1, + }) + .collect(); + let tuple = tuple.into_iter().map(atom).collect(); + let payload = serde_json::json!({ + "relation": relation, + "sources": sources, + "tuple": tuple, + "v": 1, + }); + let fact_sha256 = canonical_sha256_str(&payload.to_string()).unwrap(); + OhProjectionFact { + fact_sha256, + relation: relation.to_string(), + sources, + tuple, + v: 1, + } + } + + fn rule(id: &str, body: Vec, head: OhProjectionLiteral) -> OhProjectionRule { + let payload = serde_json::json!({ + "body": body, + "head": head, + "ruleId": id, + "v": 1, + }); + let rule_sha256 = canonical_sha256_str(&payload.to_string()).unwrap(); + OhProjectionRule { + body, + head, + rule_id: id.to_string(), + rule_sha256, + v: 1, + } + } + + fn dataset(facts: Vec) -> OhProjectionDataset { + let extractor_sha256 = canonical_sha256_str(r#"{"factPackId":"test","v":1}"#).unwrap(); + let fact_pack_sha256 = canonical_sha256_str(r#"{"factPackId":"test","v":1}"#).unwrap(); + let facts_sha256 = canonical_sha256_str(&serde_json::to_string(&facts).unwrap()).unwrap(); + let snapshot_sha256 = canonical_sha256_str(r#"{"v":1}"#).unwrap(); + let payload = serde_json::json!({ + "extractorSha256": extractor_sha256, + "factPackId": "test", + "factPackRevision": 1, + "factPackSha256": fact_pack_sha256, + "facts": facts, + "factsSha256": facts_sha256, + "snapshotSha256": snapshot_sha256, + "v": 1, + }); + let dataset_sha256 = canonical_sha256_str(&payload.to_string()).unwrap(); + OhProjectionDataset { + dataset_sha256, + extractor_sha256, + fact_pack_id: "test".to_string(), + fact_pack_revision: 1, + fact_pack_sha256, + facts, + facts_sha256, + snapshot_sha256, + v: 1, + } + } + + fn rule_pack(rules: Vec) -> OhProjectionRulePack { + let rules_sha256 = canonical_sha256_str(&serde_json::to_string(&rules).unwrap()).unwrap(); + let payload = serde_json::json!({ + "rulePackId": "test", + "rulePackRevision": 1, + "rulePackSha256": "x", + "rules": rules, + "rulesSha256": rules_sha256, + "semantics": "oh.projection.positive-datalog.v1", + "v": 1, + }); + let rule_pack_sha256 = canonical_sha256_str(&payload.to_string()).unwrap(); + OhProjectionRulePack { + rule_pack_id: "test".to_string(), + rule_pack_revision: 1, + rule_pack_sha256, + rules, + rules_sha256, + semantics: "oh.projection.positive-datalog.v1".to_string(), + v: 1, + } + } + + fn query(where_: Vec, find: Vec<&str>) -> OhProjectionQuery { + let query_sha256 = canonical_sha256_str(&serde_json::json!({ + "find": find, + "limit": 100, + "queryId": "test", + "where": where_, + "v": 1, + }).to_string()).unwrap(); + OhProjectionQuery { + find: find.into_iter().map(|s| s.to_string()).collect(), + limit: 100, + query_id: "test".to_string(), + query_sha256, + where_, + v: 1, + } + } + + #[test] + fn derives_transitive_closure() { + let dataset = dataset(vec![ + fact("edge", vec!["a", "b"], vec![("s1", "sha1")]), + fact("edge", vec!["b", "c"], vec![("s2", "sha2")]), + fact("edge", vec!["c", "d"], vec![("s3", "sha3")]), + ]); + let rules = rule_pack(vec![ + rule( + "path-direct", + vec![literal("edge", vec![variable("x"), variable("y")])], + literal("path", vec![variable("x"), variable("y")]), + ), + rule( + "path-indirect", + vec![ + literal("edge", vec![variable("x"), variable("z")]), + literal("path", vec![variable("z"), variable("y")]), + ], + literal("path", vec![variable("x"), variable("y")]), + ), + ]); + let query = query( + vec![literal("path", vec![variable("x"), variable("y")])], + vec!["x", "y"], + ); + + let result = evaluate_projection(&dataset, &rules, &query, OhProjectionEvaluationOptions { + maximum_derived_tuples: None, + maximum_proof_depth: None, + maximum_proof_nodes: None, + maximum_result_bytes: None, + maximum_rounds: None, + maximum_total_proof_nodes: None, + maximum_work_units: None, + }).unwrap(); + + assert_eq!(result.facts.derived, 6); // a->b, a->c, a->d, b->c, b->d, c->d + assert_eq!(result.output.rows.len(), 6); + } + + #[test] + fn enforces_work_unit_budget() { + let dataset = dataset(vec![ + fact("edge", vec!["a", "b"], vec![("s1", "sha1")]), + fact("edge", vec!["b", "c"], vec![("s2", "sha2")]), + ]); + let rules = rule_pack(vec![rule( + "path", + vec![literal("edge", vec![variable("x"), variable("y")])], + literal("path", vec![variable("x"), variable("y")]), + )]); + let query = query( + vec![literal("path", vec![variable("x"), variable("y")])], + vec!["x", "y"], + ); + + let result = evaluate_projection(&dataset, &rules, &query, OhProjectionEvaluationOptions { + maximum_work_units: Some(1), + ..Default::default() + }); + + assert!(matches!(result, Err(ProjectionError::WorkBudgetExceeded { .. }))); + } +} From cac2e2cf0251c106b90d303939351f371ab72122 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Thu, 17 Sep 2026 15:42:55 -0400 Subject: [PATCH 5/5] Add strict in-memory ZIP reader and raw-ABI WASM artifact - oh-archive::strict ports the Textbutler X-archive ZIP contract to Rust: EOCD/ZIP64 end records, central-directory validation, local-header consistency, data-descriptor checks, CRC-32, compression-ratio caps, NFC names, and pattern-selected in-memory extraction. - oh-archive-strict-wasm exposes it through a raw extern-C ABI with no wasm-bindgen imports, so consumers can vendor the .wasm binary directly. - oh-archive-wasm gains read_zip_entries_strict_js returning { name, bytes } objects for bindgen-based callers. --- rust/Cargo.lock | 35 + rust/Cargo.toml | 2 +- rust/oh-archive-strict-wasm/Cargo.toml | 16 + rust/oh-archive-strict-wasm/src/lib.rs | 96 +++ rust/oh-archive-wasm/Cargo.toml | 4 +- rust/oh-archive-wasm/src/lib.rs | 23 + rust/oh-archive/Cargo.toml | 3 + rust/oh-archive/src/lib.rs | 2 + rust/oh-archive/src/strict.rs | 851 +++++++++++++++++++++++++ 9 files changed, 1030 insertions(+), 2 deletions(-) create mode 100644 rust/oh-archive-strict-wasm/Cargo.toml create mode 100644 rust/oh-archive-strict-wasm/src/lib.rs create mode 100644 rust/oh-archive/src/strict.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 282c502..2b2e583 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -168,6 +168,7 @@ checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -353,10 +354,13 @@ dependencies = [ name = "oh-archive" version = "0.1.0" dependencies = [ + "crc32fast", + "flate2", "regex", "serde", "serde_json", "tempfile", + "unicode-normalization", "zip", ] @@ -370,11 +374,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "oh-archive-strict-wasm" +version = "0.1.0" +dependencies = [ + "oh-archive", + "serde_json", +] + [[package]] name = "oh-archive-wasm" version = "0.1.0" dependencies = [ + "js-sys", "oh-archive", + "serde", "serde_json", "wasm-bindgen", ] @@ -678,6 +692,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "typenum" version = "1.20.1" @@ -690,6 +710,15 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -781,6 +810,12 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 6abca40..a25f72a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi", "oh-sqlite", "oh-sqlite-napi", "oh-datalog", "oh-datalog-wasm"] +members = ["oh-canonical", "oh-canonical-wasm", "oh-canonical-napi", "oh-archive", "oh-archive-wasm", "oh-archive-napi", "oh-archive-strict-wasm", "oh-sqlite", "oh-sqlite-napi", "oh-datalog", "oh-datalog-wasm"] resolver = "3" [workspace.package] diff --git a/rust/oh-archive-strict-wasm/Cargo.toml b/rust/oh-archive-strict-wasm/Cargo.toml new file mode 100644 index 0000000..0243f6e --- /dev/null +++ b/rust/oh-archive-strict-wasm/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "oh-archive-strict-wasm" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Raw-ABI WASM artifact for the strict oh-archive reader." + +[lib] +crate-type = ["cdylib"] + +[dependencies] +oh-archive = { path = "../oh-archive" } +serde_json.workspace = true diff --git a/rust/oh-archive-strict-wasm/src/lib.rs b/rust/oh-archive-strict-wasm/src/lib.rs new file mode 100644 index 0000000..7706d23 --- /dev/null +++ b/rust/oh-archive-strict-wasm/src/lib.rs @@ -0,0 +1,96 @@ +//! Raw-ABI WASM artifact for the strict oh-archive reader. +//! +//! This crate intentionally avoids wasm-bindgen so the produced `.wasm` can be +//! vendored as a plain binary and instantiated with `WebAssembly` directly. +//! +//! ABI: +//! - `oh_archive_alloc(len)` returns a pointer to a `len`-byte buffer. +//! - `oh_archive_free(ptr, len)` releases a buffer returned by `alloc` or by +//! `oh_archive_read_strict` (for the latter, `len` is the capacity header). +//! - `oh_archive_read_strict(archive_ptr, archive_len, options_ptr, +//! options_len)` returns a pointer to a result buffer laid out as: +//! `[u32le capacity][u32le status][u32le payload_len][payload]` where +//! `capacity` is the total buffer size (header included), `status` is 0 for +//! success or 1 for an error, and `payload` is either the packed entries or +//! a UTF-8 error message. Packed entries are `[u32le count]` followed by +//! `count` repetitions of `[u32le name_len][name][u64le data_len][data]`. + +use oh_archive::strict::{read_zip_entries_strict, StrictZipOptions}; + +#[unsafe(no_mangle)] +pub extern "C" fn oh_archive_alloc(len: usize) -> *mut u8 { + if len == 0 { + return std::ptr::null_mut(); + } + let layout = std::alloc::Layout::array::(len).unwrap(); + unsafe { std::alloc::alloc(layout) } +} + +/// # Safety +/// `ptr` must come from `oh_archive_alloc` or `oh_archive_read_strict`, and +/// `len` must be the exact capacity originally allocated. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn oh_archive_free(ptr: *mut u8, len: usize) { + if ptr.is_null() || len == 0 { + return; + } + let layout = std::alloc::Layout::array::(len).unwrap(); + unsafe { std::alloc::dealloc(ptr, layout) } +} + +fn result_buffer(status: u32, payload: &[u8]) -> *mut u8 { + let capacity = 12usize.saturating_add(payload.len()); + let ptr = oh_archive_alloc(capacity); + if ptr.is_null() { + return std::ptr::null_mut(); + } + let header = [ + (capacity as u32).to_le_bytes(), + status.to_le_bytes(), + (payload.len() as u32).to_le_bytes(), + ] + .concat(); + unsafe { + std::ptr::copy_nonoverlapping(header.as_ptr(), ptr, 12); + if !payload.is_empty() { + std::ptr::copy_nonoverlapping(payload.as_ptr(), ptr.add(12), payload.len()); + } + } + ptr +} + +fn pack_entries(entries: &[oh_archive::strict::StrictZipEntry]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + for entry in entries { + out.extend_from_slice(&(entry.name.len() as u32).to_le_bytes()); + out.extend_from_slice(entry.name.as_bytes()); + out.extend_from_slice(&(entry.bytes.len() as u64).to_le_bytes()); + out.extend_from_slice(&entry.bytes); + } + out +} + +/// # Safety +/// Both `(archive_ptr, archive_len)` and `(options_ptr, options_len)` must +/// point at valid readable buffers inside the caller's WASM memory. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn oh_archive_read_strict( + archive_ptr: *const u8, + archive_len: usize, + options_ptr: *const u8, + options_len: usize, +) -> *mut u8 { + let archive = unsafe { std::slice::from_raw_parts(archive_ptr, archive_len) }; + let options_bytes = unsafe { std::slice::from_raw_parts(options_ptr, options_len) }; + let options: StrictZipOptions = match serde_json::from_slice(options_bytes) { + Ok(options) => options, + Err(error) => { + return result_buffer(1, format!("invalid options JSON: {error}").as_bytes()); + } + }; + match read_zip_entries_strict(archive, &options) { + Ok(entries) => result_buffer(0, &pack_entries(&entries)), + Err(error) => result_buffer(1, error.to_string().as_bytes()), + } +} diff --git a/rust/oh-archive-wasm/Cargo.toml b/rust/oh-archive-wasm/Cargo.toml index 67d884d..bc3dfc4 100644 --- a/rust/oh-archive-wasm/Cargo.toml +++ b/rust/oh-archive-wasm/Cargo.toml @@ -10,8 +10,10 @@ description = "WASM bindings for oh-archive." [dependencies] oh-archive = { path = "../oh-archive" } -wasm-bindgen = "0.2" +wasm-bindgen.workspace = true +serde.workspace = true serde_json.workspace = true +js-sys = "0.3" [lib] crate-type = ["cdylib"] diff --git a/rust/oh-archive-wasm/src/lib.rs b/rust/oh-archive-wasm/src/lib.rs index e11bc39..1474696 100644 --- a/rust/oh-archive-wasm/src/lib.rs +++ b/rust/oh-archive-wasm/src/lib.rs @@ -1,3 +1,4 @@ +use oh_archive::strict::{read_zip_entries_strict, StrictZipOptions}; use oh_archive::{extract_zip_entries, ExtractError, ExtractOptions}; use wasm_bindgen::prelude::*; @@ -14,6 +15,28 @@ pub fn extract_zip_entries_json(options_json: &str) -> String { } } +/// Strictly validate an in-memory ZIP/ZIP64 archive and return the selected +/// members as an array of `{ name, bytes }` objects. +#[wasm_bindgen] +pub fn read_zip_entries_strict_js(archive: &[u8], options_json: &str) -> Result { + let options: StrictZipOptions = serde_json::from_str(options_json) + .map_err(|e| JsValue::from_str(&format!("invalid options JSON: {e}")))?; + let entries = read_zip_entries_strict(archive, &options) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + let result = js_sys::Array::new_with_length(entries.len() as u32); + for (index, entry) in entries.iter().enumerate() { + let object = js_sys::Object::new(); + js_sys::Reflect::set(&object, &JsValue::from_str("name"), &JsValue::from_str(&entry.name))?; + js_sys::Reflect::set( + &object, + &JsValue::from_str("bytes"), + &js_sys::Uint8Array::from(entry.bytes.as_slice()), + )?; + result.set(index as u32, object.into()); + } + Ok(result) +} + fn serialize_error(error: ExtractError) -> String { serde_json::to_string(&error).unwrap_or_else(|_| r#"{"error":"serialization_failed"}"#.to_string()) } diff --git a/rust/oh-archive/Cargo.toml b/rust/oh-archive/Cargo.toml index cbed6ab..0fc1e9d 100644 --- a/rust/oh-archive/Cargo.toml +++ b/rust/oh-archive/Cargo.toml @@ -13,6 +13,9 @@ serde.workspace = true serde_json.workspace = true regex = "1" zip = { version = "2", default-features = false, features = ["deflate"] } +flate2 = "1" +crc32fast = "1" +unicode-normalization = "0.1" [dev-dependencies] tempfile = "3" diff --git a/rust/oh-archive/src/lib.rs b/rust/oh-archive/src/lib.rs index a3cec19..be390ff 100644 --- a/rust/oh-archive/src/lib.rs +++ b/rust/oh-archive/src/lib.rs @@ -13,6 +13,8 @@ use std::path::{Path, PathBuf}; use zip::result::ZipError; use zip::ZipArchive; +pub mod strict; + /// Options controlling archive extraction. #[derive(Debug, Clone, Deserialize)] pub struct ExtractOptions { diff --git a/rust/oh-archive/src/strict.rs b/rust/oh-archive/src/strict.rs new file mode 100644 index 0000000..983dd6c --- /dev/null +++ b/rust/oh-archive/src/strict.rs @@ -0,0 +1,851 @@ +//! Strict in-memory ZIP/ZIP64 reader. +//! +//! This is a Rust port of the bounded X-archive ZIP reader used by Textbutler +//! (`src/x-archive-zip.ts`). It validates the complete archive structure — +//! end records, central directory, local headers, data descriptors, CRC-32, +//! compression-ratio and path safety — and returns only the selected members' +//! uncompressed bytes in memory. Nothing is written to disk. + +use flate2::read::DeflateDecoder; +use regex::RegexSet; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use unicode_normalization::is_nfc; + +const EOCD: u32 = 0x0605_4b50; +const ZIP64_EOCD: u32 = 0x0606_4b50; +const ZIP64_LOCATOR: u32 = 0x0706_4b50; +const CENTRAL: u32 = 0x0201_4b50; +const LOCAL: u32 = 0x0403_4b50; +const DESCRIPTOR: u32 = 0x0807_4b50; +const ZIP64_EXTRA: u16 = 0x0001; +const STORED: u16 = 0; +const DEFLATE: u16 = 8; +const DESCRIPTOR_FLAG: u16 = 0x0008; +const UTF8_FLAG: u16 = 0x0800; +const ENCRYPTED_FLAG: u16 = 0x0001; +const STRONG_ENCRYPTION_FLAG: u16 = 0x0040; +const MASKED_HEADER_FLAG: u16 = 0x2000; +const DEFLATE_OPTION_FLAGS: u16 = 0x0006; +const UNIX_HOST: u16 = 3; +const MACOS_HOST: u16 = 19; +const UNIX_TYPE_MASK: u32 = 0o170000; +const UNIX_REGULAR: u32 = 0o100000; +const UNIX_DIRECTORY: u32 = 0o040000; +const DOS_DIRECTORY: u32 = 0x10; +const U16_MAX: u16 = 0xffff; +const U32_MAX: u32 = 0xffff_ffff; + +/// Bounds enforced while reading a strict archive. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StrictZipLimits { + /// Maximum archive size in bytes. + pub max_archive_bytes: u64, + /// Maximum uncompressed size of a selected member. + pub max_member_bytes: u64, + /// Maximum compressed size of a selected member. + pub max_compressed_member_bytes: u64, + /// Maximum number of central-directory entries. + pub max_entries: u64, + /// Maximum central-directory byte size. + pub max_central_bytes: u64, + /// Maximum total uncompressed bytes across selected members. + pub max_total_selected_bytes: u64, + /// Maximum total declared uncompressed bytes across all entries. + pub max_total_declared_bytes: u64, + /// Maximum member-name byte length. + pub max_name_bytes: usize, + /// Maximum uncompressed-to-compressed ratio for selected members. + pub max_ratio: u64, +} + +impl Default for StrictZipLimits { + fn default() -> Self { + Self { + max_archive_bytes: 16 * 1024 * 1024 * 1024, + max_member_bytes: 256 * 1024 * 1024, + max_compressed_member_bytes: 64 * 1024 * 1024, + max_entries: 100_000, + max_central_bytes: 64 * 1024 * 1024, + max_total_selected_bytes: 768 * 1024 * 1024, + max_total_declared_bytes: 64 * 1024 * 1024 * 1024, + max_name_bytes: 4 * 1024, + max_ratio: 200, + } + } +} + +/// Options controlling strict archive reads. +#[derive(Debug, Clone, Deserialize)] +pub struct StrictZipOptions { + /// Regex patterns; only entries whose full name matches at least one + /// pattern are decoded and returned. + pub patterns: Vec, + #[serde(default)] + pub limits: StrictZipLimits, +} + +/// A selected member's bytes. +#[derive(Debug, Clone, Serialize)] +pub struct StrictZipEntry { + /// Full member name inside the archive. + pub name: String, + /// Uncompressed bytes. + pub bytes: Vec, +} + +/// Errors returned by strict archive reads. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "error", rename_all = "snake_case")] +pub enum StrictZipError { + Invalid { message: String }, + Limit { message: String }, +} + +impl std::fmt::Display for StrictZipError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StrictZipError::Invalid { message } | StrictZipError::Limit { message } => { + f.write_str(message) + } + } + } +} + +impl std::error::Error for StrictZipError {} + +type Result = std::result::Result; + +fn invalid(message: impl Into) -> StrictZipError { + StrictZipError::Invalid { message: message.into() } +} + +#[derive(Debug, Clone, Copy)] +struct Directory { + count: u64, + offset: usize, + end: usize, +} + +#[derive(Debug, Clone)] +struct Entry { + name: String, + name_bytes: Vec, + directory: bool, + selected: bool, + version_made_by: u16, + version_needed: u16, + flags: u16, + method: u16, + modified_time: u16, + modified_date: u16, + crc32: u32, + compressed_size: u64, + uncompressed_size: u64, + external_attributes: u32, + local_header_offset: u64, +} + +#[derive(Debug, Clone, Copy)] +struct LocalRange { + data_offset: usize, + data_end: usize, +} + +fn checked_end(offset: usize, length: usize, label: &str) -> Result { + offset + .checked_add(length) + .ok_or_else(|| invalid(format!("X ZIP {label} has invalid bounds"))) +} + +fn read_exact<'a>(archive: &'a [u8], offset: usize, length: usize, label: &str) -> Result<&'a [u8]> { + let end = checked_end(offset, length, label)?; + archive + .get(offset..end) + .ok_or_else(|| invalid(format!("X ZIP {label} is truncated"))) +} + +fn u16_le(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes([bytes[offset], bytes[offset + 1]]) +} + +fn u32_le(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes([bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]]) +} + +fn u64_field(bytes: &[u8], offset: usize, label: &str) -> Result { + if bytes.len().saturating_sub(offset) < 8 { + return Err(invalid(format!("X ZIP {label} is truncated"))); + } + Ok(u64::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ])) +} + +fn field_value(legacy: u64, resolved: u64, sentinel: u64, label: &str) -> Result<()> { + if legacy != sentinel && legacy != resolved { + return Err(invalid(format!("X ZIP legacy {label} contradicts ZIP64 metadata"))); + } + Ok(()) +} + +fn directory_from_eocd(archive: &[u8], limits: &StrictZipLimits) -> Result { + let archive_size = archive.len(); + let tail_length = archive_size.min(22 + U16_MAX as usize + 20); + let tail_offset = archive_size - tail_length; + let tail = read_exact(archive, tail_offset, tail_length, "end records")?; + let mut candidates: Vec = Vec::new(); + for offset in (0..=tail.len() - 22).rev() { + if u32_le(tail, offset) == EOCD + && tail_offset + offset + 22 + u16_le(tail, offset + 20) as usize == archive_size + { + candidates.push(tail_offset + offset); + } + } + if candidates.len() != 1 { + return Err(invalid(if candidates.is_empty() { + "X ZIP end-of-central-directory record is missing".to_string() + } else { + "X ZIP end-of-central-directory record is ambiguous".to_string() + })); + } + let eocd_offset = candidates[0]; + let eocd = read_exact(archive, eocd_offset, archive_size - eocd_offset, "end-of-central-directory record")?; + if eocd.len() != 22 || u16_le(eocd, 20) != 0 { + return Err(invalid("X ZIP archive comments are not supported")); + } + let legacy_disk = u16_le(eocd, 4); + let legacy_central_disk = u16_le(eocd, 6); + let legacy_on_disk = u16_le(eocd, 8); + let legacy_count = u16_le(eocd, 10); + let legacy_size = u32_le(eocd, 12); + let legacy_offset = u32_le(eocd, 16); + let locator_offset = eocd_offset.wrapping_sub(20); + let has_zip64 = eocd_offset >= 20 + && read_exact(archive, locator_offset, 4, "ZIP64 locator signature") + .map(|b| u32_le(b, 0) == ZIP64_LOCATOR) + .unwrap_or(false); + if !has_zip64 { + if [legacy_disk, legacy_central_disk, legacy_on_disk, legacy_count].contains(&U16_MAX) + || [legacy_size, legacy_offset].contains(&U32_MAX) + { + return Err(invalid("X ZIP archive is missing required ZIP64 end metadata")); + } + if legacy_disk != 0 || legacy_central_disk != 0 || legacy_on_disk != legacy_count { + return Err(invalid("X ZIP multi-disk archives are not supported")); + } + if legacy_count < 1 + || legacy_count as u64 > limits.max_entries + || legacy_size as u64 > limits.max_central_bytes + { + return Err(invalid("X ZIP central directory exceeds its bounds")); + } + let end = checked_end(legacy_offset as usize, legacy_size as usize, "central directory")?; + if end != eocd_offset { + return Err(invalid("X ZIP central directory has invalid bounds")); + } + return Ok(Directory { + count: legacy_count as u64, + offset: legacy_offset as usize, + end, + }); + } + let locator = read_exact(archive, locator_offset, 20, "ZIP64 locator")?; + if u32_le(locator, 4) != 0 || u32_le(locator, 16) != 1 { + return Err(invalid("X ZIP multi-disk archives are not supported")); + } + let zip64_offset = u64_field(locator, 8, "ZIP64 end record offset")? as usize; + let zip64 = read_exact(archive, zip64_offset, 56, "ZIP64 end record")?; + if u32_le(zip64, 0) != ZIP64_EOCD || u64_field(zip64, 4, "ZIP64 end record size")? != 44 { + return Err(invalid("X ZIP64 end record has an unsupported shape")); + } + if zip64_offset + zip64.len() != locator_offset + || u32_le(zip64, 16) != 0 + || u32_le(zip64, 20) != 0 + { + return Err(invalid("X ZIP64 end record has invalid bounds or disk ownership")); + } + let on_disk = u64_field(zip64, 24, "ZIP64 entries on disk")?; + let count = u64_field(zip64, 32, "ZIP64 entry count")?; + let size = u64_field(zip64, 40, "ZIP64 central directory size")?; + let offset = u64_field(zip64, 48, "ZIP64 central directory offset")?; + if on_disk != count { + return Err(invalid("X ZIP multi-disk archives are not supported")); + } + if count < 1 || count > limits.max_entries || size > limits.max_central_bytes { + return Err(invalid("X ZIP central directory exceeds its bounds")); + } + let end = checked_end(offset as usize, size as usize, "ZIP64 central directory")?; + if end != zip64_offset { + return Err(invalid("X ZIP64 central directory has invalid bounds")); + } + field_value(legacy_disk as u64, 0, U16_MAX as u64, "disk number")?; + field_value(legacy_central_disk as u64, 0, U16_MAX as u64, "central disk number")?; + field_value(legacy_on_disk as u64, on_disk, U16_MAX as u64, "entries-on-disk count")?; + field_value(legacy_count as u64, count, U16_MAX as u64, "entry count")?; + field_value(legacy_size as u64, size, U32_MAX as u64, "central directory size")?; + field_value(legacy_offset as u64, offset, U32_MAX as u64, "central directory offset")?; + Ok(Directory { + count, + offset: offset as usize, + end, + }) +} + +fn extra_fields<'a>(bytes: &'a [u8], label: &str) -> Result> { + let mut fields: HashMap = HashMap::new(); + let mut position = 0usize; + while position < bytes.len() { + if bytes.len() - position < 4 { + return Err(invalid(format!("X ZIP {label} contains a truncated extra field"))); + } + let id = u16_le(bytes, position); + let length = u16_le(bytes, position + 2) as usize; + let next = checked_end(position + 4, length, &format!("{label} extra field"))?; + if next > bytes.len() { + return Err(invalid(format!("X ZIP {label} contains a truncated extra field"))); + } + if fields.insert(id, &bytes[position + 4..next]).is_some() { + return Err(invalid(format!("X ZIP {label} contains duplicate extra field {id}"))); + } + position = next; + } + Ok(fields) +} + +fn decode_name(name_bytes: &[u8], flags: u16, selected: bool, limits: &StrictZipLimits) -> Result<(String, bool)> { + if name_bytes.is_empty() || name_bytes.len() > limits.max_name_bytes { + return Err(invalid("X ZIP member name exceeds its bounds")); + } + let decoded: String = if (flags & UTF8_FLAG) != 0 { + String::from_utf8(name_bytes.to_vec()) + .map_err(|_| invalid("X ZIP member name is not valid UTF-8"))? + } else { + if name_bytes.iter().any(|byte| *byte > 0x7f) { + return Err(invalid("X ZIP non-UTF-8 member names must be ASCII")); + } + name_bytes.iter().map(|b| *b as char).collect() + }; + if !is_nfc(decoded.as_str()) { + return Err(invalid("X ZIP member name is not NFC-normalized")); + } + let bytes_decoded = decoded.as_bytes(); + if bytes_decoded.len() >= 3 + && bytes_decoded[0].is_ascii_alphabetic() + && bytes_decoded[1] == b':' + && bytes_decoded[2] == b'/' + || decoded.starts_with('/') + { + return Err(invalid("X ZIP member has an absolute name")); + } + if decoded.contains('\\') { + return Err(invalid("X ZIP member name contains a backslash")); + } + if decoded.chars().any(|c| (c as u32) < 0x20 || (0x7f..=0x9f).contains(&(c as u32))) { + return Err(invalid("X ZIP member name contains a control character")); + } + let directory = decoded.ends_with('/'); + let path = if directory { &decoded[..decoded.len() - 1] } else { decoded.as_str() }; + let parts: Vec<&str> = path.split('/').collect(); + if parts.iter().any(|part| *part == "." || *part == "..") + || (selected && parts.iter().any(|part| part.is_empty())) + { + return Err(invalid("X ZIP selected member has an unsafe path component")); + } + Ok((decoded, directory)) +} + +fn validate_flags(flags: u16, method: u16) -> Result<()> { + if (flags & (ENCRYPTED_FLAG | STRONG_ENCRYPTION_FLAG)) != 0 { + return Err(invalid("X ZIP encrypted members are not supported")); + } + if (flags & MASKED_HEADER_FLAG) != 0 { + return Err(invalid("X ZIP members with masked local headers are not supported")); + } + let allowed = UTF8_FLAG | DESCRIPTOR_FLAG | if method == DEFLATE { DEFLATE_OPTION_FLAGS } else { 0 }; + if (flags & !allowed) != 0 { + return Err(invalid("X ZIP member uses unsupported general-purpose flags")); + } + Ok(()) +} + +fn validate_type(entry: &Entry) -> Result<()> { + let host = entry.version_made_by >> 8; + let unix_type = if host == UNIX_HOST || host == MACOS_HOST { + (entry.external_attributes >> 16) & UNIX_TYPE_MASK + } else { + 0 + }; + if entry.directory { + if entry.crc32 != 0 + || entry.uncompressed_size != 0 + || (entry.method == STORED && entry.compressed_size != 0) + { + return Err(invalid("X ZIP directory member must expand to empty data")); + } + if unix_type != 0 && unix_type != UNIX_DIRECTORY { + return Err(invalid("X ZIP member is a symlink or another non-regular file")); + } + } else if (entry.external_attributes & DOS_DIRECTORY) != 0 + || (unix_type != 0 && unix_type != UNIX_REGULAR) + { + return Err(invalid("X ZIP member is not a regular file")); + } + Ok(()) +} + +fn central_entries( + archive: &[u8], + directory: Directory, + selection: &RegexSet, + limits: &StrictZipLimits, +) -> Result> { + let bytes = read_exact(archive, directory.offset, directory.end - directory.offset, "central directory")?; + let mut entries: Vec = Vec::new(); + let mut names: HashSet = HashSet::new(); + let mut declared: u64 = 0; + let mut selected_total: u64 = 0; + let mut position = 0usize; + for index in 0..directory.count { + if bytes.len() - position < 46 || u32_le(bytes, position) != CENTRAL { + return Err(invalid("X ZIP central directory entry is invalid or truncated")); + } + let version_made_by = u16_le(bytes, position + 4); + let version_needed = u16_le(bytes, position + 6); + let flags = u16_le(bytes, position + 8); + let method = u16_le(bytes, position + 10); + let modified_time = u16_le(bytes, position + 12); + let modified_date = u16_le(bytes, position + 14); + let checksum = u32_le(bytes, position + 16); + let compressed_legacy = u32_le(bytes, position + 20); + let uncompressed_legacy = u32_le(bytes, position + 24); + let name_length = u16_le(bytes, position + 28) as usize; + let extra_length = u16_le(bytes, position + 30) as usize; + let comment_length = u16_le(bytes, position + 32) as usize; + let disk_legacy = u16_le(bytes, position + 34); + let external_attributes = u32_le(bytes, position + 38); + let offset_legacy = u32_le(bytes, position + 42); + let next = checked_end( + position + 46, + name_length + extra_length + comment_length, + "central directory entry", + )?; + if next > bytes.len() { + return Err(invalid("X ZIP central directory entry is truncated")); + } + let name_bytes = bytes[position + 46..position + 46 + name_length].to_vec(); + let extra_start = position + 46 + name_length; + let fields = extra_fields( + &bytes[extra_start..extra_start + extra_length], + &format!("central directory entry {}", index + 1), + )?; + let zip64 = fields.get(&ZIP64_EXTRA).copied(); + let mut zip64_position = 0usize; + let mut next_zip64 = |label: &str| -> Result { + let zip64 = zip64.ok_or_else(|| invalid(format!("X ZIP {label} is missing ZIP64 metadata")))?; + let value = u64_field(zip64, zip64_position, label)?; + zip64_position += 8; + Ok(value) + }; + let uncompressed_size = if uncompressed_legacy == U32_MAX { + next_zip64("uncompressed size")? + } else { + uncompressed_legacy as u64 + }; + let compressed_size = if compressed_legacy == U32_MAX { + next_zip64("compressed size")? + } else { + compressed_legacy as u64 + }; + let local_header_offset = if offset_legacy == U32_MAX { + next_zip64("local header offset")? + } else { + offset_legacy as u64 + }; + let mut disk = disk_legacy; + if disk_legacy == U16_MAX { + let zip64_bytes = zip64 + .ok_or_else(|| invalid("X ZIP disk number is missing ZIP64 metadata"))?; + if zip64_bytes.len() - zip64_position < 4 { + return Err(invalid("X ZIP disk number is missing ZIP64 metadata")); + } + disk = u32_le(zip64_bytes, zip64_position) as u16; + zip64_position += 4; + } + if (zip64.is_none() && zip64_position != 0) + || (zip64.is_some() && zip64_position != zip64.map(|z| z.len()).unwrap_or(0)) + { + return Err(invalid("X ZIP central directory contains ambiguous ZIP64 metadata")); + } + if disk != 0 { + return Err(invalid("X ZIP multi-disk archives are not supported")); + } + validate_flags(flags, method)?; + if method != STORED && method != DEFLATE { + return Err(invalid(format!("X ZIP compression method {method} is unsupported"))); + } + let (provisional_name, provisional_directory) = + decode_name(&name_bytes, flags, false, limits)?; + let selected = !provisional_directory && selection.is_match(&provisional_name); + let (name, directory_flag) = if selected { + decode_name(&name_bytes, flags, true, limits)? + } else { + (provisional_name, provisional_directory) + }; + let entry = Entry { + name: name.clone(), + name_bytes, + directory: directory_flag, + selected, + version_made_by, + version_needed, + flags, + method, + modified_time, + modified_date, + crc32: checksum, + compressed_size, + uncompressed_size, + external_attributes, + local_header_offset, + }; + if !names.insert(name) { + return Err(invalid("X ZIP archive contains duplicate member names")); + } + if compressed_size as usize > archive.len() || local_header_offset >= directory.offset as u64 { + return Err(invalid("X ZIP member exceeds archive bounds")); + } + if method == STORED && compressed_size != uncompressed_size { + return Err(invalid("X ZIP stored member has inconsistent sizes")); + } + if selected + && (compressed_size < 1 + || uncompressed_size < 1 + || compressed_size > limits.max_compressed_member_bytes + || uncompressed_size > limits.max_member_bytes) + { + return Err(invalid("selected X ZIP member exceeds its size bounds")); + } + if selected && uncompressed_size > compressed_size.saturating_mul(limits.max_ratio) { + return Err(invalid("selected X ZIP member exceeds its compression-ratio limit")); + } + declared = declared + .checked_add(uncompressed_size) + .filter(|value| *value <= limits.max_total_declared_bytes) + .ok_or_else(|| invalid("X ZIP archive exceeds its declared uncompressed-size limit"))?; + if selected { + selected_total = selected_total + .checked_add(uncompressed_size) + .filter(|value| *value <= limits.max_total_selected_bytes) + .ok_or_else(|| invalid("selected X ZIP members exceed their total size limit"))?; + } + validate_type(&entry)?; + entries.push(entry); + position = next; + } + if position != bytes.len() { + return Err(invalid("X ZIP central directory contains unindexed data")); + } + Ok(entries) +} + +struct LocalSizes { + compressed: u64, + uncompressed: u64, + zip64: bool, +} + +fn local_sizes( + compressed: u32, + uncompressed: u32, + fields: &HashMap, + label: &str, +) -> Result { + let zip64 = fields.get(&ZIP64_EXTRA).copied(); + if compressed != U32_MAX && uncompressed != U32_MAX { + if zip64.is_some() { + return Err(invalid(format!("X ZIP {label} contains redundant ZIP64 size metadata"))); + } + return Ok(LocalSizes { + compressed: compressed as u64, + uncompressed: uncompressed as u64, + zip64: false, + }); + } + let zip64 = zip64.ok_or_else(|| invalid(format!("X ZIP {label} is missing ZIP64 size metadata")))?; + let mut position = 0usize; + let mut resolved_uncompressed = uncompressed as u64; + let mut resolved_compressed = compressed as u64; + if uncompressed == U32_MAX { + resolved_uncompressed = u64_field(zip64, position, &format!("{label} uncompressed size"))?; + position += 8; + } + if compressed == U32_MAX { + resolved_compressed = u64_field(zip64, position, &format!("{label} compressed size"))?; + position += 8; + } + if position != zip64.len() { + return Err(invalid(format!("X ZIP {label} contains ambiguous ZIP64 size metadata"))); + } + Ok(LocalSizes { + compressed: resolved_compressed, + uncompressed: resolved_uncompressed, + zip64: true, + }) +} + +fn validate_descriptor(bytes: &[u8], entry: &Entry) -> Result<()> { + let mut position = 0usize; + if bytes.len() == 16 || bytes.len() == 24 { + if u32_le(bytes, 0) != DESCRIPTOR { + return Err(invalid("X ZIP data descriptor signature is invalid")); + } + position = 4; + } else if bytes.len() != 12 && bytes.len() != 20 { + return Err(invalid("X ZIP data descriptor has an invalid length")); + } + if u32_le(bytes, position) != entry.crc32 { + return Err(invalid("X ZIP data descriptor checksum disagrees with the central directory")); + } + position += 4; + let zip64 = bytes.len() - position == 16; + let compressed = if zip64 { + u64_field(bytes, position, "descriptor compressed size")? + } else { + u32_le(bytes, position) as u64 + }; + position += if zip64 { 8 } else { 4 }; + let uncompressed = if zip64 { + u64_field(bytes, position, "descriptor uncompressed size")? + } else { + u32_le(bytes, position) as u64 + }; + if compressed != entry.compressed_size || uncompressed != entry.uncompressed_size { + return Err(invalid("X ZIP data descriptor sizes disagree with the central directory")); + } + Ok(()) +} + +fn local_ranges( + archive: &[u8], + entries: &[Entry], + central_offset: usize, +) -> Result> { + let mut ordered: Vec<&Entry> = entries.iter().collect(); + ordered.sort_by_key(|entry| entry.local_header_offset); + let mut ranges: HashMap = HashMap::new(); + let mut expected: u64 = 0; + for (index, entry) in ordered.iter().enumerate() { + let label = format!("local member {}", index + 1); + let offset = entry.local_header_offset; + if offset != expected { + return Err(invalid(if offset < expected { + "X ZIP local member ranges overlap" + } else { + "X ZIP archive contains unindexed local data" + })); + } + let offset_usize = offset as usize; + let header = read_exact(archive, offset_usize, 30, &format!("{label} header"))?; + if u32_le(header, 0) != LOCAL { + return Err(invalid("X ZIP local header signature is invalid")); + } + let version = u16_le(header, 4); + let flags = u16_le(header, 6); + let method = u16_le(header, 8); + let modified_time = u16_le(header, 10); + let modified_date = u16_le(header, 12); + let checksum = u32_le(header, 14); + let compressed_legacy = u32_le(header, 18); + let uncompressed_legacy = u32_le(header, 22); + let name_length = u16_le(header, 26) as usize; + let extra_length = u16_le(header, 28) as usize; + if version != entry.version_needed + || flags != entry.flags + || method != entry.method + || modified_time != entry.modified_time + || modified_date != entry.modified_date + { + return Err(invalid("X ZIP local header disagrees with the central directory")); + } + let variable = read_exact( + archive, + offset_usize + 30, + name_length + extra_length, + &format!("{label} fields"), + )?; + if variable[..name_length] != entry.name_bytes[..] { + return Err(invalid("X ZIP local member name disagrees with the central directory")); + } + let fields = extra_fields(&variable[name_length..], &format!("{label} header"))?; + let sizes = local_sizes(compressed_legacy, uncompressed_legacy, &fields, &format!("{label} header"))?; + if (flags & DESCRIPTOR_FLAG) == 0 { + if checksum != entry.crc32 + || sizes.compressed != entry.compressed_size + || sizes.uncompressed != entry.uncompressed_size + { + return Err(invalid("X ZIP local sizes or checksum disagree with the central directory")); + } + } else if (checksum != 0 && checksum != entry.crc32) + || (sizes.compressed != 0 && sizes.compressed != entry.compressed_size) + || (sizes.uncompressed != 0 && sizes.uncompressed != entry.uncompressed_size) + { + return Err(invalid("X ZIP local descriptor placeholders disagree with the central directory")); + } + let data_offset = checked_end(offset_usize + 30, name_length + extra_length, "local member data offset")?; + let data_end = checked_end(data_offset, entry.compressed_size as usize, "local member compressed data")?; + let next = ordered + .get(index + 1) + .map(|e| e.local_header_offset as usize) + .unwrap_or(central_offset); + if data_end > next { + return Err(invalid("X ZIP local member ranges overlap")); + } + if (flags & DESCRIPTOR_FLAG) == 0 { + if data_end != next { + return Err(invalid("X ZIP archive contains unindexed local data")); + } + } else { + let descriptor_length = next - data_end; + let allowed: &[usize] = if sizes.zip64 { &[20, 24] } else { &[12, 16] }; + if !allowed.contains(&descriptor_length) { + return Err(invalid("X ZIP data descriptor has an invalid width")); + } + validate_descriptor(read_exact(archive, data_end, descriptor_length, &format!("{label} descriptor"))?, entry)?; + } + ranges.insert(offset, LocalRange { data_offset, data_end }); + expected = next as u64; + } + if expected != central_offset as u64 { + return Err(invalid("X ZIP archive contains unindexed local data")); + } + Ok(ranges) +} + +fn read_selected(archive: &[u8], entry: &Entry, range: LocalRange, limits: &StrictZipLimits) -> Result { + let compressed = read_exact( + archive, + range.data_offset, + range.data_end - range.data_offset, + &format!("selected member {}", entry.name), + )?; + let output: Vec = if entry.method == STORED { + compressed.to_vec() + } else { + let cap = (limits.max_member_bytes + 1).min(entry.uncompressed_size + 1); + let mut decoder = DeflateDecoder::new(compressed).take(cap); + let mut buffer = Vec::with_capacity(entry.uncompressed_size as usize); + decoder + .read_to_end(&mut buffer) + .map_err(|_| invalid(format!("selected X ZIP member is invalid: {}", entry.name)))?; + buffer + }; + if output.len() as u64 != entry.uncompressed_size { + return Err(invalid("selected X ZIP member has an incorrect output size")); + } + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&output); + if hasher.finalize() != entry.crc32 { + return Err(invalid("selected X ZIP member failed its CRC-32 check")); + } + Ok(StrictZipEntry { + name: entry.name.clone(), + bytes: output, + }) +} + +/// Read and strictly validate an archive held in memory, returning only the +/// members whose names match `options.patterns`. +pub fn read_zip_entries_strict( + archive: &[u8], + options: &StrictZipOptions, +) -> Result> { + let limits = &options.limits; + if archive.is_empty() || archive.len() as u64 > limits.max_archive_bytes { + return Err(invalid("X archive size is invalid")); + } + let selection = RegexSet::new(&options.patterns) + .map_err(|e| invalid(format!("X ZIP selection patterns are invalid: {e}")))?; + let directory = directory_from_eocd(archive, limits)?; + let entries = central_entries(archive, directory, &selection, limits)?; + let ranges = local_ranges(archive, &entries, directory.offset)?; + let mut selected: HashMap = HashMap::new(); + for entry in &entries { + if !entry.selected { + continue; + } + let range = ranges + .get(&entry.local_header_offset) + .copied() + .ok_or_else(|| invalid("X ZIP selected member has no validated local range"))?; + let member = read_selected(archive, entry, range, limits)?; + if selected.insert(member.name.clone(), member).is_some() { + return Err(invalid("X ZIP archive contains a duplicate selected member")); + } + } + Ok(selected.into_values().collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn make_test_zip(entries: &[(&str, &[u8])]) -> Vec { + let mut cursor = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut cursor); + for (name, bytes) in entries { + zip.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + zip.write_all(bytes).unwrap(); + } + zip.finish().unwrap(); + } + cursor.into_inner() + } + + fn options(patterns: &[&str]) -> StrictZipOptions { + StrictZipOptions { + patterns: patterns.iter().map(|p| p.to_string()).collect(), + limits: StrictZipLimits::default(), + } + } + + #[test] + fn reads_selected_deflated_and_stored_members() { + let archive = make_test_zip(&[ + ("data/manifest.js", b"manifest-bytes"), + ("data/account.js", b"account-bytes"), + ("data/direct-messages.js", b"dm-bytes"), + ("assets/image.png", b"not-selected"), + ]); + let entries = read_zip_entries_strict(&archive, &options(&[r"^data/.*\.js$"])).unwrap(); + let names: HashSet<&str> = entries.iter().map(|e| e.name.as_str()).collect(); + assert_eq!(names.len(), 3); + assert!(names.contains("data/manifest.js")); + let manifest = entries.iter().find(|e| e.name == "data/manifest.js").unwrap(); + assert_eq!(manifest.bytes, b"manifest-bytes"); + } + + #[test] + fn rejects_truncated_archive() { + let archive = make_test_zip(&[("data/manifest.js", b"x")]); + let truncated = &archive[..archive.len() - 4]; + assert!(read_zip_entries_strict(truncated, &options(&[r"^data/.*\.js$"])).is_err()); + } + + #[test] + fn rejects_nonmatching_archives() { + let archive = make_test_zip(&[("readme.txt", b"hi")]); + let entries = read_zip_entries_strict(&archive, &options(&[r"^data/.*\.js$"])).unwrap(); + assert!(entries.is_empty()); + } +}