diff --git a/Cargo.lock b/Cargo.lock index 4a77e98..de246c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,7 @@ dependencies = [ "aden-emit", "aden-graph", "aden-parse", + "aden-store", "criterion", ] @@ -390,6 +391,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -398,9 +405,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "byteview" -version = "0.6.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" +checksum = "1c53ba0f290bfc610084c05582d9c5d421662128fc69f4bf236707af6fd321b9" [[package]] name = "cast" @@ -909,12 +916,6 @@ dependencies = [ "syn", ] -[[package]] -name = "double-ended-peekable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" - [[package]] name = "downcast-rs" version = "2.0.2" @@ -1043,17 +1044,17 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "fjall" -version = "2.11.2" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc" +checksum = "038acd422d607e0eca09e093f299f9eccf9bd097554343d93746afff81a45113" dependencies = [ - "byteorder", + "byteorder-lite", "byteview", "dashmap 6.2.1", + "flume", "log", "lsm-tree", - "path-absolutize", - "std-semaphore", + "lz4_flex", "tempfile", "xxhash-rust", ] @@ -1074,6 +1075,15 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1235,12 +1245,6 @@ dependencies = [ "wasip3", ] -[[package]] -name = "guardian" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" - [[package]] name = "half" version = "2.7.1" @@ -1661,24 +1665,22 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lsm-tree" -version = "2.10.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15" +checksum = "8ef86c3c797c10eefcc73407c43ae48c19d4df686131a8334b2895a513e91df4" dependencies = [ - "byteorder", + "byteorder-lite", + "byteview", "crossbeam-skiplist", - "double-ended-peekable", "enum_dispatch", - "guardian", "interval-heap", "log", "lz4_flex", - "path-absolutize", "quick_cache", "rustc-hash", "self_cell", + "sfa", "tempfile", - "value-log", "varint-rs", "xxhash-rust", ] @@ -1698,9 +1700,12 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] [[package]] name = "macro_rules_attribute" @@ -1993,24 +1998,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" -[[package]] -name = "path-absolutize" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" -dependencies = [ - "path-dedot", -] - -[[package]] -name = "path-dedot" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" -dependencies = [ - "once_cell", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2179,7 +2166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -2665,6 +2652,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sfa" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175" +dependencies = [ + "byteorder-lite", + "log", + "xxhash-rust", +] + [[package]] name = "sha2" version = "0.11.0" @@ -2710,6 +2708,15 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -2734,12 +2741,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "std-semaphore" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" - [[package]] name = "streaming-iterator" version = "0.1.9" @@ -3323,6 +3324,12 @@ dependencies = [ "zstd", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typeid" version = "1.0.3" @@ -3440,23 +3447,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "value-log" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" -dependencies = [ - "byteorder", - "byteview", - "interval-heap", - "log", - "path-absolutize", - "rustc-hash", - "tempfile", - "varint-rs", - "xxhash-rust", -] - [[package]] name = "varint-rs" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index cfbe7f8..8eb4982 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,12 @@ sha2 = "0.11" [dev-dependencies] criterion = "0.5" +aden-store = { path = "crates/aden-store" } +aden-core = { path = "crates/aden-core" } + +[[bench]] +name = "store_perf" +harness = false [[bench]] name = "token_density" diff --git a/NOTICE.md b/NOTICE.md index 23e3e1d..60421ea 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -234,12 +234,17 @@ Generated by `aden licenses`. - **License**: Unlicense OR MIT - **Repository**: https://github.com/BurntSushi/byteorder +### byteorder-lite v0.1.0 + +- **License**: Unlicense OR MIT +- **Repository**: https://github.com/image-rs/byteorder-lite + ### bytes v1.11.1 - **License**: MIT - **Repository**: https://github.com/tokio-rs/bytes -### byteview v0.6.1 +### byteview v0.10.1 - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/fjall-rs/byteview @@ -530,7 +535,7 @@ Generated by `aden licenses`. - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/petgraph/fixedbitset -### fjall v2.11.2 +### fjall v3.1.5 - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/fjall-rs/fjall @@ -540,6 +545,11 @@ Generated by `aden licenses`. - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/rust-lang/flate2-rs +### flume v0.12.0 + +- **License**: Apache-2.0/MIT +- **Repository**: https://github.com/zesterer/flume + ### fnv v1.0.7 - **License**: Apache-2.0 / MIT @@ -875,7 +885,7 @@ Generated by `aden licenses`. - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/rust-lang/log -### lsm-tree v2.10.4 +### lsm-tree v3.1.5 - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/fjall-rs/lsm-tree @@ -885,7 +895,7 @@ Generated by `aden licenses`. - **License**: MIT - **Repository**: https://github.com/gluon-lang/lsp-types -### lz4_flex v0.11.6 +### lz4_flex v0.13.1 - **License**: MIT - **Repository**: https://github.com/pseitz/lz4_flex @@ -1240,6 +1250,11 @@ Generated by `aden licenses`. - **License**: MIT OR Apache-2.0 - **Repository**: https://github.com/dtolnay/serde-yaml +### sfa v1.0.0 + +- **License**: MIT OR Apache-2.0 +- **Repository**: https://github.com/fjall-rs/sfa + ### sha2 v0.11.0 - **License**: MIT OR Apache-2.0 @@ -1436,6 +1451,11 @@ Generated by `aden licenses`. - **Repository**: https://github.com/kreuzberg-dev/tree-sitter-language-pack - **Note**: This crate bundles compiled grammars from numerous upstream repositories. Each grammar carries its own copyright holder and license. Full per-grammar attribution is maintained by the pack at the repository above. +### twox-hash v2.1.2 + +- **License**: MIT +- **Repository**: https://github.com/shepmaster/twox-hash + ### typenum v1.20.0 - **License**: MIT OR Apache-2.0 @@ -1501,11 +1521,6 @@ Generated by `aden licenses`. - **License**: Apache-2.0 OR MIT - **Repository**: https://github.com/uuid-rs/uuid -### value-log v1.9.0 - -- **License**: MIT OR Apache-2.0 -- **Repository**: https://github.com/fjall-rs/value-log - ### varint-rs v2.2.0 - **License**: Apache-2.0 diff --git a/benches/store_perf.rs b/benches/store_perf.rs new file mode 100644 index 0000000..a466350 --- /dev/null +++ b/benches/store_perf.rs @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Ernest Hamblen +// SPDX-License-Identifier: AGPL-3.0-or-later +//! Storage performance benchmarks. +//! +//! Two measurements: +//! 1. `store_open_drop` — open an existing fjall store and immediately drop it. +//! This was the 250ms floor per command with fjall 2.11 (monitor thread +//! unconditionally slept before checking the stop signal). With fjall 3.1+ +//! teardown uses a flume channel wake and completes in single-digit µs. +//! +//! 2. `get_all_edges_single_scan` vs `get_all_edges_per_type` — loading every +//! edge from the store. The old bridge loop called `get_edges_by_type` once +//! per EdgeType variant (32 full scans of the edges partition). The new +//! `get_all_edges` implementation does it in one pass. + +use aden_core::{Document, EdgeType, NodeType}; +use aden_store::{GraphStorage, Storage}; +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use std::collections::HashMap; +use std::path::PathBuf; + +fn temp_store_path(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("aden_bench_store_{label}")) +} + +fn seed_store(path: &str, n_docs: usize, edges_per_type: usize) { + let storage = Storage::new(path).expect("create store"); + + // Insert documents. + for i in 0..n_docs { + let doc = Document { + anchor: format!("anchor-{i}"), + node_type: NodeType::Function, + attributes: HashMap::new(), + blocks: vec![], + source_span: None, + metadata: None, + confidence: 1.0, + }; + storage.put_document(&doc).expect("put doc"); + } + + // Insert a handful of edges of every type so get_all_edges has real work. + let anchors: Vec = (0..n_docs).map(|i| format!("anchor-{i}")).collect(); + let mut bulk: Vec<(String, String, EdgeType)> = Vec::new(); + for et in &EdgeType::ALL { + for j in 0..edges_per_type { + let src = &anchors[j % n_docs]; + let dst = &anchors[(j + 1) % n_docs]; + bulk.push((src.clone(), dst.clone(), *et)); + } + } + storage.put_edges_bulk(&bulk).expect("put edges"); + storage.flush().expect("flush"); +} + +fn bench_store_open_drop(c: &mut Criterion) { + let path = temp_store_path("open_drop"); + let path_str = path.to_string_lossy().to_string(); + + // Seed once so open_existing finds a real store. + if !path.exists() { + seed_store(&path_str, 50, 3); + } + + c.bench_function("store_open_drop", |b| { + b.iter(|| { + let s = Storage::open_existing(black_box(&path_str)).expect("open"); + drop(black_box(s)); + }) + }); +} + +fn bench_get_all_edges(c: &mut Criterion) { + let path = temp_store_path("get_all_edges"); + let path_str = path.to_string_lossy().to_string(); + + if !path.exists() { + // 200 docs, 5 edges per type → 160 total edges across 32 types. + seed_store(&path_str, 200, 5); + } + + let storage = Storage::open_existing(&path_str).expect("open"); + + let mut g = c.benchmark_group("edge_load"); + + // New: single scan bucketing by type embedded in the key. + g.bench_function("single_scan", |b| { + b.iter(|| { + let edges = storage.get_all_edges().expect("get_all_edges"); + black_box(edges); + }) + }); + + // Old: one full scan per EdgeType variant (32 scans total). + g.bench_with_input( + BenchmarkId::new("per_type_loop", EdgeType::ALL.len()), + &EdgeType::ALL.len(), + |b, _| { + b.iter(|| { + let mut edges = Vec::new(); + for et in &EdgeType::ALL { + for (src, dst) in storage.get_edges_by_type(et).expect("get_edges_by_type") { + edges.push((src, dst, *et)); + } + } + black_box(edges); + }) + }, + ); + + g.finish(); +} + +criterion_group!(benches, bench_store_open_drop, bench_get_all_edges); +criterion_main!(benches); diff --git a/crates/aden-graph/src/bridge.rs b/crates/aden-graph/src/bridge.rs index a62f0c3..6cef442 100644 --- a/crates/aden-graph/src/bridge.rs +++ b/crates/aden-graph/src/bridge.rs @@ -49,17 +49,10 @@ impl GraphBridge { /// Load a graph from storage into in-memory structures. pub fn load_from_storage(storage: &S) -> Result { let docs = storage.get_all_documents()?; - let mut edges = Vec::new(); - - // The canonical variant list — a local copy here silently dropped - // every edge of a forgotten type on load (Wave 2 regression). - for edge_type in &EdgeType::ALL { - let typed_edges = storage.get_edges_by_type(edge_type)?; - for (src, dst) in typed_edges { - edges.push((src, dst, *edge_type)); - } - } - + // Single-pass: reads the edge type from each key instead of one full + // scan per type (32 scans → 1). The default trait impl falls back to + // the per-type loop for backends that don't override get_all_edges. + let edges = storage.get_all_edges()?; Ok((docs, edges)) } diff --git a/crates/aden-store/Cargo.toml b/crates/aden-store/Cargo.toml index 035a1e4..c4d8103 100644 --- a/crates/aden-store/Cargo.toml +++ b/crates/aden-store/Cargo.toml @@ -15,7 +15,7 @@ serde = { version = "1.0", features = ["derive"] } # fjall (pure-Rust LSM-tree) is aden's storage engine: vs the previous sled # backend it gave an ~8.5x smaller store on the Linux kernel (528 MB vs 4.4 GB), # faster gen, faster queries, and pulls no unmaintained transitive deps. -fjall = "2.11" +fjall = "3" [dev-dependencies] uuid = { version = "1.12", features = ["v4", "serde"] } diff --git a/crates/aden-store/src/fjall_store.rs b/crates/aden-store/src/fjall_store.rs index d9095b3..5c4d48c 100644 --- a/crates/aden-store/src/fjall_store.rs +++ b/crates/aden-store/src/fjall_store.rs @@ -8,15 +8,26 @@ //! a much smaller on-disk store, which is what matters as data pools grow. //! //! The key/value layout mirrors [`SledStorage`](crate::SledStorage) exactly; -//! sled "trees" map 1:1 onto fjall "partitions", so the two backends are +//! sled "trees" map 1:1 onto fjall "keyspaces", so the two backends are //! interchangeable behind the [`GraphStorage`] trait. +//! +//! ## Fjall 3 API mapping +//! +//! | fjall 2.x | fjall 3.x | +//! |-----------------------|--------------------------| +//! | `Keyspace` (root) | `Database` | +//! | `PartitionHandle` | `Keyspace` | +//! | `PartitionCreateOptions` | `KeyspaceCreateOptions` | +//! | `Config::new(p).open()` | `Database::builder(p).open()` | +//! | `ks.open_partition(n, opts)` | `db.keyspace(n, opts_fn)` | +//! | Iterator yields `Result<(key, val)>` | Iterator yields `Guard`; call `.into_inner()` | use crate::{ GraphStorage, KEY_SEP, StoreError, TreeName, base_key, deserialize, deserialize_document, doc_key, edge_key, incoming_key, meta_key, outgoing_key, serialize, serialize_document, }; use aden_core::{Document, EdgeType}; -use fjall::{Config, Keyspace, PartitionCreateOptions, PartitionHandle, PersistMode}; +use fjall::{Database, Keyspace, KeyspaceCreateOptions, PersistMode}; use std::collections::{HashMap, HashSet}; impl From for StoreError { @@ -27,24 +38,23 @@ impl From for StoreError { /// A Fjall (LSM-tree) backed implementation of [`GraphStorage`]. pub struct FjallStorage { - // Keyspace must outlive the partitions; kept for `persist` (flush). - keyspace: Keyspace, - docs: PartitionHandle, - edges: PartitionHandle, - outgoing: PartitionHandle, - incoming: PartitionHandle, - index: PartitionHandle, - meta: PartitionHandle, - bases: PartitionHandle, + // Database must outlive the keyspaces; kept for `persist` (flush). + db: Database, + docs: Keyspace, + edges: Keyspace, + outgoing: Keyspace, + incoming: Keyspace, + index: Keyspace, + meta: Keyspace, + bases: Keyspace, } impl FjallStorage { /// Open (or create) a Fjall store at the given path. pub fn new(path: &str) -> Result { - let keyspace = Config::new(path).open()?; - let open = |name: &str| -> Result { - keyspace - .open_partition(name, PartitionCreateOptions::default()) + let db = Database::builder(path).open()?; + let open = |name: &str| -> Result { + db.keyspace(name, KeyspaceCreateOptions::default) .map_err(StoreError::from) }; Ok(Self { @@ -55,13 +65,13 @@ impl FjallStorage { index: open(TreeName::Index.name())?, meta: open(TreeName::Meta.name())?, bases: open(TreeName::Bases.name())?, - keyspace, + db, }) } /// Open an *existing* Fjall store without creating it. /// - /// Unlike [`new`](Self::new) — which calls `fjall::Config::open()` and so + /// Unlike [`new`](Self::new) — which calls `Database::builder(path).open()` and so /// materializes the directory — this returns [`StoreError::NotFound`] when /// the store is absent. Read commands use this so that running e.g. /// `aden query` before `aden gen` yields a clear error rather than silently @@ -86,8 +96,8 @@ impl GraphStorage for FjallStorage { fn get_all_documents(&self) -> Result, StoreError> { let mut docs = HashMap::new(); - for item in self.docs.iter() { - let (key, value) = item?; + for guard in self.docs.iter() { + let (key, value) = guard.into_inner()?; let key_str = String::from_utf8_lossy(&key); if let Some(anchor) = key_str.strip_prefix("doc:") { docs.insert(anchor.to_string(), deserialize_document(&value)?); @@ -143,7 +153,8 @@ impl GraphStorage for FjallStorage { } fn put_edge(&self, src: &str, dst: &str, edge_type: EdgeType) -> Result<(), StoreError> { - self.edges.insert(edge_key(src, dst, &edge_type), [])?; + self.edges + .insert(edge_key(src, dst, &edge_type), &[] as &[u8])?; let out_key = outgoing_key(src); let mut out: Vec<(String, EdgeType)> = match self.outgoing.get(&out_key)? { @@ -167,7 +178,8 @@ impl GraphStorage for FjallStorage { let mut out_add: HashMap<&str, Vec<(String, EdgeType)>> = HashMap::new(); let mut in_add: HashMap<&str, Vec<(String, EdgeType)>> = HashMap::new(); for (src, dst, edge_type) in edges { - self.edges.insert(edge_key(src, dst, edge_type), [])?; + self.edges + .insert(edge_key(src, dst, edge_type), &[] as &[u8])?; out_add .entry(src.as_str()) .or_default() @@ -274,8 +286,8 @@ impl GraphStorage for FjallStorage { let edge_str = format!("{:?}", edge_type); let prefix = format!("edge{KEY_SEP}"); let mut edges = Vec::new(); - for item in self.edges.iter() { - let (key, _) = item?; + for guard in self.edges.iter() { + let (key, _) = guard.into_inner()?; let key_str = String::from_utf8_lossy(&key); if let Some(suffix) = key_str.strip_prefix(&prefix) { let parts: Vec<&str> = suffix.split(KEY_SEP).collect(); @@ -287,14 +299,39 @@ impl GraphStorage for FjallStorage { Ok(edges) } + fn get_all_edges(&self) -> Result, StoreError> { + // Build a debug-string → EdgeType lookup once so we can bucket by type + // in a single scan of the edges partition instead of one scan per type. + let type_map: HashMap = EdgeType::ALL + .iter() + .map(|et| (format!("{et:?}"), *et)) + .collect(); + + let prefix = format!("edge{KEY_SEP}"); + let mut edges = Vec::new(); + for guard in self.edges.iter() { + let (key, _) = guard.into_inner()?; + let key_str = String::from_utf8_lossy(&key); + if let Some(suffix) = key_str.strip_prefix(&prefix) { + let parts: Vec<&str> = suffix.split(KEY_SEP).collect(); + if parts.len() == 3 { + if let Some(&et) = type_map.get(parts[2]) { + edges.push((parts[0].to_string(), parts[1].to_string(), et)); + } + } + } + } + Ok(edges) + } + fn edge_exists(&self, src: &str, dst: &str, edge_type: &EdgeType) -> Result { Ok(self.edges.contains_key(edge_key(src, dst, edge_type))?) } fn get_all_anchors(&self) -> Result, StoreError> { let mut anchors = HashSet::new(); - for item in self.docs.iter() { - let (key, _) = item?; + for guard in self.docs.iter() { + let (key, _) = guard.into_inner()?; let key_str = String::from_utf8_lossy(&key); if let Some(anchor) = key_str.strip_prefix("doc:") { anchors.insert(anchor.to_string()); @@ -393,7 +430,10 @@ impl GraphStorage for FjallStorage { &self.index, &self.meta, ] { - let keys: Vec<_> = part.iter().filter_map(|i| i.ok().map(|(k, _)| k)).collect(); + let keys: Vec<_> = part + .iter() + .filter_map(|g| g.into_inner().ok().map(|(k, _)| k)) + .collect(); for k in keys { part.remove(k)?; } @@ -402,7 +442,7 @@ impl GraphStorage for FjallStorage { } fn flush(&self) -> Result<(), StoreError> { - self.keyspace.persist(PersistMode::SyncAll)?; + self.db.persist(PersistMode::SyncAll)?; Ok(()) } } diff --git a/crates/aden-store/src/lib.rs b/crates/aden-store/src/lib.rs index 4a31125..1e1ed2d 100644 --- a/crates/aden-store/src/lib.rs +++ b/crates/aden-store/src/lib.rs @@ -229,6 +229,21 @@ pub trait GraphStorage: Send + Sync { /// Get all edges of a type. fn get_edges_by_type(&self, edge_type: &EdgeType) -> Result, StoreError>; + /// Get all edges in one pass, bucketed by type. + /// + /// Default implementation calls `get_edges_by_type` for every variant (one + /// full scan per type). Backends should override with a single scan that + /// reads the type from the key — reducing 32 scans to 1 for graph loads. + fn get_all_edges(&self) -> Result, StoreError> { + let mut edges = Vec::new(); + for et in &EdgeType::ALL { + for (src, dst) in self.get_edges_by_type(et)? { + edges.push((src, dst, *et)); + } + } + Ok(edges) + } + /// Check if an edge exists. fn edge_exists(&self, src: &str, dst: &str, edge_type: &EdgeType) -> Result;