Skip to content

perf(store): shrink the per-key object - box ValueRepr variants (mem rounds 1-2) #262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ELares merged 3 commits into from
Jun 16, 2026
Merged

perf(store): shrink the per-key object - box ValueRepr variants (mem rounds 1-2) #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b6dfc0c
perf(store): box the collection ValueRepr variants - shrink the per-k…
ELares Jun 16, 2026
6cde13c
perf(store): box the embstr inline buffer - pointer-size ValueRepr (r…
ELares Jun 16, 2026
71f1ba0
docs(bench): record the validated single-allocation blob-entry design…
ELares Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions crates/ironcache-bench/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@
//! all three so bytes-per-key is reported per class:
//!
//! - [`EncodingClass::Int`]: a canonical i64 in decimal (e.g. `"12345"`). Stored
//! with NO value heap allocation (the integer lives inline in the object).
//! Note its per-key footprint equals embstr's: the stored-value enum is sized
//! for its largest inline variant, so int saves a heap allocation but not slot
//! bytes. `OBJECT ENCODING` -> `int`.
//! inline in the per-key object's value enum with NO value heap allocation (the
//! `int` variant is the only string-family variant that does not box its bytes).
//! `OBJECT ENCODING` -> `int`.
//! - [`EncodingClass::EmbStr`]: a short string at or below the embstr threshold
//! (44 bytes, [`ironcache_store::encoding::EMBSTR_THRESHOLD`]), stored inline in
//! the object (SSO). `OBJECT ENCODING` -> `embstr`.
//! (44 bytes, [`ironcache_store::encoding::EMBSTR_THRESHOLD`]), stored in a single
//! boxed value allocation (memory Round 2 shrank the per-key slot by boxing the
//! embstr bytes rather than carrying a fixed inline buffer). `OBJECT ENCODING` ->
//! `embstr`.
//! - [`EncodingClass::Raw`]: a longer string stored out-of-line (a separate heap
//! allocation). `OBJECT ENCODING` -> `raw`.

Expand Down
10 changes: 5 additions & 5 deletions crates/ironcache-server/src/dispatch.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2372,7 +2372,7 @@ mod tests {
ttl_present: false,
snapshot_version: 0,
};
obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
st.insert_object(0, obj);

// GET / STRLEN / GETSET against the non-string -> WRONGTYPE.
Expand Down Expand Up @@ -2421,7 +2421,7 @@ mod tests {
ttl_present: false,
snapshot_version: 0,
};
obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
st.insert_object(0, obj);

// MGET str missing lst -> [bulk("hi"), Null, Null]. The non-string yields Null,
Expand Down Expand Up @@ -2726,7 +2726,7 @@ mod tests {
ttl_present: false,
snapshot_version: 0,
};
obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
st.insert_object(0, obj);
assert_eq!(
err_line(run_on(&c, &mut s, &mut st, t, &[b"INCR", b"lst"])),
Expand Down Expand Up @@ -2769,7 +2769,7 @@ mod tests {
ttl_present: false,
snapshot_version: 0,
};
obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
st.insert_object(0, obj);
assert_eq!(
err_line(run_on(
Expand Down Expand Up @@ -4196,7 +4196,7 @@ mod tests {
ttl_present: false,
snapshot_version: 0,
};
obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
st.insert_object(0, obj);
match run_on_wheel(&c, &mut s, &mut st, &mut wheel, t, &[b"GETEX", b"lst"]) {
Value::Error(e) => assert_eq!(
Expand Down
108 changes: 46 additions & 62 deletions crates/ironcache-store/src/kvobj.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
//! command layer. The folded-header metadata (below) is laid out as the FAM
//! version will pack it, so that follow-up is a representation change only.

use crate::encoding::{Classified, EMBSTR_THRESHOLD, classify};
use crate::encoding::{Classified, classify};
use crate::scan_hash;
use bytes::Bytes;
use hashbrown::HashMap;
Expand All @@ -41,12 +41,6 @@ use ironcache_storage::{
};
use std::collections::{BTreeSet, VecDeque};

/// The inline-value buffer capacity (embstr). Matches [`EMBSTR_THRESHOLD`]; a
/// value classified as embstr fits here without a separate allocation in the
/// eventual FAM layout. In the safe rep it is a fixed-size inline array plus a
/// length, so an embstr value adds no heap allocation beyond the `KvObj` itself.
pub const INLINE_CAP: usize = EMBSTR_THRESHOLD;

/// The packed per-key header (OBJECT_LAYOUT.md "packed header and metadata bits").
///
/// In the eventual FAM layout these fields are bit-packed into a few bytes (type +
Expand Down Expand Up @@ -101,66 +95,53 @@ impl Header {
}
}

/// A small inline string buffer ([`INLINE_CAP`] bytes plus a length), the safe-rep
/// stand-in for the FAM inline-value region. An embstr value lives here with no
/// extra heap allocation.
#[derive(Debug, Clone)]
pub struct InlineBuf {
buf: [u8; INLINE_CAP],
len: u8,
}

impl InlineBuf {
/// Build from bytes that are known to fit ([`INLINE_CAP`]). Panics if too long;
/// callers (the classifier path) only construct this for embstr-sized values.
#[must_use]
pub fn from_bytes(bytes: &[u8]) -> Self {
assert!(
bytes.len() <= INLINE_CAP,
"InlineBuf overflow: {} > {INLINE_CAP}",
bytes.len()
);
let mut buf = [0u8; INLINE_CAP];
buf[..bytes.len()].copy_from_slice(bytes);
InlineBuf {
buf,
len: bytes.len() as u8,
}
}

/// The inline bytes.
#[must_use]
pub fn as_bytes(&self) -> &[u8] {
&self.buf[..self.len as usize]
}
}

/// The value representation inside a [`KvObj`] (ENCODINGS.md #112).
#[derive(Debug, Clone)]
pub enum ValueRepr {
/// An int-encoded value: the raw i64, NO value allocation (the decimal bytes
/// are materialized on read). `OBJECT ENCODING` -> int.
Int(i64),
/// A short string stored inline. `OBJECT ENCODING` -> embstr.
Inline(InlineBuf),
/// A short string (embstr). `OBJECT ENCODING` -> embstr.
///
/// BOXED (memory Round 2): the bytes live behind a `Box<[u8]>` rather than a
/// fixed inline buffer, so this variant is one pointer wide and the largest
/// `ValueRepr` variant shrinks to a `Box<[u8]>`, cutting every per-key `KvObj`
/// and the hashbrown table slot. The embstr-vs-raw distinction is the SAME (it is
/// recorded in [`Header::encoding`], NOT by the variant): a value classified as
/// embstr by [`crate::encoding::EMBSTR_THRESHOLD`] is `Inline`, a longer one is
/// [`ValueRepr::Raw`],
/// and `OBJECT ENCODING` reports `embstr` / `raw` exactly as before. Redis/Valkey
/// also heap-allocate the object body, so this is allocation-parity with redis
/// plus a smaller slot.
Inline(Box<[u8]>),
/// A long string stored out-of-line. `OBJECT ENCODING` -> raw.
Raw(Box<[u8]>),
/// A LIST value (PR-5). `OBJECT ENCODING` -> `listpack` while small, `quicklist`
/// once over the threshold (a pure function of the active repr, #40).
List(ListVal),
///
/// BOXED (memory Round 1): the four collection structs are the large `ValueRepr`
/// variants (`ListVal` 40 / `HashVal` 40 / `SetVal` 48 / `ZSetVal` 64). Holding them
/// behind a `Box` drops `ValueRepr` to the string-variant bound, which shrinks every
/// per-key `KvObj` and the hashbrown table slot. The string/int hot path
/// (`Int`/`Inline`/`Raw`) is UNBOXED so the embstr SSO is untouched; the collections
/// already heap-allocate their contents, so the `Box` is a negligible extra
/// indirection only on collection ops.
List(Box<ListVal>),
/// A HASH value (PR-6). `OBJECT ENCODING` -> `listpack` while small, `hashtable`
/// once over the entry-count OR per-element-byte threshold (a pure function of the
/// active repr, #40).
Hash(HashVal),
/// active repr, #40). BOXED (memory Round 1); see [`ValueRepr::List`].
Hash(Box<HashVal>),
/// A SET value (PR-7). `OBJECT ENCODING` -> `intset` while all-integer and small,
/// `listpack` once a non-integer member is added (and still small), `hashtable` once
/// over the entry-count OR per-member-byte threshold (a pure function of the active
/// repr, #40). The conversion is ONE-WAY (never demotes).
Set(SetVal),
/// repr, #40). The conversion is ONE-WAY (never demotes). BOXED (memory Round 1);
/// see [`ValueRepr::List`].
Set(Box<SetVal>),
/// A ZSET (sorted set) value (PR-8). `OBJECT ENCODING` -> `listpack` while small,
/// `skiplist` once over the entry-count OR per-member-byte threshold (a pure function
/// of the active repr, #40). The conversion is ONE-WAY (never demotes).
ZSet(ZSetVal),
/// of the active repr, #40). The conversion is ONE-WAY (never demotes). BOXED
/// (memory Round 1); see [`ValueRepr::List`].
ZSet(Box<ZSetVal>),
}

impl ValueRepr {
Expand All @@ -187,8 +168,9 @@ impl ValueRepr {
pub fn logical_len(&self) -> usize {
match self {
ValueRepr::Int(n) => int_decimal_len(*n),
ValueRepr::Inline(b) => b.as_bytes().len(),
ValueRepr::Raw(b) => b.len(),
// Embstr and raw both hold the value bytes behind a `Box<[u8]>`; the
// embstr-vs-raw distinction lives in `Header.encoding`, not the variant.
ValueRepr::Inline(b) | ValueRepr::Raw(b) => b.len(),
ValueRepr::List(l) => l.element_bytes(),
ValueRepr::Hash(h) => h.element_bytes(),
ValueRepr::Set(s) => s.element_bytes(),
Expand Down Expand Up @@ -1714,7 +1696,7 @@ impl KvObj {
) -> Self {
let value = match classified {
Classified::Int(n) => ValueRepr::Int(n),
Classified::EmbStr => ValueRepr::Inline(InlineBuf::from_bytes(bytes)),
Classified::EmbStr => ValueRepr::Inline(bytes.to_vec().into_boxed_slice()),
Classified::Raw => ValueRepr::Raw(bytes.to_vec().into_boxed_slice()),
};
let header = Header::new(value.encoding(), expire_at.is_some());
Expand Down Expand Up @@ -1800,7 +1782,7 @@ impl KvObj {
KvObj {
header: Header::with_type(DataType::List, encoding, expire_at.is_some()),
key: key.to_vec().into_boxed_slice(),
value: ValueRepr::List(list),
value: ValueRepr::List(Box::new(list)),
expire_at,
}
}
Expand All @@ -1814,7 +1796,7 @@ impl KvObj {
KvObj {
header: Header::with_type(DataType::Hash, encoding, expire_at.is_some()),
key: key.to_vec().into_boxed_slice(),
value: ValueRepr::Hash(hash),
value: ValueRepr::Hash(Box::new(hash)),
expire_at,
}
}
Expand All @@ -1828,7 +1810,7 @@ impl KvObj {
KvObj {
header: Header::with_type(DataType::Set, encoding, expire_at.is_some()),
key: key.to_vec().into_boxed_slice(),
value: ValueRepr::Set(set),
value: ValueRepr::Set(Box::new(set)),
expire_at,
}
}
Expand All @@ -1842,7 +1824,7 @@ impl KvObj {
KvObj {
header: Header::with_type(DataType::ZSet, encoding, expire_at.is_some()),
key: key.to_vec().into_boxed_slice(),
value: ValueRepr::ZSet(zset),
value: ValueRepr::ZSet(Box::new(zset)),
expire_at,
}
}
Expand All @@ -1861,7 +1843,9 @@ impl KvObj {
/// yields `None` -> WRONGTYPE).
pub fn as_list_mut(&mut self) -> Option<&mut ListVal> {
match &mut self.value {
ValueRepr::List(l) => Some(l),
// Deref through the `Box` (memory Round 1) to the `&mut ListVal` the
// collection trait + in-place RMW path expect.
ValueRepr::List(l) => Some(&mut **l),
_ => None,
}
}
Expand All @@ -1871,7 +1855,7 @@ impl KvObj {
/// `None` -> WRONGTYPE). The HASH analog of [`Self::as_list_mut`].
pub fn as_hash_mut(&mut self) -> Option<&mut HashVal> {
match &mut self.value {
ValueRepr::Hash(h) => Some(h),
ValueRepr::Hash(h) => Some(&mut **h),
_ => None,
}
}
Expand All @@ -1881,7 +1865,7 @@ impl KvObj {
/// -> WRONGTYPE). The SET analog of [`Self::as_list_mut`]/[`Self::as_hash_mut`].
pub fn as_set_mut(&mut self) -> Option<&mut SetVal> {
match &mut self.value {
ValueRepr::Set(s) => Some(s),
ValueRepr::Set(s) => Some(&mut **s),
_ => None,
}
}
Expand All @@ -1892,7 +1876,7 @@ impl KvObj {
/// [`Self::as_set_mut`].
pub fn as_zset_mut(&mut self) -> Option<&mut ZSetVal> {
match &mut self.value {
ValueRepr::ZSet(z) => Some(z),
ValueRepr::ZSet(z) => Some(&mut **z),
_ => None,
}
}
Expand Down Expand Up @@ -1972,7 +1956,7 @@ impl KvObj {
pub fn set_value_bytes(&mut self, bytes: &[u8]) {
self.value = match classify(bytes) {
Classified::Int(n) => ValueRepr::Int(n),
Classified::EmbStr => ValueRepr::Inline(InlineBuf::from_bytes(bytes)),
Classified::EmbStr => ValueRepr::Inline(bytes.to_vec().into_boxed_slice()),
Classified::Raw => ValueRepr::Raw(bytes.to_vec().into_boxed_slice()),
};
self.header.encoding = self.value.encoding();
Expand Down
31 changes: 13 additions & 18 deletions crates/ironcache-store/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,13 +576,9 @@ impl<E: EvictionHook, A: AccountingHook> ShardStore<E, A> {
kvobj::ValueRepr::Int(n) => {
ValueRef::from_int_bytes(obj.header.data_type, obj.expire_at, int_decimal_bytes(*n))
}
kvobj::ValueRepr::Inline(b) => ValueRef::borrowed(
obj.header.data_type,
obj.header.encoding,
obj.expire_at,
b.as_bytes(),
),
kvobj::ValueRepr::Raw(b) => {
// Embstr and raw both borrow their bytes the same way; the embstr-vs-raw
// distinction is carried by `obj.header.encoding`, not the variant.
kvobj::ValueRepr::Inline(b) | kvobj::ValueRepr::Raw(b) => {
ValueRef::borrowed(obj.header.data_type, obj.header.encoding, obj.expire_at, b)
}
// A LIST/HASH/SET is not byte-readable as a string: the command layer only
Expand Down Expand Up @@ -612,13 +608,9 @@ impl<E: EvictionHook, A: AccountingHook> ShardStore<E, A> {
obj.expire_at,
int_decimal_bytes(*n),
),
kvobj::ValueRepr::Inline(b) => OccupiedEntry::borrowed(
obj.header.data_type,
obj.header.encoding,
obj.expire_at,
b.as_bytes(),
),
kvobj::ValueRepr::Raw(b) => {
// Embstr and raw both borrow their bytes the same way; the embstr-vs-raw
// distinction is carried by `obj.header.encoding`, not the variant.
kvobj::ValueRepr::Inline(b) | kvobj::ValueRepr::Raw(b) => {
OccupiedEntry::borrowed(obj.header.data_type, obj.header.encoding, obj.expire_at, b)
}
// A LIST/HASH/SET observed through the READ-ONLY rmw arm (e.g. a numeric RMW
Expand Down Expand Up @@ -823,17 +815,20 @@ impl<E: EvictionHook, A: AccountingHook> Store for ShardStore<E, A> {
// would each take and drop a fresh `&mut` and obscure the dispatch) so each
// collection type maps to exactly one arm.
let entry = match &mut obj.value {
// The collection variants are boxed (memory Round 1); deref through the
// `Box` (`&mut **`) to the concrete `&mut *Val`, which then coerces to the
// `&mut dyn *Value` trait object the typed view constructors take.
kvobj::ValueRepr::List(l) => {
RmwEntry::OccupiedMut(OccupiedEntryMut::list(encoding, expire_at, l))
RmwEntry::OccupiedMut(OccupiedEntryMut::list(encoding, expire_at, &mut **l))
}
kvobj::ValueRepr::Hash(h) => {
RmwEntry::OccupiedMut(OccupiedEntryMut::hash(encoding, expire_at, h))
RmwEntry::OccupiedMut(OccupiedEntryMut::hash(encoding, expire_at, &mut **h))
}
kvobj::ValueRepr::Set(s) => {
RmwEntry::OccupiedMut(OccupiedEntryMut::set(encoding, expire_at, s))
RmwEntry::OccupiedMut(OccupiedEntryMut::set(encoding, expire_at, &mut **s))
}
kvobj::ValueRepr::ZSet(z) => {
RmwEntry::OccupiedMut(OccupiedEntryMut::zset(encoding, expire_at, z))
RmwEntry::OccupiedMut(OccupiedEntryMut::zset(encoding, expire_at, &mut **z))
}
kvobj::ValueRepr::Int(_)
| kvobj::ValueRepr::Inline(_)
Expand Down
4 changes: 2 additions & 2 deletions crates/ironcache-store/tests/keyspace.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use ironcache_storage::{
UnixMillis,
};
use ironcache_store::ShardStore;
use ironcache_store::kvobj::{Header, InlineBuf, KvObj, ValueRepr};
use ironcache_store::kvobj::{Header, KvObj, ValueRepr};
use std::collections::HashSet;

const NOW: UnixMillis = UnixMillis(1_000);
Expand Down Expand Up @@ -256,7 +256,7 @@ fn scan_type_filter_selects_by_data_type() {
ttl_present: false,
snapshot_version: 0,
};
lst.value = ValueRepr::Inline(InlineBuf::from_bytes(b"x"));
lst.value = ValueRepr::Inline(Box::from(&b"x"[..]));
s.insert_object(0, lst);

let mut out = Vec::new();
Expand Down
Loading
Loading