perf(store): shrink the per-key object - box ValueRepr variants (mem rounds 1-2) (#262)

Box the collection + embstr ValueRepr variants: ValueRepr 72->24, KvObj 112->64, table slot 128->80. vs redis 8.8.0: bytes-per-key 526.7->386.85 (gap 2.41x->1.77x), qps 71.4k->77.9k. Zero behavior change (full suite green); perf-gate verified the memory drop (-17 to -35% per class) with no qps regression. Next lever (single-alloc blob entry) scoped in OPTIMIZATION_LOG.

Signed-off-by: Zeke <ezequiel.lares@outlook.com>
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ELares

crates/ironcache-bench/src/lib.rs

@@ -22,13 +22,14 @@
 //! all three so bytes-per-key is reported per class:
 //!
 //! - [`EncodingClass::Int`]: a canonical i64 in decimal (e.g. `"12345"`). Stored
-//!   with NO value heap allocation (the integer lives inline in the object).
-//!   Note its per-key footprint equals embstr's: the stored-value enum is sized
-//!   for its largest inline variant, so int saves a heap allocation but not slot
-//!   bytes. `OBJECT ENCODING` -> `int`.
+//!   inline in the per-key object's value enum with NO value heap allocation (the
+//!   `int` variant is the only string-family variant that does not box its bytes).
+//!   `OBJECT ENCODING` -> `int`.
 //! - [`EncodingClass::EmbStr`]: a short string at or below the embstr threshold
-//!   (44 bytes, [`ironcache_store::encoding::EMBSTR_THRESHOLD`]), stored inline in
-//!   the object (SSO). `OBJECT ENCODING` -> `embstr`.
+//!   (44 bytes, [`ironcache_store::encoding::EMBSTR_THRESHOLD`]), stored in a single
+//!   boxed value allocation (memory Round 2 shrank the per-key slot by boxing the
+//!   embstr bytes rather than carrying a fixed inline buffer). `OBJECT ENCODING` ->
+//!   `embstr`.
 //! - [`EncodingClass::Raw`]: a longer string stored out-of-line (a separate heap
 //!   allocation). `OBJECT ENCODING` -> `raw`.
 

crates/ironcache-server/src/dispatch.rs

@@ -2372,7 +2372,7 @@ mod tests {
             ttl_present: false,
             snapshot_version: 0,
         };
-        obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
+        obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
         st.insert_object(0, obj);
 
         // GET / STRLEN / GETSET against the non-string -> WRONGTYPE.
@@ -2421,7 +2421,7 @@ mod tests {
             ttl_present: false,
             snapshot_version: 0,
         };
-        obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
+        obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
         st.insert_object(0, obj);
 
         // MGET str missing lst -> [bulk("hi"), Null, Null]. The non-string yields Null,
@@ -2726,7 +2726,7 @@ mod tests {
             ttl_present: false,
             snapshot_version: 0,
         };
-        obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
+        obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
         st.insert_object(0, obj);
         assert_eq!(
             err_line(run_on(&c, &mut s, &mut st, t, &[b"INCR", b"lst"])),
@@ -2769,7 +2769,7 @@ mod tests {
             ttl_present: false,
             snapshot_version: 0,
         };
-        obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
+        obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
         st.insert_object(0, obj);
         assert_eq!(
             err_line(run_on(
@@ -4196,7 +4196,7 @@ mod tests {
             ttl_present: false,
             snapshot_version: 0,
         };
-        obj.value = ValueRepr::Inline(ironcache_store::kvobj::InlineBuf::from_bytes(b"x"));
+        obj.value = ValueRepr::Inline(Box::from(&b"x"[..]));
         st.insert_object(0, obj);
         match run_on_wheel(&c, &mut s, &mut st, &mut wheel, t, &[b"GETEX", b"lst"]) {
             Value::Error(e) => assert_eq!(

crates/ironcache-store/src/kvobj.rs

@@ -24,7 +24,7 @@
 //! command layer. The folded-header metadata (below) is laid out as the FAM
 //! version will pack it, so that follow-up is a representation change only.
 
-use crate::encoding::{Classified, EMBSTR_THRESHOLD, classify};
+use crate::encoding::{Classified, classify};
 use crate::scan_hash;
 use bytes::Bytes;
 use hashbrown::HashMap;
@@ -41,12 +41,6 @@ use ironcache_storage::{
 };
 use std::collections::{BTreeSet, VecDeque};
 
-/// The inline-value buffer capacity (embstr). Matches [`EMBSTR_THRESHOLD`]; a
-/// value classified as embstr fits here without a separate allocation in the
-/// eventual FAM layout. In the safe rep it is a fixed-size inline array plus a
-/// length, so an embstr value adds no heap allocation beyond the `KvObj` itself.
-pub const INLINE_CAP: usize = EMBSTR_THRESHOLD;
-
 /// The packed per-key header (OBJECT_LAYOUT.md "packed header and metadata bits").
 ///
 /// In the eventual FAM layout these fields are bit-packed into a few bytes (type +
@@ -101,66 +95,53 @@ impl Header {
     }
 }
 
-/// A small inline string buffer ([`INLINE_CAP`] bytes plus a length), the safe-rep
-/// stand-in for the FAM inline-value region. An embstr value lives here with no
-/// extra heap allocation.
-#[derive(Debug, Clone)]
-pub struct InlineBuf {
-    buf: [u8; INLINE_CAP],
-    len: u8,
-}
-
-impl InlineBuf {
-    /// Build from bytes that are known to fit ([`INLINE_CAP`]). Panics if too long;
-    /// callers (the classifier path) only construct this for embstr-sized values.
-    #[must_use]
-    pub fn from_bytes(bytes: &[u8]) -> Self {
-        assert!(
-            bytes.len() <= INLINE_CAP,
-            "InlineBuf overflow: {} > {INLINE_CAP}",
-            bytes.len()
-        );
-        let mut buf = [0u8; INLINE_CAP];
-        buf[..bytes.len()].copy_from_slice(bytes);
-        InlineBuf {
-            buf,
-            len: bytes.len() as u8,
-        }
-    }
-
-    /// The inline bytes.
-    #[must_use]
-    pub fn as_bytes(&self) -> &[u8] {
-        &self.buf[..self.len as usize]
-    }
-}
-
 /// The value representation inside a [`KvObj`] (ENCODINGS.md #112).
 #[derive(Debug, Clone)]
 pub enum ValueRepr {
     /// An int-encoded value: the raw i64, NO value allocation (the decimal bytes
     /// are materialized on read). `OBJECT ENCODING` -> int.
     Int(i64),
-    /// A short string stored inline. `OBJECT ENCODING` -> embstr.
-    Inline(InlineBuf),
+    /// A short string (embstr). `OBJECT ENCODING` -> embstr.
+    ///
+    /// BOXED (memory Round 2): the bytes live behind a `Box<[u8]>` rather than a
+    /// fixed inline buffer, so this variant is one pointer wide and the largest
+    /// `ValueRepr` variant shrinks to a `Box<[u8]>`, cutting every per-key `KvObj`
+    /// and the hashbrown table slot. The embstr-vs-raw distinction is the SAME (it is
+    /// recorded in [`Header::encoding`], NOT by the variant): a value classified as
+    /// embstr by [`crate::encoding::EMBSTR_THRESHOLD`] is `Inline`, a longer one is
+    /// [`ValueRepr::Raw`],
+    /// and `OBJECT ENCODING` reports `embstr` / `raw` exactly as before. Redis/Valkey
+    /// also heap-allocate the object body, so this is allocation-parity with redis
+    /// plus a smaller slot.
+    Inline(Box<[u8]>),
     /// A long string stored out-of-line. `OBJECT ENCODING` -> raw.
     Raw(Box<[u8]>),
     /// A LIST value (PR-5). `OBJECT ENCODING` -> `listpack` while small, `quicklist`
     /// once over the threshold (a pure function of the active repr, #40).
-    List(ListVal),
+    ///
+    /// BOXED (memory Round 1): the four collection structs are the large `ValueRepr`
+    /// variants (`ListVal` 40 / `HashVal` 40 / `SetVal` 48 / `ZSetVal` 64). Holding them
+    /// behind a `Box` drops `ValueRepr` to the string-variant bound, which shrinks every
+    /// per-key `KvObj` and the hashbrown table slot. The string/int hot path
+    /// (`Int`/`Inline`/`Raw`) is UNBOXED so the embstr SSO is untouched; the collections
+    /// already heap-allocate their contents, so the `Box` is a negligible extra
+    /// indirection only on collection ops.
+    List(Box<ListVal>),
     /// A HASH value (PR-6). `OBJECT ENCODING` -> `listpack` while small, `hashtable`
     /// once over the entry-count OR per-element-byte threshold (a pure function of the
-    /// active repr, #40).
-    Hash(HashVal),
+    /// active repr, #40). BOXED (memory Round 1); see [`ValueRepr::List`].
+    Hash(Box<HashVal>),
     /// A SET value (PR-7). `OBJECT ENCODING` -> `intset` while all-integer and small,
     /// `listpack` once a non-integer member is added (and still small), `hashtable` once
     /// over the entry-count OR per-member-byte threshold (a pure function of the active
-    /// repr, #40). The conversion is ONE-WAY (never demotes).
-    Set(SetVal),
+    /// repr, #40). The conversion is ONE-WAY (never demotes). BOXED (memory Round 1);
+    /// see [`ValueRepr::List`].
+    Set(Box<SetVal>),
     /// A ZSET (sorted set) value (PR-8). `OBJECT ENCODING` -> `listpack` while small,
     /// `skiplist` once over the entry-count OR per-member-byte threshold (a pure function
-    /// of the active repr, #40). The conversion is ONE-WAY (never demotes).
-    ZSet(ZSetVal),
+    /// of the active repr, #40). The conversion is ONE-WAY (never demotes). BOXED
+    /// (memory Round 1); see [`ValueRepr::List`].
+    ZSet(Box<ZSetVal>),
 }
 
 impl ValueRepr {
@@ -187,8 +168,9 @@ impl ValueRepr {
     pub fn logical_len(&self) -> usize {
         match self {
             ValueRepr::Int(n) => int_decimal_len(*n),
-            ValueRepr::Inline(b) => b.as_bytes().len(),
-            ValueRepr::Raw(b) => b.len(),
+            // Embstr and raw both hold the value bytes behind a `Box<[u8]>`; the
+            // embstr-vs-raw distinction lives in `Header.encoding`, not the variant.
+            ValueRepr::Inline(b) | ValueRepr::Raw(b) => b.len(),
             ValueRepr::List(l) => l.element_bytes(),
             ValueRepr::Hash(h) => h.element_bytes(),
             ValueRepr::Set(s) => s.element_bytes(),
@@ -1714,7 +1696,7 @@ impl KvObj {
     ) -> Self {
         let value = match classified {
             Classified::Int(n) => ValueRepr::Int(n),
-            Classified::EmbStr => ValueRepr::Inline(InlineBuf::from_bytes(bytes)),
+            Classified::EmbStr => ValueRepr::Inline(bytes.to_vec().into_boxed_slice()),
             Classified::Raw => ValueRepr::Raw(bytes.to_vec().into_boxed_slice()),
         };
         let header = Header::new(value.encoding(), expire_at.is_some());
@@ -1800,7 +1782,7 @@ impl KvObj {
         KvObj {
             header: Header::with_type(DataType::List, encoding, expire_at.is_some()),
             key: key.to_vec().into_boxed_slice(),
-            value: ValueRepr::List(list),
+            value: ValueRepr::List(Box::new(list)),
             expire_at,
         }
     }
@@ -1814,7 +1796,7 @@ impl KvObj {
         KvObj {
             header: Header::with_type(DataType::Hash, encoding, expire_at.is_some()),
             key: key.to_vec().into_boxed_slice(),
-            value: ValueRepr::Hash(hash),
+            value: ValueRepr::Hash(Box::new(hash)),
             expire_at,
         }
     }
@@ -1828,7 +1810,7 @@ impl KvObj {
         KvObj {
             header: Header::with_type(DataType::Set, encoding, expire_at.is_some()),
             key: key.to_vec().into_boxed_slice(),
-            value: ValueRepr::Set(set),
+            value: ValueRepr::Set(Box::new(set)),
             expire_at,
         }
     }
@@ -1842,7 +1824,7 @@ impl KvObj {
         KvObj {
             header: Header::with_type(DataType::ZSet, encoding, expire_at.is_some()),
             key: key.to_vec().into_boxed_slice(),
-            value: ValueRepr::ZSet(zset),
+            value: ValueRepr::ZSet(Box::new(zset)),
             expire_at,
         }
     }
@@ -1861,7 +1843,9 @@ impl KvObj {
     /// yields `None` -> WRONGTYPE).
     pub fn as_list_mut(&mut self) -> Option<&mut ListVal> {
         match &mut self.value {
-            ValueRepr::List(l) => Some(l),
+            // Deref through the `Box` (memory Round 1) to the `&mut ListVal` the
+            // collection trait + in-place RMW path expect.
+            ValueRepr::List(l) => Some(&mut **l),
             _ => None,
         }
     }
@@ -1871,7 +1855,7 @@ impl KvObj {
     /// `None` -> WRONGTYPE). The HASH analog of [`Self::as_list_mut`].
     pub fn as_hash_mut(&mut self) -> Option<&mut HashVal> {
         match &mut self.value {
-            ValueRepr::Hash(h) => Some(h),
+            ValueRepr::Hash(h) => Some(&mut **h),
             _ => None,
         }
     }
@@ -1881,7 +1865,7 @@ impl KvObj {
     /// -> WRONGTYPE). The SET analog of [`Self::as_list_mut`]/[`Self::as_hash_mut`].
     pub fn as_set_mut(&mut self) -> Option<&mut SetVal> {
         match &mut self.value {
-            ValueRepr::Set(s) => Some(s),
+            ValueRepr::Set(s) => Some(&mut **s),
             _ => None,
         }
     }
@@ -1892,7 +1876,7 @@ impl KvObj {
     /// [`Self::as_set_mut`].
     pub fn as_zset_mut(&mut self) -> Option<&mut ZSetVal> {
         match &mut self.value {
-            ValueRepr::ZSet(z) => Some(z),
+            ValueRepr::ZSet(z) => Some(&mut **z),
             _ => None,
         }
     }
@@ -1972,7 +1956,7 @@ impl KvObj {
     pub fn set_value_bytes(&mut self, bytes: &[u8]) {
         self.value = match classify(bytes) {
             Classified::Int(n) => ValueRepr::Int(n),
-            Classified::EmbStr => ValueRepr::Inline(InlineBuf::from_bytes(bytes)),
+            Classified::EmbStr => ValueRepr::Inline(bytes.to_vec().into_boxed_slice()),
             Classified::Raw => ValueRepr::Raw(bytes.to_vec().into_boxed_slice()),
         };
         self.header.encoding = self.value.encoding();

crates/ironcache-store/src/lib.rs

@@ -576,13 +576,9 @@ impl<E: EvictionHook, A: AccountingHook> ShardStore<E, A> {
             kvobj::ValueRepr::Int(n) => {
                 ValueRef::from_int_bytes(obj.header.data_type, obj.expire_at, int_decimal_bytes(*n))
             }
-            kvobj::ValueRepr::Inline(b) => ValueRef::borrowed(
-                obj.header.data_type,
-                obj.header.encoding,
-                obj.expire_at,
-                b.as_bytes(),
-            ),
-            kvobj::ValueRepr::Raw(b) => {
+            // Embstr and raw both borrow their bytes the same way; the embstr-vs-raw
+            // distinction is carried by `obj.header.encoding`, not the variant.
+            kvobj::ValueRepr::Inline(b) | kvobj::ValueRepr::Raw(b) => {
                 ValueRef::borrowed(obj.header.data_type, obj.header.encoding, obj.expire_at, b)
             }
             // A LIST/HASH/SET is not byte-readable as a string: the command layer only
@@ -612,13 +608,9 @@ impl<E: EvictionHook, A: AccountingHook> ShardStore<E, A> {
                 obj.expire_at,
                 int_decimal_bytes(*n),
             ),
-            kvobj::ValueRepr::Inline(b) => OccupiedEntry::borrowed(
-                obj.header.data_type,
-                obj.header.encoding,
-                obj.expire_at,
-                b.as_bytes(),
-            ),
-            kvobj::ValueRepr::Raw(b) => {
+            // Embstr and raw both borrow their bytes the same way; the embstr-vs-raw
+            // distinction is carried by `obj.header.encoding`, not the variant.
+            kvobj::ValueRepr::Inline(b) | kvobj::ValueRepr::Raw(b) => {
                 OccupiedEntry::borrowed(obj.header.data_type, obj.header.encoding, obj.expire_at, b)
             }
             // A LIST/HASH/SET observed through the READ-ONLY rmw arm (e.g. a numeric RMW
@@ -823,17 +815,20 @@ impl<E: EvictionHook, A: AccountingHook> Store for ShardStore<E, A> {
             // would each take and drop a fresh `&mut` and obscure the dispatch) so each
             // collection type maps to exactly one arm.
             let entry = match &mut obj.value {
+                // The collection variants are boxed (memory Round 1); deref through the
+                // `Box` (`&mut **`) to the concrete `&mut *Val`, which then coerces to the
+                // `&mut dyn *Value` trait object the typed view constructors take.
                 kvobj::ValueRepr::List(l) => {
-                    RmwEntry::OccupiedMut(OccupiedEntryMut::list(encoding, expire_at, l))
+                    RmwEntry::OccupiedMut(OccupiedEntryMut::list(encoding, expire_at, &mut **l))
                 }
                 kvobj::ValueRepr::Hash(h) => {
-                    RmwEntry::OccupiedMut(OccupiedEntryMut::hash(encoding, expire_at, h))
+                    RmwEntry::OccupiedMut(OccupiedEntryMut::hash(encoding, expire_at, &mut **h))
                 }
                 kvobj::ValueRepr::Set(s) => {
-                    RmwEntry::OccupiedMut(OccupiedEntryMut::set(encoding, expire_at, s))
+                    RmwEntry::OccupiedMut(OccupiedEntryMut::set(encoding, expire_at, &mut **s))
                 }
                 kvobj::ValueRepr::ZSet(z) => {
-                    RmwEntry::OccupiedMut(OccupiedEntryMut::zset(encoding, expire_at, z))
+                    RmwEntry::OccupiedMut(OccupiedEntryMut::zset(encoding, expire_at, &mut **z))
                 }
                 kvobj::ValueRepr::Int(_)
                 | kvobj::ValueRepr::Inline(_)

crates/ironcache-store/tests/keyspace.rs

@@ -13,7 +13,7 @@ use ironcache_storage::{
     UnixMillis,
 };
 use ironcache_store::ShardStore;
-use ironcache_store::kvobj::{Header, InlineBuf, KvObj, ValueRepr};
+use ironcache_store::kvobj::{Header, KvObj, ValueRepr};
 use std::collections::HashSet;
 
 const NOW: UnixMillis = UnixMillis(1_000);
@@ -256,7 +256,7 @@ fn scan_type_filter_selects_by_data_type() {
         ttl_present: false,
         snapshot_version: 0,
     };
-    lst.value = ValueRepr::Inline(InlineBuf::from_bytes(b"x"));
+    lst.value = ValueRepr::Inline(Box::from(&b"x"[..]));
     s.insert_object(0, lst);
 
     let mut out = Vec::new();