clean used utxos from the long term cache

Cargo.toml

@@ -21,4 +21,4 @@ tokio-postgres = "0.7.10"
 prometheus = "0.13.3"
 
 [workspace.package]
-version = "3.0.0"
+version = "3.0.3"

components/runes/src/db/cache/index_cache.rs

@@ -1,4 +1,8 @@
-use std::{collections::HashMap, num::NonZeroUsize, str::FromStr};
+use std::{
+    collections::{HashMap, HashSet},
+    num::NonZeroUsize,
+    str::FromStr,
+};
 
 use bitcoin::{Network, ScriptBuf};
 use bitcoind::{try_debug, try_warn, types::bitcoin::TxIn, utils::Context};
@@ -35,6 +39,8 @@ pub struct IndexCache {
     /// Same as above but only for the current block. We use a `HashMap` instead of an LRU cache to make sure we keep all outputs
     /// in memory while we index this block. Must be cleared every time a new block is processed.
     block_output_cache: HashMap<(String, u32), HashMap<RuneId, Vec<InputRuneBalance>>>,
+    /// Tracks UTXOs consumed from output_cache during this block. These will be removed after successful commit.
+    consumed_utxos: HashSet<(String, u32)>,
     /// Holds a single transaction's rune cache. Must be cleared every time a new transaction is processed.
     tx_cache: TransactionCache,
     /// Keeps rows that have not yet been inserted in the DB.
@@ -52,6 +58,7 @@ impl IndexCache {
             rune_total_mints_cache: LruCache::new(cap),
             output_cache: LruCache::new(cap),
             block_output_cache: HashMap::new(),
+            consumed_utxos: HashSet::new(),
             tx_cache: TransactionCache::new(
                 TransactionLocation {
                     network,
@@ -89,6 +96,7 @@ impl IndexCache {
             tx_inputs,
             &self.block_output_cache,
             &mut self.output_cache,
+            &mut self.consumed_utxos,
             db_tx,
             ctx,
         )
@@ -127,6 +135,22 @@ impl IndexCache {
         );
     }
 
+    /// Removes all consumed UTXOs from the output LRU cache.
+    pub fn clear_consumed_utxos_from_cache(&mut self, ctx: &Context) {
+        let removed_count = self.consumed_utxos.len();
+        for utxo_key in self.consumed_utxos.drain() {
+            // Remove from LRU cache
+            self.output_cache.pop(&utxo_key);
+        }
+        if removed_count > 0 {
+            try_debug!(
+                ctx,
+                "Cleaned {} spent UTXOs from output cache",
+                removed_count
+            );
+        }
+    }
+
     pub async fn apply_runestone(
         &mut self,
         runestone: &Runestone,

components/runes/src/db/cache/utils.rs

@@ -1,4 +1,4 @@
-use std::collections::{HashMap, VecDeque};
+use std::collections::{HashMap, HashSet, VecDeque};
 
 use bitcoin::{Address, ScriptBuf};
 use bitcoind::{try_debug, try_warn, types::bitcoin::TxIn, utils::Context};
@@ -22,12 +22,14 @@ use crate::db::{
 /// * `inputs` - Raw transaction inputs
 /// * `block_output_cache` - Cache with output balances produced by the current block
 /// * `output_cache` - LRU cache with output balances
+/// * `consumed_utxos` - HashSet to track UTXOs consumed from output_cache for later cleanup
 /// * `db_tx` - DB transaction
 /// * `ctx` - Context
 pub async fn input_rune_balances_from_tx_inputs(
     inputs: &Vec<TxIn>,
     block_output_cache: &HashMap<(String, u32), HashMap<RuneId, Vec<InputRuneBalance>>>,
     output_cache: &mut LruCache<(String, u32), HashMap<RuneId, Vec<InputRuneBalance>>>,
+    consumed_utxos: &mut HashSet<(String, u32)>,
     db_tx: &mut Transaction<'_>,
     ctx: &Context,
 ) -> HashMap<RuneId, VecDeque<InputRuneBalance>> {
@@ -42,8 +44,12 @@ pub async fn input_rune_balances_from_tx_inputs(
         let k = (tx_id.clone(), vout);
         if let Some(map) = block_output_cache.get(&k) {
             indexed_input_runes.insert(i as u32, map.clone());
+            // Track block cache UTXO for removal from LRU after successful commit
+            consumed_utxos.insert(k);
         } else if let Some(map) = output_cache.get(&k) {
             indexed_input_runes.insert(i as u32, map.clone());
+            // Track LRU cache UTXO for removal after successful commit
+            consumed_utxos.insert(k);
         } else {
             cache_misses.push((i as u32, tx_id, vout));
         }
@@ -666,7 +672,7 @@ mod test {
     }
 
     mod input_balances {
-        use std::num::NonZeroUsize;
+        use std::{collections::HashSet, num::NonZeroUsize};
 
         use bitcoind::{
             types::{
@@ -711,6 +717,7 @@ mod test {
                             }
             };
             let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap());
+            let mut consumed_utxos = HashSet::new();
             let ctx = Context::empty();
 
             let mut pg_client = pg_test_client(true, &ctx).await;
@@ -719,6 +726,7 @@ mod test {
                 &inputs,
                 &block_output_cache,
                 &mut output_cache,
+                &mut consumed_utxos,
                 &mut db_tx,
                 &ctx,
             )
@@ -762,6 +770,7 @@ mod test {
                     rune_id => vec![InputRuneBalance { address: None, amount: 2000 }]
                 },
             );
+            let mut consumed_utxos = HashSet::new();
             let ctx = Context::empty();
 
             let mut pg_client = pg_test_client(true, &ctx).await;
@@ -770,6 +779,7 @@ mod test {
                 &inputs,
                 &block_output_cache,
                 &mut output_cache,
+                &mut consumed_utxos,
                 &mut db_tx,
                 &ctx,
             )
@@ -804,6 +814,7 @@ mod test {
             let rune_id = RuneId::new(840000, 25).unwrap();
             let block_output_cache = hashmap! {};
             let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap());
+            let mut consumed_utxos = HashSet::new();
             let ctx = Context::empty();
 
             let mut pg_client = pg_test_client(true, &ctx).await;
@@ -829,6 +840,7 @@ mod test {
                 &inputs,
                 &block_output_cache,
                 &mut output_cache,
+                &mut consumed_utxos,
                 &mut db_tx,
                 &ctx,
             )
@@ -862,6 +874,7 @@ mod test {
             }];
             let block_output_cache = hashmap! {};
             let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap());
+            let mut consumed_utxos = HashSet::new();
             let ctx = Context::empty();
 
             let mut pg_client = pg_test_client(true, &ctx).await;
@@ -870,6 +883,7 @@ mod test {
                 &inputs,
                 &block_output_cache,
                 &mut output_cache,
+                &mut consumed_utxos,
                 &mut db_tx,
                 &ctx,
             )

components/runes/src/db/index.rs

@@ -183,6 +183,7 @@ pub async fn index_block(
         .commit()
         .await
         .expect("Unable to commit pg transaction");
+    index_cache.clear_consumed_utxos_from_cache(ctx);
     prometheus.metrics_record_rune_db_write_time(rune_db_write_start.elapsed().as_millis() as f64);
 
     prometheus.metrics_record_runes_etching_per_block(etching_count);