From 1ae3cdcb766cff21b34f11fd39852dc9e93a47be Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 17 Jul 2025 18:31:33 +0300 Subject: [PATCH] clean used utxos from the long term cache --- Cargo.lock | 2 +- Cargo.toml | 2 +- components/runes/src/db/cache/index_cache.rs | 26 +++++++++++++++++++- components/runes/src/db/cache/utils.rs | 18 ++++++++++++-- components/runes/src/db/index.rs | 1 + 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3a29eb3..92397f3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,7 +725,7 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cli" -version = "3.0.0" +version = "3.0.3" dependencies = [ "bitcoind", "clap 3.2.25", diff --git a/Cargo.toml b/Cargo.toml index 31da5915..eb892856 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,4 +21,4 @@ tokio-postgres = "0.7.10" prometheus = "0.13.3" [workspace.package] -version = "3.0.0" +version = "3.0.3" diff --git a/components/runes/src/db/cache/index_cache.rs b/components/runes/src/db/cache/index_cache.rs index 75e2edae..bdd2037e 100644 --- a/components/runes/src/db/cache/index_cache.rs +++ b/components/runes/src/db/cache/index_cache.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, num::NonZeroUsize, str::FromStr}; +use std::{ + collections::{HashMap, HashSet}, + num::NonZeroUsize, + str::FromStr, +}; use bitcoin::{Network, ScriptBuf}; use bitcoind::{try_debug, try_warn, types::bitcoin::TxIn, utils::Context}; @@ -35,6 +39,8 @@ pub struct IndexCache { /// Same as above but only for the current block. We use a `HashMap` instead of an LRU cache to make sure we keep all outputs /// in memory while we index this block. Must be cleared every time a new block is processed. block_output_cache: HashMap<(String, u32), HashMap>>, + /// Tracks UTXOs consumed from output_cache during this block. These will be removed after successful commit. + consumed_utxos: HashSet<(String, u32)>, /// Holds a single transaction's rune cache. Must be cleared every time a new transaction is processed. tx_cache: TransactionCache, /// Keeps rows that have not yet been inserted in the DB. @@ -52,6 +58,7 @@ impl IndexCache { rune_total_mints_cache: LruCache::new(cap), output_cache: LruCache::new(cap), block_output_cache: HashMap::new(), + consumed_utxos: HashSet::new(), tx_cache: TransactionCache::new( TransactionLocation { network, @@ -89,6 +96,7 @@ impl IndexCache { tx_inputs, &self.block_output_cache, &mut self.output_cache, + &mut self.consumed_utxos, db_tx, ctx, ) @@ -127,6 +135,22 @@ impl IndexCache { ); } + /// Removes all consumed UTXOs from the output LRU cache. + pub fn clear_consumed_utxos_from_cache(&mut self, ctx: &Context) { + let removed_count = self.consumed_utxos.len(); + for utxo_key in self.consumed_utxos.drain() { + // Remove from LRU cache + self.output_cache.pop(&utxo_key); + } + if removed_count > 0 { + try_debug!( + ctx, + "Cleaned {} spent UTXOs from output cache", + removed_count + ); + } + } + pub async fn apply_runestone( &mut self, runestone: &Runestone, diff --git a/components/runes/src/db/cache/utils.rs b/components/runes/src/db/cache/utils.rs index fb33833d..942d1302 100644 --- a/components/runes/src/db/cache/utils.rs +++ b/components/runes/src/db/cache/utils.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use bitcoin::{Address, ScriptBuf}; use bitcoind::{try_debug, try_warn, types::bitcoin::TxIn, utils::Context}; @@ -22,12 +22,14 @@ use crate::db::{ /// * `inputs` - Raw transaction inputs /// * `block_output_cache` - Cache with output balances produced by the current block /// * `output_cache` - LRU cache with output balances +/// * `consumed_utxos` - HashSet to track UTXOs consumed from output_cache for later cleanup /// * `db_tx` - DB transaction /// * `ctx` - Context pub async fn input_rune_balances_from_tx_inputs( inputs: &Vec, block_output_cache: &HashMap<(String, u32), HashMap>>, output_cache: &mut LruCache<(String, u32), HashMap>>, + consumed_utxos: &mut HashSet<(String, u32)>, db_tx: &mut Transaction<'_>, ctx: &Context, ) -> HashMap> { @@ -42,8 +44,12 @@ pub async fn input_rune_balances_from_tx_inputs( let k = (tx_id.clone(), vout); if let Some(map) = block_output_cache.get(&k) { indexed_input_runes.insert(i as u32, map.clone()); + // Track block cache UTXO for removal from LRU after successful commit + consumed_utxos.insert(k); } else if let Some(map) = output_cache.get(&k) { indexed_input_runes.insert(i as u32, map.clone()); + // Track LRU cache UTXO for removal after successful commit + consumed_utxos.insert(k); } else { cache_misses.push((i as u32, tx_id, vout)); } @@ -666,7 +672,7 @@ mod test { } mod input_balances { - use std::num::NonZeroUsize; + use std::{collections::HashSet, num::NonZeroUsize}; use bitcoind::{ types::{ @@ -711,6 +717,7 @@ mod test { } }; let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap()); + let mut consumed_utxos = HashSet::new(); let ctx = Context::empty(); let mut pg_client = pg_test_client(true, &ctx).await; @@ -719,6 +726,7 @@ mod test { &inputs, &block_output_cache, &mut output_cache, + &mut consumed_utxos, &mut db_tx, &ctx, ) @@ -762,6 +770,7 @@ mod test { rune_id => vec![InputRuneBalance { address: None, amount: 2000 }] }, ); + let mut consumed_utxos = HashSet::new(); let ctx = Context::empty(); let mut pg_client = pg_test_client(true, &ctx).await; @@ -770,6 +779,7 @@ mod test { &inputs, &block_output_cache, &mut output_cache, + &mut consumed_utxos, &mut db_tx, &ctx, ) @@ -804,6 +814,7 @@ mod test { let rune_id = RuneId::new(840000, 25).unwrap(); let block_output_cache = hashmap! {}; let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap()); + let mut consumed_utxos = HashSet::new(); let ctx = Context::empty(); let mut pg_client = pg_test_client(true, &ctx).await; @@ -829,6 +840,7 @@ mod test { &inputs, &block_output_cache, &mut output_cache, + &mut consumed_utxos, &mut db_tx, &ctx, ) @@ -862,6 +874,7 @@ mod test { }]; let block_output_cache = hashmap! {}; let mut output_cache = LruCache::new(NonZeroUsize::new(1).unwrap()); + let mut consumed_utxos = HashSet::new(); let ctx = Context::empty(); let mut pg_client = pg_test_client(true, &ctx).await; @@ -870,6 +883,7 @@ mod test { &inputs, &block_output_cache, &mut output_cache, + &mut consumed_utxos, &mut db_tx, &ctx, ) diff --git a/components/runes/src/db/index.rs b/components/runes/src/db/index.rs index 603f0c18..c675ec3a 100644 --- a/components/runes/src/db/index.rs +++ b/components/runes/src/db/index.rs @@ -183,6 +183,7 @@ pub async fn index_block( .commit() .await .expect("Unable to commit pg transaction"); + index_cache.clear_consumed_utxos_from_cache(ctx); prometheus.metrics_record_rune_db_write_time(rune_db_write_start.elapsed().as_millis() as f64); prometheus.metrics_record_runes_etching_per_block(etching_count);