diff --git a/.opencode/plans/nv2-experiment-plan.md b/.opencode/plans/nv2-experiment-plan.md new file mode 100644 index 0000000000..54ac7742a8 --- /dev/null +++ b/.opencode/plans/nv2-experiment-plan.md @@ -0,0 +1,194 @@ +# NV-2 execution plan: Experiment + upstream issue + +Approved by user: benchmark branch → run → GitHub issue → update audit record. +Blocked by plan-mode permissions; execute when switched to normal mode. + +## Steps + +1. **Create** `chainstate/test-suite/benches/reorg_growth.rs` (source below) and add to `chainstate/test-suite/Cargo.toml` after the existing `[[bench]]`: + ```toml + [[bench]] + name = "reorg_growth" + harness = false + ``` +2. **Run**: `cargo bench --offline -p chainstate-test-suite --bench reorg_growth` (release profile; deps cached). Capture table. +3. **Branch + push**: `git checkout -b bench/pos-sidechain-reorg-growth`, commit both files, push to origin. +4. **Issue** on mintlayer/mintlayer-core: trace + refinements (reorg-before-consensus ordering = zero-marginal-cost garbage triggers; no trust pre-filter; losing branches persisted) + bench table + fix options (branch-state cache / configured connect-side cap / partial pre-checks). +5. **Update** `~/security-audit-skill/mintlayer-core/run-1/NEEDS-VALIDATION.md` NV-2 entry with refinements + links. + +## Bench source (composed, signatures verified against framework.rs / pos_block_builder.rs / benches.rs) + +Key facts used: `tf.chainstate` public (framework.rs:62); `make_pos_block_builder()` (framework.rs:105); builder `with_parent/with_stake_pool_id/with_stake_spending_key/with_vrf_key/build` (pos_block_builder.rs:155-291); `process_block` via `ChainstateInterface` (chainstate re-exported from test-framework's use: `use chainstate::{BlockSource, ChainstateError, chainstate_interface::ChainstateInterface}`); setup copied from `pos_reorg` bench (benches.rs:51-78). + +```rust +// (license header: MIT, same as benches.rs) + +use std::time::Instant; + +use chainstate::{BlockSource, chainstate_interface::ChainstateInterface}; +use chainstate_test_framework::TestFramework; +use common::{ + chain::{Block, Destination, GenBlock, PoolId, config::create_unit_test_config, stakelock::StakePoolData}, + primitives::{Amount, BlockDistance, H256, Id, Idable, per_thousand::PerThousand}, +}; +use crypto::{ + key::{KeyKind, PrivateKey}, + vrf::{VRFKeyKind, VRFPrivateKey}, +}; +use test_utils::random::make_seedable_rng; + +const MAIN_CHAIN_MARGIN: usize = 10; +const WINDOW: usize = 5; + +fn make_pos_chain(rng: &mut impl CryptoRng) -> (TestFramework, PoolId, PrivateKey, VRFPrivateKey) { + let (staking_sk, staking_pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let (vrf_sk, vrf_pk) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + + let genesis_pool_id = PoolId::new(H256::random_using(rng)); + let stake_pool_pledge = create_unit_test_config().min_stake_pool_pledge(); + let stake_pool_data = StakePoolData::new( + stake_pool_pledge, + Destination::PublicKey(staking_pk), + vrf_pk, + Destination::AnyoneCanSpend, + PerThousand::new(1000).unwrap(), + Amount::ZERO, + ); + + let chain_config = chainstate_test_framework::create_chain_config_with_staking_pool( + rng, + Amount::from_atoms(1000), + genesis_pool_id, + stake_pool_data, + ) + .max_depth_for_reorg(BlockDistance::new(5000)) + .build(); + let target_block_time = chain_config.target_block_spacing(); + + let mut tf = TestFramework::builder(rng).with_chain_config(chain_config).build(); + tf.progress_time_seconds_since_epoch(target_block_time.as_secs()); + + (tf, genesis_pool_id, staking_sk, vrf_sk) +} + +fn build_side_block( + tf: &mut TestFramework, + rng: &mut impl CryptoRng, + parent: &Id, + pool_id: PoolId, + staking_sk: &PrivateKey, + vrf_sk: &VRFPrivateKey, +) -> Block { + tf.make_pos_block_builder() + .with_parent(*parent) + .with_stake_pool_id(pool_id) + .with_stake_spending_key(staking_sk.clone()) + .with_vrf_key(vrf_sk.clone()) + .build(rng) +} + +/// Structurally valid PoS block signed by a random wrong key: passes +/// parent/size/checkpoint/reorg-depth checks, triggers the full in-memory +/// reorg, and only fails consensus validation afterwards. +fn build_garbage_block( + tf: &mut TestFramework, + rng: &mut impl CryptoRng, + parent: &Id, + pool_id: PoolId, +) -> Block { + let (wrong_sk, _) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let (wrong_vrf_sk, _) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + tf.make_pos_block_builder() + .with_parent(*parent) + .with_stake_pool_id(pool_id) + .with_stake_spending_key(wrong_sk) + .with_vrf_key(wrong_vrf_sk) + .build(rng) +} + +fn report_row(label: &str, samples: &[f64]) { + let n = samples.len(); + let mean: f64 = samples.iter().sum::() / n as f64; + let mut sorted = samples.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let median = sorted[n / 2]; + println!("{label:>28} | mean {mean:>10.3} ms | median {median:>10.3} ms"); +} + +fn run_case(n: usize, rng: &mut impl CryptoRng) { + let (mut tf, pool_id, staking_sk, vrf_sk) = make_pos_chain(rng); + + let common_block_id = tf + .create_chain_pos(rng, &tf.genesis().get_id().into(), 5, pool_id, &staking_sk, &vrf_sk) + .unwrap(); + + let started = Instant::now(); + tf.create_chain_pos(rng, &common_block_id, n + MAIN_CHAIN_MARGIN, pool_id, &staking_sk, &vrf_sk) + .unwrap(); + println!( + "N={n}: main chain of {} built in {:.2}s", + n + MAIN_CHAIN_MARGIN, + started.elapsed().as_secs_f64() + ); + + // Losing-branch side chain: submit blocks one at a time, timing process_block only. + let mut prev = common_block_id; + let mut window_samples: Vec = Vec::with_capacity(WINDOW); + let quarter = (n / 4).max(WINDOW); + let mut checkpoints: Vec = [1, quarter, quarter * 2, quarter * 3, n] + .into_iter() + .map(|x| x.max(1).min(n)) + .collect(); + checkpoints.sort(); + checkpoints.dedup(); + + let total_start = Instant::now(); + for i in 1..=n { + let block = build_side_block(&mut tf, rng, &prev, pool_id, &staking_sk, &vrf_sk); + let block_id: Id = block.get_id().into(); + + let t = Instant::now(); + tf.chainstate.process_block(block, BlockSource::Local).unwrap(); + window_samples.push(t.elapsed().as_secs_f64() * 1000.0); + + if Some(&i) == checkpoints.first() { + let label = format!("side arrival #{i} (branch len {})", i - 1); + report_row(&label, &window_samples); + window_samples.clear(); + checkpoints.remove(0); + } + prev = block_id; + } + println!( + "N={n}: side chain of {n} accepted in {:.2}s", + total_start.elapsed().as_secs_f64() + ); + + // Trigger phase: garbage blocks pointing at the side-chain tip. Each is + // rejected, but only after paying the full in-memory reorg. + let mut garbage_samples = Vec::with_capacity(WINDOW); + for _ in 0..WINDOW { + let garbage = build_garbage_block(&mut tf, rng, &prev, pool_id); + let t = Instant::now(); + let res = tf.chainstate.process_block(garbage, BlockSource::Local); + let elapsed = t.elapsed().as_secs_f64() * 1000.0; + assert!(res.is_err(), "garbage block was unexpectedly accepted"); + garbage_samples.push(elapsed); + } + report_row(&format!("garbage @branch len {n}"), &garbage_samples); + println!(); +} + +fn main() { + let mut rng = make_seedable_rng(4242.into()); + println!("PoS side-chain growth cost (per-arrival process_block, in-memory store)\n"); + for n in [100usize, 200, 400] { + run_case(n, &mut rng); + } +} +``` + +## Notes +- Garbage blocks use wrong staking/VRF keys (structurally valid, consensus-invalid) — distinct keys give distinct ids, no seal grinding needed. +- Side chain length ≤ main chain length (margin 10) so it stays a losing branch; per-arrival reorg cost is paid regardless of trust (verified: no trust pre-filter, reorganize_in_memory at chainstateref/mod.rs:688 runs before validate_consensus at :703). +- If compile fails on imports (e.g. `Block`/`GenBlock` paths), check `common::chain` re-exports used by framework.rs imports. diff --git a/Cargo.lock b/Cargo.lock index 83d80726c4..2a9bd19b8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,7 @@ dependencies = [ "api-server-common", "async-trait", "axum 0.7.9", + "base64 0.22.1", "build_utils", "chainstate-test-framework", "clap", diff --git a/api-server/CHANGELOG.md b/api-server/CHANGELOG.md index f3d88a2211..c5b77528a7 100644 --- a/api-server/CHANGELOG.md +++ b/api-server/CHANGELOG.md @@ -16,6 +16,7 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/ Transactions seen in the node's mempool are bridged into `tx_seen` events by the web server. ### Changed +- The `items` query parameter is now validated the same way on every paginated `v2` endpoint: `items=0` is rejected with `400 invalid num items` instead of returning an empty page. Previously the offset-based endpoints returned an empty listing for `items=0`; the new keyset-paginated endpoints (pools, transactions, holders, order book) reject it from the start. - The api-server storage version was bumped from 25 to 26 (new `ml.emitted_events` table); the scanner re-initializes the database when it finds a different version, as before. Full resync is required. - The stream event retention pruning no longer deletes events that the event pump has not consumed yet: the pump records its progress in the database and the pruning never overtakes it (with a hard limit of 100k retained events before the first progress record or during a pump outage, logged as an error when it kicks in). - A terminated streaming background task (the event pump or the mempool bridge) now brings the web server process down instead of leaving the event stream silently dead while the REST endpoints keep working. diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 9d59e68b36..7b6a839049 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -26,6 +26,7 @@ use itertools::Itertools as _; use common::{ address::Address, + address::traits::Addressable, chain::{ Block, ChainConfig, DelegationId, Destination, Genesis, OrderId, PoolId, Transaction, UtxoOutPoint, @@ -37,8 +38,9 @@ use common::{ use crate::storage::storage_api::{ AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, - FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoLock, UtxoWithExtraInfo, + FungibleTokenData, LockedUtxo, NftWithOwner, ORDER_BOOK_MAX_ORDERS, Order, PoolBlockStats, + PoolDataWithExtraInfo, TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoLock, UtxoWithExtraInfo, block_aux_data::{BlockAuxData, BlockWithExtraData}, }; @@ -60,6 +62,13 @@ impl Ord for TokenTransactionOrderedByTxId { } #[derive(Debug, Clone)] +/// An in-memory api-server storage, meant for tests. +/// +/// The listing reads (pool listings, top holders, order book) scan and sort their full +/// tables on every call instead of maintaining the incrementally-updated caches the +/// postgres backend has (`ml.latest_pool_data_cache`, `ml.address_amount`, +/// `ml.latest_orders_cache`); the contracts (ordering, filters, caps) are identical, +/// only the asymptotics differ, which is acceptable for test-sized data. struct ApiServerInMemoryStorage { block_table: BTreeMap, BlockWithExtraData>, block_aux_data_table: BTreeMap, BlockAuxData>, @@ -85,6 +94,9 @@ struct ApiServerInMemoryStorage { genesis_block: Arc>, number_of_coin_decimals: u8, storage_version: u32, + // kept so that order/pool identifiers can be encoded exactly like the postgres + // backend does (bech32 address strings), keeping tie-break orders identical + chain_config: ChainConfig, } impl ApiServerInMemoryStorage { @@ -112,9 +124,33 @@ impl ApiServerInMemoryStorage { genesis_block: chain_config.genesis_block().clone(), number_of_coin_decimals: chain_config.coin_decimals(), storage_version: CURRENT_STORAGE_VERSION, + chain_config: chain_config.clone(), } } + /// Encoded (bech32) form of an addressable object id — the exact string the + /// postgres backend stores and orders by, keeping tie-break orders identical + /// across backends. Shared by the pool-id and order-id sort keys. + fn id_sort_key( + &self, + id: T, + label: &str, + ) -> Result { + Address::new(&self.chain_config, id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!("invalid {label} {id}: {e}")) + }) + .map(|address| address.as_str().to_owned()) + } + + fn pool_id_sort_key(&self, pool_id: &PoolId) -> Result { + self.id_sort_key(*pool_id, "pool id") + } + + fn order_id_sort_key(&self, order_id: &OrderId) -> Result { + self.id_sort_key(*order_id, "order id") + } + fn is_initialized(&self) -> Result { Ok(true) } @@ -133,6 +169,38 @@ impl ApiServerInMemoryStorage { ) } + fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + let mut holders = self + .address_balance_table + .iter() + .filter_map(|(address, by_coin_or_token)| { + by_coin_or_token + .get(&coin_or_token_id) + .and_then(|by_height| by_height.values().last()) + .filter(|amount| **amount != Amount::ZERO) + .map(|amount| (address.clone(), *amount)) + }) + .collect::>(); + + holders.sort_by(|(l_address, l_amount), (r_address, r_amount)| { + (r_amount, r_address.as_str()).cmp(&(l_amount, l_address.as_str())) + }); + + if let Some((cursor_address, cursor_amount)) = cursor { + holders.retain(|(address, amount)| { + *amount < cursor_amount + || (*amount == cursor_amount && address.as_str() < cursor_address.as_str()) + }); + } + + Ok(holders.into_iter().take(len as usize).collect()) + } + fn get_address_balances( &self, address: &str, @@ -518,6 +586,101 @@ impl ApiServerInMemoryStorage { Ok(latest_orders) } + fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + let entries: Vec<_> = self + .orders_table + .iter() + .filter_map(|(order_id, by_height)| Some((order_id, by_height.values().last()?))) + .filter(|(_order_id, order)| { + order.ask_currency == ask_currency + && order.give_currency == give_currency + && !order.is_frozen + && order.ask_balance != Amount::ZERO + && order.give_balance != Amount::ZERO + }) + .collect(); + + // deepest orders first, ties broken by the encoded order id (byte order) and + // capped exactly like the postgres query; has_more reports the truncation. + // the sort key is encoded once per entry, not once per comparison + let mut decorated: Vec<(String, Amount, Amount)> = entries + .into_iter() + .map(|(order_id, order)| { + Ok(( + self.order_id_sort_key(order_id)?, + order.ask_balance, + order.give_balance, + )) + }) + .collect::, ApiServerStorageError>>()?; + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(key_a, ask_a, _), (key_b, ask_b, _)| { + (Reverse(ask_a.into_atoms()), Reverse(&**key_a)) + .cmp(&(Reverse(ask_b.into_atoms()), Reverse(&**key_b))) + }); + let has_more = decorated.len() > ORDER_BOOK_MAX_ORDERS; + decorated.truncate(ORDER_BOOK_MAX_ORDERS); + + Ok(( + decorated + .into_iter() + .map(|(_key, ask_balance, give_balance)| (ask_balance, give_balance)) + .collect(), + has_more, + )) + } + + /// The live pools (non-zero pledge), newest creation height first, ties broken + /// by the encoded pool address in DESCENDING byte order (`Reverse`, matching + /// the postgres listing's `pool_id COLLATE "C" DESC` tie-break); the sort key + /// is encoded once per entry. Shared by the offset and the keyset (cursor) + /// listings so their ordering cannot diverge. + fn sorted_live_pools( + &self, + ) -> Result, ApiServerStorageError> + { + let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self + .pool_data_table + .iter() + .filter_map(|(pool_id, by_height)| { + // a pool in the table always carries at least one versioned row; empty + // maps (never produced by the current write paths) are skipped instead + // of panicking in a read path, and the cheap zero-balance filter runs + // before the (allocating) sort-key encoding, so decommissioned pools + // skip the bech32 work entirely + let (Some(created_height), Some(latest_data)) = + (by_height.keys().next(), by_height.values().last()) + else { + return None; + }; + let created_height = *created_height; + let latest_data = latest_data.clone(); + latest_data.staker_balance().is_ok_and(|b| b != Amount::ZERO).then_some(( + *pool_id, + created_height, + latest_data, + )) + }) + .map(|(pool_id, created_height, latest_data)| { + Ok(( + created_height, + self.pool_id_sort_key(&pool_id)?, + pool_id, + latest_data, + )) + }) + .collect::, ApiServerStorageError>>()?; + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { + (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) + }); + Ok(decorated) + } + fn get_latest_pool_ids( &self, len: u32, @@ -525,25 +688,45 @@ impl ApiServerInMemoryStorage { ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; - let mut pool_data: Vec<_> = self - .pool_data_table + let decorated = self.sorted_live_pools()?; + if offset >= decorated.len() { + return Ok(vec![]); + } + + let latest_pools = decorated[offset..std::cmp::min(offset + len, decorated.len())] .iter() - .map(|(pool_id, by_height)| { - let created_height = by_height.keys().next().expect("not empty"); - let latest_data = by_height.values().last().expect("not empty"); - (pool_id, (created_height, latest_data)) - }) - .filter(|(_pool_id, data)| !data.1.is_decommissioned()) + .map(|(_height, _key, pool_id, data)| (*pool_id, data.clone())) .collect(); - pool_data.sort_by_key(|(_, (height, _data))| Reverse(*height)); - if offset >= pool_data.len() { - return Ok(vec![]); - } + Ok(latest_pools) + } - let latest_pools = pool_data[offset..std::cmp::min(offset + len, pool_data.len())] + fn get_latest_pool_ids_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let len = len as usize; + let mut decorated = self.sorted_live_pools()?; + let cursor_key = cursor + .map(|(cursor_height, cursor_pool_id)| { + Ok::<_, ApiServerStorageError>(( + cursor_height, + self.pool_id_sort_key(&cursor_pool_id)?, + )) + }) + .transpose()?; + decorated.retain(|(height, key, _pool_id, _data)| match &cursor_key { + Some((cursor_height, cursor_pool_key)) => { + *height < *cursor_height || (*height == *cursor_height && *key < *cursor_pool_key) + } + None => true, + }); + + let latest_pools = decorated .iter() - .map(|(pool_id, data)| (**pool_id, (data.1).clone())) + .take(len) + .map(|(height, _key, pool_id, data)| (*height, *pool_id, data.clone())) .collect(); Ok(latest_pools) @@ -556,21 +739,40 @@ impl ApiServerInMemoryStorage { ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; - let mut pool_data: Vec<_> = self + // deepest pledge first, ties broken by the encoded pool address (byte order), + // exactly like the postgres listing; the balance and the sort key are decoded + // once per entry, and a pool whose stored balance fails to decode is surfaced + // as a storage error instead of silently shrinking the listing (the + // empty-history skip below stays silent because no write path produces one) + let mut decorated: Vec<(Amount, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() - .map(|(pool_id, by_height)| (pool_id, by_height.values().last().expect("not empty"))) - .filter(|(_pool_id, data)| !data.is_decommissioned()) - .collect(); - - pool_data.sort_by_key(|(_, data)| Reverse(data.staker_balance().expect("no overflow"))); - if offset >= pool_data.len() { + .filter_map(|(pool_id, by_height)| { + let data = by_height.values().last()?.clone(); + let balance = match data.staker_balance() { + Ok(balance) => balance, + Err(e) => { + return Some(Err(ApiServerStorageError::DeserializationError(format!( + "pool {pool_id} has an invalid staker balance: {e}" + )))); + } + }; + (balance != Amount::ZERO) + .then(|| Ok((balance, self.pool_id_sort_key(pool_id)?, *pool_id, data))) + }) + .collect::, ApiServerStorageError>>()?; + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(balance_a, key_a, _, _), (balance_b, key_b, _, _)| { + (Reverse(balance_a.into_atoms()), Reverse(&**key_a)) + .cmp(&(Reverse(balance_b.into_atoms()), Reverse(&**key_b))) + }); + if offset >= decorated.len() { return Ok(vec![]); } - let latest_pools = pool_data[offset..std::cmp::min(offset + len, pool_data.len())] + let latest_pools = decorated[offset..std::cmp::min(offset + len, decorated.len())] .iter() - .map(|(pool_id, data)| (**pool_id, (*data).clone())) + .map(|(_balance, _key, pool_id, data)| (*pool_id, data.clone())) .collect(); Ok(latest_pools) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index 546f7e1951..865882ed35 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -46,6 +46,15 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_address_balance(address, coin_or_token_id) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) + } + async fn get_address_balances( &self, address: &str, @@ -151,6 +160,14 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_latest_pool_ids(len, offset) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_latest_pool_ids_before(len, cursor) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -317,4 +334,12 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } + + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + self.transaction.get_order_book_entries(ask_currency, give_currency) + } } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index 5039ae013d..d82c1d3d98 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -331,6 +331,15 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_address_balance(address, coin_or_token_id) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) + } + async fn get_address_balances( &self, address: &str, @@ -467,6 +476,14 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_latest_pool_ids(len, offset) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_latest_pool_ids_before(len, cursor) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -598,4 +615,12 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } + + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + self.transaction.get_order_book_entries(ask_currency, give_currency) + } } diff --git a/api-server/api-server-common/src/storage/impls/mod.rs b/api-server/api-server-common/src/storage/impls/mod.rs index c224d5b546..3e91df11ec 100644 --- a/api-server/api-server-common/src/storage/impls/mod.rs +++ b/api-server/api-server-common/src/storage/impls/mod.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub const CURRENT_STORAGE_VERSION: u32 = 26; +pub const CURRENT_STORAGE_VERSION: u32 = 27; pub mod in_memory; pub mod postgres; diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index ee6e3b8d5e..942d47a40e 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -37,8 +37,9 @@ use crate::storage::{ impls::CURRENT_STORAGE_VERSION, storage_api::{ AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, - FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoWithExtraInfo, + FungibleTokenData, LockedUtxo, NftWithOwner, ORDER_BOOK_MAX_ORDERS, Order, PoolBlockStats, + PoolDataWithExtraInfo, TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoWithExtraInfo, block_aux_data::{BlockAuxData, BlockWithExtraData}, }, }; @@ -54,6 +55,51 @@ pub struct QueryFromConnection<'a, 'b> { tx: &'a PooledConnection<'b, PostgresConnectionManager>, } +/// Largest `$n` placeholder index across up to four query-text fragments +/// (const-evaluable); used to tie bound-parameter array lengths to the assembled +/// query text at compile time. +#[allow(dead_code)] +const fn placeholder_count(a: &str, b: &str, c: &str, d: &str) -> usize { + let mut largest = placeholder_count_1(a); + let candidate = placeholder_count_1(b); + if candidate > largest { + largest = candidate; + } + let candidate = placeholder_count_1(c); + if candidate > largest { + largest = candidate; + } + let candidate = placeholder_count_1(d); + if candidate > largest { + largest = candidate; + } + largest +} + +/// `placeholder_count` for a single fragment; counts DISTINCT placeholder indices +/// (`$n` with the largest n), not occurrences, because a placeholder index is +/// bound exactly once regardless of how often the query text references it. +#[allow(dead_code)] +const fn placeholder_count_1(text: &str) -> usize { + let bytes = text.as_bytes(); + let mut largest = 0usize; + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'$' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { + // single-digit placeholder indices only; the largest bound parameter + // in these queries is single-digit by construction + let digit = (bytes[i + 1] - b'0') as usize; + if digit > largest { + largest = digit; + } + i += 2; + } else { + i += 1; + } + } + largest +} + impl<'a, 'b> QueryFromConnection<'a, 'b> { fn get_table_exists_query(table_name: &str) -> String { format!( @@ -97,6 +143,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map_err(|_| ApiServerStorageError::TxGlobalIndexTooHigh(tx_global_index)) } + /// Whether the schema exists and carries a storage version. Note that a `true` + /// result does NOT mean the stored version is current: the caller must compare + /// [`Self::get_storage_version`] with the current version and reinitialize when + /// they differ (the scanner daemon does exactly that). pub async fn is_initialized(&mut self) -> Result { let query_str = Self::get_table_exists_query("misc_data"); let row_count = self @@ -183,6 +233,78 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) } + /// Top holders of an asset, ordered by the balance (descending, ties broken by the + /// address); keyset (cursor) pagination, the cursor is the (amount, address) of the + /// last returned holder, exclusive. Addresses with a zero balance are excluded. + pub async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + // the cursor and no-cursor variants share one query text: the cursor + // predicate is appended from a constant, so the ordering and the cursor + // comparison cannot drift apart between the two branches (all query parts + // are compile-time constants; only bound parameters carry runtime values) + const QUERY_HEAD: &str = r#" + SELECT address, amount::text + FROM ml.address_amount + WHERE coin_or_token_id = $1 AND amount != 0"#; + const CURSOR_PREDICATE: &str = r#" + AND (amount < $2::text::numeric + OR (amount = $2::text::numeric + AND address COLLATE "C" < $3))"#; + const QUERY_TAIL: &str = r#" + ORDER BY amount DESC, address COLLATE "C" DESC + LIMIT "#; + // the largest placeholder of each query variant, derived from the ACTUAL + // constants above at compile time (const-concatenation): editing a constant + // without adjusting the matching parameter array fails to compile, including + // in release builds + const CURSOR_PARAM_COUNT: usize = + placeholder_count(QUERY_HEAD, CURSOR_PREDICATE, QUERY_TAIL, "$4;"); + const NO_CURSOR_PARAM_COUNT: usize = placeholder_count(QUERY_HEAD, QUERY_TAIL, "$2;", ""); + let coin_or_token_id = coin_or_token_id.encode(); + let len = len as i64; + let rows = match cursor { + Some((address, amount)) => { + let amount_str = amount.into_atoms().to_string(); + let params: [&(dyn tokio_postgres::types::ToSql + Sync); CURSOR_PARAM_COUNT] = + [&coin_or_token_id, &amount_str, &address, &len]; + self.tx + .query( + &format!("{QUERY_HEAD}{CURSOR_PREDICATE}{QUERY_TAIL}$4;"), + ¶ms, + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } + None => { + let params: [&(dyn tokio_postgres::types::ToSql + Sync); NO_CURSOR_PARAM_COUNT] = + [&coin_or_token_id, &len]; + self.tx + .query(&format!("{QUERY_HEAD}{QUERY_TAIL}$2;"), ¶ms) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } + }; + + rows.into_iter() + .map(|row| -> Result<(String, Amount), ApiServerStorageError> { + let address: String = row.get(0); + let amount_str: String = row.get(1); + // fail fast on a corrupt row rather than silently returning wrong + // balances; the message carries the offending value for diagnosis + let atoms = amount_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid amount {amount_str} for address {address}: {e}" + )) + })?; + Ok((address, Amount::from_atoms(atoms))) + }) + .collect() + } + pub async fn get_address_balances( &self, address: &str, @@ -272,15 +394,15 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result<(), ApiServerStorageError> { let height = Self::block_height_to_postgres_friendly(block_height); - self.tx - .execute( - "DELETE FROM ml.address_balance WHERE block_height > $1;", - &[&height], - ) - .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // the versioned-history delete and the holders re-derivation form one + // dependency unit (the re-derivation is only correct immediately after + // the delete); keeping the two in a single helper makes that coupling + // structural instead of comment-enforced + self.del_address_balance_and_rederive_holders(height).await?; - // delete and update the address balance from the cache table + // roll back the latest-address-balance cache of the disconnected + // blocks; this is only correct because the versioned delete inside the + // helper above already ran within this same rollback call self.tx .execute( r#" @@ -322,6 +444,139 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { Ok(()) } + /// Delete the versioned address-balance rows above the height and re-derive + /// the holders (`ml.address_amount`) rows from what remains; set-based, in a + /// constant number of round trips, so the two steps cannot drift apart. + async fn del_address_balance_and_rederive_holders( + &mut self, + height: i64, + ) -> Result<(), ApiServerStorageError> { + self.tx + .execute( + "DELETE FROM ml.address_balance WHERE block_height > $1;", + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + // roll back the holders table rows of the disconnected blocks; set-based, in a + // constant number of round trips: delete the rolled-back rows, drop rows whose + // whole versioned history was orphaned, then re-derive and re-insert the rest + // in bulk (the numeric amount is decoded from the versioned balances). + // NOTE: this re-derivation is only correct because the versioned + // `ml.address_balance` rows above the height were deleted by the statement + // above, within this same call; the ordering of the two statements is + // load-bearing. `top_address_amounts` in the storage test suite guards this by + // rolling back through this function and asserting the re-derived listing. + let deleted_holders_rows = self + .tx + .query( + "DELETE FROM ml.address_amount WHERE block_height > $1 RETURNING address, coin_or_token_id;", + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + if !deleted_holders_rows.is_empty() { + let addresses: Vec = + deleted_holders_rows.iter().map(|row| row.get(0)).collect(); + let coin_or_token_ids: Vec> = + deleted_holders_rows.iter().map(|row| row.get(1)).collect(); + + // rows whose versioned history was fully orphaned leave the table + self.tx + .execute( + r#" + DELETE FROM ml.address_amount aa + WHERE (aa.address, aa.coin_or_token_id) IN ( + SELECT address, coin_or_token_id + FROM unnest($1::text[], $2::bytea[]) AS d(address, coin_or_token_id) + ) + AND NOT EXISTS ( + SELECT 1 FROM ml.address_balance ab + WHERE ab.address = aa.address AND ab.coin_or_token_id = aa.coin_or_token_id + ); + "#, + &[&addresses, &coin_or_token_ids], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + // the latest remaining versioned balance of every affected pair + let latest_versioned = self + .tx + .query( + r#" + SELECT DISTINCT ON (a.address, a.coin_or_token_id) + a.address, a.coin_or_token_id, a.block_height, a.amount + FROM ml.address_balance a + JOIN unnest($1::text[], $2::bytea[]) AS d(address, coin_or_token_id) + ON a.address = d.address AND a.coin_or_token_id = d.coin_or_token_id + ORDER BY a.address, a.coin_or_token_id, a.block_height DESC; + "#, + &[&addresses, &coin_or_token_ids], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + let mut upsert_addresses: Vec = Vec::with_capacity(latest_versioned.len()); + let mut upsert_coin_or_token_ids: Vec> = + Vec::with_capacity(latest_versioned.len()); + let mut upsert_block_heights: Vec = Vec::with_capacity(latest_versioned.len()); + let mut upsert_amounts: Vec = Vec::with_capacity(latest_versioned.len()); + + for row in latest_versioned { + let amount: Vec = row.get(3); + let amount = Amount::decode_all(&mut amount.as_slice()).map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Amount deserialization failed: {e}" + )) + })?; + upsert_addresses.push(row.get(0)); + upsert_coin_or_token_ids.push(row.get(1)); + upsert_block_heights.push(row.get::<_, i64>(2)); + upsert_amounts.push(amount.into_atoms().to_string()); + } + + self.tx + .execute( + r#" + INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) + SELECT address, coin_or_token_id, block_height, amount + FROM unnest($1::text[], $2::bytea[], $3::bigint[], $4::text[]::numeric[]) + AS t(address, coin_or_token_id, block_height, amount) + ON CONFLICT (address, coin_or_token_id) DO UPDATE + SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount + WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + -- unlike the pool cache there is no LEAST/GREATEST envelope here, + -- by design: the holders listing orders by `amount DESC`, which + -- rollback does not touch, so a stale cache row can only be wrong + -- in VALUE, never out of order. The height guard `<=` lets any + -- write at the same or newer height than the cached row replace + -- it (including same-height reorg rewrites); cache rows older + -- than the incoming versioned row are exactly the rows this + -- write corrects. A cache row NEWER than the versioned table can + -- only exist if a rollback was interrupted after truncating the + -- versioned history but before rolling the cache back — the + -- rollback helper (del_address_balance_and_rederive_holders) + -- always runs both steps in one transaction, so that state is + -- unreachable outside a crash mid-transaction, which postgres + -- atomicity also rules out. + "#, + &[ + &upsert_addresses, + &upsert_coin_or_token_ids, + &upsert_block_heights, + &upsert_amounts, + ], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + } + + Ok(()) + } + pub async fn del_address_locked_balance_above_height( &mut self, block_height: BlockHeight, @@ -348,6 +603,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result<(), ApiServerStorageError> { let height = Self::block_height_to_postgres_friendly(block_height); + // the versioned insert and the holders-cache upsert below both run on every + // balance write: the cache row is rewritten even when the amount is unchanged + // (the height guard still updates it), which is intentional — re-parsing the + // listing from the versioned table per read would cost more than the + // redundant upsert (the listing is served from this cache alone) self.tx .execute( r#" @@ -380,6 +640,23 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // the holders table tracks the same latest balance, but with the amount in the + // NUMERIC form so the holders listing can order the addresses by it + let amount_str = amount.into_atoms().to_string(); + self.tx + .execute( + r#" + INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) + VALUES ($1, $2, $3, $4::text::numeric) + ON CONFLICT (address, coin_or_token_id) DO UPDATE + SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount + WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + "#, + &[&address.to_string(), &coin_or_token_id.encode(), &height, &amount_str], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + let CoinOrTokenId::TokenId(token_id) = coin_or_token_id else { return Ok(()); }; @@ -778,6 +1055,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; + // index for the keyset (cursor) pagination of the transaction listing + self.just_execute( + "CREATE INDEX transactions_tx_global_index ON ml.transactions (tx_global_index DESC);", + ) + .await?; + self.just_execute( "CREATE TABLE ml.address_balance ( address TEXT NOT NULL, @@ -1051,6 +1334,104 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; + // Latest state of every pool; maintained alongside ml.pool_data by the scanner so + // that the pool listings do not have to derive the newest state of each pool from + // the full versioned history (which is touched at every PoS block). + self.just_execute( + "CREATE TABLE ml.latest_pool_data_cache ( + pool_id TEXT COLLATE \"C\" NOT NULL, + block_height bigint NOT NULL, + creation_block_height bigint NOT NULL, + staker_balance NUMERIC NOT NULL, + data bytea NOT NULL, + PRIMARY KEY (pool_id) + );", + ) + .await?; + + // index for the keyset pagination of the pool listing (by creation height) + self.just_execute( + "CREATE INDEX latest_pool_data_created_index + ON ml.latest_pool_data_cache (creation_block_height DESC, pool_id DESC);", + ) + .await?; + + // index for the pool listing sorted by pledge; includes the tie-break column + // so the listing's ORDER BY is index-satisfiable (matching + // latest_pool_data_created_index) + self.just_execute( + "CREATE INDEX latest_pool_data_staker_balance_index + ON ml.latest_pool_data_cache (staker_balance DESC, pool_id DESC);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX latest_pool_data_block_height_index + ON ml.latest_pool_data_cache (block_height DESC);", + ) + .await?; + + // Latest balance of every (address, asset) pair as an orderable numeric amount; + // maintained alongside ml.address_balance by the scanner so that the top holders + // listing can be served by an index scan instead of decoding every balance. + self.just_execute( + "CREATE TABLE ml.address_amount ( + address TEXT COLLATE \"C\" NOT NULL, + coin_or_token_id bytea NOT NULL, + block_height bigint NOT NULL, + amount NUMERIC NOT NULL, + PRIMARY KEY (address, coin_or_token_id) + );", + ) + .await?; + + // index for the keyset pagination of the holders listing + self.just_execute( + "CREATE INDEX address_amount_top_holders_index + ON ml.address_amount (coin_or_token_id, amount DESC, address DESC);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX address_amount_block_height ON ml.address_amount (block_height DESC);", + ) + .await?; + + // Latest state of every order; maintained alongside ml.orders by the scanner so + // that the order book does not have to derive the newest state of each order from + // the full versioned history. + self.just_execute( + "CREATE TABLE ml.latest_orders_cache ( + order_id TEXT COLLATE \"C\" NOT NULL, + block_height bigint NOT NULL, + creation_block_height bigint NOT NULL, + ask_currency bytea NOT NULL, + ask_balance NUMERIC NOT NULL, + give_currency bytea NOT NULL, + give_balance NUMERIC NOT NULL, + frozen BOOLEAN NOT NULL, + PRIMARY KEY (order_id) + );", + ) + .await?; + + // index for the order book pair lookups; the trailing columns cover the + // book read's ordering (deepest ask first, encoded order id as tie-break) so + // no separate sort of the pair's live orders is needed + self.just_execute( + "CREATE INDEX latest_orders_currencies_index + ON ml.latest_orders_cache (ask_currency, give_currency, ask_balance DESC, order_id DESC);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX latest_orders_block_height_index ON ml.latest_orders_cache (block_height DESC);", + ) + .await?; + // Append-only log of stream events for real-time clients; the primary key index also // serves the `WHERE id > $1 ORDER BY id` reads of the event pump. self.just_execute( @@ -1318,8 +1699,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { }; let pool_id: String = data.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let balance: String = data.get(1); let spend_destination: Vec = data.get(2); @@ -1379,12 +1764,20 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map(|row| { let delegation_id: String = row.get(0); let delegation_id = - Address::::from_string(chain_config, delegation_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + Address::::from_string(chain_config, &delegation_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid delegation id {delegation_id}: {e}" + )) + })? .into_object(); let pool_id: String = row.get(1); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let balance: String = row.get(2); let spend_destination: Vec = row.get(3); @@ -1574,6 +1967,56 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // delete and re-derive the pool rows of the latest-state cache from the versioned + // table (a deleted row whose pool still exists below the disconnected height must + // not be lost) + self.tx + .execute( + r#" + WITH deleted_cache_rows AS ( + DELETE FROM ml.latest_pool_data_cache + WHERE block_height > $1 + RETURNING pool_id + ), + latest_pool_data_for_deleted AS ( + SELECT + dcr.pool_id, + ld.block_height, + created.creation_block_height, + ld.staker_balance, + ld.data + FROM deleted_cache_rows dcr + CROSS JOIN LATERAL ( + SELECT + pool_id, + block_height, + staker_balance, + data + FROM ml.pool_data d_inner + WHERE d_inner.pool_id = dcr.pool_id + ORDER BY block_height DESC + LIMIT 1 + ) AS ld + CROSS JOIN LATERAL ( + SELECT MIN(p_min.block_height) AS creation_block_height + FROM ml.pool_data p_min + WHERE p_min.pool_id = dcr.pool_id + ) AS created + ) + INSERT INTO ml.latest_pool_data_cache (pool_id, block_height, creation_block_height, staker_balance, data) + SELECT + pool_id, + block_height, + creation_block_height, + staker_balance::numeric, + data + FROM latest_pool_data_for_deleted; + "#, + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -1631,7 +2074,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let delegation_id_str: String = row.get(0); let delegation_id = Address::::from_string(chain_config, &delegation_id_str) - .map_err(|_| ApiServerStorageError::AddressableError)? + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid delegation id {delegation_id_str}: {e}" + )) + })? .into_object(); let balance: String = row.get(1); let spend_destination: Vec = row.get(2); @@ -1709,6 +2156,36 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) } + fn decode_pool_id_and_data_rows( + rows: Vec, + chain_config: &ChainConfig, + ) -> Result, ApiServerStorageError> { + rows.into_iter() + .map( + |row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { + let pool_id: String = row.get(0); + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? + .into_object(); + let pool_data: Vec = row.get(1); + let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Pool data deserialization failed: {}", + e + )) + })?; + + Ok((pool_id, pool_data)) + }, + ) + .collect() + } + pub async fn get_latest_pool_data( &self, len: u32, @@ -1717,30 +2194,90 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; - self.tx + let rows = self + .tx .query( r#" - SELECT sub.pool_id, data - FROM ( - SELECT pool_id, data, staker_balance, block_height, ROW_NUMBER() OVER(PARTITION BY pool_id ORDER BY block_height DESC) as newest - FROM ml.pool_data - ) AS sub INNER JOIN (SELECT pool_id, MIN(block_height) AS created_height FROM ml.pool_data GROUP BY pool_id) as created ON sub.pool_id = created.pool_id - WHERE newest = 1 AND staker_balance::NUMERIC != 0 - ORDER BY created_height DESC + SELECT pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC OFFSET $1 LIMIT $2; "#, &[&offset, &len], ) .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? - .into_iter() - .map(|row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { - let pool_id: String = row.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + Self::decode_pool_id_and_data_rows(rows, chain_config) + } + + /// Keyset (cursor) pagination of the pool listing by creation height; the cursor is + /// the (creation height, pool id) of the last returned pool, exclusive. + pub async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + chain_config: &ChainConfig, + ) -> Result, ApiServerStorageError> { + let len = len as i64; + let rows = match cursor { + Some((creation_height, pool_id)) => { + let creation_height = Self::block_height_to_postgres_friendly(creation_height); + let pool_id = Address::new(chain_config, pool_id) + .map_err(|_| ApiServerStorageError::AddressableError)?; + self.tx + .query( + r#" + SELECT creation_block_height, pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + AND (creation_block_height < $1 + OR (creation_block_height = $1 + AND pool_id COLLATE "C" < $2)) + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC + LIMIT $3; + "#, + &[&creation_height, &pool_id.as_str(), &len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } + None => self + .tx + .query( + r#" + SELECT creation_block_height, pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC + LIMIT $1; + "#, + &[&len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?, + }; + + rows.into_iter() + .map(|row| -> Result<(BlockHeight, PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { + let creation_block_height: i64 = row.get(0); + let creation_block_height = u64::try_from(creation_block_height).map_err(|_| { + ApiServerStorageError::DeserializationError(format!( + "invalid negative creation block height: {creation_block_height}" + )) + })?; + let creation_block_height = BlockHeight::new(creation_block_height); + let pool_id: String = row.get(1); + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); - let pool_data: Vec = row.get(1); + let pool_data: Vec = row.get(2); let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()).map_err(|e| { ApiServerStorageError::DeserializationError(format!( "Pool data deserialization failed: {}", @@ -1748,7 +2285,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { )) })?; - Ok((pool_id, pool_data)) + Ok((creation_block_height, pool_id, pool_data)) }) .collect() } @@ -1761,40 +2298,23 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; - self.tx + let rows = self + .tx .query( r#" SELECT pool_id, data - FROM ( - SELECT pool_id, data, staker_balance, ROW_NUMBER() OVER(PARTITION BY pool_id ORDER BY block_height DESC) as newest - FROM ml.pool_data - ) AS sub - WHERE newest = 1 AND staker_balance::NUMERIC != 0 - ORDER BY staker_balance DESC + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + ORDER BY staker_balance DESC, pool_id COLLATE "C" DESC OFFSET $1 LIMIT $2; "#, &[&offset, &len], ) .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? - .into_iter() - .map(|row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { - let pool_id: String = row.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? - .into_object(); - let pool_data: Vec = row.get(1); - let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()).map_err(|e| { - ApiServerStorageError::DeserializationError(format!( - "Pool data deserialization failed: {}", - e - )) - })?; + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - Ok((pool_id, pool_data)) - }) - .collect() + Self::decode_pool_id_and_data_rows(rows, chain_config) } pub async fn set_pool_data_at_height( @@ -1822,6 +2342,36 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // maintain the latest-state cache used by the pool listings as the envelope of + // the versioned table: the creation height is the lowest height the pool was + // ever written at, the latest state is the highest one (in the normal + // genesis-ascending scan that is simply the first and the last write). The + // update is monotone — the same-height write of a reorg replaces the cached + // state (the `>=` CASE branches), a late lower-height write only corrects the + // creation height — so a pool first seen mid-history (a re-seed that started at + // a higher height) gets its creation height corrected when the lower rows + // arrive, matching the `MIN(block_height)` re-derivation of the rollback path + // and the in-memory backend. + self.tx + .execute( + r#" + INSERT INTO ml.latest_pool_data_cache (pool_id, block_height, creation_block_height, staker_balance, data) + VALUES ($1, $2, $2, $3::text::numeric, $4) + ON CONFLICT (pool_id) DO UPDATE + SET block_height = GREATEST(ml.latest_pool_data_cache.block_height, EXCLUDED.block_height), + creation_block_height = LEAST(ml.latest_pool_data_cache.creation_block_height, EXCLUDED.creation_block_height), + staker_balance = CASE + WHEN EXCLUDED.block_height >= ml.latest_pool_data_cache.block_height THEN EXCLUDED.staker_balance + ELSE ml.latest_pool_data_cache.staker_balance END, + data = CASE + WHEN EXCLUDED.block_height >= ml.latest_pool_data_cache.block_height THEN EXCLUDED.data + ELSE ml.latest_pool_data_cache.data END; + "#, + &[&pool_id.as_str(), &height, &amount_str, &pool_data.encode()], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -2650,19 +3200,36 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &self, coin_or_token_id: CoinOrTokenId, ) -> Result, ApiServerStorageError> { + // Note: the newest value of each statistic is read with one indexed lookup per + // statistic instead of a `ROW_NUMBER()` window over the whole table: the primary + // key is (statistic, coin_or_token_id, block_height), so a lookup that filters by + // the coin or token id alone cannot use the key prefix and degrades into a scan of + // every historical row of the asset (the circulating supply and the staked amount + // are touched at every PoS block, so there can be millions of such rows). let rows = self .tx .query( + // DISTINCT ON picks the highest row per requested statistic; a + // statistic with no rows at all is silently omitted from the result + // (the endpoints treat a missing statistic as absent rather than + // zero) — keep the endpoint defaults in sync with this behavior r#" - SELECT sub.statistic, sub.amount + SELECT statistic, amount FROM ( - SELECT statistic, amount, ROW_NUMBER() OVER(PARTITION BY statistic ORDER BY block_height DESC) as newest + SELECT DISTINCT ON (statistic) + statistic, amount FROM ml.statistics - WHERE coin_or_token_id = $1 - ) AS sub - WHERE newest = 1; + WHERE statistic = ANY($1) AND coin_or_token_id = $2 + ORDER BY statistic, block_height DESC + ) AS sub; "#, - &[&coin_or_token_id.encode()], + &[ + &CoinOrTokenStatistic::VARIANTS + .iter() + .map(|variant| variant.to_string()) + .collect::>(), + &coin_or_token_id.encode(), + ], ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; @@ -2982,6 +3549,42 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // maintain the latest-orders cache as a mirror of the latest versioned state; + // concluded orders (both balances zeroed) are stored as well, they are filtered + // out by the order book reads + let ask_balance = order.ask_balance.into_atoms().to_string(); + let give_balance = order.give_balance.into_atoms().to_string(); + + self.tx + .execute( + r#" + INSERT INTO ml.latest_orders_cache + (order_id, block_height, creation_block_height, ask_currency, ask_balance, give_currency, give_balance, frozen) + VALUES ($1, $2, $3, $4, $5::text::numeric, $6, $7::text::numeric, $8) + ON CONFLICT (order_id) DO UPDATE + SET block_height = EXCLUDED.block_height, + creation_block_height = EXCLUDED.creation_block_height, + ask_currency = EXCLUDED.ask_currency, + ask_balance = EXCLUDED.ask_balance, + give_currency = EXCLUDED.give_currency, + give_balance = EXCLUDED.give_balance, + frozen = EXCLUDED.frozen + WHERE ml.latest_orders_cache.block_height <= EXCLUDED.block_height; + "#, + &[ + &order_id.as_str(), + &height, + &creation_block_height, + &order.ask_currency.encode(), + &ask_balance, + &order.give_currency.encode(), + &give_balance, + &order.is_frozen, + ], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -2996,9 +3599,109 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // roll back the latest-orders cache rows of the disconnected blocks; the cache is + // a mirror of the latest versioned state (concluded orders included, filtered by + // the order book reads), so the surviving versioned history of every order with a + // deleted cache row fully determines its restored state. + // + // No ON CONFLICT clause is needed: `order_id` is the primary key of the cache + // table and the deleting CTE returns each affected order exactly once, so the + // lateral join inserts at most one row per order. The lateral join can also + // produce NO row for a deleted cache row only if that order has no surviving + // versioned history at all — impossible in practice, because every cache row is + // written from a versioned row (the upsert's `WHERE block_height <= + // EXCLUDED.block_height` guard keeps one cache row per order), and rollback + // heights only ever move to heights where the versioned rows already exist. + self.tx + .execute( + r#" + WITH deleted_cache_rows AS ( + DELETE FROM ml.latest_orders_cache WHERE block_height > $1 RETURNING order_id + ) + INSERT INTO ml.latest_orders_cache + (order_id, block_height, creation_block_height, ask_currency, ask_balance, give_currency, give_balance, frozen) + SELECT + o.order_id, o.block_height, o.creation_block_height, o.ask_currency, + o.ask_balance::numeric, o.give_currency, o.give_balance::numeric, o.frozen + FROM deleted_cache_rows d + CROSS JOIN LATERAL ( + SELECT + o.order_id, o.block_height, o.creation_block_height, + o.ask_currency, o.ask_balance, o.give_currency, + o.give_balance, o.frozen + FROM ml.orders o + WHERE o.order_id = d.order_id + ORDER BY o.block_height DESC + LIMIT 1 + ) o; + "#, + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } + /// Remaining balances of the live (not concluded, not frozen) orders of a trading + /// pair, as (ask_balance, give_balance) pairs, ordered by ask balance (descending, + /// ties broken by the order id in descending byte order); capped at + /// [`ORDER_BOOK_MAX_ORDERS`] entries, `has_more` reports truncation. + pub async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + let ask_currency = ask_currency.encode(); + let give_currency = give_currency.encode(); + // one extra row detects truncation + let cap = ORDER_BOOK_MAX_ORDERS as i64 + 1; + + let rows = self + .tx + .query( + r#" + SELECT ask_balance::text, give_balance::text, order_id + FROM ml.latest_orders_cache + WHERE ask_currency = $1 AND give_currency = $2 + AND frozen = FALSE + AND ask_balance != 0 AND give_balance != 0 + ORDER BY ask_balance DESC, order_id DESC + LIMIT $3; + "#, + &[&ask_currency, &give_currency, &cap], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + let has_more = rows.len() > ORDER_BOOK_MAX_ORDERS; + + rows.into_iter() + .take(ORDER_BOOK_MAX_ORDERS) + .map(|row| -> Result<(Amount, Amount), ApiServerStorageError> { + let ask_str: String = row.get(0); + let give_str: String = row.get(1); + let order_id: String = row.get(2); + + // fail fast on a corrupt row rather than silently returning wrong + // balances; the message carries the offending value for diagnosis + let ask = ask_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid ask balance {ask_str} for order {order_id}: {e}" + )) + })?; + let give = give_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid give balance {give_str} for order {order_id}: {e}" + )) + })?; + + Ok((Amount::from_atoms(ask), Amount::from_atoms(give))) + }) + .collect::, _>>() + .map(|entries| (entries, has_more)) + } + pub async fn get_orders_by_height( &self, len: u32, @@ -3256,8 +3959,12 @@ fn decode_order_from_row( let creation_block_height: i64 = data.get("creation_block_height"); let is_frozen: bool = data.get("frozen"); - let order_id = Address::::from_string(chain_config, order_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let order_id = Address::::from_string(chain_config, &order_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid addressable id {order_id}: {e}" + )) + })? .into_object(); let initially_asked = Amount::from_fixedpoint_str(&initially_asked, 0).ok_or_else(|| { diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 6fa3514681..7f133b11e0 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -64,6 +64,16 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_top_address_amounts(coin_or_token_id, len, cursor).await + } + async fn get_address_balances( &self, address: &str, @@ -262,6 +272,17 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_latest_pool_data_before(len, cursor, &self.chain_config).await?; + + Ok(res) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -450,6 +471,15 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_order_book_entries(ask_currency, give_currency).await + } + async fn read_stream_events_after( &self, last_seen_id: StreamEventId, diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index c9a53f3c28..63690bf4d0 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -465,6 +465,16 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_top_address_amounts(coin_or_token_id, len, cursor).await + } + async fn get_address_balances( &self, address: &str, @@ -638,6 +648,17 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_latest_pool_data_before(len, cursor, &self.chain_config).await?; + + Ok(res) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -847,6 +868,15 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_order_book_entries(ask_currency, give_currency).await + } + async fn read_stream_events_after( &self, last_seen_id: StreamEventId, diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 05863da479..4bd2aa4852 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -40,6 +40,12 @@ use serialization::{Decode, Encode}; use self::block_aux_data::{BlockAuxData, BlockWithExtraData}; use crate::streaming::{StreamEvent, StreamEventId, StreamEventReadError}; +/// Hard cap on the number of live orders a single order-book query may consume; +/// both backends return at most this many entries, largest ask balance first, and +/// when the cap truncates the book, the smallest-balance orders are omitted; +/// `has_more` signals a truncated result. +pub const ORDER_BOOK_MAX_ORDERS: usize = 10_000; + pub mod block_aux_data; #[allow(dead_code)] @@ -83,6 +89,20 @@ pub enum CoinOrTokenStatistic { Preminted, } +impl CoinOrTokenStatistic { + /// Order matters: the array is serialized into the `ANY($1)` statistics filter + /// as text (via `to_string`), so it must stay in sync with the enum variants + /// AND their string spellings. + pub const VARIANTS: [Self; 4] = { + // compile-time exhaustiveness guard: adding a variant without updating the + // array below fails to compile here because this match has no wildcard + match Self::CirculatingSupply { + Self::CirculatingSupply | Self::Staked | Self::Burned | Self::Preminted => {} + } + [Self::CirculatingSupply, Self::Staked, Self::Burned, Self::Preminted] + }; +} + impl FromStr for CoinOrTokenStatistic { type Err = ApiServerStorageError; @@ -602,6 +622,19 @@ pub trait ApiServerStorageRead: Sync { coin_or_token_id: CoinOrTokenId, ) -> Result, ApiServerStorageError>; + /// Returns up to `len` holders of the asset with the largest balances, ordered by + /// the balance (descending, ties broken by the address in descending BYTE order); + /// keyset (cursor) pagination, the cursor is the (address, amount) of the last + /// returned holder, exclusive, with the amount encoded as a plain atom-count + /// string (`into_atoms().to_string()`) — not a decimal/fixed-point form. + /// Addresses with a zero balance are excluded. + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(String, Amount)>, + ) -> Result, ApiServerStorageError>; + async fn get_address_balances( &self, address: &str, @@ -686,6 +719,15 @@ pub trait ApiServerStorageRead: Sync { offset: u64, ) -> Result, ApiServerStorageError>; + /// Keyset (cursor) pagination of the pool listing by creation height (descending); + /// the cursor is the (creation height, pool id) of the last returned pool, exclusive. + /// Returns the creation height of each pool alongside its id and data. + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError>; + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -802,6 +844,19 @@ pub trait ApiServerStorageRead: Sync { offset: u64, ) -> Result, ApiServerStorageError>; + /// Remaining balances of the live (not concluded, not frozen) orders of a trading + /// pair, as (ask_balance, give_balance) pairs; the price of each order is derived + /// from these in the web layer. At most [`ORDER_BOOK_MAX_ORDERS`] entries + /// are returned, ordered by ask balance (descending, ties broken by the order id + /// in descending byte order); `has_more` reports whether the cap truncated the + /// book (the entries with the largest ask balance are returned; when the cap + /// truncates the book, the smallest-balance orders are omitted). + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError>; + /// Read the stream events with an id greater than `last_seen_id`, in ascending id order. /// /// Note: backends that don't support stream events simply return an empty list. @@ -847,6 +902,10 @@ pub trait ApiServerStorageWrite: ApiServerStorageRead { block_height: BlockHeight, ) -> Result<(), ApiServerStorageError>; + /// Stores the balance under the string encoding carried by the given address: + /// scanning constructs addresses via `Address::new`, so the stored encoding is + /// the canonical one, which the byte-ordered tie-break comparison of the + /// top-addresses listing relies on. async fn set_address_balance_at_height( &mut self, address: &Address, diff --git a/api-server/scanner-lib/src/sync/tests/mod.rs b/api-server/scanner-lib/src/sync/tests/mod.rs index 63aa80cb15..29a343c36c 100644 --- a/api-server/scanner-lib/src/sync/tests/mod.rs +++ b/api-server/scanner-lib/src/sync/tests/mod.rs @@ -29,7 +29,8 @@ use tokio::sync::mpsc; use api_server_common::storage::{ impls::in_memory::transactional::TransactionalApiServerInMemoryStorage, storage_api::{ - ApiServerStorageRead, ApiServerStorageWrite, ApiServerTransactionRw, Transactional, + ApiServerStorageRead, ApiServerStorageWrite, ApiServerTransactionRw, CoinOrTokenStatistic, + Transactional, }, }; use chainstate::{BlockSource, ChainInfo}; @@ -1037,6 +1038,143 @@ async fn reorg_locked_balance(#[case] seed: Seed) { drop(db_tx); } +// `get_all_statistic` omits statistics that were never written; absence means the +// reverted-to value, i.e. zero — e.g. after a reorg the burned amount is rolled back +// and the Burned key disappears entirely, which this helper must report as zero +fn statistic_value( + stats: &BTreeMap, + statistic: CoinOrTokenStatistic, +) -> Amount { + stats.get(&statistic).copied().unwrap_or(Amount::ZERO) +} + +#[rstest] +#[trace] +#[case(test_utils::random::Seed::from_entropy())] +#[tokio::test] +async fn reorg_statistics(#[case] seed: Seed) { + let mut rng = make_seedable_rng(seed); + + let mut tf = TestFramework::builder(&mut rng).build(); + + let chain_config = Arc::clone(tf.chainstate.get_chain_config()); + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + let mut local_state = BlockchainState::new(chain_config.clone(), storage); + local_state.scan_genesis(chain_config.genesis_block().as_ref()).await.unwrap(); + + let target_block_time = chain_config.target_block_spacing(); + + let genesis_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + let genesis_burned = statistic_value(&genesis_stats, CoinOrTokenStatistic::Burned); + let genesis_circulating_supply = + statistic_value(&genesis_stats, CoinOrTokenStatistic::CirculatingSupply); + + // a block with a coin burn, which increases the burned supply and decreases the + // circulating supply + let burned_amount = Amount::from_atoms(5); + let transaction = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::BlockReward(chain_config.genesis_block_id()), + 0, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::Burn(OutputValue::Coin(burned_amount))) + .build(); + + tf.progress_time_seconds_since_epoch(target_block_time.as_secs()); + let block = tf + .make_block_builder() + .with_parent(chain_config.genesis_block_id()) + .with_transactions(vec![transaction]) + .build(&mut rng); + + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + let block_height = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_best_block() + .await + .unwrap() + .block_height(); + local_state.scan_blocks(block_height, vec![block.clone()]).await.unwrap(); + + let post_burn_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&post_burn_stats, CoinOrTokenStatistic::Burned), + (genesis_burned + burned_amount).expect("no overflow in test amounts") + ); + assert_eq!( + statistic_value(&post_burn_stats, CoinOrTokenStatistic::CirculatingSupply), + (genesis_circulating_supply - burned_amount).expect("no underflow in test amounts") + ); + + // reorg away the block; the statistics must return to the genesis values + local_state.scan_blocks(block_height, vec![]).await.unwrap(); + + let rolled_back_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&rolled_back_stats, CoinOrTokenStatistic::Burned), + genesis_burned + ); + assert_eq!( + statistic_value(&rolled_back_stats, CoinOrTokenStatistic::CirculatingSupply), + genesis_circulating_supply + ); + + // scan the block again; the statistics must be restored + local_state.scan_blocks(block_height, vec![block]).await.unwrap(); + + let restored_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&restored_stats, CoinOrTokenStatistic::Burned), + statistic_value(&post_burn_stats, CoinOrTokenStatistic::Burned) + ); + assert_eq!( + statistic_value(&restored_stats, CoinOrTokenStatistic::CirculatingSupply), + statistic_value(&post_burn_stats, CoinOrTokenStatistic::CirculatingSupply) + ); +} + #[allow(clippy::too_many_arguments)] fn create_block( rng: &mut impl CryptoRng, diff --git a/api-server/stack-test-suite/tests/v2/orders.rs b/api-server/stack-test-suite/tests/v2/orders.rs index ddcaff70be..f16bcb2f6c 100644 --- a/api-server/stack-test-suite/tests/v2/orders.rs +++ b/api-server/stack-test-suite/tests/v2/orders.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use api_web_server::api::json_helpers::amount_to_json; use common::chain::{ AccountCommand, AccountNonce, ChainstateUpgradeBuilder, OrderAccountCommand, OrderData, OrdersVersion, make_order_id, @@ -177,7 +178,7 @@ async fn create_fill_conclude_order(#[case] seed: Seed, #[case] version: OrdersV } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let (block1_id, tx1_id, block2_id, tx2_id, block3_id, tx3_id) = rx.await.unwrap(); @@ -200,6 +201,250 @@ async fn create_fill_conclude_order(#[case] seed: Seed, #[case] version: OrdersV shutdown_task(task).await; } +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn order_pair_book(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + // Issue and mint some tokens to create orders of both sides of the + // coin/token pair + let tokens_amount = Amount::from_atoms(1000); + let issue_and_mint_result = + helpers::issue_and_mint_tokens_from_genesis(tokens_amount, &mut rng, &mut tf); + let token_id = issue_and_mint_result.token_id; + let minted = issue_and_mint_result.minted_tokens; + + let ask_coin = Amount::from_atoms(10); + + // asks 10 coins for 10 tokens (a price of 1 token per coin) + let tx_1 = TransactionBuilder::new() + .add_input( + TxInput::Utxo(issue_and_mint_result.tokens_outpoint), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::Coin(ask_coin), + OutputValue::TokenV1(token_id, ask_coin), + )))) + .add_output(TxOutput::Transfer( + OutputValue::TokenV1(token_id, (minted - ask_coin).unwrap()), + Destination::AnyoneCanSpend, + )) + .build(); + let block1 = tf.make_block_builder().add_transaction(tx_1.clone()).build(&mut rng); + tf.process_block(block1.clone(), BlockSource::Local).unwrap(); + + // asks 10 coins for 30 tokens (a price of 3 tokens per coin) + let tx_2 = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::Transaction(tx_1.transaction().get_id()), + 1, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::Coin(ask_coin), + OutputValue::TokenV1(token_id, Amount::from_atoms(30)), + )))) + .add_output(TxOutput::Transfer( + OutputValue::TokenV1( + token_id, + ((minted - ask_coin).unwrap() - Amount::from_atoms(30)).unwrap(), + ), + Destination::AnyoneCanSpend, + )) + .build(); + let block2 = tf.make_block_builder().add_transaction(tx_2).build(&mut rng); + tf.process_block(block2.clone(), BlockSource::Local).unwrap(); + + // a bid of the coin/token pair: asks 5 tokens for 5 coins + let tx_3 = TransactionBuilder::new() + .add_input( + TxInput::Utxo(issue_and_mint_result.change_outpoint.clone()), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::TokenV1(token_id, Amount::from_atoms(5)), + OutputValue::Coin(Amount::from_atoms(5)), + )))) + .add_output(TxOutput::Transfer( + OutputValue::Coin( + (chainstate_test_framework::get_output_value( + tf.chainstate + .utxo(&issue_and_mint_result.change_outpoint) + .unwrap() + .unwrap() + .output(), + ) + .unwrap() + .coin_amount() + .unwrap() + - Amount::from_atoms(5)) + .unwrap(), + ), + Destination::AnyoneCanSpend, + )) + .build(); + let block3 = tf.make_block_builder().add_transaction(tx_3).build(&mut rng); + tf.process_block(block3.clone(), BlockSource::Local).unwrap(); + + _ = tx.send(( + chain_config.coin_ticker().to_owned(), + Address::new(&chain_config, token_id).unwrap().into_string(), + chain_config.coin_decimals(), + )); + + vec![ + issue_and_mint_result.issue_block, + issue_and_mint_result.mint_block, + block1, + block2, + block3, + ] + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await.expect("web server failed"); + }); + + let (ml, tkn, coin_decimals) = + rx.await.expect("the web server task sends the decimals parameters"); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the ask book of ML_TKN, paginated by one level per request + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&items=1"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_some()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(10), coin_decimals) + ); + + // the next page + let cursor = body["next_cursor"].as_str().unwrap(); + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&items=1&cursor={cursor}"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_none()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "3/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(10), coin_decimals) + ); + } + + // the bid book of ML_TKN has the single reversed order + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=bid"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_none()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(5), coin_decimals) + ); + } + + // the reversed pair sees the bid order as its ask + { + let url = format!("/api/v2/order/pair/{tkn}_{ml}/book?side=ask"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + } + + // an invalid side is rejected + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=nonsense"); + let (status, body) = get_json(url).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&cursor=garbage"); + let (status, body) = get_json(url).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + shutdown_task(task).await; +} + #[rstest] #[trace] #[case(Seed::from_entropy())] @@ -284,7 +529,7 @@ async fn order_pairs(#[case] seed: Seed) { } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let (order_id, ml, tkn) = rx.await.unwrap(); diff --git a/api-server/stack-test-suite/tests/v2/pools.rs b/api-server/stack-test-suite/tests/v2/pools.rs index 6fc5c37ad0..ecdebfae51 100644 --- a/api-server/stack-test-suite/tests/v2/pools.rs +++ b/api-server/stack-test-suite/tests/v2/pools.rs @@ -336,3 +336,168 @@ async fn ok(#[case] seed: Seed) { shutdown_task(task).await; } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn pools_cursor_pagination(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let mut stake_pool_outpoint = UtxoOutPoint::new( + OutPointSourceId::BlockReward(tf.genesis().get_id().into()), + 0, + ); + let mut available_amount = ((chain_config.min_stake_pool_pledge() * 10).unwrap() + + Amount::from_atoms(10000)) + .unwrap(); + + let mut pool_ids = vec![]; + let mut blocks = vec![]; + for _ in 0..3 { + let (transfer_outpoint, _stake_pool_data, pool_id, block) = prepare_stake_pool( + stake_pool_outpoint, + &mut rng, + &mut available_amount, + &mut tf, + ); + stake_pool_outpoint = transfer_outpoint; + pool_ids.push(pool_id); + blocks.push(block); + } + + let pool_ids = pool_ids + .iter() + .map(|pool_id| Address::new(&chain_config, *pool_id).unwrap().into_string()) + .collect::>(); + _ = tx.send(pool_ids); + + blocks + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await.expect("web server failed"); + }); + + let created_pools = rx.await.unwrap(); + let expected_order = created_pools.iter().rev().collect::>(); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the whole list, unpaged + { + let (status, body) = get_json("/api/v2/pool?offset=0&items=100".to_owned()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr = body.as_array().unwrap(); + assert_eq!(arr.len(), 3); + for (expected_pool_id, json) in expected_order.iter().zip(arr) { + assert_eq!( + json.get("pool_id").unwrap().as_str().unwrap(), + *expected_pool_id + ); + } + } + + // the same list, walked with the cursor + { + let mut url = "/api/v2/pool?items=1&cursor=".to_owned(); + let mut paged_order = vec![]; + loop { + // bounded by the known pool count so a self-referencing cursor fails + // instead of hanging + assert!( + paged_order.len() < expected_order.len(), + "cursor walk exceeded the expected pool count" + ); + let (status, body) = get_json(url.clone()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + paged_order.push(items[0]["pool_id"].as_str().unwrap().to_owned()); + + match body["next_cursor"].as_str() { + Some(cursor) => url = format!("/api/v2/pool?items=1&cursor={cursor}"), + None => break, + } + } + assert_eq!(paged_order.len(), 3); + assert_eq!( + paged_order, + expected_order.iter().map(|s| s.as_str()).collect::>() + ); + } + + // a cursor is only supported by the default sort order + { + let (status, body) = get_json("/api/v2/pool?sort=by_pledge&cursor=xyz".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let (status, body) = get_json("/api/v2/pool?cursor=garbage".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + // a zero page size is rejected + { + let (status, body) = get_json("/api/v2/pool?items=0".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid number of items"); + } + + shutdown_task(task).await; +} diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 9df59e8d4e..8d88c3a92b 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -441,3 +441,194 @@ async fn ok_coins(#[case] seed: Seed) { shutdown_task(task).await; } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn coin_holders(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let (_, alice_pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let (_, bob_pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let alice_destination = Destination::PublicKeyHash(PublicKeyHash::from(&alice_pk)); + let bob_destination = Destination::PublicKeyHash(PublicKeyHash::from(&bob_pk)); + + let transaction = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::BlockReward(tf.genesis().get_id().into()), + 0, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::Transfer( + OutputValue::Coin(Amount::from_atoms(100)), + alice_destination.clone(), + )) + .add_output(TxOutput::Transfer( + OutputValue::Coin(Amount::from_atoms(50)), + bob_destination.clone(), + )) + .build(); + + let block = tf.make_block_builder().add_transaction(transaction).build(&mut rng); + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + + let alice_address = Address::new(&chain_config, alice_destination).unwrap(); + let bob_address = Address::new(&chain_config, bob_destination).unwrap(); + _ = tx.send((alice_address, bob_address, chain_config.coin_decimals())); + + vec![block] + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, false).await.expect("web server failed"); + }); + + let (alice_address, bob_address, coin_decimals) = rx.await.unwrap(); + let alice_atoms = amount_to_json(Amount::from_atoms(100), coin_decimals); + let bob_atoms = amount_to_json(Amount::from_atoms(50), coin_decimals); + + // the holder count is known from an unpaged request; the walk below is + // bounded by it so a self-referencing cursor fails instead of hanging + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/coin/holders?items=100", + addr.ip(), + addr.port() + )) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let body: serde_json::Value = serde_json::from_str(&response.text().await.unwrap()).unwrap(); + let num_holders = body["items"].as_array().unwrap().len(); + assert!(num_holders >= 2); + // the baseline is fetched with the server's maximum page size; a larger list + // would be silently truncated and the bound below would be wrong + assert!( + num_holders < 100, + "the baseline fetch was truncated by the server page cap (MAX_NUM_ITEMS)" + ); + + // walk the whole holders list, one item per request + let mut url = "/api/v2/statistics/coin/holders?items=1".to_owned(); + let mut holders = BTreeMap::::new(); + let mut prev_amount: Option = None; + loop { + // bounded by the known holder count so a self-referencing cursor fails + // instead of hanging + assert!( + holders.len() < num_holders, + "the holders cursor walk exceeded the expected number of holders" + ); + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + assert_eq!(response.status(), 200, "Failed getting coin holders"); + + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + + let address = items[0]["address"].as_str().unwrap().to_owned(); + assert!(!holders.contains_key(&address), "duplicate holder address"); + let amount = items[0]["amount"]["atoms"].as_str().unwrap().to_owned(); + + // the amounts are ordered, from the biggest to the smallest + if let Some(prev) = prev_amount.replace(amount.clone()) { + // the amounts are encoded as integer atom strings by `amount_to_json`; a + // parse failure means that serialization contract changed + let (prev, amount) = ( + prev.parse::().expect("atoms is an integer string"), + amount.parse::().expect("atoms is an integer string"), + ); + assert!(prev >= amount); + } + holders.insert(address, items[0]["amount"].clone()); + + match body["next_cursor"].as_str() { + Some(cursor) => { + url = format!("/api/v2/statistics/coin/holders?items=1&cursor={cursor}"); + } + None => break, + } + } + + assert_eq!(holders.len(), num_holders); + assert_eq!(holders.get(alice_address.as_str()), Some(&alice_atoms)); + assert_eq!(holders.get(bob_address.as_str()), Some(&bob_atoms)); + + // an unknown token has no holders + let mut rng = make_seedable_rng(seed); + let token_id = TokenId::new(H256::random_using(&mut rng)); + let token_id = Address::::new(&create_unit_test_config(), token_id).unwrap(); + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/token/{}/holders", + addr.ip(), + addr.port(), + token_id.as_str() + )) + .await + .unwrap(); + assert_eq!(response.status(), 404); + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Token not found"); + + // an invalid cursor is rejected + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/coin/holders?cursor=garbage", + addr.ip(), + addr.port() + )) + .await + .unwrap(); + assert_eq!(response.status(), 400); + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + + shutdown_task(task).await; +} diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 90cabea401..c788fbd8fd 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -308,3 +308,158 @@ fn compare_body(body: &Value, expected_transaction: &Value) { &expected_transaction["tx_global_index"] ); } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn cursor_pagination(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let n_blocks = 10; + + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let chainstate_block_ids = tf + .create_chain_return_ids(&tf.genesis().get_id().into(), n_blocks, &mut rng) + .unwrap(); + + _ = tx.send(()); + + chainstate_block_ids + .iter() + .map(|id| tf.block(tf.to_chain_block_id(id))) + .collect::>() + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await.expect("web server failed") + }); + + let () = rx.await.unwrap(); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the whole list, unpaged + let (status, body) = get_json("/api/v2/transaction?offset=0&items=100".to_owned()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr = body.as_array().unwrap(); + let expected_txs = + arr.iter().map(|tx| tx["id"].as_str().unwrap().to_owned()).collect::>(); + assert!(!expected_txs.is_empty()); + let num_tx = expected_txs.len(); + // the baseline is fetched with the server's maximum page size; a larger list + // would be silently truncated and the comparison below would be vacuous + assert!( + num_tx < 100, + "the baseline fetch was truncated by the server page cap (MAX_NUM_ITEMS)" + ); + + // the same list, walked with the cursor; the walk is bounded by the known + // baseline length so a self-referencing cursor fails instead of hanging + let mut url = "/api/v2/transaction?items=1&cursor=".to_owned(); + let mut paged_txs = vec![]; + loop { + assert!( + paged_txs.len() < num_tx, + "the cursor walk exceeded the expected number of transactions" + ); + let (status, body) = get_json(url.clone()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + paged_txs.push(items[0]["id"].as_str().unwrap().to_owned()); + + match body["next_cursor"].as_str() { + Some(cursor) => url = format!("/api/v2/transaction?items=1&cursor={cursor}"), + None => break, + } + } + assert_eq!(paged_txs.len(), num_tx); + assert_eq!(paged_txs, expected_txs); + + // an offset mode cannot be combined with a cursor + { + let (status, body) = + get_json("/api/v2/transaction?offset_mode=legacy&cursor=xyz".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let (status, body) = get_json("/api/v2/transaction?cursor=garbage".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + // a zero page size is rejected + { + let (status, body) = get_json("/api/v2/transaction?items=0".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid number of items"); + } + + // a cursor minted by another endpoint is rejected instead of silently misread + { + let (status, tx_body) = get_json("/api/v2/transaction?items=1&cursor=".to_owned()).await; + assert_eq!(status, 200); + let tx_body: serde_json::Value = serde_json::from_str(&tx_body).unwrap(); + let foreign_cursor = tx_body["next_cursor"].as_str().unwrap(); + let (status, body) = + get_json(format!("/api/v2/pool?items=1&cursor={foreign_cursor}")).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + shutdown_task(task).await; +} diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 7395508a23..902c8d8a06 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2198,6 +2198,480 @@ async fn orders<'a, S: for<'b> Transactional<'b>>(rng: &mut impl CryptoRng, stor ); } +/// Keyset (cursor) pagination of the pool listing over the latest pool data. +async fn pool_data_keyset( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = create_unit_test_config(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + // three pools created at heights 10, 20, 30 + let (pool1_id, pool2_id, pool3_id) = ( + PoolId::random_using(&mut rng), + PoolId::random_using(&mut rng), + PoolId::random_using(&mut rng), + ); + let (pool1_data, pool2_data, pool3_data) = ( + make_pool_data(&mut rng, Amount::from_atoms(100)), + make_pool_data(&mut rng, Amount::from_atoms(200)), + make_pool_data(&mut rng, Amount::from_atoms(300)), + ); + let (h10, h20, h30) = ( + BlockHeight::new(10), + BlockHeight::new(20), + BlockHeight::new(30), + ); + db_tx.set_pool_data_at_height(pool1_id, &pool1_data, h10).await.unwrap(); + db_tx.set_pool_data_at_height(pool2_id, &pool2_data, h20).await.unwrap(); + db_tx.set_pool_data_at_height(pool3_id, &pool3_data, h30).await.unwrap(); + + // the creation height is the height of the first insert + let expect = |expected: &[(BlockHeight, PoolId, &PoolDataWithExtraInfo)]| { + expected + .iter() + .map(|(height, pool_id, data)| (*height, *pool_id, (*data).clone())) + .collect::>() + }; + + // ordered by the creation height (descending) + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h20, pool2_id, &pool2_data), + (h10, pool1_id, &pool1_data) + ]) + ); + + // a later update does not change the creation height + let pool1_updated = make_pool_data(&mut rng, Amount::from_atoms(111)); + db_tx + .set_pool_data_at_height(pool1_id, &pool1_updated, h30.next_height()) + .await + .unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h20, pool2_id, &pool2_data), + (h10, pool1_id, &pool1_updated), + ]) + ); + + // a late lower-height write corrects the creation height to the true minimum: the + // backends agree on min-height semantics (matching the `MIN(block_height)` + // re-derivation of the rollback path), which keeps a mid-history re-seed from + // silently mis-ordering the listing. The latest data is still the highest height's. + let pool2_early = make_pool_data(&mut rng, Amount::from_atoms(222)); + let h5 = BlockHeight::new(5); + db_tx.set_pool_data_at_height(pool2_id, &pool2_early, h5).await.unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h10, pool1_id, &pool1_updated), + (h5, pool2_id, &pool2_data), + ]) + ); + + // cursor pagination + let page1 = db_tx.get_latest_pool_data_before(2, None).await.unwrap(); + assert_eq!( + page1, + expect(&[(h30, pool3_id, &pool3_data), (h10, pool1_id, &pool1_updated)]) + ); + + let page2 = db_tx.get_latest_pool_data_before(2, Some((h10, pool1_id))).await.unwrap(); + assert_eq!(page2, expect(&[(h5, pool2_id, &pool2_data)])); + + let end = db_tx.get_latest_pool_data_before(2, Some((h5, pool2_id))).await.unwrap(); + assert!(end.is_empty()); + + // pools with a zero staker balance (decommissioned) are not listed + let pool3_decommissioned = make_pool_data(&mut rng, Amount::ZERO); + db_tx + .set_pool_data_at_height(pool3_id, &pool3_decommissioned, h30.next_height()) + .await + .unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[(h10, pool1_id, &pool1_updated), (h5, pool2_id, &pool2_data)]) + ); + + // the rollback restores the latest data below the deleted height, keeping the + // original (minimum) creation height; the late h5 write of pool2 survives + db_tx.del_pools_above_height(h30).await.unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h10, pool1_id, &pool1_data), + (h5, pool2_id, &pool2_data), + ]) + ); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +/// Holders of an asset: the top address amounts with keyset (cursor) pagination. +async fn top_address_amounts( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = create_unit_test_config(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + let (addr1, addr2, addr3, addr4) = ( + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + ); + + let token_id = TokenId::random_using(&mut rng); + let h1 = BlockHeight::new(1); + + // register the token (the scanner stores the issuance, including the decimals, before + // any balances) + let token_data = FungibleTokenData { + token_ticker: "XXXX".as_bytes().to_vec(), + number_of_decimals: 18, + metadata_uri: "http://uri".as_bytes().to_vec(), + circulating_supply: Amount::ZERO, + total_supply: TokenTotalSupply::Unlimited, + is_locked: false, + frozen: IsTokenFrozen::No(IsTokenFreezable::Yes), + authority: Destination::PublicKeyHash(PublicKeyHash::from( + &PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr).1, + )), + next_nonce: AccountNonce::new(0), + }; + db_tx.set_fungible_token_issuance(token_id, h1, token_data).await.unwrap(); + + // coin holders: addr2 (300), addr3 and addr4 (200 each, so the encoded-address + // tie-break is exercised), addr1 (100) + let coin_holders: Vec<(Address, Amount)> = vec![ + (addr1.clone(), Amount::from_atoms(100)), + (addr2.clone(), Amount::from_atoms(300)), + (addr3.clone(), Amount::from_atoms(200)), + (addr4.clone(), Amount::from_atoms(200)), + ]; + for (address, amount) in &coin_holders { + db_tx + .set_address_balance_at_height(address, *amount, CoinOrTokenId::Coin, h1) + .await + .unwrap(); + } + // a token holder + db_tx + .set_address_balance_at_height( + &addr1, + Amount::from_atoms(50), + CoinOrTokenId::TokenId(token_id), + h1, + ) + .await + .unwrap(); + + // NOTE: no re-sorting before the asserts — the ordering (balance descending, the + // encoded address descending as tie-break) is exactly what the contract requires + let expect = |expected: &[(&Address, Amount)]| { + expected + .iter() + .map(|(address, amount)| (address.as_str().to_owned(), *amount)) + .collect::>() + }; + + // equal balances tie-break by the encoded address, descending byte order + let mut tie = [addr3.as_str(), addr4.as_str()]; + tie.sort_unstable(); + let (tie_low, tie_high) = (tie[0], tie[1]); + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + all, + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] + ); + + // the token holders are independent of the coin ones + let token_holders = db_tx + .get_top_address_amounts(CoinOrTokenId::TokenId(token_id), 10, None) + .await + .unwrap(); + assert_eq!(token_holders, expect(&[(&addr1, Amount::from_atoms(50))])); + + // cursor pagination over the coin holders + let page = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 2, None).await.unwrap(); + assert_eq!( + page, + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + ] + ); + let (last_address, last_amount) = page[1].clone(); + + let next_page = db_tx + .get_top_address_amounts( + CoinOrTokenId::Coin, + 2, + Some((last_address.clone(), last_amount)), + ) + .await + .unwrap(); + assert_eq!( + next_page, + vec![ + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] + ); + let (last_address2, last_amount2) = next_page[1].clone(); + + let end = db_tx + .get_top_address_amounts(CoinOrTokenId::Coin, 2, Some((last_address2, last_amount2))) + .await + .unwrap(); + assert!(end.is_empty()); + + // a balance update changes the ranking + db_tx + .set_address_balance_at_height( + &addr1, + Amount::from_atoms(400), + CoinOrTokenId::Coin, + h1.next_height(), + ) + .await + .unwrap(); + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + all, + vec![ + (addr1.as_str().to_owned(), Amount::from_atoms(400)), + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + ] + ); + + // the rollback restores the previous balances; this also guards the ordering + // coupling in `del_address_balance_above_height`: the holders cache + // re-derivation is only correct because the versioned balances above the height + // are deleted first, within the same call + db_tx.del_address_balance_above_height(h1).await.unwrap(); + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + all, + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] + ); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +/// The order book entries of a trading pair: only the live (not concluded, not frozen) +/// orders with non-zero remaining balances are returned. +async fn order_book_entries( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = common::chain::config::create_regtest(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + let token1 = TokenId::random_using(&mut rng); + let token2 = TokenId::random_using(&mut rng); + + // three live orders of the (Coin, token1) pair and one of another pair + let (order1_id, order1) = random_order( + &mut rng, + BlockHeight::new(1), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order2_id, order2) = random_order( + &mut rng, + BlockHeight::new(2), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order3_id, order3) = random_order( + &mut rng, + BlockHeight::new(3), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order4_id, order4) = random_order( + &mut rng, + BlockHeight::new(4), + CoinOrTokenId::TokenId(token1), + CoinOrTokenId::TokenId(token2), + ); + for (order_id, order) in [ + (&order1_id, &order1), + (&order2_id, &order2), + (&order3_id, &order3), + (&order4_id, &order4), + ] { + db_tx + .set_order_at_height(*order_id, order, order.creation_block_height) + .await + .unwrap(); + } + + let sort_entries = |mut entries: Vec<(Amount, Amount)>| { + entries.sort_by_key(|(ask_balance, give_balance)| (*ask_balance, *give_balance)); + entries + }; + let expected_pair = sort_entries(vec![ + (order1.ask_balance, order1.give_balance), + (order2.ask_balance, order2.give_balance), + (order3.ask_balance, order3.give_balance), + ]); + + let (entries, has_more) = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + // the entries come back with the deepest orders first (the ordering contract) + assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); + assert!(!has_more); + // TODO: cover has_more == true (ORDER_BOOK_MAX_ORDERS truncation) once the cap is + // injectable (test-only constructor parameter or a configurable limit), e.g. via + // a small cap and a handful of seeded orders + assert_eq!(sort_entries(entries.clone()), expected_pair); + + // the reverse pair is empty + let (entries, has_more) = db_tx + .get_order_book_entries(CoinOrTokenId::TokenId(token1), CoinOrTokenId::Coin) + .await + .unwrap(); + assert!(entries.is_empty()); + assert!(!has_more); + + // a fully filled (concluded) order is no longer part of the book + let block_height = BlockHeight::new(100); + let order1_concluded = + order1.clone().fill(&chain_config, block_height, order1.ask_balance).conclude(); + db_tx + .set_order_at_height(order1_id, &order1_concluded, block_height.next_height()) + .await + .unwrap(); + let (entries, has_more) = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!( + sort_entries(entries), + sort_entries(vec![ + (order2.ask_balance, order2.give_balance), + (order3.ask_balance, order3.give_balance), + ]) + ); + assert!(!has_more); + + // a frozen order is not part of the book either + let order2_frozen = order2.clone().freeze(); + db_tx + .set_order_at_height(order2_id, &order2_frozen, block_height.next_height()) + .await + .unwrap(); + let (entries, has_more) = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!( + sort_entries(entries), + vec![(order3.ask_balance, order3.give_balance)] + ); + assert!(!has_more); + + // the rollback restores the original book + db_tx.del_orders_above_height(block_height).await.unwrap(); + let (entries, has_more) = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!(sort_entries(entries.clone()), expected_pair); + assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); + assert!(!has_more); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +fn make_pool_data(rng: &mut impl CryptoRng, staker_balance: Amount) -> PoolDataWithExtraInfo { + let (_, vrf_pk) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + let (_, pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let pool_data = PoolData::new( + Destination::PublicKey(pk), + staker_balance, + Amount::ZERO, + vrf_pk, + PerThousand::new(500).unwrap(), + Amount::from_atoms(100), + ); + PoolDataWithExtraInfo { + pool_data, + delegations_balance: Amount::ZERO, + } +} + +fn make_address( + chain_config: &common::chain::ChainConfig, + rng: &mut impl CryptoRng, +) -> Address { + let (_, pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + Address::new( + chain_config, + Destination::PublicKeyHash(PublicKeyHash::from(&pk)), + ) + .unwrap() +} + fn random_order( rng: &mut impl CryptoRng, creation_height: BlockHeight, @@ -2234,6 +2708,9 @@ where vec![ make_test!(initialization, storage_maker.clone()), make_test!(set_get, storage_maker.clone()), + make_test!(pool_data_keyset, storage_maker.clone()), + make_test!(top_address_amounts, storage_maker.clone()), + make_test!(order_book_entries, storage_maker.clone()), make_test!(stream_events_append_and_read, storage_maker), ] .into_iter() diff --git a/api-server/web-server/Cargo.toml b/api-server/web-server/Cargo.toml index c9b028f97c..a85649a59c 100644 --- a/api-server/web-server/Cargo.toml +++ b/api-server/web-server/Cargo.toml @@ -20,6 +20,7 @@ mempool = { path = "../../mempool" } axum.workspace = true async-trait.workspace = true +base64.workspace = true ctor.workspace = true clap = { workspace = true, features = ["derive"] } futures = { workspace = true, default-features = false } diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs new file mode 100644 index 0000000000..5b113fe23a --- /dev/null +++ b/api-server/web-server/src/api/cursor.rs @@ -0,0 +1,123 @@ +// Copyright (c) 2023 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Opaque keyset pagination cursors. +//! +//! A cursor marks the position of the last item of a page within the sorted result +//! set of a listing endpoint: the next page consists of the items strictly following +//! that position. A cursor carries the sort keys of the item as strings plus a +//! tie-break id and is rendered as an URL-safe, unpadded base64 encoding of its JSON +//! form, so that the internal sort order of the endpoint does not leak to the clients +//! and stays free to change. + +use base64::Engine; +use serde::{Deserialize, Serialize}; + +use crate::error::ApiServerWebServerClientError; + +/// The largest accepted encoded cursor length. Legitimate cursors carry a handful of +/// short keys (well under this bound); the cap rejects oversized unauthenticated input +/// before the base64 decode. +const MAX_CURSOR_LEN: usize = 1024; + +/// The largest number of sort keys accepted in a decoded cursor. Every endpoint +/// validates its own key count after decoding; this bound additionally rejects +/// payloads carrying arbitrary numbers of keys for ANY tag at decode time, so a +/// fabricated cursor cannot smuggle an unbounded key vector past the shared +/// entry point. +const MAX_CURSOR_KEYS: usize = 16; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Cursor { + /// Identifier of the endpoint (and sort schema) the cursor belongs to; a cursor + /// minted by another endpoint is rejected instead of silently misread. + tag: String, + /// Sort keys of the item the cursor points to. + keys: Vec, + /// Unique tie-break id of the item. + id: String, +} + +impl Cursor { + pub fn new(tag: &str, keys: Vec, id: String) -> Self { + Self { + tag: tag.to_owned(), + keys, + id, + } + } + + /// Checks the endpoint tag and returns the sort keys and the tie-break id. + pub fn into_parts( + self, + tag: &str, + ) -> Result<(Vec, String), ApiServerWebServerClientError> { + if self.tag != tag { + return Err(ApiServerWebServerClientError::InvalidCursor); + } + Ok((self.keys, self.id)) + } + + pub fn encode(&self) -> String { + base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_string(self).expect("serialization must not fail")) + } + + pub fn decode(encoded: &str) -> Result { + // a cursor carries a handful of short keys; reject oversized payloads before + // the base64 decode as a cheap guard against unauthenticated input bloat + if encoded.len() > MAX_CURSOR_LEN { + return Err(ApiServerWebServerClientError::InvalidCursor); + } + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| ApiServerWebServerClientError::InvalidCursor)?; + serde_json::from_slice::(&decoded) + .map_err(|_| ApiServerWebServerClientError::InvalidCursor)? + .validate_keys() + } + + /// Rejects cursors with an implausibly large key vector before any endpoint + /// logic inspects the keys; each endpoint then still checks its exact count. + fn validate_keys(self) -> Result { + if self.keys.len() > MAX_CURSOR_KEYS { + return Err(ApiServerWebServerClientError::InvalidCursor); + } + Ok(self) + } + + /// Decodes a cursor request parameter. An empty value starts the listing from the + /// beginning, so that a client can walk all the pages of a result set uniformly. + pub fn decode_start(encoded: &str) -> Result, ApiServerWebServerClientError> { + if encoded.is_empty() { + Ok(None) + } else { + Self::decode(encoded).map(Some) + } + } +} + +/// Renders the listing response of a cursor-based request: the items of the page plus +/// the cursor of the last returned item, `null` when the result set is exhausted. +pub fn paged_response( + items: Vec, + next_cursor: Option, +) -> serde_json::Value { + serde_json::json!({ + "items": items, + "next_cursor": next_cursor.map(|cursor| cursor.encode()), + }) +} diff --git a/api-server/web-server/src/api/mod.rs b/api-server/web-server/src/api/mod.rs index ccf14cba80..6bcde1a303 100644 --- a/api-server/web-server/src/api/mod.rs +++ b/api-server/web-server/src/api/mod.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod cursor; pub mod json_helpers; pub mod stream; pub mod v2; diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b5d3e08d1b..24b8daa309 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -15,9 +15,12 @@ use crate::{ MempoolQueryClient, TxSubmitClient, - api::json_helpers::{ - self, TokenDecimals, amount_to_json, block_header_to_json, pool_data_to_json, - to_tx_json_with_block_info, tx_to_json, txoutput_to_json, utxo_outpoint_to_json, + api::{ + cursor::{self, Cursor}, + json_helpers::{ + self, TokenDecimals, amount_to_json, block_header_to_json, pool_data_to_json, + to_tx_json_with_block_info, tx_to_json, txoutput_to_json, utxo_outpoint_to_json, + }, }, error::{ ApiServerWebServerClientError, ApiServerWebServerError, ApiServerWebServerForbiddenError, @@ -37,6 +40,7 @@ use axum::{ routing::{get, post}, }; use common::{ + Uint256, address::Address, chain::{ Block, ChainConfig, Destination, OutPointSourceId, SignedTransaction, Transaction, @@ -45,7 +49,7 @@ use common::{ make_token_id, tokens::{IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, TokenId}, }, - primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable}, + primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable, rational::Rational}, }; use hex::ToHex; use serde::Deserialize; @@ -137,7 +141,9 @@ pub fn routes< let router = router .route("/statistics/coin", get(coin_statistics)) - .route("/statistics/token/:id", get(token_statistics)); + .route("/statistics/coin/holders", get(coin_holders)) + .route("/statistics/token/:id", get(token_statistics)) + .route("/statistics/token/:id/holders", get(token_holders)); let router = router .route("/token", get(token_ids)) @@ -150,6 +156,7 @@ pub fn routes< .route("/order", get(orders)) .route("/order/:id", get(order)) .route("/order/pair/:pair", get(order_pair)) + .route("/order/pair/:pair/book", get(order_pair_book)) // Note: the real-time event stream is exposed together with the v2 endpoints, since the // events reference data that is served by them. .route("/stream", get(super::stream::stream_events)) @@ -741,11 +748,22 @@ pub async fn transactions( State(state): State, Arc>>, ) -> Result { const OFFSET_MODE: &str = "offset_mode"; + const CURSOR: &str = "cursor"; let offset_mode = params .get(OFFSET_MODE) .map(|mode| OffsetMode::from_str(mode)) .transpose()? .unwrap_or(OffsetMode::Legacy); + + let cursor_param = params.get(CURSOR); + if cursor_param.is_some() && params.contains_key(OFFSET_MODE) { + // the cursor walks the global index of the transactions, the offset modes are + // the legacy ways of walking the listing + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + let offset_and_items = get_offset_and_items(¶ms)?; let db_tx = state.db.transaction_ro().await.map_err(|e| { @@ -753,25 +771,111 @@ pub async fn transactions( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - let txs = match offset_mode { - OffsetMode::Absolute => { - db_tx - .get_transactions_with_block_info_before_tx_global_index( - offset_and_items.items, - offset_and_items.offset, + let fetch_len = offset_and_items.items.saturating_add(1); + + let (txs, next_cursor) = if let Some(cursor_str) = cursor_param { + let txs = match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + // the tie-break id of the cursor is ignored for this tag: the global + // transaction index is unique per transaction, so the index key alone is + // already a total order (the pools and holders endpoints need the + // tie-break because their amounts can repeat) + let (keys, _tie_break_id) = cursor.into_parts("transactions")?; + // the transactions cursor carries exactly one key; reject anything + // else instead of silently ignoring extra entries + ensure!( + keys.len() == 1, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); + let tx_global_index = + keys.first().expect("the cursor carries exactly one key, checked above"); + let tx_global_index = u64::from_str(tx_global_index).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; + // reject out-of-range indices as a client error before they reach the + // storage layer, which maps the overflow to an internal server error + if i64::try_from(tx_global_index).is_err() { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + + db_tx + .get_transactions_with_block_info_before_tx_global_index( + fetch_len, + tx_global_index, + ) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + } + // an empty cursor starts the listing from the beginning; the cursor and + // offset variants of the listing must agree on the (newest-first, + // ascending global index) order for the cursor boundary to stay + // consistent — the backends derive both orderings from the same scanned + // main-chain state, so they cannot diverge without a storage bug + None => db_tx.get_transactions_with_block_info(fetch_len, 0).await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, ) - .await - } - OffsetMode::Legacy => { - db_tx - .get_transactions_with_block_info(offset_and_items.items, offset_and_items.offset) - .await + })?, + }; + + // the shared validator guarantees items > 0 and the fetch above requests + // one extra row, so the indexing below stays in bounds; keep those + // cross-function invariants locally asserted + debug_assert!(offset_and_items.items > 0); + let has_next_page = txs.len() > offset_and_items.items as usize; + debug_assert!( + txs.len() <= offset_and_items.items as usize + 1, + "the storage returned at most the requested page plus the extra row" + ); + let next_cursor = has_next_page.then(|| { + let last_returned = &txs[offset_and_items.items as usize - 1]; + Cursor::new( + "transactions", + vec![last_returned.tx_global_index.to_string()], + last_returned.tx_info.tx.transaction().get_id().to_hash().encode_hex::(), + ) + }); + let txs = txs.into_iter().take(offset_and_items.items as usize).collect(); + + (txs, next_cursor) + } else { + let txs = match offset_mode { + OffsetMode::Absolute => { + db_tx + .get_transactions_with_block_info_before_tx_global_index( + offset_and_items.items, + offset_and_items.offset, + ) + .await + } + OffsetMode::Legacy => { + db_tx + .get_transactions_with_block_info( + offset_and_items.items, + offset_and_items.offset, + ) + .await + } } - } - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })?; + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + (txs, None) + }; let tip_height = best_block(&state).await?.block_height(); let txs = txs @@ -787,7 +891,11 @@ pub async fn transactions( }) .collect(); - Ok(Json(serde_json::Value::Array(txs))) + Ok(Json(match next_cursor { + Some(next_cursor) => cursor::paged_response(txs, Some(next_cursor)), + None if cursor_param.is_some() => cursor::paged_response(txs, None), + None => serde_json::Value::Array(txs), + })) } pub async fn transaction< @@ -1325,11 +1433,19 @@ impl FromStr for PoolSorting { } } +/// List pools, newest creation height first (or deepest pledge first with +/// `sort=by_pledge`). +/// +/// Note: pages are only stable once the scanner is fully caught up. A late +/// lower-height write (reorg catch-up) adjusts a pool's creation height in the +/// backing cache, so a keyset walk performed concurrently with such a write may +/// skip or repeat that pool. pub async fn pools( Query(params): Query>, State(state): State, Arc>>, ) -> Result { const SORT: &str = "sort"; + const CURSOR: &str = "cursor"; let sort = params .get(SORT) @@ -1337,6 +1453,14 @@ pub async fn pools( .transpose()? .unwrap_or(PoolSorting::ByHeight); + let cursor_param = params.get(CURSOR); + if cursor_param.is_some() && !matches!(sort, PoolSorting::ByHeight) { + // the cursor orders pools by creation height, the only ordering it supports + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + let offset_and_items = get_offset_and_items(¶ms)?; let db_tx = state.db.transaction_ro().await.map_err(|e| { @@ -1344,35 +1468,147 @@ pub async fn pools( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - let pools = match sort { - PoolSorting::ByHeight => db_tx - .get_latest_pool_data(offset_and_items.items, offset_and_items.offset) - .await - .map_err(|e| { + let pools_json = if let Some(cursor_str) = cursor_param { + let cursor = match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, id) = cursor.into_parts("pools")?; + // the pools cursor carries exactly one key; reject anything else + // instead of silently ignoring extra entries + ensure!( + keys.len() == 1, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); + + match keys.first() { + Some(creation_height) if !id.is_empty() => { + let creation_height = u64::from_str(creation_height).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; + // reject out-of-range heights as a client error before they + // reach the storage layer, which maps the overflow to a panic + if i64::try_from(creation_height).is_err() { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + // re-encode through Address::new so the id used as the + // keyset boundary is the canonical lowercase form: bech32 + // accepts all-uppercase strings and `from_string` keeps the + // input verbatim, so an all-uppercase cursor id would + // validate but silently break the byte-wise comparison + // against the canonical strings in storage + let pool_address = + Address::from_string(&state.chain_config, &id).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; + let pool_address = + Address::new(&state.chain_config, pool_address.into_object()).map_err( + |_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + }, + )?; + let pool_id = pool_address.into_object(); + Some((BlockHeight::new(creation_height), pool_id)) + } + _ => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + } + } + // an empty cursor starts the listing from the beginning + None => None, + }; + + // an extra item tells whether the result set continues past the page + let items_to_fetch = offset_and_items.items.saturating_add(1); + let pools = + db_tx.get_latest_pool_data_before(items_to_fetch, cursor).await.map_err(|e| { logging::log::error!("internal error: {e}"); ApiServerWebServerError::ServerError( ApiServerWebServerServerError::InternalServerError, ) - })?, - PoolSorting::ByPledge => db_tx - .get_pool_data_with_largest_staker_balance( - offset_and_items.items, - offset_and_items.offset, - ) - .await - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError( - ApiServerWebServerServerError::InternalServerError, + })?; + + let has_next_page = pools.len() > offset_and_items.items as usize; + let pools = pools.into_iter().take(offset_and_items.items as usize).collect::>(); + + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking + // should a future caller ever bypass it + let next_cursor = if has_next_page && !pools.is_empty() { + let (creation_height, pool_id, _pool_data) = + pools.last().expect("the page is not empty"); + // the pool id comes from the storage, so encoding cannot fail; map the + // error anyway instead of risking a panic inside a request handler + let pool_id_str = Address::new(&state.chain_config, *pool_id) + .map_err(|e| { + logging::log::error!("internal error: pool id encoding failed: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + .as_str() + .to_owned(); + Some(Cursor::new( + "pools", + vec![creation_height.into_int().to_string()], + pool_id_str, + )) + } else { + None + }; + + let items = pools + .into_iter() + .map(|(_creation_height, pool_id, pool_data)| { + pool_data_to_json(&state.chain_config, pool_data, pool_id) + }) + .collect(); + + cursor::paged_response(items, next_cursor) + } else { + let pools = match sort { + PoolSorting::ByHeight => db_tx + .get_latest_pool_data(offset_and_items.items, offset_and_items.offset) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })?, + PoolSorting::ByPledge => db_tx + .get_pool_data_with_largest_staker_balance( + offset_and_items.items, + offset_and_items.offset, ) - })?, - }; + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })?, + }; - let pools = pools - .into_iter() - .map(|(pool_id, pool_data)| pool_data_to_json(&state.chain_config, pool_data, pool_id)); + let pools = pools + .into_iter() + .map(|(pool_id, pool_data)| pool_data_to_json(&state.chain_config, pool_data, pool_id)); - Ok(Json(pools.collect::>())) + serde_json::Value::Array(pools.collect()) + }; + + Ok(Json(pools_json)) } pub async fn pool( @@ -1734,6 +1970,180 @@ pub async fn token_statistics( }))) } +/// Re-encode an address string in its canonical bech32 form. `Address::from_string` +/// keeps the input verbatim, so an all-uppercase encoding would validate fine yet break +/// the byte-order keyset comparison against the canonical lowercase strings in storage; +/// re-encoding (as the pools listing does for pool ids) removes that. Returns `None` if +/// the string is not a valid address. +fn canonical_address_string(chain_config: &ChainConfig, address: &str) -> Option { + let parsed = Address::::from_string(chain_config, address).ok()?; + Address::new(chain_config, parsed.into_object()) + .ok() + .map(|address| address.into_string()) +} + +/// Shared implementation of the holders endpoints: the top holders of an asset, +/// ordered by the balance, with optional keyset (cursor) pagination. +async fn holders_response( + params: &BTreeMap, + state: &ApiServerWebServerState, Arc>, + coin_or_token_id: CoinOrTokenId, + number_of_decimals: u8, +) -> Result { + const CURSOR: &str = "cursor"; + + let offset_and_items = get_offset_and_items(params)?; + + let cursor = match params.get(CURSOR) { + Some(cursor_str) => match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, tie_break_id) = cursor.into_parts("holders")?; + // `Amount::from_atoms` does not range-check, so reject atoms above + // the representable maximum up front: a fabricated boundary amount + // would otherwise be handed to the storage layer unchecked + let amount = match keys.first() { + Some(atoms) => u128::from_str(atoms) + .ok() + .filter(|atoms| *atoms <= Amount::MAX.into_atoms()) + .map(Amount::from_atoms) + .ok_or(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ))?, + None => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + }; + // the tie-break is an encoded address; reject garbage up front and + // canonicalize the encoding: `from_string` keeps the input verbatim, so + // an all-uppercase cursor would otherwise be compared byte-wise against + // the canonical lowercase strings in storage and mis-order the keyset + let tie_break_id = canonical_address_string(&state.chain_config, &tie_break_id) + .ok_or(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ))?; + Some((tie_break_id, amount)) + } + None => None, + }, + None => None, + }; + + // an extra item is requested to detect the presence of the next page + let fetch_len = offset_and_items.items.saturating_add(1); + + let mut holders = state + .db + .transaction_ro() + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })? + .get_top_address_amounts(coin_or_token_id, fetch_len, cursor) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let has_next_page = holders.len() > offset_and_items.items as usize; + holders.truncate(offset_and_items.items as usize); + + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking should a + // future caller ever bypass it + let next_cursor = if has_next_page && !holders.is_empty() { + let (address, amount) = holders.last().expect("the page is not empty"); + // re-encode so the cursor id is canonical and guaranteed to round-trip through + // the decode-side validation; a stored address that fails to re-encode is a + // storage anomaly, surfaced as a server error instead of a cursor the client + // could not consume on the next page + let address = match canonical_address_string(&state.chain_config, address) { + Some(address) => address, + None => { + logging::log::error!("non-canonical address in the holders listing: {address}"); + return Err(ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + )); + } + }; + Some(Cursor::new( + "holders", + vec![amount.into_atoms().to_string()], + address, + )) + } else { + None + }; + + let items = holders + .into_iter() + .map(|(address, amount)| { + json!({ + "address": address, + "amount": amount_to_json(amount, number_of_decimals), + }) + }) + .collect(); + + Ok(cursor::paged_response(items, next_cursor)) +} + +pub async fn coin_holders( + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + let response = holders_response( + ¶ms, + &state, + CoinOrTokenId::Coin, + state.chain_config.coin_decimals(), + ) + .await?; + + Ok(Json(response)) +} + +pub async fn token_holders( + Path(token_id): Path, + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + let token_id = Address::from_string(&state.chain_config, token_id) + .map_err(|_| { + ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidTokenId) + })? + .into_object(); + + let tx = state.db.transaction_ro().await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let token_decimals = tx + .get_token_num_decimals(token_id) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })? + .ok_or(ApiServerWebServerError::NotFound( + ApiServerWebServerNotFoundError::TokenNotFound, + ))?; + + let response = holders_response( + ¶ms, + &state, + CoinOrTokenId::TokenId(token_id), + token_decimals, + ) + .await?; + + Ok(Json(response)) +} + pub async fn token_ids( Query(params): Query>, State(state): State, Arc>>, @@ -1983,7 +2393,7 @@ pub async fn order_pair( |s: &str| -> Result<(CoinOrTokenId, CoinOrTokenId), ApiServerWebServerError> { let parts: Vec<_> = s.split("_").collect(); ensure!( - parts.len() == 2, + parts.len() == 2 && parts.iter().all(|part| !part.is_empty()), ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidOrderTradingPair ) @@ -2024,6 +2434,364 @@ pub async fn order_pair( Ok(Json(orders.collect::>())) } +/// The order book of a trading pair: the live orders of the pair aggregated into price +/// levels. `side` selects the side of the book: `ask` lists the orders asking for the +/// base currency (first in the pair) while giving the quote currency, `bid` the reverse. +/// The price of a level is the remaining quote amount per remaining base unit (a +/// rational number, in atoms); `amount` is the remaining base amount at the level. The +/// ask book is ordered by the ascending price, the bid book by the descending price; +/// keyset (cursor) pagination over the levels. +/// +/// The levels are computed from a fresh storage snapshot on every request, so a +/// paginated walk is not a consistent snapshot of a moving book: a level whose +/// remaining balance changes after a cursor was issued can be skipped or duplicated +/// on the next page. Clients needing a consistent view should re-fetch from the +/// start (an empty cursor). +/// +/// The levels are aggregated from at most `ORDER_BOOK_MAX_ORDERS` orders per request +/// (the additive `truncated` flag reports when that cap was hit). The flag is +/// informational: levels beyond the cap cannot be reached by continuing the cursor +/// walk. +/// +/// The `price.decimal` field is truncated (floored) toward zero; the exact price is +/// available in the `price.atoms` field. Cursors are side-specific: one minted for +/// `side=ask` is rejected with `invalid cursor` on `side=bid` and vice versa. +pub async fn order_pair_book( + Path(pair): Path, + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + const SIDE: &str = "side"; + const CURSOR: &str = "cursor"; + const SIDE_ASK: &str = "ask"; + const SIDE_BID: &str = "bid"; + + let parse_currency = |s: &str| -> Result { + // the coin ticker is matched case-insensitively for client convenience, while + // token ids are exact bech32 strings (bech32 decoding is case-sensitive by + // design) — an accepted asymmetry, not an oversight + if s.to_uppercase() == state.chain_config.coin_ticker() { + Ok(CoinOrTokenId::Coin) + } else { + let token_id = Address::from_string(&state.chain_config, s) + .map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidTokenId, + ) + })? + .into_object(); + Ok(CoinOrTokenId::TokenId(token_id)) + } + }; + + let parts: Vec<_> = pair.split("_").collect(); + ensure!( + parts.len() == 2 && parts.iter().all(|part| !part.is_empty()), + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidOrderTradingPair + ) + ); + let base = parse_currency(parts[0])?; + let quote = parse_currency(parts[1])?; + + // an ask of the (base, quote) pair is an order asking for the base and giving the + // quote; a bid is the reverse. In both cases the price is expressed in the quote + // currency per one base unit. + let (ask_currency, give_currency, base_is_ask) = match params.get(SIDE).map(String::as_str) { + Some(SIDE_ASK) => (base, quote, true), + Some(SIDE_BID) => (quote, base, false), + _ => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + }; + + // the cursor tag is side-specific: the same price walks in opposite directions on + // the two sides, so a cursor minted for one side must be rejected on the other + let cursor_tag = if base_is_ask { "book-ask" } else { "book-bid" }; + + let offset_and_items = get_offset_and_items(¶ms)?; + + // the cursor is the (numerator, denominator) of the last returned level price + let cursor_price = match params.get(CURSOR) { + Some(cursor_str) => match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, tie_break_id) = cursor.into_parts(cursor_tag)?; + // the book cursor keys are exactly (numerator, denominator) and the + // tie-break id is always empty; reject anything else up front + ensure!( + tie_break_id.is_empty(), + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); + let parse_key = |key: Option<&String>| { + key.and_then(|key| u128::from_str(key).ok()).ok_or( + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ), + ) + }; + let numer = parse_key(keys.first())?; + let denom = parse_key(keys.get(1))?; + ensure!( + denom != 0, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); + Some(Rational::new(numer, denom)) + } + None => None, + }, + None => None, + }; + + let db_tx = state.db.transaction_ro().await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let (entries, has_more_orders) = + db_tx.get_order_book_entries(ask_currency, give_currency).await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + // aggregate the remaining balances into price levels; the storage layer caps + // the input at ORDER_BOOK_MAX_ORDERS entries, so each request (regardless of + // the requested page size) pays one O(ORDER_BOOK_MAX_ORDERS) scan+aggregate — + // acceptable for the capped input, but worth knowing before adding pages to + // the walk + let mut levels = BTreeMap::, u128>::new(); + for (ask_balance, give_balance) in entries { + // defensive: the storage layer filters these out, but a zero balance would + // yield a zero-denominator price whose `Ord` compares equal to every other + // key and silently corrupts the level map, so skip it here as well + if ask_balance == Amount::ZERO || give_balance == Amount::ZERO { + continue; + } + let (price, base_amount) = if base_is_ask { + ( + Rational::new(give_balance.into_atoms(), ask_balance.into_atoms()), + ask_balance.into_atoms(), + ) + } else { + ( + Rational::new(ask_balance.into_atoms(), give_balance.into_atoms()), + give_balance.into_atoms(), + ) + }; + // reduce the fraction so equal prices always map to the same level; a zero + // denominator would corrupt the level ordering, so it maps to an internal + // server error instead of degrading silently + let price = reduce_rational(price).ok_or_else(|| { + logging::log::error!("order book price with a zero denominator reached reduction"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let level = levels.entry(price).or_insert(0); + *level = level.checked_add(base_amount).ok_or_else(|| { + logging::log::error!("order book level amount overflow"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + } + + // the ask book is ordered by the ascending price, the bid book by the descending one + let mut levels: Vec<_> = levels.into_iter().collect(); + if !base_is_ask { + levels.reverse(); + } + + if let Some(cursor_price) = cursor_price { + levels.retain(|(price, _amount)| { + if base_is_ask { + *price > cursor_price + } else { + *price < cursor_price + } + }); + } + + let has_next_page = levels.len() > offset_and_items.items as usize; + levels.truncate(offset_and_items.items as usize); + + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking should a + // future caller ever bypass it. When the storage cap truncated the book + // (`has_more_orders`), no cursor is minted: the levels in hand are an incomplete + // aggregation, and walking them would present a partial book as authoritative. + let next_cursor = (has_next_page && !levels.is_empty() && !has_more_orders).then(|| { + let (price, _amount) = levels.last().expect("the page is not empty"); + Cursor::new( + cursor_tag, + vec![price.numer().to_string(), price.denom().to_string()], + String::new(), + ) + }); + + let decimals = currency_decimals_for_pair(&db_tx, &base, "e, &state.chain_config).await?; + let base_decimals = decimals[&base]; + let quote_decimals = decimals["e]; + + let items = levels + .into_iter() + .map( + |(price, level_amount)| -> Result { + let (numer, denom) = (*price.numer(), *price.denom()); + // the price in atoms is the quote amount per one base atom; to express it + // in the currency units, scale the numerator by the base currency's + // decimals (and represent the result with the quote currency's decimals) + let price_decimal = + price_to_decimal_string(numer, denom, base_decimals, quote_decimals)?; + + Ok(json!({ + "price": { + "decimal": price_decimal, + "atoms": format!("{numer}/{denom}"), + }, + "amount": amount_to_json(Amount::from_atoms(level_amount), base_decimals), + })) + }, + ) + .collect::, ApiServerWebServerError>>()?; + + // the hard cap on aggregated live orders may have truncated the book; surface + // that to the client as an additive field so a partial book is never presented + // as complete + let response = if has_more_orders { + let mut response = cursor::paged_response(items, next_cursor); + response["truncated"] = json!(true); + response + } else { + cursor::paged_response(items, next_cursor) + }; + + Ok(Json(response)) +} + +/// The exact decimal representation of a price given as `numer/denom` quote atoms per +/// base atom: `numer * 10^base_decimals / denom`, displayed with `quote_decimals` +/// fractional digits, computed in exact u256 arithmetic (no floating point). +/// +/// The result is truncated (floored) toward zero, so it can be slightly lower than the +/// exact level price; the exact value is available alongside in the `atoms` field. +fn price_to_decimal_string( + numer: u128, + denom: u128, + base_decimals: u8, + quote_decimals: u8, +) -> Result { + // the price equals (numer * 10^base_decimals / denom) * 10^-quote_decimals; the + // digits are computed exactly in u256 arithmetic (the scaled numerator fits a + // u256 for any realistic decimal count and is checked anyway), because a + // financial price must not silently lose precision through floating point + // 10^base_decimals by exponentiation-by-squaring (`Uint256` has no pow); at most + // eight squarings for the maximum 255 decimals a token can declare + let overflow = || { + logging::log::error!("order book price scaling overflow"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + }; + let ten = Uint256::from_u64(10); + let (mut scale, mut base, mut exp) = (Uint256::from_u64(1), ten, u32::from(base_decimals)); + loop { + if exp & 1 == 1 { + scale = scale.checked_mul(&base).ok_or_else(overflow)?; + } + exp >>= 1; + if exp == 0 { + break; + } + base = base.checked_mul(&base).ok_or_else(overflow)?; + } + let scaled = Uint256::from(numer).checked_mul(&scale).ok_or_else(overflow)?; + let scaled = scaled.checked_div(&Uint256::from(denom)).expect("denominator is not zero"); + + // the decimal digits of the scaled price, least significant last, by repeated + // division by ten + let mut digits = Vec::new(); + let mut value = scaled; + while value != Uint256::ZERO { + let rem = value.checked_rem(&ten).expect("ten is not zero"); + let rem = u128::try_from(rem).expect("a digit fits a u128"); + digits.push(u8::try_from(rem).expect("a digit fits a u8") + b'0'); + value = value.checked_div(&ten).expect("ten is not zero"); + } + if digits.is_empty() { + digits.push(b'0'); + } + digits.reverse(); + let digits = String::from_utf8(digits).expect("digits are ascii"); + + // place the decimal point `quote_decimals` digits from the right + let point = usize::from(quote_decimals); + let price = if point == 0 { + digits + } else if digits.len() > point { + let (int_part, frac_part) = digits.split_at(digits.len() - point); + format!("{int_part}.{frac_part}") + } else { + format!("0.{}{digits}", "0".repeat(point - digits.len())) + }; + Ok(price) +} + +fn reduce_rational(rational: Rational) -> Option> { + let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); + // the caller skips zero-balance entries, so a zero denominator (a price that + // would compare equal to every other level and corrupt the ordering) must + // never reach this function; the debug assertion keeps the two invariants + // tied together, and the None return lets the caller surface the broken + // invariant instead of silently degrading in release builds + debug_assert_ne!(denom, 0); + if denom == 0 { + return None; + } + while denom != 0 { + (numer, denom) = (denom, numer % denom); + } + if numer == 0 { + // gcd(0, 0) == 0; only reachable when both the numerator and the denominator + // are zero, i.e. the same bypassed caller invariant as above + return None; + } + Some(Rational::new( + *rational.numer() / numer, + *rational.denom() / numer, + )) +} + +async fn currency_decimals_for_pair( + db_tx: &impl ApiServerStorageRead, + base: &CoinOrTokenId, + quote: &CoinOrTokenId, + chain_config: &ChainConfig, +) -> Result, ApiServerWebServerError> { + let mut decimals = BTreeMap::new(); + decimals.insert(CoinOrTokenId::Coin, chain_config.coin_decimals()); + + for currency in [base, quote] { + if let CoinOrTokenId::TokenId(token_id) = currency { + let token_decimals = db_tx + .get_token_num_decimals(*token_id) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + .ok_or(ApiServerWebServerError::NotFound( + ApiServerWebServerNotFoundError::TokenNotFound, + ))?; + decimals.insert(*currency, token_decimals); + } + } + + Ok(decimals) +} + struct OffsetAndItems { offset: u64, items: u32, @@ -2054,6 +2822,13 @@ fn get_offset_and_items( ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) })? .unwrap_or(DEFAULT_NUM_ITEMS); + // `items=0` is rejected here for every endpoint using this parser, including the + // legacy offset-mode listings which used to return an empty page; the behavior + // change is noted in the CHANGELOG + ensure!( + items > 0, + ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) + ); ensure!( items <= MAX_NUM_ITEMS, ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) diff --git a/api-server/web-server/src/error.rs b/api-server/web-server/src/error.rs index edf807cda2..ecac9def40 100644 --- a/api-server/web-server/src/error.rs +++ b/api-server/web-server/src/error.rs @@ -117,6 +117,8 @@ pub enum ApiServerWebServerClientError { InvalidOrderId, #[error("Invalid order trading pair")] InvalidOrderTradingPair, + #[error("Invalid cursor")] + InvalidCursor, } #[allow(dead_code)]