From e36cc984fce63e32b86841928c8a1890dd1f05e5 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 00:09:21 +0200 Subject: [PATCH 01/22] feat(indexer): supply counters, keyset pagination, holders, order book Supply statistics (closes the indexer part of #2129): - get_all_statistic now resolves each counter (circulating supply, staked, burned, preminted) with an indexed per-statistic latest lookup instead of a window scan over the whole statistics history, making /statistics/coin and /statistics/token/{id} O(1) per counter. The counters themselves were already maintained incrementally and reorg-safe per block height, so no new supply tables are needed; total minted remains derivable as circulating supply + burned. - storage version bumped to 27: existing deployments rescan from genesis, which rebuilds the new caches in the same pass. Keyset pagination for /pool and /transaction: - ml.latest_pool_data_cache holds the newest state of each pool (including its creation height), maintained by the scanner and rolled back on reorgs, so /pool no longer scans the whole per-block pool history; the pledge sort now orders by a numeric column instead of a textual byte comparison. - both endpoints accept an opaque URL-safe cursor (next_cursor field in the response, empty cursor = first page); the offset protocol keeps working. - an index on ml.transactions.tx_global_index backs the transaction cursor. Holders endpoints: - the scanner maintains ml.address_amount (latest balance per address and asset, numeric column, rolled back on reorgs). - new endpoints /statistics/coin/holders and /statistics/token/{id}/holders list holders by balance with cursor pagination. Order book endpoint: - the scanner maintains ml.latest_orders_cache (remaining ask/give balances per order, rolled back on reorgs; concluded orders drop out via zero balances, frozen orders are excluded from reads). - new endpoint /order/pair/{pair}/book?side=ask|bid aggregates remaining balances into price levels (remaining give/ask ratio) with cursor pagination; /order/pair/{pair} stays unchanged. --- Cargo.lock | 1 + .../src/storage/impls/in_memory/mod.rs | 89 +++ .../impls/in_memory/transactional/read.rs | 25 + .../impls/in_memory/transactional/write.rs | 25 + .../src/storage/impls/mod.rs | 2 +- .../src/storage/impls/postgres/queries.rs | 575 +++++++++++++++-- .../impls/postgres/transactional/read.rs | 30 + .../impls/postgres/transactional/write.rs | 30 + .../src/storage/storage_api/mod.rs | 34 + api-server/scanner-lib/src/sync/tests/mod.rs | 139 ++++- .../stack-test-suite/tests/v2/orders.rs | 244 ++++++++ api-server/stack-test-suite/tests/v2/pools.rs | 151 +++++ .../stack-test-suite/tests/v2/statistics.rs | 167 +++++ .../stack-test-suite/tests/v2/transactions.rs | 123 ++++ api-server/storage-test-suite/src/basic.rs | 434 +++++++++++++ api-server/web-server/Cargo.toml | 1 + api-server/web-server/src/api/cursor.rs | 80 +++ api-server/web-server/src/api/mod.rs | 1 + api-server/web-server/src/api/v2.rs | 580 ++++++++++++++++-- api-server/web-server/src/error.rs | 2 + 20 files changed, 2644 insertions(+), 89 deletions(-) create mode 100644 api-server/web-server/src/api/cursor.rs diff --git a/Cargo.lock b/Cargo.lock index 83d80726c4..2a9bd19b8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,7 @@ dependencies = [ "api-server-common", "async-trait", "axum 0.7.9", + "base64 0.22.1", "build_utils", "chainstate-test-framework", "clap", diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 9d59e68b36..2c811cbaef 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -133,6 +133,38 @@ impl ApiServerInMemoryStorage { ) } + fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + let mut holders = self + .address_balance_table + .iter() + .filter_map(|(address, by_coin_or_token)| { + by_coin_or_token + .get(&coin_or_token_id) + .and_then(|by_height| by_height.values().last()) + .filter(|amount| **amount != Amount::ZERO) + .map(|amount| (address.clone(), *amount)) + }) + .collect::>(); + + holders.sort_by(|(l_address, l_amount), (r_address, r_amount)| { + (r_amount, r_address.as_str()).cmp(&(l_amount, l_address.as_str())) + }); + + if let Some((cursor_amount, cursor_address)) = cursor { + holders.retain(|(address, amount)| { + *amount < cursor_amount + || (*amount == cursor_amount && address.as_str() < cursor_address.as_str()) + }); + } + + Ok(holders.into_iter().take(len as usize).collect::>()) + } + fn get_address_balances( &self, address: &str, @@ -518,6 +550,28 @@ impl ApiServerInMemoryStorage { Ok(latest_orders) } + fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + let entries = self + .orders_table + .values() + .filter_map(|by_height| by_height.values().last()) + .filter(|order| { + order.ask_currency == ask_currency + && order.give_currency == give_currency + && !order.is_frozen + && order.ask_balance != Amount::ZERO + && order.give_balance != Amount::ZERO + }) + .map(|order| (order.ask_balance, order.give_balance)) + .collect(); + + Ok(entries) + } + fn get_latest_pool_ids( &self, len: u32, @@ -549,6 +603,41 @@ impl ApiServerInMemoryStorage { Ok(latest_pools) } + fn get_latest_pool_ids_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let len = len as usize; + let mut pool_data: Vec<_> = self + .pool_data_table + .iter() + .map(|(pool_id, by_height)| { + let created_height = by_height.keys().next().expect("not empty"); + let latest_data = by_height.values().last().expect("not empty"); + (pool_id, (*created_height, latest_data)) + }) + .filter(|(_pool_id, data)| !data.1.is_decommissioned()) + .collect(); + + // newest first, ties broken by pool id so the keyset cursor is deterministic + pool_data.sort_by_key(|(pool_id, (height, _data))| Reverse((*height, **pool_id))); + pool_data.retain(|(pool_id, (height, _data))| match cursor { + Some((cursor_height, cursor_pool_id)) => { + *height < cursor_height || (*height == cursor_height && **pool_id < cursor_pool_id) + } + None => true, + }); + + let latest_pools = pool_data + .iter() + .take(len) + .map(|(pool_id, (height, data))| (*height, **pool_id, (*data).clone())) + .collect(); + + Ok(latest_pools) + } + fn get_pool_data_with_largest_staker_balance( &self, len: u32, diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index 546f7e1951..4deb1b524b 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -46,6 +46,15 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_address_balance(address, coin_or_token_id) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) + } + async fn get_address_balances( &self, address: &str, @@ -151,6 +160,14 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { self.transaction.get_latest_pool_ids(len, offset) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_latest_pool_ids_before(len, cursor) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -317,4 +334,12 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } + + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + self.transaction.get_order_book_entries(ask_currency, give_currency) + } } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index 5039ae013d..22fc166b2c 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -331,6 +331,15 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_address_balance(address, coin_or_token_id) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) + } + async fn get_address_balances( &self, address: &str, @@ -467,6 +476,14 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { self.transaction.get_latest_pool_ids(len, offset) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + self.transaction.get_latest_pool_ids_before(len, cursor) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -598,4 +615,12 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { ) -> Result, ApiServerStorageError> { self.transaction.get_orders_for_trading_pair(pair, len, offset) } + + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + self.transaction.get_order_book_entries(ask_currency, give_currency) + } } diff --git a/api-server/api-server-common/src/storage/impls/mod.rs b/api-server/api-server-common/src/storage/impls/mod.rs index c224d5b546..3e91df11ec 100644 --- a/api-server/api-server-common/src/storage/impls/mod.rs +++ b/api-server/api-server-common/src/storage/impls/mod.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub const CURRENT_STORAGE_VERSION: u32 = 26; +pub const CURRENT_STORAGE_VERSION: u32 = 27; pub mod in_memory; pub mod postgres; diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index ee6e3b8d5e..3a77ebe772 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -183,6 +183,65 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) } + /// Top holders of an asset, ordered by the balance (descending, ties broken by the + /// address); keyset (cursor) pagination, the cursor is the (amount, address) of the + /// last returned holder, exclusive. Addresses with a zero balance are excluded. + pub async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + let coin_or_token_id = coin_or_token_id.encode(); + let len = len as i64; + let rows = match cursor { + Some((amount, address)) => { + let amount_str = amount.into_atoms().to_string(); + self.tx + .query( + r#" + SELECT address, amount::text + FROM ml.address_amount + WHERE coin_or_token_id = $1 AND amount != 0 + AND (amount, address) < ($2::text::numeric, $3) + ORDER BY amount DESC, address DESC + LIMIT $4; + "#, + &[&coin_or_token_id, &amount_str, &address, &len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } + None => self + .tx + .query( + r#" + SELECT address, amount::text + FROM ml.address_amount + WHERE coin_or_token_id = $1 AND amount != 0 + ORDER BY amount DESC, address DESC + LIMIT $2; + "#, + &[&coin_or_token_id, &len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?, + }; + + rows.into_iter() + .map(|row| -> Result<(String, Amount), ApiServerStorageError> { + let address: String = row.get(0); + let amount_str: String = row.get(1); + let atoms = amount_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid amount {amount_str}: {e}" + )) + })?; + Ok((address, Amount::from_atoms(atoms))) + }) + .collect() + } + pub async fn get_address_balances( &self, address: &str, @@ -319,6 +378,68 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // roll back the holders table rows of the disconnected blocks; the numeric + // amount is derived from the versioned balances, hence the row-by-row handling + let deleted_holders_rows = self + .tx + .query( + "DELETE FROM ml.address_amount WHERE block_height > $1 RETURNING address, coin_or_token_id;", + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + for row in deleted_holders_rows { + let address: String = row.get(0); + let coin_or_token_id: Vec = row.get(1); + + let latest_versioned = self + .tx + .query_opt( + "SELECT block_height, amount FROM ml.address_balance WHERE address = $1 AND coin_or_token_id = $2 ORDER BY block_height DESC LIMIT 1;", + &[&address, &coin_or_token_id], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + match latest_versioned { + Some(latest) => { + let block_height: i64 = latest.get(0); + let amount: Vec = latest.get(1); + let amount = Amount::decode_all(&mut amount.as_slice()).map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Amount deserialization failed: {e}" + )) + })?; + let amount_str = amount.into_atoms().to_string(); + + self.tx + .execute( + r#" + INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) + VALUES ($1, $2, $3, $4::text::numeric) + ON CONFLICT (address, coin_or_token_id) DO UPDATE + SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount + WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + "#, + &[&address, &coin_or_token_id, &block_height, &amount_str], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + } + None => { + // the whole history of the address and the asset was orphaned + self.tx + .execute( + "DELETE FROM ml.address_amount WHERE address = $1 AND coin_or_token_id = $2;", + &[&address, &coin_or_token_id], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + } + } + } + Ok(()) } @@ -380,6 +501,23 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // the holders table tracks the same latest balance, but with the amount in the + // NUMERIC form so the holders listing can order the addresses by it + let amount_str = amount.into_atoms().to_string(); + self.tx + .execute( + r#" + INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) + VALUES ($1, $2, $3, $4::text::numeric) + ON CONFLICT (address, coin_or_token_id) DO UPDATE + SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount + WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + "#, + &[&address.to_string(), &coin_or_token_id.encode(), &height, &amount_str], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + let CoinOrTokenId::TokenId(token_id) = coin_or_token_id else { return Ok(()); }; @@ -778,6 +916,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; + // index for the keyset (cursor) pagination of the transaction listing + self.just_execute( + "CREATE INDEX transactions_tx_global_index ON ml.transactions (tx_global_index DESC);", + ) + .await?; + self.just_execute( "CREATE TABLE ml.address_balance ( address TEXT NOT NULL, @@ -1051,6 +1195,100 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; + // Latest state of every pool; maintained alongside ml.pool_data by the scanner so + // that the pool listings do not have to derive the newest state of each pool from + // the full versioned history (which is touched at every PoS block). + self.just_execute( + "CREATE TABLE ml.latest_pool_data_cache ( + pool_id TEXT NOT NULL, + block_height bigint NOT NULL, + creation_block_height bigint NOT NULL, + staker_balance NUMERIC NOT NULL, + data bytea NOT NULL, + PRIMARY KEY (pool_id) + );", + ) + .await?; + + // index for the keyset pagination of the pool listing (by creation height) + self.just_execute( + "CREATE INDEX latest_pool_data_created_index + ON ml.latest_pool_data_cache (creation_block_height DESC, pool_id DESC);", + ) + .await?; + + // index for the pool listing sorted by pledge + self.just_execute( + "CREATE INDEX latest_pool_data_staker_balance_index + ON ml.latest_pool_data_cache (staker_balance DESC);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX latest_pool_data_block_height_index + ON ml.latest_pool_data_cache (block_height DESC);", + ) + .await?; + + // Latest balance of every (address, asset) pair as an orderable numeric amount; + // maintained alongside ml.address_balance by the scanner so that the top holders + // listing can be served by an index scan instead of decoding every balance. + self.just_execute( + "CREATE TABLE ml.address_amount ( + address TEXT NOT NULL, + coin_or_token_id bytea NOT NULL, + block_height bigint NOT NULL, + amount NUMERIC NOT NULL, + PRIMARY KEY (address, coin_or_token_id) + );", + ) + .await?; + + // index for the keyset pagination of the holders listing + self.just_execute( + "CREATE INDEX address_amount_top_holders_index + ON ml.address_amount (coin_or_token_id, amount DESC, address DESC);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX address_amount_block_height ON ml.address_amount (block_height DESC);", + ) + .await?; + + // Latest state of every order; maintained alongside ml.orders by the scanner so + // that the order book does not have to derive the newest state of each order from + // the full versioned history. + self.just_execute( + "CREATE TABLE ml.latest_orders_cache ( + order_id TEXT NOT NULL, + block_height bigint NOT NULL, + creation_block_height bigint NOT NULL, + ask_currency bytea NOT NULL, + ask_balance NUMERIC NOT NULL, + give_currency bytea NOT NULL, + give_balance NUMERIC NOT NULL, + frozen BOOLEAN NOT NULL, + PRIMARY KEY (order_id) + );", + ) + .await?; + + // index for the order book pair lookups + self.just_execute( + "CREATE INDEX latest_orders_currencies_index + ON ml.latest_orders_cache (ask_currency, give_currency);", + ) + .await?; + + // index for reorgs + self.just_execute( + "CREATE INDEX latest_orders_block_height_index ON ml.latest_orders_cache (block_height DESC);", + ) + .await?; + // Append-only log of stream events for real-time clients; the primary key index also // serves the `WHERE id > $1 ORDER BY id` reads of the event pump. self.just_execute( @@ -1574,6 +1812,56 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // delete and re-derive the pool rows of the latest-state cache from the versioned + // table (a deleted row whose pool still exists below the disconnected height must + // not be lost) + self.tx + .execute( + r#" + WITH deleted_cache_rows AS ( + DELETE FROM ml.latest_pool_data_cache + WHERE block_height > $1 + RETURNING pool_id + ), + latest_pool_data_for_deleted AS ( + SELECT + dcr.pool_id, + ld.block_height, + created.creation_block_height, + ld.staker_balance, + ld.data + FROM deleted_cache_rows dcr + CROSS JOIN LATERAL ( + SELECT + pool_id, + block_height, + staker_balance, + data + FROM ml.pool_data d_inner + WHERE d_inner.pool_id = dcr.pool_id + ORDER BY block_height DESC + LIMIT 1 + ) AS ld + CROSS JOIN LATERAL ( + SELECT MIN(p_min.block_height) AS creation_block_height + FROM ml.pool_data p_min + WHERE p_min.pool_id = dcr.pool_id + ) AS created + ) + INSERT INTO ml.latest_pool_data_cache (pool_id, block_height, creation_block_height, staker_balance, data) + SELECT + pool_id, + block_height, + creation_block_height, + staker_balance::numeric, + data + FROM latest_pool_data_for_deleted; + "#, + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -1709,6 +1997,32 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) } + fn decode_pool_id_and_data_rows( + rows: Vec, + chain_config: &ChainConfig, + ) -> Result, ApiServerStorageError> { + rows.into_iter() + .map( + |row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { + let pool_id: String = row.get(0); + let pool_id = Address::::from_string(chain_config, pool_id) + .map_err(|_| ApiServerStorageError::AddressableError)? + .into_object(); + let pool_data: Vec = row.get(1); + let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Pool data deserialization failed: {}", + e + )) + })?; + + Ok((pool_id, pool_data)) + }, + ) + .collect() + } + pub async fn get_latest_pool_data( &self, len: u32, @@ -1717,30 +2031,79 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; - self.tx + let rows = self + .tx .query( r#" - SELECT sub.pool_id, data - FROM ( - SELECT pool_id, data, staker_balance, block_height, ROW_NUMBER() OVER(PARTITION BY pool_id ORDER BY block_height DESC) as newest - FROM ml.pool_data - ) AS sub INNER JOIN (SELECT pool_id, MIN(block_height) AS created_height FROM ml.pool_data GROUP BY pool_id) as created ON sub.pool_id = created.pool_id - WHERE newest = 1 AND staker_balance::NUMERIC != 0 - ORDER BY created_height DESC + SELECT pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + ORDER BY creation_block_height DESC, pool_id DESC OFFSET $1 LIMIT $2; "#, &[&offset, &len], ) .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? - .into_iter() - .map(|row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { - let pool_id: String = row.get(0); + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + Self::decode_pool_id_and_data_rows(rows, chain_config) + } + + /// Keyset (cursor) pagination of the pool listing by creation height; the cursor is + /// the (creation height, pool id) of the last returned pool, exclusive. + pub async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + chain_config: &ChainConfig, + ) -> Result, ApiServerStorageError> { + let len = len as i64; + let rows = match cursor { + Some((creation_height, pool_id)) => { + let creation_height = Self::block_height_to_postgres_friendly(creation_height); + let pool_id = Address::new(chain_config, pool_id) + .map_err(|_| ApiServerStorageError::AddressableError)?; + self.tx + .query( + r#" + SELECT creation_block_height, pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + AND (creation_block_height, pool_id) < ($1, $2) + ORDER BY creation_block_height DESC, pool_id DESC + LIMIT $3; + "#, + &[&creation_height, &pool_id.as_str(), &len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } + None => self + .tx + .query( + r#" + SELECT creation_block_height, pool_id, data + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 + ORDER BY creation_block_height DESC, pool_id DESC + LIMIT $1; + "#, + &[&len], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?, + }; + + rows.into_iter() + .map(|row| -> Result<(BlockHeight, PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { + let creation_block_height: i64 = row.get(0); + let creation_block_height = BlockHeight::new(creation_block_height as u64); + let pool_id: String = row.get(1); let pool_id = Address::::from_string(chain_config, pool_id) .map_err(|_| ApiServerStorageError::AddressableError)? .into_object(); - let pool_data: Vec = row.get(1); + let pool_data: Vec = row.get(2); let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()).map_err(|e| { ApiServerStorageError::DeserializationError(format!( "Pool data deserialization failed: {}", @@ -1748,7 +2111,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { )) })?; - Ok((pool_id, pool_data)) + Ok((creation_block_height, pool_id, pool_data)) }) .collect() } @@ -1761,15 +2124,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result, ApiServerStorageError> { let len = len as i64; let offset = offset as i64; - self.tx + let rows = self + .tx .query( r#" SELECT pool_id, data - FROM ( - SELECT pool_id, data, staker_balance, ROW_NUMBER() OVER(PARTITION BY pool_id ORDER BY block_height DESC) as newest - FROM ml.pool_data - ) AS sub - WHERE newest = 1 AND staker_balance::NUMERIC != 0 + FROM ml.latest_pool_data_cache + WHERE staker_balance != 0 ORDER BY staker_balance DESC OFFSET $1 LIMIT $2; @@ -1777,24 +2138,9 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &[&offset, &len], ) .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? - .into_iter() - .map(|row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { - let pool_id: String = row.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? - .into_object(); - let pool_data: Vec = row.get(1); - let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()).map_err(|e| { - ApiServerStorageError::DeserializationError(format!( - "Pool data deserialization failed: {}", - e - )) - })?; + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - Ok((pool_id, pool_data)) - }) - .collect() + Self::decode_pool_id_and_data_rows(rows, chain_config) } pub async fn set_pool_data_at_height( @@ -1822,6 +2168,25 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // maintain the latest-state cache used by the pool listings; the creation height + // is the height of the first write of the pool (the scanner writes pools from + // genesis, so the first write is the creation) and is not modified on conflict. + self.tx + .execute( + r#" + INSERT INTO ml.latest_pool_data_cache (pool_id, block_height, creation_block_height, staker_balance, data) + VALUES ($1, $2, $2, $3::text::numeric, $4) + ON CONFLICT (pool_id) DO UPDATE + SET block_height = EXCLUDED.block_height, + staker_balance = EXCLUDED.staker_balance, + data = EXCLUDED.data + WHERE ml.latest_pool_data_cache.block_height <= EXCLUDED.block_height; + "#, + &[&pool_id.as_str(), &height, &amount_str, &pool_data.encode()], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -2650,19 +3015,32 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &self, coin_or_token_id: CoinOrTokenId, ) -> Result, ApiServerStorageError> { + // Note: the newest value of each statistic is read with one indexed lookup per + // statistic instead of a `ROW_NUMBER()` window over the whole table: the primary + // key is (statistic, coin_or_token_id, block_height), so a lookup that filters by + // the coin or token id alone cannot use the key prefix and degrades into a scan of + // every historical row of the asset (the circulating supply and the staked amount + // are touched at every PoS block, so there can be millions of such rows). let rows = self .tx .query( r#" - SELECT sub.statistic, sub.amount + SELECT statistic, amount FROM ( - SELECT statistic, amount, ROW_NUMBER() OVER(PARTITION BY statistic ORDER BY block_height DESC) as newest + SELECT DISTINCT ON (statistic) + statistic, amount FROM ml.statistics - WHERE coin_or_token_id = $1 - ) AS sub - WHERE newest = 1; + WHERE statistic = ANY($1) AND coin_or_token_id = $2 + ORDER BY statistic, block_height DESC + ) AS sub; "#, - &[&coin_or_token_id.encode()], + &[ + &CoinOrTokenStatistic::VARIANTS + .iter() + .map(|variant| variant.to_string()) + .collect::>(), + &coin_or_token_id.encode(), + ], ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; @@ -2982,6 +3360,42 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // maintain the latest-orders cache as a mirror of the latest versioned state; + // concluded orders (both balances zeroed) are stored as well, they are filtered + // out by the order book reads + let ask_balance = order.ask_balance.into_atoms().to_string(); + let give_balance = order.give_balance.into_atoms().to_string(); + + self.tx + .execute( + r#" + INSERT INTO ml.latest_orders_cache + (order_id, block_height, creation_block_height, ask_currency, ask_balance, give_currency, give_balance, frozen) + VALUES ($1, $2, $3, $4, $5::text::numeric, $6, $7::text::numeric, $8) + ON CONFLICT (order_id) DO UPDATE + SET block_height = EXCLUDED.block_height, + creation_block_height = EXCLUDED.creation_block_height, + ask_currency = EXCLUDED.ask_currency, + ask_balance = EXCLUDED.ask_balance, + give_currency = EXCLUDED.give_currency, + give_balance = EXCLUDED.give_balance, + frozen = EXCLUDED.frozen + WHERE ml.latest_orders_cache.block_height <= EXCLUDED.block_height; + "#, + &[ + &order_id.as_str(), + &height, + &creation_block_height, + &order.ask_currency.encode(), + &ask_balance, + &order.give_currency.encode(), + &give_balance, + &order.is_frozen, + ], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } @@ -2996,9 +3410,84 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // roll back the latest-orders cache rows of the disconnected blocks; the cache is + // a mirror of the latest versioned state (concluded orders included, filtered by + // the order book reads), so the surviving versioned history of every order with a + // deleted cache row fully determines its restored state + self.tx + .execute( + r#" + WITH deleted_cache_rows AS ( + DELETE FROM ml.latest_orders_cache WHERE block_height > $1 RETURNING order_id + ) + INSERT INTO ml.latest_orders_cache + (order_id, block_height, creation_block_height, ask_currency, ask_balance, give_currency, give_balance, frozen) + SELECT + o.order_id, o.block_height, o.creation_block_height, o.ask_currency, + o.ask_balance::numeric, o.give_currency, o.give_balance::numeric, o.frozen + FROM deleted_cache_rows d + CROSS JOIN LATERAL ( + SELECT * FROM ml.orders + WHERE order_id = d.order_id + ORDER BY block_height DESC + LIMIT 1 + ) o; + "#, + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) } + /// Remaining balances of all the live (not concluded, not frozen) orders of a + /// trading pair, as (ask_balance, give_balance) pairs; the price of each order is + /// derived from these in the web layer. + pub async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + let ask_currency = ask_currency.encode(); + let give_currency = give_currency.encode(); + + let rows = self + .tx + .query( + r#" + SELECT ask_balance::text, give_balance::text + FROM ml.latest_orders_cache + WHERE ask_currency = $1 AND give_currency = $2 + AND frozen = FALSE + AND ask_balance != 0 AND give_balance != 0; + "#, + &[&ask_currency, &give_currency], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + + rows.into_iter() + .map(|row| -> Result<(Amount, Amount), ApiServerStorageError> { + let ask_str: String = row.get(0); + let give_str: String = row.get(1); + + let ask = ask_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid ask balance {ask_str}: {e}" + )) + })?; + let give = give_str.parse::().map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid give balance {give_str}: {e}" + )) + })?; + + Ok((Amount::from_atoms(ask), Amount::from_atoms(give))) + }) + .collect() + } + pub async fn get_orders_by_height( &self, len: u32, diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 6fa3514681..15a835eed7 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -64,6 +64,16 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_top_address_amounts(coin_or_token_id, len, cursor).await + } + async fn get_address_balances( &self, address: &str, @@ -262,6 +272,17 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_latest_pool_data_before(len, cursor, &self.chain_config).await?; + + Ok(res) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -450,6 +471,15 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { Ok(res) } + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_order_book_entries(ask_currency, give_currency).await + } + async fn read_stream_events_after( &self, last_seen_id: StreamEventId, diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index c9a53f3c28..23cf339db5 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -465,6 +465,16 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_top_address_amounts(coin_or_token_id, len, cursor).await + } + async fn get_address_balances( &self, address: &str, @@ -638,6 +648,17 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + let res = conn.get_latest_pool_data_before(len, cursor, &self.chain_config).await?; + + Ok(res) + } + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -847,6 +868,15 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { Ok(res) } + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError> { + let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); + conn.get_order_book_entries(ask_currency, give_currency).await + } + async fn read_stream_events_after( &self, last_seen_id: StreamEventId, diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 05863da479..b10f13a508 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -83,6 +83,11 @@ pub enum CoinOrTokenStatistic { Preminted, } +impl CoinOrTokenStatistic { + pub const VARIANTS: [Self; 4] = + [Self::CirculatingSupply, Self::Staked, Self::Burned, Self::Preminted]; +} + impl FromStr for CoinOrTokenStatistic { type Err = ApiServerStorageError; @@ -602,6 +607,17 @@ pub trait ApiServerStorageRead: Sync { coin_or_token_id: CoinOrTokenId, ) -> Result, ApiServerStorageError>; + /// Returns up to `len` holders of the asset with the largest balances, ordered by + /// the balance (descending, ties broken by the address descending); keyset (cursor) + /// pagination, the cursor is the (amount, address) of the last returned holder, + /// exclusive. Addresses with a zero balance are excluded. + async fn get_top_address_amounts( + &self, + coin_or_token_id: CoinOrTokenId, + len: u32, + cursor: Option<(Amount, String)>, + ) -> Result, ApiServerStorageError>; + async fn get_address_balances( &self, address: &str, @@ -686,6 +702,15 @@ pub trait ApiServerStorageRead: Sync { offset: u64, ) -> Result, ApiServerStorageError>; + /// Keyset (cursor) pagination of the pool listing by creation height (descending); + /// the cursor is the (creation height, pool id) of the last returned pool, exclusive. + /// Returns the creation height of each pool alongside its id and data. + async fn get_latest_pool_data_before( + &self, + len: u32, + cursor: Option<(BlockHeight, PoolId)>, + ) -> Result, ApiServerStorageError>; + async fn get_pool_data_with_largest_staker_balance( &self, len: u32, @@ -802,6 +827,15 @@ pub trait ApiServerStorageRead: Sync { offset: u64, ) -> Result, ApiServerStorageError>; + /// Remaining balances of all the live (not concluded, not frozen) orders of a + /// trading pair, as (ask_balance, give_balance) pairs; the price of each order is + /// derived from these in the web layer. + async fn get_order_book_entries( + &self, + ask_currency: CoinOrTokenId, + give_currency: CoinOrTokenId, + ) -> Result, ApiServerStorageError>; + /// Read the stream events with an id greater than `last_seen_id`, in ascending id order. /// /// Note: backends that don't support stream events simply return an empty list. diff --git a/api-server/scanner-lib/src/sync/tests/mod.rs b/api-server/scanner-lib/src/sync/tests/mod.rs index 63aa80cb15..8b0117d088 100644 --- a/api-server/scanner-lib/src/sync/tests/mod.rs +++ b/api-server/scanner-lib/src/sync/tests/mod.rs @@ -29,7 +29,8 @@ use tokio::sync::mpsc; use api_server_common::storage::{ impls::in_memory::transactional::TransactionalApiServerInMemoryStorage, storage_api::{ - ApiServerStorageRead, ApiServerStorageWrite, ApiServerTransactionRw, Transactional, + ApiServerStorageRead, ApiServerStorageWrite, ApiServerTransactionRw, CoinOrTokenStatistic, + Transactional, }, }; use chainstate::{BlockSource, ChainInfo}; @@ -1037,6 +1038,142 @@ async fn reorg_locked_balance(#[case] seed: Seed) { drop(db_tx); } +// `get_all_statistic` omits the statistics that were never written; the unwritten ones +// are treated as zero here +fn statistic_value( + stats: &BTreeMap, + statistic: CoinOrTokenStatistic, +) -> Amount { + stats.get(&statistic).copied().unwrap_or(Amount::ZERO) +} + +#[rstest] +#[trace] +#[case(test_utils::random::Seed::from_entropy())] +#[tokio::test] +async fn reorg_statistics(#[case] seed: Seed) { + let mut rng = make_seedable_rng(seed); + + let mut tf = TestFramework::builder(&mut rng).build(); + + let chain_config = Arc::clone(tf.chainstate.get_chain_config()); + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + let mut local_state = BlockchainState::new(chain_config.clone(), storage); + local_state.scan_genesis(chain_config.genesis_block().as_ref()).await.unwrap(); + + let target_block_time = chain_config.target_block_spacing(); + + let genesis_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + let genesis_burned = statistic_value(&genesis_stats, CoinOrTokenStatistic::Burned); + let genesis_circulating_supply = + statistic_value(&genesis_stats, CoinOrTokenStatistic::CirculatingSupply); + + // a block with a coin burn, which increases the burned supply and decreases the + // circulating supply + let burned_amount = Amount::from_atoms(5); + let transaction = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::BlockReward(chain_config.genesis_block_id()), + 0, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::Burn(OutputValue::Coin(burned_amount))) + .build(); + + tf.progress_time_seconds_since_epoch(target_block_time.as_secs()); + let block = tf + .make_block_builder() + .with_parent(chain_config.genesis_block_id()) + .with_transactions(vec![transaction]) + .build(&mut rng); + + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + let block_height = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_best_block() + .await + .unwrap() + .block_height(); + local_state.scan_blocks(block_height, vec![block.clone()]).await.unwrap(); + + let post_burn_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&post_burn_stats, CoinOrTokenStatistic::Burned), + (genesis_burned + burned_amount).expect("no overflow in test amounts") + ); + assert_eq!( + statistic_value(&post_burn_stats, CoinOrTokenStatistic::CirculatingSupply), + (genesis_circulating_supply - burned_amount).expect("no underflow in test amounts") + ); + + // reorg away the block; the statistics must return to the genesis values + local_state.scan_blocks(block_height, vec![]).await.unwrap(); + + let rolled_back_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&rolled_back_stats, CoinOrTokenStatistic::Burned), + genesis_burned + ); + assert_eq!( + statistic_value(&rolled_back_stats, CoinOrTokenStatistic::CirculatingSupply), + genesis_circulating_supply + ); + + // scan the block again; the statistics must be restored + local_state.scan_blocks(block_height, vec![block]).await.unwrap(); + + let restored_stats = local_state + .storage() + .transaction_ro() + .await + .unwrap() + .get_all_statistic(CoinOrTokenId::Coin) + .await + .unwrap(); + assert_eq!( + statistic_value(&restored_stats, CoinOrTokenStatistic::Burned), + statistic_value(&post_burn_stats, CoinOrTokenStatistic::Burned) + ); + assert_eq!( + statistic_value(&restored_stats, CoinOrTokenStatistic::CirculatingSupply), + statistic_value(&post_burn_stats, CoinOrTokenStatistic::CirculatingSupply) + ); +} + #[allow(clippy::too_many_arguments)] fn create_block( rng: &mut impl CryptoRng, diff --git a/api-server/stack-test-suite/tests/v2/orders.rs b/api-server/stack-test-suite/tests/v2/orders.rs index ddcaff70be..3d805783a7 100644 --- a/api-server/stack-test-suite/tests/v2/orders.rs +++ b/api-server/stack-test-suite/tests/v2/orders.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use api_web_server::api::json_helpers::amount_to_json; use common::chain::{ AccountCommand, AccountNonce, ChainstateUpgradeBuilder, OrderAccountCommand, OrderData, OrdersVersion, make_order_id, @@ -200,6 +201,249 @@ async fn create_fill_conclude_order(#[case] seed: Seed, #[case] version: OrdersV shutdown_task(task).await; } +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn order_pair_book(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + // Issue and mint some tokens to create orders of both sides of the + // coin/token pair + let tokens_amount = Amount::from_atoms(1000); + let issue_and_mint_result = + helpers::issue_and_mint_tokens_from_genesis(tokens_amount, &mut rng, &mut tf); + let token_id = issue_and_mint_result.token_id; + let minted = issue_and_mint_result.minted_tokens; + + let ask_coin = Amount::from_atoms(10); + + // asks 10 coins for 10 tokens (a price of 1 token per coin) + let tx_1 = TransactionBuilder::new() + .add_input( + TxInput::Utxo(issue_and_mint_result.tokens_outpoint), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::Coin(ask_coin), + OutputValue::TokenV1(token_id, ask_coin), + )))) + .add_output(TxOutput::Transfer( + OutputValue::TokenV1(token_id, (minted - ask_coin).unwrap()), + Destination::AnyoneCanSpend, + )) + .build(); + let block1 = tf.make_block_builder().add_transaction(tx_1.clone()).build(&mut rng); + tf.process_block(block1.clone(), BlockSource::Local).unwrap(); + + // asks 10 coins for 30 tokens (a price of 3 tokens per coin) + let tx_2 = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::Transaction(tx_1.transaction().get_id()), + 1, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::Coin(ask_coin), + OutputValue::TokenV1(token_id, Amount::from_atoms(30)), + )))) + .add_output(TxOutput::Transfer( + OutputValue::TokenV1( + token_id, + ((minted - ask_coin).unwrap() - Amount::from_atoms(30)).unwrap(), + ), + Destination::AnyoneCanSpend, + )) + .build(); + let block2 = tf.make_block_builder().add_transaction(tx_2).build(&mut rng); + tf.process_block(block2.clone(), BlockSource::Local).unwrap(); + + // a bid of the coin/token pair: asks 5 tokens for 5 coins + let tx_3 = TransactionBuilder::new() + .add_input( + TxInput::Utxo(issue_and_mint_result.change_outpoint.clone()), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::CreateOrder(Box::new(OrderData::new( + Destination::AnyoneCanSpend, + OutputValue::TokenV1(token_id, Amount::from_atoms(5)), + OutputValue::Coin(Amount::from_atoms(5)), + )))) + .add_output(TxOutput::Transfer( + OutputValue::Coin( + (chainstate_test_framework::get_output_value( + tf.chainstate + .utxo(&issue_and_mint_result.change_outpoint) + .unwrap() + .unwrap() + .output(), + ) + .unwrap() + .coin_amount() + .unwrap() + - Amount::from_atoms(5)) + .unwrap(), + ), + Destination::AnyoneCanSpend, + )) + .build(); + let block3 = tf.make_block_builder().add_transaction(tx_3).build(&mut rng); + tf.process_block(block3.clone(), BlockSource::Local).unwrap(); + + _ = tx.send(( + chain_config.coin_ticker().to_owned(), + Address::new(&chain_config, token_id).unwrap().into_string(), + chain_config.coin_decimals(), + )); + + vec![ + issue_and_mint_result.issue_block, + issue_and_mint_result.mint_block, + block1, + block2, + block3, + ] + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await + }); + + let (ml, tkn, coin_decimals) = rx.await.unwrap(); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the ask book of ML_TKN, paginated by one level per request + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&items=1"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_some()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(10), coin_decimals) + ); + + // the next page + let cursor = body["next_cursor"].as_str().unwrap(); + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&items=1&cursor={cursor}"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_none()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "3/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(10), coin_decimals) + ); + } + + // the bid book of ML_TKN has the single reversed order + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=bid"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(body["next_cursor"].as_str().is_none()); + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + assert_eq!( + items[0]["amount"], + amount_to_json(Amount::from_atoms(5), coin_decimals) + ); + } + + // the reversed pair sees the bid order as its ask + { + let url = format!("/api/v2/order/pair/{tkn}_{ml}/book?side=ask"); + let (status, body) = get_json(url).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["price"]["atoms"], "1/1"); + } + + // an invalid side is rejected + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=nonsense"); + let (status, body) = get_json(url).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let url = format!("/api/v2/order/pair/{ml}_{tkn}/book?side=ask&cursor=garbage"); + let (status, body) = get_json(url).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + shutdown_task(task).await; +} + #[rstest] #[trace] #[case(Seed::from_entropy())] diff --git a/api-server/stack-test-suite/tests/v2/pools.rs b/api-server/stack-test-suite/tests/v2/pools.rs index 6fc5c37ad0..2ce217df4a 100644 --- a/api-server/stack-test-suite/tests/v2/pools.rs +++ b/api-server/stack-test-suite/tests/v2/pools.rs @@ -336,3 +336,154 @@ async fn ok(#[case] seed: Seed) { shutdown_task(task).await; } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn pools_cursor_pagination(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let mut stake_pool_outpoint = UtxoOutPoint::new( + OutPointSourceId::BlockReward(tf.genesis().get_id().into()), + 0, + ); + let mut available_amount = ((chain_config.min_stake_pool_pledge() * 10).unwrap() + + Amount::from_atoms(10000)) + .unwrap(); + + let mut pool_ids = vec![]; + let mut blocks = vec![]; + for _ in 0..3 { + let (transfer_outpoint, _stake_pool_data, pool_id, block) = prepare_stake_pool( + stake_pool_outpoint, + &mut rng, + &mut available_amount, + &mut tf, + ); + stake_pool_outpoint = transfer_outpoint; + pool_ids.push(pool_id); + blocks.push(block); + } + + let pool_ids = pool_ids + .iter() + .map(|pool_id| Address::new(&chain_config, *pool_id).unwrap().into_string()) + .collect::>(); + _ = tx.send(pool_ids); + + blocks + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await + }); + + let created_pools = rx.await.unwrap(); + let expected_order = created_pools.iter().rev().collect::>(); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the whole list, unpaged + { + let (status, body) = get_json("/api/v2/pool?offset=0&items=100".to_owned()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr = body.as_array().unwrap(); + assert_eq!(arr.len(), 3); + for (expected_pool_id, json) in expected_order.iter().zip(arr) { + assert_eq!( + json.get("pool_id").unwrap().as_str().unwrap(), + *expected_pool_id + ); + } + } + + // the same list, walked with the cursor + { + let mut url = "/api/v2/pool?items=1&cursor=".to_owned(); + let mut paged_order = vec![]; + loop { + let (status, body) = get_json(url.clone()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + paged_order.push(items[0]["pool_id"].as_str().unwrap().to_owned()); + + match body["next_cursor"].as_str() { + Some(cursor) => url = format!("/api/v2/pool?items=1&cursor={cursor}"), + None => break, + } + } + assert_eq!(paged_order.len(), 3); + assert_eq!( + paged_order, + expected_order.iter().map(|s| s.as_str()).collect::>() + ); + } + + // a cursor is only supported by the default sort order + { + let (status, body) = get_json("/api/v2/pool?sort=by_pledge&cursor=xyz".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let (status, body) = get_json("/api/v2/pool?cursor=garbage".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + shutdown_task(task).await; +} diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 9df59e8d4e..31c48c4e5e 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -441,3 +441,170 @@ async fn ok_coins(#[case] seed: Seed) { shutdown_task(task).await; } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn coin_holders(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let (_, alice_pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let (_, bob_pk) = PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr); + let alice_destination = Destination::PublicKeyHash(PublicKeyHash::from(&alice_pk)); + let bob_destination = Destination::PublicKeyHash(PublicKeyHash::from(&bob_pk)); + + let transaction = TransactionBuilder::new() + .add_input( + TxInput::from_utxo( + OutPointSourceId::BlockReward(tf.genesis().get_id().into()), + 0, + ), + InputWitness::NoSignature(None), + ) + .add_output(TxOutput::Transfer( + OutputValue::Coin(Amount::from_atoms(100)), + alice_destination.clone(), + )) + .add_output(TxOutput::Transfer( + OutputValue::Coin(Amount::from_atoms(50)), + bob_destination.clone(), + )) + .build(); + + let block = tf.make_block_builder().add_transaction(transaction).build(&mut rng); + tf.process_block(block.clone(), BlockSource::Local).unwrap(); + + let alice_address = Address::new(&chain_config, alice_destination).unwrap(); + let bob_address = Address::new(&chain_config, bob_destination).unwrap(); + _ = tx.send(( + alice_address.clone(), + bob_address.clone(), + chain_config.coin_decimals(), + )); + + (alice_address, bob_address, vec![block]) + }; + let (_alice_address, _bob_address, chainstate_blocks) = chainstate_blocks; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, false).await + }); + + let (alice_address, bob_address, coin_decimals) = rx.await.unwrap(); + let alice_atoms = amount_to_json(Amount::from_atoms(100), coin_decimals); + let bob_atoms = amount_to_json(Amount::from_atoms(50), coin_decimals); + + // walk the whole holders list, one item per request + let mut url = "/api/v2/statistics/coin/holders?items=1".to_owned(); + let mut holders = BTreeMap::::new(); + let mut prev_amount: Option = None; + loop { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + assert_eq!(response.status(), 200, "Failed getting coin holders"); + + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + + let address = items[0]["address"].as_str().unwrap().to_owned(); + assert!(!holders.contains_key(&address), "duplicate holder address"); + let amount = items[0]["amount"]["atoms"].as_str().unwrap().to_owned(); + + // the amounts are ordered, from the biggest to the smallest + if let Some(prev) = prev_amount.replace(amount.clone()) { + let (prev, amount) = ( + prev.parse::().unwrap(), + amount.parse::().unwrap(), + ); + assert!(prev >= amount); + } + holders.insert(address, items[0]["amount"].clone()); + + match body["next_cursor"].as_str() { + Some(cursor) => { + url = format!("/api/v2/statistics/coin/holders?items=1&cursor={cursor}"); + } + None => break, + } + } + + assert_eq!(holders.get(alice_address.as_str()), Some(&alice_atoms)); + assert_eq!(holders.get(bob_address.as_str()), Some(&bob_atoms)); + + // an unknown token has no holders + let mut rng = make_seedable_rng(seed); + let token_id = TokenId::new(H256::random_using(&mut rng)); + let token_id = Address::::new(&create_unit_test_config(), token_id).unwrap(); + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/token/{}/holders", + addr.ip(), + addr.port(), + token_id.as_str() + )) + .await + .unwrap(); + assert_eq!(response.status(), 404); + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Token not found"); + + // an invalid cursor is rejected + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/coin/holders?cursor=garbage", + addr.ip(), + addr.port() + )) + .await + .unwrap(); + assert_eq!(response.status(), 400); + let body = response.text().await.unwrap(); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + + shutdown_task(task).await; +} diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 90cabea401..11e13f05ae 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -308,3 +308,126 @@ fn compare_body(body: &Value, expected_transaction: &Value) { &expected_transaction["tx_global_index"] ); } + +#[rstest] +#[trace] +#[case(Seed::from_entropy())] +#[tokio::test] +async fn cursor_pagination(#[case] seed: Seed) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let (tx, rx) = tokio::sync::oneshot::channel(); + + let task = tokio::spawn(async move { + let web_server_state = { + let mut rng = make_seedable_rng(seed); + let n_blocks = 10; + + let chain_config = create_unit_test_config(); + + let chainstate_blocks = { + let mut tf = TestFramework::builder(&mut rng) + .with_chain_config(chain_config.clone()) + .build(); + + let chainstate_block_ids = tf + .create_chain_return_ids(&tf.genesis().get_id().into(), n_blocks, &mut rng) + .unwrap(); + + _ = tx.send(()); + + chainstate_block_ids + .iter() + .map(|id| tf.block(tf.to_chain_block_id(id))) + .collect::>() + }; + + let storage = { + let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); + + let mut db_tx = storage.transaction_rw().await.unwrap(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + db_tx.commit().await.unwrap(); + + storage + }; + + let chain_config = Arc::new(chain_config); + let mut local_node = BlockchainState::new(Arc::clone(&chain_config), storage); + local_node.scan_genesis(chain_config.genesis_block()).await.unwrap(); + local_node.scan_blocks(BlockHeight::new(0), chainstate_blocks).await.unwrap(); + + ApiServerWebServerState { + db: Arc::new(local_node.storage().clone_storage().await), + chain_config: Arc::clone(&chain_config), + rpc: Arc::new(DummyRPC {}), + cached_values: Arc::new(CachedValues { + feerate_points: RwLock::new((get_time(), vec![])), + }), + time_getter: Default::default(), + stream_events: Default::default(), + } + }; + + web_server(listener, web_server_state, true).await.expect("web server failed") + }); + + let () = rx.await.unwrap(); + + let get_json = |url: String| async move { + let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) + .await + .unwrap(); + (response.status(), response.text().await.unwrap()) + }; + + // the whole list, unpaged + let (status, body) = get_json("/api/v2/transaction?offset=0&items=100".to_owned()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + let arr = body.as_array().unwrap(); + let expected_txs = + arr.iter().map(|tx| tx["id"].as_str().unwrap().to_owned()).collect::>(); + assert!(!expected_txs.is_empty()); + let num_tx = expected_txs.len(); + + // the same list, walked with the cursor + let mut url = "/api/v2/transaction?items=1&cursor=".to_owned(); + let mut paged_txs = vec![]; + loop { + let (status, body) = get_json(url.clone()).await; + assert_eq!(status, 200); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + + let items = body["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + paged_txs.push(items[0]["id"].as_str().unwrap().to_owned()); + + match body["next_cursor"].as_str() { + Some(cursor) => url = format!("/api/v2/transaction?items=1&cursor={cursor}"), + None => break, + } + } + assert_eq!(paged_txs.len(), num_tx); + assert_eq!(paged_txs, expected_txs); + + // an offset mode cannot be combined with a cursor + { + let (status, body) = + get_json("/api/v2/transaction?offset_mode=legacy&cursor=xyz".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Bad request"); + } + + // an invalid cursor is rejected + { + let (status, body) = get_json("/api/v2/transaction?cursor=garbage".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + + shutdown_task(task).await; +} diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 7395508a23..f3c9452bd0 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2198,6 +2198,437 @@ async fn orders<'a, S: for<'b> Transactional<'b>>(rng: &mut impl CryptoRng, stor ); } +/// Keyset (cursor) pagination of the pool listing over the latest pool data. +async fn pool_data_keyset( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = create_unit_test_config(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + // three pools created at heights 10, 20, 30 + let (pool1_id, pool2_id, pool3_id) = ( + PoolId::random_using(&mut rng), + PoolId::random_using(&mut rng), + PoolId::random_using(&mut rng), + ); + let (pool1_data, pool2_data, pool3_data) = ( + make_pool_data(&mut rng, Amount::from_atoms(100)), + make_pool_data(&mut rng, Amount::from_atoms(200)), + make_pool_data(&mut rng, Amount::from_atoms(300)), + ); + let (h10, h20, h30) = ( + BlockHeight::new(10), + BlockHeight::new(20), + BlockHeight::new(30), + ); + db_tx.set_pool_data_at_height(pool1_id, &pool1_data, h10).await.unwrap(); + db_tx.set_pool_data_at_height(pool2_id, &pool2_data, h20).await.unwrap(); + db_tx.set_pool_data_at_height(pool3_id, &pool3_data, h30).await.unwrap(); + + // the creation height is the height of the first insert + let expect = |expected: &[(BlockHeight, PoolId, &PoolDataWithExtraInfo)]| { + expected + .iter() + .map(|(height, pool_id, data)| (*height, *pool_id, (*data).clone())) + .collect::>() + }; + + // ordered by the creation height (descending) + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h20, pool2_id, &pool2_data), + (h10, pool1_id, &pool1_data) + ]) + ); + + // a later update does not change the creation height + let pool1_updated = make_pool_data(&mut rng, Amount::from_atoms(111)); + db_tx + .set_pool_data_at_height(pool1_id, &pool1_updated, h30.next_height()) + .await + .unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h20, pool2_id, &pool2_data), + (h10, pool1_id, &pool1_updated), + ]) + ); + + // cursor pagination + let page1 = db_tx.get_latest_pool_data_before(2, None).await.unwrap(); + assert_eq!( + page1, + expect(&[(h30, pool3_id, &pool3_data), (h20, pool2_id, &pool2_data)]) + ); + + let page2 = db_tx.get_latest_pool_data_before(2, Some((h20, pool2_id))).await.unwrap(); + assert_eq!(page2, expect(&[(h10, pool1_id, &pool1_updated)])); + + let end = db_tx.get_latest_pool_data_before(2, Some((h10, pool1_id))).await.unwrap(); + assert!(end.is_empty()); + + // pools with a zero staker balance (decommissioned) are not listed + let pool3_decommissioned = make_pool_data(&mut rng, Amount::ZERO); + db_tx + .set_pool_data_at_height(pool3_id, &pool3_decommissioned, h30.next_height()) + .await + .unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[(h20, pool2_id, &pool2_data), (h10, pool1_id, &pool1_updated)]) + ); + + // the rollback restores the latest data below the deleted height, keeping the + // original creation height + db_tx.del_pools_above_height(h30).await.unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h20, pool2_id, &pool2_data), + (h10, pool1_id, &pool1_data) + ]) + ); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +/// Holders of an asset: the top address amounts with keyset (cursor) pagination. +async fn top_address_amounts( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = create_unit_test_config(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + let (addr1, addr2, addr3, addr4) = ( + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + make_address(&chain_config, &mut rng), + ); + + let token_id = TokenId::random_using(&mut rng); + let h1 = BlockHeight::new(1); + + // register the token (the scanner stores the issuance, including the decimals, before + // any balances) + let token_data = FungibleTokenData { + token_ticker: "XXXX".as_bytes().to_vec(), + number_of_decimals: 18, + metadata_uri: "http://uri".as_bytes().to_vec(), + circulating_supply: Amount::ZERO, + total_supply: TokenTotalSupply::Unlimited, + is_locked: false, + frozen: IsTokenFrozen::No(IsTokenFreezable::Yes), + authority: Destination::PublicKeyHash(PublicKeyHash::from( + &PrivateKey::new_from_rng(&mut rng, KeyKind::Secp256k1Schnorr).1, + )), + next_nonce: AccountNonce::new(0), + }; + db_tx.set_fungible_token_issuance(token_id, h1, token_data).await.unwrap(); + + // coin holders: addr2 (300), addr3 (200), addr1 (100); addr4 has a zero balance + let coin_holders: Vec<(Address, Amount)> = vec![ + (addr1.clone(), Amount::from_atoms(100)), + (addr2.clone(), Amount::from_atoms(300)), + (addr3.clone(), Amount::from_atoms(200)), + (addr4.clone(), Amount::ZERO), + ]; + for (address, amount) in &coin_holders { + db_tx + .set_address_balance_at_height(address, *amount, CoinOrTokenId::Coin, h1) + .await + .unwrap(); + } + // a token holder + db_tx + .set_address_balance_at_height( + &addr1, + Amount::from_atoms(50), + CoinOrTokenId::TokenId(token_id), + h1, + ) + .await + .unwrap(); + + let sorted_by_amount = |mut holders: Vec<(String, Amount)>| { + holders.sort_by(|(address_l, amount_l), (address_r, amount_r)| { + (amount_r, address_r.as_str()).cmp(&(amount_l, address_l.as_str())) + }); + holders + }; + let expect = |expected: &[(&Address, Amount)]| { + expected + .iter() + .map(|(address, amount)| (address.as_str().to_owned(), *amount)) + .collect::>() + }; + + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + sorted_by_amount(all), + expect(&[ + (&addr2, Amount::from_atoms(300)), + (&addr3, Amount::from_atoms(200)), + (&addr1, Amount::from_atoms(100)) + ]) + ); + + // the token holders are independent of the coin ones + let token_holders = db_tx + .get_top_address_amounts(CoinOrTokenId::TokenId(token_id), 10, None) + .await + .unwrap(); + assert_eq!(token_holders, expect(&[(&addr1, Amount::from_atoms(50))])); + + // cursor pagination over the coin holders + let page = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 2, None).await.unwrap(); + assert_eq!(sorted_by_amount(page.clone()).len(), 2); + let (last_address, last_amount) = { + let mut sorted = sorted_by_amount(page); + let (address, amount) = sorted.pop().unwrap(); + (address, amount) + }; + + let next_page = db_tx + .get_top_address_amounts( + CoinOrTokenId::Coin, + 2, + Some((last_amount, last_address.clone())), + ) + .await + .unwrap(); + assert_eq!(next_page.len(), 1); + let (last_address2, last_amount2) = next_page[0].clone(); + + let end = db_tx + .get_top_address_amounts(CoinOrTokenId::Coin, 2, Some((last_amount2, last_address2))) + .await + .unwrap(); + assert!(end.is_empty()); + + // a balance update changes the ranking + db_tx + .set_address_balance_at_height( + &addr1, + Amount::from_atoms(400), + CoinOrTokenId::Coin, + h1.next_height(), + ) + .await + .unwrap(); + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + sorted_by_amount(all), + expect(&[ + (&addr1, Amount::from_atoms(400)), + (&addr2, Amount::from_atoms(300)), + (&addr3, Amount::from_atoms(200)) + ]) + ); + + // the rollback restores the previous balances + db_tx.del_address_balance_above_height(h1).await.unwrap(); + let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); + assert_eq!( + sorted_by_amount(all), + expect(&[ + (&addr2, Amount::from_atoms(300)), + (&addr3, Amount::from_atoms(200)), + (&addr1, Amount::from_atoms(100)) + ]) + ); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +/// The order book entries of a trading pair: only the live (not concluded, not frozen) +/// orders with non-zero remaining balances are returned. +async fn order_book_entries( + storage_maker: Arc, + seed_maker: Box Seed + Send>, +) -> Result<(), Failed> +where + S: ApiServerStorage, + Fut: Future + Send + 'static, + F: Fn() -> Fut, +{ + let mut rng = make_seedable_rng(seed_maker()); + + let mut storage = storage_maker().await; + let mut db_tx = storage.transaction_rw().await.unwrap(); + let chain_config = common::chain::config::create_regtest(); + db_tx.reinitialize_storage(&chain_config).await.unwrap(); + + let token1 = TokenId::random_using(&mut rng); + let token2 = TokenId::random_using(&mut rng); + + // three live orders of the (Coin, token1) pair and one of another pair + let (order1_id, order1) = random_order( + &mut rng, + BlockHeight::new(1), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order2_id, order2) = random_order( + &mut rng, + BlockHeight::new(2), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order3_id, order3) = random_order( + &mut rng, + BlockHeight::new(3), + CoinOrTokenId::Coin, + CoinOrTokenId::TokenId(token1), + ); + let (order4_id, order4) = random_order( + &mut rng, + BlockHeight::new(4), + CoinOrTokenId::TokenId(token1), + CoinOrTokenId::TokenId(token2), + ); + for (order_id, order) in [ + (&order1_id, &order1), + (&order2_id, &order2), + (&order3_id, &order3), + (&order4_id, &order4), + ] { + db_tx + .set_order_at_height(*order_id, order, order.creation_block_height) + .await + .unwrap(); + } + + let sort_entries = |mut entries: Vec<(Amount, Amount)>| { + entries.sort_by_key(|(ask_balance, give_balance)| (*ask_balance, *give_balance)); + entries + }; + let expected_pair = sort_entries(vec![ + (order1.ask_balance, order1.give_balance), + (order2.ask_balance, order2.give_balance), + (order3.ask_balance, order3.give_balance), + ]); + + let entries = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!(sort_entries(entries), expected_pair); + + // the reverse pair is empty + let entries = db_tx + .get_order_book_entries(CoinOrTokenId::TokenId(token1), CoinOrTokenId::Coin) + .await + .unwrap(); + assert!(entries.is_empty()); + + // a fully filled (concluded) order is no longer part of the book + let block_height = BlockHeight::new(100); + let order1_concluded = + order1.clone().fill(&chain_config, block_height, order1.ask_balance).conclude(); + db_tx + .set_order_at_height(order1_id, &order1_concluded, block_height.next_height()) + .await + .unwrap(); + let entries = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!( + sort_entries(entries), + sort_entries(vec![ + (order2.ask_balance, order2.give_balance), + (order3.ask_balance, order3.give_balance), + ]) + ); + + // a frozen order is not part of the book either + let order2_frozen = order2.clone().freeze(); + db_tx + .set_order_at_height(order2_id, &order2_frozen, block_height.next_height()) + .await + .unwrap(); + let entries = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!( + sort_entries(entries), + vec![(order3.ask_balance, order3.give_balance)] + ); + + // the rollback restores the original book + db_tx.del_orders_above_height(block_height).await.unwrap(); + let entries = db_tx + .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) + .await + .unwrap(); + assert_eq!(sort_entries(entries), expected_pair); + + db_tx.commit().await.unwrap(); + Ok(()) +} + +fn make_pool_data(rng: &mut impl CryptoRng, staker_balance: Amount) -> PoolDataWithExtraInfo { + let (_, vrf_pk) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + let (_, pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let pool_data = PoolData::new( + Destination::PublicKey(pk), + staker_balance, + Amount::ZERO, + vrf_pk, + PerThousand::new(500).unwrap(), + Amount::from_atoms(100), + ); + PoolDataWithExtraInfo { + pool_data, + delegations_balance: Amount::ZERO, + } +} + +fn make_address( + chain_config: &common::chain::ChainConfig, + rng: &mut impl CryptoRng, +) -> Address { + let (_, pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + Address::new( + chain_config, + Destination::PublicKeyHash(PublicKeyHash::from(&pk)), + ) + .unwrap() +} + fn random_order( rng: &mut impl CryptoRng, creation_height: BlockHeight, @@ -2234,6 +2665,9 @@ where vec![ make_test!(initialization, storage_maker.clone()), make_test!(set_get, storage_maker.clone()), + make_test!(pool_data_keyset, storage_maker.clone()), + make_test!(top_address_amounts, storage_maker.clone()), + make_test!(order_book_entries, storage_maker.clone()), make_test!(stream_events_append_and_read, storage_maker), ] .into_iter() diff --git a/api-server/web-server/Cargo.toml b/api-server/web-server/Cargo.toml index c9b028f97c..a85649a59c 100644 --- a/api-server/web-server/Cargo.toml +++ b/api-server/web-server/Cargo.toml @@ -20,6 +20,7 @@ mempool = { path = "../../mempool" } axum.workspace = true async-trait.workspace = true +base64.workspace = true ctor.workspace = true clap = { workspace = true, features = ["derive"] } futures = { workspace = true, default-features = false } diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs new file mode 100644 index 0000000000..7e3eb77d49 --- /dev/null +++ b/api-server/web-server/src/api/cursor.rs @@ -0,0 +1,80 @@ +// Copyright (c) 2023 RBB S.r.l +// opensource@mintlayer.org +// SPDX-License-Identifier: MIT +// Licensed under the MIT License; +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Opaque keyset pagination cursors. +//! +//! A cursor marks the position of the last item of a page within the sorted result +//! set of a listing endpoint: the next page consists of the items strictly following +//! that position. A cursor carries the sort keys of the item as strings plus a +//! tie-break id and is rendered as an URL-safe, unpadded base64 encoding of its JSON +//! form, so that the internal sort order of the endpoint does not leak to the clients +//! and stays free to change. + +use base64::Engine; +use serde::{Deserialize, Serialize}; + +use crate::error::ApiServerWebServerClientError; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Cursor { + /// Sort keys of the item the cursor points to. + keys: Vec, + /// Unique tie-break id of the item. + id: String, +} + +impl Cursor { + pub fn new(keys: Vec, id: String) -> Self { + Self { keys, id } + } + + pub fn into_parts(self) -> (Vec, String) { + (self.keys, self.id) + } + + pub fn encode(&self) -> String { + base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_string(self).expect("serialization must not fail")) + } + + pub fn decode(encoded: &str) -> Result { + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| ApiServerWebServerClientError::InvalidCursor)?; + serde_json::from_slice(&decoded).map_err(|_| ApiServerWebServerClientError::InvalidCursor) + } + + /// Decodes a cursor request parameter. An empty value starts the listing from the + /// beginning, so that a client can walk all the pages of a result set uniformly. + pub fn decode_start(encoded: &str) -> Result, ApiServerWebServerClientError> { + if encoded.is_empty() { + Ok(None) + } else { + Self::decode(encoded).map(Some) + } + } +} + +/// Renders the listing response of a cursor-based request: the items of the page plus +/// the cursor of the last returned item, `null` when the result set is exhausted. +pub fn paged_response( + items: Vec, + next_cursor: Option, +) -> serde_json::Value { + serde_json::json!({ + "items": items, + "next_cursor": next_cursor.map(|cursor| cursor.encode()), + }) +} diff --git a/api-server/web-server/src/api/mod.rs b/api-server/web-server/src/api/mod.rs index ccf14cba80..6bcde1a303 100644 --- a/api-server/web-server/src/api/mod.rs +++ b/api-server/web-server/src/api/mod.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod cursor; pub mod json_helpers; pub mod stream; pub mod v2; diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b5d3e08d1b..b994d7c7d0 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -15,9 +15,12 @@ use crate::{ MempoolQueryClient, TxSubmitClient, - api::json_helpers::{ - self, TokenDecimals, amount_to_json, block_header_to_json, pool_data_to_json, - to_tx_json_with_block_info, tx_to_json, txoutput_to_json, utxo_outpoint_to_json, + api::{ + cursor::{self, Cursor}, + json_helpers::{ + self, TokenDecimals, amount_to_json, block_header_to_json, pool_data_to_json, + to_tx_json_with_block_info, tx_to_json, txoutput_to_json, utxo_outpoint_to_json, + }, }, error::{ ApiServerWebServerClientError, ApiServerWebServerError, ApiServerWebServerForbiddenError, @@ -45,7 +48,7 @@ use common::{ make_token_id, tokens::{IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, TokenId}, }, - primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable}, + primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable, rational::Rational}, }; use hex::ToHex; use serde::Deserialize; @@ -137,7 +140,9 @@ pub fn routes< let router = router .route("/statistics/coin", get(coin_statistics)) - .route("/statistics/token/:id", get(token_statistics)); + .route("/statistics/coin/holders", get(coin_holders)) + .route("/statistics/token/:id", get(token_statistics)) + .route("/statistics/token/:id/holders", get(token_holders)); let router = router .route("/token", get(token_ids)) @@ -150,6 +155,7 @@ pub fn routes< .route("/order", get(orders)) .route("/order/:id", get(order)) .route("/order/pair/:pair", get(order_pair)) + .route("/order/pair/:pair/book", get(order_pair_book)) // Note: the real-time event stream is exposed together with the v2 endpoints, since the // events reference data that is served by them. .route("/stream", get(super::stream::stream_events)) @@ -741,11 +747,22 @@ pub async fn transactions( State(state): State, Arc>>, ) -> Result { const OFFSET_MODE: &str = "offset_mode"; + const CURSOR: &str = "cursor"; let offset_mode = params .get(OFFSET_MODE) .map(|mode| OffsetMode::from_str(mode)) .transpose()? .unwrap_or(OffsetMode::Legacy); + + let cursor_param = params.get(CURSOR); + if cursor_param.is_some() && params.contains_key(OFFSET_MODE) { + // the cursor walks the global index of the transactions, the offset modes are + // the legacy ways of walking the listing + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + let offset_and_items = get_offset_and_items(¶ms)?; let db_tx = state.db.transaction_ro().await.map_err(|e| { @@ -753,25 +770,82 @@ pub async fn transactions( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - let txs = match offset_mode { - OffsetMode::Absolute => { - db_tx - .get_transactions_with_block_info_before_tx_global_index( - offset_and_items.items, - offset_and_items.offset, + let fetch_len = offset_and_items.items.saturating_add(1); + + let (txs, next_cursor) = if let Some(cursor_str) = cursor_param { + let txs = match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, _tie_break_id) = cursor.into_parts(); + let tx_global_index = keys.first().map(|index| index.to_owned()).ok_or( + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ), + )?; + let tx_global_index = u64::from_str(&tx_global_index).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; + + db_tx + .get_transactions_with_block_info_before_tx_global_index( + fetch_len, + tx_global_index, + ) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + } + // an empty cursor starts the listing from the beginning + None => db_tx.get_transactions_with_block_info(fetch_len, 0).await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, ) - .await - } - OffsetMode::Legacy => { - db_tx - .get_transactions_with_block_info(offset_and_items.items, offset_and_items.offset) - .await + })?, + }; + + let has_next_page = txs.len() > offset_and_items.items as usize; + let next_cursor = has_next_page.then(|| { + let last_returned = &txs[offset_and_items.items as usize - 1]; + Cursor::new( + vec![last_returned.tx_global_index.to_string()], + last_returned.tx_info.tx.transaction().get_id().to_hash().encode_hex::(), + ) + }); + let txs = txs.into_iter().take(offset_and_items.items as usize).collect(); + + (txs, next_cursor) + } else { + let txs = match offset_mode { + OffsetMode::Absolute => { + db_tx + .get_transactions_with_block_info_before_tx_global_index( + offset_and_items.items, + offset_and_items.offset, + ) + .await + } + OffsetMode::Legacy => { + db_tx + .get_transactions_with_block_info( + offset_and_items.items, + offset_and_items.offset, + ) + .await + } } - } - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })?; + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + (txs, None) + }; let tip_height = best_block(&state).await?.block_height(); let txs = txs @@ -787,7 +861,11 @@ pub async fn transactions( }) .collect(); - Ok(Json(serde_json::Value::Array(txs))) + Ok(Json(match next_cursor { + Some(next_cursor) => cursor::paged_response(txs, Some(next_cursor)), + None if cursor_param.is_some() => cursor::paged_response(txs, None), + None => serde_json::Value::Array(txs), + })) } pub async fn transaction< @@ -1330,6 +1408,7 @@ pub async fn pools( State(state): State, Arc>>, ) -> Result { const SORT: &str = "sort"; + const CURSOR: &str = "cursor"; let sort = params .get(SORT) @@ -1337,6 +1416,14 @@ pub async fn pools( .transpose()? .unwrap_or(PoolSorting::ByHeight); + let cursor_param = params.get(CURSOR); + if cursor_param.is_some() && !matches!(sort, PoolSorting::ByHeight) { + // the cursor orders pools by creation height, the only ordering it supports + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + let offset_and_items = get_offset_and_items(¶ms)?; let db_tx = state.db.transaction_ro().await.map_err(|e| { @@ -1344,35 +1431,104 @@ pub async fn pools( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - let pools = match sort { - PoolSorting::ByHeight => db_tx - .get_latest_pool_data(offset_and_items.items, offset_and_items.offset) - .await - .map_err(|e| { + let pools_json = if let Some(cursor_str) = cursor_param { + let cursor = match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, id) = cursor.into_parts(); + + match keys.first() { + Some(creation_height) if !id.is_empty() => { + let creation_height = u64::from_str(creation_height).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; + let pool_id = Address::from_string(&state.chain_config, &id) + .map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })? + .into_object(); + Some((BlockHeight::new(creation_height), pool_id)) + } + _ => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + } + } + // an empty cursor starts the listing from the beginning + None => None, + }; + + // an extra item tells whether the result set continues past the page + let items_to_fetch = offset_and_items.items.saturating_add(1); + let pools = + db_tx.get_latest_pool_data_before(items_to_fetch, cursor).await.map_err(|e| { logging::log::error!("internal error: {e}"); ApiServerWebServerError::ServerError( ApiServerWebServerServerError::InternalServerError, ) - })?, - PoolSorting::ByPledge => db_tx - .get_pool_data_with_largest_staker_balance( - offset_and_items.items, - offset_and_items.offset, + })?; + + let has_next_page = pools.len() > offset_and_items.items as usize; + let pools = pools.into_iter().take(offset_and_items.items as usize).collect::>(); + + let next_cursor = has_next_page.then(|| { + let (creation_height, pool_id, _pool_data) = + pools.last().expect("a next page implies a last item"); + Cursor::new( + vec![creation_height.into_int().to_string()], + Address::new(&state.chain_config, *pool_id) + .expect("no error in encoding") + .as_str() + .to_owned(), ) - .await - .map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError( - ApiServerWebServerServerError::InternalServerError, + }); + + let items = pools + .into_iter() + .map(|(_creation_height, pool_id, pool_data)| { + pool_data_to_json(&state.chain_config, pool_data, pool_id) + }) + .collect(); + + cursor::paged_response(items, next_cursor) + } else { + let pools = match sort { + PoolSorting::ByHeight => db_tx + .get_latest_pool_data(offset_and_items.items, offset_and_items.offset) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })?, + PoolSorting::ByPledge => db_tx + .get_pool_data_with_largest_staker_balance( + offset_and_items.items, + offset_and_items.offset, ) - })?, - }; + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })?, + }; - let pools = pools - .into_iter() - .map(|(pool_id, pool_data)| pool_data_to_json(&state.chain_config, pool_data, pool_id)); + let pools = pools + .into_iter() + .map(|(pool_id, pool_data)| pool_data_to_json(&state.chain_config, pool_data, pool_id)); - Ok(Json(pools.collect::>())) + serde_json::Value::Array(pools.collect()) + }; + + Ok(Json(pools_json)) } pub async fn pool( @@ -1734,6 +1890,133 @@ pub async fn token_statistics( }))) } +/// Shared implementation of the holders endpoints: the top holders of an asset, +/// ordered by the balance, with optional keyset (cursor) pagination. +async fn holders_response( + params: &BTreeMap, + state: &ApiServerWebServerState, Arc>, + coin_or_token_id: CoinOrTokenId, + number_of_decimals: u8, +) -> Result { + const CURSOR: &str = "cursor"; + + let offset_and_items = get_offset_and_items(params)?; + + let cursor = match params.get(CURSOR) { + Some(cursor_str) => match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, tie_break_id) = cursor.into_parts(); + let amount = match keys.first() { + Some(atoms) => u128::from_str(atoms).map(Amount::from_atoms).map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?, + None => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } + }; + Some((amount, tie_break_id)) + } + None => None, + }, + None => None, + }; + + // an extra item is requested to detect the presence of the next page + let fetch_len = offset_and_items.items.saturating_add(1); + + let mut holders = state + .db + .transaction_ro() + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })? + .get_top_address_amounts(coin_or_token_id, fetch_len, cursor) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let has_next_page = holders.len() > offset_and_items.items as usize; + holders.truncate(offset_and_items.items as usize); + + let next_cursor = has_next_page.then(|| { + let (address, amount) = holders.last().expect("at least one item"); + Cursor::new(vec![amount.into_atoms().to_string()], address.clone()) + }); + + let items = holders + .into_iter() + .map(|(address, amount)| { + json!({ + "address": address, + "amount": amount_to_json(amount, number_of_decimals), + }) + }) + .collect(); + + Ok(cursor::paged_response(items, next_cursor)) +} + +pub async fn coin_holders( + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + let response = holders_response( + ¶ms, + &state, + CoinOrTokenId::Coin, + state.chain_config.coin_decimals(), + ) + .await?; + + Ok(Json(response)) +} + +pub async fn token_holders( + Path(token_id): Path, + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + let token_id = Address::from_string(&state.chain_config, token_id) + .map_err(|_| { + ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidTokenId) + })? + .into_object(); + + let tx = state.db.transaction_ro().await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let token_decimals = tx + .get_token_num_decimals(token_id) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })? + .ok_or(ApiServerWebServerError::NotFound( + ApiServerWebServerNotFoundError::TokenNotFound, + ))?; + + let response = holders_response( + ¶ms, + &state, + CoinOrTokenId::TokenId(token_id), + token_decimals, + ) + .await?; + + Ok(Json(response)) +} + pub async fn token_ids( Query(params): Query>, State(state): State, Arc>>, @@ -2024,6 +2307,215 @@ pub async fn order_pair( Ok(Json(orders.collect::>())) } +/// The order book of a trading pair: the live orders of the pair aggregated into price +/// levels. `side` selects the side of the book: `ask` lists the orders asking for the +/// base currency (first in the pair) while giving the quote currency, `bid` the reverse. +/// The price of a level is the remaining quote amount per remaining base unit (a +/// rational number, in atoms); `amount` is the remaining base amount at the level. The +/// ask book is ordered by the ascending price, the bid book by the descending price; +/// keyset (cursor) pagination over the levels. +pub async fn order_pair_book( + Path(pair): Path, + Query(params): Query>, + State(state): State, Arc>>, +) -> Result { + const SIDE: &str = "side"; + const CURSOR: &str = "cursor"; + const SIDE_ASK: &str = "ask"; + const SIDE_BID: &str = "bid"; + + let parse_currency = |s: &str| -> Result { + if s.to_uppercase() == state.chain_config.coin_ticker() { + Ok(CoinOrTokenId::Coin) + } else { + let token_id = Address::from_string(&state.chain_config, s) + .map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidTokenId, + ) + })? + .into_object(); + Ok(CoinOrTokenId::TokenId(token_id)) + } + }; + + let parts: Vec<_> = pair.split("_").collect(); + ensure!( + parts.len() == 2, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidOrderTradingPair + ) + ); + let base = parse_currency(parts[0])?; + let quote = parse_currency(parts[1])?; + + // an ask of the (base, quote) pair is an order asking for the base and giving the + // quote; a bid is the reverse. In both cases the price is expressed in the quote + // currency per one base unit. + let (ask_currency, give_currency, base_is_ask) = match params.get(SIDE).map(String::as_str) { + Some(SIDE_ASK) => (base, quote, true), + Some(SIDE_BID) => (quote, base, false), + _ => { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::BadRequest, + )); + } + }; + + let offset_and_items = get_offset_and_items(¶ms)?; + + // the cursor is the (numerator, denominator) of the last returned level price + let cursor_price = match params.get(CURSOR) { + Some(cursor_str) => match Cursor::decode_start(cursor_str)? { + Some(cursor) => { + let (keys, _tie_break_id) = cursor.into_parts(); + let parse_key = |key: Option<&String>| { + key.and_then(|key| u128::from_str(key).ok()).ok_or( + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ), + ) + }; + let numer = parse_key(keys.first())?; + let denom = parse_key(keys.get(1))?; + ensure!( + denom != 0, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); + Some(Rational::new(numer, denom)) + } + None => None, + }, + None => None, + }; + + let db_tx = state.db.transaction_ro().await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + let entries = db_tx.get_order_book_entries(ask_currency, give_currency).await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + + // aggregate the remaining balances into price levels + let mut levels = BTreeMap::, u128>::new(); + for (ask_balance, give_balance) in entries { + let (price, base_amount) = if base_is_ask { + ( + Rational::new(give_balance.into_atoms(), ask_balance.into_atoms()), + ask_balance.into_atoms(), + ) + } else { + ( + Rational::new(ask_balance.into_atoms(), give_balance.into_atoms()), + give_balance.into_atoms(), + ) + }; + // reduce the fraction so equal prices always map to the same level + let price = reduce_rational(price); + + let level = levels.entry(price).or_insert(0); + *level = level.checked_add(base_amount).ok_or_else(|| { + logging::log::error!("order book level amount overflow"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + } + + // the ask book is ordered by the ascending price, the bid book by the descending one + let mut levels: Vec<_> = levels.into_iter().collect(); + if !base_is_ask { + levels.reverse(); + } + + if let Some(cursor_price) = cursor_price { + levels.retain(|(price, _amount)| { + if base_is_ask { + *price > cursor_price + } else { + *price < cursor_price + } + }); + } + + let has_next_page = levels.len() > offset_and_items.items as usize; + levels.truncate(offset_and_items.items as usize); + + let next_cursor = has_next_page.then(|| { + let (price, _amount) = levels.last().expect("at least one level"); + Cursor::new( + vec![price.numer().to_string(), price.denom().to_string()], + String::new(), + ) + }); + + let decimals = currency_decimals_for_pair(&db_tx, &base, "e, &state.chain_config).await?; + let base_decimals = decimals[&base]; + let quote_decimals = decimals["e]; + + let items = levels + .into_iter() + .map(|(price, level_amount)| { + let (numer, denom) = (*price.numer(), *price.denom()); + // the price in atoms is the quote amount per one base atom; to express it + // in the currency units, scale both sides by the other currency's decimals + let price_decimal = numer as f64 * 10f64.powi(base_decimals as i32) + / (denom as f64 * 10f64.powi(quote_decimals as i32)); + + json!({ + "price": { + "decimal": price_decimal.to_string(), + "atoms": format!("{numer}/{denom}"), + }, + "amount": amount_to_json(Amount::from_atoms(level_amount), base_decimals), + }) + }) + .collect(); + + Ok(Json(cursor::paged_response(items, next_cursor))) +} + +fn reduce_rational(rational: Rational) -> Rational { + let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); + while denom != 0 { + (numer, denom) = (denom, numer % denom); + } + Rational::new(*rational.numer() / numer, *rational.denom() / numer) +} + +async fn currency_decimals_for_pair( + db_tx: &impl ApiServerStorageRead, + base: &CoinOrTokenId, + quote: &CoinOrTokenId, + chain_config: &ChainConfig, +) -> Result, ApiServerWebServerError> { + let mut decimals = BTreeMap::new(); + decimals.insert(CoinOrTokenId::Coin, chain_config.coin_decimals()); + + for currency in [base, quote] { + if let CoinOrTokenId::TokenId(token_id) = currency { + let token_decimals = db_tx + .get_token_num_decimals(*token_id) + .await + .map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + .ok_or(ApiServerWebServerError::NotFound( + ApiServerWebServerNotFoundError::TokenNotFound, + ))?; + decimals.insert(*currency, token_decimals); + } + } + + Ok(decimals) +} + struct OffsetAndItems { offset: u64, items: u32, diff --git a/api-server/web-server/src/error.rs b/api-server/web-server/src/error.rs index edf807cda2..ecac9def40 100644 --- a/api-server/web-server/src/error.rs +++ b/api-server/web-server/src/error.rs @@ -117,6 +117,8 @@ pub enum ApiServerWebServerClientError { InvalidOrderId, #[error("Invalid order trading pair")] InvalidOrderTradingPair, + #[error("Invalid cursor")] + InvalidCursor, } #[allow(dead_code)] From de27544e4188d2eb59ce843bb9758087eb575516 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 00:21:43 +0200 Subject: [PATCH 02/22] fix(indexer): empty pages, pool filters, tie-break order, exact prices Addresses the first automated review pass: - items=0 is now rejected by the shared pagination parameter parser: the cursor endpoints fetch items+1 rows and read the last returned row to build the next cursor, so an accepted zero page size made all four cursor endpoints panic remotely (requests return 400 instead). - the in-memory pool listings now filter pools by a non-zero staker balance, matching the postgres implementation (decommissioned is not equivalent: pledge/rewards can be zero while the staker balance is not, and vice versa). - the holders' address and the pool id columns are declared with the C collation so ordering and cursor comparisons are plain byte order, deterministic across database clusters and identical to the in-memory backend's ordering. - the order book price decimal field is computed with exact integer arithmetic (u128 mantissa scaled in a u256) instead of f64; the fraction string is built by the shared decimal formatter. An f64 remains only as an unreachable-in-practice fallback if the exact mantissa overflows u128. - endpoint tests: zero page size is rejected on /pool and /transaction. --- .../src/storage/impls/in_memory/mod.rs | 6 +-- .../src/storage/impls/postgres/queries.rs | 4 +- api-server/stack-test-suite/tests/v2/pools.rs | 8 ++++ .../stack-test-suite/tests/v2/transactions.rs | 8 ++++ api-server/web-server/src/api/v2.rs | 42 ++++++++++++++++--- 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 2c811cbaef..183e738e86 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -587,7 +587,7 @@ impl ApiServerInMemoryStorage { let latest_data = by_height.values().last().expect("not empty"); (pool_id, (created_height, latest_data)) }) - .filter(|(_pool_id, data)| !data.1.is_decommissioned()) + .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); pool_data.sort_by_key(|(_, (height, _data))| Reverse(*height)); @@ -617,7 +617,7 @@ impl ApiServerInMemoryStorage { let latest_data = by_height.values().last().expect("not empty"); (pool_id, (*created_height, latest_data)) }) - .filter(|(_pool_id, data)| !data.1.is_decommissioned()) + .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); // newest first, ties broken by pool id so the keyset cursor is deterministic @@ -649,7 +649,7 @@ impl ApiServerInMemoryStorage { .pool_data_table .iter() .map(|(pool_id, by_height)| (pool_id, by_height.values().last().expect("not empty"))) - .filter(|(_pool_id, data)| !data.is_decommissioned()) + .filter(|data| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); pool_data.sort_by_key(|(_, data)| Reverse(data.staker_balance().expect("no overflow"))); diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 3a77ebe772..ddede69f85 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -1200,7 +1200,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // the full versioned history (which is touched at every PoS block). self.just_execute( "CREATE TABLE ml.latest_pool_data_cache ( - pool_id TEXT NOT NULL, + pool_id TEXT COLLATE \"C\" NOT NULL, block_height bigint NOT NULL, creation_block_height bigint NOT NULL, staker_balance NUMERIC NOT NULL, @@ -1236,7 +1236,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // listing can be served by an index scan instead of decoding every balance. self.just_execute( "CREATE TABLE ml.address_amount ( - address TEXT NOT NULL, + address TEXT COLLATE \"C\" NOT NULL, coin_or_token_id bytea NOT NULL, block_height bigint NOT NULL, amount NUMERIC NOT NULL, diff --git a/api-server/stack-test-suite/tests/v2/pools.rs b/api-server/stack-test-suite/tests/v2/pools.rs index 2ce217df4a..6931e28b77 100644 --- a/api-server/stack-test-suite/tests/v2/pools.rs +++ b/api-server/stack-test-suite/tests/v2/pools.rs @@ -485,5 +485,13 @@ async fn pools_cursor_pagination(#[case] seed: Seed) { assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); } + // a zero page size is rejected + { + let (status, body) = get_json("/api/v2/pool?items=0".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid number of items"); + } + shutdown_task(task).await; } diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 11e13f05ae..18bc4f991a 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -429,5 +429,13 @@ async fn cursor_pagination(#[case] seed: Seed) { assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); } + // a zero page size is rejected + { + let (status, body) = get_json("/api/v2/transaction?items=0".to_owned()).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid number of items"); + } + shutdown_task(task).await; } diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b994d7c7d0..596ede051c 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -40,6 +40,7 @@ use axum::{ routing::{get, post}, }; use common::{ + Uint256, address::Address, chain::{ Block, ChainConfig, Destination, OutPointSourceId, SignedTransaction, Transaction, @@ -48,7 +49,9 @@ use common::{ make_token_id, tokens::{IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, TokenId}, }, - primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable, rational::Rational}, + primitives::{ + Amount, BlockHeight, CoinOrTokenId, DecimalAmount, H256, Id, Idable, rational::Rational, + }, }; use hex::ToHex; use serde::Deserialize; @@ -2461,13 +2464,14 @@ pub async fn order_pair_book( .map(|(price, level_amount)| { let (numer, denom) = (*price.numer(), *price.denom()); // the price in atoms is the quote amount per one base atom; to express it - // in the currency units, scale both sides by the other currency's decimals - let price_decimal = numer as f64 * 10f64.powi(base_decimals as i32) - / (denom as f64 * 10f64.powi(quote_decimals as i32)); + // in the currency units, scale the numerator by the base currency's + // decimals (and represent the result with the quote currency's decimals) + let price_decimal = + price_to_decimal_string(numer, denom, base_decimals, quote_decimals); json!({ "price": { - "decimal": price_decimal.to_string(), + "decimal": price_decimal, "atoms": format!("{numer}/{denom}"), }, "amount": amount_to_json(Amount::from_atoms(level_amount), base_decimals), @@ -2478,6 +2482,30 @@ pub async fn order_pair_book( Ok(Json(cursor::paged_response(items, next_cursor))) } +/// The exact decimal representation of a price given as `numer/denom` quote atoms per +/// base atom: `numer * 10^base_decimals / denom`, displayed with `quote_decimals` +/// fractional digits. Falls back to an f64 only if the exact mantissa exceeds u128. +fn price_to_decimal_string( + numer: u128, + denom: u128, + base_decimals: u8, + quote_decimals: u8, +) -> String { + let mantissa = 10u128 + .checked_pow(u32::from(base_decimals)) + .and_then(|base_scale| Uint256::from(numer).checked_mul(&Uint256::from(base_scale))) + .and_then(|scaled| scaled.checked_div(&Uint256::from(denom))) + .and_then(|mantissa| u128::try_from(mantissa).ok()); + match mantissa { + Some(mantissa) => DecimalAmount::from_uint_decimal(mantissa, quote_decimals).to_string(), + None => { + let price = numer as f64 * 10f64.powi(base_decimals as i32) + / (denom as f64 * 10f64.powi(quote_decimals as i32)); + price.to_string() + } + } +} + fn reduce_rational(rational: Rational) -> Rational { let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); while denom != 0 { @@ -2546,6 +2574,10 @@ fn get_offset_and_items( ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) })? .unwrap_or(DEFAULT_NUM_ITEMS); + ensure!( + items > 0, + ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) + ); ensure!( items <= MAX_NUM_ITEMS, ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) From c1de28dfbe367e5abd4e6ffa1753eeeac10820b2 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 00:39:35 +0200 Subject: [PATCH 03/22] fix(indexer): batch holder rollback, bounded book, cursor hardening - set-based reorg rollback for the holders table (constant round trips instead of per-row queries) - cap order book aggregation at 10k deepest orders per query - validate the holders cursor tie-break address; guard reduce_rational against zero denominators - exact integer price formatting via 256-bit truncation (no float fallback inconsistency) - tests: items=0 rejected on transaction/pool listings, exact holder page content, baseline truncation assert --- .../src/storage/impls/postgres/queries.rs | 131 ++++++++++++------ .../stack-test-suite/tests/v2/transactions.rs | 3 + api-server/storage-test-suite/src/basic.rs | 7 +- api-server/web-server/src/api/v2.rs | 53 +++++-- 4 files changed, 136 insertions(+), 58 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index ddede69f85..3a07224a5b 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -55,6 +55,11 @@ pub struct QueryFromConnection<'a, 'b> { } impl<'a, 'b> QueryFromConnection<'a, 'b> { + /// Safety bound for the order book aggregation: at most this many live orders + /// (the deepest ones first) contribute to a single request, capping the cost + /// of one book query regardless of how many orders a pair accumulates. + const ORDER_BOOK_MAX_ORDERS: i64 = 10_000; + fn get_table_exists_query(table_name: &str) -> String { format!( "SELECT COALESCE( ( @@ -378,8 +383,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - // roll back the holders table rows of the disconnected blocks; the numeric - // amount is derived from the versioned balances, hence the row-by-row handling + // roll back the holders table rows of the disconnected blocks; set-based, in a + // constant number of round trips: delete the rolled-back rows, drop rows whose + // whole versioned history was orphaned, then re-derive and re-insert the rest + // in bulk (the numeric amount is decoded from the versioned balances) let deleted_holders_rows = self .tx .query( @@ -389,55 +396,87 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - for row in deleted_holders_rows { - let address: String = row.get(0); - let coin_or_token_id: Vec = row.get(1); + if !deleted_holders_rows.is_empty() { + let addresses: Vec = + deleted_holders_rows.iter().map(|row| row.get(0)).collect(); + let coin_or_token_ids: Vec> = + deleted_holders_rows.iter().map(|row| row.get(1)).collect(); + + // rows whose versioned history was fully orphaned leave the table + self.tx + .execute( + r#" + DELETE FROM ml.address_amount aa + WHERE (aa.address, aa.coin_or_token_id) IN ( + SELECT address, coin_or_token_id + FROM unnest($1::text[], $2::bytea[]) AS d(address, coin_or_token_id) + ) + AND NOT EXISTS ( + SELECT 1 FROM ml.address_balance ab + WHERE ab.address = aa.address AND ab.coin_or_token_id = aa.coin_or_token_id + ); + "#, + &[&addresses, &coin_or_token_ids], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // the latest remaining versioned balance of every affected pair let latest_versioned = self .tx - .query_opt( - "SELECT block_height, amount FROM ml.address_balance WHERE address = $1 AND coin_or_token_id = $2 ORDER BY block_height DESC LIMIT 1;", - &[&address, &coin_or_token_id], + .query( + r#" + SELECT DISTINCT ON (a.address, a.coin_or_token_id) + a.address, a.coin_or_token_id, a.block_height, a.amount + FROM ml.address_balance a + JOIN unnest($1::text[], $2::bytea[]) AS d(address, coin_or_token_id) + ON a.address = d.address AND a.coin_or_token_id = d.coin_or_token_id + ORDER BY a.address, a.coin_or_token_id, a.block_height DESC; + "#, + &[&addresses, &coin_or_token_ids], ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - match latest_versioned { - Some(latest) => { - let block_height: i64 = latest.get(0); - let amount: Vec = latest.get(1); - let amount = Amount::decode_all(&mut amount.as_slice()).map_err(|e| { - ApiServerStorageError::DeserializationError(format!( - "Amount deserialization failed: {e}" - )) - })?; - let amount_str = amount.into_atoms().to_string(); - - self.tx - .execute( - r#" - INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) - VALUES ($1, $2, $3, $4::text::numeric) - ON CONFLICT (address, coin_or_token_id) DO UPDATE - SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount - WHERE ml.address_amount.block_height <= EXCLUDED.block_height; - "#, - &[&address, &coin_or_token_id, &block_height, &amount_str], - ) - .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - } - None => { - // the whole history of the address and the asset was orphaned - self.tx - .execute( - "DELETE FROM ml.address_amount WHERE address = $1 AND coin_or_token_id = $2;", - &[&address, &coin_or_token_id], - ) - .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - } + let mut upsert_addresses: Vec = Vec::with_capacity(latest_versioned.len()); + let mut upsert_coin_or_token_ids: Vec> = + Vec::with_capacity(latest_versioned.len()); + let mut upsert_block_heights: Vec = Vec::with_capacity(latest_versioned.len()); + let mut upsert_amounts: Vec = Vec::with_capacity(latest_versioned.len()); + + for row in latest_versioned { + let amount: Vec = row.get(3); + let amount = Amount::decode_all(&mut amount.as_slice()).map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "Amount deserialization failed: {e}" + )) + })?; + upsert_addresses.push(row.get(0)); + upsert_coin_or_token_ids.push(row.get(1)); + upsert_block_heights.push(row.get::<_, i64>(2)); + upsert_amounts.push(amount.into_atoms().to_string()); } + + self.tx + .execute( + r#" + INSERT INTO ml.address_amount (address, coin_or_token_id, block_height, amount) + SELECT address, coin_or_token_id, block_height, amount + FROM unnest($1::text[], $2::bytea[], $3::bigint[], $4::text[]::numeric[]) + AS t(address, coin_or_token_id, block_height, amount) + ON CONFLICT (address, coin_or_token_id) DO UPDATE + SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount + WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + "#, + &[ + &upsert_addresses, + &upsert_coin_or_token_ids, + &upsert_block_heights, + &upsert_amounts, + ], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; } Ok(()) @@ -3460,9 +3499,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { FROM ml.latest_orders_cache WHERE ask_currency = $1 AND give_currency = $2 AND frozen = FALSE - AND ask_balance != 0 AND give_balance != 0; + AND ask_balance != 0 AND give_balance != 0 + ORDER BY ask_balance DESC + LIMIT $3; "#, - &[&ask_currency, &give_currency], + &[&ask_currency, &give_currency, &Self::ORDER_BOOK_MAX_ORDERS], ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 18bc4f991a..c6b4e7a429 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -391,6 +391,9 @@ async fn cursor_pagination(#[case] seed: Seed) { arr.iter().map(|tx| tx["id"].as_str().unwrap().to_owned()).collect::>(); assert!(!expected_txs.is_empty()); let num_tx = expected_txs.len(); + // the baseline is fetched with the server's maximum page size; a larger list + // would be silently truncated and the comparison below would be vacuous + assert!(num_tx < 100); // the same list, walked with the cursor let mut url = "/api/v2/transaction?items=1&cursor=".to_owned(); diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index f3c9452bd0..d54f5d0fc3 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2412,7 +2412,10 @@ where // cursor pagination over the coin holders let page = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 2, None).await.unwrap(); - assert_eq!(sorted_by_amount(page.clone()).len(), 2); + assert_eq!( + sorted_by_amount(page.clone()), + expect(&[(&addr2, Amount::from_atoms(300)), (&addr3, Amount::from_atoms(200))]) + ); let (last_address, last_amount) = { let mut sorted = sorted_by_amount(page); let (address, amount) = sorted.pop().unwrap(); @@ -2427,7 +2430,7 @@ where ) .await .unwrap(); - assert_eq!(next_page.len(), 1); + assert_eq!(next_page, expect(&[(&addr1, Amount::from_atoms(100))])); let (last_address2, last_amount2) = next_page[0].clone(); let end = db_tx diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 596ede051c..0a4c8f34b5 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1921,6 +1921,14 @@ async fn holders_response( )); } }; + // the tie-break is an encoded address; reject garbage up front + let _validated = + Address::::from_string(&state.chain_config, &tie_break_id) + .map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })?; Some((amount, tie_break_id)) } None => None, @@ -2491,23 +2499,46 @@ fn price_to_decimal_string( base_decimals: u8, quote_decimals: u8, ) -> String { - let mantissa = 10u128 - .checked_pow(u32::from(base_decimals)) - .and_then(|base_scale| Uint256::from(numer).checked_mul(&Uint256::from(base_scale))) + let as_f64 = || { + let price = numer as f64 * 10f64.powi(base_decimals as i32) + / (denom as f64 * 10f64.powi(quote_decimals as i32)); + price.to_string() + }; + + // mantissa = numer * 10^base_decimals / denom so that the price equals + // mantissa * 10^-quote_decimals; computed in a u256 because numer can be close + // to u128::MAX and is scaled by up to 10^38 + let Some(base_scale) = 10u128.checked_pow(u32::from(base_decimals)) else { + return as_f64(); + }; + let Some(mut mantissa) = Uint256::from(numer) + .checked_mul(&Uint256::from(base_scale)) .and_then(|scaled| scaled.checked_div(&Uint256::from(denom))) - .and_then(|mantissa| u128::try_from(mantissa).ok()); - match mantissa { - Some(mantissa) => DecimalAmount::from_uint_decimal(mantissa, quote_decimals).to_string(), - None => { - let price = numer as f64 * 10f64.powi(base_decimals as i32) - / (denom as f64 * 10f64.powi(quote_decimals as i32)); - price.to_string() + else { + return as_f64(); + }; + // shrink a mantissa that does not fit a u128 by dropping its least significant + // digits, each drop compensated by one fewer decimal place; this cannot exhaust + // the decimals for any real price, so the f64 tail is unreachable in practice + let mut decimals = quote_decimals; + let mantissa = loop { + if let Ok(mantissa) = u128::try_from(mantissa) { + break mantissa; } - } + if decimals == 0 { + return as_f64(); + } + mantissa = mantissa.checked_div(&Uint256::from(10u128)).expect("ten is not zero"); + decimals -= 1; + }; + DecimalAmount::from_uint_decimal(mantissa, decimals).to_string() } fn reduce_rational(rational: Rational) -> Rational { let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); + if denom == 0 { + return rational; + } while denom != 0 { (numer, denom) = (denom, numer % denom); } From 95dcaf9f2c9490dcb61a93f610ad7b77880bb165 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 01:07:59 +0200 Subject: [PATCH 04/22] fix(indexer): book parity, tie-breaks, no silent truncation - order-book reads return (entries, has_more) with the 10k cap enforced in both backends; the endpoint surfaces truncation as an additive "truncated" field - deterministic tie-breaks everywhere: order ids and pool ids sort by their encoded (bech32) address strings in byte order in both backends (in-memory storage now keeps the chain config for that); explicit COLLATE "C" in the holders ORDER BY - in-memory pool listings sort by the same keys as postgres (creation height then encoded address; pledge then encoded address) - tests assert the raw backend order instead of re-sorting --- .../src/storage/impls/in_memory/mod.rs | 99 ++++++++++++++++--- .../impls/in_memory/transactional/read.rs | 2 +- .../impls/in_memory/transactional/write.rs | 2 +- .../src/storage/impls/postgres/queries.rs | 39 ++++---- .../impls/postgres/transactional/read.rs | 2 +- .../impls/postgres/transactional/write.rs | 2 +- .../src/storage/storage_api/mod.rs | 22 +++-- api-server/storage-test-suite/src/basic.rs | 44 ++++----- api-server/web-server/src/api/v2.rs | 48 ++++++--- 9 files changed, 180 insertions(+), 80 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 183e738e86..3842ac9f63 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -37,8 +37,9 @@ use common::{ use crate::storage::storage_api::{ AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, - FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoLock, UtxoWithExtraInfo, + FungibleTokenData, LockedUtxo, NftWithOwner, ORDER_BOOK_MAX_ORDERS, Order, PoolBlockStats, + PoolDataWithExtraInfo, TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoLock, UtxoWithExtraInfo, block_aux_data::{BlockAuxData, BlockWithExtraData}, }; @@ -85,6 +86,9 @@ struct ApiServerInMemoryStorage { genesis_block: Arc>, number_of_coin_decimals: u8, storage_version: u32, + // kept so that order/pool identifiers can be encoded exactly like the postgres + // backend does (bech32 address strings), keeping tie-break orders identical + chain_config: ChainConfig, } impl ApiServerInMemoryStorage { @@ -112,9 +116,27 @@ impl ApiServerInMemoryStorage { genesis_block: chain_config.genesis_block().clone(), number_of_coin_decimals: chain_config.coin_decimals(), storage_version: CURRENT_STORAGE_VERSION, + chain_config: chain_config.clone(), } } + /// Encoded (bech32) form of a pool id — the exact string the postgres backend + /// stores and orders by, keeping tie-break orders identical across backends. + fn pool_id_sort_key(&self, pool_id: &PoolId) -> String { + Address::new(&self.chain_config, *pool_id) + .expect("encoding of a pool id stored in the in-memory storage cannot fail") + .as_str() + .to_owned() + } + + /// Encoded (bech32) form of an order id — see `pool_id_sort_key`. + fn order_id_sort_key(&self, order_id: &OrderId) -> String { + Address::new(&self.chain_config, *order_id) + .expect("encoding of an order id stored in the in-memory storage cannot fail") + .as_str() + .to_owned() + } + fn is_initialized(&self) -> Result { Ok(true) } @@ -554,22 +576,41 @@ impl ApiServerInMemoryStorage { &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { - let entries = self + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { + let mut entries: Vec<_> = self .orders_table - .values() - .filter_map(|by_height| by_height.values().last()) - .filter(|order| { + .iter() + .filter_map(|(order_id, by_height)| Some((order_id, by_height.values().last()?))) + .filter(|(_order_id, order)| { order.ask_currency == ask_currency && order.give_currency == give_currency && !order.is_frozen && order.ask_balance != Amount::ZERO && order.give_balance != Amount::ZERO }) - .map(|order| (order.ask_balance, order.give_balance)) .collect(); - Ok(entries) + // deepest orders first, ties broken by the encoded order id (byte order) and + // capped exactly like the postgres query; has_more reports the truncation + entries.sort_by(|(order_id_a, order_a), (order_id_b, order_b)| { + order_b + .ask_balance + .into_atoms() + .cmp(&order_a.ask_balance.into_atoms()) + .then_with(|| { + self.order_id_sort_key(order_id_b).cmp(&self.order_id_sort_key(order_id_a)) + }) + }); + let has_more = entries.len() > ORDER_BOOK_MAX_ORDERS; + entries.truncate(ORDER_BOOK_MAX_ORDERS); + + Ok(( + entries + .into_iter() + .map(|(_order_id, order)| (order.ask_balance, order.give_balance)) + .collect(), + has_more, + )) } fn get_latest_pool_ids( @@ -590,7 +631,13 @@ impl ApiServerInMemoryStorage { .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); - pool_data.sort_by_key(|(_, (height, _data))| Reverse(*height)); + // newest first, ties broken by the encoded pool address (byte order), exactly + // like the postgres listing + pool_data.sort_by(|(pool_id_a, (height_a, _)), (pool_id_b, (height_b, _))| { + height_b.cmp(height_a).then_with(|| { + self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) + }) + }); if offset >= pool_data.len() { return Ok(vec![]); } @@ -620,11 +667,21 @@ impl ApiServerInMemoryStorage { .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); - // newest first, ties broken by pool id so the keyset cursor is deterministic - pool_data.sort_by_key(|(pool_id, (height, _data))| Reverse((*height, **pool_id))); - pool_data.retain(|(pool_id, (height, _data))| match cursor { - Some((cursor_height, cursor_pool_id)) => { - *height < cursor_height || (*height == cursor_height && **pool_id < cursor_pool_id) + // newest first, ties broken by the encoded pool address (byte order) so the + // keyset cursor is deterministic and backend-independent + pool_data.sort_by(|(pool_id_a, (height_a, _)), (pool_id_b, (height_b, _))| { + height_b.cmp(height_a).then_with(|| { + self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) + }) + }); + let cursor_key = cursor.map(|(cursor_height, cursor_pool_id)| { + (cursor_height, self.pool_id_sort_key(&cursor_pool_id)) + }); + pool_data.retain(|(pool_id, (height, _data))| match &cursor_key { + Some((cursor_height, cursor_pool_key)) => { + *height < *cursor_height + || (*height == *cursor_height + && self.pool_id_sort_key(pool_id) < *cursor_pool_key) } None => true, }); @@ -652,7 +709,17 @@ impl ApiServerInMemoryStorage { .filter(|data| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) .collect(); - pool_data.sort_by_key(|(_, data)| Reverse(data.staker_balance().expect("no overflow"))); + // deepest pledge first, ties broken by the encoded pool address (byte order), + // exactly like the postgres listing + pool_data.sort_by(|(pool_id_a, data_a), (pool_id_b, data_b)| { + data_b + .staker_balance() + .expect("no overflow") + .cmp(&data_a.staker_balance().expect("no overflow")) + .then_with(|| { + self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) + }) + }); if offset >= pool_data.len() { return Ok(vec![]); } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index 4deb1b524b..e0835f1890 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -339,7 +339,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { self.transaction.get_order_book_entries(ask_currency, give_currency) } } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index 22fc166b2c..01d2f8f109 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -620,7 +620,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { self.transaction.get_order_book_entries(ask_currency, give_currency) } } diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 3a07224a5b..1c42339e3d 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -37,8 +37,9 @@ use crate::storage::{ impls::CURRENT_STORAGE_VERSION, storage_api::{ AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation, - FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo, - TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoWithExtraInfo, + FungibleTokenData, LockedUtxo, NftWithOwner, ORDER_BOOK_MAX_ORDERS, Order, PoolBlockStats, + PoolDataWithExtraInfo, TokenTransaction, TransactionInfo, TransactionWithBlockInfo, Utxo, + UtxoWithExtraInfo, block_aux_data::{BlockAuxData, BlockWithExtraData}, }, }; @@ -55,11 +56,6 @@ pub struct QueryFromConnection<'a, 'b> { } impl<'a, 'b> QueryFromConnection<'a, 'b> { - /// Safety bound for the order book aggregation: at most this many live orders - /// (the deepest ones first) contribute to a single request, capping the cost - /// of one book query regardless of how many orders a pair accumulates. - const ORDER_BOOK_MAX_ORDERS: i64 = 10_000; - fn get_table_exists_query(table_name: &str) -> String { format!( "SELECT COALESCE( ( @@ -209,7 +205,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { FROM ml.address_amount WHERE coin_or_token_id = $1 AND amount != 0 AND (amount, address) < ($2::text::numeric, $3) - ORDER BY amount DESC, address DESC + ORDER BY amount DESC, address COLLATE "C" DESC LIMIT $4; "#, &[&coin_or_token_id, &amount_str, &address, &len], @@ -224,7 +220,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT address, amount::text FROM ml.address_amount WHERE coin_or_token_id = $1 AND amount != 0 - ORDER BY amount DESC, address DESC + ORDER BY amount DESC, address COLLATE "C" DESC LIMIT $2; "#, &[&coin_or_token_id, &len], @@ -1302,7 +1298,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // the full versioned history. self.just_execute( "CREATE TABLE ml.latest_orders_cache ( - order_id TEXT NOT NULL, + order_id TEXT COLLATE \"C\" NOT NULL, block_height bigint NOT NULL, creation_block_height bigint NOT NULL, ask_currency bytea NOT NULL, @@ -2170,7 +2166,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT pool_id, data FROM ml.latest_pool_data_cache WHERE staker_balance != 0 - ORDER BY staker_balance DESC + ORDER BY staker_balance DESC, pool_id DESC OFFSET $1 LIMIT $2; "#, @@ -3480,16 +3476,19 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { Ok(()) } - /// Remaining balances of all the live (not concluded, not frozen) orders of a - /// trading pair, as (ask_balance, give_balance) pairs; the price of each order is - /// derived from these in the web layer. + /// Remaining balances of the live (not concluded, not frozen) orders of a trading + /// pair, as (ask_balance, give_balance) pairs, ordered by ask balance (descending, + /// ties broken by the order id in descending byte order); capped at + /// [`ORDER_BOOK_MAX_ORDERS`] entries, `has_more` reports truncation. pub async fn get_order_book_entries( &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { let ask_currency = ask_currency.encode(); let give_currency = give_currency.encode(); + // one extra row detects truncation + let cap = ORDER_BOOK_MAX_ORDERS as i64 + 1; let rows = self .tx @@ -3500,15 +3499,18 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { WHERE ask_currency = $1 AND give_currency = $2 AND frozen = FALSE AND ask_balance != 0 AND give_balance != 0 - ORDER BY ask_balance DESC + ORDER BY ask_balance DESC, order_id DESC LIMIT $3; "#, - &[&ask_currency, &give_currency, &Self::ORDER_BOOK_MAX_ORDERS], + &[&ask_currency, &give_currency, &cap], ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + let has_more = rows.len() > ORDER_BOOK_MAX_ORDERS; + rows.into_iter() + .take(ORDER_BOOK_MAX_ORDERS) .map(|row| -> Result<(Amount, Amount), ApiServerStorageError> { let ask_str: String = row.get(0); let give_str: String = row.get(1); @@ -3526,7 +3528,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { Ok((Amount::from_atoms(ask), Amount::from_atoms(give))) }) - .collect() + .collect::, _>>() + .map(|entries| (entries, has_more)) } pub async fn get_orders_by_height( diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 15a835eed7..2f6e81b339 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -475,7 +475,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); conn.get_order_book_entries(ask_currency, give_currency).await } diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index 23cf339db5..368638a3ef 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -872,7 +872,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError> { + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); conn.get_order_book_entries(ask_currency, give_currency).await } diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index b10f13a508..16c6b78481 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -40,6 +40,11 @@ use serialization::{Decode, Encode}; use self::block_aux_data::{BlockAuxData, BlockWithExtraData}; use crate::streaming::{StreamEvent, StreamEventId, StreamEventReadError}; +/// Hard cap on the number of live orders a single order-book query may consume; +/// both backends return at most this many entries, deepest (largest ask balance) +/// first, with `has_more` signaling a truncated result. +pub const ORDER_BOOK_MAX_ORDERS: usize = 10_000; + pub mod block_aux_data; #[allow(dead_code)] @@ -608,9 +613,9 @@ pub trait ApiServerStorageRead: Sync { ) -> Result, ApiServerStorageError>; /// Returns up to `len` holders of the asset with the largest balances, ordered by - /// the balance (descending, ties broken by the address descending); keyset (cursor) - /// pagination, the cursor is the (amount, address) of the last returned holder, - /// exclusive. Addresses with a zero balance are excluded. + /// the balance (descending, ties broken by the address in descending BYTE order); + /// keyset (cursor) pagination, the cursor is the (amount, address) of the last + /// returned holder, exclusive. Addresses with a zero balance are excluded. async fn get_top_address_amounts( &self, coin_or_token_id: CoinOrTokenId, @@ -827,14 +832,17 @@ pub trait ApiServerStorageRead: Sync { offset: u64, ) -> Result, ApiServerStorageError>; - /// Remaining balances of all the live (not concluded, not frozen) orders of a - /// trading pair, as (ask_balance, give_balance) pairs; the price of each order is - /// derived from these in the web layer. + /// Remaining balances of the live (not concluded, not frozen) orders of a trading + /// pair, as (ask_balance, give_balance) pairs; the price of each order is derived + /// from these in the web layer. At most [`ORDER_BOOK_MAX_ORDERS`] entries + /// are returned, ordered by ask balance (descending, ties broken by the order id + /// in descending byte order); `has_more` reports whether the cap truncated the + /// book (the entries are the deepest orders, the remainder is omitted). async fn get_order_book_entries( &self, ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, - ) -> Result, ApiServerStorageError>; + ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError>; /// Read the stream events with an id greater than `last_seen_id`, in ascending id order. /// diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index d54f5d0fc3..9b2168a69f 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2380,12 +2380,8 @@ where .await .unwrap(); - let sorted_by_amount = |mut holders: Vec<(String, Amount)>| { - holders.sort_by(|(address_l, amount_l), (address_r, amount_r)| { - (amount_r, address_r.as_str()).cmp(&(amount_l, address_l.as_str())) - }); - holders - }; + // NOTE: no re-sorting before the asserts — the ordering (balance descending, the + // encoded address descending as tie-break) is exactly what the contract requires let expect = |expected: &[(&Address, Amount)]| { expected .iter() @@ -2395,7 +2391,7 @@ where let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( - sorted_by_amount(all), + all, expect(&[ (&addr2, Amount::from_atoms(300)), (&addr3, Amount::from_atoms(200)), @@ -2413,14 +2409,10 @@ where // cursor pagination over the coin holders let page = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 2, None).await.unwrap(); assert_eq!( - sorted_by_amount(page.clone()), + page, expect(&[(&addr2, Amount::from_atoms(300)), (&addr3, Amount::from_atoms(200))]) ); - let (last_address, last_amount) = { - let mut sorted = sorted_by_amount(page); - let (address, amount) = sorted.pop().unwrap(); - (address, amount) - }; + let (last_address, last_amount) = page[1].clone(); let next_page = db_tx .get_top_address_amounts( @@ -2451,7 +2443,7 @@ where .unwrap(); let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( - sorted_by_amount(all), + all, expect(&[ (&addr1, Amount::from_atoms(400)), (&addr2, Amount::from_atoms(300)), @@ -2463,7 +2455,7 @@ where db_tx.del_address_balance_above_height(h1).await.unwrap(); let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( - sorted_by_amount(all), + all, expect(&[ (&addr2, Amount::from_atoms(300)), (&addr3, Amount::from_atoms(200)), @@ -2543,18 +2535,22 @@ where (order3.ask_balance, order3.give_balance), ]); - let entries = db_tx + let (entries, has_more) = db_tx .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) .await .unwrap(); - assert_eq!(sort_entries(entries), expected_pair); + // the entries come back with the deepest orders first (the ordering contract) + assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); + assert!(!has_more); + assert_eq!(sort_entries(entries.clone()), expected_pair); // the reverse pair is empty - let entries = db_tx + let (entries, has_more) = db_tx .get_order_book_entries(CoinOrTokenId::TokenId(token1), CoinOrTokenId::Coin) .await .unwrap(); assert!(entries.is_empty()); + assert!(!has_more); // a fully filled (concluded) order is no longer part of the book let block_height = BlockHeight::new(100); @@ -2564,7 +2560,7 @@ where .set_order_at_height(order1_id, &order1_concluded, block_height.next_height()) .await .unwrap(); - let entries = db_tx + let (entries, has_more) = db_tx .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) .await .unwrap(); @@ -2575,6 +2571,7 @@ where (order3.ask_balance, order3.give_balance), ]) ); + assert!(!has_more); // a frozen order is not part of the book either let order2_frozen = order2.clone().freeze(); @@ -2582,7 +2579,7 @@ where .set_order_at_height(order2_id, &order2_frozen, block_height.next_height()) .await .unwrap(); - let entries = db_tx + let (entries, has_more) = db_tx .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) .await .unwrap(); @@ -2590,14 +2587,17 @@ where sort_entries(entries), vec![(order3.ask_balance, order3.give_balance)] ); + assert!(!has_more); // the rollback restores the original book db_tx.del_orders_above_height(block_height).await.unwrap(); - let entries = db_tx + let (entries, has_more) = db_tx .get_order_book_entries(CoinOrTokenId::Coin, CoinOrTokenId::TokenId(token1)) .await .unwrap(); - assert_eq!(sort_entries(entries), expected_pair); + assert_eq!(sort_entries(entries.clone()), expected_pair); + assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); + assert!(!has_more); db_tx.commit().await.unwrap(); Ok(()) diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 0a4c8f34b5..579a5c19f4 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1479,17 +1479,27 @@ pub async fn pools( let has_next_page = pools.len() > offset_and_items.items as usize; let pools = pools.into_iter().take(offset_and_items.items as usize).collect::>(); - let next_cursor = has_next_page.then(|| { + let next_cursor = if has_next_page { let (creation_height, pool_id, _pool_data) = pools.last().expect("a next page implies a last item"); - Cursor::new( + // the pool id comes from the storage, so encoding cannot fail; map the + // error anyway instead of risking a panic inside a request handler + let pool_id_str = Address::new(&state.chain_config, *pool_id) + .map_err(|e| { + logging::log::error!("internal error: pool id encoding failed: {e}"); + ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + ) + })? + .as_str() + .to_owned(); + Some(Cursor::new( vec![creation_height.into_int().to_string()], - Address::new(&state.chain_config, *pool_id) - .expect("no error in encoding") - .as_str() - .to_owned(), - ) - }); + pool_id_str, + )) + } else { + None + }; let items = pools .into_iter() @@ -2407,10 +2417,11 @@ pub async fn order_pair_book( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - let entries = db_tx.get_order_book_entries(ask_currency, give_currency).await.map_err(|e| { - logging::log::error!("internal error: {e}"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })?; + let (entries, has_more_orders) = + db_tx.get_order_book_entries(ask_currency, give_currency).await.map_err(|e| { + logging::log::error!("internal error: {e}"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; // aggregate the remaining balances into price levels let mut levels = BTreeMap::, u128>::new(); @@ -2487,7 +2498,18 @@ pub async fn order_pair_book( }) .collect(); - Ok(Json(cursor::paged_response(items, next_cursor))) + // the hard cap on aggregated live orders may have truncated the book; surface + // that to the client as an additive field so a partial book is never presented + // as complete + let response = if has_more_orders { + let mut response = cursor::paged_response(items, next_cursor); + response["truncated"] = json!(true); + response + } else { + cursor::paged_response(items, next_cursor) + }; + + Ok(Json(response)) } /// The exact decimal representation of a price given as `numer/denom` quote atoms per From 9c39d5e8c97ad99a91126cc26eea2f2ad7b363cb Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 01:26:59 +0200 Subject: [PATCH 05/22] fix(indexer): sort-key encoding, robust pledge sort, bounded test walks - encode order-id/pool-id sort keys once per entry in the in-memory backend (decorate-sort-undecorate) instead of once per comparison - the pledge listing decodes the balance once and treats an unparseable balance as absent instead of expecting in a production sort - the order-book handler defensively skips zero-balance entries so a future storage change cannot corrupt the level ordering - cursor-walk endpoint tests are bounded by the known item count --- .../src/storage/impls/in_memory/mod.rs | 144 ++++++++++-------- api-server/stack-test-suite/tests/v2/pools.rs | 6 + .../stack-test-suite/tests/v2/statistics.rs | 21 +++ .../stack-test-suite/tests/v2/transactions.rs | 7 +- api-server/web-server/src/api/v2.rs | 6 + 5 files changed, 117 insertions(+), 67 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 3842ac9f63..0de0552b8f 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -577,7 +577,7 @@ impl ApiServerInMemoryStorage { ask_currency: CoinOrTokenId, give_currency: CoinOrTokenId, ) -> Result<(Vec<(Amount, Amount)>, bool), ApiServerStorageError> { - let mut entries: Vec<_> = self + let entries: Vec<_> = self .orders_table .iter() .filter_map(|(order_id, by_height)| Some((order_id, by_height.values().last()?))) @@ -591,23 +591,28 @@ impl ApiServerInMemoryStorage { .collect(); // deepest orders first, ties broken by the encoded order id (byte order) and - // capped exactly like the postgres query; has_more reports the truncation - entries.sort_by(|(order_id_a, order_a), (order_id_b, order_b)| { - order_b - .ask_balance - .into_atoms() - .cmp(&order_a.ask_balance.into_atoms()) - .then_with(|| { - self.order_id_sort_key(order_id_b).cmp(&self.order_id_sort_key(order_id_a)) - }) + // capped exactly like the postgres query; has_more reports the truncation. + // the sort key is encoded once per entry, not once per comparison + let mut decorated: Vec<(String, Amount, Amount)> = entries + .into_iter() + .map(|(order_id, order)| { + ( + self.order_id_sort_key(order_id), + order.ask_balance, + order.give_balance, + ) + }) + .collect(); + decorated.sort_by_key(|(key, ask_balance, _give_balance)| { + (Reverse(ask_balance.into_atoms()), Reverse(key.clone())) }); - let has_more = entries.len() > ORDER_BOOK_MAX_ORDERS; - entries.truncate(ORDER_BOOK_MAX_ORDERS); + let has_more = decorated.len() > ORDER_BOOK_MAX_ORDERS; + decorated.truncate(ORDER_BOOK_MAX_ORDERS); Ok(( - entries + decorated .into_iter() - .map(|(_order_id, order)| (order.ask_balance, order.give_balance)) + .map(|(_key, ask_balance, give_balance)| (ask_balance, give_balance)) .collect(), has_more, )) @@ -620,31 +625,34 @@ impl ApiServerInMemoryStorage { ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; - let mut pool_data: Vec<_> = self + // newest first, ties broken by the encoded pool address (byte order), exactly + // like the postgres listing; the sort key is encoded once per entry + let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() .map(|(pool_id, by_height)| { - let created_height = by_height.keys().next().expect("not empty"); - let latest_data = by_height.values().last().expect("not empty"); - (pool_id, (created_height, latest_data)) + let created_height = *by_height.keys().next().expect("not empty"); + let latest_data = by_height.values().last().expect("not empty").clone(); + ( + created_height, + self.pool_id_sort_key(pool_id), + *pool_id, + latest_data, + ) }) - .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) - .collect(); - - // newest first, ties broken by the encoded pool address (byte order), exactly - // like the postgres listing - pool_data.sort_by(|(pool_id_a, (height_a, _)), (pool_id_b, (height_b, _))| { - height_b.cmp(height_a).then_with(|| { - self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) + .filter(|(_height, _key, _pool_id, data)| { + data.staker_balance().is_ok_and(|b| b != Amount::ZERO) }) - }); - if offset >= pool_data.len() { + .collect(); + decorated + .sort_by_key(|(height, key, _pool_id, _data)| (Reverse(*height), Reverse(key.clone()))); + if offset >= decorated.len() { return Ok(vec![]); } - let latest_pools = pool_data[offset..std::cmp::min(offset + len, pool_data.len())] + let latest_pools = decorated[offset..std::cmp::min(offset + len, decorated.len())] .iter() - .map(|(pool_id, data)| (**pool_id, (data.1).clone())) + .map(|(_height, _key, pool_id, data)| (*pool_id, data.clone())) .collect(); Ok(latest_pools) @@ -656,40 +664,42 @@ impl ApiServerInMemoryStorage { cursor: Option<(BlockHeight, PoolId)>, ) -> Result, ApiServerStorageError> { let len = len as usize; - let mut pool_data: Vec<_> = self + // newest first, ties broken by the encoded pool address (byte order) so the + // keyset cursor is deterministic and backend-independent; the sort key is + // encoded once per entry + let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() .map(|(pool_id, by_height)| { - let created_height = by_height.keys().next().expect("not empty"); - let latest_data = by_height.values().last().expect("not empty"); - (pool_id, (*created_height, latest_data)) + let created_height = *by_height.keys().next().expect("not empty"); + let latest_data = by_height.values().last().expect("not empty").clone(); + ( + created_height, + self.pool_id_sort_key(pool_id), + *pool_id, + latest_data, + ) }) - .filter(|(_pool_id, data)| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) - .collect(); - - // newest first, ties broken by the encoded pool address (byte order) so the - // keyset cursor is deterministic and backend-independent - pool_data.sort_by(|(pool_id_a, (height_a, _)), (pool_id_b, (height_b, _))| { - height_b.cmp(height_a).then_with(|| { - self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) + .filter(|(_height, _key, _pool_id, data)| { + data.staker_balance().is_ok_and(|b| b != Amount::ZERO) }) - }); + .collect(); + decorated + .sort_by_key(|(height, key, _pool_id, _data)| (Reverse(*height), Reverse(key.clone()))); let cursor_key = cursor.map(|(cursor_height, cursor_pool_id)| { (cursor_height, self.pool_id_sort_key(&cursor_pool_id)) }); - pool_data.retain(|(pool_id, (height, _data))| match &cursor_key { + decorated.retain(|(height, key, _pool_id, _data)| match &cursor_key { Some((cursor_height, cursor_pool_key)) => { - *height < *cursor_height - || (*height == *cursor_height - && self.pool_id_sort_key(pool_id) < *cursor_pool_key) + *height < *cursor_height || (*height == *cursor_height && *key < *cursor_pool_key) } None => true, }); - let latest_pools = pool_data + let latest_pools = decorated .iter() .take(len) - .map(|(pool_id, (height, data))| (*height, **pool_id, (*data).clone())) + .map(|(height, _key, pool_id, data)| (*height, *pool_id, data.clone())) .collect(); Ok(latest_pools) @@ -702,31 +712,33 @@ impl ApiServerInMemoryStorage { ) -> Result, ApiServerStorageError> { let len = len as usize; let offset = offset as usize; - let mut pool_data: Vec<_> = self + // deepest pledge first, ties broken by the encoded pool address (byte order), + // exactly like the postgres listing; the balance and the sort key are decoded + // once per entry, and an unparseable/overflowed balance is treated as absent + let mut decorated: Vec<(Amount, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() - .map(|(pool_id, by_height)| (pool_id, by_height.values().last().expect("not empty"))) - .filter(|data| data.1.staker_balance().is_ok_and(|b| b != Amount::ZERO)) + .filter_map(|(pool_id, by_height)| { + let data = by_height.values().last()?.clone(); + let balance = data.staker_balance().ok()?; + (balance != Amount::ZERO).then_some(( + balance, + self.pool_id_sort_key(pool_id), + *pool_id, + data, + )) + }) .collect(); - - // deepest pledge first, ties broken by the encoded pool address (byte order), - // exactly like the postgres listing - pool_data.sort_by(|(pool_id_a, data_a), (pool_id_b, data_b)| { - data_b - .staker_balance() - .expect("no overflow") - .cmp(&data_a.staker_balance().expect("no overflow")) - .then_with(|| { - self.pool_id_sort_key(pool_id_b).cmp(&self.pool_id_sort_key(pool_id_a)) - }) + decorated.sort_by_key(|(balance, key, _pool_id, _data)| { + (Reverse(balance.into_atoms()), Reverse(key.clone())) }); - if offset >= pool_data.len() { + if offset >= decorated.len() { return Ok(vec![]); } - let latest_pools = pool_data[offset..std::cmp::min(offset + len, pool_data.len())] + let latest_pools = decorated[offset..std::cmp::min(offset + len, decorated.len())] .iter() - .map(|(pool_id, data)| (**pool_id, (*data).clone())) + .map(|(_balance, _key, pool_id, data)| (*pool_id, data.clone())) .collect(); Ok(latest_pools) diff --git a/api-server/stack-test-suite/tests/v2/pools.rs b/api-server/stack-test-suite/tests/v2/pools.rs index 6931e28b77..bada674abd 100644 --- a/api-server/stack-test-suite/tests/v2/pools.rs +++ b/api-server/stack-test-suite/tests/v2/pools.rs @@ -449,6 +449,12 @@ async fn pools_cursor_pagination(#[case] seed: Seed) { let mut url = "/api/v2/pool?items=1&cursor=".to_owned(); let mut paged_order = vec![]; loop { + // bounded by the known pool count so a self-referencing cursor fails + // instead of hanging + assert!( + paged_order.len() < expected_order.len(), + "cursor walk exceeded the expected pool count" + ); let (status, body) = get_json(url.clone()).await; assert_eq!(status, 200); let body: serde_json::Value = serde_json::from_str(&body).unwrap(); diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 31c48c4e5e..dce0cc6e56 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -535,11 +535,31 @@ async fn coin_holders(#[case] seed: Seed) { let alice_atoms = amount_to_json(Amount::from_atoms(100), coin_decimals); let bob_atoms = amount_to_json(Amount::from_atoms(50), coin_decimals); + // the holder count is known from an unpaged request; the walk below is + // bounded by it so a self-referencing cursor fails instead of hanging + let response = reqwest::get(format!( + "http://{}:{}/api/v2/statistics/coin/holders?items=100", + addr.ip(), + addr.port() + )) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let body: serde_json::Value = serde_json::from_str(&response.text().await.unwrap()).unwrap(); + let num_holders = body["items"].as_array().unwrap().len(); + assert!(num_holders >= 2); + // walk the whole holders list, one item per request let mut url = "/api/v2/statistics/coin/holders?items=1".to_owned(); let mut holders = BTreeMap::::new(); let mut prev_amount: Option = None; loop { + // bounded by the known holder count so a self-referencing cursor fails + // instead of hanging + assert!( + holders.len() < num_holders, + "the holders cursor walk exceeded the expected number of holders" + ); let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) .await .unwrap(); @@ -573,6 +593,7 @@ async fn coin_holders(#[case] seed: Seed) { } } + assert_eq!(holders.len(), num_holders); assert_eq!(holders.get(alice_address.as_str()), Some(&alice_atoms)); assert_eq!(holders.get(bob_address.as_str()), Some(&bob_atoms)); diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index c6b4e7a429..bf4a991dbb 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -395,10 +395,15 @@ async fn cursor_pagination(#[case] seed: Seed) { // would be silently truncated and the comparison below would be vacuous assert!(num_tx < 100); - // the same list, walked with the cursor + // the same list, walked with the cursor; the walk is bounded by the known + // baseline length so a self-referencing cursor fails instead of hanging let mut url = "/api/v2/transaction?items=1&cursor=".to_owned(); let mut paged_txs = vec![]; loop { + assert!( + paged_txs.len() < num_tx, + "the cursor walk exceeded the expected number of transactions" + ); let (status, body) = get_json(url.clone()).await; assert_eq!(status, 200); let body: serde_json::Value = serde_json::from_str(&body).unwrap(); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 579a5c19f4..364b8fd2be 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -2426,6 +2426,12 @@ pub async fn order_pair_book( // aggregate the remaining balances into price levels let mut levels = BTreeMap::, u128>::new(); for (ask_balance, give_balance) in entries { + // defensive: the storage layer filters these out, but a zero balance would + // yield a zero-denominator price whose `Ord` compares equal to every other + // key and silently corrupts the level map, so skip it here as well + if ask_balance == Amount::ZERO || give_balance == Amount::ZERO { + continue; + } let (price, base_amount) = if base_is_ask { ( Rational::new(give_balance.into_atoms(), ask_balance.into_atoms()), From eeb1503a4e05ace14aa930188f739994908f4761 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 01:48:08 +0200 Subject: [PATCH 06/22] fix(indexer): explicit collations, book index, exact price digits - the holders and pools keyset predicates expand the row comparison so the byte-order (C) collation of the tie-break is explicit rather than relying on the column default; the pool ORDER BYs pin it too - latest_orders_currencies_index now covers the book read's ordering so the pair scan needs no separate sort - the order-book price decimal is rendered exactly (u256 digit extraction) with no floating-point fallback; an impossible scaling overflow becomes an internal server error - the storage test pins the equal-balance tie-break and its cursor boundary --- .../src/storage/impls/postgres/queries.rs | 20 +-- api-server/storage-test-suite/src/basic.rs | 57 ++++++--- api-server/web-server/src/api/v2.rs | 114 ++++++++++-------- 3 files changed, 111 insertions(+), 80 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 1c42339e3d..b31fec3ef5 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -204,7 +204,9 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT address, amount::text FROM ml.address_amount WHERE coin_or_token_id = $1 AND amount != 0 - AND (amount, address) < ($2::text::numeric, $3) + AND (amount < $2::text::numeric + OR (amount = $2::text::numeric + AND address COLLATE "C" < $3)) ORDER BY amount DESC, address COLLATE "C" DESC LIMIT $4; "#, @@ -1311,10 +1313,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; - // index for the order book pair lookups + // index for the order book pair lookups; the trailing columns cover the + // book read's ordering (deepest ask first, encoded order id as tie-break) so + // no separate sort of the pair's live orders is needed self.just_execute( "CREATE INDEX latest_orders_currencies_index - ON ml.latest_orders_cache (ask_currency, give_currency);", + ON ml.latest_orders_cache (ask_currency, give_currency, ask_balance DESC, order_id DESC);", ) .await?; @@ -2073,7 +2077,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT pool_id, data FROM ml.latest_pool_data_cache WHERE staker_balance != 0 - ORDER BY creation_block_height DESC, pool_id DESC + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC OFFSET $1 LIMIT $2; "#, @@ -2105,8 +2109,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT creation_block_height, pool_id, data FROM ml.latest_pool_data_cache WHERE staker_balance != 0 - AND (creation_block_height, pool_id) < ($1, $2) - ORDER BY creation_block_height DESC, pool_id DESC + AND (creation_block_height < $1 + OR (creation_block_height = $1 + AND pool_id COLLATE "C" < $2)) + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC LIMIT $3; "#, &[&creation_height, &pool_id.as_str(), &len], @@ -2166,7 +2172,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT pool_id, data FROM ml.latest_pool_data_cache WHERE staker_balance != 0 - ORDER BY staker_balance DESC, pool_id DESC + ORDER BY staker_balance DESC, pool_id COLLATE "C" DESC OFFSET $1 LIMIT $2; "#, diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 9b2168a69f..3bd885c2c3 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2356,12 +2356,13 @@ where }; db_tx.set_fungible_token_issuance(token_id, h1, token_data).await.unwrap(); - // coin holders: addr2 (300), addr3 (200), addr1 (100); addr4 has a zero balance + // coin holders: addr2 (300), addr3 and addr4 (200 each, so the encoded-address + // tie-break is exercised), addr1 (100) let coin_holders: Vec<(Address, Amount)> = vec![ (addr1.clone(), Amount::from_atoms(100)), (addr2.clone(), Amount::from_atoms(300)), (addr3.clone(), Amount::from_atoms(200)), - (addr4.clone(), Amount::ZERO), + (addr4.clone(), Amount::from_atoms(200)), ]; for (address, amount) in &coin_holders { db_tx @@ -2389,14 +2390,19 @@ where .collect::>() }; + // equal balances tie-break by the encoded address, descending byte order + let mut tie = [addr3.as_str(), addr4.as_str()]; + tie.sort_unstable(); + let (tie_low, tie_high) = (tie[0], tie[1]); let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( all, - expect(&[ - (&addr2, Amount::from_atoms(300)), - (&addr3, Amount::from_atoms(200)), - (&addr1, Amount::from_atoms(100)) - ]) + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] ); // the token holders are independent of the coin ones @@ -2410,7 +2416,10 @@ where let page = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 2, None).await.unwrap(); assert_eq!( page, - expect(&[(&addr2, Amount::from_atoms(300)), (&addr3, Amount::from_atoms(200))]) + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + ] ); let (last_address, last_amount) = page[1].clone(); @@ -2422,8 +2431,14 @@ where ) .await .unwrap(); - assert_eq!(next_page, expect(&[(&addr1, Amount::from_atoms(100))])); - let (last_address2, last_amount2) = next_page[0].clone(); + assert_eq!( + next_page, + vec![ + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] + ); + let (last_address2, last_amount2) = next_page[1].clone(); let end = db_tx .get_top_address_amounts(CoinOrTokenId::Coin, 2, Some((last_amount2, last_address2))) @@ -2444,11 +2459,12 @@ where let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( all, - expect(&[ - (&addr1, Amount::from_atoms(400)), - (&addr2, Amount::from_atoms(300)), - (&addr3, Amount::from_atoms(200)) - ]) + vec![ + (addr1.as_str().to_owned(), Amount::from_atoms(400)), + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + ] ); // the rollback restores the previous balances @@ -2456,11 +2472,12 @@ where let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( all, - expect(&[ - (&addr2, Amount::from_atoms(300)), - (&addr3, Amount::from_atoms(200)), - (&addr1, Amount::from_atoms(100)) - ]) + vec![ + (addr2.as_str().to_owned(), Amount::from_atoms(300)), + (tie_high.to_owned(), Amount::from_atoms(200)), + (tie_low.to_owned(), Amount::from_atoms(200)), + (addr1.as_str().to_owned(), Amount::from_atoms(100)), + ] ); db_tx.commit().await.unwrap(); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 364b8fd2be..b579623ced 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -49,9 +49,7 @@ use common::{ make_token_id, tokens::{IsTokenFreezable, IsTokenFrozen, IsTokenUnfreezable, TokenId}, }, - primitives::{ - Amount, BlockHeight, CoinOrTokenId, DecimalAmount, H256, Id, Idable, rational::Rational, - }, + primitives::{Amount, BlockHeight, CoinOrTokenId, H256, Id, Idable, rational::Rational}, }; use hex::ToHex; use serde::Deserialize; @@ -2486,23 +2484,25 @@ pub async fn order_pair_book( let items = levels .into_iter() - .map(|(price, level_amount)| { - let (numer, denom) = (*price.numer(), *price.denom()); - // the price in atoms is the quote amount per one base atom; to express it - // in the currency units, scale the numerator by the base currency's - // decimals (and represent the result with the quote currency's decimals) - let price_decimal = - price_to_decimal_string(numer, denom, base_decimals, quote_decimals); - - json!({ - "price": { - "decimal": price_decimal, - "atoms": format!("{numer}/{denom}"), - }, - "amount": amount_to_json(Amount::from_atoms(level_amount), base_decimals), - }) - }) - .collect(); + .map( + |(price, level_amount)| -> Result { + let (numer, denom) = (*price.numer(), *price.denom()); + // the price in atoms is the quote amount per one base atom; to express it + // in the currency units, scale the numerator by the base currency's + // decimals (and represent the result with the quote currency's decimals) + let price_decimal = + price_to_decimal_string(numer, denom, base_decimals, quote_decimals)?; + + Ok(json!({ + "price": { + "decimal": price_decimal, + "atoms": format!("{numer}/{denom}"), + }, + "amount": amount_to_json(Amount::from_atoms(level_amount), base_decimals), + })) + }, + ) + .collect::, ApiServerWebServerError>>()?; // the hard cap on aggregated live orders may have truncated the book; surface // that to the client as an additive field so a partial book is never presented @@ -2526,40 +2526,48 @@ fn price_to_decimal_string( denom: u128, base_decimals: u8, quote_decimals: u8, -) -> String { - let as_f64 = || { - let price = numer as f64 * 10f64.powi(base_decimals as i32) - / (denom as f64 * 10f64.powi(quote_decimals as i32)); - price.to_string() - }; - - // mantissa = numer * 10^base_decimals / denom so that the price equals - // mantissa * 10^-quote_decimals; computed in a u256 because numer can be close - // to u128::MAX and is scaled by up to 10^38 - let Some(base_scale) = 10u128.checked_pow(u32::from(base_decimals)) else { - return as_f64(); - }; - let Some(mut mantissa) = Uint256::from(numer) - .checked_mul(&Uint256::from(base_scale)) - .and_then(|scaled| scaled.checked_div(&Uint256::from(denom))) - else { - return as_f64(); - }; - // shrink a mantissa that does not fit a u128 by dropping its least significant - // digits, each drop compensated by one fewer decimal place; this cannot exhaust - // the decimals for any real price, so the f64 tail is unreachable in practice - let mut decimals = quote_decimals; - let mantissa = loop { - if let Ok(mantissa) = u128::try_from(mantissa) { - break mantissa; - } - if decimals == 0 { - return as_f64(); - } - mantissa = mantissa.checked_div(&Uint256::from(10u128)).expect("ten is not zero"); - decimals -= 1; +) -> Result { + // the price equals (numer * 10^base_decimals / denom) * 10^-quote_decimals; the + // digits are computed exactly in u256 arithmetic (the scaled numerator fits a + // u256 for any realistic decimal count and is checked anyway), because a + // financial price must not silently lose precision through floating point + let ten = Uint256::from_u64(10); + let mut scaled = Uint256::from(numer); + for _ in 0..base_decimals { + scaled = scaled.checked_mul(&ten).ok_or_else(|| { + logging::log::error!("order book price scaling overflow"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; + } + let scaled = scaled.checked_div(&Uint256::from(denom)).expect("denominator is not zero"); + + // the decimal digits of the scaled price, least significant last, by repeated + // division by ten + let mut digits = Vec::new(); + let mut value = scaled; + while value != Uint256::ZERO { + let rem = value.checked_rem(&ten).expect("ten is not zero"); + let rem = u128::try_from(rem).expect("a digit fits a u128"); + digits.push(u8::try_from(rem).expect("a digit fits a u8") + b'0'); + value = value.checked_div(&ten).expect("ten is not zero"); + } + if digits.is_empty() { + digits.push(b'0'); + } + digits.reverse(); + let digits = String::from_utf8(digits).expect("digits are ascii"); + + // place the decimal point `quote_decimals` digits from the right + let point = usize::from(quote_decimals); + let price = if point == 0 { + digits + } else if digits.len() > point { + let (int_part, frac_part) = digits.split_at(digits.len() - point); + format!("{int_part}.{frac_part}") + } else { + format!("0.{}{digits}", "0".repeat(point - digits.len())) }; - DecimalAmount::from_uint_decimal(mantissa, decimals).to_string() + Ok(price) } fn reduce_rational(rational: Rational) -> Rational { From 4c25c29f3eb5324e74fa610985a8cc64b1a41534 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 02:09:25 +0200 Subject: [PATCH 07/22] fix(indexer): exhaustiveness guard, explicit columns, cursor bounds - VARIANTS gets a compile-time exhaustiveness guard so a new statistic variant cannot be silently omitted from the latest-state lookup - the orders-cache reorg restore names its columns instead of SELECT * - a transactions cursor index that does not fit an i64 is rejected as an invalid cursor (client error) before reaching the storage layer - the holders test baseline checks itself against the page limit; the order book handler documents that a walk is not a consistent snapshot of a moving book --- .../src/storage/impls/postgres/queries.rs | 10 +++++++--- .../src/storage/storage_api/mod.rs | 10 ++++++++-- api-server/stack-test-suite/tests/v2/statistics.rs | 3 +++ api-server/web-server/src/api/v2.rs | 13 +++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index b31fec3ef5..b3e53fb8f2 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -3468,9 +3468,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { o.ask_balance::numeric, o.give_currency, o.give_balance::numeric, o.frozen FROM deleted_cache_rows d CROSS JOIN LATERAL ( - SELECT * FROM ml.orders - WHERE order_id = d.order_id - ORDER BY block_height DESC + SELECT + o.order_id, o.block_height, o.creation_block_height, + o.ask_currency, o.ask_balance, o.give_currency, + o.give_balance, o.frozen + FROM ml.orders o + WHERE o.order_id = d.order_id + ORDER BY o.block_height DESC LIMIT 1 ) o; "#, diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 16c6b78481..9889a426c4 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -89,8 +89,14 @@ pub enum CoinOrTokenStatistic { } impl CoinOrTokenStatistic { - pub const VARIANTS: [Self; 4] = - [Self::CirculatingSupply, Self::Staked, Self::Burned, Self::Preminted]; + pub const VARIANTS: [Self; 4] = { + // compile-time exhaustiveness guard: adding a variant without updating the + // array below fails to compile here because this match has no wildcard + match Self::CirculatingSupply { + Self::CirculatingSupply | Self::Staked | Self::Burned | Self::Preminted => {} + } + [Self::CirculatingSupply, Self::Staked, Self::Burned, Self::Preminted] + }; } impl FromStr for CoinOrTokenStatistic { diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index dce0cc6e56..7db23ec068 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -548,6 +548,9 @@ async fn coin_holders(#[case] seed: Seed) { let body: serde_json::Value = serde_json::from_str(&response.text().await.unwrap()).unwrap(); let num_holders = body["items"].as_array().unwrap().len(); assert!(num_holders >= 2); + // the baseline is fetched with the server's maximum page size; a larger list + // would be silently truncated and the bound below would be wrong + assert!(num_holders < 100); // walk the whole holders list, one item per request let mut url = "/api/v2/statistics/coin/holders?items=1".to_owned(); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b579623ced..9096e4cd35 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -787,6 +787,13 @@ pub async fn transactions( ApiServerWebServerClientError::InvalidCursor, ) })?; + // reject out-of-range indices as a client error before they reach the + // storage layer, which maps the overflow to an internal server error + if i64::try_from(tx_global_index).is_err() { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } db_tx .get_transactions_with_block_info_before_tx_global_index( @@ -2333,6 +2340,12 @@ pub async fn order_pair( /// rational number, in atoms); `amount` is the remaining base amount at the level. The /// ask book is ordered by the ascending price, the bid book by the descending price; /// keyset (cursor) pagination over the levels. +/// +/// The levels are computed from a fresh storage snapshot on every request, so a +/// paginated walk is not a consistent snapshot of a moving book: a level whose +/// remaining balance changes after a cursor was issued can be skipped or duplicated +/// on the next page. Clients needing a consistent view should re-fetch from the +/// start (an empty cursor). pub async fn order_pair_book( Path(pair): Path, Query(params): Query>, From 674fab659e03a4f07bdfc35d4a7c3f1e1f215bb1 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 02:30:46 +0200 Subject: [PATCH 08/22] fix(indexer): endpoint-tagged cursors, invariants, docs - cursors carry an endpoint tag and reject unknown fields, so a cursor minted by one listing can no longer be decoded by another (with a test) - reduce_rational pins its zero-denominator invariant with a debug assertion - is_initialized documents that true does not mean the storage version is current; the caller must compare versions and reinitialize on mismatch --- .../src/storage/impls/postgres/queries.rs | 4 ++++ .../stack-test-suite/tests/v2/transactions.rs | 13 ++++++++++ api-server/web-server/src/api/cursor.rs | 23 ++++++++++++++---- api-server/web-server/src/api/v2.rs | 24 ++++++++++++++----- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index b3e53fb8f2..22927820cf 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -98,6 +98,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map_err(|_| ApiServerStorageError::TxGlobalIndexTooHigh(tx_global_index)) } + /// Whether the schema exists and carries a storage version. Note that a `true` + /// result does NOT mean the stored version is current: the caller must compare + /// [`Self::get_storage_version`] with the current version and reinitialize when + /// they differ (the scanner daemon does exactly that). pub async fn is_initialized(&mut self) -> Result { let query_str = Self::get_table_exists_query("misc_data"); let row_count = self diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index bf4a991dbb..14c765bd8e 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -445,5 +445,18 @@ async fn cursor_pagination(#[case] seed: Seed) { assert_eq!(body["error"].as_str().unwrap(), "Invalid number of items"); } + // a cursor minted by another endpoint is rejected instead of silently misread + { + let (status, tx_body) = get_json("/api/v2/transaction?items=1&cursor=".to_owned()).await; + assert_eq!(status, 200); + let tx_body: serde_json::Value = serde_json::from_str(&tx_body).unwrap(); + let foreign_cursor = tx_body["next_cursor"].as_str().unwrap(); + let (status, body) = + get_json(format!("/api/v2/pool?items=1&cursor={foreign_cursor}")).await; + assert_eq!(status, 400); + let body: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(body["error"].as_str().unwrap(), "Invalid cursor"); + } + shutdown_task(task).await; } diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs index 7e3eb77d49..69a4ea95ce 100644 --- a/api-server/web-server/src/api/cursor.rs +++ b/api-server/web-server/src/api/cursor.rs @@ -28,7 +28,11 @@ use serde::{Deserialize, Serialize}; use crate::error::ApiServerWebServerClientError; #[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Cursor { + /// Identifier of the endpoint (and sort schema) the cursor belongs to; a cursor + /// minted by another endpoint is rejected instead of silently misread. + tag: String, /// Sort keys of the item the cursor points to. keys: Vec, /// Unique tie-break id of the item. @@ -36,12 +40,23 @@ pub struct Cursor { } impl Cursor { - pub fn new(keys: Vec, id: String) -> Self { - Self { keys, id } + pub fn new(tag: &str, keys: Vec, id: String) -> Self { + Self { + tag: tag.to_owned(), + keys, + id, + } } - pub fn into_parts(self) -> (Vec, String) { - (self.keys, self.id) + /// Checks the endpoint tag and returns the sort keys and the tie-break id. + pub fn into_parts( + self, + tag: &str, + ) -> Result<(Vec, String), ApiServerWebServerClientError> { + if self.tag != tag { + return Err(ApiServerWebServerClientError::InvalidCursor); + } + Ok((self.keys, self.id)) } pub fn encode(&self) -> String { diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 9096e4cd35..b4b33b2b29 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -776,7 +776,7 @@ pub async fn transactions( let (txs, next_cursor) = if let Some(cursor_str) = cursor_param { let txs = match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, _tie_break_id) = cursor.into_parts(); + let (keys, _tie_break_id) = cursor.into_parts("transactions")?; let tx_global_index = keys.first().map(|index| index.to_owned()).ok_or( ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, @@ -821,6 +821,7 @@ pub async fn transactions( let next_cursor = has_next_page.then(|| { let last_returned = &txs[offset_and_items.items as usize - 1]; Cursor::new( + "transactions", vec![last_returned.tx_global_index.to_string()], last_returned.tx_info.tx.transaction().get_id().to_hash().encode_hex::(), ) @@ -1442,7 +1443,7 @@ pub async fn pools( let pools_json = if let Some(cursor_str) = cursor_param { let cursor = match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, id) = cursor.into_parts(); + let (keys, id) = cursor.into_parts("pools")?; match keys.first() { Some(creation_height) if !id.is_empty() => { @@ -1499,6 +1500,7 @@ pub async fn pools( .as_str() .to_owned(); Some(Cursor::new( + "pools", vec![creation_height.into_int().to_string()], pool_id_str, )) @@ -1923,7 +1925,7 @@ async fn holders_response( let cursor = match params.get(CURSOR) { Some(cursor_str) => match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, tie_break_id) = cursor.into_parts(); + let (keys, tie_break_id) = cursor.into_parts("holders")?; let amount = match keys.first() { Some(atoms) => u128::from_str(atoms).map(Amount::from_atoms).map_err(|_| { ApiServerWebServerError::ClientError( @@ -1974,7 +1976,11 @@ async fn holders_response( let next_cursor = has_next_page.then(|| { let (address, amount) = holders.last().expect("at least one item"); - Cursor::new(vec![amount.into_atoms().to_string()], address.clone()) + Cursor::new( + "holders", + vec![amount.into_atoms().to_string()], + address.clone(), + ) }); let items = holders @@ -2400,7 +2406,7 @@ pub async fn order_pair_book( let cursor_price = match params.get(CURSOR) { Some(cursor_str) => match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, _tie_break_id) = cursor.into_parts(); + let (keys, _tie_break_id) = cursor.into_parts("book")?; let parse_key = |key: Option<&String>| { key.and_then(|key| u128::from_str(key).ok()).ok_or( ApiServerWebServerError::ClientError( @@ -2486,6 +2492,7 @@ pub async fn order_pair_book( let next_cursor = has_next_page.then(|| { let (price, _amount) = levels.last().expect("at least one level"); Cursor::new( + "book", vec![price.numer().to_string(), price.denom().to_string()], String::new(), ) @@ -2533,7 +2540,7 @@ pub async fn order_pair_book( /// The exact decimal representation of a price given as `numer/denom` quote atoms per /// base atom: `numer * 10^base_decimals / denom`, displayed with `quote_decimals` -/// fractional digits. Falls back to an f64 only if the exact mantissa exceeds u128. +/// fractional digits, computed in exact u256 arithmetic (no floating point). fn price_to_decimal_string( numer: u128, denom: u128, @@ -2585,6 +2592,11 @@ fn price_to_decimal_string( fn reduce_rational(rational: Rational) -> Rational { let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); + // the caller skips zero-balance entries, so a zero denominator (a price that + // would compare equal to every other level and corrupt the ordering) must + // never reach this function; the debug assertion keeps the two invariants + // tied together + debug_assert_ne!(denom, 0); if denom == 0 { return rational; } From 165afb789c7aa1cfc765afb264da6c12b31b1053 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 02:53:55 +0200 Subject: [PATCH 09/22] fix(indexer): pool cursor bounds, in-memory error paths, sort clones - a pools cursor height that does not fit an i64 is rejected as an invalid cursor (client error) before reaching the storage layer - the in-memory sort-key helpers propagate an encoding error instead of panicking on a mismatched chain config, matching the postgres backend - the decorated sorts compare references instead of cloning string keys --- .../src/storage/impls/in_memory/mod.rs | 91 ++++++++++--------- api-server/web-server/src/api/v2.rs | 7 ++ 2 files changed, 55 insertions(+), 43 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 0de0552b8f..f4476d2af6 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -122,19 +122,17 @@ impl ApiServerInMemoryStorage { /// Encoded (bech32) form of a pool id — the exact string the postgres backend /// stores and orders by, keeping tie-break orders identical across backends. - fn pool_id_sort_key(&self, pool_id: &PoolId) -> String { + fn pool_id_sort_key(&self, pool_id: &PoolId) -> Result { Address::new(&self.chain_config, *pool_id) - .expect("encoding of a pool id stored in the in-memory storage cannot fail") - .as_str() - .to_owned() + .map_err(|_| ApiServerStorageError::AddressableError) + .map(|address| address.as_str().to_owned()) } /// Encoded (bech32) form of an order id — see `pool_id_sort_key`. - fn order_id_sort_key(&self, order_id: &OrderId) -> String { + fn order_id_sort_key(&self, order_id: &OrderId) -> Result { Address::new(&self.chain_config, *order_id) - .expect("encoding of an order id stored in the in-memory storage cannot fail") - .as_str() - .to_owned() + .map_err(|_| ApiServerStorageError::AddressableError) + .map(|address| address.as_str().to_owned()) } fn is_initialized(&self) -> Result { @@ -596,15 +594,17 @@ impl ApiServerInMemoryStorage { let mut decorated: Vec<(String, Amount, Amount)> = entries .into_iter() .map(|(order_id, order)| { - ( - self.order_id_sort_key(order_id), + Ok(( + self.order_id_sort_key(order_id)?, order.ask_balance, order.give_balance, - ) + )) }) - .collect(); - decorated.sort_by_key(|(key, ask_balance, _give_balance)| { - (Reverse(ask_balance.into_atoms()), Reverse(key.clone())) + .collect::, ApiServerStorageError>>()?; + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(key_a, ask_a, _), (key_b, ask_b, _)| { + (Reverse(ask_a.into_atoms()), Reverse(&**key_a)) + .cmp(&(Reverse(ask_b.into_atoms()), Reverse(&**key_b))) }); let has_more = decorated.len() > ORDER_BOOK_MAX_ORDERS; decorated.truncate(ORDER_BOOK_MAX_ORDERS); @@ -633,19 +633,21 @@ impl ApiServerInMemoryStorage { .map(|(pool_id, by_height)| { let created_height = *by_height.keys().next().expect("not empty"); let latest_data = by_height.values().last().expect("not empty").clone(); - ( + Ok(( created_height, - self.pool_id_sort_key(pool_id), + self.pool_id_sort_key(pool_id)?, *pool_id, latest_data, - ) - }) - .filter(|(_height, _key, _pool_id, data)| { - data.staker_balance().is_ok_and(|b| b != Amount::ZERO) + )) }) - .collect(); - decorated - .sort_by_key(|(height, key, _pool_id, _data)| (Reverse(*height), Reverse(key.clone()))); + .collect::, ApiServerStorageError>>()?; + decorated.retain(|(_height, _key, _pool_id, data)| { + data.staker_balance().is_ok_and(|b| b != Amount::ZERO) + }); + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { + (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) + }); if offset >= decorated.len() { return Ok(vec![]); } @@ -673,22 +675,27 @@ impl ApiServerInMemoryStorage { .map(|(pool_id, by_height)| { let created_height = *by_height.keys().next().expect("not empty"); let latest_data = by_height.values().last().expect("not empty").clone(); - ( + Ok(( created_height, - self.pool_id_sort_key(pool_id), + self.pool_id_sort_key(pool_id)?, *pool_id, latest_data, - ) - }) - .filter(|(_height, _key, _pool_id, data)| { - data.staker_balance().is_ok_and(|b| b != Amount::ZERO) + )) }) - .collect(); - decorated - .sort_by_key(|(height, key, _pool_id, _data)| (Reverse(*height), Reverse(key.clone()))); - let cursor_key = cursor.map(|(cursor_height, cursor_pool_id)| { - (cursor_height, self.pool_id_sort_key(&cursor_pool_id)) + .collect::, ApiServerStorageError>>()?; + decorated.retain(|(_height, _key, _pool_id, data)| { + data.staker_balance().is_ok_and(|b| b != Amount::ZERO) }); + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { + (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) + }); + let cursor_key = match cursor { + Some((cursor_height, cursor_pool_id)) => { + Some((cursor_height, self.pool_id_sort_key(&cursor_pool_id)?)) + } + None => None, + }; decorated.retain(|(height, key, _pool_id, _data)| match &cursor_key { Some((cursor_height, cursor_pool_key)) => { *height < *cursor_height || (*height == *cursor_height && *key < *cursor_pool_key) @@ -721,16 +728,14 @@ impl ApiServerInMemoryStorage { .filter_map(|(pool_id, by_height)| { let data = by_height.values().last()?.clone(); let balance = data.staker_balance().ok()?; - (balance != Amount::ZERO).then_some(( - balance, - self.pool_id_sort_key(pool_id), - *pool_id, - data, - )) + (balance != Amount::ZERO) + .then(|| Ok((balance, self.pool_id_sort_key(pool_id)?, *pool_id, data))) }) - .collect(); - decorated.sort_by_key(|(balance, key, _pool_id, _data)| { - (Reverse(balance.into_atoms()), Reverse(key.clone())) + .collect::, ApiServerStorageError>>()?; + // comparing references avoids re-allocating sort keys + decorated.sort_by(|(balance_a, key_a, _, _), (balance_b, key_b, _, _)| { + (Reverse(balance_a.into_atoms()), Reverse(&**key_a)) + .cmp(&(Reverse(balance_b.into_atoms()), Reverse(&**key_b))) }); if offset >= decorated.len() { return Ok(vec![]); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index b4b33b2b29..4e642d4afa 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1452,6 +1452,13 @@ pub async fn pools( ApiServerWebServerClientError::InvalidCursor, ) })?; + // reject out-of-range heights as a client error before they + // reach the storage layer, which maps the overflow to a panic + if i64::try_from(creation_height).is_err() { + return Err(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + )); + } let pool_id = Address::from_string(&state.chain_config, &id) .map_err(|_| { ApiServerWebServerError::ClientError( From 2c74e885b79b590ec1e43f03335fbc3c97453b7b Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 03:12:22 +0200 Subject: [PATCH 10/22] chore(indexer): trigger final review pass From 6ee00f28ddd59f71883032138efbf1867d0fd8df Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sat, 3 Oct 2026 08:45:15 +0200 Subject: [PATCH 11/22] fix(indexer): correct license header in cursor module --- api-server/web-server/src/api/cursor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs index 69a4ea95ce..78d0b8b25d 100644 --- a/api-server/web-server/src/api/cursor.rs +++ b/api-server/web-server/src/api/cursor.rs @@ -5,7 +5,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://opensource.org/licenses/MIT +// https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, From 84847e0c941c134071a8ef63bc733603594274d5 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 12:33:41 +0200 Subject: [PATCH 12/22] fix(api-server): canonicalize holders cursors, dedupe pool sort - Canonicalize the address id when minting and decoding holders cursors: Address::from_string keeps the input verbatim and bech32 accepts an all-uppercase encoding, which would corrupt the byte-order keyset comparison. Re-encode via Address::new like the pools listing does. - Extract sorted_live_pools() shared by the offset and keyset pool listings, removing the duplicated decorate/filter/sort block, and document the in-memory backend's full-scan cost characteristics. - Comment the intentional <= conflict guard on the latest_pool_data_cache upsert (same-height reorg replacement). - Document the items=0 -> 400 behavior change across paginated v2 endpoints in the changelog, and that the order book truncated flag is informational (levels beyond the cap are unreachable by cursor walk). --- api-server/CHANGELOG.md | 1 + .../src/storage/impls/in_memory/mod.rs | 58 +++++++++---------- .../src/storage/impls/postgres/queries.rs | 5 ++ api-server/web-server/src/api/v2.rs | 42 +++++++++++--- 4 files changed, 67 insertions(+), 39 deletions(-) diff --git a/api-server/CHANGELOG.md b/api-server/CHANGELOG.md index f3d88a2211..c5b77528a7 100644 --- a/api-server/CHANGELOG.md +++ b/api-server/CHANGELOG.md @@ -16,6 +16,7 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/ Transactions seen in the node's mempool are bridged into `tx_seen` events by the web server. ### Changed +- The `items` query parameter is now validated the same way on every paginated `v2` endpoint: `items=0` is rejected with `400 invalid num items` instead of returning an empty page. Previously the offset-based endpoints returned an empty listing for `items=0`; the new keyset-paginated endpoints (pools, transactions, holders, order book) reject it from the start. - The api-server storage version was bumped from 25 to 26 (new `ml.emitted_events` table); the scanner re-initializes the database when it finds a different version, as before. Full resync is required. - The stream event retention pruning no longer deletes events that the event pump has not consumed yet: the pump records its progress in the database and the pruning never overtakes it (with a hard limit of 100k retained events before the first progress record or during a pump outage, logged as an error when it kicks in). - A terminated streaming background task (the event pump or the mempool bridge) now brings the web server process down instead of leaving the event stream silently dead while the REST endpoints keep working. diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index f4476d2af6..6f7407e86c 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -61,6 +61,13 @@ impl Ord for TokenTransactionOrderedByTxId { } #[derive(Debug, Clone)] +/// An in-memory api-server storage, meant for tests. +/// +/// The listing reads (pool listings, top holders, order book) scan and sort their full +/// tables on every call instead of maintaining the incrementally-updated caches the +/// postgres backend has (`ml.latest_pool_data_cache`, `ml.address_amount`, +/// `ml.latest_orders_cache`); the contracts (ordering, filters, caps) are identical, +/// only the asymptotics differ, which is acceptable for test-sized data. struct ApiServerInMemoryStorage { block_table: BTreeMap, BlockWithExtraData>, block_aux_data_table: BTreeMap, BlockAuxData>, @@ -618,15 +625,14 @@ impl ApiServerInMemoryStorage { )) } - fn get_latest_pool_ids( + /// The live pools (non-zero pledge), newest creation height first, ties broken by + /// the encoded pool address (byte order), exactly like the postgres listing; the + /// sort key is encoded once per entry. Shared by the offset and the keyset + /// (cursor) listings so their ordering cannot diverge. + fn sorted_live_pools( &self, - len: u32, - offset: u64, - ) -> Result, ApiServerStorageError> { - let len = len as usize; - let offset = offset as usize; - // newest first, ties broken by the encoded pool address (byte order), exactly - // like the postgres listing; the sort key is encoded once per entry + ) -> Result, ApiServerStorageError> + { let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() @@ -648,6 +654,17 @@ impl ApiServerInMemoryStorage { decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) }); + Ok(decorated) + } + + fn get_latest_pool_ids( + &self, + len: u32, + offset: u64, + ) -> Result, ApiServerStorageError> { + let len = len as usize; + let offset = offset as usize; + let decorated = self.sorted_live_pools()?; if offset >= decorated.len() { return Ok(vec![]); } @@ -666,30 +683,7 @@ impl ApiServerInMemoryStorage { cursor: Option<(BlockHeight, PoolId)>, ) -> Result, ApiServerStorageError> { let len = len as usize; - // newest first, ties broken by the encoded pool address (byte order) so the - // keyset cursor is deterministic and backend-independent; the sort key is - // encoded once per entry - let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self - .pool_data_table - .iter() - .map(|(pool_id, by_height)| { - let created_height = *by_height.keys().next().expect("not empty"); - let latest_data = by_height.values().last().expect("not empty").clone(); - Ok(( - created_height, - self.pool_id_sort_key(pool_id)?, - *pool_id, - latest_data, - )) - }) - .collect::, ApiServerStorageError>>()?; - decorated.retain(|(_height, _key, _pool_id, data)| { - data.staker_balance().is_ok_and(|b| b != Amount::ZERO) - }); - // comparing references avoids re-allocating sort keys - decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { - (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) - }); + let mut decorated = self.sorted_live_pools()?; let cursor_key = match cursor { Some((cursor_height, cursor_pool_id)) => { Some((cursor_height, self.pool_id_sort_key(&cursor_pool_id)?)) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 22927820cf..66e619747c 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -2216,6 +2216,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // maintain the latest-state cache used by the pool listings; the creation height // is the height of the first write of the pool (the scanner writes pools from // genesis, so the first write is the creation) and is not modified on conflict. + // + // The `<=` (not `<`) guard intentionally lets a same-height write win, which a + // same-height reorg needs to replace the cached row. This relies on the scanner + // writing each pool's state for a given height exactly once per reorg state, so + // an equal-height conflict can only ever be that replacement. self.tx .execute( r#" diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 4e642d4afa..e328527d96 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1917,6 +1917,18 @@ pub async fn token_statistics( }))) } +/// Re-encode an address string in its canonical bech32 form. `Address::from_string` +/// keeps the input verbatim, so an all-uppercase encoding would validate fine yet break +/// the byte-order keyset comparison against the canonical lowercase strings in storage; +/// re-encoding (as the pools listing does for pool ids) removes that. Returns `None` if +/// the string is not a valid address. +fn canonical_address_string(chain_config: &ChainConfig, address: &str) -> Option { + let parsed = Address::::from_string(chain_config, address).ok()?; + Address::new(chain_config, parsed.into_object()) + .ok() + .map(|address| address.into_string()) +} + /// Shared implementation of the holders endpoints: the top holders of an asset, /// ordered by the balance, with optional keyset (cursor) pagination. async fn holders_response( @@ -1945,14 +1957,24 @@ async fn holders_response( )); } }; - // the tie-break is an encoded address; reject garbage up front - let _validated = + // the tie-break is an encoded address; reject garbage up front and + // canonicalize the encoding: `from_string` keeps the input verbatim, so + // an all-uppercase cursor would otherwise be compared byte-wise against + // the canonical lowercase strings in storage and mis-order the keyset + let validated = Address::::from_string(&state.chain_config, &tie_break_id) .map_err(|_| { ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, ) })?; + let tie_break_id = Address::new(&state.chain_config, validated.into_object()) + .map_err(|_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + })? + .into_string(); Some((amount, tie_break_id)) } None => None, @@ -1983,11 +2005,12 @@ async fn holders_response( let next_cursor = has_next_page.then(|| { let (address, amount) = holders.last().expect("at least one item"); - Cursor::new( - "holders", - vec![amount.into_atoms().to_string()], - address.clone(), - ) + // re-encode so the cursor id is canonical and guaranteed to round-trip through + // the decode-side validation; fall back to the stored string (canonical in + // practice) rather than failing the listing on a storage anomaly + let address = canonical_address_string(&state.chain_config, address) + .unwrap_or_else(|| address.clone()); + Cursor::new("holders", vec![amount.into_atoms().to_string()], address) }); let items = holders @@ -2359,6 +2382,11 @@ pub async fn order_pair( /// remaining balance changes after a cursor was issued can be skipped or duplicated /// on the next page. Clients needing a consistent view should re-fetch from the /// start (an empty cursor). +/// +/// The levels are aggregated from at most `ORDER_BOOK_MAX_ORDERS` orders per request +/// (the additive `truncated` flag reports when that cap was hit). The flag is +/// informational: levels beyond the cap cannot be reached by continuing the cursor +/// walk. pub async fn order_pair_book( Path(pair): Path, Query(params): Query>, From 2bcd1b3fc3f4d2bf9549a1174c1fc3b10fc6c5d9 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 13:37:33 +0200 Subject: [PATCH 13/22] fix(api-server): side-scoped book cursors, envelope pool upsert Make the order book cursor tag side-specific so a cursor minted for one side is rejected on the other, guard next-cursor minting against an empty page instead of expecting, log on the zero-denominator price fallback, document the floored decimal price and the ignored transaction cursor tie-break, and use expect with context in tests. The postgres pool cache upsert becomes a monotone envelope update (min creation height, max latest state) so late lower-height writes correct the creation height like the in-memory backend and the rollback path; covered by a new shared pool_data_keyset block and a rollback-coupling note in top_address_amounts. --- .../src/storage/impls/postgres/queries.rs | 37 ++++++++++------ .../stack-test-suite/tests/v2/statistics.rs | 11 +++-- .../stack-test-suite/tests/v2/transactions.rs | 5 ++- api-server/storage-test-suite/src/basic.rs | 38 ++++++++++++---- api-server/web-server/src/api/v2.rs | 43 +++++++++++++++---- 5 files changed, 100 insertions(+), 34 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 66e619747c..5072ef66e1 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -388,7 +388,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // roll back the holders table rows of the disconnected blocks; set-based, in a // constant number of round trips: delete the rolled-back rows, drop rows whose // whole versioned history was orphaned, then re-derive and re-insert the rest - // in bulk (the numeric amount is decoded from the versioned balances) + // in bulk (the numeric amount is decoded from the versioned balances). + // NOTE: this re-derivation is only correct because the versioned + // `ml.address_balance` rows above the height were deleted by the statement + // above, within this same call; the ordering of the two statements is + // load-bearing. `top_address_amounts` in the storage test suite guards this by + // rolling back through this function and asserting the re-derived listing. let deleted_holders_rows = self .tx .query( @@ -2213,24 +2218,30 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; - // maintain the latest-state cache used by the pool listings; the creation height - // is the height of the first write of the pool (the scanner writes pools from - // genesis, so the first write is the creation) and is not modified on conflict. - // - // The `<=` (not `<`) guard intentionally lets a same-height write win, which a - // same-height reorg needs to replace the cached row. This relies on the scanner - // writing each pool's state for a given height exactly once per reorg state, so - // an equal-height conflict can only ever be that replacement. + // maintain the latest-state cache used by the pool listings as the envelope of + // the versioned table: the creation height is the lowest height the pool was + // ever written at, the latest state is the highest one (in the normal + // genesis-ascending scan that is simply the first and the last write). The + // update is monotone — the same-height write of a reorg replaces the cached + // state (the `>=` CASE branches), a late lower-height write only corrects the + // creation height — so a pool first seen mid-history (a re-seed that started at + // a higher height) gets its creation height corrected when the lower rows + // arrive, matching the `MIN(block_height)` re-derivation of the rollback path + // and the in-memory backend. self.tx .execute( r#" INSERT INTO ml.latest_pool_data_cache (pool_id, block_height, creation_block_height, staker_balance, data) VALUES ($1, $2, $2, $3::text::numeric, $4) ON CONFLICT (pool_id) DO UPDATE - SET block_height = EXCLUDED.block_height, - staker_balance = EXCLUDED.staker_balance, - data = EXCLUDED.data - WHERE ml.latest_pool_data_cache.block_height <= EXCLUDED.block_height; + SET block_height = GREATEST(ml.latest_pool_data_cache.block_height, EXCLUDED.block_height), + creation_block_height = LEAST(ml.latest_pool_data_cache.creation_block_height, EXCLUDED.creation_block_height), + staker_balance = CASE + WHEN EXCLUDED.block_height >= ml.latest_pool_data_cache.block_height THEN EXCLUDED.staker_balance + ELSE ml.latest_pool_data_cache.staker_balance END, + data = CASE + WHEN EXCLUDED.block_height >= ml.latest_pool_data_cache.block_height THEN EXCLUDED.data + ELSE ml.latest_pool_data_cache.data END; "#, &[&pool_id.as_str(), &height, &amount_str, &pool_data.encode()], ) diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 7db23ec068..6a8818cb9d 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -550,7 +550,10 @@ async fn coin_holders(#[case] seed: Seed) { assert!(num_holders >= 2); // the baseline is fetched with the server's maximum page size; a larger list // would be silently truncated and the bound below would be wrong - assert!(num_holders < 100); + assert!( + num_holders < 100, + "the baseline fetch was truncated by the server page cap (MAX_NUM_ITEMS)" + ); // walk the whole holders list, one item per request let mut url = "/api/v2/statistics/coin/holders?items=1".to_owned(); @@ -580,9 +583,11 @@ async fn coin_holders(#[case] seed: Seed) { // the amounts are ordered, from the biggest to the smallest if let Some(prev) = prev_amount.replace(amount.clone()) { + // the amounts are encoded as integer atom strings by `amount_to_json`; a + // parse failure means that serialization contract changed let (prev, amount) = ( - prev.parse::().unwrap(), - amount.parse::().unwrap(), + prev.parse::().expect("atoms is an integer string"), + amount.parse::().expect("atoms is an integer string"), ); assert!(prev >= amount); } diff --git a/api-server/stack-test-suite/tests/v2/transactions.rs b/api-server/stack-test-suite/tests/v2/transactions.rs index 14c765bd8e..c788fbd8fd 100644 --- a/api-server/stack-test-suite/tests/v2/transactions.rs +++ b/api-server/stack-test-suite/tests/v2/transactions.rs @@ -393,7 +393,10 @@ async fn cursor_pagination(#[case] seed: Seed) { let num_tx = expected_txs.len(); // the baseline is fetched with the server's maximum page size; a larger list // would be silently truncated and the comparison below would be vacuous - assert!(num_tx < 100); + assert!( + num_tx < 100, + "the baseline fetch was truncated by the server page cap (MAX_NUM_ITEMS)" + ); // the same list, walked with the cursor; the walk is bounded by the known // baseline length so a self-referencing cursor fails instead of hanging diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 3bd885c2c3..0a5289a4ed 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2270,17 +2270,34 @@ where ]) ); + // a late lower-height write corrects the creation height to the true minimum: the + // backends agree on min-height semantics (matching the `MIN(block_height)` + // re-derivation of the rollback path), which keeps a mid-history re-seed from + // silently mis-ordering the listing. The latest data is still the highest height's. + let pool2_early = make_pool_data(&mut rng, Amount::from_atoms(222)); + let h5 = BlockHeight::new(5); + db_tx.set_pool_data_at_height(pool2_id, &pool2_early, h5).await.unwrap(); + let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); + assert_eq!( + all, + expect(&[ + (h30, pool3_id, &pool3_data), + (h10, pool1_id, &pool1_updated), + (h5, pool2_id, &pool2_data), + ]) + ); + // cursor pagination let page1 = db_tx.get_latest_pool_data_before(2, None).await.unwrap(); assert_eq!( page1, - expect(&[(h30, pool3_id, &pool3_data), (h20, pool2_id, &pool2_data)]) + expect(&[(h30, pool3_id, &pool3_data), (h10, pool1_id, &pool1_updated)]) ); - let page2 = db_tx.get_latest_pool_data_before(2, Some((h20, pool2_id))).await.unwrap(); - assert_eq!(page2, expect(&[(h10, pool1_id, &pool1_updated)])); + let page2 = db_tx.get_latest_pool_data_before(2, Some((h10, pool1_id))).await.unwrap(); + assert_eq!(page2, expect(&[(h5, pool2_id, &pool2_data)])); - let end = db_tx.get_latest_pool_data_before(2, Some((h10, pool1_id))).await.unwrap(); + let end = db_tx.get_latest_pool_data_before(2, Some((h5, pool2_id))).await.unwrap(); assert!(end.is_empty()); // pools with a zero staker balance (decommissioned) are not listed @@ -2292,19 +2309,19 @@ where let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); assert_eq!( all, - expect(&[(h20, pool2_id, &pool2_data), (h10, pool1_id, &pool1_updated)]) + expect(&[(h10, pool1_id, &pool1_updated), (h5, pool2_id, &pool2_data)]) ); // the rollback restores the latest data below the deleted height, keeping the - // original creation height + // original (minimum) creation height; the late h5 write of pool2 survives db_tx.del_pools_above_height(h30).await.unwrap(); let all = db_tx.get_latest_pool_data_before(10, None).await.unwrap(); assert_eq!( all, expect(&[ (h30, pool3_id, &pool3_data), - (h20, pool2_id, &pool2_data), - (h10, pool1_id, &pool1_data) + (h10, pool1_id, &pool1_data), + (h5, pool2_id, &pool2_data), ]) ); @@ -2467,7 +2484,10 @@ where ] ); - // the rollback restores the previous balances + // the rollback restores the previous balances; this also guards the ordering + // coupling in `del_address_balance_above_height`: the holders cache + // re-derivation is only correct because the versioned balances above the height + // are deleted first, within the same call db_tx.del_address_balance_above_height(h1).await.unwrap(); let all = db_tx.get_top_address_amounts(CoinOrTokenId::Coin, 10, None).await.unwrap(); assert_eq!( diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index e328527d96..854f2e8464 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -776,6 +776,10 @@ pub async fn transactions( let (txs, next_cursor) = if let Some(cursor_str) = cursor_param { let txs = match Cursor::decode_start(cursor_str)? { Some(cursor) => { + // the tie-break id of the cursor is ignored for this tag: the global + // transaction index is unique per transaction, so the index key alone is + // already a total order (the pools and holders endpoints need the + // tie-break because their amounts can repeat) let (keys, _tie_break_id) = cursor.into_parts("transactions")?; let tx_global_index = keys.first().map(|index| index.to_owned()).ok_or( ApiServerWebServerError::ClientError( @@ -1492,9 +1496,12 @@ pub async fn pools( let has_next_page = pools.len() > offset_and_items.items as usize; let pools = pools.into_iter().take(offset_and_items.items as usize).collect::>(); - let next_cursor = if has_next_page { + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking + // should a future caller ever bypass it + let next_cursor = if has_next_page && !pools.is_empty() { let (creation_height, pool_id, _pool_data) = - pools.last().expect("a next page implies a last item"); + pools.last().expect("the page is not empty"); // the pool id comes from the storage, so encoding cannot fail; map the // error anyway instead of risking a panic inside a request handler let pool_id_str = Address::new(&state.chain_config, *pool_id) @@ -2003,8 +2010,11 @@ async fn holders_response( let has_next_page = holders.len() > offset_and_items.items as usize; holders.truncate(offset_and_items.items as usize); - let next_cursor = has_next_page.then(|| { - let (address, amount) = holders.last().expect("at least one item"); + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking should a + // future caller ever bypass it + let next_cursor = (has_next_page && !holders.is_empty()).then(|| { + let (address, amount) = holders.last().expect("the page is not empty"); // re-encode so the cursor id is canonical and guaranteed to round-trip through // the decode-side validation; fall back to the stored string (canonical in // practice) rather than failing the listing on a storage anomaly @@ -2387,6 +2397,10 @@ pub async fn order_pair( /// (the additive `truncated` flag reports when that cap was hit). The flag is /// informational: levels beyond the cap cannot be reached by continuing the cursor /// walk. +/// +/// The `price.decimal` field is truncated (floored) toward zero; the exact price is +/// available in the `price.atoms` field. Cursors are side-specific: one minted for +/// `side=ask` is rejected with `invalid cursor` on `side=bid` and vice versa. pub async fn order_pair_book( Path(pair): Path, Query(params): Query>, @@ -2435,13 +2449,17 @@ pub async fn order_pair_book( } }; + // the cursor tag is side-specific: the same price walks in opposite directions on + // the two sides, so a cursor minted for one side must be rejected on the other + let cursor_tag = if base_is_ask { "book-ask" } else { "book-bid" }; + let offset_and_items = get_offset_and_items(¶ms)?; // the cursor is the (numerator, denominator) of the last returned level price let cursor_price = match params.get(CURSOR) { Some(cursor_str) => match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, _tie_break_id) = cursor.into_parts("book")?; + let (keys, _tie_break_id) = cursor.into_parts(cursor_tag)?; let parse_key = |key: Option<&String>| { key.and_then(|key| u128::from_str(key).ok()).ok_or( ApiServerWebServerError::ClientError( @@ -2524,10 +2542,13 @@ pub async fn order_pair_book( let has_next_page = levels.len() > offset_and_items.items as usize; levels.truncate(offset_and_items.items as usize); - let next_cursor = has_next_page.then(|| { - let (price, _amount) = levels.last().expect("at least one level"); + // a next page implies a non-empty page, unless `items` was zero; the shared + // validator rejects that, but degrade to "no cursor" instead of panicking should a + // future caller ever bypass it + let next_cursor = (has_next_page && !levels.is_empty()).then(|| { + let (price, _amount) = levels.last().expect("the page is not empty"); Cursor::new( - "book", + cursor_tag, vec![price.numer().to_string(), price.denom().to_string()], String::new(), ) @@ -2576,6 +2597,9 @@ pub async fn order_pair_book( /// The exact decimal representation of a price given as `numer/denom` quote atoms per /// base atom: `numer * 10^base_decimals / denom`, displayed with `quote_decimals` /// fractional digits, computed in exact u256 arithmetic (no floating point). +/// +/// The result is truncated (floored) toward zero, so it can be slightly lower than the +/// exact level price; the exact value is available alongside in the `atoms` field. fn price_to_decimal_string( numer: u128, denom: u128, @@ -2633,6 +2657,9 @@ fn reduce_rational(rational: Rational) -> Rational { // tied together debug_assert_ne!(denom, 0); if denom == 0 { + // the caller skips zero-balance entries, so reaching this branch means that + // invariant was broken elsewhere; log it instead of silently degrading + logging::log::error!("order book price with a zero denominator reached reduction"); return rational; } while denom != 0 { From 2c5ecbbaacdf529cfe5abd4b517d066bc02af48d Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 13:49:11 +0200 Subject: [PATCH 14/22] fix(api-server): cursor cap, pool filter order, holders error path Cap the encoded cursor length before the base64 decode, filter zero-balance pools before the bech32 sort-key encoding, compute the order book price scale by exponentiation, fail the holders listing with a server error when a stored address cannot be re-encoded canonically, collate the pool listing tie-break explicitly, and document the offset-mode items guard and the coin ticker case-insensitivity. No behavioral change for well-formed requests. --- .../src/storage/impls/in_memory/mod.rs | 23 +++++--- .../src/storage/impls/postgres/queries.rs | 2 +- api-server/web-server/src/api/cursor.rs | 10 ++++ api-server/web-server/src/api/v2.rs | 57 ++++++++++++++----- 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 6f7407e86c..69875e067d 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -636,20 +636,27 @@ impl ApiServerInMemoryStorage { let mut decorated: Vec<(BlockHeight, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() - .map(|(pool_id, by_height)| { - let created_height = *by_height.keys().next().expect("not empty"); - let latest_data = by_height.values().last().expect("not empty").clone(); + .filter_map(|(pool_id, by_height)| { + // a pool in the table always carries at least one versioned row; the + // cheap zero-balance filter runs before the (allocating) sort-key + // encoding, so decommissioned pools skip the bech32 work entirely + let created_height = *by_height.keys().next().expect("a pool has versioned rows"); + let latest_data = by_height.values().last().expect("a pool has versioned rows"); + latest_data.staker_balance().is_ok_and(|b| b != Amount::ZERO).then_some(( + *pool_id, + created_height, + latest_data.clone(), + )) + }) + .map(|(pool_id, created_height, latest_data)| { Ok(( created_height, - self.pool_id_sort_key(pool_id)?, - *pool_id, + self.pool_id_sort_key(&pool_id)?, + pool_id, latest_data, )) }) .collect::, ApiServerStorageError>>()?; - decorated.retain(|(_height, _key, _pool_id, data)| { - data.staker_balance().is_ok_and(|b| b != Amount::ZERO) - }); // comparing references avoids re-allocating sort keys decorated.sort_by(|(height_a, key_a, _, _), (height_b, key_b, _, _)| { (Reverse(*height_a), Reverse(&**key_a)).cmp(&(Reverse(*height_b), Reverse(&**key_b))) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 5072ef66e1..aa7ed76c03 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -2136,7 +2136,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { SELECT creation_block_height, pool_id, data FROM ml.latest_pool_data_cache WHERE staker_balance != 0 - ORDER BY creation_block_height DESC, pool_id DESC + ORDER BY creation_block_height DESC, pool_id COLLATE "C" DESC LIMIT $1; "#, &[&len], diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs index 78d0b8b25d..d585f384a9 100644 --- a/api-server/web-server/src/api/cursor.rs +++ b/api-server/web-server/src/api/cursor.rs @@ -27,6 +27,11 @@ use serde::{Deserialize, Serialize}; use crate::error::ApiServerWebServerClientError; +/// The largest accepted encoded cursor length. Legitimate cursors carry a handful of +/// short keys (well under this bound); the cap rejects oversized unauthenticated input +/// before the base64 decode. +const MAX_CURSOR_LEN: usize = 1024; + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Cursor { @@ -65,6 +70,11 @@ impl Cursor { } pub fn decode(encoded: &str) -> Result { + // a cursor carries a handful of short keys; reject oversized payloads before + // the base64 decode as a cheap guard against unauthenticated input bloat + if encoded.len() > MAX_CURSOR_LEN { + return Err(ApiServerWebServerClientError::InvalidCursor); + } let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD .decode(encoded) .map_err(|_| ApiServerWebServerClientError::InvalidCursor)?; diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 854f2e8464..022ba9d079 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -2013,15 +2013,29 @@ async fn holders_response( // a next page implies a non-empty page, unless `items` was zero; the shared // validator rejects that, but degrade to "no cursor" instead of panicking should a // future caller ever bypass it - let next_cursor = (has_next_page && !holders.is_empty()).then(|| { + let next_cursor = if has_next_page && !holders.is_empty() { let (address, amount) = holders.last().expect("the page is not empty"); // re-encode so the cursor id is canonical and guaranteed to round-trip through - // the decode-side validation; fall back to the stored string (canonical in - // practice) rather than failing the listing on a storage anomaly - let address = canonical_address_string(&state.chain_config, address) - .unwrap_or_else(|| address.clone()); - Cursor::new("holders", vec![amount.into_atoms().to_string()], address) - }); + // the decode-side validation; a stored address that fails to re-encode is a + // storage anomaly, surfaced as a server error instead of a cursor the client + // could not consume on the next page + let address = match canonical_address_string(&state.chain_config, address) { + Some(address) => address, + None => { + logging::log::error!("non-canonical address in the holders listing: {address}"); + return Err(ApiServerWebServerError::ServerError( + ApiServerWebServerServerError::InternalServerError, + )); + } + }; + Some(Cursor::new( + "holders", + vec![amount.into_atoms().to_string()], + address, + )) + } else { + None + }; let items = holders .into_iter() @@ -2412,6 +2426,9 @@ pub async fn order_pair_book( const SIDE_BID: &str = "bid"; let parse_currency = |s: &str| -> Result { + // the coin ticker is matched case-insensitively for client convenience, while + // token ids are exact bech32 strings (bech32 decoding is case-sensitive by + // design) — an accepted asymmetry, not an oversight if s.to_uppercase() == state.chain_config.coin_ticker() { Ok(CoinOrTokenId::Coin) } else { @@ -2610,14 +2627,25 @@ fn price_to_decimal_string( // digits are computed exactly in u256 arithmetic (the scaled numerator fits a // u256 for any realistic decimal count and is checked anyway), because a // financial price must not silently lose precision through floating point + // 10^base_decimals by exponentiation-by-squaring (`Uint256` has no pow); at most + // eight squarings for the maximum 255 decimals a token can declare + let overflow = || { + logging::log::error!("order book price scaling overflow"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + }; let ten = Uint256::from_u64(10); - let mut scaled = Uint256::from(numer); - for _ in 0..base_decimals { - scaled = scaled.checked_mul(&ten).ok_or_else(|| { - logging::log::error!("order book price scaling overflow"); - ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) - })?; + let (mut scale, mut base, mut exp) = (Uint256::from_u64(1), ten, u32::from(base_decimals)); + loop { + if exp & 1 == 1 { + scale = scale.checked_mul(&base).ok_or_else(overflow)?; + } + exp >>= 1; + if exp == 0 { + break; + } + base = base.checked_mul(&base).ok_or_else(overflow)?; } + let scaled = Uint256::from(numer).checked_mul(&scale).ok_or_else(overflow)?; let scaled = scaled.checked_div(&Uint256::from(denom)).expect("denominator is not zero"); // the decimal digits of the scaled price, least significant last, by repeated @@ -2728,6 +2756,9 @@ fn get_offset_and_items( ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) })? .unwrap_or(DEFAULT_NUM_ITEMS); + // `items=0` is rejected here for every endpoint using this parser, including the + // legacy offset-mode listings which used to return an empty page; the behavior + // change is noted in the CHANGELOG ensure!( items > 0, ApiServerWebServerError::ClientError(ApiServerWebServerClientError::InvalidNumItems) From dc2bdc25009db375af6bf26402696c23c678a684 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 13:59:21 +0200 Subject: [PATCH 15/22] fix(api-server): align holders cursor tuple, tighten listing guards Mirror the holders cursor tuple to the returned row shape (address, amount) across the trait, both backends and call sites, reuse canonical_address_string on the decode side, guard the reduce against a zero gcd, reject empty trading-pair segments with the accurate error, extend the pledge index with the tie-break column, skip empty versioned maps in the in-memory pool listing, and drop unused test bindings. No behavioral change for well-formed requests. --- .../src/storage/impls/in_memory/mod.rs | 15 ++++++---- .../impls/in_memory/transactional/read.rs | 2 +- .../impls/in_memory/transactional/write.rs | 2 +- .../src/storage/impls/postgres/queries.rs | 10 ++++--- .../impls/postgres/transactional/read.rs | 2 +- .../impls/postgres/transactional/write.rs | 2 +- .../src/storage/storage_api/mod.rs | 2 +- .../stack-test-suite/tests/v2/statistics.rs | 2 +- api-server/storage-test-suite/src/basic.rs | 4 +-- api-server/web-server/src/api/v2.rs | 30 ++++++++----------- 10 files changed, 37 insertions(+), 34 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 69875e067d..19d04ccc54 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -164,7 +164,7 @@ impl ApiServerInMemoryStorage { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { let mut holders = self .address_balance_table @@ -182,7 +182,7 @@ impl ApiServerInMemoryStorage { (r_amount, r_address.as_str()).cmp(&(l_amount, l_address.as_str())) }); - if let Some((cursor_amount, cursor_address)) = cursor { + if let Some((cursor_address, cursor_amount)) = cursor { holders.retain(|(address, amount)| { *amount < cursor_amount || (*amount == cursor_amount && address.as_str() < cursor_address.as_str()) @@ -637,9 +637,14 @@ impl ApiServerInMemoryStorage { .pool_data_table .iter() .filter_map(|(pool_id, by_height)| { - // a pool in the table always carries at least one versioned row; the - // cheap zero-balance filter runs before the (allocating) sort-key - // encoding, so decommissioned pools skip the bech32 work entirely + // a pool in the table always carries at least one versioned row; empty + // maps (never produced by the current write paths) are skipped instead + // of panicking in a read path, and the cheap zero-balance filter runs + // before the (allocating) sort-key encoding, so decommissioned pools + // skip the bech32 work entirely + if by_height.is_empty() { + return None; + } let created_height = *by_height.keys().next().expect("a pool has versioned rows"); let latest_data = by_height.values().last().expect("a pool has versioned rows"); latest_data.staker_balance().is_ok_and(|b| b != Amount::ZERO).then_some(( diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs index e0835f1890..865882ed35 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs @@ -50,7 +50,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) } diff --git a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs index 01d2f8f109..d82c1d3d98 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs @@ -335,7 +335,7 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { self.transaction.get_top_address_amounts(coin_or_token_id, len, cursor) } diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index aa7ed76c03..5843edcb52 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -195,12 +195,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { let coin_or_token_id = coin_or_token_id.encode(); let len = len as i64; let rows = match cursor { - Some((amount, address)) => { + Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); self.tx .query( @@ -1263,10 +1263,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) .await?; - // index for the pool listing sorted by pledge + // index for the pool listing sorted by pledge; includes the tie-break column + // so the listing's ORDER BY is index-satisfiable (matching + // latest_pool_data_created_index) self.just_execute( "CREATE INDEX latest_pool_data_staker_balance_index - ON ml.latest_pool_data_cache (staker_balance DESC);", + ON ml.latest_pool_data_cache (staker_balance DESC, pool_id DESC);", ) .await?; diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs index 2f6e81b339..7f133b11e0 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/read.rs @@ -68,7 +68,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRo<'_> { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); conn.get_top_address_amounts(coin_or_token_id, len, cursor).await diff --git a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs index 368638a3ef..63690bf4d0 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/transactional/write.rs @@ -469,7 +469,7 @@ impl ApiServerStorageRead for ApiServerPostgresTransactionalRw<'_> { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { let conn = QueryFromConnection::new(self.connection.as_ref().expect(CONN_ERR)); conn.get_top_address_amounts(coin_or_token_id, len, cursor).await diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 9889a426c4..fbfbde4370 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -626,7 +626,7 @@ pub trait ApiServerStorageRead: Sync { &self, coin_or_token_id: CoinOrTokenId, len: u32, - cursor: Option<(Amount, String)>, + cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError>; async fn get_address_balances( diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 6a8818cb9d..5ee2450a77 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -498,7 +498,7 @@ async fn coin_holders(#[case] seed: Seed) { (alice_address, bob_address, vec![block]) }; - let (_alice_address, _bob_address, chainstate_blocks) = chainstate_blocks; + let (_, _, chainstate_blocks) = chainstate_blocks; let storage = { let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 0a5289a4ed..1dd7f640b6 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2444,7 +2444,7 @@ where .get_top_address_amounts( CoinOrTokenId::Coin, 2, - Some((last_amount, last_address.clone())), + Some((last_address.clone(), last_amount)), ) .await .unwrap(); @@ -2458,7 +2458,7 @@ where let (last_address2, last_amount2) = next_page[1].clone(); let end = db_tx - .get_top_address_amounts(CoinOrTokenId::Coin, 2, Some((last_amount2, last_address2))) + .get_top_address_amounts(CoinOrTokenId::Coin, 2, Some((last_address2, last_amount2))) .await .unwrap(); assert!(end.is_empty()); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 022ba9d079..5950368739 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1968,21 +1968,11 @@ async fn holders_response( // canonicalize the encoding: `from_string` keeps the input verbatim, so // an all-uppercase cursor would otherwise be compared byte-wise against // the canonical lowercase strings in storage and mis-order the keyset - let validated = - Address::::from_string(&state.chain_config, &tie_break_id) - .map_err(|_| { - ApiServerWebServerError::ClientError( - ApiServerWebServerClientError::InvalidCursor, - ) - })?; - let tie_break_id = Address::new(&state.chain_config, validated.into_object()) - .map_err(|_| { - ApiServerWebServerError::ClientError( - ApiServerWebServerClientError::InvalidCursor, - ) - })? - .into_string(); - Some((amount, tie_break_id)) + let tie_break_id = canonical_address_string(&state.chain_config, &tie_break_id) + .ok_or(ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ))?; + Some((tie_break_id, amount)) } None => None, }, @@ -2352,7 +2342,7 @@ pub async fn order_pair( |s: &str| -> Result<(CoinOrTokenId, CoinOrTokenId), ApiServerWebServerError> { let parts: Vec<_> = s.split("_").collect(); ensure!( - parts.len() == 2, + parts.len() == 2 && parts.iter().all(|part| !part.is_empty()), ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidOrderTradingPair ) @@ -2445,7 +2435,7 @@ pub async fn order_pair_book( let parts: Vec<_> = pair.split("_").collect(); ensure!( - parts.len() == 2, + parts.len() == 2 && parts.iter().all(|part| !part.is_empty()), ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidOrderTradingPair ) @@ -2693,6 +2683,12 @@ fn reduce_rational(rational: Rational) -> Rational { while denom != 0 { (numer, denom) = (denom, numer % denom); } + if numer == 0 { + // gcd(0, 0) == 0; only reachable when both the numerator and the denominator + // are zero, i.e. the same bypassed caller invariant as above — return the + // input instead of dividing by zero + return rational; + } Rational::new(*rational.numer() / numer, *rational.denom() / numer) } From 9e2f201585900618b6a35eeac5496bd0cc923a2d Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 14:18:09 +0200 Subject: [PATCH 16/22] fix(api-server): surface storage corruption, harden postgres rollback Propagate an undecodable pool staker balance as a storage error instead of silently shrinking the in-memory listing, fold the versioned balance delete and the holders re-derivation into one helper so the ordering dependency is structural, include the failing address/order in balance parse errors, check the creation block height cast, note the DISTINCT ON omission semantics, document pool listing page stability and the canonical address encoding contract, and expect instead of unwrap in the orders test setup. --- .../src/storage/impls/in_memory/mod.rs | 13 ++++- .../src/storage/impls/postgres/queries.rs | 54 ++++++++++++++----- .../src/storage/storage_api/mod.rs | 4 ++ .../stack-test-suite/tests/v2/orders.rs | 3 +- api-server/web-server/src/api/v2.rs | 7 +++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 19d04ccc54..11da91d7a0 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -727,13 +727,22 @@ impl ApiServerInMemoryStorage { let offset = offset as usize; // deepest pledge first, ties broken by the encoded pool address (byte order), // exactly like the postgres listing; the balance and the sort key are decoded - // once per entry, and an unparseable/overflowed balance is treated as absent + // once per entry, and a pool whose stored balance fails to decode is surfaced + // as a storage error instead of silently shrinking the listing (the + // empty-history skip below stays silent because no write path produces one) let mut decorated: Vec<(Amount, String, PoolId, PoolDataWithExtraInfo)> = self .pool_data_table .iter() .filter_map(|(pool_id, by_height)| { let data = by_height.values().last()?.clone(); - let balance = data.staker_balance().ok()?; + let balance = match data.staker_balance() { + Ok(balance) => balance, + Err(e) => { + return Some(Err(ApiServerStorageError::DeserializationError(format!( + "pool {pool_id} has an invalid staker balance: {e}" + )))); + } + }; (balance != Amount::ZERO) .then(|| Ok((balance, self.pool_id_sort_key(pool_id)?, *pool_id, data))) }) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 5843edcb52..e4b9839ac7 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -241,7 +241,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let amount_str: String = row.get(1); let atoms = amount_str.parse::().map_err(|e| { ApiServerStorageError::DeserializationError(format!( - "invalid amount {amount_str}: {e}" + "invalid amount {amount_str} for address {address}: {e}" )) })?; Ok((address, Amount::from_atoms(atoms))) @@ -338,15 +338,15 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result<(), ApiServerStorageError> { let height = Self::block_height_to_postgres_friendly(block_height); - self.tx - .execute( - "DELETE FROM ml.address_balance WHERE block_height > $1;", - &[&height], - ) - .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // the versioned-history delete and the holders re-derivation form one + // dependency unit (the re-derivation is only correct immediately after + // the delete); keeping the two in a single helper makes that coupling + // structural instead of comment-enforced + self.del_address_balance_and_rederive_holders(height).await?; - // delete and update the address balance from the cache table + // roll back the latest-address-balance cache of the disconnected + // blocks; this is only correct because the versioned delete inside the + // helper above already ran within this same rollback call self.tx .execute( r#" @@ -385,6 +385,24 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + Ok(()) + } + + /// Delete the versioned address-balance rows above the height and re-derive + /// the holders (`ml.address_amount`) rows from what remains; set-based, in a + /// constant number of round trips, so the two steps cannot drift apart. + async fn del_address_balance_and_rederive_holders( + &mut self, + height: i64, + ) -> Result<(), ApiServerStorageError> { + self.tx + .execute( + "DELETE FROM ml.address_balance WHERE block_height > $1;", + &[&height], + ) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?; + // roll back the holders table rows of the disconnected blocks; set-based, in a // constant number of round trips: delete the rolled-back rows, drop rows whose // whole versioned history was orphaned, then re-derive and re-insert the rest @@ -2150,7 +2168,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { rows.into_iter() .map(|row| -> Result<(BlockHeight, PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { let creation_block_height: i64 = row.get(0); - let creation_block_height = BlockHeight::new(creation_block_height as u64); + let creation_block_height = u64::try_from(creation_block_height).map_err(|_| { + ApiServerStorageError::DeserializationError(format!( + "invalid negative creation block height: {creation_block_height}" + )) + })?; + let creation_block_height = BlockHeight::new(creation_block_height); let pool_id: String = row.get(1); let pool_id = Address::::from_string(chain_config, pool_id) .map_err(|_| ApiServerStorageError::AddressableError)? @@ -3087,6 +3110,10 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let rows = self .tx .query( + // DISTINCT ON picks the highest row per requested statistic; a + // statistic with no rows at all is silently omitted from the result + // (the endpoints treat a missing statistic as absent rather than + // zero) — keep the endpoint defaults in sync with this behavior r#" SELECT statistic, amount FROM ( @@ -3526,7 +3553,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .tx .query( r#" - SELECT ask_balance::text, give_balance::text + SELECT ask_balance::text, give_balance::text, order_id FROM ml.latest_orders_cache WHERE ask_currency = $1 AND give_currency = $2 AND frozen = FALSE @@ -3546,15 +3573,16 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map(|row| -> Result<(Amount, Amount), ApiServerStorageError> { let ask_str: String = row.get(0); let give_str: String = row.get(1); + let order_id: String = row.get(2); let ask = ask_str.parse::().map_err(|e| { ApiServerStorageError::DeserializationError(format!( - "invalid ask balance {ask_str}: {e}" + "invalid ask balance {ask_str} for order {order_id}: {e}" )) })?; let give = give_str.parse::().map_err(|e| { ApiServerStorageError::DeserializationError(format!( - "invalid give balance {give_str}: {e}" + "invalid give balance {give_str} for order {order_id}: {e}" )) })?; diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index fbfbde4370..b525f1f6ce 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -895,6 +895,10 @@ pub trait ApiServerStorageWrite: ApiServerStorageRead { block_height: BlockHeight, ) -> Result<(), ApiServerStorageError>; + /// Stores the balance under the string encoding carried by the given address: + /// scanning constructs addresses via `Address::new`, so the stored encoding is + /// the canonical one, which the byte-ordered tie-break comparison of the + /// top-addresses listing relies on. async fn set_address_balance_at_height( &mut self, address: &Address, diff --git a/api-server/stack-test-suite/tests/v2/orders.rs b/api-server/stack-test-suite/tests/v2/orders.rs index 3d805783a7..a941dfbf3c 100644 --- a/api-server/stack-test-suite/tests/v2/orders.rs +++ b/api-server/stack-test-suite/tests/v2/orders.rs @@ -352,7 +352,8 @@ async fn order_pair_book(#[case] seed: Seed) { web_server(listener, web_server_state, true).await }); - let (ml, tkn, coin_decimals) = rx.await.unwrap(); + let (ml, tkn, coin_decimals) = + rx.await.expect("the web server task sends the decimals parameters"); let get_json = |url: String| async move { let response = reqwest::get(format!("http://{}:{}{url}", addr.ip(), addr.port())) diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 5950368739..f6c5c88f34 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -1416,6 +1416,13 @@ impl FromStr for PoolSorting { } } +/// List pools, newest creation height first (or deepest pledge first with +/// `sort=by_pledge`). +/// +/// Note: pages are only stable once the scanner is fully caught up. A late +/// lower-height write (reorg catch-up) adjusts a pool's creation height in the +/// backing cache, so a keyset walk performed concurrently with such a write may +/// skip or repeat that pool. pub async fn pools( Query(params): Query>, State(state): State, Arc>>, From 4a50e8590f7d8a7c06c1f47649b05140b6dd4237 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 14:27:24 +0200 Subject: [PATCH 17/22] fix(api-server): dedupe holders query, detailed address decode errors Share one constant query text between the cursor and no-cursor top-addresses variants so the ordering and the cursor predicate cannot drift apart, include the offending identifier and the underlying error in address-decode failures instead of the bare AddressableError, skip empty versioned maps without expect in the in-memory pledge listing, document the transaction listing ordering contract, and simplify the statistics test bootstrap. No behavioral change for well-formed storage. --- .../src/storage/impls/in_memory/mod.rs | 12 +-- .../src/storage/impls/postgres/queries.rs | 88 ++++++++++++------- .../stack-test-suite/tests/v2/statistics.rs | 9 +- api-server/web-server/src/api/v2.rs | 6 +- 4 files changed, 72 insertions(+), 43 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 11da91d7a0..6eee7e15c4 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -642,15 +642,17 @@ impl ApiServerInMemoryStorage { // of panicking in a read path, and the cheap zero-balance filter runs // before the (allocating) sort-key encoding, so decommissioned pools // skip the bech32 work entirely - if by_height.is_empty() { + let (Some(created_height), Some(latest_data)) = + (by_height.keys().next(), by_height.values().last()) + else { return None; - } - let created_height = *by_height.keys().next().expect("a pool has versioned rows"); - let latest_data = by_height.values().last().expect("a pool has versioned rows"); + }; + let created_height = *created_height; + let latest_data = latest_data.clone(); latest_data.staker_balance().is_ok_and(|b| b != Amount::ZERO).then_some(( *pool_id, created_height, - latest_data.clone(), + latest_data, )) }) .map(|(pool_id, created_height, latest_data)| { diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index e4b9839ac7..29b427a5b2 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -199,21 +199,27 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result, ApiServerStorageError> { let coin_or_token_id = coin_or_token_id.encode(); let len = len as i64; + // the cursor and no-cursor variants share one query text: the cursor + // predicate is appended from a constant, so the ordering and the cursor + // comparison cannot drift apart between the two branches (all query parts + // are compile-time constants; only bound parameters carry runtime values) + const QUERY_HEAD: &str = r#" + SELECT address, amount::text + FROM ml.address_amount + WHERE coin_or_token_id = $1 AND amount != 0"#; + const CURSOR_PREDICATE: &str = r#" + AND (amount < $2::text::numeric + OR (amount = $2::text::numeric + AND address COLLATE "C" < $3))"#; + const QUERY_TAIL: &str = r#" + ORDER BY amount DESC, address COLLATE "C" DESC + LIMIT "#; let rows = match cursor { Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); self.tx .query( - r#" - SELECT address, amount::text - FROM ml.address_amount - WHERE coin_or_token_id = $1 AND amount != 0 - AND (amount < $2::text::numeric - OR (amount = $2::text::numeric - AND address COLLATE "C" < $3)) - ORDER BY amount DESC, address COLLATE "C" DESC - LIMIT $4; - "#, + &format!("{QUERY_HEAD}{CURSOR_PREDICATE}{QUERY_TAIL}$4;"), &[&coin_or_token_id, &amount_str, &address, &len], ) .await @@ -222,13 +228,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { None => self .tx .query( - r#" - SELECT address, amount::text - FROM ml.address_amount - WHERE coin_or_token_id = $1 AND amount != 0 - ORDER BY amount DESC, address COLLATE "C" DESC - LIMIT $2; - "#, + &format!("{QUERY_HEAD}{QUERY_TAIL}$2;"), &[&coin_or_token_id, &len], ) .await @@ -1624,8 +1624,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { }; let pool_id: String = data.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let balance: String = data.get(1); let spend_destination: Vec = data.get(2); @@ -1685,12 +1689,20 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map(|row| { let delegation_id: String = row.get(0); let delegation_id = - Address::::from_string(chain_config, delegation_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + Address::::from_string(chain_config, &delegation_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid delegation id {delegation_id}: {e}" + )) + })? .into_object(); let pool_id: String = row.get(1); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let balance: String = row.get(2); let spend_destination: Vec = row.get(3); @@ -1987,7 +1999,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let delegation_id_str: String = row.get(0); let delegation_id = Address::::from_string(chain_config, &delegation_id_str) - .map_err(|_| ApiServerStorageError::AddressableError)? + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid delegation id {delegation_id_str}: {e}" + )) + })? .into_object(); let balance: String = row.get(1); let spend_destination: Vec = row.get(2); @@ -2073,8 +2089,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map( |row| -> Result<(PoolId, PoolDataWithExtraInfo), ApiServerStorageError> { let pool_id: String = row.get(0); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let pool_data: Vec = row.get(1); let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()) @@ -2175,8 +2195,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { })?; let creation_block_height = BlockHeight::new(creation_block_height); let pool_id: String = row.get(1); - let pool_id = Address::::from_string(chain_config, pool_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let pool_id = Address::::from_string(chain_config, &pool_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + })? .into_object(); let pool_data: Vec = row.get(2); let pool_data = PoolDataWithExtraInfo::decode_all(&mut pool_data.as_slice()).map_err(|e| { @@ -3849,8 +3873,12 @@ fn decode_order_from_row( let creation_block_height: i64 = data.get("creation_block_height"); let is_frozen: bool = data.get("frozen"); - let order_id = Address::::from_string(chain_config, order_id) - .map_err(|_| ApiServerStorageError::AddressableError)? + let order_id = Address::::from_string(chain_config, &order_id) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid addressable id {order_id}: {e}" + )) + })? .into_object(); let initially_asked = Amount::from_fixedpoint_str(&initially_asked, 0).ok_or_else(|| { diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 5ee2450a77..9e2636f3e2 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -490,15 +490,10 @@ async fn coin_holders(#[case] seed: Seed) { let alice_address = Address::new(&chain_config, alice_destination).unwrap(); let bob_address = Address::new(&chain_config, bob_destination).unwrap(); - _ = tx.send(( - alice_address.clone(), - bob_address.clone(), - chain_config.coin_decimals(), - )); + _ = tx.send((alice_address, bob_address, chain_config.coin_decimals())); - (alice_address, bob_address, vec![block]) + vec![block] }; - let (_, _, chainstate_blocks) = chainstate_blocks; let storage = { let mut storage = TransactionalApiServerInMemoryStorage::new(&chain_config); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index f6c5c88f34..c0cda99013 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -812,7 +812,11 @@ pub async fn transactions( ) })? } - // an empty cursor starts the listing from the beginning + // an empty cursor starts the listing from the beginning; the cursor and + // offset variants of the listing must agree on the (newest-first, + // ascending global index) order for the cursor boundary to stay + // consistent — the backends derive both orderings from the same scanned + // main-chain state, so they cannot diverge without a storage bug None => db_tx.get_transactions_with_block_info(fetch_len, 0).await.map_err(|e| { logging::log::error!("internal error: {e}"); ApiServerWebServerError::ServerError( From 56c6b91b0d9f45b2a14c715359c21baeabda97aa Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 14:53:40 +0200 Subject: [PATCH 18/22] fix(api-server): canonicalize pool cursors, tighten cursor keys Re-encode pool-id cursors through Address::new so keyset boundaries use canonical bech32; document orders-cache rollback invariant; propagate staker-balance decode errors in sorted_live_pools; validate cursor key counts; add context to decode errors. --- .opencode/plans/nv2-experiment-plan.md | 194 ++++++++++++++++++ .../src/storage/impls/in_memory/mod.rs | 6 +- .../src/storage/impls/postgres/queries.rs | 36 +++- .../src/storage/storage_api/mod.rs | 2 +- .../stack-test-suite/tests/v2/orders.rs | 6 +- api-server/storage-test-suite/src/basic.rs | 3 + api-server/web-server/src/api/v2.rs | 30 ++- 7 files changed, 258 insertions(+), 19 deletions(-) create mode 100644 .opencode/plans/nv2-experiment-plan.md diff --git a/.opencode/plans/nv2-experiment-plan.md b/.opencode/plans/nv2-experiment-plan.md new file mode 100644 index 0000000000..54ac7742a8 --- /dev/null +++ b/.opencode/plans/nv2-experiment-plan.md @@ -0,0 +1,194 @@ +# NV-2 execution plan: Experiment + upstream issue + +Approved by user: benchmark branch → run → GitHub issue → update audit record. +Blocked by plan-mode permissions; execute when switched to normal mode. + +## Steps + +1. **Create** `chainstate/test-suite/benches/reorg_growth.rs` (source below) and add to `chainstate/test-suite/Cargo.toml` after the existing `[[bench]]`: + ```toml + [[bench]] + name = "reorg_growth" + harness = false + ``` +2. **Run**: `cargo bench --offline -p chainstate-test-suite --bench reorg_growth` (release profile; deps cached). Capture table. +3. **Branch + push**: `git checkout -b bench/pos-sidechain-reorg-growth`, commit both files, push to origin. +4. **Issue** on mintlayer/mintlayer-core: trace + refinements (reorg-before-consensus ordering = zero-marginal-cost garbage triggers; no trust pre-filter; losing branches persisted) + bench table + fix options (branch-state cache / configured connect-side cap / partial pre-checks). +5. **Update** `~/security-audit-skill/mintlayer-core/run-1/NEEDS-VALIDATION.md` NV-2 entry with refinements + links. + +## Bench source (composed, signatures verified against framework.rs / pos_block_builder.rs / benches.rs) + +Key facts used: `tf.chainstate` public (framework.rs:62); `make_pos_block_builder()` (framework.rs:105); builder `with_parent/with_stake_pool_id/with_stake_spending_key/with_vrf_key/build` (pos_block_builder.rs:155-291); `process_block` via `ChainstateInterface` (chainstate re-exported from test-framework's use: `use chainstate::{BlockSource, ChainstateError, chainstate_interface::ChainstateInterface}`); setup copied from `pos_reorg` bench (benches.rs:51-78). + +```rust +// (license header: MIT, same as benches.rs) + +use std::time::Instant; + +use chainstate::{BlockSource, chainstate_interface::ChainstateInterface}; +use chainstate_test_framework::TestFramework; +use common::{ + chain::{Block, Destination, GenBlock, PoolId, config::create_unit_test_config, stakelock::StakePoolData}, + primitives::{Amount, BlockDistance, H256, Id, Idable, per_thousand::PerThousand}, +}; +use crypto::{ + key::{KeyKind, PrivateKey}, + vrf::{VRFKeyKind, VRFPrivateKey}, +}; +use test_utils::random::make_seedable_rng; + +const MAIN_CHAIN_MARGIN: usize = 10; +const WINDOW: usize = 5; + +fn make_pos_chain(rng: &mut impl CryptoRng) -> (TestFramework, PoolId, PrivateKey, VRFPrivateKey) { + let (staking_sk, staking_pk) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let (vrf_sk, vrf_pk) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + + let genesis_pool_id = PoolId::new(H256::random_using(rng)); + let stake_pool_pledge = create_unit_test_config().min_stake_pool_pledge(); + let stake_pool_data = StakePoolData::new( + stake_pool_pledge, + Destination::PublicKey(staking_pk), + vrf_pk, + Destination::AnyoneCanSpend, + PerThousand::new(1000).unwrap(), + Amount::ZERO, + ); + + let chain_config = chainstate_test_framework::create_chain_config_with_staking_pool( + rng, + Amount::from_atoms(1000), + genesis_pool_id, + stake_pool_data, + ) + .max_depth_for_reorg(BlockDistance::new(5000)) + .build(); + let target_block_time = chain_config.target_block_spacing(); + + let mut tf = TestFramework::builder(rng).with_chain_config(chain_config).build(); + tf.progress_time_seconds_since_epoch(target_block_time.as_secs()); + + (tf, genesis_pool_id, staking_sk, vrf_sk) +} + +fn build_side_block( + tf: &mut TestFramework, + rng: &mut impl CryptoRng, + parent: &Id, + pool_id: PoolId, + staking_sk: &PrivateKey, + vrf_sk: &VRFPrivateKey, +) -> Block { + tf.make_pos_block_builder() + .with_parent(*parent) + .with_stake_pool_id(pool_id) + .with_stake_spending_key(staking_sk.clone()) + .with_vrf_key(vrf_sk.clone()) + .build(rng) +} + +/// Structurally valid PoS block signed by a random wrong key: passes +/// parent/size/checkpoint/reorg-depth checks, triggers the full in-memory +/// reorg, and only fails consensus validation afterwards. +fn build_garbage_block( + tf: &mut TestFramework, + rng: &mut impl CryptoRng, + parent: &Id, + pool_id: PoolId, +) -> Block { + let (wrong_sk, _) = PrivateKey::new_from_rng(rng, KeyKind::Secp256k1Schnorr); + let (wrong_vrf_sk, _) = VRFPrivateKey::new_from_rng(rng, VRFKeyKind::Schnorrkel); + tf.make_pos_block_builder() + .with_parent(*parent) + .with_stake_pool_id(pool_id) + .with_stake_spending_key(wrong_sk) + .with_vrf_key(wrong_vrf_sk) + .build(rng) +} + +fn report_row(label: &str, samples: &[f64]) { + let n = samples.len(); + let mean: f64 = samples.iter().sum::() / n as f64; + let mut sorted = samples.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let median = sorted[n / 2]; + println!("{label:>28} | mean {mean:>10.3} ms | median {median:>10.3} ms"); +} + +fn run_case(n: usize, rng: &mut impl CryptoRng) { + let (mut tf, pool_id, staking_sk, vrf_sk) = make_pos_chain(rng); + + let common_block_id = tf + .create_chain_pos(rng, &tf.genesis().get_id().into(), 5, pool_id, &staking_sk, &vrf_sk) + .unwrap(); + + let started = Instant::now(); + tf.create_chain_pos(rng, &common_block_id, n + MAIN_CHAIN_MARGIN, pool_id, &staking_sk, &vrf_sk) + .unwrap(); + println!( + "N={n}: main chain of {} built in {:.2}s", + n + MAIN_CHAIN_MARGIN, + started.elapsed().as_secs_f64() + ); + + // Losing-branch side chain: submit blocks one at a time, timing process_block only. + let mut prev = common_block_id; + let mut window_samples: Vec = Vec::with_capacity(WINDOW); + let quarter = (n / 4).max(WINDOW); + let mut checkpoints: Vec = [1, quarter, quarter * 2, quarter * 3, n] + .into_iter() + .map(|x| x.max(1).min(n)) + .collect(); + checkpoints.sort(); + checkpoints.dedup(); + + let total_start = Instant::now(); + for i in 1..=n { + let block = build_side_block(&mut tf, rng, &prev, pool_id, &staking_sk, &vrf_sk); + let block_id: Id = block.get_id().into(); + + let t = Instant::now(); + tf.chainstate.process_block(block, BlockSource::Local).unwrap(); + window_samples.push(t.elapsed().as_secs_f64() * 1000.0); + + if Some(&i) == checkpoints.first() { + let label = format!("side arrival #{i} (branch len {})", i - 1); + report_row(&label, &window_samples); + window_samples.clear(); + checkpoints.remove(0); + } + prev = block_id; + } + println!( + "N={n}: side chain of {n} accepted in {:.2}s", + total_start.elapsed().as_secs_f64() + ); + + // Trigger phase: garbage blocks pointing at the side-chain tip. Each is + // rejected, but only after paying the full in-memory reorg. + let mut garbage_samples = Vec::with_capacity(WINDOW); + for _ in 0..WINDOW { + let garbage = build_garbage_block(&mut tf, rng, &prev, pool_id); + let t = Instant::now(); + let res = tf.chainstate.process_block(garbage, BlockSource::Local); + let elapsed = t.elapsed().as_secs_f64() * 1000.0; + assert!(res.is_err(), "garbage block was unexpectedly accepted"); + garbage_samples.push(elapsed); + } + report_row(&format!("garbage @branch len {n}"), &garbage_samples); + println!(); +} + +fn main() { + let mut rng = make_seedable_rng(4242.into()); + println!("PoS side-chain growth cost (per-arrival process_block, in-memory store)\n"); + for n in [100usize, 200, 400] { + run_case(n, &mut rng); + } +} +``` + +## Notes +- Garbage blocks use wrong staking/VRF keys (structurally valid, consensus-invalid) — distinct keys give distinct ids, no seal grinding needed. +- Side chain length ≤ main chain length (margin 10) so it stays a losing branch; per-arrival reorg cost is paid regardless of trust (verified: no trust pre-filter, reorganize_in_memory at chainstateref/mod.rs:688 runs before validate_consensus at :703). +- If compile fails on imports (e.g. `Block`/`GenBlock` paths), check `common::chain` re-exports used by framework.rs imports. diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 6eee7e15c4..0f1b20f3e2 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -131,7 +131,11 @@ impl ApiServerInMemoryStorage { /// stores and orders by, keeping tie-break orders identical across backends. fn pool_id_sort_key(&self, pool_id: &PoolId) -> Result { Address::new(&self.chain_config, *pool_id) - .map_err(|_| ApiServerStorageError::AddressableError) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid pool id {pool_id}: {e}" + )) + }) .map(|address| address.as_str().to_owned()) } diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 29b427a5b2..8ec7d015cc 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -217,22 +217,29 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let rows = match cursor { Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); + let params: [&(dyn tokio_postgres::types::ToSql + Sync); 4] = + [&coin_or_token_id, &amount_str, &address, &len]; + // the largest placeholder in the assembled query text must match the + // number of bound parameters, so a future edit that reorders the + // constants or the array cannot silently misbind the placeholders + debug_assert_eq!(params.len(), 4); self.tx .query( &format!("{QUERY_HEAD}{CURSOR_PREDICATE}{QUERY_TAIL}$4;"), - &[&coin_or_token_id, &amount_str, &address, &len], + ¶ms, ) .await .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? } - None => self - .tx - .query( - &format!("{QUERY_HEAD}{QUERY_TAIL}$2;"), - &[&coin_or_token_id, &len], - ) - .await - .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?, + None => { + let params: [&(dyn tokio_postgres::types::ToSql + Sync); 2] = + [&coin_or_token_id, &len]; + debug_assert_eq!(params.len(), 2); + self.tx + .query(&format!("{QUERY_HEAD}{QUERY_TAIL}$2;"), ¶ms) + .await + .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? + } }; rows.into_iter() @@ -3527,7 +3534,16 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { // roll back the latest-orders cache rows of the disconnected blocks; the cache is // a mirror of the latest versioned state (concluded orders included, filtered by // the order book reads), so the surviving versioned history of every order with a - // deleted cache row fully determines its restored state + // deleted cache row fully determines its restored state. + // + // No ON CONFLICT clause is needed: `order_id` is the primary key of the cache + // table and the deleting CTE returns each affected order exactly once, so the + // lateral join inserts at most one row per order. The lateral join can also + // produce NO row for a deleted cache row only if that order has no surviving + // versioned history at all — impossible in practice, because every cache row is + // written from a versioned row (the upsert's `WHERE block_height <= + // EXCLUDED.block_height` guard keeps one cache row per order), and rollback + // heights only ever move to heights where the versioned rows already exist. self.tx .execute( r#" diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index b525f1f6ce..1dced13458 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -620,7 +620,7 @@ pub trait ApiServerStorageRead: Sync { /// Returns up to `len` holders of the asset with the largest balances, ordered by /// the balance (descending, ties broken by the address in descending BYTE order); - /// keyset (cursor) pagination, the cursor is the (amount, address) of the last + /// keyset (cursor) pagination, the cursor is the (address, amount) of the last /// returned holder, exclusive. Addresses with a zero balance are excluded. async fn get_top_address_amounts( &self, diff --git a/api-server/stack-test-suite/tests/v2/orders.rs b/api-server/stack-test-suite/tests/v2/orders.rs index a941dfbf3c..f16bcb2f6c 100644 --- a/api-server/stack-test-suite/tests/v2/orders.rs +++ b/api-server/stack-test-suite/tests/v2/orders.rs @@ -178,7 +178,7 @@ async fn create_fill_conclude_order(#[case] seed: Seed, #[case] version: OrdersV } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let (block1_id, tx1_id, block2_id, tx2_id, block3_id, tx3_id) = rx.await.unwrap(); @@ -349,7 +349,7 @@ async fn order_pair_book(#[case] seed: Seed) { } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let (ml, tkn, coin_decimals) = @@ -529,7 +529,7 @@ async fn order_pairs(#[case] seed: Seed) { } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let (order_id, ml, tkn) = rx.await.unwrap(); diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 1dd7f640b6..2d53860819 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2579,6 +2579,9 @@ where // the entries come back with the deepest orders first (the ordering contract) assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); assert!(!has_more); + // TODO: cover has_more == true (ORDER_BOOK_MAX_ORDERS truncation), e.g. via a + // test-injectable cap; the constant is not injectable today and seeding 10k + // orders is impractical, so the truncation path stays unexercised assert_eq!(sort_entries(entries.clone()), expected_pair); // the reverse pair is empty diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index c0cda99013..a8050d9b0e 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -781,6 +781,14 @@ pub async fn transactions( // already a total order (the pools and holders endpoints need the // tie-break because their amounts can repeat) let (keys, _tie_break_id) = cursor.into_parts("transactions")?; + // the transactions cursor carries exactly one key; reject anything + // else instead of silently ignoring extra entries + ensure!( + keys.len() == 1, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); let tx_global_index = keys.first().map(|index| index.to_owned()).ok_or( ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, @@ -1474,13 +1482,27 @@ pub async fn pools( ApiServerWebServerClientError::InvalidCursor, )); } - let pool_id = Address::from_string(&state.chain_config, &id) - .map_err(|_| { + // re-encode through Address::new so the id used as the + // keyset boundary is the canonical lowercase form: bech32 + // accepts all-uppercase strings and `from_string` keeps the + // input verbatim, so an all-uppercase cursor id would + // validate but silently break the byte-wise comparison + // against the canonical strings in storage + let pool_address = + Address::from_string(&state.chain_config, &id).map_err(|_| { ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, ) - })? - .into_object(); + })?; + let pool_address = + Address::new(&state.chain_config, pool_address.into_object()).map_err( + |_| { + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor, + ) + }, + )?; + let pool_id = pool_address.into_object(); Some((BlockHeight::new(creation_height), pool_id)) } _ => { From 4b0c0c8a96b29c70437ec74067512280eb1a4a8d Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 15:24:37 +0200 Subject: [PATCH 19/22] fix(api-server): range-check cursor amounts, harden order book reduction Reject holders cursors above the max representable amount; make reduce_rational return Option so a zero denominator surfaces as an internal error; validate pools cursor key count; add debug asserts to the transactions pagination bounds; document holders upsert staleness policy; compile-time parameter-count check for the top-holders queries; expect the pools test web server. --- .../src/storage/impls/in_memory/mod.rs | 14 +++-- .../src/storage/impls/postgres/queries.rs | 57 +++++++++++++++--- .../src/storage/storage_api/mod.rs | 8 ++- api-server/stack-test-suite/tests/v2/pools.rs | 2 +- api-server/web-server/src/api/v2.rs | 58 ++++++++++++++----- 5 files changed, 106 insertions(+), 33 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 0f1b20f3e2..dcbb6f0d9e 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -702,12 +702,14 @@ impl ApiServerInMemoryStorage { ) -> Result, ApiServerStorageError> { let len = len as usize; let mut decorated = self.sorted_live_pools()?; - let cursor_key = match cursor { - Some((cursor_height, cursor_pool_id)) => { - Some((cursor_height, self.pool_id_sort_key(&cursor_pool_id)?)) - } - None => None, - }; + let cursor_key = cursor + .map(|(cursor_height, cursor_pool_id)| { + Ok::<_, ApiServerStorageError>(( + cursor_height, + self.pool_id_sort_key(&cursor_pool_id)?, + )) + }) + .transpose()?; decorated.retain(|(height, key, _pool_id, _data)| match &cursor_key { Some((cursor_height, cursor_pool_key)) => { *height < *cursor_height || (*height == *cursor_height && *key < *cursor_pool_key) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 8ec7d015cc..d9a196c590 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -55,6 +55,24 @@ pub struct QueryFromConnection<'a, 'b> { tx: &'a PooledConnection<'b, PostgresConnectionManager>, } +/// Number of `$n`-style placeholders in a query text (const-evaluable); used to +/// tie bound-parameter array lengths to the assembled query text at compile time. +#[allow(dead_code)] +const fn placeholder_count(text: &str) -> usize { + let bytes = text.as_bytes(); + let mut count = 0; + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'$' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { + count += 1; + i += 2; + } else { + i += 1; + } + } + count +} + impl<'a, 'b> QueryFromConnection<'a, 'b> { fn get_table_exists_query(table_name: &str) -> String { format!( @@ -214,15 +232,31 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { const QUERY_TAIL: &str = r#" ORDER BY amount DESC, address COLLATE "C" DESC LIMIT "#; + // the number of bound parameters of each query variant, checked at compile + // time (const-evaluated): a future edit that reorders the constants or + // changes the parameter arrays cannot silently misbind the placeholders, + // including in release builds + const CURSOR_PARAM_COUNT: usize = 4; + const NO_CURSOR_PARAM_COUNT: usize = 2; + // (compile-time cross-check that the counter counts placeholders correctly; + // the unit type is the const block's value) + const _PLACEHOLDER_COUNT_SANITY: () = { + assert!( + NO_CURSOR_PARAM_COUNT + == placeholder_count( + "SELECT address, amount::text\nFROM ml.address_amount\nWHERE coin_or_token_id = $1 AND amount != 0\nORDER BY amount DESC, address COLLATE \"C\" DESC\nLIMIT $2;", + ), + "placeholder counter drifted" + ); + // const blocks must evaluate to the annotated type; the check itself is + // the point of the constant + let _: () = (); + }; let rows = match cursor { Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); - let params: [&(dyn tokio_postgres::types::ToSql + Sync); 4] = + let params: [&(dyn tokio_postgres::types::ToSql + Sync); CURSOR_PARAM_COUNT] = [&coin_or_token_id, &amount_str, &address, &len]; - // the largest placeholder in the assembled query text must match the - // number of bound parameters, so a future edit that reorders the - // constants or the array cannot silently misbind the placeholders - debug_assert_eq!(params.len(), 4); self.tx .query( &format!("{QUERY_HEAD}{CURSOR_PREDICATE}{QUERY_TAIL}$4;"), @@ -232,9 +266,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))? } None => { - let params: [&(dyn tokio_postgres::types::ToSql + Sync); 2] = + let params: [&(dyn tokio_postgres::types::ToSql + Sync); NO_CURSOR_PARAM_COUNT] = [&coin_or_token_id, &len]; - debug_assert_eq!(params.len(), 2); self.tx .query(&format!("{QUERY_HEAD}{QUERY_TAIL}$2;"), ¶ms) .await @@ -246,6 +279,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { .map(|row| -> Result<(String, Amount), ApiServerStorageError> { let address: String = row.get(0); let amount_str: String = row.get(1); + // fail fast on a corrupt row rather than silently returning wrong + // balances; the message carries the offending value for diagnosis let atoms = amount_str.parse::().map_err(|e| { ApiServerStorageError::DeserializationError(format!( "invalid amount {amount_str} for address {address}: {e}" @@ -499,6 +534,12 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ON CONFLICT (address, coin_or_token_id) DO UPDATE SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount WHERE ml.address_amount.block_height <= EXCLUDED.block_height; + -- unlike the pool cache there is no LEAST/GREATEST envelope here: + -- the holders listing orders by `amount DESC`, which the rollback + -- does not touch, so a stale cache row can only be wrong in value + -- and any next write of the same (address, coin) from the versioned + -- table replaces it (`<=` includes same-height reorg rewrites); + -- the ordering cache invariant the pools needed does not apply "#, &[ &upsert_addresses, @@ -3615,6 +3656,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { let give_str: String = row.get(1); let order_id: String = row.get(2); + // fail fast on a corrupt row rather than silently returning wrong + // balances; the message carries the offending value for diagnosis let ask = ask_str.parse::().map_err(|e| { ApiServerStorageError::DeserializationError(format!( "invalid ask balance {ask_str} for order {order_id}: {e}" diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 1dced13458..40a8989ad4 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -41,8 +41,9 @@ use self::block_aux_data::{BlockAuxData, BlockWithExtraData}; use crate::streaming::{StreamEvent, StreamEventId, StreamEventReadError}; /// Hard cap on the number of live orders a single order-book query may consume; -/// both backends return at most this many entries, deepest (largest ask balance) -/// first, with `has_more` signaling a truncated result. +/// both backends return at most this many entries, largest ask balance first, and +/// when the cap truncates the book, the smallest-balance orders are omitted; +/// `has_more` signals a truncated result. pub const ORDER_BOOK_MAX_ORDERS: usize = 10_000; pub mod block_aux_data; @@ -843,7 +844,8 @@ pub trait ApiServerStorageRead: Sync { /// from these in the web layer. At most [`ORDER_BOOK_MAX_ORDERS`] entries /// are returned, ordered by ask balance (descending, ties broken by the order id /// in descending byte order); `has_more` reports whether the cap truncated the - /// book (the entries are the deepest orders, the remainder is omitted). + /// book (the entries with the largest ask balance are returned; when the cap + /// truncates the book, the smallest-balance orders are omitted). async fn get_order_book_entries( &self, ask_currency: CoinOrTokenId, diff --git a/api-server/stack-test-suite/tests/v2/pools.rs b/api-server/stack-test-suite/tests/v2/pools.rs index bada674abd..ecdebfae51 100644 --- a/api-server/stack-test-suite/tests/v2/pools.rs +++ b/api-server/stack-test-suite/tests/v2/pools.rs @@ -416,7 +416,7 @@ async fn pools_cursor_pagination(#[case] seed: Seed) { } }; - web_server(listener, web_server_state, true).await + web_server(listener, web_server_state, true).await.expect("web server failed"); }); let created_pools = rx.await.unwrap(); diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index a8050d9b0e..cdf490dcd8 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -833,7 +833,15 @@ pub async fn transactions( })?, }; + // the shared validator guarantees items > 0 and the fetch above requests + // one extra row, so the indexing below stays in bounds; keep those + // cross-function invariants locally asserted + debug_assert!(offset_and_items.items > 0); let has_next_page = txs.len() > offset_and_items.items as usize; + debug_assert!( + !has_next_page || txs.len() > offset_and_items.items as usize, + "the extra fetched row is present when a next page exists" + ); let next_cursor = has_next_page.then(|| { let last_returned = &txs[offset_and_items.items as usize - 1]; Cursor::new( @@ -1467,6 +1475,14 @@ pub async fn pools( let cursor = match Cursor::decode_start(cursor_str)? { Some(cursor) => { let (keys, id) = cursor.into_parts("pools")?; + // the pools cursor carries exactly one key; reject anything else + // instead of silently ignoring extra entries + ensure!( + keys.len() == 1, + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); match keys.first() { Some(creation_height) if !id.is_empty() => { @@ -1985,12 +2001,17 @@ async fn holders_response( Some(cursor_str) => match Cursor::decode_start(cursor_str)? { Some(cursor) => { let (keys, tie_break_id) = cursor.into_parts("holders")?; + // `Amount::from_atoms` does not range-check, so reject atoms above + // the representable maximum up front: a fabricated boundary amount + // would otherwise be handed to the storage layer unchecked let amount = match keys.first() { - Some(atoms) => u128::from_str(atoms).map(Amount::from_atoms).map_err(|_| { - ApiServerWebServerError::ClientError( + Some(atoms) => u128::from_str(atoms) + .ok() + .filter(|atoms| *atoms <= Amount::MAX.into_atoms()) + .map(Amount::from_atoms) + .ok_or(ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, - ) - })?, + ))?, None => { return Err(ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, @@ -2553,8 +2574,13 @@ pub async fn order_pair_book( give_balance.into_atoms(), ) }; - // reduce the fraction so equal prices always map to the same level - let price = reduce_rational(price); + // reduce the fraction so equal prices always map to the same level; a zero + // denominator would corrupt the level ordering, so it maps to an internal + // server error instead of degrading silently + let price = reduce_rational(price).ok_or_else(|| { + logging::log::error!("order book price with a zero denominator reached reduction"); + ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) + })?; let level = levels.entry(price).or_insert(0); *level = level.checked_add(base_amount).ok_or_else(|| { @@ -2700,29 +2726,29 @@ fn price_to_decimal_string( Ok(price) } -fn reduce_rational(rational: Rational) -> Rational { +fn reduce_rational(rational: Rational) -> Option> { let (mut numer, mut denom) = (*rational.numer(), *rational.denom()); // the caller skips zero-balance entries, so a zero denominator (a price that // would compare equal to every other level and corrupt the ordering) must // never reach this function; the debug assertion keeps the two invariants - // tied together + // tied together, and the None return lets the caller surface the broken + // invariant instead of silently degrading in release builds debug_assert_ne!(denom, 0); if denom == 0 { - // the caller skips zero-balance entries, so reaching this branch means that - // invariant was broken elsewhere; log it instead of silently degrading - logging::log::error!("order book price with a zero denominator reached reduction"); - return rational; + return None; } while denom != 0 { (numer, denom) = (denom, numer % denom); } if numer == 0 { // gcd(0, 0) == 0; only reachable when both the numerator and the denominator - // are zero, i.e. the same bypassed caller invariant as above — return the - // input instead of dividing by zero - return rational; + // are zero, i.e. the same bypassed caller invariant as above + return None; } - Rational::new(*rational.numer() / numer, *rational.denom() / numer) + Some(Rational::new( + *rational.numer() / numer, + *rational.denom() / numer, + )) } async fn currency_decimals_for_pair( From efddcf003ef34effd75b0caf2a382bda2b78763c Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 15:45:14 +0200 Subject: [PATCH 20/22] fix(api-server): derive query param counts from real SQL constants Compile-time placeholder checks now read the actual query constants instead of a duplicated literal; balance-write cache upsert rationale documented; in-memory sort-key errors carry context; statistics VARIANTS ordering contract documented; holders cursor doc specifies atom-string encoding; order book aggregation cost documented; test web servers expected consistently. --- .../src/storage/impls/in_memory/mod.rs | 17 ++-- .../src/storage/impls/postgres/queries.rs | 91 ++++++++++++------- .../src/storage/storage_api/mod.rs | 7 +- .../stack-test-suite/tests/v2/statistics.rs | 2 +- api-server/storage-test-suite/src/basic.rs | 6 +- api-server/web-server/src/api/v2.rs | 15 +-- 6 files changed, 88 insertions(+), 50 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index dcbb6f0d9e..19376c6f0a 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -142,7 +142,11 @@ impl ApiServerInMemoryStorage { /// Encoded (bech32) form of an order id — see `pool_id_sort_key`. fn order_id_sort_key(&self, order_id: &OrderId) -> Result { Address::new(&self.chain_config, *order_id) - .map_err(|_| ApiServerStorageError::AddressableError) + .map_err(|e| { + ApiServerStorageError::DeserializationError(format!( + "invalid order id {order_id}: {e}" + )) + }) .map(|address| address.as_str().to_owned()) } @@ -193,7 +197,7 @@ impl ApiServerInMemoryStorage { }); } - Ok(holders.into_iter().take(len as usize).collect::>()) + Ok(holders.into_iter().take(len as usize).collect()) } fn get_address_balances( @@ -629,10 +633,11 @@ impl ApiServerInMemoryStorage { )) } - /// The live pools (non-zero pledge), newest creation height first, ties broken by - /// the encoded pool address (byte order), exactly like the postgres listing; the - /// sort key is encoded once per entry. Shared by the offset and the keyset - /// (cursor) listings so their ordering cannot diverge. + /// The live pools (non-zero pledge), newest creation height first, ties broken + /// by the encoded pool address in DESCENDING byte order (`Reverse`, matching + /// the postgres listing's `pool_id COLLATE "C" DESC` tie-break); the sort key + /// is encoded once per entry. Shared by the offset and the keyset (cursor) + /// listings so their ordering cannot diverge. fn sorted_live_pools( &self, ) -> Result, ApiServerStorageError> diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index d9a196c590..991d4c1b6e 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -55,22 +55,49 @@ pub struct QueryFromConnection<'a, 'b> { tx: &'a PooledConnection<'b, PostgresConnectionManager>, } -/// Number of `$n`-style placeholders in a query text (const-evaluable); used to -/// tie bound-parameter array lengths to the assembled query text at compile time. +/// Largest `$n` placeholder index across up to four query-text fragments +/// (const-evaluable); used to tie bound-parameter array lengths to the assembled +/// query text at compile time. #[allow(dead_code)] -const fn placeholder_count(text: &str) -> usize { +const fn placeholder_count(a: &str, b: &str, c: &str, d: &str) -> usize { + let mut largest = placeholder_count_1(a); + let candidate = placeholder_count_1(b); + if candidate > largest { + largest = candidate; + } + let candidate = placeholder_count_1(c); + if candidate > largest { + largest = candidate; + } + let candidate = placeholder_count_1(d); + if candidate > largest { + largest = candidate; + } + largest +} + +/// `placeholder_count` for a single fragment; counts DISTINCT placeholder indices +/// (`$n` with the largest n), not occurrences, because a placeholder index is +/// bound exactly once regardless of how often the query text references it. +#[allow(dead_code)] +const fn placeholder_count_1(text: &str) -> usize { let bytes = text.as_bytes(); - let mut count = 0; + let mut largest = 0usize; let mut i = 0; while i < bytes.len() { if bytes[i] == b'$' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { - count += 1; + // single-digit placeholder indices only; the largest bound parameter + // in these queries is single-digit by construction + let digit = (bytes[i + 1] - b'0') as usize; + if digit > largest { + largest = digit; + } i += 2; } else { i += 1; } } - count + largest } impl<'a, 'b> QueryFromConnection<'a, 'b> { @@ -232,26 +259,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { const QUERY_TAIL: &str = r#" ORDER BY amount DESC, address COLLATE "C" DESC LIMIT "#; - // the number of bound parameters of each query variant, checked at compile - // time (const-evaluated): a future edit that reorders the constants or - // changes the parameter arrays cannot silently misbind the placeholders, - // including in release builds - const CURSOR_PARAM_COUNT: usize = 4; - const NO_CURSOR_PARAM_COUNT: usize = 2; - // (compile-time cross-check that the counter counts placeholders correctly; - // the unit type is the const block's value) - const _PLACEHOLDER_COUNT_SANITY: () = { - assert!( - NO_CURSOR_PARAM_COUNT - == placeholder_count( - "SELECT address, amount::text\nFROM ml.address_amount\nWHERE coin_or_token_id = $1 AND amount != 0\nORDER BY amount DESC, address COLLATE \"C\" DESC\nLIMIT $2;", - ), - "placeholder counter drifted" - ); - // const blocks must evaluate to the annotated type; the check itself is - // the point of the constant - let _: () = (); - }; + // the largest placeholder of each query variant, derived from the ACTUAL + // constants above at compile time (const-concatenation): editing a constant + // without adjusting the matching parameter array fails to compile, including + // in release builds + const CURSOR_PARAM_COUNT: usize = + placeholder_count(QUERY_HEAD, CURSOR_PREDICATE, QUERY_TAIL, "$4;"); + const NO_CURSOR_PARAM_COUNT: usize = placeholder_count(QUERY_HEAD, QUERY_TAIL, "$2;", ""); let rows = match cursor { Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); @@ -534,12 +548,20 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ON CONFLICT (address, coin_or_token_id) DO UPDATE SET block_height = EXCLUDED.block_height, amount = EXCLUDED.amount WHERE ml.address_amount.block_height <= EXCLUDED.block_height; - -- unlike the pool cache there is no LEAST/GREATEST envelope here: - -- the holders listing orders by `amount DESC`, which the rollback - -- does not touch, so a stale cache row can only be wrong in value - -- and any next write of the same (address, coin) from the versioned - -- table replaces it (`<=` includes same-height reorg rewrites); - -- the ordering cache invariant the pools needed does not apply + -- unlike the pool cache there is no LEAST/GREATEST envelope here, + -- by design: the holders listing orders by `amount DESC`, which + -- rollback does not touch, so a stale cache row can only be wrong + -- in VALUE, never out of order. The height guard `<=` lets any + -- write at the same or newer height than the cached row replace + -- it (including same-height reorg rewrites); cache rows older + -- than the incoming versioned row are exactly the rows this + -- write corrects. A cache row NEWER than the versioned table can + -- only exist if a rollback was interrupted after truncating the + -- versioned history but before rolling the cache back — the + -- rollback helper (del_address_balance_and_rederive_holders) + -- always runs both steps in one transaction, so that state is + -- unreachable outside a crash mid-transaction, which postgres + -- atomicity also rules out. "#, &[ &upsert_addresses, @@ -581,6 +603,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { ) -> Result<(), ApiServerStorageError> { let height = Self::block_height_to_postgres_friendly(block_height); + // the versioned insert and the holders-cache upsert below both run on every + // balance write: the cache row is rewritten even when the amount is unchanged + // (the height guard still updates it), which is intentional — re-parsing the + // listing from the versioned table per read would cost more than the + // redundant upsert (the listing is served from this cache alone) self.tx .execute( r#" diff --git a/api-server/api-server-common/src/storage/storage_api/mod.rs b/api-server/api-server-common/src/storage/storage_api/mod.rs index 40a8989ad4..4bd2aa4852 100644 --- a/api-server/api-server-common/src/storage/storage_api/mod.rs +++ b/api-server/api-server-common/src/storage/storage_api/mod.rs @@ -90,6 +90,9 @@ pub enum CoinOrTokenStatistic { } impl CoinOrTokenStatistic { + /// Order matters: the array is serialized into the `ANY($1)` statistics filter + /// as text (via `to_string`), so it must stay in sync with the enum variants + /// AND their string spellings. pub const VARIANTS: [Self; 4] = { // compile-time exhaustiveness guard: adding a variant without updating the // array below fails to compile here because this match has no wildcard @@ -622,7 +625,9 @@ pub trait ApiServerStorageRead: Sync { /// Returns up to `len` holders of the asset with the largest balances, ordered by /// the balance (descending, ties broken by the address in descending BYTE order); /// keyset (cursor) pagination, the cursor is the (address, amount) of the last - /// returned holder, exclusive. Addresses with a zero balance are excluded. + /// returned holder, exclusive, with the amount encoded as a plain atom-count + /// string (`into_atoms().to_string()`) — not a decimal/fixed-point form. + /// Addresses with a zero balance are excluded. async fn get_top_address_amounts( &self, coin_or_token_id: CoinOrTokenId, diff --git a/api-server/stack-test-suite/tests/v2/statistics.rs b/api-server/stack-test-suite/tests/v2/statistics.rs index 9e2636f3e2..8d88c3a92b 100644 --- a/api-server/stack-test-suite/tests/v2/statistics.rs +++ b/api-server/stack-test-suite/tests/v2/statistics.rs @@ -523,7 +523,7 @@ async fn coin_holders(#[case] seed: Seed) { } }; - web_server(listener, web_server_state, false).await + web_server(listener, web_server_state, false).await.expect("web server failed"); }); let (alice_address, bob_address, coin_decimals) = rx.await.unwrap(); diff --git a/api-server/storage-test-suite/src/basic.rs b/api-server/storage-test-suite/src/basic.rs index 2d53860819..902c8d8a06 100644 --- a/api-server/storage-test-suite/src/basic.rs +++ b/api-server/storage-test-suite/src/basic.rs @@ -2579,9 +2579,9 @@ where // the entries come back with the deepest orders first (the ordering contract) assert!(entries.windows(2).all(|pair| pair[0].0 >= pair[1].0)); assert!(!has_more); - // TODO: cover has_more == true (ORDER_BOOK_MAX_ORDERS truncation), e.g. via a - // test-injectable cap; the constant is not injectable today and seeding 10k - // orders is impractical, so the truncation path stays unexercised + // TODO: cover has_more == true (ORDER_BOOK_MAX_ORDERS truncation) once the cap is + // injectable (test-only constructor parameter or a configurable limit), e.g. via + // a small cap and a handful of seeded orders assert_eq!(sort_entries(entries.clone()), expected_pair); // the reverse pair is empty diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index cdf490dcd8..4ccaada544 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -789,12 +789,9 @@ pub async fn transactions( ApiServerWebServerClientError::InvalidCursor ) ); - let tx_global_index = keys.first().map(|index| index.to_owned()).ok_or( - ApiServerWebServerError::ClientError( - ApiServerWebServerClientError::InvalidCursor, - ), - )?; - let tx_global_index = u64::from_str(&tx_global_index).map_err(|_| { + let tx_global_index = + keys.first().expect("the cursor carries exactly one key, checked above"); + let tx_global_index = u64::from_str(tx_global_index).map_err(|_| { ApiServerWebServerError::ClientError( ApiServerWebServerClientError::InvalidCursor, ) @@ -2554,7 +2551,11 @@ pub async fn order_pair_book( ApiServerWebServerError::ServerError(ApiServerWebServerServerError::InternalServerError) })?; - // aggregate the remaining balances into price levels + // aggregate the remaining balances into price levels; the storage layer caps + // the input at ORDER_BOOK_MAX_ORDERS entries, so each request (regardless of + // the requested page size) pays one O(ORDER_BOOK_MAX_ORDERS) scan+aggregate — + // acceptable for the capped input, but worth knowing before adding pages to + // the walk let mut levels = BTreeMap::, u128>::new(); for (ask_balance, give_balance) in entries { // defensive: the storage layer filters these out, but a zero balance would From b0ca55549b12877584ea74f841eb70700274d5ae Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 16:00:34 +0200 Subject: [PATCH 21/22] fix(api-server): bound cursor keys, gate book continuation Cursor decode rejects key vectors above a small constant bound; the order book cursor walk stops (no next_cursor) when the storage cap truncated the aggregation, so a partial book is never continued; the book cursor rejects a non-empty tie-break id; the transactions pagination debug assert checks the fetch bound instead of restating the page condition; scanner statistics tests fail loudly on a missing key; the in-memory id sort keys share one helper. --- .../src/storage/impls/in_memory/mod.rs | 31 ++++++++++--------- api-server/scanner-lib/src/sync/tests/mod.rs | 9 ++++-- api-server/web-server/src/api/cursor.rs | 20 +++++++++++- api-server/web-server/src/api/v2.rs | 20 +++++++++--- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs index 19376c6f0a..7b6a839049 100644 --- a/api-server/api-server-common/src/storage/impls/in_memory/mod.rs +++ b/api-server/api-server-common/src/storage/impls/in_memory/mod.rs @@ -26,6 +26,7 @@ use itertools::Itertools as _; use common::{ address::Address, + address::traits::Addressable, chain::{ Block, ChainConfig, DelegationId, Destination, Genesis, OrderId, PoolId, Transaction, UtxoOutPoint, @@ -127,27 +128,27 @@ impl ApiServerInMemoryStorage { } } - /// Encoded (bech32) form of a pool id — the exact string the postgres backend - /// stores and orders by, keeping tie-break orders identical across backends. - fn pool_id_sort_key(&self, pool_id: &PoolId) -> Result { - Address::new(&self.chain_config, *pool_id) + /// Encoded (bech32) form of an addressable object id — the exact string the + /// postgres backend stores and orders by, keeping tie-break orders identical + /// across backends. Shared by the pool-id and order-id sort keys. + fn id_sort_key( + &self, + id: T, + label: &str, + ) -> Result { + Address::new(&self.chain_config, id) .map_err(|e| { - ApiServerStorageError::DeserializationError(format!( - "invalid pool id {pool_id}: {e}" - )) + ApiServerStorageError::DeserializationError(format!("invalid {label} {id}: {e}")) }) .map(|address| address.as_str().to_owned()) } - /// Encoded (bech32) form of an order id — see `pool_id_sort_key`. + fn pool_id_sort_key(&self, pool_id: &PoolId) -> Result { + self.id_sort_key(*pool_id, "pool id") + } + fn order_id_sort_key(&self, order_id: &OrderId) -> Result { - Address::new(&self.chain_config, *order_id) - .map_err(|e| { - ApiServerStorageError::DeserializationError(format!( - "invalid order id {order_id}: {e}" - )) - }) - .map(|address| address.as_str().to_owned()) + self.id_sort_key(*order_id, "order id") } fn is_initialized(&self) -> Result { diff --git a/api-server/scanner-lib/src/sync/tests/mod.rs b/api-server/scanner-lib/src/sync/tests/mod.rs index 8b0117d088..86eef4c7cf 100644 --- a/api-server/scanner-lib/src/sync/tests/mod.rs +++ b/api-server/scanner-lib/src/sync/tests/mod.rs @@ -1039,12 +1039,17 @@ async fn reorg_locked_balance(#[case] seed: Seed) { } // `get_all_statistic` omits the statistics that were never written; the unwritten ones -// are treated as zero here +// are treated as zero here, but the callers assert on keys that MUST be present after +// a burn and its rollback (Burned, CirculatingSupply), so absence is a failure, not a +// zero fn statistic_value( stats: &BTreeMap, statistic: CoinOrTokenStatistic, ) -> Amount { - stats.get(&statistic).copied().unwrap_or(Amount::ZERO) + stats + .get(&statistic) + .copied() + .unwrap_or_else(|| panic!("{statistic} is missing from the storage statistics")) } #[rstest] diff --git a/api-server/web-server/src/api/cursor.rs b/api-server/web-server/src/api/cursor.rs index d585f384a9..5b113fe23a 100644 --- a/api-server/web-server/src/api/cursor.rs +++ b/api-server/web-server/src/api/cursor.rs @@ -32,6 +32,13 @@ use crate::error::ApiServerWebServerClientError; /// before the base64 decode. const MAX_CURSOR_LEN: usize = 1024; +/// The largest number of sort keys accepted in a decoded cursor. Every endpoint +/// validates its own key count after decoding; this bound additionally rejects +/// payloads carrying arbitrary numbers of keys for ANY tag at decode time, so a +/// fabricated cursor cannot smuggle an unbounded key vector past the shared +/// entry point. +const MAX_CURSOR_KEYS: usize = 16; + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Cursor { @@ -78,7 +85,18 @@ impl Cursor { let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD .decode(encoded) .map_err(|_| ApiServerWebServerClientError::InvalidCursor)?; - serde_json::from_slice(&decoded).map_err(|_| ApiServerWebServerClientError::InvalidCursor) + serde_json::from_slice::(&decoded) + .map_err(|_| ApiServerWebServerClientError::InvalidCursor)? + .validate_keys() + } + + /// Rejects cursors with an implausibly large key vector before any endpoint + /// logic inspects the keys; each endpoint then still checks its exact count. + fn validate_keys(self) -> Result { + if self.keys.len() > MAX_CURSOR_KEYS { + return Err(ApiServerWebServerClientError::InvalidCursor); + } + Ok(self) } /// Decodes a cursor request parameter. An empty value starts the listing from the diff --git a/api-server/web-server/src/api/v2.rs b/api-server/web-server/src/api/v2.rs index 4ccaada544..24b8daa309 100644 --- a/api-server/web-server/src/api/v2.rs +++ b/api-server/web-server/src/api/v2.rs @@ -836,8 +836,8 @@ pub async fn transactions( debug_assert!(offset_and_items.items > 0); let has_next_page = txs.len() > offset_and_items.items as usize; debug_assert!( - !has_next_page || txs.len() > offset_and_items.items as usize, - "the extra fetched row is present when a next page exists" + txs.len() <= offset_and_items.items as usize + 1, + "the storage returned at most the requested page plus the extra row" ); let next_cursor = has_next_page.then(|| { let last_returned = &txs[offset_and_items.items as usize - 1]; @@ -2517,7 +2517,15 @@ pub async fn order_pair_book( let cursor_price = match params.get(CURSOR) { Some(cursor_str) => match Cursor::decode_start(cursor_str)? { Some(cursor) => { - let (keys, _tie_break_id) = cursor.into_parts(cursor_tag)?; + let (keys, tie_break_id) = cursor.into_parts(cursor_tag)?; + // the book cursor keys are exactly (numerator, denominator) and the + // tie-break id is always empty; reject anything else up front + ensure!( + tie_break_id.is_empty(), + ApiServerWebServerError::ClientError( + ApiServerWebServerClientError::InvalidCursor + ) + ); let parse_key = |key: Option<&String>| { key.and_then(|key| u128::from_str(key).ok()).ok_or( ApiServerWebServerError::ClientError( @@ -2611,8 +2619,10 @@ pub async fn order_pair_book( // a next page implies a non-empty page, unless `items` was zero; the shared // validator rejects that, but degrade to "no cursor" instead of panicking should a - // future caller ever bypass it - let next_cursor = (has_next_page && !levels.is_empty()).then(|| { + // future caller ever bypass it. When the storage cap truncated the book + // (`has_more_orders`), no cursor is minted: the levels in hand are an incomplete + // aggregation, and walking them would present a partial book as authoritative. + let next_cursor = (has_next_page && !levels.is_empty() && !has_more_orders).then(|| { let (price, _amount) = levels.last().expect("the page is not empty"); Cursor::new( cursor_tag, From 82f3cfd6fa92ea47cb8cb207ed11dcf004c4b0c4 Mon Sep 17 00:00:00 2001 From: nullPointerEnjoyer Date: Sun, 4 Oct 2026 20:58:55 +0200 Subject: [PATCH 22/22] fix(api-server): fix static check failures Move query-text constants before the executable statements in the holders query (items_after_statements is denied on production code); restore the reorg statistics helper to treat a missing statistic as zero, which is the intended meaning after a rollback, with the rationale documented. --- .../src/storage/impls/postgres/queries.rs | 4 ++-- api-server/scanner-lib/src/sync/tests/mod.rs | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/api-server/api-server-common/src/storage/impls/postgres/queries.rs b/api-server/api-server-common/src/storage/impls/postgres/queries.rs index 991d4c1b6e..942d47a40e 100644 --- a/api-server/api-server-common/src/storage/impls/postgres/queries.rs +++ b/api-server/api-server-common/src/storage/impls/postgres/queries.rs @@ -242,8 +242,6 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { len: u32, cursor: Option<(String, Amount)>, ) -> Result, ApiServerStorageError> { - let coin_or_token_id = coin_or_token_id.encode(); - let len = len as i64; // the cursor and no-cursor variants share one query text: the cursor // predicate is appended from a constant, so the ordering and the cursor // comparison cannot drift apart between the two branches (all query parts @@ -266,6 +264,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> { const CURSOR_PARAM_COUNT: usize = placeholder_count(QUERY_HEAD, CURSOR_PREDICATE, QUERY_TAIL, "$4;"); const NO_CURSOR_PARAM_COUNT: usize = placeholder_count(QUERY_HEAD, QUERY_TAIL, "$2;", ""); + let coin_or_token_id = coin_or_token_id.encode(); + let len = len as i64; let rows = match cursor { Some((address, amount)) => { let amount_str = amount.into_atoms().to_string(); diff --git a/api-server/scanner-lib/src/sync/tests/mod.rs b/api-server/scanner-lib/src/sync/tests/mod.rs index 86eef4c7cf..29a343c36c 100644 --- a/api-server/scanner-lib/src/sync/tests/mod.rs +++ b/api-server/scanner-lib/src/sync/tests/mod.rs @@ -1038,18 +1038,14 @@ async fn reorg_locked_balance(#[case] seed: Seed) { drop(db_tx); } -// `get_all_statistic` omits the statistics that were never written; the unwritten ones -// are treated as zero here, but the callers assert on keys that MUST be present after -// a burn and its rollback (Burned, CirculatingSupply), so absence is a failure, not a -// zero +// `get_all_statistic` omits statistics that were never written; absence means the +// reverted-to value, i.e. zero — e.g. after a reorg the burned amount is rolled back +// and the Burned key disappears entirely, which this helper must report as zero fn statistic_value( stats: &BTreeMap, statistic: CoinOrTokenStatistic, ) -> Amount { - stats - .get(&statistic) - .copied() - .unwrap_or_else(|| panic!("{statistic} is missing from the storage statistics")) + stats.get(&statistic).copied().unwrap_or(Amount::ZERO) } #[rstest]