From 0fded971e975b947b758b4167e8921ec082c9f38 Mon Sep 17 00:00:00 2001 From: Nic Date: Mon, 29 Dec 2025 11:04:49 +0100 Subject: [PATCH 01/28] Add missing properties --- testnet/settings-test.json | 3 ++- testnet/testchain.json | 47 +++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/testnet/settings-test.json b/testnet/settings-test.json index ef00f162a..7dcf71cc8 100755 --- a/testnet/settings-test.json +++ b/testnet/settings-test.json @@ -6,6 +6,7 @@ "singleNodeTestnet": false, "minPeerVersion": "4.5.2", "allowConnectionsWithOlderPeerVersions": false, + "balanceRecorderEnabled": true, "bitcoinNet": "TEST3", "litecoinNet": "TEST3", "dogecoinNet": "TEST3", @@ -32,4 +33,4 @@ "0.0.0.0/0", "::/0" ] -} +} \ No newline at end of file diff --git a/testnet/testchain.json b/testnet/testchain.json index 06f5df122..02d1e7d8a 100644 --- a/testnet/testchain.json +++ b/testnet/testchain.json @@ -81,32 +81,41 @@ "minutesPerBlock": 1 }, "featureTriggers": { + "adminQueryFixHeight": 9999999999999, + "adminsReplaceFoundersHeight": 9999999999999, + "aggregateSignatureTimestamp": 0, + "arbitraryOptionalFeeTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 0, - "shareBinFix": 0, - "sharesByLevelV2Height": 0, - "rewardShareLimitTimestamp": 0, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0, - "transactionV6Timestamp": 9999999999999, + "cancelSellNameValidationTimestamp": 9999999999999, + "chatReferenceTimestamp": 0, + "decreaseOnlineAccountsDifficultyTimestamp": 1731958200000, "disableReferenceTimestamp": 0, - "aggregateSignatureTimestamp": 0, + "disableRewardshareHeight": 8450, + "disableTransferPrivsTimestamp": 9999999999990, + "enableRewardshareHeight": 11400, + "enableTransferPrivsTimestamp": 9999999999999, + "feeValidationFixTimestamp": 0, + "fixBatchRewardHeight": 9999999999999, + "groupMemberCheckHeight": 11200, + "ignoreLevelForRewardShareHeight": 9999999999999, "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, + "mintedBlocksAdjustmentRemovalHeight": 2206300, + "multipleNamesPerAccountHeight": 10, + "newBlockSigHeight": 0, + "nullGroupMembershipHeight": 20, "onlineAccountMinterLevelValidationHeight": 0, + "onlyMintWithNameHeight": 8500, + "removeOnlyMintWithNameHeight": 9999999999999, + "rewardShareLimitTimestamp": 0, "selfSponsorshipAlgoV1Height": 9999999, "selfSponsorshipAlgoV2Height": 9999900, "selfSponsorshipAlgoV3Height": 9999900, - "feeValidationFixTimestamp": 0, - "chatReferenceTimestamp": 0, - "arbitraryOptionalFeeTimestamp": 0, - "unconfirmableRewardSharesHeight": 9999999, - "disableTransferPrivsTimestamp": 9999999999990, - "enableTransferPrivsTimestamp": 9999999999999, - "cancelSellNameValidationTimestamp": 9999999999999, - "disableRewardshareHeight": 8450, - "enableRewardshareHeight": 11400, - "onlyMintWithNameHeight": 8500, - "groupMemberCheckHeight": 11200 + "shareBinFix": 0, + "sharesByLevelV2Height": 0, + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 9999999999999, + "unconfirmableRewardSharesHeight": 9999999 }, "genesisInfo": { "version": 4, @@ -2678,4 +2687,4 @@ { "type": "GENESIS", "recipient": "QU7EUWDZz7qJVPih3wL9RKTHRfPFy4ASHC", "amount": 10 } ] } -} +} \ No newline at end of file From b0f482613406c05c6e848b7ad3db436dcd383ba0 Mon Sep 17 00:00:00 2001 From: Nic Date: Mon, 29 Dec 2025 11:06:21 +0100 Subject: [PATCH 02/28] Add catches to manage gracefully the shutdown process --- .../arbitrary/ArbitraryDataCacheManager.java | 11 +++++ .../controller/repository/BlockArchiver.java | 7 ++++ .../OnlineAccountsSignaturesTrimmer.java | 7 ++++ .../repository/hsqldb/HSQLDBRepository.java | 40 +++++++++++++++++-- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index 9accd9c71..b9c78d199 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -63,6 +63,13 @@ public void run() { // Process queue processResourceQueue(); + } catch (InterruptedException e) { + // Check if we're shutting down + if (Controller.isStopping()) { + LOGGER.info("Arbitrary Data Cache Manager shutting down"); + break; + } + LOGGER.warn("Arbitrary Data Cache Manager interrupted, retrying...", e); } catch (Exception e) { LOGGER.error(e.getMessage(), e); Thread.sleep(600_000L); // wait 10 minutes to continue @@ -71,6 +78,10 @@ public void run() { // Clear queue before terminating thread processResourceQueue(); + } catch (InterruptedException e) { + if (!Controller.isStopping()) { + LOGGER.error("Arbitrary Data Cache Manager interrupted unexpectedly", e); + } } catch (Exception e) { LOGGER.error(e.getMessage(), e); } diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index 01cf40edd..60d875378 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -45,6 +45,13 @@ public void run() { repository.discardChanges(); return; } + } catch (InterruptedException e) { + if (Controller.isStopping()) { + LOGGER.info("Block Archiving shutting down"); + } else { + LOGGER.error("Block Archiving interrupted during initialization. Restart ASAP. Report this error immediately to the developers.", e); + } + return; } catch (Exception e) { LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e); return; diff --git a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java index c2d37e14f..22893088d 100644 --- a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java @@ -35,6 +35,13 @@ public void run() { Thread.sleep(INITIAL_SLEEP_PERIOD); trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + } catch (InterruptedException e) { + if (Controller.isStopping()) { + LOGGER.info("Online Accounts Signatures Trimming shutting down"); + } else { + LOGGER.error("Online Accounts Signatures Trimming interrupted during initialization. Restart ASAP. Report this error immediately to the developers.", e); + } + return; } catch (Exception e) { LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e); return; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 2bf88657d..a4972bcb2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -574,7 +574,7 @@ private PreparedStatement cachePreparedStatement(String sql) throws SQLException * which we never close, which means HSQLDB also caches a parsed, * prepared statement that can be reused for subsequent * calls to HSQLDB.prepareStatement(sql). - * + * * See org.hsqldb.StatementManager for more details. */ PreparedStatement preparedStatement = this.preparedStatementCache.get(sql); @@ -586,9 +586,19 @@ private PreparedStatement cachePreparedStatement(String sql) throws SQLException preparedStatement = this.connection.prepareStatement(sql); this.preparedStatementCache.put(sql, preparedStatement); } else { - // Clean up ready for reuse - preparedStatement.clearBatch(); - preparedStatement.clearParameters(); + try { + // Clean up ready for reuse + preparedStatement.clearBatch(); + preparedStatement.clearParameters(); + } catch (SQLException e) { + // Connection may have been closed, try to recreate the statement + if (this.connection == null || this.connection.isClosed()) { + throw new SQLException("Connection is closed", e); + } + // Statement is closed but connection is still open, recreate + preparedStatement = this.connection.prepareStatement(sql); + this.preparedStatementCache.put(sql, preparedStatement); + } } return preparedStatement; @@ -964,6 +974,18 @@ public SQLException examineException(SQLException e) { } private void assertEmptyTransaction(String context) throws DataException { + // If connection is already closed, skip this check + try { + if (this.connection == null || this.connection.isClosed()) { + LOGGER.debug(() -> String.format("Skipping transaction check after %s - connection already closed", context)); + return; + } + } catch (SQLException e) { + // If we can't check if connection is closed, assume it is and skip + LOGGER.debug(() -> String.format("Unable to check connection status after %s, skipping transaction check", context)); + return; + } + String sql = "SELECT transaction, transaction_size FROM information_schema.system_sessions WHERE session_id = ?"; try { @@ -993,6 +1015,16 @@ private void assertEmptyTransaction(String context) throws DataException { } } } catch (SQLException e) { + // During shutdown, the connection might be closed by another thread + // Check if this is the case and log appropriately + try { + if (this.connection == null || this.connection.isClosed()) { + LOGGER.debug(() -> String.format("Connection closed while checking repository status after %s", context)); + return; + } + } catch (SQLException ignored) { + // Ignore - we'll throw the original exception below + } throw new DataException("Error checking repository status after " + context, e); } } From 10823a86c52601fa4c25f8d26506312d8ecb5560 Mon Sep 17 00:00:00 2001 From: Nic Date: Mon, 29 Dec 2025 11:42:30 +0100 Subject: [PATCH 03/28] Shutdown TradeBot --- src/main/java/org/qortal/controller/Controller.java | 3 +++ .../org/qortal/controller/tradebot/TradeBot.java | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 8b147bf43..64f4f7a92 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1173,6 +1173,9 @@ public void shutdown() { // We were interrupted while waiting for thread to join } + LOGGER.info("Shutting down TradeBot"); + TradeBot.getInstance().shutdown(); + // Make sure we're the only thread modifying the blockchain when shutting down the repository ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); try { diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index c17e57589..cecbca3e9 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -137,6 +137,19 @@ public static synchronized TradeBot getInstance() { return instance; } + public void shutdown() { + try { + LOGGER.info("Shutting down TradeBot scheduler"); + tradePresenceMessageScheduler.shutdownNow(); + if (!tradePresenceMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("TradeBot scheduler did not terminate in time"); + } + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while waiting for TradeBot scheduler to terminate", e); + Thread.currentThread().interrupt(); + } + } + public ACCT getAcctUsingAtData(ATData atData) { byte[] codeHash = atData.getCodeHash(); if (codeHash == null) From 4998ab149d636991202eb056d0a59c9c98f544cf Mon Sep 17 00:00:00 2001 From: Nic Date: Mon, 29 Dec 2025 15:14:32 +0100 Subject: [PATCH 04/28] Restore unsorted featureTriggers section --- testnet/testchain.json | 45 +++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/testnet/testchain.json b/testnet/testchain.json index 02d1e7d8a..ed57aacbf 100644 --- a/testnet/testchain.json +++ b/testnet/testchain.json @@ -81,41 +81,32 @@ "minutesPerBlock": 1 }, "featureTriggers": { - "adminQueryFixHeight": 9999999999999, - "adminsReplaceFoundersHeight": 9999999999999, - "aggregateSignatureTimestamp": 0, - "arbitraryOptionalFeeTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 0, + "shareBinFix": 0, + "sharesByLevelV2Height": 0, + "rewardShareLimitTimestamp": 0, "calcChainWeightTimestamp": 0, - "cancelSellNameValidationTimestamp": 9999999999999, - "chatReferenceTimestamp": 0, - "decreaseOnlineAccountsDifficultyTimestamp": 1731958200000, + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 9999999999999, "disableReferenceTimestamp": 0, - "disableRewardshareHeight": 8450, - "disableTransferPrivsTimestamp": 9999999999990, - "enableRewardshareHeight": 11400, - "enableTransferPrivsTimestamp": 9999999999999, - "feeValidationFixTimestamp": 0, - "fixBatchRewardHeight": 9999999999999, - "groupMemberCheckHeight": 11200, - "ignoreLevelForRewardShareHeight": 9999999999999, + "aggregateSignatureTimestamp": 0, "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, - "mintedBlocksAdjustmentRemovalHeight": 2206300, - "multipleNamesPerAccountHeight": 10, - "newBlockSigHeight": 0, - "nullGroupMembershipHeight": 20, "onlineAccountMinterLevelValidationHeight": 0, - "onlyMintWithNameHeight": 8500, - "removeOnlyMintWithNameHeight": 9999999999999, - "rewardShareLimitTimestamp": 0, "selfSponsorshipAlgoV1Height": 9999999, "selfSponsorshipAlgoV2Height": 9999900, "selfSponsorshipAlgoV3Height": 9999900, - "shareBinFix": 0, - "sharesByLevelV2Height": 0, - "transactionV5Timestamp": 0, - "transactionV6Timestamp": 9999999999999, - "unconfirmableRewardSharesHeight": 9999999 + "feeValidationFixTimestamp": 0, + "chatReferenceTimestamp": 0, + "arbitraryOptionalFeeTimestamp": 0, + "unconfirmableRewardSharesHeight": 9999999, + "disableTransferPrivsTimestamp": 9999999999990, + "enableTransferPrivsTimestamp": 9999999999999, + "cancelSellNameValidationTimestamp": 9999999999999, + "disableRewardshareHeight": 8450, + "enableRewardshareHeight": 11400, + "onlyMintWithNameHeight": 8500, + "groupMemberCheckHeight": 11200 }, "genesisInfo": { "version": 4, From 028c290e7b0a4e2cb4793bffd57617d445ad7319 Mon Sep 17 00:00:00 2001 From: Nic Date: Tue, 30 Dec 2025 19:02:02 +0100 Subject: [PATCH 05/28] Add missing properties to make testnet start --- testnet/settings-test.json | 3 +-- testnet/testchain.json | 11 ++++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/testnet/settings-test.json b/testnet/settings-test.json index 7dcf71cc8..61729c439 100755 --- a/testnet/settings-test.json +++ b/testnet/settings-test.json @@ -3,10 +3,9 @@ "apiPort": 62391, "bindAddress": "0.0.0.0", "isTestNet": true, - "singleNodeTestnet": false, + "singleNodeTestnet": true, "minPeerVersion": "4.5.2", "allowConnectionsWithOlderPeerVersions": false, - "balanceRecorderEnabled": true, "bitcoinNet": "TEST3", "litecoinNet": "TEST3", "dogecoinNet": "TEST3", diff --git a/testnet/testchain.json b/testnet/testchain.json index ed57aacbf..99c4e2311 100644 --- a/testnet/testchain.json +++ b/testnet/testchain.json @@ -106,7 +106,16 @@ "disableRewardshareHeight": 8450, "enableRewardshareHeight": 11400, "onlyMintWithNameHeight": 8500, - "groupMemberCheckHeight": 11200 + "groupMemberCheckHeight": 11200, + "decreaseOnlineAccountsDifficultyTimestamp": 9999999999999, + "removeOnlyMintWithNameHeight": 9999999999999, + "fixBatchRewardHeight": 9999999999999, + "adminsReplaceFoundersHeight": 0, + "nullGroupMembershipHeight": 0, + "ignoreLevelForRewardShareHeight": 0, + "adminQueryFixHeight": 0, + "multipleNamesPerAccountHeight": 0, + "mintedBlocksAdjustmentRemovalHeight": 0 }, "genesisInfo": { "version": 4, From b3ea65dbd56b6941626094bec313ad21a2c5e97b Mon Sep 17 00:00:00 2001 From: Nic Date: Tue, 30 Dec 2025 22:19:36 +0100 Subject: [PATCH 06/28] Revert to false --- testnet/settings-test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testnet/settings-test.json b/testnet/settings-test.json index 61729c439..f454b82f0 100755 --- a/testnet/settings-test.json +++ b/testnet/settings-test.json @@ -3,7 +3,7 @@ "apiPort": 62391, "bindAddress": "0.0.0.0", "isTestNet": true, - "singleNodeTestnet": true, + "singleNodeTestnet": false, "minPeerVersion": "4.5.2", "allowConnectionsWithOlderPeerVersions": false, "bitcoinNet": "TEST3", From 756e8ebceb29ed2828fa3dc85ae2e52c1a507776 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 12:48:19 +0100 Subject: [PATCH 07/28] Add check for controller shutdown to avoid electrum connections --- src/main/java/org/qortal/crosschain/ElectrumX.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 88ef7deb0..97ba9b738 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -9,6 +9,7 @@ import org.json.simple.JSONObject; import org.json.simple.JSONValue; import org.qortal.api.resource.CrossChainUtils; +import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.utils.BitTwiddling; @@ -880,8 +881,8 @@ private void makeMoreConnections() { } private void connectRemainingServers() { - // while there are remaining servers and less than the maximum connections - while( !this.remainingServers.isEmpty() && this.connections.size() < MAXIMUM_CONNECTIONS ) { + // if the controller is not stopping and while there are remaining servers and less than the maximum connections + while( !Controller.isStopping() && !this.remainingServers.isEmpty() && this.connections.size() < MAXIMUM_CONNECTIONS ) { ChainableServer server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); makeConnection(server, this.getClass().getSimpleName()); From 41449ccec54a7af3bab8fd6709e379a695988371 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 12:50:04 +0100 Subject: [PATCH 08/28] Add logger info --- .../org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java index 2ddabf8d1..5647bd602 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -45,6 +45,7 @@ public HSQLDBRepositoryFactory(String connectionUrl) throws DataException { // Check no-one else is accessing database try (Connection connection = DriverManager.getConnection(this.connectionUrl)) { // We only need to check we can obtain connection. It will be auto-closed. + LOGGER.info("Checking database connection..."); } catch (SQLException e) { Throwable cause = e.getCause(); if (!(cause instanceof HsqlException)) From e33b4fdf98d45de1fd2294f6bbe33ee01974871b Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 12:50:31 +0100 Subject: [PATCH 09/28] Remove duplicated condition --- .../controller/hsqldb/HSQLDBBalanceRecorder.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java index 43e7c5425..1c7f89ae2 100644 --- a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java +++ b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java @@ -45,18 +45,12 @@ private HSQLDBBalanceRecorder( int priorityRequested, int frequency, int capacit public static Optional getInstance() { if( SINGLETON == null ) { - SINGLETON = new HSQLDBBalanceRecorder( Settings.getInstance().getBalanceRecorderPriority(), Settings.getInstance().getBalanceRecorderFrequency(), Settings.getInstance().getBalanceRecorderCapacity() ); - - } - else if( SINGLETON == null ) { - - return Optional.empty(); } return Optional.of(SINGLETON); @@ -72,13 +66,11 @@ public void run() { public List getLatestDynamics(int limit, long offset) { - List latest = this.balanceDynamics.stream() + return this.balanceDynamics.stream() .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed()) .skip(offset) .limit(limit) .collect(Collectors.toList()); - - return latest; } public List getRanges(Integer offset, Integer limit, Boolean reverse) { From 6b88993d0e92f255686f1fce4e30dc4bd9ad6f8c Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 13:14:57 +0100 Subject: [PATCH 10/28] Add timers shutdown --- .../repository/hsqldb/HSQLDBCacheUtils.java | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index bee629b8f..99ea49672 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -62,6 +62,9 @@ public int compare(ArbitraryResourceData data1, ArbitraryResourceData data2) { }; private static final String DEFAULT_IDENTIFIER = "default"; private static final int ZERO = 0; + private static Timer dbCacheTimer; + private static Timer balanceRecorderTimer; + public static final String DB_CACHE_TIMER = "DB Cache Timer"; public static final String DB_CACHE_TIMER_TASK = "DB Cache Timer Task"; public static final String BALANCE_RECORDER_TIMER = "Balance Recorder Timer"; @@ -460,7 +463,7 @@ private static boolean passQuery(Predicate predicate, ArbitraryResourceD */ public static void startCaching(int priorityRequested, int frequency) { - Timer timer = buildTimer(DB_CACHE_TIMER, priorityRequested); + dbCacheTimer = buildTimer(DB_CACHE_TIMER, priorityRequested); TimerTask task = new TimerTask() { @Override @@ -468,6 +471,11 @@ public void run() { Thread.currentThread().setName(DB_CACHE_TIMER_TASK); + // Exit gracefully if shutting down + if (Controller.isStopping()) { + return; + } + try (final Repository respository = RepositoryManager.getRepository()) { fillCache(ArbitraryResourceCache.getInstance(), respository); } @@ -478,7 +486,7 @@ public void run() { }; // delay 1 second - timer.scheduleAtFixedRate(task, 1000, frequency * 1000); + dbCacheTimer.scheduleAtFixedRate(task, 1000, frequency * 1000); } /** @@ -497,7 +505,7 @@ public static void startRecordingBalances( int frequency, int capacity) { - Timer timer = buildTimer(BALANCE_RECORDER_TIMER, priorityRequested); + balanceRecorderTimer = buildTimer(BALANCE_RECORDER_TIMER, priorityRequested); TimerTask task = new TimerTask() { @Override @@ -505,6 +513,11 @@ public void run() { Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK); + // Exit gracefully if shutting down + if (Controller.isStopping()) { + return; + } + int currentHeight = recordCurrentBalances(balancesByHeight); LOGGER.debug("recorded balances: height = " + currentHeight); @@ -542,7 +555,7 @@ public void run() { }; // wait 5 minutes - timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000); + balanceRecorderTimer.scheduleAtFixedRate(task, 300_000, frequency * 60_000); } private static void produceBalanceDynamics(int currentHeight, Optional priorHeight, boolean isRewardDistribution, ConcurrentHashMap> balancesByHeight, CopyOnWriteArrayList balanceDynamics, int capacity) { @@ -860,4 +873,29 @@ public static List getAccountBalances(Repository repository) return data; } + + /** + * Shutdown all timers + * + * Cancels all running Timer tasks to allow clean shutdown + */ + public static void shutdown() { + LOGGER.info("Shutting down HSQLDBCacheUtils timers..."); + + if (dbCacheTimer != null) { + dbCacheTimer.cancel(); + dbCacheTimer.purge(); + dbCacheTimer = null; + LOGGER.info("DB Cache Timer shutdown"); + } + + if (balanceRecorderTimer != null) { + balanceRecorderTimer.cancel(); + balanceRecorderTimer.purge(); + balanceRecorderTimer = null; + LOGGER.info("Balance Recorder Timer shutdown"); + } + + LOGGER.info("HSQLDBCacheUtils timers shut down complete"); + } } \ No newline at end of file From 4af27616efa1bc1b901763ef468a564bfb3e3de2 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 13:35:39 +0100 Subject: [PATCH 11/28] Change logger type --- .../java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index 99ea49672..e1174158e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -886,14 +886,14 @@ public static void shutdown() { dbCacheTimer.cancel(); dbCacheTimer.purge(); dbCacheTimer = null; - LOGGER.info("DB Cache Timer shutdown"); + LOGGER.debug("DB Cache Timer shutdown"); } if (balanceRecorderTimer != null) { balanceRecorderTimer.cancel(); balanceRecorderTimer.purge(); balanceRecorderTimer = null; - LOGGER.info("Balance Recorder Timer shutdown"); + LOGGER.debug("Balance Recorder Timer shutdown"); } LOGGER.info("HSQLDBCacheUtils timers shut down complete"); From 4f0d448afc535e15dbaa241cfc29ee09fa1a66da Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 14:26:42 +0100 Subject: [PATCH 12/28] Add shutdown method --- .../arbitrary/ArbitraryMetadataManager.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 76e448955..45a9a4e98 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -398,6 +398,11 @@ public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) { private void processNetworkGetArbitraryMetadataMessage() { + // Exit gracefully if shutting down + if (Controller.isStopping()) { + return; + } + try { List messagesToProcess; synchronized (lock) { @@ -531,4 +536,29 @@ private void processNetworkGetArbitraryMetadataMessage() { LOGGER.error(e.getMessage(), e); } } + + /** + * Shutdown the scheduler + * + * Stops the scheduled executor service to allow clean shutdown + */ + public void shutdown() { + LOGGER.info("Shutting down ArbitraryMetadataManager scheduler..."); + + if (!scheduler.isShutdown()) { + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) { + scheduler.shutdownNow(); + LOGGER.debug("Scheduler forced shutdown"); + } + } catch (InterruptedException e) { + scheduler.shutdownNow(); + Thread.currentThread().interrupt(); + } + LOGGER.debug("Scheduler shutdown complete"); + } + + LOGGER.info("ArbitraryMetadataManager scheduler shutdown complete"); + } } From 396259ae340260ff35c0e7dd1855bd3d49ded116 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 14:28:55 +0100 Subject: [PATCH 13/28] Improve logger messages, invoke shutdown methos for HSQLDBCacheUtils and ArbitraryMetadataManager --- .../org/qortal/controller/Controller.java | 39 ++++++++++++------- .../repository/hsqldb/HSQLDBCacheUtils.java | 2 +- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 64f4f7a92..247067196 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -37,6 +37,7 @@ import org.qortal.network.PeerAddress; import org.qortal.network.message.*; import org.qortal.repository.*; +import org.qortal.repository.hsqldb.HSQLDBCacheUtils; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; @@ -419,7 +420,7 @@ public static void main(String[] args) { } if( Settings.getInstance().isDbCacheEnabled() ) { - LOGGER.info("Db Cache Starting ..."); + LOGGER.info("Starting Db Cache..."); HSQLDBDataCacheManager hsqldbDataCacheManager = new HSQLDBDataCacheManager(); hsqldbDataCacheManager.start(); } @@ -427,7 +428,7 @@ public static void main(String[] args) { LOGGER.info("Db Cache Disabled"); } - LOGGER.info("Arbitrary Indexing Starting ..."); + LOGGER.info("Starting Arbitrary Indexing..."); ArbitraryIndexUtils.startCaching( Settings.getInstance().getArbitraryIndexingPriority(), Settings.getInstance().getArbitraryIndexingFrequency() @@ -437,7 +438,7 @@ public static void main(String[] args) { Optional recorder = HSQLDBBalanceRecorder.getInstance(); if( recorder.isPresent() ) { - LOGGER.info("Balance Recorder Starting ..."); + LOGGER.info("Starting Balance Recorder..."); recorder.get().start(); } else { @@ -445,7 +446,7 @@ public static void main(String[] args) { } } else { - LOGGER.info("Balance Recorder Disabled"); + LOGGER.info("Balance Recorder disabled"); } } catch (DataException e) { // If exception has no cause or message then repository is in use by some other process. @@ -465,17 +466,17 @@ public static void main(String[] args) { // Rebuild Names table and check database integrity (if enabled) NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + LOGGER.info("Rebuilding all names..."); namesDatabaseIntegrityCheck.rebuildAllNames(); if (Settings.getInstance().isNamesIntegrityCheckEnabled()) { + LOGGER.info("Running database integrity check..."); namesDatabaseIntegrityCheck.runIntegrityCheck(); } - - LOGGER.info("Validating blockchain"); + LOGGER.info("Validating blockchain..."); try { BlockChain.validate(); - Controller.getInstance().refillLatestBlocksCache(); - LOGGER.info(String.format("Our chain height at start-up: %d", Controller.getInstance().getChainHeight())); + LOGGER.info("Chain height at start-up: {}", Controller.getInstance().getChainHeight()); } catch (DataException e) { LOGGER.error("Couldn't validate blockchain", e); Gui.getInstance().fatalError("Blockchain validation issue", e); @@ -561,7 +562,6 @@ public void run() { ); } - LOGGER.info("Starting online accounts manager"); OnlineAccountsManager.getInstance().start(); @@ -595,7 +595,7 @@ public void run() { } if (Settings.getInstance().isGatewayEnabled()) { - LOGGER.info(String.format("Starting gateway service on port %d", Settings.getInstance().getGatewayPort())); + LOGGER.info("Starting gateway service on port {}", Settings.getInstance().getGatewayPort()); try { GatewayService gatewayService = GatewayService.getInstance(); gatewayService.start(); @@ -608,7 +608,7 @@ public void run() { } if (Settings.getInstance().isDomainMapEnabled()) { - LOGGER.info(String.format("Starting domain map service on port %d", Settings.getInstance().getDomainMapPort())); + LOGGER.info("Starting domain map service on port {}", Settings.getInstance().getDomainMapPort()); try { DomainMapService domainMapService = DomainMapService.getInstance(); domainMapService.start(); @@ -778,12 +778,12 @@ public void run() { if (ntpTime != null) { if (ntpTime != now) // Only log if non-zero offset - LOGGER.info(String.format("Adjusting system time by NTP offset: %dms", ntpTime - now)); + LOGGER.info("Adjusting system time by NTP offset: {}ms", ntpTime - now); ntpCheckTimestamp = now + NTP_POST_SYNC_CHECK_PERIOD; requestSysTrayUpdate = true; } else { - LOGGER.info(String.format("No NTP offset yet")); + LOGGER.info("No NTP offset yet"); ntpCheckTimestamp = now + NTP_PRE_SYNC_CHECK_PERIOD; // We can't do much without a valid NTP time continue; @@ -1116,6 +1116,11 @@ public void shutdown() { LOGGER.info("Shutting down synchronizer"); Synchronizer.getInstance().shutdown(); + try { + Synchronizer.getInstance().join(); + } catch (InterruptedException e) { + // We were interrupted while waiting for thread to join + } LOGGER.info("Shutting down API"); ApiService.getInstance().stop(); @@ -1176,6 +1181,14 @@ public void shutdown() { LOGGER.info("Shutting down TradeBot"); TradeBot.getInstance().shutdown(); + // Shutdown database cache timers before closing repository + LOGGER.info("Shutting down database cache timers"); + HSQLDBCacheUtils.shutdown(); + + // Shutdown arbitrary metadata manager scheduler before closing repository + LOGGER.info("Shutting down arbitrary metadata manager"); + ArbitraryMetadataManager.getInstance().shutdown(); + // Make sure we're the only thread modifying the blockchain when shutting down the repository ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); try { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index e1174158e..f5f7f0622 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -896,6 +896,6 @@ public static void shutdown() { LOGGER.debug("Balance Recorder Timer shutdown"); } - LOGGER.info("HSQLDBCacheUtils timers shut down complete"); + LOGGER.info("HSQLDBCacheUtils timers shutdown complete"); } } \ No newline at end of file From 64564aa7d0b730587de79c7d5b398aa1fa77d272 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 4 Jan 2026 15:39:20 +0100 Subject: [PATCH 14/28] Improve shutdown --- .../controller/TransactionImporter.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 6f9fa1774..37aa8af13 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -92,6 +92,27 @@ public void run() { public void shutdown() { isStopping = true; this.interrupt(); + + // Shutdown all schedulers + LOGGER.info("Shutting down TransactionImporter schedulers"); + try { + getTransactionMessageScheduler.shutdownNow(); + getUnconfirmedTransactionsMessageScheduler.shutdownNow(); + signatureMessageScheduler.shutdownNow(); + + if (!getTransactionMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("getTransactionMessageScheduler did not terminate in time"); + } + if (!getUnconfirmedTransactionsMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("getUnconfirmedTransactionsMessageScheduler did not terminate in time"); + } + if (!signatureMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("signatureMessageScheduler did not terminate in time"); + } + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while waiting for TransactionImporter schedulers to terminate", e); + Thread.currentThread().interrupt(); + } } @@ -397,6 +418,9 @@ public void onNetworkGetTransactionMessage(Peer peer, Message message) { } private void processNetworkGetTransactionMessages() { + if (Controller.isStopping()) { + return; + } try { List messagesToProcess; @@ -490,7 +514,6 @@ private static void sendTransactionMessage(String signature58, TransactionData d // Scheduled executor service to process messages every second private final ScheduledExecutorService getUnconfirmedTransactionsMessageScheduler = Executors.newScheduledThreadPool(1); - public void onNetworkGetUnconfirmedTransactionsMessage(Peer peer, Message message) { synchronized (getUnconfirmedTransactionsMessageLock) { getUnconfirmedTransactionsMessageList.add(new PeerMessage(peer, message)); @@ -498,6 +521,9 @@ public void onNetworkGetUnconfirmedTransactionsMessage(Peer peer, Message messag } private void processNetworkGetUnconfirmedTransactionsMessages() { + if (Controller.isStopping()) { + return; + } List messagesToProcess; synchronized (getUnconfirmedTransactionsMessageLock) { @@ -542,6 +568,9 @@ public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { } public void processNetworkTransactionSignaturesMessage() { + if (Controller.isStopping()) { + return; + } try { List messagesToProcess; From a33bdf8a2eb4c71c6fcf1ee1e7ec7d9f9a2a670f Mon Sep 17 00:00:00 2001 From: crowetic Date: Mon, 12 Jan 2026 17:53:41 -0800 Subject: [PATCH 15/28] Added connection spinup upon need only for Electrum-based connections. Added automatically generated header for connection to electrum servers on a per-connection basis to prevent banning based on header. Added back DigiByte servers from allseeingeye for cipig.net servers. Added modifications to release note generation script Added additional litecoin servers and removed broken ones. --- .../java/org/qortal/crosschain/Digibyte.java | 17 +- .../org/qortal/crosschain/ElectrumServer.java | 9 + .../java/org/qortal/crosschain/ElectrumX.java | 466 +++++++++++++----- .../java/org/qortal/crosschain/Litecoin.java | 85 ++-- .../generate-release-notes.sh | 3 +- 5 files changed, 410 insertions(+), 170 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Digibyte.java b/src/main/java/org/qortal/crosschain/Digibyte.java index 9ee1f06a7..cf600b808 100644 --- a/src/main/java/org/qortal/crosschain/Digibyte.java +++ b/src/main/java/org/qortal/crosschain/Digibyte.java @@ -43,14 +43,15 @@ public NetworkParameters getParams() { @Override public Collection getServers() { - return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=dgb - new Server("electrum.qortal.link", Server.ConnectionType.SSL, 55002), - new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20059), - new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20059), - new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20059) - ); + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=dgb + new Server("electrum.qortal.link", Server.ConnectionType.SSL, 55002), + new Server("electrum.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20059) + ); } @Override diff --git a/src/main/java/org/qortal/crosschain/ElectrumServer.java b/src/main/java/org/qortal/crosschain/ElectrumServer.java index bcf399983..d17316087 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumServer.java +++ b/src/main/java/org/qortal/crosschain/ElectrumServer.java @@ -18,6 +18,7 @@ public class ElectrumServer { private Socket socket; private Scanner scanner; private int nextId = 1; + private String clientName; private ChainableServerConnectionRecorder recorder; @@ -60,6 +61,14 @@ public int incrementNextId() { return nextId++; } + public String getClientName() { + return this.clientName; + } + + public void setClientName(String clientName) { + this.clientName = clientName; + } + public String write(byte[] bytes, String id) throws IOException { synchronized (this.serverLock) { diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index aedc93eb8..3b9738551 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -23,11 +23,16 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ThreadLocalRandom; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -/** ElectrumX network support for querying Bitcoiny-related info like block headers, transaction outputs, etc. */ +/** + * ElectrumX network support for querying Bitcoiny-related info like block + * headers, transaction outputs, etc. + */ public class ElectrumX extends BitcoinyBlockchainProvider { public static final String NULL_RESPONSE_FROM_ELECTRUM_X_SERVER = "Null response from ElectrumX server"; @@ -37,27 +42,35 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // See: https://electrumx.readthedocs.io/en/latest/protocol-changes.html private static final double MIN_PROTOCOL_VERSION = 1.2; private static final double MAX_PROTOCOL_VERSION = 2.0; // Higher than current latest, for hopeful future-proofing - private static final String CLIENT_NAME = "Qortal"; + private static final int MIN_TARGET_CONNECTIONS = 2; + private static final int DEFAULT_TARGET_CONNECTIONS = 3; + private static final int MAX_TARGET_CONNECTIONS = 12; private static final int BLOCK_HEADER_LENGTH = 80; - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" - private static final Pattern DAEMON_ERROR_REGEX = Pattern.compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such + // mempool or blockchain transaction. Use gettransaction for wallet + // transactions.'})" + private static final Pattern DAEMON_ERROR_REGEX = Pattern + .compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content - /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ + /** + * Error message sent by some ElectrumX servers when they don't support + * returning verbose transactions. + */ private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; private static final int RESPONSE_TIME_READINGS = 5; private static final long MAX_AVG_RESPONSE_TIME = 2000L; // ms public static final String MISSING_FEATURES_ERROR = "MISSING FEATURES ERROR"; public static final String EXPECTED_GENESIS_ERROR = "EXPECTED GENESIS ERROR"; - public static final int MINIMUM_CONNECTIONS = 30; - public static final int MAXIMUM_CONNECTIONS = 50; + private static final long IDLE_DISCONNECT_MS = 2 * 60 * 1000L; private ChainableServerConnectionRecorder recorder = new ChainableServerConnectionRecorder(100); // the minimum number of connections targeted for this foreign blockchain private int minimumConnections; + private final int maximumConnections; public static class Server implements ChainableServer { String hostname; @@ -134,8 +147,10 @@ public String toString() { return String.format("%s:%s:%d", this.connectionType.name(), this.hostname, this.port); } } + private Set servers = Collections.synchronizedSet(new HashSet<>()); - private List remainingServers = new ArrayList<>(); // this is only accessed in the scheduling thread, so it is not thread safe + private List remainingServers = new ArrayList<>(); // this is only accessed in the scheduling thread, + // so it is not thread safe private Set uselessServers = Collections.synchronizedSet(new HashSet<>()); private Set connections = Collections.synchronizedSet(new HashSet<>()); @@ -148,13 +163,14 @@ public String toString() { private static final int TX_CACHE_SIZE = 1000; - private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { - // This method is called just after a new entry has been added - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > TX_CACHE_SIZE; - } - }); + private final Map transactionCache = Collections + .synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > TX_CACHE_SIZE; + } + }); // Scheduled executor service to make connections private final ScheduledExecutorService scheduleMakeConnections = Executors.newScheduledThreadPool(1); @@ -165,20 +181,33 @@ public boolean removeEldestEntry(Map.Entry eldest) // Scheduled executor service to monitor connections private final ScheduledExecutorService scheduleMonitorConnections = Executors.newScheduledThreadPool(1); + private final Object connectionManagementLock = new Object(); + private final Object connectionListLock = new Object(); + private volatile boolean connectionManagementStarted = false; + private volatile long lastRpcTimeMs = 0L; + private final AtomicInteger inFlightRpcCount = new AtomicInteger(0); + // Constructors - public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { + public ElectrumX(String netId, String genesisHash, Collection initialServerList, + Map defaultPorts) { this.netId = netId; this.expectedGenesisHash = genesisHash; this.servers.addAll(initialServerList); this.defaultPorts.putAll(defaultPorts); - // the minimum is set to roughly 10% of the initial count - this.minimumConnections = (initialServerList.size() / 10) + 1; + int listSize = initialServerList.size(); + if (listSize == 0) { + this.maximumConnections = 0; + this.minimumConnections = 0; + return; + } - scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); - scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); - scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); + int scaledTarget = (listSize / 10) + 1; + int targetConnections = clamp(scaledTarget, MIN_TARGET_CONNECTIONS, MAX_TARGET_CONNECTIONS); + targetConnections = Math.min(listSize, Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS)); + this.maximumConnections = targetConnections; + this.minimumConnections = Math.max(1, Math.min(listSize, Math.max(1, targetConnections / 2))); } // Methods for use by other classes @@ -196,20 +225,23 @@ public String getNetId() { /** * Returns current blockchain height. *

+ * * @throws ForeignBlockchainException if error occurs */ @Override public int getCurrentHeight() throws ForeignBlockchainException { Object blockObj = this.rpc("blockchain.headers.subscribe").getResponse(); if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); + throw new ForeignBlockchainException.NetworkException( + "Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); JSONObject blockJson = (JSONObject) blockObj; Object heightObj = blockJson.get("height"); if (!(heightObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException("Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); + throw new ForeignBlockchainException.NetworkException( + "Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); return ((Long) heightObj).intValue(); } @@ -217,23 +249,28 @@ public int getCurrentHeight() throws ForeignBlockchainException { /** * Returns list of raw blocks, starting from startHeight inclusive. *

+ * * @throws ForeignBlockchainException if error occurs */ @Override public List getCompactBlocks(int startHeight, int count) throws ForeignBlockchainException { - throw new ForeignBlockchainException("getCompactBlocks not implemented for ElectrumX due to being specific to zcash"); + throw new ForeignBlockchainException( + "getCompactBlocks not implemented for ElectrumX due to being specific to zcash"); } /** - * Returns list of raw block headers, starting from startHeight inclusive. + * Returns list of raw block headers, starting from startHeight + * inclusive. *

+ * * @throws ForeignBlockchainException if error occurs */ @Override public List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException { Object blockObj = this.rpc("blockchain.block.headers", startHeight, count).getResponse(); if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.block.headers RPC"); + throw new ForeignBlockchainException.NetworkException( + "Unexpected output from ElectrumX blockchain.block.headers RPC"); JSONObject blockJson = (JSONObject) blockObj; @@ -241,7 +278,8 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig Object hexObj = blockJson.get("hex"); if (!(countObj instanceof Long) || !(hexObj instanceof String)) - throw new ForeignBlockchainException.NetworkException("Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); + throw new ForeignBlockchainException.NetworkException( + "Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); long returnedCount = (Long) countObj; String hex = (String) hexObj; @@ -250,12 +288,18 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig byte[] raw = HashCode.fromString(hex).asBytes(); - // Most chains use a fixed length 80 byte header, so block headers can be split up by dividing the hex into - // 80-byte segments. However, some chains such as DOGE use variable length headers due to AuxPoW or other - // reasons. In these cases we can identify the start of each block header by the location of the block version - // numbers. Each block starts with a version number, and for DOGE this is easily identifiable (6422788) at the - // time of writing (Jul 2021). If we encounter a chain that is using more generic version numbers (e.g. 1) - // and can't be used to accurately identify block indexes, then there are sufficient checks to ensure an + // Most chains use a fixed length 80 byte header, so block headers can be split + // up by dividing the hex into + // 80-byte segments. However, some chains such as DOGE use variable length + // headers due to AuxPoW or other + // reasons. In these cases we can identify the start of each block header by the + // location of the block version + // numbers. Each block starts with a version number, and for DOGE this is easily + // identifiable (6422788) at the + // time of writing (Jul 2021). If we encounter a chain that is using more + // generic version numbers (e.g. 1) + // and can't be used to accurately identify block indexes, then there are + // sufficient checks to ensure an // exception is thrown. if (raw.length == returnedCount * BLOCK_HEADER_LENGTH) { @@ -263,8 +307,7 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig for (int i = 0; i < returnedCount; ++i) { rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); } - } - else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { + } else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { // Assume AuxPoW variable length header (DOGE) int referenceVersion = BitTwiddling.intFromLEBytes(raw, 0); // DOGE uses 6422788 at time of commit (Jul 2021) for (int i = 0; i < raw.length - 4; ++i) { @@ -275,19 +318,22 @@ else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { } // Ensure that we found the correct number of block headers if (rawBlockHeaders.size() != count) { - throw new ForeignBlockchainException.NetworkException("Unexpected raw header contents in JSON from ElectrumX blockchain.block.headers RPC."); + throw new ForeignBlockchainException.NetworkException( + "Unexpected raw header contents in JSON from ElectrumX blockchain.block.headers RPC."); } - } - else if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) { - throw new ForeignBlockchainException.NetworkException("Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); + } else if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) { + throw new ForeignBlockchainException.NetworkException( + "Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); } return rawBlockHeaders; } /** - * Returns list of raw block timestamps, starting from startHeight inclusive. + * Returns list of raw block timestamps, starting from startHeight + * inclusive. *

+ * * @throws ForeignBlockchainException if error occurs */ @Override @@ -299,6 +345,7 @@ public List getBlockTimestamps(int startHeight, int count) throws ForeignB /** * Returns confirmed balance, based on passed payment script. *

+ * * @return confirmed balance, or zero if script unknown * @throws ForeignBlockchainException if there was an error */ @@ -307,16 +354,19 @@ public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()).getResponse(); + Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()) + .getResponse(); if (!(balanceObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); + throw new ForeignBlockchainException.NetworkException( + "Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); JSONObject balanceJson = (JSONObject) balanceObj; Object confirmedBalanceObj = balanceJson.get("confirmed"); if (!(confirmedBalanceObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException("Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); + throw new ForeignBlockchainException.NetworkException( + "Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); return (Long) balanceJson.get("confirmed"); } @@ -324,6 +374,7 @@ public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException /** * Returns confirmed balance, based on passed base58 encoded address. *

+ * * @return confirmed balance, or zero if address unknown * @throws ForeignBlockchainException if there was an error */ @@ -335,11 +386,13 @@ public long getConfirmedAddressBalance(String base58Address) throws ForeignBlock /** * Returns list of unspent outputs pertaining to passed address. *

+ * * @return list of unspent outputs, or empty list if address unknown * @throws ForeignBlockchainException if there was an error. */ @Override - public List getUnspentOutputs(String address, boolean includeUnconfirmed) throws ForeignBlockchainException { + public List getUnspentOutputs(String address, boolean includeUnconfirmed) + throws ForeignBlockchainException { byte[] script = this.blockchain.addressToScriptPubKey(address); return this.getUnspentOutputs(script, includeUnconfirmed); } @@ -347,24 +400,29 @@ public List getUnspentOutputs(String address, boolean includeUnco /** * Returns list of unspent outputs pertaining to passed payment script. *

+ * * @return list of unspent outputs, or empty list if script unknown * @throws ForeignBlockchainException if there was an error. */ @Override - public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { + public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) + throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()).getResponse(); + Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()) + .getResponse(); if (!(unspentJson instanceof JSONArray)) - throw new ForeignBlockchainException("Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); + throw new ForeignBlockchainException( + "Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); List unspentOutputs = new ArrayList<>(); for (Object rawUnspent : (JSONArray) unspentJson) { JSONObject unspent = (JSONObject) rawUnspent; int height = ((Long) unspent.get("height")).intValue(); - // We only want unspent outputs from confirmed transactions (and definitely not mempool duplicates with height 0) + // We only want unspent outputs from confirmed transactions (and definitely not + // mempool duplicates with height 0) if (!includeUnconfirmed && height <= 0) continue; @@ -384,7 +442,7 @@ public List getUnspentOutputs(byte[] script, boolean includeUncon * NOTE: Do not mutate returned byte[]! * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException { @@ -392,7 +450,8 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException try { rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false).getResponse(); } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain + // transaction. Use gettransaction for wallet transactions.'}) if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) throw new ForeignBlockchainException.NotFoundException(e.getMessage()); @@ -400,7 +459,8 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException } if (!(rawTransactionHex instanceof String)) - throw new ForeignBlockchainException.NetworkException("Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException( + "Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); return HashCode.fromString((String) rawTransactionHex).asBytes(); } @@ -411,7 +471,7 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException * NOTE: Do not mutate returned byte[]! * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException { @@ -421,8 +481,9 @@ public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException /** * Returns transaction info for passed transaction hash. *

+ * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { @@ -439,7 +500,8 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai serverResponse = this.rpc("blockchain.transaction.get", txHash, true); transactionObj = serverResponse.getResponse(); } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain + // transaction. Use gettransaction for wallet transactions.'}) if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) throw new ForeignBlockchainException.NotFoundException(e.getMessage()); @@ -448,17 +510,20 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } while (transactionObj == null); if (!(transactionObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException( + "Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); JSONObject transactionJson = (JSONObject) transactionObj; Object inputsObj = transactionJson.get("vin"); if (!(inputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException( + "Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); Object outputsObj = transactionJson.get("vout"); if (!(outputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException( + "Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); try { int size = ((Long) transactionJson.get("size")).intValue(); @@ -509,9 +574,12 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } // For the purposes of Qortal we require all outputs to contain addresses - // Some servers omit this info, causing problems down the line with balance calculations - // Update: it turns out that they were just using a different key - "address" instead of "addresses" - // The code below can remain in place, just in case a peer returns a missing address in the future + // Some servers omit this info, causing problems down the line with balance + // calculations + // Update: it turns out that they were just using a different key - "address" + // instead of "addresses" + // The code below can remain in place, just in case a peer returns a missing + // address in the future if (addresses == null || addresses.isEmpty()) { final String message = String.format("No output addresses returned for transaction %s", txHash); LOGGER.warn("{}: No output addresses returned for transaction {}", this.blockchain.getCurrencyCode(), txHash); @@ -526,7 +594,7 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs); // Save into cache, if and only if it has been confirmed - if( transaction.timestamp != null ) { + if (transaction.timestamp != null) { transactionCache.put(txHash, transaction); } @@ -536,25 +604,30 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } this.connections.remove(serverResponse.getElectrumServer()); - serverResponse.getElectrumServer().closeServer(this.getClass().getSimpleName(), "Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); + serverResponse.getElectrumServer().closeServer(this.getClass().getSimpleName(), + "Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); return getTransaction(txHash); } /** * Returns list of transactions, relating to passed payment script. *

+ * * @return list of related transactions, or empty list if script unknown * @throws ForeignBlockchainException if error occurs */ @Override - public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { + public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) + throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - ElectrumServerResponse serverResponse = this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); + ElectrumServerResponse serverResponse = this.rpc("blockchain.scripthash.get_history", + HashCode.fromBytes(scriptHash).toString()); Object transactionsJson = serverResponse.getResponse(); if (!(transactionsJson instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); + throw new ForeignBlockchainException.NetworkException( + "Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); List transactionHashes = new ArrayList<>(); @@ -575,32 +648,39 @@ public List getAddressTransactions(byte[] script, boolean inclu } @Override - public List getAddressBitcoinyTransactions(String address, boolean includeUnconfirmed) throws ForeignBlockchainException { - // FUTURE: implement this if needed. For now we use getAddressTransactions() + getTransaction() + public List getAddressBitcoinyTransactions(String address, boolean includeUnconfirmed) + throws ForeignBlockchainException { + // FUTURE: implement this if needed. For now we use getAddressTransactions() + + // getTransaction() throw new ForeignBlockchainException("getAddressBitcoinyTransactions not yet implemented for ElectrumX"); } /** * Broadcasts raw transaction to network. *

+ * * @throws ForeignBlockchainException if error occurs */ @Override public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException { - Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()).getResponse(); + Object rawBroadcastResult = this + .rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()).getResponse(); // We're expecting a simple string that is the transaction hash if (!(rawBroadcastResult instanceof String)) - throw new ForeignBlockchainException.NetworkException("Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); + throw new ForeignBlockchainException.NetworkException( + "Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); } // Class-private utility methods /** - * Query current server for its list of peer servers, and return those we can parse. + * Query current server for its list of peer servers, and return those we can + * parse. *

+ * * @throws ForeignBlockchainException - * @throws ClassCastException to be handled by caller + * @throws ClassCastException to be handled by caller */ private Set serverPeersSubscribe() { Set newServers = new HashSet<>(); @@ -608,14 +688,16 @@ private Set serverPeersSubscribe() { List electrumServers = acquireServers(); try { - for( ElectrumServer electrumServer : electrumServers ) { + for (ElectrumServer electrumServer : electrumServers) { Object peers = this.connectedRpc(electrumServer, "server.peers.subscribe"); - if( peers == null ) continue; + if (peers == null) + continue; Object peersObject = Objects.requireNonNull(peers); - if( !(peersObject instanceof JSONArray) ) continue; + if (!(peersObject instanceof JSONArray)) + continue; for (Object rawPeer : (JSONArray) peersObject) { @@ -669,7 +751,7 @@ private Set serverPeersSubscribe() { } catch (Exception e) { LOGGER.error(e.getMessage(), e); } finally { - for( ElectrumServer server : electrumServers ) { + for (ElectrumServer server : electrumServers) { releaseServer(server); } } @@ -681,8 +763,7 @@ private ElectrumServer acquireServer() throws ForeignBlockchainException { try { return this.availableConnections.take(); - } - catch( InterruptedException e ) { + } catch (InterruptedException e) { throw new ForeignBlockchainException(e.getMessage()); } } @@ -733,28 +814,89 @@ public static List drainRandomly(BlockingQueue queue, int numToDrain) return drainedList; } - private void releaseServer( ElectrumServer server ) { + private void releaseServer(ElectrumServer server) { // if the connection is still open - if( this.connections.contains(server)) + if (this.connections.contains(server)) this.availableConnections.add(server); } + private boolean isIdle() { + if (this.inFlightRpcCount.get() > 0) { + return false; + } + if (this.lastRpcTimeMs <= 0L) { + return true; + } + return System.currentTimeMillis() - this.lastRpcTimeMs > IDLE_DISCONNECT_MS; + } + + private void closeAllConnections(String reason) { + synchronized (this.connectionListLock) { + for (ElectrumServer server : new HashSet<>(this.connections)) { + this.connections.remove(server); + server.closeServer(this.getClass().getSimpleName(), reason); + } + this.availableConnections.clear(); + this.remainingServers.clear(); + } + } + + /** + * Ensure the connection maintenance threads are running and initial connections + * exist. + */ + private void ensureConnectionManagementStarted() { + if (this.connectionManagementStarted) { + return; + } + + boolean shouldInit = false; + synchronized (this.connectionManagementLock) { + if (!this.connectionManagementStarted) { + this.connectionManagementStarted = true; + shouldInit = true; + } + } + + if (!shouldInit) { + return; + } + + startMakingConnections(); + + this.scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); + this.scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); + this.scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); + } + /** - *

Performs RPC call, with automatic reconnection to different server if needed. + *

+ * Performs RPC call, with automatic reconnection to different server if needed. *

+ * * @param method String representation of the RPC call value * @param params a list of Objects passed to the method of the Remote Server * @return "result" object from within JSON output - * @throws ForeignBlockchainException if server returns error or something goes wrong + * @throws ForeignBlockchainException if server returns error or something goes + * wrong */ - private ElectrumServerResponse rpc(String method, Object...params) throws ForeignBlockchainException { + private ElectrumServerResponse rpc(String method, Object... params) throws ForeignBlockchainException { + this.inFlightRpcCount.incrementAndGet(); + this.lastRpcTimeMs = System.currentTimeMillis(); + try { + ensureConnectionManagementStarted(); + if (this.availableConnections.isEmpty()) { + LOGGER.debug("{} no available ElectrumX connections; starting connections on demand", + this.blockchain.getCurrencyCode()); + startMakingConnections(); + } ElectrumServer electrumServer = acquireServer(); Object response = null; - while(response == null) { + while (response == null) { response = connectedRpc(electrumServer, method, params); @@ -762,7 +904,8 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig if (!this.availableConnections.isEmpty()) { long averageResponseTime = electrumServer.averageResponseTime(); if (averageResponseTime > MAX_AVG_RESPONSE_TIME) { - String message = String.format("Slow average response time %dms from %s - trying another server...", averageResponseTime, electrumServer.getServer()); + String message = String.format("Slow average response time %dms from %s - trying another server...", + averageResponseTime, electrumServer.getServer()); LOGGER.info(message); electrumServer.closeServer(this.getClass().getSimpleName(), message); break; @@ -771,6 +914,7 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig if (response != null) { releaseServer(electrumServer); + this.lastRpcTimeMs = System.currentTimeMillis(); return new ElectrumServerResponse(electrumServer, response); } @@ -785,6 +929,9 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig // Failed to perform RPC - maybe lack of servers? LOGGER.info("Error: No connected Electrum servers when trying to make RPC call"); throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); + } finally { + this.inFlightRpcCount.decrementAndGet(); + } } /** @@ -794,14 +941,19 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig */ private void monitorConnections() { + if (this.isIdle() && !this.connections.isEmpty()) { + LOGGER.info("{} idle; closing {} ElectrumX connections", this.blockchain.getCurrencyCode(), + this.connections.size()); + this.closeAllConnections("idle timeout"); + } + LOGGER.info( - "{} {} available connections, {} total servers, {} total connections, {} useless servers", - this.blockchain.getCurrencyCode(), - this.availableConnections.size(), - this.servers.size(), - this.connections.size(), - this.uselessServers.size() - ); + "{} {} available connections, {} total servers, {} total connections, {} useless servers", + this.blockchain.getCurrencyCode(), + this.availableConnections.size(), + this.servers.size(), + this.connections.size(), + this.uselessServers.size()); } /** @@ -812,7 +964,10 @@ private void monitorConnections() { private void makeConnections() { try { - if( this.connections.isEmpty() ) { + if (this.isIdle()) { + return; + } + if (this.connections.isEmpty()) { startMakingConnections(); } @@ -825,12 +980,16 @@ private void makeConnections() { /** * Recover Connections * - * If connection count is below the minimum, then recover connections from the initial list. + * If connection count is below the minimum, then recover connections from the + * initial list. */ private void recoverConnections() { try { - if( this.connections.size() < this.minimumConnections ) { + if (this.isIdle()) { + return; + } + if (this.connections.size() < this.minimumConnections) { LOGGER.debug("{} recovering connections", this.blockchain.currencyCode); startMakingConnections(); LOGGER.debug("{} recovered {} connections", this.blockchain.currencyCode, this.connections.size()); @@ -844,10 +1003,12 @@ private void recoverConnections() { * Start Making Connections */ private void startMakingConnections() { - - // assume there are no server to get peers from, so we must start from the base list - this.remainingServers.clear(); - this.remainingServers.addAll(this.servers); + synchronized (this.connectionListLock) { + // assume there are no server to get peers from, so we must start from the base + // list + this.remainingServers.clear(); + this.remainingServers.addAll(this.servers); + } connectRemainingServers(); } @@ -860,19 +1021,22 @@ private void startMakingConnections() { private void makeMoreConnections() { // if we need more connections - if(this.connections.size() < MINIMUM_CONNECTIONS) { + if (this.connections.size() < this.maximumConnections) { // Ask for more servers Set moreServers = serverPeersSubscribe(); - // Add all servers to base list - this.servers.addAll(moreServers); + synchronized (this.connectionListLock) { + // Add all servers to base list + this.servers.addAll(moreServers); - // add base list to remaining list - this.remainingServers.addAll(this.servers); + // add base list to remaining list + this.remainingServers.addAll(this.servers); - // remove servers that this node is already connected to - this.remainingServers.removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); + // remove servers that this node is already connected to + this.remainingServers + .removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); + } // try connecting the remaining servers connectRemainingServers(); @@ -881,13 +1045,39 @@ private void makeMoreConnections() { private void connectRemainingServers() { // while there are remaining servers and less than the maximum connections - while( !this.remainingServers.isEmpty() && this.connections.size() < MAXIMUM_CONNECTIONS ) { - ChainableServer server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); + while (true) { + ChainableServer server; + synchronized (this.connectionListLock) { + if (this.remainingServers.isEmpty() || this.connections.size() >= this.maximumConnections) { + return; + } + server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); + } makeConnection(server, this.getClass().getSimpleName()); } } + private static int clamp(int value, int min, int max) { + if (value < min) { + return min; + } + if (value > max) { + return max; + } + return value; + } + + private static String randomClientName() { + final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + StringBuilder name = new StringBuilder(12); + ThreadLocalRandom random = ThreadLocalRandom.current(); + for (int i = 0; i < 12; i++) { + name.append(alphabet.charAt(random.nextInt(alphabet.length()))); + } + return name.toString(); + } + private Optional makeConnection(ChainableServer server, String requestedBy) { LOGGER.debug(() -> String.format("Connecting to %s %s", server, this.blockchain.currencyCode)); @@ -896,38 +1086,44 @@ private Optional makeConnection(ChainableServer serve int timeout = 5000; // ms ElectrumServer electrumServer = ElectrumServer.createInstance(server, endpoint, timeout, this.recorder); + electrumServer.setClientName(randomClientName()); // All connections need to start with a version negotiation this.connectedRpc(electrumServer, "server.version"); - // Check connection is suitable by asking for server features, including genesis block hash + // Check connection is suitable by asking for server features, including genesis + // block hash JSONObject featuresJson = (JSONObject) this.connectedRpc(electrumServer, "server.features"); - if (featuresJson == null ) - return Optional.of( recorder.recordConnection(server, requestedBy, true, false, MISSING_FEATURES_ERROR) ); + if (featuresJson == null) + return Optional.of(recorder.recordConnection(server, requestedBy, true, false, MISSING_FEATURES_ERROR)); try { double protocol_min = CrossChainUtils.getVersionDecimal(featuresJson, "protocol_min"); if (protocol_min < MIN_PROTOCOL_VERSION) - return Optional.of( recorder.recordConnection(server, requestedBy, true, false, "old version: protocol_min = " + protocol_min + " < MIN_PROTOCOL_VERSION = " + MIN_PROTOCOL_VERSION) ); + return Optional.of(recorder.recordConnection(server, requestedBy, true, false, + "old version: protocol_min = " + protocol_min + " < MIN_PROTOCOL_VERSION = " + MIN_PROTOCOL_VERSION)); } catch (NumberFormatException e) { - return Optional.of( recorder.recordConnection(server, requestedBy,true, false,featuresJson.get("protocol_min").toString() + " is not a valid version")); + return Optional.of(recorder.recordConnection(server, requestedBy, true, false, + featuresJson.get("protocol_min").toString() + " is not a valid version")); } catch (NullPointerException e) { - return Optional.of( recorder.recordConnection(server, requestedBy,true, false,"server version not available: protocol_min")); + return Optional.of( + recorder.recordConnection(server, requestedBy, true, false, "server version not available: protocol_min")); } - if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) - return Optional.of( recorder.recordConnection(server, requestedBy, true, false, EXPECTED_GENESIS_ERROR) ); + if (this.expectedGenesisHash != null + && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) + return Optional.of(recorder.recordConnection(server, requestedBy, true, false, EXPECTED_GENESIS_ERROR)); LOGGER.debug(() -> String.format("Connected to %s %s", server, this.blockchain.currencyCode)); this.connections.add(electrumServer); this.availableConnections.add(electrumServer); - return Optional.of( this.recorder.recordConnection( server, requestedBy, true, true, EMPTY) ); + return Optional.of(this.recorder.recordConnection(server, requestedBy, true, true, EMPTY)); } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { // Didn't work, try another server... - return Optional.of( this.recorder.recordConnection( server, requestedBy, true, false, CrossChainUtils.getNotes(e))); - } catch( Exception e ) { + return Optional.of(this.recorder.recordConnection(server, requestedBy, true, false, CrossChainUtils.getNotes(e))); + } catch (Exception e) { LOGGER.error(e.getMessage(), e); return Optional.empty(); } @@ -936,13 +1132,15 @@ private Optional makeConnection(ChainableServer serve /** * Perform RPC using currently connected server. *

+ * * @param method * @param params * @return response Object, or null if server fails to respond * @throws ForeignBlockchainException if server returns error */ @SuppressWarnings("unchecked") - private Object connectedRpc(ElectrumServer server, String method, Object...params) throws ForeignBlockchainException { + private Object connectedRpc(ElectrumServer server, String method, Object... params) + throws ForeignBlockchainException { JSONObject requestJson = new JSONObject(); String id = UUID.randomUUID().toString(); requestJson.put("id", id); @@ -954,7 +1152,12 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param // server.version needs additional params to negotiate a version if (method.equals("server.version")) { - requestParams.add(CLIENT_NAME); + String clientName = server.getClientName(); + if (clientName == null) { + clientName = randomClientName(); + server.setClientName(clientName); + } + requestParams.add(clientName); List versions = new ArrayList<>(); DecimalFormat df = new DecimalFormat("#.#"); versions.add(df.format(MIN_PROTOCOL_VERSION)); @@ -982,10 +1185,10 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param } long endTime = System.currentTimeMillis(); - long responseTime = endTime-startTime; + long responseTime = endTime - startTime; LOGGER.trace(() -> String.format("Request: %s Response: %s", request, response)); - LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime)); + LOGGER.trace(() -> String.format("Time taken: %dms", endTime - startTime)); if (response.isEmpty()) // Empty response - try another server? @@ -1003,13 +1206,15 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param Object errorObj = responseJson.get("error"); if (errorObj != null) { if (errorObj instanceof String) { - LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", server.getServer(), method, (String) errorObj)); + LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", + server.getServer(), method, (String) errorObj)); // Try another server return null; } if (!(errorObj instanceof JSONObject)) { - LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", server.getServer(), method)); + LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", + server.getServer(), method)); // Try another server return null; } @@ -1019,7 +1224,9 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param Object messageObj = errorJson.get("message"); if (!(messageObj instanceof String)) { - LOGGER.debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", server.getServer(), method)); + LOGGER + .debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", + server.getServer(), method)); // Try another server return null; } @@ -1027,7 +1234,9 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param String message = (String) messageObj; // Some error 'messages' are actually wrapped upstream bitcoind errors: - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such + // mempool or blockchain transaction. Use gettransaction for wallet + // transactions.'})" // We want to detect these and extract the upstream error code for caller's use Matcher messageMatcher = DAEMON_ERROR_REGEX.matcher(message); if (messageMatcher.find()) @@ -1035,7 +1244,8 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param int daemonErrorCode = Integer.parseInt(messageMatcher.group(1)); throw new ForeignBlockchainException.NetworkException(daemonErrorCode, message, server.getServer()); } catch (NumberFormatException e) { - // We couldn't parse the error code integer? Fall-through to generic exception... + // We couldn't parse the error code integer? Fall-through to generic + // exception... } throw new ForeignBlockchainException.NetworkException(message, server.getServer()); @@ -1046,7 +1256,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param @Override public Set getServers() { - return new HashSet<>(this.servers ); + return new HashSet<>(this.servers); } @Override diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index b27e77f18..958ea801b 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -32,8 +32,10 @@ public class Litecoin extends Bitcoiny { private static final long MAINNET_FEE = 1000L; private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST - private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); - public static final LitecoinMainNetParamsP2ShOverride MAIN_NET_PARAMS_P2SH_OVERRIDE = new LitecoinMainNetParamsP2ShOverride(50); + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>( + ElectrumX.Server.ConnectionType.class); + public static final LitecoinMainNetParamsP2ShOverride MAIN_NET_PARAMS_P2SH_OVERRIDE = new LitecoinMainNetParamsP2ShOverride( + 50); static { DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); @@ -50,17 +52,30 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=ltc - new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), - new Server("electrum.qortal.link", Server.ConnectionType.SSL, 50002), - new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), - new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("ltc.rentonrisk.com", Server.ConnectionType.SSL, 50002) - ); + // Servers chosen on NO BASIS WHATSOEVER from various sources! + // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=ltc + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 50002), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.jochen-hoenicke.de", Server.ConnectionType.SSL, 50091), + new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("electrum.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), + new Server("fury.fiatfaucet.com", Server.ConnectionType.SSL, 50002), + new Server("ltc-electrum.cakewallet.com", Server.ConnectionType.SSL, 50002), + new Server("litecoin.stackwallet.com", Server.ConnectionType.SSL, 20063), + new Server("ltc.aftrek.org", Server.ConnectionType.SSL, 50002), + new Server("137.184.250.112", Server.ConnectionType.SSL, 50002), + new Server("146.190.15.65", Server.ConnectionType.SSL, 50002), + new Server("157.230.64.188", Server.ConnectionType.SSL, 50002), + new Server("209.38.53.75", Server.ConnectionType.SSL, 50002), + new Server("24.199.78.132", Server.ConnectionType.SSL, 50002), + new Server("5.78.97.174", Server.ConnectionType.SSL, 50002), + new Server("188.166.208.106", Server.ConnectionType.SSL, 50002), + new Server("5.161.216.180", Server.ConnectionType.SSL, 50002)); } @Override @@ -82,9 +97,8 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002) - ); + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002)); } @Override @@ -106,9 +120,8 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - new Server("localhost", Server.ConnectionType.TCP, 50001), - new Server("localhost", Server.ConnectionType.SSL, 50002) - ); + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002)); } @Override @@ -134,8 +147,11 @@ public void setFeeRequired(long feeRequired) { } public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; } @@ -145,7 +161,8 @@ public void setFeeRequired(long feeRequired) { // Constructors and instance - private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, + String currencyCode) { super(blockchain, bitcoinjContext, currencyCode, DEFAULT_FEE_PER_KB); this.litecoinNet = litecoinNet; @@ -156,7 +173,8 @@ public static synchronized Litecoin getInstance() { if (instance == null) { LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet(); - BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), + litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); Context bitcoinjContext = new Context(litecoinNet.getParams()); instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); @@ -181,10 +199,12 @@ public long getMinimumOrderAmount() { } /** - * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp. + * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic + * timestamp. * * @param timestamp optional milliseconds since epoch, or null for 'now' - * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + * @return sats per 1000bytes, or throws ForeignBlockchainException if something + * went wrong */ @Override public long getP2shFee(Long timestamp) throws ForeignBlockchainException { @@ -199,7 +219,7 @@ public long getFeeRequired() { @Override public void setFeeRequired(long fee) { - this.litecoinNet.setFeeRequired( fee ); + this.litecoinNet.setFeeRequired(fee); } /** @@ -225,7 +245,8 @@ public boolean isCurrentP2ShAddress(String address) { /** * Convert Current P2SH Address * - * Convert a p2sh address conforming the current standard prefix 'M', to the internal standard here + * Convert a p2sh address conforming the current standard prefix 'M', to the + * internal standard here * using prefix '3' * * @param address the p2sh address, starts with 'M' @@ -234,10 +255,9 @@ public boolean isCurrentP2ShAddress(String address) { */ public String convertCurrentP2ShAddress(String address) { - if( isCurrentP2ShAddress(address) ) { + if (isCurrentP2ShAddress(address)) { return convertP2SHAddress(address, MAIN_NET_PARAMS_P2SH_OVERRIDE, this.params); - } - else { + } else { throw new AddressFormatException("this is not a current p2sh address for Litecoin"); } } @@ -248,12 +268,13 @@ public String convertCurrentP2ShAddress(String address) { * Convert p2sh address from one network standard to another. * * @param p2shAddress the p2sh address - * @param fromParams the existing standard - * @param toParams the desired standard + * @param fromParams the existing standard + * @param toParams the desired standard * * @return the p2sh conforming to the desired standard */ - private static String convertP2SHAddress(String p2shAddress, NetworkParameters fromParams, NetworkParameters toParams) { + private static String convertP2SHAddress(String p2shAddress, NetworkParameters fromParams, + NetworkParameters toParams) { try { // decode the P2SH address Address address = LegacyAddress.fromBase58(fromParams, p2shAddress); @@ -269,4 +290,4 @@ private static String convertP2SHAddress(String p2shAddress, NetworkParameters f return null; } } -} \ No newline at end of file +} diff --git a/tools/auto-update-scripts/generate-release-notes.sh b/tools/auto-update-scripts/generate-release-notes.sh index c0f44f390..1e4e21c8e 100755 --- a/tools/auto-update-scripts/generate-release-notes.sh +++ b/tools/auto-update-scripts/generate-release-notes.sh @@ -126,7 +126,7 @@ fi # Get changelog between previous and current commit echo "Generating changelog between ${PREV_BUMP_COMMIT} and ${CURRENT_BUMP_COMMIT}..." -CHANGELOG=$(curl -s "https://api.github.com/repos/${REPO}/compare/${PREV_BUMP_COMMIT}...${CURRENT_BUMP_COMMIT}" | jq -r '.commits[] | "- " + .sha[0:7] + " " + .commit.message') +CHANGELOG=$(curl -s "https://api.github.com/repos/${REPO}/compare/${PREV_BUMP_COMMIT}...${CURRENT_BUMP_COMMIT}" | jq -r '.commits[] | .sha[0:7] as $sha | (.commit.message | split("\n")) as $lines | ($lines[0]) as $title | ($lines[1:] | map(select(length > 0))) as $body | "- " + $title + "\n - " + $sha + (if ($body | length) > 0 then "\n " + ($body | join("\n ")) else "" end)') # Fetch latest commit timestamp from GitHub API for final file timestamping COMMIT_API_URL="https://api.github.com/repos/${REPO}/commits?sha=${BRANCH}&per_page=1" @@ -235,4 +235,3 @@ Packed with \`7z a -r -tzip qortal.zip qortal/\` EOF echo "Release notes generated: release-notes.txt" - From d63d02a42e5c9f3eb3aa1fb4295193bcbd05fa3a Mon Sep 17 00:00:00 2001 From: crowetic Date: Mon, 12 Jan 2026 18:10:03 -0800 Subject: [PATCH 16/28] removed unnecessary formatting changes --- .../java/org/qortal/crosschain/ElectrumX.java | 466 +++++------------- .../java/org/qortal/crosschain/Litecoin.java | 85 ++-- 2 files changed, 160 insertions(+), 391 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 3b9738551..aedc93eb8 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -23,16 +23,11 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.ThreadLocalRandom; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -/** - * ElectrumX network support for querying Bitcoiny-related info like block - * headers, transaction outputs, etc. - */ +/** ElectrumX network support for querying Bitcoiny-related info like block headers, transaction outputs, etc. */ public class ElectrumX extends BitcoinyBlockchainProvider { public static final String NULL_RESPONSE_FROM_ELECTRUM_X_SERVER = "Null response from ElectrumX server"; @@ -42,35 +37,27 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // See: https://electrumx.readthedocs.io/en/latest/protocol-changes.html private static final double MIN_PROTOCOL_VERSION = 1.2; private static final double MAX_PROTOCOL_VERSION = 2.0; // Higher than current latest, for hopeful future-proofing - private static final int MIN_TARGET_CONNECTIONS = 2; - private static final int DEFAULT_TARGET_CONNECTIONS = 3; - private static final int MAX_TARGET_CONNECTIONS = 12; + private static final String CLIENT_NAME = "Qortal"; private static final int BLOCK_HEADER_LENGTH = 80; - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such - // mempool or blockchain transaction. Use gettransaction for wallet - // transactions.'})" - private static final Pattern DAEMON_ERROR_REGEX = Pattern - .compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + private static final Pattern DAEMON_ERROR_REGEX = Pattern.compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content - /** - * Error message sent by some ElectrumX servers when they don't support - * returning verbose transactions. - */ + /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; private static final int RESPONSE_TIME_READINGS = 5; private static final long MAX_AVG_RESPONSE_TIME = 2000L; // ms public static final String MISSING_FEATURES_ERROR = "MISSING FEATURES ERROR"; public static final String EXPECTED_GENESIS_ERROR = "EXPECTED GENESIS ERROR"; - private static final long IDLE_DISCONNECT_MS = 2 * 60 * 1000L; + public static final int MINIMUM_CONNECTIONS = 30; + public static final int MAXIMUM_CONNECTIONS = 50; private ChainableServerConnectionRecorder recorder = new ChainableServerConnectionRecorder(100); // the minimum number of connections targeted for this foreign blockchain private int minimumConnections; - private final int maximumConnections; public static class Server implements ChainableServer { String hostname; @@ -147,10 +134,8 @@ public String toString() { return String.format("%s:%s:%d", this.connectionType.name(), this.hostname, this.port); } } - private Set servers = Collections.synchronizedSet(new HashSet<>()); - private List remainingServers = new ArrayList<>(); // this is only accessed in the scheduling thread, - // so it is not thread safe + private List remainingServers = new ArrayList<>(); // this is only accessed in the scheduling thread, so it is not thread safe private Set uselessServers = Collections.synchronizedSet(new HashSet<>()); private Set connections = Collections.synchronizedSet(new HashSet<>()); @@ -163,14 +148,13 @@ public String toString() { private static final int TX_CACHE_SIZE = 1000; - private final Map transactionCache = Collections - .synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { - // This method is called just after a new entry has been added - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > TX_CACHE_SIZE; - } - }); + private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > TX_CACHE_SIZE; + } + }); // Scheduled executor service to make connections private final ScheduledExecutorService scheduleMakeConnections = Executors.newScheduledThreadPool(1); @@ -181,33 +165,20 @@ public boolean removeEldestEntry(Map.Entry eldest) // Scheduled executor service to monitor connections private final ScheduledExecutorService scheduleMonitorConnections = Executors.newScheduledThreadPool(1); - private final Object connectionManagementLock = new Object(); - private final Object connectionListLock = new Object(); - private volatile boolean connectionManagementStarted = false; - private volatile long lastRpcTimeMs = 0L; - private final AtomicInteger inFlightRpcCount = new AtomicInteger(0); - // Constructors - public ElectrumX(String netId, String genesisHash, Collection initialServerList, - Map defaultPorts) { + public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { this.netId = netId; this.expectedGenesisHash = genesisHash; this.servers.addAll(initialServerList); this.defaultPorts.putAll(defaultPorts); - int listSize = initialServerList.size(); - if (listSize == 0) { - this.maximumConnections = 0; - this.minimumConnections = 0; - return; - } + // the minimum is set to roughly 10% of the initial count + this.minimumConnections = (initialServerList.size() / 10) + 1; - int scaledTarget = (listSize / 10) + 1; - int targetConnections = clamp(scaledTarget, MIN_TARGET_CONNECTIONS, MAX_TARGET_CONNECTIONS); - targetConnections = Math.min(listSize, Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS)); - this.maximumConnections = targetConnections; - this.minimumConnections = Math.max(1, Math.min(listSize, Math.max(1, targetConnections / 2))); + scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); + scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); + scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); } // Methods for use by other classes @@ -225,23 +196,20 @@ public String getNetId() { /** * Returns current blockchain height. *

- * * @throws ForeignBlockchainException if error occurs */ @Override public int getCurrentHeight() throws ForeignBlockchainException { Object blockObj = this.rpc("blockchain.headers.subscribe").getResponse(); if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException( - "Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); JSONObject blockJson = (JSONObject) blockObj; Object heightObj = blockJson.get("height"); if (!(heightObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException( - "Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); return ((Long) heightObj).intValue(); } @@ -249,28 +217,23 @@ public int getCurrentHeight() throws ForeignBlockchainException { /** * Returns list of raw blocks, starting from startHeight inclusive. *

- * * @throws ForeignBlockchainException if error occurs */ @Override public List getCompactBlocks(int startHeight, int count) throws ForeignBlockchainException { - throw new ForeignBlockchainException( - "getCompactBlocks not implemented for ElectrumX due to being specific to zcash"); + throw new ForeignBlockchainException("getCompactBlocks not implemented for ElectrumX due to being specific to zcash"); } /** - * Returns list of raw block headers, starting from startHeight - * inclusive. + * Returns list of raw block headers, starting from startHeight inclusive. *

- * * @throws ForeignBlockchainException if error occurs */ @Override public List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException { Object blockObj = this.rpc("blockchain.block.headers", startHeight, count).getResponse(); if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException( - "Unexpected output from ElectrumX blockchain.block.headers RPC"); + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.block.headers RPC"); JSONObject blockJson = (JSONObject) blockObj; @@ -278,8 +241,7 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig Object hexObj = blockJson.get("hex"); if (!(countObj instanceof Long) || !(hexObj instanceof String)) - throw new ForeignBlockchainException.NetworkException( - "Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); long returnedCount = (Long) countObj; String hex = (String) hexObj; @@ -288,18 +250,12 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig byte[] raw = HashCode.fromString(hex).asBytes(); - // Most chains use a fixed length 80 byte header, so block headers can be split - // up by dividing the hex into - // 80-byte segments. However, some chains such as DOGE use variable length - // headers due to AuxPoW or other - // reasons. In these cases we can identify the start of each block header by the - // location of the block version - // numbers. Each block starts with a version number, and for DOGE this is easily - // identifiable (6422788) at the - // time of writing (Jul 2021). If we encounter a chain that is using more - // generic version numbers (e.g. 1) - // and can't be used to accurately identify block indexes, then there are - // sufficient checks to ensure an + // Most chains use a fixed length 80 byte header, so block headers can be split up by dividing the hex into + // 80-byte segments. However, some chains such as DOGE use variable length headers due to AuxPoW or other + // reasons. In these cases we can identify the start of each block header by the location of the block version + // numbers. Each block starts with a version number, and for DOGE this is easily identifiable (6422788) at the + // time of writing (Jul 2021). If we encounter a chain that is using more generic version numbers (e.g. 1) + // and can't be used to accurately identify block indexes, then there are sufficient checks to ensure an // exception is thrown. if (raw.length == returnedCount * BLOCK_HEADER_LENGTH) { @@ -307,7 +263,8 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig for (int i = 0; i < returnedCount; ++i) { rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); } - } else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { + } + else if (raw.length > returnedCount * BLOCK_HEADER_LENGTH) { // Assume AuxPoW variable length header (DOGE) int referenceVersion = BitTwiddling.intFromLEBytes(raw, 0); // DOGE uses 6422788 at time of commit (Jul 2021) for (int i = 0; i < raw.length - 4; ++i) { @@ -318,22 +275,19 @@ public List getRawBlockHeaders(int startHeight, int count) throws Foreig } // Ensure that we found the correct number of block headers if (rawBlockHeaders.size() != count) { - throw new ForeignBlockchainException.NetworkException( - "Unexpected raw header contents in JSON from ElectrumX blockchain.block.headers RPC."); + throw new ForeignBlockchainException.NetworkException("Unexpected raw header contents in JSON from ElectrumX blockchain.block.headers RPC."); } - } else if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) { - throw new ForeignBlockchainException.NetworkException( - "Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); + } + else if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) { + throw new ForeignBlockchainException.NetworkException("Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); } return rawBlockHeaders; } /** - * Returns list of raw block timestamps, starting from startHeight - * inclusive. + * Returns list of raw block timestamps, starting from startHeight inclusive. *

- * * @throws ForeignBlockchainException if error occurs */ @Override @@ -345,7 +299,6 @@ public List getBlockTimestamps(int startHeight, int count) throws ForeignB /** * Returns confirmed balance, based on passed payment script. *

- * * @return confirmed balance, or zero if script unknown * @throws ForeignBlockchainException if there was an error */ @@ -354,19 +307,16 @@ public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()) - .getResponse(); + Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()).getResponse(); if (!(balanceObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException( - "Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); JSONObject balanceJson = (JSONObject) balanceObj; Object confirmedBalanceObj = balanceJson.get("confirmed"); if (!(confirmedBalanceObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException( - "Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); + throw new ForeignBlockchainException.NetworkException("Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); return (Long) balanceJson.get("confirmed"); } @@ -374,7 +324,6 @@ public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException /** * Returns confirmed balance, based on passed base58 encoded address. *

- * * @return confirmed balance, or zero if address unknown * @throws ForeignBlockchainException if there was an error */ @@ -386,13 +335,11 @@ public long getConfirmedAddressBalance(String base58Address) throws ForeignBlock /** * Returns list of unspent outputs pertaining to passed address. *

- * * @return list of unspent outputs, or empty list if address unknown * @throws ForeignBlockchainException if there was an error. */ @Override - public List getUnspentOutputs(String address, boolean includeUnconfirmed) - throws ForeignBlockchainException { + public List getUnspentOutputs(String address, boolean includeUnconfirmed) throws ForeignBlockchainException { byte[] script = this.blockchain.addressToScriptPubKey(address); return this.getUnspentOutputs(script, includeUnconfirmed); } @@ -400,29 +347,24 @@ public List getUnspentOutputs(String address, boolean includeUnco /** * Returns list of unspent outputs pertaining to passed payment script. *

- * * @return list of unspent outputs, or empty list if script unknown * @throws ForeignBlockchainException if there was an error. */ @Override - public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) - throws ForeignBlockchainException { + public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()) - .getResponse(); + Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()).getResponse(); if (!(unspentJson instanceof JSONArray)) - throw new ForeignBlockchainException( - "Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); + throw new ForeignBlockchainException("Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); List unspentOutputs = new ArrayList<>(); for (Object rawUnspent : (JSONArray) unspentJson) { JSONObject unspent = (JSONObject) rawUnspent; int height = ((Long) unspent.get("height")).intValue(); - // We only want unspent outputs from confirmed transactions (and definitely not - // mempool duplicates with height 0) + // We only want unspent outputs from confirmed transactions (and definitely not mempool duplicates with height 0) if (!includeUnconfirmed && height <= 0) continue; @@ -442,7 +384,7 @@ public List getUnspentOutputs(byte[] script, boolean includeUncon * NOTE: Do not mutate returned byte[]! * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException { @@ -450,8 +392,7 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException try { rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false).getResponse(); } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain - // transaction. Use gettransaction for wallet transactions.'}) + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) throw new ForeignBlockchainException.NotFoundException(e.getMessage()); @@ -459,8 +400,7 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException } if (!(rawTransactionHex instanceof String)) - throw new ForeignBlockchainException.NetworkException( - "Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException("Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); return HashCode.fromString((String) rawTransactionHex).asBytes(); } @@ -471,7 +411,7 @@ public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException * NOTE: Do not mutate returned byte[]! * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException { @@ -481,9 +421,8 @@ public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException /** * Returns transaction info for passed transaction hash. *

- * * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs + * @throws ForeignBlockchainException if error occurs */ @Override public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { @@ -500,8 +439,7 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai serverResponse = this.rpc("blockchain.transaction.get", txHash, true); transactionObj = serverResponse.getResponse(); } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain - // transaction. Use gettransaction for wallet transactions.'}) + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) throw new ForeignBlockchainException.NotFoundException(e.getMessage()); @@ -510,20 +448,17 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } while (transactionObj == null); if (!(transactionObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException( - "Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException("Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); JSONObject transactionJson = (JSONObject) transactionObj; Object inputsObj = transactionJson.get("vin"); if (!(inputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException( - "Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); Object outputsObj = transactionJson.get("vout"); if (!(outputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException( - "Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); try { int size = ((Long) transactionJson.get("size")).intValue(); @@ -574,12 +509,9 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } // For the purposes of Qortal we require all outputs to contain addresses - // Some servers omit this info, causing problems down the line with balance - // calculations - // Update: it turns out that they were just using a different key - "address" - // instead of "addresses" - // The code below can remain in place, just in case a peer returns a missing - // address in the future + // Some servers omit this info, causing problems down the line with balance calculations + // Update: it turns out that they were just using a different key - "address" instead of "addresses" + // The code below can remain in place, just in case a peer returns a missing address in the future if (addresses == null || addresses.isEmpty()) { final String message = String.format("No output addresses returned for transaction %s", txHash); LOGGER.warn("{}: No output addresses returned for transaction {}", this.blockchain.getCurrencyCode(), txHash); @@ -594,7 +526,7 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs); // Save into cache, if and only if it has been confirmed - if (transaction.timestamp != null) { + if( transaction.timestamp != null ) { transactionCache.put(txHash, transaction); } @@ -604,30 +536,25 @@ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchai } this.connections.remove(serverResponse.getElectrumServer()); - serverResponse.getElectrumServer().closeServer(this.getClass().getSimpleName(), - "Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); + serverResponse.getElectrumServer().closeServer(this.getClass().getSimpleName(), "Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); return getTransaction(txHash); } /** * Returns list of transactions, relating to passed payment script. *

- * * @return list of related transactions, or empty list if script unknown * @throws ForeignBlockchainException if error occurs */ @Override - public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) - throws ForeignBlockchainException { + public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { byte[] scriptHash = Crypto.digest(script); Bytes.reverse(scriptHash); - ElectrumServerResponse serverResponse = this.rpc("blockchain.scripthash.get_history", - HashCode.fromBytes(scriptHash).toString()); + ElectrumServerResponse serverResponse = this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); Object transactionsJson = serverResponse.getResponse(); if (!(transactionsJson instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException( - "Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); + throw new ForeignBlockchainException.NetworkException("Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); List transactionHashes = new ArrayList<>(); @@ -648,39 +575,32 @@ public List getAddressTransactions(byte[] script, boolean inclu } @Override - public List getAddressBitcoinyTransactions(String address, boolean includeUnconfirmed) - throws ForeignBlockchainException { - // FUTURE: implement this if needed. For now we use getAddressTransactions() + - // getTransaction() + public List getAddressBitcoinyTransactions(String address, boolean includeUnconfirmed) throws ForeignBlockchainException { + // FUTURE: implement this if needed. For now we use getAddressTransactions() + getTransaction() throw new ForeignBlockchainException("getAddressBitcoinyTransactions not yet implemented for ElectrumX"); } /** * Broadcasts raw transaction to network. *

- * * @throws ForeignBlockchainException if error occurs */ @Override public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException { - Object rawBroadcastResult = this - .rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()).getResponse(); + Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()).getResponse(); // We're expecting a simple string that is the transaction hash if (!(rawBroadcastResult instanceof String)) - throw new ForeignBlockchainException.NetworkException( - "Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); + throw new ForeignBlockchainException.NetworkException("Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); } // Class-private utility methods /** - * Query current server for its list of peer servers, and return those we can - * parse. + * Query current server for its list of peer servers, and return those we can parse. *

- * * @throws ForeignBlockchainException - * @throws ClassCastException to be handled by caller + * @throws ClassCastException to be handled by caller */ private Set serverPeersSubscribe() { Set newServers = new HashSet<>(); @@ -688,16 +608,14 @@ private Set serverPeersSubscribe() { List electrumServers = acquireServers(); try { - for (ElectrumServer electrumServer : electrumServers) { + for( ElectrumServer electrumServer : electrumServers ) { Object peers = this.connectedRpc(electrumServer, "server.peers.subscribe"); - if (peers == null) - continue; + if( peers == null ) continue; Object peersObject = Objects.requireNonNull(peers); - if (!(peersObject instanceof JSONArray)) - continue; + if( !(peersObject instanceof JSONArray) ) continue; for (Object rawPeer : (JSONArray) peersObject) { @@ -751,7 +669,7 @@ private Set serverPeersSubscribe() { } catch (Exception e) { LOGGER.error(e.getMessage(), e); } finally { - for (ElectrumServer server : electrumServers) { + for( ElectrumServer server : electrumServers ) { releaseServer(server); } } @@ -763,7 +681,8 @@ private ElectrumServer acquireServer() throws ForeignBlockchainException { try { return this.availableConnections.take(); - } catch (InterruptedException e) { + } + catch( InterruptedException e ) { throw new ForeignBlockchainException(e.getMessage()); } } @@ -814,89 +733,28 @@ public static List drainRandomly(BlockingQueue queue, int numToDrain) return drainedList; } - private void releaseServer(ElectrumServer server) { + private void releaseServer( ElectrumServer server ) { // if the connection is still open - if (this.connections.contains(server)) + if( this.connections.contains(server)) this.availableConnections.add(server); } - private boolean isIdle() { - if (this.inFlightRpcCount.get() > 0) { - return false; - } - if (this.lastRpcTimeMs <= 0L) { - return true; - } - return System.currentTimeMillis() - this.lastRpcTimeMs > IDLE_DISCONNECT_MS; - } - - private void closeAllConnections(String reason) { - synchronized (this.connectionListLock) { - for (ElectrumServer server : new HashSet<>(this.connections)) { - this.connections.remove(server); - server.closeServer(this.getClass().getSimpleName(), reason); - } - this.availableConnections.clear(); - this.remainingServers.clear(); - } - } - - /** - * Ensure the connection maintenance threads are running and initial connections - * exist. - */ - private void ensureConnectionManagementStarted() { - if (this.connectionManagementStarted) { - return; - } - - boolean shouldInit = false; - synchronized (this.connectionManagementLock) { - if (!this.connectionManagementStarted) { - this.connectionManagementStarted = true; - shouldInit = true; - } - } - - if (!shouldInit) { - return; - } - - startMakingConnections(); - - this.scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); - this.scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); - this.scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); - } - /** - *

- * Performs RPC call, with automatic reconnection to different server if needed. + *

Performs RPC call, with automatic reconnection to different server if needed. *

- * * @param method String representation of the RPC call value * @param params a list of Objects passed to the method of the Remote Server * @return "result" object from within JSON output - * @throws ForeignBlockchainException if server returns error or something goes - * wrong + * @throws ForeignBlockchainException if server returns error or something goes wrong */ - private ElectrumServerResponse rpc(String method, Object... params) throws ForeignBlockchainException { - this.inFlightRpcCount.incrementAndGet(); - this.lastRpcTimeMs = System.currentTimeMillis(); - try { - ensureConnectionManagementStarted(); - if (this.availableConnections.isEmpty()) { - LOGGER.debug("{} no available ElectrumX connections; starting connections on demand", - this.blockchain.getCurrencyCode()); - startMakingConnections(); - } + private ElectrumServerResponse rpc(String method, Object...params) throws ForeignBlockchainException { ElectrumServer electrumServer = acquireServer(); Object response = null; - while (response == null) { + while(response == null) { response = connectedRpc(electrumServer, method, params); @@ -904,8 +762,7 @@ private ElectrumServerResponse rpc(String method, Object... params) throws Forei if (!this.availableConnections.isEmpty()) { long averageResponseTime = electrumServer.averageResponseTime(); if (averageResponseTime > MAX_AVG_RESPONSE_TIME) { - String message = String.format("Slow average response time %dms from %s - trying another server...", - averageResponseTime, electrumServer.getServer()); + String message = String.format("Slow average response time %dms from %s - trying another server...", averageResponseTime, electrumServer.getServer()); LOGGER.info(message); electrumServer.closeServer(this.getClass().getSimpleName(), message); break; @@ -914,7 +771,6 @@ private ElectrumServerResponse rpc(String method, Object... params) throws Forei if (response != null) { releaseServer(electrumServer); - this.lastRpcTimeMs = System.currentTimeMillis(); return new ElectrumServerResponse(electrumServer, response); } @@ -929,9 +785,6 @@ private ElectrumServerResponse rpc(String method, Object... params) throws Forei // Failed to perform RPC - maybe lack of servers? LOGGER.info("Error: No connected Electrum servers when trying to make RPC call"); throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); - } finally { - this.inFlightRpcCount.decrementAndGet(); - } } /** @@ -941,19 +794,14 @@ private ElectrumServerResponse rpc(String method, Object... params) throws Forei */ private void monitorConnections() { - if (this.isIdle() && !this.connections.isEmpty()) { - LOGGER.info("{} idle; closing {} ElectrumX connections", this.blockchain.getCurrencyCode(), - this.connections.size()); - this.closeAllConnections("idle timeout"); - } - LOGGER.info( - "{} {} available connections, {} total servers, {} total connections, {} useless servers", - this.blockchain.getCurrencyCode(), - this.availableConnections.size(), - this.servers.size(), - this.connections.size(), - this.uselessServers.size()); + "{} {} available connections, {} total servers, {} total connections, {} useless servers", + this.blockchain.getCurrencyCode(), + this.availableConnections.size(), + this.servers.size(), + this.connections.size(), + this.uselessServers.size() + ); } /** @@ -964,10 +812,7 @@ private void monitorConnections() { private void makeConnections() { try { - if (this.isIdle()) { - return; - } - if (this.connections.isEmpty()) { + if( this.connections.isEmpty() ) { startMakingConnections(); } @@ -980,16 +825,12 @@ private void makeConnections() { /** * Recover Connections * - * If connection count is below the minimum, then recover connections from the - * initial list. + * If connection count is below the minimum, then recover connections from the initial list. */ private void recoverConnections() { try { - if (this.isIdle()) { - return; - } - if (this.connections.size() < this.minimumConnections) { + if( this.connections.size() < this.minimumConnections ) { LOGGER.debug("{} recovering connections", this.blockchain.currencyCode); startMakingConnections(); LOGGER.debug("{} recovered {} connections", this.blockchain.currencyCode, this.connections.size()); @@ -1003,12 +844,10 @@ private void recoverConnections() { * Start Making Connections */ private void startMakingConnections() { - synchronized (this.connectionListLock) { - // assume there are no server to get peers from, so we must start from the base - // list - this.remainingServers.clear(); - this.remainingServers.addAll(this.servers); - } + + // assume there are no server to get peers from, so we must start from the base list + this.remainingServers.clear(); + this.remainingServers.addAll(this.servers); connectRemainingServers(); } @@ -1021,22 +860,19 @@ private void startMakingConnections() { private void makeMoreConnections() { // if we need more connections - if (this.connections.size() < this.maximumConnections) { + if(this.connections.size() < MINIMUM_CONNECTIONS) { // Ask for more servers Set moreServers = serverPeersSubscribe(); - synchronized (this.connectionListLock) { - // Add all servers to base list - this.servers.addAll(moreServers); + // Add all servers to base list + this.servers.addAll(moreServers); - // add base list to remaining list - this.remainingServers.addAll(this.servers); + // add base list to remaining list + this.remainingServers.addAll(this.servers); - // remove servers that this node is already connected to - this.remainingServers - .removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); - } + // remove servers that this node is already connected to + this.remainingServers.removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); // try connecting the remaining servers connectRemainingServers(); @@ -1045,39 +881,13 @@ private void makeMoreConnections() { private void connectRemainingServers() { // while there are remaining servers and less than the maximum connections - while (true) { - ChainableServer server; - synchronized (this.connectionListLock) { - if (this.remainingServers.isEmpty() || this.connections.size() >= this.maximumConnections) { - return; - } - server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); - } + while( !this.remainingServers.isEmpty() && this.connections.size() < MAXIMUM_CONNECTIONS ) { + ChainableServer server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); makeConnection(server, this.getClass().getSimpleName()); } } - private static int clamp(int value, int min, int max) { - if (value < min) { - return min; - } - if (value > max) { - return max; - } - return value; - } - - private static String randomClientName() { - final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - StringBuilder name = new StringBuilder(12); - ThreadLocalRandom random = ThreadLocalRandom.current(); - for (int i = 0; i < 12; i++) { - name.append(alphabet.charAt(random.nextInt(alphabet.length()))); - } - return name.toString(); - } - private Optional makeConnection(ChainableServer server, String requestedBy) { LOGGER.debug(() -> String.format("Connecting to %s %s", server, this.blockchain.currencyCode)); @@ -1086,44 +896,38 @@ private Optional makeConnection(ChainableServer serve int timeout = 5000; // ms ElectrumServer electrumServer = ElectrumServer.createInstance(server, endpoint, timeout, this.recorder); - electrumServer.setClientName(randomClientName()); // All connections need to start with a version negotiation this.connectedRpc(electrumServer, "server.version"); - // Check connection is suitable by asking for server features, including genesis - // block hash + // Check connection is suitable by asking for server features, including genesis block hash JSONObject featuresJson = (JSONObject) this.connectedRpc(electrumServer, "server.features"); - if (featuresJson == null) - return Optional.of(recorder.recordConnection(server, requestedBy, true, false, MISSING_FEATURES_ERROR)); + if (featuresJson == null ) + return Optional.of( recorder.recordConnection(server, requestedBy, true, false, MISSING_FEATURES_ERROR) ); try { double protocol_min = CrossChainUtils.getVersionDecimal(featuresJson, "protocol_min"); if (protocol_min < MIN_PROTOCOL_VERSION) - return Optional.of(recorder.recordConnection(server, requestedBy, true, false, - "old version: protocol_min = " + protocol_min + " < MIN_PROTOCOL_VERSION = " + MIN_PROTOCOL_VERSION)); + return Optional.of( recorder.recordConnection(server, requestedBy, true, false, "old version: protocol_min = " + protocol_min + " < MIN_PROTOCOL_VERSION = " + MIN_PROTOCOL_VERSION) ); } catch (NumberFormatException e) { - return Optional.of(recorder.recordConnection(server, requestedBy, true, false, - featuresJson.get("protocol_min").toString() + " is not a valid version")); + return Optional.of( recorder.recordConnection(server, requestedBy,true, false,featuresJson.get("protocol_min").toString() + " is not a valid version")); } catch (NullPointerException e) { - return Optional.of( - recorder.recordConnection(server, requestedBy, true, false, "server version not available: protocol_min")); + return Optional.of( recorder.recordConnection(server, requestedBy,true, false,"server version not available: protocol_min")); } - if (this.expectedGenesisHash != null - && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) - return Optional.of(recorder.recordConnection(server, requestedBy, true, false, EXPECTED_GENESIS_ERROR)); + if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) + return Optional.of( recorder.recordConnection(server, requestedBy, true, false, EXPECTED_GENESIS_ERROR) ); LOGGER.debug(() -> String.format("Connected to %s %s", server, this.blockchain.currencyCode)); this.connections.add(electrumServer); this.availableConnections.add(electrumServer); - return Optional.of(this.recorder.recordConnection(server, requestedBy, true, true, EMPTY)); + return Optional.of( this.recorder.recordConnection( server, requestedBy, true, true, EMPTY) ); } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { // Didn't work, try another server... - return Optional.of(this.recorder.recordConnection(server, requestedBy, true, false, CrossChainUtils.getNotes(e))); - } catch (Exception e) { + return Optional.of( this.recorder.recordConnection( server, requestedBy, true, false, CrossChainUtils.getNotes(e))); + } catch( Exception e ) { LOGGER.error(e.getMessage(), e); return Optional.empty(); } @@ -1132,15 +936,13 @@ private Optional makeConnection(ChainableServer serve /** * Perform RPC using currently connected server. *

- * * @param method * @param params * @return response Object, or null if server fails to respond * @throws ForeignBlockchainException if server returns error */ @SuppressWarnings("unchecked") - private Object connectedRpc(ElectrumServer server, String method, Object... params) - throws ForeignBlockchainException { + private Object connectedRpc(ElectrumServer server, String method, Object...params) throws ForeignBlockchainException { JSONObject requestJson = new JSONObject(); String id = UUID.randomUUID().toString(); requestJson.put("id", id); @@ -1152,12 +954,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para // server.version needs additional params to negotiate a version if (method.equals("server.version")) { - String clientName = server.getClientName(); - if (clientName == null) { - clientName = randomClientName(); - server.setClientName(clientName); - } - requestParams.add(clientName); + requestParams.add(CLIENT_NAME); List versions = new ArrayList<>(); DecimalFormat df = new DecimalFormat("#.#"); versions.add(df.format(MIN_PROTOCOL_VERSION)); @@ -1185,10 +982,10 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para } long endTime = System.currentTimeMillis(); - long responseTime = endTime - startTime; + long responseTime = endTime-startTime; LOGGER.trace(() -> String.format("Request: %s Response: %s", request, response)); - LOGGER.trace(() -> String.format("Time taken: %dms", endTime - startTime)); + LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime)); if (response.isEmpty()) // Empty response - try another server? @@ -1206,15 +1003,13 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para Object errorObj = responseJson.get("error"); if (errorObj != null) { if (errorObj instanceof String) { - LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", - server.getServer(), method, (String) errorObj)); + LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", server.getServer(), method, (String) errorObj)); // Try another server return null; } if (!(errorObj instanceof JSONObject)) { - LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", - server.getServer(), method)); + LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", server.getServer(), method)); // Try another server return null; } @@ -1224,9 +1019,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para Object messageObj = errorJson.get("message"); if (!(messageObj instanceof String)) { - LOGGER - .debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", - server.getServer(), method)); + LOGGER.debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", server.getServer(), method)); // Try another server return null; } @@ -1234,9 +1027,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para String message = (String) messageObj; // Some error 'messages' are actually wrapped upstream bitcoind errors: - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such - // mempool or blockchain transaction. Use gettransaction for wallet - // transactions.'})" + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" // We want to detect these and extract the upstream error code for caller's use Matcher messageMatcher = DAEMON_ERROR_REGEX.matcher(message); if (messageMatcher.find()) @@ -1244,8 +1035,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para int daemonErrorCode = Integer.parseInt(messageMatcher.group(1)); throw new ForeignBlockchainException.NetworkException(daemonErrorCode, message, server.getServer()); } catch (NumberFormatException e) { - // We couldn't parse the error code integer? Fall-through to generic - // exception... + // We couldn't parse the error code integer? Fall-through to generic exception... } throw new ForeignBlockchainException.NetworkException(message, server.getServer()); @@ -1256,7 +1046,7 @@ private Object connectedRpc(ElectrumServer server, String method, Object... para @Override public Set getServers() { - return new HashSet<>(this.servers); + return new HashSet<>(this.servers ); } @Override diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index 958ea801b..b27e77f18 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -32,10 +32,8 @@ public class Litecoin extends Bitcoiny { private static final long MAINNET_FEE = 1000L; private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST - private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>( - ElectrumX.Server.ConnectionType.class); - public static final LitecoinMainNetParamsP2ShOverride MAIN_NET_PARAMS_P2SH_OVERRIDE = new LitecoinMainNetParamsP2ShOverride( - 50); + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + public static final LitecoinMainNetParamsP2ShOverride MAIN_NET_PARAMS_P2SH_OVERRIDE = new LitecoinMainNetParamsP2ShOverride(50); static { DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); @@ -52,30 +50,17 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=ltc - new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), - new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 50002), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), - new Server("electrum.jochen-hoenicke.de", Server.ConnectionType.SSL, 50091), - new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), - new Server("electrum.petrkr.net", Server.ConnectionType.SSL, 60002), - new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), - new Server("fury.fiatfaucet.com", Server.ConnectionType.SSL, 50002), - new Server("ltc-electrum.cakewallet.com", Server.ConnectionType.SSL, 50002), - new Server("litecoin.stackwallet.com", Server.ConnectionType.SSL, 20063), - new Server("ltc.aftrek.org", Server.ConnectionType.SSL, 50002), - new Server("137.184.250.112", Server.ConnectionType.SSL, 50002), - new Server("146.190.15.65", Server.ConnectionType.SSL, 50002), - new Server("157.230.64.188", Server.ConnectionType.SSL, 50002), - new Server("209.38.53.75", Server.ConnectionType.SSL, 50002), - new Server("24.199.78.132", Server.ConnectionType.SSL, 50002), - new Server("5.78.97.174", Server.ConnectionType.SSL, 50002), - new Server("188.166.208.106", Server.ConnectionType.SSL, 50002), - new Server("5.161.216.180", Server.ConnectionType.SSL, 50002)); + // Servers chosen on NO BASIS WHATSOEVER from various sources! + // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=ltc + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.qortal.link", Server.ConnectionType.SSL, 50002), + new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("ltc.rentonrisk.com", Server.ConnectionType.SSL, 50002) + ); } @Override @@ -97,8 +82,9 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002)); + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002) + ); } @Override @@ -120,8 +106,9 @@ public NetworkParameters getParams() { @Override public Collection getServers() { return Arrays.asList( - new Server("localhost", Server.ConnectionType.TCP, 50001), - new Server("localhost", Server.ConnectionType.SSL, 50002)); + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002) + ); } @Override @@ -147,11 +134,8 @@ public void setFeeRequired(long feeRequired) { } public abstract NetworkParameters getParams(); - public abstract Collection getServers(); - public abstract String getGenesisHash(); - public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; } @@ -161,8 +145,7 @@ public void setFeeRequired(long feeRequired) { // Constructors and instance - private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, - String currencyCode) { + private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { super(blockchain, bitcoinjContext, currencyCode, DEFAULT_FEE_PER_KB); this.litecoinNet = litecoinNet; @@ -173,8 +156,7 @@ public static synchronized Litecoin getInstance() { if (instance == null) { LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet(); - BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), - litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); Context bitcoinjContext = new Context(litecoinNet.getParams()); instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); @@ -199,12 +181,10 @@ public long getMinimumOrderAmount() { } /** - * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic - * timestamp. + * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp. * * @param timestamp optional milliseconds since epoch, or null for 'now' - * @return sats per 1000bytes, or throws ForeignBlockchainException if something - * went wrong + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong */ @Override public long getP2shFee(Long timestamp) throws ForeignBlockchainException { @@ -219,7 +199,7 @@ public long getFeeRequired() { @Override public void setFeeRequired(long fee) { - this.litecoinNet.setFeeRequired(fee); + this.litecoinNet.setFeeRequired( fee ); } /** @@ -245,8 +225,7 @@ public boolean isCurrentP2ShAddress(String address) { /** * Convert Current P2SH Address * - * Convert a p2sh address conforming the current standard prefix 'M', to the - * internal standard here + * Convert a p2sh address conforming the current standard prefix 'M', to the internal standard here * using prefix '3' * * @param address the p2sh address, starts with 'M' @@ -255,9 +234,10 @@ public boolean isCurrentP2ShAddress(String address) { */ public String convertCurrentP2ShAddress(String address) { - if (isCurrentP2ShAddress(address)) { + if( isCurrentP2ShAddress(address) ) { return convertP2SHAddress(address, MAIN_NET_PARAMS_P2SH_OVERRIDE, this.params); - } else { + } + else { throw new AddressFormatException("this is not a current p2sh address for Litecoin"); } } @@ -268,13 +248,12 @@ public String convertCurrentP2ShAddress(String address) { * Convert p2sh address from one network standard to another. * * @param p2shAddress the p2sh address - * @param fromParams the existing standard - * @param toParams the desired standard + * @param fromParams the existing standard + * @param toParams the desired standard * * @return the p2sh conforming to the desired standard */ - private static String convertP2SHAddress(String p2shAddress, NetworkParameters fromParams, - NetworkParameters toParams) { + private static String convertP2SHAddress(String p2shAddress, NetworkParameters fromParams, NetworkParameters toParams) { try { // decode the P2SH address Address address = LegacyAddress.fromBase58(fromParams, p2shAddress); @@ -290,4 +269,4 @@ private static String convertP2SHAddress(String p2shAddress, NetworkParameters f return null; } } -} +} \ No newline at end of file From f31f3783d4a9aca5dd8a5ecbd8e74fc4f35bf034 Mon Sep 17 00:00:00 2001 From: crowetic Date: Mon, 12 Jan 2026 18:10:27 -0800 Subject: [PATCH 17/28] removed unnecessary formatting changes --- .../java/org/qortal/crosschain/Digibyte.java | 18 +- .../java/org/qortal/crosschain/ElectrumX.java | 163 +++++++++++++++--- .../java/org/qortal/crosschain/Litecoin.java | 21 ++- 3 files changed, 168 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Digibyte.java b/src/main/java/org/qortal/crosschain/Digibyte.java index cf600b808..67ac4f108 100644 --- a/src/main/java/org/qortal/crosschain/Digibyte.java +++ b/src/main/java/org/qortal/crosschain/Digibyte.java @@ -43,15 +43,15 @@ public NetworkParameters getParams() { @Override public Collection getServers() { - return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=dgb - new Server("electrum.qortal.link", Server.ConnectionType.SSL, 55002), - new Server("electrum.cipig.net", Server.ConnectionType.SSL, 20059), - new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20059), - new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20059), - new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20059) - ); + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=dgb + new Server("electrum.qortal.link", Server.ConnectionType.SSL, 55002), + new Server("electrum.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20059), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20059) + ); } @Override diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index aedc93eb8..0bda0da72 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -22,7 +22,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -37,7 +39,9 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // See: https://electrumx.readthedocs.io/en/latest/protocol-changes.html private static final double MIN_PROTOCOL_VERSION = 1.2; private static final double MAX_PROTOCOL_VERSION = 2.0; // Higher than current latest, for hopeful future-proofing - private static final String CLIENT_NAME = "Qortal"; + private static final int MIN_TARGET_CONNECTIONS = 2; + private static final int DEFAULT_TARGET_CONNECTIONS = 3; + private static final int MAX_TARGET_CONNECTIONS = 12; private static final int BLOCK_HEADER_LENGTH = 80; @@ -51,13 +55,13 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private static final long MAX_AVG_RESPONSE_TIME = 2000L; // ms public static final String MISSING_FEATURES_ERROR = "MISSING FEATURES ERROR"; public static final String EXPECTED_GENESIS_ERROR = "EXPECTED GENESIS ERROR"; - public static final int MINIMUM_CONNECTIONS = 30; - public static final int MAXIMUM_CONNECTIONS = 50; + private static final long IDLE_DISCONNECT_MS = 2 * 60 * 1000L; private ChainableServerConnectionRecorder recorder = new ChainableServerConnectionRecorder(100); // the minimum number of connections targeted for this foreign blockchain private int minimumConnections; + private final int maximumConnections; public static class Server implements ChainableServer { String hostname; @@ -165,6 +169,12 @@ public boolean removeEldestEntry(Map.Entry eldest) // Scheduled executor service to monitor connections private final ScheduledExecutorService scheduleMonitorConnections = Executors.newScheduledThreadPool(1); + private final Object connectionManagementLock = new Object(); + private final Object connectionListLock = new Object(); + private volatile boolean connectionManagementStarted = false; + private volatile long lastRpcTimeMs = 0L; + private final AtomicInteger inFlightRpcCount = new AtomicInteger(0); + // Constructors public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { @@ -173,12 +183,18 @@ public ElectrumX(String netId, String genesisHash, Collection initialSer this.servers.addAll(initialServerList); this.defaultPorts.putAll(defaultPorts); - // the minimum is set to roughly 10% of the initial count - this.minimumConnections = (initialServerList.size() / 10) + 1; + int listSize = initialServerList.size(); + if (listSize == 0) { + this.maximumConnections = 0; + this.minimumConnections = 0; + return; + } - scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); - scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); - scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); + int scaledTarget = (listSize / 10) + 1; + int targetConnections = clamp(scaledTarget, MIN_TARGET_CONNECTIONS, MAX_TARGET_CONNECTIONS); + targetConnections = Math.min(listSize, Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS)); + this.maximumConnections = targetConnections; + this.minimumConnections = Math.max(1, Math.min(listSize, Math.max(1, targetConnections / 2))); } // Methods for use by other classes @@ -740,6 +756,54 @@ private void releaseServer( ElectrumServer server ) { this.availableConnections.add(server); } + private boolean isIdle() { + if (this.inFlightRpcCount.get() > 0) { + return false; + } + if (this.lastRpcTimeMs <= 0L) { + return true; + } + return System.currentTimeMillis() - this.lastRpcTimeMs > IDLE_DISCONNECT_MS; + } + + private void closeAllConnections(String reason) { + synchronized (this.connectionListLock) { + for (ElectrumServer server : new HashSet<>(this.connections)) { + this.connections.remove(server); + server.closeServer(this.getClass().getSimpleName(), reason); + } + this.availableConnections.clear(); + this.remainingServers.clear(); + } + } + + /** + * Ensure the connection maintenance threads are running and initial connections exist. + */ + private void ensureConnectionManagementStarted() { + if (this.connectionManagementStarted) { + return; + } + + boolean shouldInit = false; + synchronized (this.connectionManagementLock) { + if (!this.connectionManagementStarted) { + this.connectionManagementStarted = true; + shouldInit = true; + } + } + + if (!shouldInit) { + return; + } + + startMakingConnections(); + + scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); + scheduleRecoverConnections.scheduleWithFixedDelay(this::recoverConnections, 120, 10, TimeUnit.SECONDS); + scheduleMonitorConnections.scheduleWithFixedDelay(this::monitorConnections, 1, 10, TimeUnit.MINUTES); + } + /** *

Performs RPC call, with automatic reconnection to different server if needed. *

@@ -749,6 +813,14 @@ private void releaseServer( ElectrumServer server ) { * @throws ForeignBlockchainException if server returns error or something goes wrong */ private ElectrumServerResponse rpc(String method, Object...params) throws ForeignBlockchainException { + this.inFlightRpcCount.incrementAndGet(); + this.lastRpcTimeMs = System.currentTimeMillis(); + try { + ensureConnectionManagementStarted(); + if (this.availableConnections.isEmpty()) { + LOGGER.debug("{} no available ElectrumX connections; starting connections on demand", this.blockchain.getCurrencyCode()); + startMakingConnections(); + } ElectrumServer electrumServer = acquireServer(); @@ -771,6 +843,7 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig if (response != null) { releaseServer(electrumServer); + this.lastRpcTimeMs = System.currentTimeMillis(); return new ElectrumServerResponse(electrumServer, response); } @@ -785,6 +858,9 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig // Failed to perform RPC - maybe lack of servers? LOGGER.info("Error: No connected Electrum servers when trying to make RPC call"); throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); + } finally { + this.inFlightRpcCount.decrementAndGet(); + } } /** @@ -794,6 +870,11 @@ private ElectrumServerResponse rpc(String method, Object...params) throws Foreig */ private void monitorConnections() { + if (this.isIdle() && !this.connections.isEmpty()) { + LOGGER.info("{} idle; closing {} ElectrumX connections", this.blockchain.getCurrencyCode(), this.connections.size()); + this.closeAllConnections("idle timeout"); + } + LOGGER.info( "{} {} available connections, {} total servers, {} total connections, {} useless servers", this.blockchain.getCurrencyCode(), @@ -812,6 +893,9 @@ private void monitorConnections() { private void makeConnections() { try { + if (this.isIdle()) { + return; + } if( this.connections.isEmpty() ) { startMakingConnections(); } @@ -830,6 +914,9 @@ private void makeConnections() { private void recoverConnections() { try { + if (this.isIdle()) { + return; + } if( this.connections.size() < this.minimumConnections ) { LOGGER.debug("{} recovering connections", this.blockchain.currencyCode); startMakingConnections(); @@ -846,8 +933,10 @@ private void recoverConnections() { private void startMakingConnections() { // assume there are no server to get peers from, so we must start from the base list - this.remainingServers.clear(); - this.remainingServers.addAll(this.servers); + synchronized (this.connectionListLock) { + this.remainingServers.clear(); + this.remainingServers.addAll(this.servers); + } connectRemainingServers(); } @@ -860,19 +949,21 @@ private void startMakingConnections() { private void makeMoreConnections() { // if we need more connections - if(this.connections.size() < MINIMUM_CONNECTIONS) { + if(this.connections.size() < this.maximumConnections) { // Ask for more servers Set moreServers = serverPeersSubscribe(); - // Add all servers to base list - this.servers.addAll(moreServers); + synchronized (this.connectionListLock) { + // Add all servers to base list + this.servers.addAll(moreServers); - // add base list to remaining list - this.remainingServers.addAll(this.servers); + // add base list to remaining list + this.remainingServers.addAll(this.servers); - // remove servers that this node is already connected to - this.remainingServers.removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); + // remove servers that this node is already connected to + this.remainingServers.removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); + } // try connecting the remaining servers connectRemainingServers(); @@ -881,13 +972,39 @@ private void makeMoreConnections() { private void connectRemainingServers() { // while there are remaining servers and less than the maximum connections - while( !this.remainingServers.isEmpty() && this.connections.size() < MAXIMUM_CONNECTIONS ) { - ChainableServer server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); + while (true) { + ChainableServer server; + synchronized (this.connectionListLock) { + if (this.remainingServers.isEmpty() || this.connections.size() >= this.maximumConnections) { + return; + } + server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); + } makeConnection(server, this.getClass().getSimpleName()); } } + private static int clamp(int value, int min, int max) { + if (value < min) { + return min; + } + if (value > max) { + return max; + } + return value; + } + + private static String randomClientName() { + final String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + StringBuilder name = new StringBuilder(12); + ThreadLocalRandom random = ThreadLocalRandom.current(); + for (int i = 0; i < 12; i++) { + name.append(alphabet.charAt(random.nextInt(alphabet.length()))); + } + return name.toString(); + } + private Optional makeConnection(ChainableServer server, String requestedBy) { LOGGER.debug(() -> String.format("Connecting to %s %s", server, this.blockchain.currencyCode)); @@ -896,6 +1013,7 @@ private Optional makeConnection(ChainableServer serve int timeout = 5000; // ms ElectrumServer electrumServer = ElectrumServer.createInstance(server, endpoint, timeout, this.recorder); + electrumServer.setClientName(randomClientName()); // All connections need to start with a version negotiation this.connectedRpc(electrumServer, "server.version"); @@ -954,7 +1072,12 @@ private Object connectedRpc(ElectrumServer server, String method, Object...param // server.version needs additional params to negotiate a version if (method.equals("server.version")) { - requestParams.add(CLIENT_NAME); + String clientName = server.getClientName(); + if (clientName == null) { + clientName = randomClientName(); + server.setClientName(clientName); + } + requestParams.add(clientName); List versions = new ArrayList<>(); DecimalFormat df = new DecimalFormat("#.#"); versions.add(df.format(MIN_PROTOCOL_VERSION)); diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index b27e77f18..5a40e4145 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -53,13 +53,24 @@ public Collection getServers() { // Servers chosen on NO BASIS WHATSOEVER from various sources! // Status verified at https://1209k.com/bitcoin-eye/ele.php?chain=ltc new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 50002), new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), - new Server("electrum.qortal.link", Server.ConnectionType.SSL, 50002), + new Server("electrum.jochen-hoenicke.de", Server.ConnectionType.SSL, 50091), new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), - new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), - new Server("ltc.rentonrisk.com", Server.ConnectionType.SSL, 50002) + new Server("electrum.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), + new Server("fury.fiatfaucet.com", Server.ConnectionType.SSL, 50002), + new Server("ltc-electrum.cakewallet.com", Server.ConnectionType.SSL, 50002), + new Server("litecoin.stackwallet.com", Server.ConnectionType.SSL, 20063), + new Server("ltc.aftrek.org", Server.ConnectionType.SSL, 50002), + new Server("137.184.250.112", Server.ConnectionType.SSL, 50002), + new Server("146.190.15.65", Server.ConnectionType.SSL, 50002), + new Server("157.230.64.188", Server.ConnectionType.SSL, 50002), + new Server("209.38.53.75", Server.ConnectionType.SSL, 50002), + new Server("24.199.78.132", Server.ConnectionType.SSL, 50002), + new Server("5.78.97.174", Server.ConnectionType.SSL, 50002), + new Server("188.166.208.106", Server.ConnectionType.SSL, 50002), + new Server("5.161.216.180", Server.ConnectionType.SSL, 50002) ); } From 14bacf29f8335506d2ec4e12dcb33df699e43693 Mon Sep 17 00:00:00 2001 From: crowetic Date: Tue, 13 Jan 2026 16:39:49 -0800 Subject: [PATCH 18/28] =?UTF-8?q?On=E2=80=91demand=20connection=20manageme?= =?UTF-8?q?nt=20with=20idle=20disconnects=20and=20in=E2=80=91flight=20RPC?= =?UTF-8?q?=20tracking.=20ElectrumX.java=20Per=E2=80=91connection=20random?= =?UTF-8?q?ized=20client=20name=20for=20server.version.=20ElectrumX.java?= =?UTF-8?q?=20+=20ElectrumServer.java=20Server=20scoring=20+=20selection:?= =?UTF-8?q?=20probe=20servers=20on=20first=20demand,=20score=20by=20latenc?= =?UTF-8?q?y=20+=20failures,=20connect=20to=20top=2075%=20by=20score,=20up?= =?UTF-8?q?date=20on=20peer=20discovery.=20ElectrumX.java=20Logging:=20tar?= =?UTF-8?q?get=20connection=20counts,=20selection=20counts,=20top/bottom?= =?UTF-8?q?=20scores=20(only=20when=20changed;=20includes=20connection=20c?= =?UTF-8?q?ount).=20ElectrumX.java=20Thread=E2=80=91safety=20fixes=20aroun?= =?UTF-8?q?d=20response=20time=20averages=20and=20list=20snapshots.=20Elec?= =?UTF-8?q?trumX.java=20Early=20SSL=20handshake=20to=20surface=20TLS=20fai?= =?UTF-8?q?lures=20immediately.=20ElectrumServer.java=20Server=20lists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Litecoin: updated ElectrumX mainnet servers (added/remapped list). Litecoin.java Digibyte: re‑added electrum.cipig.net (line 20059). Digibyte.java Release notes script Changelog formatting now includes title, SHA, and multiline body bullets. generate-release-notes.sh --- .../org/qortal/crosschain/ElectrumServer.java | 5 + .../java/org/qortal/crosschain/ElectrumX.java | 214 ++++++++++++++++-- 2 files changed, 194 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumServer.java b/src/main/java/org/qortal/crosschain/ElectrumServer.java index d17316087..cbb40e9b9 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumServer.java +++ b/src/main/java/org/qortal/crosschain/ElectrumServer.java @@ -2,6 +2,7 @@ import org.qortal.crypto.TrustlessSSLSocketFactory; +import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; import java.io.IOException; import java.net.Socket; @@ -45,6 +46,10 @@ private void init(ChainableServer server, SocketAddress endpoint, int timeout, C if (this.server.getConnectionType() == ElectrumX.Server.ConnectionType.SSL) { SSLSocketFactory factory = TrustlessSSLSocketFactory.getSocketFactory(); this.socket = factory.createSocket(this.socket, server.getHostName(), server.getPort(), true); + this.socket.setSoTimeout(timeout); + this.socket.setTcpNoDelay(true); + this.socket.getOutputStream().flush(); + ((SSLSocket) this.socket).startHandshake(); } this.scanner = new Scanner(this.socket.getInputStream()); diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 0bda0da72..2b124427b 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -19,6 +19,7 @@ import java.text.DecimalFormat; import java.util.*; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; @@ -41,7 +42,9 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private static final double MAX_PROTOCOL_VERSION = 2.0; // Higher than current latest, for hopeful future-proofing private static final int MIN_TARGET_CONNECTIONS = 2; private static final int DEFAULT_TARGET_CONNECTIONS = 3; - private static final int MAX_TARGET_CONNECTIONS = 12; + private static final double TARGET_CONNECTIONS_FRACTION = 0.75d; + private static final int PROBE_TIMEOUT_MS = 2000; + private static final long FAILURE_PENALTY_MS = 5000L; private static final int BLOCK_HEADER_LENGTH = 80; @@ -53,6 +56,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private static final int RESPONSE_TIME_READINGS = 5; private static final long MAX_AVG_RESPONSE_TIME = 2000L; // ms + private static final long UNKNOWN_RESPONSE_PENALTY_MS = MAX_AVG_RESPONSE_TIME * 5; public static final String MISSING_FEATURES_ERROR = "MISSING FEATURES ERROR"; public static final String EXPECTED_GENESIS_ERROR = "EXPECTED GENESIS ERROR"; private static final long IDLE_DISCONNECT_MS = 2 * 60 * 1000L; @@ -61,7 +65,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // the minimum number of connections targeted for this foreign blockchain private int minimumConnections; - private final int maximumConnections; + private int maximumConnections; public static class Server implements ChainableServer { String hostname; @@ -87,11 +91,15 @@ public void addResponseTime(long responseTime) { @Override public long averageResponseTime() { - if (this.responseTimes.size() < RESPONSE_TIME_READINGS) { + List snapshot; + synchronized (this.responseTimes) { + snapshot = new ArrayList<>(this.responseTimes); + } + if (snapshot.size() < RESPONSE_TIME_READINGS) { // Not enough readings yet return 0L; } - OptionalDouble average = this.responseTimes.stream().mapToDouble(a -> a).average(); + OptionalDouble average = snapshot.stream().filter(Objects::nonNull).mapToDouble(a -> a).average(); if (average.isPresent()) { return Double.valueOf(average.getAsDouble()).longValue(); } @@ -174,6 +182,10 @@ public boolean removeEldestEntry(Map.Entry eldest) private volatile boolean connectionManagementStarted = false; private volatile long lastRpcTimeMs = 0L; private final AtomicInteger inFlightRpcCount = new AtomicInteger(0); + private final Map serverFailureCounts = new ConcurrentHashMap<>(); + private final Set probedServers = ConcurrentHashMap.newKeySet(); + private volatile boolean initialProbeCompleted = false; + private volatile String lastScoreExtremesDigest = ""; // Constructors @@ -183,18 +195,7 @@ public ElectrumX(String netId, String genesisHash, Collection initialSer this.servers.addAll(initialServerList); this.defaultPorts.putAll(defaultPorts); - int listSize = initialServerList.size(); - if (listSize == 0) { - this.maximumConnections = 0; - this.minimumConnections = 0; - return; - } - - int scaledTarget = (listSize / 10) + 1; - int targetConnections = clamp(scaledTarget, MIN_TARGET_CONNECTIONS, MAX_TARGET_CONNECTIONS); - targetConnections = Math.min(listSize, Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS)); - this.maximumConnections = targetConnections; - this.minimumConnections = Math.max(1, Math.min(listSize, Math.max(1, targetConnections / 2))); + updateConnectionTargets(initialServerList.size()); } // Methods for use by other classes @@ -777,6 +778,131 @@ private void closeAllConnections(String reason) { } } + private long averageConnectedResponseTime() { + long total = 0L; + int count = 0; + synchronized (this.connections) { + for (ElectrumServer server : this.connections) { + long responseTime = server.averageResponseTime(); + if (responseTime > 0) { + total += responseTime; + count++; + } + } + } + return count == 0 ? 0L : total / count; + } + + private void updateConnectionTargets(int listSize) { + if (listSize <= 0) { + this.maximumConnections = 0; + this.minimumConnections = 0; + LOGGER.info("{} has no ElectrumX servers configured", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode()); + return; + } + + int targetConnections = (int) Math.ceil(listSize * TARGET_CONNECTIONS_FRACTION); + targetConnections = Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS); + int minTarget = Math.min(MIN_TARGET_CONNECTIONS, listSize); + targetConnections = clamp(targetConnections, minTarget, listSize); + this.maximumConnections = targetConnections; + this.minimumConnections = Math.max(1, Math.min(listSize, Math.max(1, targetConnections / 2))); + + LOGGER.info("{} targets {} connections (min {}), listSize {}, avgResponse {}ms", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), this.maximumConnections, this.minimumConnections, listSize, averageConnectedResponseTime()); + } + + private long scoreServer(ChainableServer server) { + long averageResponse = server.averageResponseTime(); + long latencyScore = averageResponse > 0 ? averageResponse : UNKNOWN_RESPONSE_PENALTY_MS; + int failures = this.serverFailureCounts.getOrDefault(server, 0); + return latencyScore + (failures * FAILURE_PENALTY_MS); + } + + private List selectPreferredServers(int maxServers) { + List snapshot; + synchronized (this.connectionListLock) { + snapshot = new ArrayList<>(this.servers); + } + if (snapshot.isEmpty() || maxServers <= 0) { + return Collections.emptyList(); + } + snapshot.sort(Comparator.comparingLong(this::scoreServer)); + logScoreExtremes(snapshot); + int limit = Math.min(maxServers, snapshot.size()); + return new ArrayList<>(snapshot.subList(0, limit)); + } + + private void logScoreExtremes(List sortedServers) { + int limit = Math.min(3, sortedServers.size()); + if (limit == 0) { + return; + } + + StringBuilder best = new StringBuilder(); + StringBuilder worst = new StringBuilder(); + for (int i = 0; i < limit; i++) { + if (i > 0) { + best.append(", "); + } + ChainableServer server = sortedServers.get(i); + best.append(server).append(":").append(scoreServer(server)).append("ms"); + } + for (int i = sortedServers.size() - limit; i < sortedServers.size(); i++) { + if (i > sortedServers.size() - limit) { + worst.append(", "); + } + ChainableServer server = sortedServers.get(i); + worst.append(server).append(":").append(scoreServer(server)).append("ms"); + } + + String digest = best.toString() + "|" + worst.toString() + "|" + this.connections.size(); + if (digest.equals(this.lastScoreExtremesDigest)) { + return; + } + this.lastScoreExtremesDigest = digest; + + LOGGER.info("{} top {} ElectrumX servers: {}", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), limit, best); + LOGGER.info("{} bottom {} ElectrumX servers: {}", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), limit, worst); + } + + private void recordFailure(ChainableServer server) { + this.serverFailureCounts.merge(server, 1, Integer::sum); + } + + private void recordSuccess(ChainableServer server) { + this.serverFailureCounts.put(server, 0); + } + + private void probeServers(Collection servers) { + for (ChainableServer server : servers) { + if (this.probedServers.add(server)) { + probeServer(server); + } + } + } + + private void probeServer(ChainableServer server) { + ElectrumServer electrumServer = null; + try { + SocketAddress endpoint = new InetSocketAddress(server.getHostName(), server.getPort()); + electrumServer = ElectrumServer.createInstance(server, endpoint, PROBE_TIMEOUT_MS, this.recorder); + electrumServer.setClientName(randomClientName()); + + Object response = connectedRpc(electrumServer, "server.version"); + if (response != null) { + recordSuccess(server); + } else { + recordFailure(server); + } + } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { + recordFailure(server); + } finally { + if (electrumServer != null) { + electrumServer.closeServer(this.getClass().getSimpleName(), "probe"); + } + } + } + /** * Ensure the connection maintenance threads are running and initial connections exist. */ @@ -797,6 +923,18 @@ private void ensureConnectionManagementStarted() { return; } + if (!this.initialProbeCompleted) { + List serversSnapshot; + synchronized (this.connectionListLock) { + serversSnapshot = new ArrayList<>(this.servers); + } + if (!serversSnapshot.isEmpty()) { + LOGGER.info("{} probing {} ElectrumX servers for initial scoring", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), serversSnapshot.size()); + probeServers(serversSnapshot); + } + this.initialProbeCompleted = true; + } + startMakingConnections(); scheduleMakeConnections.scheduleWithFixedDelay(this::makeConnections, 1, 3600, TimeUnit.SECONDS); @@ -876,11 +1014,12 @@ private void monitorConnections() { } LOGGER.info( - "{} {} available connections, {} total servers, {} total connections, {} useless servers", + "{} {} available connections, {} total servers, {} total connections (target {}), {} useless servers", this.blockchain.getCurrencyCode(), this.availableConnections.size(), this.servers.size(), this.connections.size(), + this.maximumConnections, this.uselessServers.size() ); } @@ -935,7 +1074,10 @@ private void startMakingConnections() { // assume there are no server to get peers from, so we must start from the base list synchronized (this.connectionListLock) { this.remainingServers.clear(); - this.remainingServers.addAll(this.servers); + updateConnectionTargets(this.servers.size()); + List preferredServers = selectPreferredServers(this.maximumConnections); + LOGGER.info("{} selecting {} of {} ElectrumX servers by score", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), preferredServers.size(), this.servers.size()); + this.remainingServers.addAll(preferredServers); } connectRemainingServers(); @@ -953,15 +1095,27 @@ private void makeMoreConnections() { // Ask for more servers Set moreServers = serverPeersSubscribe(); + List newlyAdded = new ArrayList<>(); synchronized (this.connectionListLock) { - // Add all servers to base list + for (Server server : moreServers) { + if (!this.servers.contains(server)) { + newlyAdded.add(server); + } + } this.servers.addAll(moreServers); + } - // add base list to remaining list - this.remainingServers.addAll(this.servers); + if (!newlyAdded.isEmpty()) { + LOGGER.info("{} probing {} newly discovered ElectrumX servers", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), newlyAdded.size()); + probeServers(newlyAdded); + } - // remove servers that this node is already connected to + synchronized (this.connectionListLock) { + updateConnectionTargets(this.servers.size()); + List preferredServers = selectPreferredServers(this.maximumConnections); + this.remainingServers.clear(); + this.remainingServers.addAll(preferredServers); this.remainingServers.removeAll(this.connections.stream().map(ElectrumServer::getServer).collect(Collectors.toList())); } @@ -1021,29 +1175,39 @@ private Optional makeConnection(ChainableServer serve // Check connection is suitable by asking for server features, including genesis block hash JSONObject featuresJson = (JSONObject) this.connectedRpc(electrumServer, "server.features"); - if (featuresJson == null ) + if (featuresJson == null ) { + recordFailure(server); return Optional.of( recorder.recordConnection(server, requestedBy, true, false, MISSING_FEATURES_ERROR) ); + } try { double protocol_min = CrossChainUtils.getVersionDecimal(featuresJson, "protocol_min"); - if (protocol_min < MIN_PROTOCOL_VERSION) + if (protocol_min < MIN_PROTOCOL_VERSION) { + recordFailure(server); return Optional.of( recorder.recordConnection(server, requestedBy, true, false, "old version: protocol_min = " + protocol_min + " < MIN_PROTOCOL_VERSION = " + MIN_PROTOCOL_VERSION) ); + } } catch (NumberFormatException e) { + recordFailure(server); return Optional.of( recorder.recordConnection(server, requestedBy,true, false,featuresJson.get("protocol_min").toString() + " is not a valid version")); } catch (NullPointerException e) { + recordFailure(server); return Optional.of( recorder.recordConnection(server, requestedBy,true, false,"server version not available: protocol_min")); } - if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) + if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) { + recordFailure(server); return Optional.of( recorder.recordConnection(server, requestedBy, true, false, EXPECTED_GENESIS_ERROR) ); + } + recordSuccess(server); LOGGER.debug(() -> String.format("Connected to %s %s", server, this.blockchain.currencyCode)); this.connections.add(electrumServer); this.availableConnections.add(electrumServer); return Optional.of( this.recorder.recordConnection( server, requestedBy, true, true, EMPTY) ); } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { // Didn't work, try another server... + recordFailure(server); return Optional.of( this.recorder.recordConnection( server, requestedBy, true, false, CrossChainUtils.getNotes(e))); } catch( Exception e ) { LOGGER.error(e.getMessage(), e); From 6b4cb8f9985be66b603a59a61afe9497dc636ae7 Mon Sep 17 00:00:00 2001 From: crowetic Date: Tue, 13 Jan 2026 19:41:43 -0800 Subject: [PATCH 19/28] Added Null Pointer Exception prevention and retry logic for failed connections. Retaining connect-on-demand overall. --- .../java/org/qortal/crosschain/ElectrumX.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 2b124427b..8fd3606e9 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -44,6 +44,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private static final int DEFAULT_TARGET_CONNECTIONS = 3; private static final double TARGET_CONNECTIONS_FRACTION = 0.75d; private static final int PROBE_TIMEOUT_MS = 2000; + private static final long PROBE_RETRY_MS = 5 * 60 * 1000L; private static final long FAILURE_PENALTY_MS = 5000L; private static final int BLOCK_HEADER_LENGTH = 80; @@ -183,7 +184,7 @@ public boolean removeEldestEntry(Map.Entry eldest) private volatile long lastRpcTimeMs = 0L; private final AtomicInteger inFlightRpcCount = new AtomicInteger(0); private final Map serverFailureCounts = new ConcurrentHashMap<>(); - private final Set probedServers = ConcurrentHashMap.newKeySet(); + private final Map serverLastProbeTime = new ConcurrentHashMap<>(); private volatile boolean initialProbeCompleted = false; private volatile String lastScoreExtremesDigest = ""; @@ -802,6 +803,9 @@ private void updateConnectionTargets(int listSize) { } int targetConnections = (int) Math.ceil(listSize * TARGET_CONNECTIONS_FRACTION); + if (listSize > 30) { + targetConnections = 30; + } targetConnections = Math.max(targetConnections, DEFAULT_TARGET_CONNECTIONS); int minTarget = Math.min(MIN_TARGET_CONNECTIONS, listSize); targetConnections = clamp(targetConnections, minTarget, listSize); @@ -874,10 +878,14 @@ private void recordSuccess(ChainableServer server) { } private void probeServers(Collection servers) { + long now = System.currentTimeMillis(); for (ChainableServer server : servers) { - if (this.probedServers.add(server)) { - probeServer(server); + Long lastProbe = this.serverLastProbeTime.get(server); + if (lastProbe != null && now - lastProbe < PROBE_RETRY_MS) { + continue; } + this.serverLastProbeTime.put(server, now); + probeServer(server); } } @@ -1057,9 +1065,14 @@ private void recoverConnections() { return; } if( this.connections.size() < this.minimumConnections ) { - LOGGER.debug("{} recovering connections", this.blockchain.currencyCode); + LOGGER.debug("{} recovering connections", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode()); + List serversSnapshot; + synchronized (this.connectionListLock) { + serversSnapshot = new ArrayList<>(this.servers); + } + probeServers(serversSnapshot); startMakingConnections(); - LOGGER.debug("{} recovered {} connections", this.blockchain.currencyCode, this.connections.size()); + LOGGER.debug("{} recovered {} connections", this.blockchain == null ? "ElectrumX" : this.blockchain.getCurrencyCode(), this.connections.size()); } } catch (Exception e) { LOGGER.error(e.getMessage(), e); From 53ce930352298bc9b892257582659675e5c09dfd Mon Sep 17 00:00:00 2001 From: crowetic Date: Tue, 13 Jan 2026 19:56:37 -0800 Subject: [PATCH 20/28] Fixed reading synchronized list without locking it by removing stream usage that was iterating while mutations could happen. --- .../ChainableServerConnectionRecorder.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ChainableServerConnectionRecorder.java b/src/main/java/org/qortal/crosschain/ChainableServerConnectionRecorder.java index 23697c125..47dc9ff9e 100644 --- a/src/main/java/org/qortal/crosschain/ChainableServerConnectionRecorder.java +++ b/src/main/java/org/qortal/crosschain/ChainableServerConnectionRecorder.java @@ -21,13 +21,18 @@ public ChainableServerConnection recordConnection( ChainableServerConnection connection = new ChainableServerConnection(server, requestedBy, open, success, System.currentTimeMillis(), notes); - connections.add(connection); - - if( connections.size() > limit) { - ChainableServerConnection firstConnection - = connections.stream().sorted(Comparator.comparing(ChainableServerConnection::getCurrentTimeMillis)) - .findFirst().get(); - connections.remove(firstConnection); + synchronized (connections) { + connections.add(connection); + + if (connections.size() > limit) { + ChainableServerConnection firstConnection = connections.get(0); + for (ChainableServerConnection candidate : connections) { + if (candidate.getCurrentTimeMillis() < firstConnection.getCurrentTimeMillis()) { + firstConnection = candidate; + } + } + connections.remove(firstConnection); + } } return connection; } From 532ef2205653f5ec8b3a9ca4c0ef1226a48c1d0c Mon Sep 17 00:00:00 2001 From: crowetic Date: Tue, 13 Jan 2026 20:04:56 -0800 Subject: [PATCH 21/28] added fix for 'cannot invoke intvalue' in follower thread by checking for non-null block height. --- src/main/java/org/qortal/controller/arbitrary/Follower.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/Follower.java b/src/main/java/org/qortal/controller/arbitrary/Follower.java index 228640f4c..6a06cc5d8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/Follower.java +++ b/src/main/java/org/qortal/controller/arbitrary/Follower.java @@ -68,7 +68,8 @@ private void fetch(OptionalInt limit) { final int blockHeightThreshold = repository.getBlockRepository().getBlockchainHeight() - limit.getAsInt(); transactionsInReverseOrder - = latestArbitraryTransactionsByName.stream().filter(tx -> tx.getBlockHeight() > blockHeightThreshold) + = latestArbitraryTransactionsByName.stream() + .filter(tx -> tx.getBlockHeight() != null && tx.getBlockHeight() > blockHeightThreshold) .collect(Collectors.toList()); } else { transactionsInReverseOrder = latestArbitraryTransactionsByName; From b9396ed36dbaab9ea3dbef408bc3f9468051c9a0 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 17 Jan 2026 14:26:51 -0800 Subject: [PATCH 22/28] skip foreign fees with unavailable AT data when processing foreign fees --- src/main/java/org/qortal/controller/ForeignFeesManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java index 3763b7b30..71fb7ca7f 100644 --- a/src/main/java/org/qortal/controller/ForeignFeesManager.java +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -415,6 +415,9 @@ private void processForeignFeesImportQueue() { ATData atData = repository.getATRepository().fromATAddress(atAddress); + // if AT data is not available, then continue on to the next AT + if( atData == null ) continue; + LOGGER.debug("verify signer for atAddress = " + atAddress); // determine if the creator authorized the foreign fee From 6cf7d2dd8e08d777130c513cf1e2afd191bdfdcf Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 18 Jan 2026 15:37:21 +0100 Subject: [PATCH 23/28] Catches the InterruptedException in method fetchAllMetadata --- .../org/qortal/controller/arbitrary/ArbitraryDataManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 8f0bf7081..f4b9d752a 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -449,6 +449,10 @@ private void fetchAllMetadata() throws InterruptedException { ); } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + } catch (InterruptedException e) { + // Thread interrupted during shutdown - restore interrupt status and exit + Thread.currentThread().interrupt(); + return; } catch (Exception e) { LOGGER.error(e.getMessage(), e); } From 53958e5ae76b6a3084cd714a98afbe49a4ae3798 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 18 Jan 2026 16:02:07 +0100 Subject: [PATCH 24/28] Improve performances of rebuildAllNames --- .../NamesDatabaseIntegrityCheck.java | 157 +++++++++++++++--- 1 file changed, 134 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 359819764..67772c3b2 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -30,6 +30,7 @@ public class NamesDatabaseIntegrityCheck { ); private List nameTransactions = new ArrayList<>(); + private Map> transactionsByNameCache = null; public int rebuildName(String name, Repository repository) { @@ -130,18 +131,55 @@ public int rebuildName(String name, Repository repository) { public int rebuildAllNames() { int modificationCount = 0; + long startTime = System.currentTimeMillis(); + try (final Repository repository = RepositoryManager.getRepository()) { - List names = this.fetchAllNames(repository); // TODO: de-duplicate, to speed up this process + // Build cache of all transactions by name to avoid repeated database queries + long cacheStartTime = System.currentTimeMillis(); + this.buildTransactionsByNameCache(repository); + long cacheEndTime = System.currentTimeMillis(); + LOGGER.info("Cache built in {} ms", cacheEndTime - cacheStartTime); + + List names = this.fetchAllNames(repository); + int totalNames = names.size(); + LOGGER.info("Rebuilding {} names...", totalNames); + + int processedCount = 0; + int logInterval = Math.max(1, totalNames / 10); // Log every 10% + for (String name : names) { modificationCount += this.rebuildName(name, repository); + processedCount++; + + // Log progress every logInterval names + if (processedCount % logInterval == 0 || processedCount == totalNames) { + long elapsedTime = System.currentTimeMillis() - startTime; + double percentComplete = (processedCount * 100.0) / totalNames; + long estimatedTotalTime = (long) (elapsedTime / (processedCount / (double) totalNames)); + long estimatedTimeRemaining = estimatedTotalTime - elapsedTime; + + LOGGER.info("Progress: {}/{} names ({:.1f}%) - Elapsed: {} ms - Est. remaining: {} ms", + processedCount, totalNames, percentComplete, + elapsedTime, estimatedTimeRemaining); + } } + + long saveStartTime = System.currentTimeMillis(); repository.saveChanges(); + long saveEndTime = System.currentTimeMillis(); + LOGGER.info("Changes saved in {} ms", saveEndTime - saveStartTime); + + // Clear cache after use + this.transactionsByNameCache = null; + + long totalTime = System.currentTimeMillis() - startTime; + LOGGER.info("Rebuild completed: {} modifications in {} ms ({} seconds)", + modificationCount, totalTime, totalTime / 1000.0); } catch (DataException e) { LOGGER.info("Error when running integrity check for all names: {}", e.getMessage()); } - //LOGGER.info("modificationCount: {}", modificationCount); return modificationCount; } @@ -307,7 +345,86 @@ private void fetchAllNameTransactions(Repository repository) throws DataExceptio this.nameTransactions = nameTransactions; } + private void buildTransactionsByNameCache(Repository repository) throws DataException { + LOGGER.info("Building transaction cache for all names..."); + this.transactionsByNameCache = new HashMap<>(); + + // Fetch all name transactions if not already fetched + if (this.nameTransactions.isEmpty()) { + this.fetchAllNameTransactions(repository); + } + + // Group all transactions by the names they involve + for (TransactionData transactionData : this.nameTransactions) { + // Filter out unconfirmed transactions + if (transactionData.getBlockHeight() == null || transactionData.getBlockHeight() <= 0) { + continue; + } + + Set involvedNames = new HashSet<>(); + + if (transactionData instanceof RegisterNameTransactionData) { + RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; + involvedNames.add(registerNameTransactionData.getName()); + String reducedName = Unicode.sanitize(registerNameTransactionData.getName()); + if (reducedName != null && !reducedName.equals(registerNameTransactionData.getName())) { + involvedNames.add(reducedName); + } + } + else if (transactionData instanceof UpdateNameTransactionData) { + UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; + involvedNames.add(updateNameTransactionData.getName()); + if (updateNameTransactionData.getNewName() != null) { + involvedNames.add(updateNameTransactionData.getNewName()); + String reducedNewName = Unicode.sanitize(updateNameTransactionData.getNewName()); + if (reducedNewName != null && !reducedNewName.isEmpty()) { + involvedNames.add(reducedNewName); + } + } + } + else if (transactionData instanceof BuyNameTransactionData) { + BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; + involvedNames.add(buyNameTransactionData.getName()); + } + else if (transactionData instanceof SellNameTransactionData) { + SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; + involvedNames.add(sellNameTransactionData.getName()); + } + else if (transactionData instanceof CancelSellNameTransactionData) { + CancelSellNameTransactionData cancelSellNameTransactionData = (CancelSellNameTransactionData) transactionData; + involvedNames.add(cancelSellNameTransactionData.getName()); + } + + // Add this transaction to all involved names + for (String involvedName : involvedNames) { + if (involvedName == null || involvedName.isEmpty()) { + continue; + } + this.transactionsByNameCache.computeIfAbsent(involvedName, k -> new ArrayList<>()).add(transactionData); + } + } + + // Sort all transaction lists by block height and timestamp + for (List transactions : this.transactionsByNameCache.values()) { + sortTransactions(transactions); + } + + LOGGER.info("Transaction cache built for {} unique names", this.transactionsByNameCache.size()); + } + public List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { + // Use cache if available + if (this.transactionsByNameCache != null) { + List cachedTransactions = this.transactionsByNameCache.get(name); + if (cachedTransactions != null) { + // Return a copy to avoid external modifications + return new ArrayList<>(cachedTransactions); + } + // If not in cache, return empty list (all names should be in cache when it's built) + return new ArrayList<>(); + } + + // Fall back to database queries if cache not available List signatures = new ArrayList<>(); String reducedName = Unicode.sanitize(name); @@ -361,7 +478,7 @@ private TransactionData fetchLatestModificationTransactionInvolvingName(String r } private List fetchAllNames(Repository repository) throws DataException { - List names = new ArrayList<>(); + Set namesSet = new HashSet<>(); // Fetch all the confirmed name transactions if (this.nameTransactions.isEmpty()) { @@ -372,46 +489,39 @@ private List fetchAllNames(Repository repository) throws DataException { if ((transactionData instanceof RegisterNameTransactionData)) { RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; - if (!names.contains(registerNameTransactionData.getName())) { - names.add(registerNameTransactionData.getName()); - } + namesSet.add(registerNameTransactionData.getName()); } if ((transactionData instanceof UpdateNameTransactionData)) { UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; - if (!names.contains(updateNameTransactionData.getName())) { - names.add(updateNameTransactionData.getName()); - } - if (!names.contains(updateNameTransactionData.getNewName())) { - names.add(updateNameTransactionData.getNewName()); + namesSet.add(updateNameTransactionData.getName()); + if (updateNameTransactionData.getNewName() != null) { + namesSet.add(updateNameTransactionData.getNewName()); } } if ((transactionData instanceof BuyNameTransactionData)) { BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; - if (!names.contains(buyNameTransactionData.getName())) { - names.add(buyNameTransactionData.getName()); - } + namesSet.add(buyNameTransactionData.getName()); } if ((transactionData instanceof SellNameTransactionData)) { SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; - if (!names.contains(sellNameTransactionData.getName())) { - names.add(sellNameTransactionData.getName()); - } + namesSet.add(sellNameTransactionData.getName()); } if ((transactionData instanceof CancelSellNameTransactionData)) { CancelSellNameTransactionData cancelSellNameTransactionData = (CancelSellNameTransactionData) transactionData; - if (!names.contains(cancelSellNameTransactionData.getName())) { - names.add(cancelSellNameTransactionData.getName()); - } + namesSet.add(cancelSellNameTransactionData.getName()); } } - return names; + return new ArrayList<>(namesSet); } private int addAdditionalTransactionsRelatingToName(List transactions, String name, Repository repository) throws DataException { int added = 0; + // Use a HashSet for O(1) lookups when checking for existing transactions + Set existingTransactions = new HashSet<>(transactions); + // If this name has been updated at any point, we need to add transactions from the other names to the sequence - List otherNames = new ArrayList<>(); + Set otherNames = new HashSet<>(); List updateNameTransactions = transactions.stream().filter(t -> t.getType() == TransactionType.UPDATE_NAME).collect(Collectors.toList()); for (TransactionData transactionData : updateNameTransactions) { UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; @@ -431,9 +541,10 @@ private int addAdditionalTransactionsRelatingToName(List transa for (String otherName : otherNames) { List otherNameTransactions = this.fetchAllTransactionsInvolvingName(otherName, repository); for (TransactionData otherNameTransactionData : otherNameTransactions) { - if (!transactions.contains(otherNameTransactionData)) { + if (!existingTransactions.contains(otherNameTransactionData)) { // Add new transaction relating to other name transactions.add(otherNameTransactionData); + existingTransactions.add(otherNameTransactionData); added++; } } From 08378f9d957496edc2d045ca99c1d50b71f4dce8 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 18 Jan 2026 16:18:50 +0100 Subject: [PATCH 25/28] Improve progress logging --- .../controller/repository/NamesDatabaseIntegrityCheck.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 67772c3b2..67b985deb 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -158,9 +158,9 @@ public int rebuildAllNames() { long estimatedTotalTime = (long) (elapsedTime / (processedCount / (double) totalNames)); long estimatedTimeRemaining = estimatedTotalTime - elapsedTime; - LOGGER.info("Progress: {}/{} names ({:.1f}%) - Elapsed: {} ms - Est. remaining: {} ms", + LOGGER.info(String.format("Progress: %d/%d names (%.1f%%) - Elapsed: %d ms - Est. remaining: %d ms", processedCount, totalNames, percentComplete, - elapsedTime, estimatedTimeRemaining); + elapsedTime, estimatedTimeRemaining)); } } From 22902014e38db37128aae09779a72f9e4f0fafd0 Mon Sep 17 00:00:00 2001 From: Nic Date: Sun, 18 Jan 2026 16:42:53 +0100 Subject: [PATCH 26/28] Add clearer logging --- .../org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java index 5647bd602..6c7f9de30 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -43,9 +43,10 @@ public HSQLDBRepositoryFactory(String connectionUrl) throws DataException { this.connectionUrl = connectionUrl; // Check no-one else is accessing database + LOGGER.info("Opening database connection (this may take a while if replaying transaction logs)..."); try (Connection connection = DriverManager.getConnection(this.connectionUrl)) { // We only need to check we can obtain connection. It will be auto-closed. - LOGGER.info("Checking database connection..."); + LOGGER.info("Database connection established"); } catch (SQLException e) { Throwable cause = e.getCause(); if (!(cause instanceof HsqlException)) From d7f6f77686658bfa896ae3456a517a250f4cbcec Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 18 Jan 2026 15:06:10 -0800 Subject: [PATCH 27/28] removing group chat messages from non-members --- .../repository/hsqldb/HSQLDBChatRepository.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 66b2447a2..ad2fc04df 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -7,6 +7,7 @@ import org.qortal.data.chat.ActiveChats.DirectChat; import org.qortal.data.chat.ActiveChats.GroupChat; import org.qortal.data.chat.ChatMessage; +import org.qortal.data.group.GroupMemberData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.repository.ChatRepository; import org.qortal.repository.DataException; @@ -16,6 +17,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import static org.qortal.data.chat.ChatMessage.Encoding; @@ -149,6 +151,17 @@ else if (hasChatReference != null && !hasChatReference) { chatMessages.add(chatMessage); } while (resultSet.next()); + // if this is a group chat, then ensure that the sender is in the group + if( txGroupId != null && txGroupId > 0 ) { + List members + = this.repository.getGroupRepository() + .getGroupMembers(txGroupId).stream() + .map(GroupMemberData::getMember) + .collect(Collectors.toList()); + + chatMessages.removeIf( data -> !members.contains(data.getSender()) ); + } + return chatMessages; } catch (SQLException e) { throw new DataException("Unable to fetch matching chat transactions from repository", e); From 5b15867357ff3f9bcc15a2684ae357718a9b77dc Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 21 Jan 2026 16:11:35 -0800 Subject: [PATCH 28/28] general chat transactions are rejected when sent and are discarded when received --- .../java/org/qortal/api/resource/TransactionsResource.java | 5 +++++ .../org/qortal/api/websocket/ChatMessagesWebSocket.java | 6 ++++++ .../java/org/qortal/controller/TransactionImporter.java | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index f6f82153c..964d38d6e 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -832,6 +832,11 @@ public String processTransaction(String rawBytes58, @HeaderParam(ApiService.API_ if (transactionData == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + // general chat transactions are invalid + if( transactionData.getType() == TransactionType.CHAT && transactionData.getTxGroupId() == 0 && transactionData.getRecipient() == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + try (final Repository repository = RepositoryManager.getRepository()) { Transaction transaction = Transaction.fromData(repository, transactionData); diff --git a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java index 4340ad581..abfecfd32 100644 --- a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java @@ -45,6 +45,12 @@ public void onWebSocketConnect(Session session) { if (txGroupIds != null && txGroupIds.size() == 1) { int txGroupId = Integer.parseInt(txGroupIds.get(0)); + // reject general chat + if( txGroupId == 0 ) { + session.close(4001, "invalid criteria"); + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { List chatMessages = repository.getChatRepository().getMessagesMatchingCriteria( null, diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 37aa8af13..80f9402dd 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -266,6 +266,13 @@ private void importTransactionsInQueue() { return; } + // discard general chat transactions, chat transactions with no group and no recipient + sigValidTransactions.removeIf( + transactionData -> transactionData.getType() == Transaction.TransactionType.CHAT && + transactionData.getTxGroupId() == 0 && + transactionData.getRecipient() == null + ); + if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { // Prioritize syncing, and don't attempt to lock return;