Optimizations, General Chat Removal, Bug Fixes

src/main/java/org/qortal/api/resource/TransactionsResource.java

@@ -832,6 +832,11 @@ public String processTransaction(String rawBytes58, @HeaderParam(ApiService.API_
 		if (transactionData == null)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
 
+		// general chat transactions are invalid
+		if( transactionData.getType() == TransactionType.CHAT && transactionData.getTxGroupId() == 0 && transactionData.getRecipient() == null) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			Transaction transaction = Transaction.fromData(repository, transactionData);
 

src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java

@@ -45,6 +45,12 @@ public void onWebSocketConnect(Session session) {
 		if (txGroupIds != null && txGroupIds.size() == 1) {
 			int txGroupId = Integer.parseInt(txGroupIds.get(0));
 
+			// reject general chat
+			if( txGroupId == 0 ) {
+				session.close(4001, "invalid criteria");
+				return;
+			}
+
 			try (final Repository repository = RepositoryManager.getRepository()) {
 				List<ChatMessage> chatMessages = repository.getChatRepository().getMessagesMatchingCriteria(
 						null,

src/main/java/org/qortal/controller/Controller.java

@@ -37,6 +37,7 @@
 import org.qortal.network.PeerAddress;
 import org.qortal.network.message.*;
 import org.qortal.repository.*;
+import org.qortal.repository.hsqldb.HSQLDBCacheUtils;
 import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.Transaction;
@@ -423,15 +424,15 @@ public static void main(String[] args) {
 			}
 
 			if( Settings.getInstance().isDbCacheEnabled() ) {
-				LOGGER.info("Db Cache Starting ...");
+				LOGGER.info("Starting Db Cache...");
 				HSQLDBDataCacheManager hsqldbDataCacheManager = new HSQLDBDataCacheManager();
 				hsqldbDataCacheManager.start();
 			}
 			else {
 				LOGGER.info("Db Cache Disabled");
 			}
 
-			LOGGER.info("Arbitrary Indexing Starting ...");
+			LOGGER.info("Starting Arbitrary Indexing...");
 			ArbitraryIndexUtils.startCaching(
 				Settings.getInstance().getArbitraryIndexingPriority(),
 				Settings.getInstance().getArbitraryIndexingFrequency()
@@ -441,15 +442,15 @@ public static void main(String[] args) {
 				Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
 
 				if( recorder.isPresent() ) {
-					LOGGER.info("Balance Recorder Starting ...");
+					LOGGER.info("Starting Balance Recorder...");
 					recorder.get().start();
 				}
 				else {
 					LOGGER.info("Balance Recorder won't start.");
 				}
 			}
 			else {
-				LOGGER.info("Balance Recorder Disabled");
+				LOGGER.info("Balance Recorder disabled");
 			}
 		} catch (DataException e) {
 			// If exception has no cause or message then repository is in use by some other process.
@@ -469,17 +470,17 @@ public static void main(String[] args) {
 
 			// Rebuild Names table and check database integrity (if enabled)
 			NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck();
+			LOGGER.info("Rebuilding all names...");
 			namesDatabaseIntegrityCheck.rebuildAllNames();
 			if (Settings.getInstance().isNamesIntegrityCheckEnabled()) {
+				LOGGER.info("Running database integrity check...");
 				namesDatabaseIntegrityCheck.runIntegrityCheck();
 			}
-
-			LOGGER.info("Validating blockchain");
+			LOGGER.info("Validating blockchain...");
 			try {
 				BlockChain.validate();
-
 				Controller.getInstance().refillLatestBlocksCache();
-				LOGGER.info(String.format("Our chain height at start-up: %d", Controller.getInstance().getChainHeight()));
+				LOGGER.info("Chain height at start-up: {}", Controller.getInstance().getChainHeight());
 			} catch (DataException e) {
 				LOGGER.error("Couldn't validate blockchain", e);
 				Gui.getInstance().fatalError("Blockchain validation issue", e);
@@ -565,7 +566,6 @@ public void run() {
 			);
 		}
 
-
 		LOGGER.info("Starting online accounts manager");
 		OnlineAccountsManager.getInstance().start();
 
@@ -599,7 +599,7 @@ public void run() {
 		}
 
 		if (Settings.getInstance().isGatewayEnabled()) {
-			LOGGER.info(String.format("Starting gateway service on port %d", Settings.getInstance().getGatewayPort()));
+			LOGGER.info("Starting gateway service on port {}", Settings.getInstance().getGatewayPort());
 			try {
 				GatewayService gatewayService = GatewayService.getInstance();
 				gatewayService.start();
@@ -612,7 +612,7 @@ public void run() {
 		}
 
 		if (Settings.getInstance().isDomainMapEnabled()) {
-			LOGGER.info(String.format("Starting domain map service on port %d", Settings.getInstance().getDomainMapPort()));
+			LOGGER.info("Starting domain map service on port {}", Settings.getInstance().getDomainMapPort());
 			try {
 				DomainMapService domainMapService = DomainMapService.getInstance();
 				domainMapService.start();
@@ -782,12 +782,12 @@ public void run() {
 					if (ntpTime != null) {
 						if (ntpTime != now)
 							// Only log if non-zero offset
-							LOGGER.info(String.format("Adjusting system time by NTP offset: %dms", ntpTime - now));
+							LOGGER.info("Adjusting system time by NTP offset: {}ms", ntpTime - now);
 
 						ntpCheckTimestamp = now + NTP_POST_SYNC_CHECK_PERIOD;
 						requestSysTrayUpdate = true;
 					} else {
-						LOGGER.info(String.format("No NTP offset yet"));
+						LOGGER.info("No NTP offset yet");
 						ntpCheckTimestamp = now + NTP_PRE_SYNC_CHECK_PERIOD;
 						// We can't do much without a valid NTP time
 						continue;
@@ -1120,6 +1120,11 @@ public void shutdown() {
 
 				LOGGER.info("Shutting down synchronizer");
 				Synchronizer.getInstance().shutdown();
+				try {
+					Synchronizer.getInstance().join();
+				} catch (InterruptedException e) {
+					// We were interrupted while waiting for thread to join
+				}
 
 				LOGGER.info("Shutting down API");
 				ApiService.getInstance().stop();
@@ -1177,6 +1182,17 @@ public void shutdown() {
 					// We were interrupted while waiting for thread to join
 				}
 
+				LOGGER.info("Shutting down TradeBot");
+				TradeBot.getInstance().shutdown();
+
+				// Shutdown database cache timers before closing repository
+				LOGGER.info("Shutting down database cache timers");
+				HSQLDBCacheUtils.shutdown();
+
+				// Shutdown arbitrary metadata manager scheduler before closing repository
+				LOGGER.info("Shutting down arbitrary metadata manager");
+				ArbitraryMetadataManager.getInstance().shutdown();
+
 				// Make sure we're the only thread modifying the blockchain when shutting down the repository
 				ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
 				try {

src/main/java/org/qortal/controller/ForeignFeesManager.java

@@ -415,6 +415,9 @@ private void processForeignFeesImportQueue() {
 
                     ATData atData = repository.getATRepository().fromATAddress(atAddress);
 
+                    // if AT data is not available, then continue on to the next AT
+                    if( atData == null ) continue;
+
                     LOGGER.debug("verify signer for atAddress = " + atAddress);
 
                     // determine if the creator authorized the foreign fee

src/main/java/org/qortal/controller/TransactionImporter.java

@@ -92,6 +92,27 @@ public void run() {
     public void shutdown() {
         isStopping = true;
         this.interrupt();
+
+        // Shutdown all schedulers
+        LOGGER.info("Shutting down TransactionImporter schedulers");
+        try {
+            getTransactionMessageScheduler.shutdownNow();
+            getUnconfirmedTransactionsMessageScheduler.shutdownNow();
+            signatureMessageScheduler.shutdownNow();
+
+            if (!getTransactionMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) {
+                LOGGER.warn("getTransactionMessageScheduler did not terminate in time");
+            }
+            if (!getUnconfirmedTransactionsMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) {
+                LOGGER.warn("getUnconfirmedTransactionsMessageScheduler did not terminate in time");
+            }
+            if (!signatureMessageScheduler.awaitTermination(5, TimeUnit.SECONDS)) {
+                LOGGER.warn("signatureMessageScheduler did not terminate in time");
+            }
+        } catch (InterruptedException e) {
+            LOGGER.warn("Interrupted while waiting for TransactionImporter schedulers to terminate", e);
+            Thread.currentThread().interrupt();
+        }
     }
 
 
@@ -245,6 +266,13 @@ private void importTransactionsInQueue() {
             return;
         }
 
+        // discard general chat transactions, chat transactions with no group and no recipient
+        sigValidTransactions.removeIf(
+                transactionData -> transactionData.getType() == Transaction.TransactionType.CHAT &&
+                        transactionData.getTxGroupId() == 0 &&
+                        transactionData.getRecipient() == null
+        );
+
         if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) {
             // Prioritize syncing, and don't attempt to lock
             return;
@@ -397,6 +425,9 @@ public void onNetworkGetTransactionMessage(Peer peer, Message message) {
     }
 
     private void processNetworkGetTransactionMessages() {
+        if (Controller.isStopping()) {
+            return;
+        }
 
         try {
             List<PeerMessage> messagesToProcess;
@@ -490,14 +521,16 @@ private static void sendTransactionMessage(String signature58, TransactionData d
     // Scheduled executor service to process messages every second
     private final ScheduledExecutorService getUnconfirmedTransactionsMessageScheduler = Executors.newScheduledThreadPool(1);
 
-
     public void onNetworkGetUnconfirmedTransactionsMessage(Peer peer, Message message) {
         synchronized (getUnconfirmedTransactionsMessageLock) {
             getUnconfirmedTransactionsMessageList.add(new PeerMessage(peer, message));
         }
     }
 
     private void processNetworkGetUnconfirmedTransactionsMessages() {
+        if (Controller.isStopping()) {
+            return;
+        }
 
         List<PeerMessage> messagesToProcess;
         synchronized (getUnconfirmedTransactionsMessageLock) {
@@ -542,6 +575,9 @@ public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) {
     }
 
     public void processNetworkTransactionSignaturesMessage() {
+        if (Controller.isStopping()) {
+            return;
+        }
 
         try {
             List<PeerMessage> messagesToProcess;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java

@@ -63,6 +63,13 @@ public void run() {
 
                     // Process queue
                     processResourceQueue();
+                } catch (InterruptedException e) {
+                    // Check if we're shutting down
+                    if (Controller.isStopping()) {
+                        LOGGER.info("Arbitrary Data Cache Manager shutting down");
+                        break;
+                    }
+                    LOGGER.warn("Arbitrary Data Cache Manager interrupted, retrying...", e);
                 } catch (Exception e) {
                     LOGGER.error(e.getMessage(), e);
                     Thread.sleep(600_000L); // wait 10 minutes to continue
@@ -71,6 +78,10 @@ public void run() {
 
             // Clear queue before terminating thread
             processResourceQueue();
+        } catch (InterruptedException e) {
+            if (!Controller.isStopping()) {
+                LOGGER.error("Arbitrary Data Cache Manager interrupted unexpectedly", e);
+            }
         } catch (Exception e) {
             LOGGER.error(e.getMessage(), e);
         }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -449,6 +449,10 @@ private void fetchAllMetadata() throws InterruptedException {
 				);
 			} catch (DataException e) {
 				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+			} catch (InterruptedException e) {
+				// Thread interrupted during shutdown - restore interrupt status and exit
+				Thread.currentThread().interrupt();
+				return;
 			} catch (Exception e) {
 				LOGGER.error(e.getMessage(), e);
 			}

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -398,6 +398,11 @@ public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) {
 
     private void processNetworkGetArbitraryMetadataMessage() {
 
+        // Exit gracefully if shutting down
+        if (Controller.isStopping()) {
+            return;
+        }
+
         try {
             List<PeerMessage> messagesToProcess;
             synchronized (lock) {
@@ -531,4 +536,29 @@ private void processNetworkGetArbitraryMetadataMessage() {
             LOGGER.error(e.getMessage(), e);
         }
     }
+
+    /**
+     * Shutdown the scheduler
+     *
+     * Stops the scheduled executor service to allow clean shutdown
+     */
+    public void shutdown() {
+        LOGGER.info("Shutting down ArbitraryMetadataManager scheduler...");
+
+        if (!scheduler.isShutdown()) {
+            scheduler.shutdown();
+            try {
+                if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) {
+                    scheduler.shutdownNow();
+                    LOGGER.debug("Scheduler forced shutdown");
+                }
+            } catch (InterruptedException e) {
+                scheduler.shutdownNow();
+                Thread.currentThread().interrupt();
+            }
+            LOGGER.debug("Scheduler shutdown complete");
+        }
+
+        LOGGER.info("ArbitraryMetadataManager scheduler shutdown complete");
+    }
 }

src/main/java/org/qortal/controller/arbitrary/Follower.java

@@ -68,7 +68,8 @@ private void fetch(OptionalInt limit) {
                         final int blockHeightThreshold = repository.getBlockRepository().getBlockchainHeight() - limit.getAsInt();
 
                         transactionsInReverseOrder
-                                = latestArbitraryTransactionsByName.stream().filter(tx -> tx.getBlockHeight() > blockHeightThreshold)
+                                = latestArbitraryTransactionsByName.stream()
+                                .filter(tx -> tx.getBlockHeight() != null && tx.getBlockHeight() > blockHeightThreshold)
                                 .collect(Collectors.toList());
                     } else {
                         transactionsInReverseOrder = latestArbitraryTransactionsByName;

src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java

@@ -45,18 +45,12 @@ private HSQLDBBalanceRecorder( int priorityRequested, int frequency, int capacit
     public static Optional<HSQLDBBalanceRecorder> getInstance() {
 
         if( SINGLETON == null ) {
-
             SINGLETON
                 = new HSQLDBBalanceRecorder(
                     Settings.getInstance().getBalanceRecorderPriority(),
                     Settings.getInstance().getBalanceRecorderFrequency(),
                     Settings.getInstance().getBalanceRecorderCapacity()
             );
-
-        }
-        else if( SINGLETON == null ) {
-
-            return Optional.empty();
         }
 
         return Optional.of(SINGLETON);
@@ -72,13 +66,11 @@ public void run() {
 
     public List<BlockHeightRangeAddressAmounts> getLatestDynamics(int limit, long offset) {
 
-        List<BlockHeightRangeAddressAmounts> latest = this.balanceDynamics.stream()
+        return this.balanceDynamics.stream()
                 .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed())
                 .skip(offset)
                 .limit(limit)
                 .collect(Collectors.toList());
-
-        return latest;
     }
 
     public List<BlockHeightRange> getRanges(Integer offset, Integer limit, Boolean reverse) {

src/main/java/org/qortal/controller/repository/BlockArchiver.java

@@ -45,6 +45,13 @@ public void run() {
 				repository.discardChanges();
 				return;
 			}
+		} catch (InterruptedException e) {
+			if (Controller.isStopping()) {
+				LOGGER.info("Block Archiving shutting down");
+			} else {
+				LOGGER.error("Block Archiving interrupted during initialization. Restart ASAP. Report this error immediately to the developers.", e);
+			}
+			return;
 		} catch (Exception e) {
 			LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			return;