Migrate to getwalletstatus RPC for syncing wallets

qa/rpc-tests/test_framework/test_framework.py

@@ -12,7 +12,6 @@
 import sys
 import shutil
 import tempfile
-import time
 import traceback
 
 from .config import ZebraArgs
@@ -131,21 +130,15 @@ def split_network(self):
 
     def sync_all(self, do_mempool_sync = True):
         if self.is_network_split:
-            sync_blocks(self.nodes[:2])
-            sync_blocks(self.nodes[2:])
+            sync_blocks(self.nodes[:2], None if self.wallets is None else self.wallets[:2])
+            sync_blocks(self.nodes[2:], None if self.wallets is None else self.wallets[2:])
             if do_mempool_sync:
-                sync_mempools(self.nodes[:2])
-                sync_mempools(self.nodes[2:])
+                sync_mempools(self.nodes[:2], None if self.wallets is None else self.wallets[:2])
+                sync_mempools(self.nodes[2:], None if self.wallets is None else self.wallets[2:])
         else:
-            sync_blocks(self.nodes)
+            sync_blocks(self.nodes, self.wallets)
             if do_mempool_sync:
-                sync_mempools(self.nodes)
-
-        # TODO: Sync wallets inside `sync_blocks`
-        # TODO: Use `getwalletstatus` in all sync issues
-        # https://github.com/zcash/wallet/issues/316
-        if self.num_wallets > 0:
-            time.sleep(2)
+                sync_mempools(self.nodes, self.wallets)
 
     def join_network(self):
         """

qa/rpc-tests/test_framework/util.py

@@ -161,7 +161,7 @@ def hex_str_to_bytes(hex_str):
 def str_to_b64str(string):
     return b64encode(string.encode('utf-8')).decode('ascii')
 
-def sync_blocks(rpc_connections, wait=0.125, timeout=60, allow_different_tips=False):
+def sync_blocks(nodes, wallets=None, wait=0.125, timeout=60, allow_different_tips=False):
     """
     Wait until everybody has the same tip, and has notified
     all internal listeners of them.
@@ -171,31 +171,72 @@ def sync_blocks(rpc_connections, wait=0.125, timeout=60, allow_different_tips=Fa
     """
     while timeout > 0:
         if allow_different_tips:
-            tips = [ x.getblockcount() for x in rpc_connections ]
+            tips = [ x.getblockcount() for x in nodes ]
         else:
-            tips = [ x.getbestblockhash() for x in rpc_connections ]
+            tips = [ x.getbestblockhash() for x in nodes ]
         if tips == [ tips[0] ]*len(tips):
+            if not wallets:
+                return True
             break
         time.sleep(wait)
         timeout -= wait
-    return True
 
-def sync_mempools(rpc_connections, wait=0.5, timeout=60):
+    if wallets:
+        # Now that the block counts are in sync, wait for the internal
+        # notifications to finish. `getwalletstatus` omits `wallet_tip` until
+        # the wallet has a committed tip, so treat its absence as "not synced
+        # yet" and keep polling instead of raising KeyError.
+        while timeout > 0:
+            wallet_status = [ x.getwalletstatus() for x in wallets ]
+            if all('wallet_tip' in w for w in wallet_status):
+                key = 'height' if allow_different_tips else 'blockhash'
+                wallet_node_tips = [ w['node_tip'][key] for w in wallet_status ]
+                wallet_tips = [ w['wallet_tip'][key] for w in wallet_status ]
+                if tips == wallet_node_tips and tips == wallet_tips:
+                    return True
+            time.sleep(wait)
+            timeout -= wait
+
+    print('Node tips:', tips)
+    print('Wallet statuses:', wallet_status)
+    raise AssertionError("Block sync failed")
+
+def sync_mempools(nodes, wallets=None, wait=0.5, timeout=60):
     """
     Wait until everybody has the same transactions in their memory
     pools, and has notified all internal listeners of them
+
+    Returns `True` when all wallets are in synced, or if no wallet is given.
     """
     while timeout > 0:
-        pool = set(rpc_connections[0].getrawmempool())
+        pool = set(nodes[0].getrawmempool())
         num_match = 1
-        for i in range(1, len(rpc_connections)):
-            if set(rpc_connections[i].getrawmempool()) == pool:
+        for i in range(1, len(nodes)):
+            if set(nodes[i].getrawmempool()) == pool:
                 num_match = num_match+1
-        if num_match == len(rpc_connections):
+        if num_match == len(nodes):
+            if not wallets:
+                return True
             break
         time.sleep(wait)
         timeout -= wait
-    return True
+
+    if wallets:
+        # Now that the mempools are in sync, wait for the internal
+        # notifications to finish. `getwalletstatus` omits `wallet_tip` until
+        # the wallet has a committed tip, so treat its absence as "not synced
+        # yet" and keep polling instead of raising KeyError.
+        while timeout > 0:
+            wallet_status = [ x.getwalletstatus() for x in wallets ]
+            if all('wallet_tip' in w for w in wallet_status):
+                tips = [ (w['node_tip']['blockhash'], w['wallet_tip']['blockhash']) for w in wallet_status ]
+                if tips == [ tips[0] ]*len(tips) and tips[0][0] == tips[0][1]:
+                    return True
+            time.sleep(wait)
+            timeout -= wait
+
+    print('Wallet view of tips:', wallet_status)
+    raise AssertionError("Mempool sync failed")
 
 bitcoind_processes = {}
 
@@ -438,10 +479,15 @@ def rebuild_cache():
                 sys.stderr.write("Error connecting to "+rpc_url_wallet(i)+"\n")
                 sys.exit(1)
 
-        # Wait for zallets to synchronize with the nodes
-        # TODO: Use `getwalletstatus` in all sync issues
-        # https://github.com/zcash/wallet/issues/316
-        time.sleep(10)
+        # Wait for zallets to synchronize with the nodes. `getwalletstatus`
+        # omits `wallet_tip` until the wallet has a committed tip.
+        while True:
+            wallet_status = [ x.getwalletstatus() for x in wallets ]
+            if all('wallet_tip' in w for w in wallet_status):
+                tips = [ (w['node_tip']['blockhash'], w['wallet_tip']['blockhash']) for w in wallet_status ]
+                if tips == [ tips[0] ]*len(tips) and tips[0][0] == tips[0][1]:
+                    break
+            time.sleep(0.25)
 
         # Shut them down, and clean up cache directories:
         stop_wallets(wallets)

qa/rpc-tests/wallet.py

@@ -5,7 +5,6 @@
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
 #from decimal import Decimal
-import time
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal, assert_true
@@ -26,6 +25,9 @@ def run_test(self):
         node_balance = self.nodes[0].getaddressbalance(transparent_address)
         assert_equal(node_balance['balance'], 625000000)
 
+        # Wait for the wallet to scan and commit the coinbase block.
+        self.sync_all()
+
         # Zallet can see the balance.
         wallet_balance = self.wallets[0].z_gettotalbalance(1, True)
         # TODO: Result is a string (https://github.com/zcash/wallet/issues/15)
@@ -34,8 +36,8 @@ def run_test(self):
         # Mine another block
         self.nodes[0].generate(1)
 
-        # Wait for the wallet to sync
-        time.sleep(1)
+        # Wait for the wallet to scan and commit the new block.
+        self.sync_all()
 
         node_balance = self.nodes[0].getaddressbalance(transparent_address)
         assert_equal(node_balance['balance'], 1250000000)