[release/5.x] Cherry pick: Consolidate single execution path for begin_private_recovery() (#6912) (#6948)

Co-authored-by: Eddy Ashton <edashton@microsoft.com>

Author: maxtropets

src/node/node_state.h

@@ -147,7 +147,6 @@ namespace ccf
     std::vector<ccf::kv::Version> view_history;
     ::consensus::Index last_recovered_signed_idx = 0;
     RecoveredEncryptedLedgerSecrets recovered_encrypted_ledger_secrets = {};
-    LedgerSecretsMap recovered_ledger_secrets = {};
     ::consensus::Index last_recovered_idx = 0;
     static const size_t recovery_batch_size = 100;
 
@@ -1147,7 +1146,6 @@ namespace ccf
       // Open the service
       if (consensus->can_replicate())
       {
-        setup_one_off_secret_hook();
         auto tx = network.tables->create_tx();
 
         // Clear recovery shares that were submitted to initiate the recovery
@@ -1493,18 +1491,24 @@ namespace ccf
       }
     }
 
+    // Decrypts chain of ledger secrets, and writes those to the ledger
+    // encrypted for each node. On a commit hook for this write, each node
+    // (including this one!) will begin_private_recovery().
     void initiate_private_recovery(ccf::kv::Tx& tx) override
     {
       std::lock_guard<pal::Mutex> guard(lock);
       sm.expect(NodeStartupState::partOfPublicNetwork);
 
-      recovered_ledger_secrets = share_manager.restore_recovery_shares_info(
-        tx, recovered_encrypted_ledger_secrets);
+      LedgerSecretsMap recovered_ledger_secrets =
+        share_manager.restore_recovery_shares_info(
+          tx, recovered_encrypted_ledger_secrets);
 
       // Broadcast decrypted ledger secrets to other nodes for them to
       // initiate private recovery too
       LedgerSecretsBroadcast::broadcast_some(
-        network, self, tx, recovered_ledger_secrets);
+        InternalTablesAccess::get_trusted_nodes(tx),
+        tx.wo(network.secrets),
+        std::move(recovered_ledger_secrets));
     }
 
     //
@@ -1699,7 +1703,9 @@ namespace ccf
       auto new_ledger_secret = make_ledger_secret();
       share_manager.issue_recovery_shares(tx, new_ledger_secret);
       LedgerSecretsBroadcast::broadcast_new(
-        network, tx, std::move(new_ledger_secret));
+        InternalTablesAccess::get_trusted_nodes(tx),
+        tx.wo(network.secrets),
+        std::move(new_ledger_secret));
 
       return true;
     }
@@ -1991,14 +1997,11 @@ namespace ccf
         threading::get_current_thread_id(), std::move(msg));
     }
 
-    void backup_initiate_private_recovery()
+    void begin_private_recovery()
     {
-      if (!consensus->is_backup())
-        return;
-
       sm.expect(NodeStartupState::partOfPublicNetwork);
 
-      LOG_INFO_FMT("Initiating end of recovery (backup)");
+      LOG_INFO_FMT("Beginning private recovery");
 
       setup_private_recovery_store();
 
@@ -2121,42 +2124,18 @@ namespace ccf
               // recovery protocol (backup only)
               network.ledger_secrets->restore_historical(
                 std::move(restored_ledger_secrets));
-              backup_initiate_private_recovery();
+              begin_private_recovery();
               return;
             }
           }
-        }));
-
-      network.tables->set_global_hook(
-        network.encrypted_submitted_shares.get_name(),
-        network.encrypted_submitted_shares.wrap_commit_hook(
-          [this](
-            ccf::kv::Version hook_version,
-            const EncryptedSubmittedShares::Write& w) {
-            // Initiate recovery procedure from global hook, once all recovery
-            // shares have been submitted (i.e. recovered_ledger_secrets is
-            // set)
-            if (!recovered_ledger_secrets.empty())
-            {
-              network.ledger_secrets->restore_historical(
-                std::move(recovered_ledger_secrets));
 
-              LOG_INFO_FMT("Initiating end of recovery (primary)");
-
-              setup_private_recovery_store();
-              reset_recovery_hook();
-
-              // Start reading private security domain of ledger
-              last_recovered_idx = recovery_store->current_version();
-              read_ledger_entries(
-                last_recovered_idx + 1,
-                last_recovered_idx + recovery_batch_size);
-
-              sm.advance(NodeStartupState::readingPrivateLedger);
-            }
-
-            return;
-          }));
+          LOG_INFO_FMT(
+            "Found no ledger secrets for this node ({}) in global commit hook "
+            "for {} @ {}",
+            self,
+            network.secrets.get_name(),
+            hook_version);
+        }));
 
       network.tables->set_global_hook(
         network.nodes.get_name(),

src/node/secret_broadcast.h

@@ -5,7 +5,6 @@
 #include "ccf/crypto/key_wrap.h"
 #include "ccf/crypto/rsa_key_pair.h"
 #include "ledger_secrets.h"
-#include "network_state.h"
 #include "service/internal_tables_access.h"
 
 #include <optional>
@@ -15,17 +14,16 @@ namespace ccf
   class LedgerSecretsBroadcast
   {
   public:
+    using SecretsWriteHandle = ccf::Secrets::WriteOnlyHandle;
+
     static void broadcast_some(
-      NetworkState& network,
-      NodeId self,
-      ccf::kv::Tx& tx,
+      std::map<NodeId, NodeInfo>&& nodes,
+      SecretsWriteHandle* secrets,
       const LedgerSecretsMap& some_ledger_secrets)
     {
-      auto secrets = tx.rw(network.secrets);
-
       LedgerSecretsForNodes secrets_for_nodes;
 
-      for (auto [nid, ni] : InternalTablesAccess::get_trusted_nodes(tx, self))
+      for (auto [nid, ni] : nodes)
       {
         std::vector<EncryptedLedgerSecret> ledger_secrets_for_node;
 
@@ -46,15 +44,13 @@ namespace ccf
     }
 
     static void broadcast_new(
-      NetworkState& network,
-      ccf::kv::Tx& tx,
+      std::map<NodeId, NodeInfo>&& nodes,
+      SecretsWriteHandle* secrets,
       LedgerSecretPtr&& new_ledger_secret)
     {
-      auto secrets = tx.rw(network.secrets);
-
       LedgerSecretsForNodes secrets_for_nodes;
 
-      for (auto [nid, ni] : InternalTablesAccess::get_trusted_nodes(tx))
+      for (auto [nid, ni] : nodes)
       {
         std::vector<EncryptedLedgerSecret> ledger_secrets_for_node;
 

src/service/internal_tables_access.h

@@ -282,36 +282,30 @@ namespace ccf
       node->put(id, node_info);
     }
 
-    static auto get_trusted_nodes(
-      ccf::kv::ReadOnlyTx& tx,
-      std::optional<NodeId> self_to_exclude = std::nullopt)
+    static std::map<NodeId, NodeInfo> get_trusted_nodes(ccf::kv::ReadOnlyTx& tx)
     {
-      // Returns the list of trusted nodes. If self_to_exclude is set,
-      // self_to_exclude is not included in the list of returned nodes.
       std::map<NodeId, NodeInfo> active_nodes;
 
       auto nodes = tx.ro<ccf::Nodes>(Tables::NODES);
 
-      nodes->foreach([&active_nodes, &nodes, self_to_exclude](
-                       const NodeId& nid, const NodeInfo& ni) {
-        if (
-          ni.status == ccf::NodeStatus::TRUSTED &&
-          (!self_to_exclude.has_value() || self_to_exclude.value() != nid))
-        {
-          active_nodes[nid] = ni;
-        }
-        else if (ni.status == ccf::NodeStatus::RETIRED)
-        {
-          // If a node is retired, but knowledge of their retirement has not yet
-          // been globally committed, they are still considered active.
-          auto cni = nodes->get_globally_committed(nid);
-          if (cni.has_value() && !cni->retired_committed)
+      nodes->foreach(
+        [&active_nodes, &nodes](const NodeId& nid, const NodeInfo& ni) {
+          if (ni.status == ccf::NodeStatus::TRUSTED)
           {
             active_nodes[nid] = ni;
           }
-        }
-        return true;
-      });
+          else if (ni.status == ccf::NodeStatus::RETIRED)
+          {
+            // If a node is retired, but knowledge of their retirement has not
+            // yet been globally committed, they are still considered active.
+            auto cni = nodes->get_globally_committed(nid);
+            if (cni.has_value() && !cni->retired_committed)
+            {
+              active_nodes[nid] = ni;
+            }
+          }
+          return true;
+        });
 
       return active_nodes;
     }

src/service/tables/secrets.h

@@ -31,8 +31,8 @@ namespace ccf
   using EncryptedLedgerSecrets = std::vector<EncryptedLedgerSecret>;
   using LedgerSecretsForNodes = std::map<NodeId, EncryptedLedgerSecrets>;
 
-  // This map is used to communicate encrypted ledger secrets from the primary
-  // to the backups during recovery (past secrets) and re-keying (new secret)
+  // This map is used to broadcast encrypted ledger secrets to all nodes, during
+  // recovery (past secrets) and re-keying (new secret)
   using Secrets = ServiceValue<LedgerSecretsForNodes>;
   namespace Tables
   {