Add periodic snapshot file cleanup (#7747)

Copilot · achamayou · Copilot · web-flow · commit b79409eb8c34 · 2026-03-27T15:05:26.000Z
Co-authored-by: copilot-swe-agent[bot] &lt;198982749+Copilot@users.noreply.github.com&gt;
Co-authored-by: achamayou &lt;4016369+achamayou@users.noreply.github.com&gt;
Co-authored-by: Amaury Chamayou &lt;amchamay@microsoft.com&gt;
Co-authored-by: Amaury Chamayou &lt;amaury@xargs.fr&gt;
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Added `files_cleanup.max_snapshots` configuration option to limit the number of committed snapshot files retained on disk. When the number of committed snapshots exceeds this value, the oldest snapshots (by sequence number) are automatically deleted. The value must be at least 1 if set.
+- Added `files_cleanup.interval` configuration option (default `"30s"`) to periodically scan the snapshot directory and delete old committed snapshots exceeding `max_snapshots`. This ensures backup nodes (which receive snapshots via `backup_fetch`) also prune old snapshots. Only effective when `max_snapshots` is set.
 - Added `POST /node/snapshot:create`, gated by the `SnapshotCreate` RPC interface operator feature, to create a snapshot via an operator endpoint rather than a governance action.
 - Added `make_cose_verifier_from_pem_cert()` and `make_cose_verifier_from_der_cert()` that accept certificates in a known format. The existing `make_cose_verifier_cert()` is renamed to `make_cose_verifier_any_cert()` (#7768).
 
diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json
@@ -546,6 +546,24 @@
       "description": "This section includes configuration for the snapshot directories and files",
       "additionalProperties": false
     },
+    "files_cleanup": {
+      "type": "object",
+      "properties": {
+        "max_snapshots": {
+          "type": ["integer", "null"],
+          "default": null,
+          "description": "Maximum number of committed snapshot files to retain. When the number of committed snapshots exceeds this value, the oldest snapshots are deleted. Must be at least 1 if set. If null or unset, no automated snapshot garbage collection is performed.",
+          "minimum": 1
+        },
+        "interval": {
+          "type": "string",
+          "default": "30s",
+          "description": "Time interval at which to scan the snapshot directory and delete old committed snapshots in excess of max_snapshots. This periodic cleanup executes regardless of the node's status (primary or backup)."
+        }
+      },
+      "description": "This section includes configuration for periodic cleanup of old files (snapshots, ledger chunks)",
+      "additionalProperties": false
+    },
     "logging": {
       "type": "object",
       "properties": {
diff --git a/doc/operations/ledger_snapshot.rst b/doc/operations/ledger_snapshot.rst
@@ -178,6 +178,8 @@ Committed snapshot files are named ``snapshot_<seqno>_<evidence_seqno>.committed
 
 Uncommitted snapshot files, i.e. those whose evidence has not yet been committed, are named ``snapshot_<seqno>_<evidence_seqno>``. These files will be ignored by CCF when joining or recovering a service as no evidence can attest of their validity.
 
+.. note:: The ``files_cleanup.max_snapshots`` configuration entry can be used to limit the number of committed snapshot files retained on disk. When the number of committed snapshots exceeds this value, the oldest snapshots (by sequence number) are automatically deleted. This is useful to control the local persistent storage footprint of a node. The value must be at least 1 if set.
+
 Join or Recover From Snapshot
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/include/ccf/node/startup_config.h b/include/ccf/node/startup_config.h
@@ -116,6 +116,15 @@ namespace ccf
       bool operator==(const Snapshots&) const = default;
     };
     Snapshots snapshots = {};
+
+    struct FilesCleanup
+    {
+      std::optional<size_t> max_snapshots = std::nullopt;
+      ccf::ds::TimeString interval = {"30s"};
+
+      bool operator==(const FilesCleanup&) const = default;
+    };
+    FilesCleanup files_cleanup = {};
   };
 
   struct RecoveryDecisionProtocolConfig
diff --git a/src/common/configuration.h b/src/common/configuration.h
@@ -114,6 +114,11 @@ namespace ccf
     read_only_directory,
     backup_fetch);
 
+  DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CCFConfig::FilesCleanup);
+  DECLARE_JSON_REQUIRED_FIELDS(CCFConfig::FilesCleanup);
+  DECLARE_JSON_OPTIONAL_FIELDS(
+    CCFConfig::FilesCleanup, max_snapshots, interval);
+
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CCFConfig);
   DECLARE_JSON_REQUIRED_FIELDS(CCFConfig, network);
   DECLARE_JSON_OPTIONAL_FIELDS(
@@ -126,6 +131,7 @@ namespace ccf
     jwt,
     attestation,
     snapshots,
+    files_cleanup,
     node_to_node_message_limit,
     historical_cache_soft_limit);
 
diff --git a/src/host/run.cpp b/src/host/run.cpp
@@ -30,6 +30,7 @@
 #include "enclave/entry_points.h"
 #include "handle_ring_buffer.h"
 #include "host/env.h"
+#include "host/snapshot_cleanup_timer.h"
 #include "http/curl.h"
 #include "json_schema.h"
 #include "lfs_file_handler.h"
@@ -620,6 +621,18 @@ namespace ccf
       config.snapshots.read_only_directory);
     snapshots.register_message_handlers(buffer_processor.get_dispatcher());
 
+    std::optional<asynchost::SnapshotCleanupTimer> snapshot_cleanup;
+    if (
+      config.files_cleanup.max_snapshots.has_value() &&
+      std::chrono::milliseconds(config.files_cleanup.interval) >
+        std::chrono::milliseconds::zero())
+    {
+      snapshot_cleanup.emplace(
+        std::chrono::milliseconds(config.files_cleanup.interval),
+        config.snapshots.directory,
+        config.files_cleanup.max_snapshots.value());
+    }
+
     // handle LFS-related messages from the enclave
     asynchost::LFSFileHandler lfs_file_handler(
       writer_factory.create_writer_to_inside());
diff --git a/src/host/snapshot_cleanup_timer.h b/src/host/snapshot_cleanup_timer.h
@@ -0,0 +1,124 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "snapshots/filenames.h"
+#include "timer.h"
+
+#include <filesystem>
+#include <string>
+
+namespace asynchost
+{
+  class SnapshotCleanupImpl
+  {
+  private:
+    std::filesystem::path dir;
+    size_t max_retained;
+
+    struct CleanupWork
+    {
+      std::filesystem::path dir;
+      size_t max_retained;
+    };
+
+    static void cleanup_old_snapshots(
+      const std::filesystem::path& dir, size_t max_retained)
+    {
+      std::vector<std::filesystem::path> directories{dir};
+      decltype(snapshots::find_committed_snapshots_in_directories(
+        directories)) committed;
+      try
+      {
+        committed =
+          snapshots::find_committed_snapshots_in_directories(directories);
+      }
+      catch (const std::filesystem::filesystem_error& e)
+      {
+        LOG_FAIL_FMT(
+          "Failed to list committed snapshots in {}: {}", dir, e.what());
+        return;
+      }
+      catch (const std::exception& e)
+      {
+        LOG_FAIL_FMT(
+          "Unexpected error while listing committed snapshots in {}: {}",
+          dir,
+          e.what());
+        return;
+      }
+
+      if (committed.size() > max_retained)
+      {
+        // committed is sorted descending by snapshot index, so the
+        // oldest are at the end
+        for (auto it = committed.rbegin();
+             it != committed.rend() - max_retained;
+             ++it)
+        {
+          const auto& path = it->second;
+          LOG_INFO_FMT(
+            "Deleting old snapshot {} (retaining {})",
+            path.filename(),
+            max_retained);
+          std::error_code ec;
+          std::filesystem::remove(path, ec);
+          if (ec)
+          {
+            LOG_FAIL_FMT(
+              "Failed to delete old snapshot {}: {}",
+              path.filename(),
+              ec.message());
+          }
+        }
+      }
+    }
+
+    static void on_cleanup_work(uv_work_t* req)
+    {
+      auto* work = static_cast<CleanupWork*>(req->data);
+      cleanup_old_snapshots(work->dir, work->max_retained);
+    }
+
+    static void on_cleanup_work_done(uv_work_t* req, int /*status*/)
+    {
+      auto* work = static_cast<CleanupWork*>(req->data);
+      LOG_DEBUG_FMT("Snapshot cleanup completed");
+      delete work; // NOLINT(cppcoreguidelines-owning-memory)
+      delete req; // NOLINT(cppcoreguidelines-owning-memory)
+    }
+
+  public:
+    SnapshotCleanupImpl(const std::string& dir_, size_t max_retained_) :
+      dir(dir_),
+      max_retained(max_retained_)
+    {
+      if (max_retained < 1)
+      {
+        throw std::logic_error(fmt::format(
+          "files_cleanup.max_snapshots must be at least 1, got {}",
+          max_retained));
+      }
+    }
+
+    void on_timer()
+    {
+      // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+      auto* work = new CleanupWork{.dir = dir, .max_retained = max_retained};
+      // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+      auto* req = new uv_work_t;
+      req->data = work;
+      int rc = uv_queue_work(
+        uv_default_loop(), req, &on_cleanup_work, &on_cleanup_work_done);
+      if (rc < 0)
+      {
+        LOG_FAIL_FMT(
+          "Failed to queue snapshot cleanup work: {}", uv_strerror(rc));
+        delete work; // NOLINT(cppcoreguidelines-owning-memory)
+        delete req; // NOLINT(cppcoreguidelines-owning-memory)
+      }
+    }
+  };
+
+  using SnapshotCleanupTimer = proxy_ptr<Timer<SnapshotCleanupImpl>>;
+}
diff --git a/src/snapshots/snapshot_manager.h b/src/snapshots/snapshot_manager.h
@@ -24,7 +24,6 @@ namespace snapshots
 
     const fs::path snapshot_dir;
     const std::optional<fs::path> read_snapshot_dir = std::nullopt;
-
     struct PendingSnapshot
     {
       ::consensus::Index evidence_idx;
@@ -62,6 +61,9 @@ namespace snapshots
       }
     }
 
+    SnapshotManager(const SnapshotManager&) = delete;
+    SnapshotManager& operator=(const SnapshotManager&) = delete;
+
     [[nodiscard]] fs::path get_main_directory() const
     {
       return snapshot_dir;
diff --git a/tests/config.jinja b/tests/config.jinja
@@ -73,7 +73,12 @@
       "target_rpc_interface": "{{ backup_snapshot_fetch_target_rpc_interface }}"{% endif %}{% if backup_snapshot_fetch_max_size %},
       "max_size": "{{ backup_snapshot_fetch_max_size }}"{% endif %}
     }{% endif %}
-  },
+  },{% if files_cleanup_max_snapshots or files_cleanup_interval %}
+  "files_cleanup":
+  {
+    {% if files_cleanup_max_snapshots %}"max_snapshots": {{ files_cleanup_max_snapshots }}{% endif %}{% if files_cleanup_max_snapshots and files_cleanup_interval %},
+    {% endif %}{% if files_cleanup_interval %}"interval": "{{ files_cleanup_interval }}"{% endif %}
+  },{% endif %}
   "logging":
   {
     "host_level": "{{ host_log_level }}",
diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py
diff --git a/tests/infra/network.py b/tests/infra/network.py
diff --git a/tests/infra/node.py b/tests/infra/node.py

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,6 @@ namespace snapshots`
`24`	`24`
`25`	`25`	`const fs::path snapshot_dir;`
`26`	`26`	`const std::optional<fs::path> read_snapshot_dir = std::nullopt;`
`27`		`-`
`28`	`27`	`struct PendingSnapshot`
`29`	`28`	`{`
`30`	`29`	`::consensus::Index evidence_idx;`
`@@ -62,6 +61,9 @@ namespace snapshots`
`62`	`61`	`}`
`63`	`62`	`}`
`64`	`63`
	`64`	`+ SnapshotManager(const SnapshotManager&) = delete;`
	`65`	`+ SnapshotManager& operator=(const SnapshotManager&) = delete;`
	`66`	`+`
`65`	`67`	`[[nodiscard]] fs::path get_main_directory() const`
`66`	`68`	`{`
`67`	`69`	`return snapshot_dir;`