fix(runtime-core): let shutdown close wait for draining leases

crates/tracedecay-dashboard-api/src/cloud.rs

@@ -185,6 +185,13 @@ pub fn flush_pending(amount: u64) -> Option<u64> {
     FLUSH_PENDING.get().and_then(|flush| flush(amount))
 }
 
+/// Set by test harnesses so no spawned CLI or daemon asks GitHub for the
+/// latest release; the advisory version check then has no answer.
+pub const NO_RELEASE_CHECK_ENV: &str = "TRACEDECAY_NO_RELEASE_CHECK";
+
 pub fn fetch_latest_version() -> Option<Result<String, ReleaseLookupError>> {
+    if std::env::var_os(NO_RELEASE_CHECK_ENV).is_some() {
+        return None;
+    }
     FETCH_LATEST_VERSION.get().map(|fetch| fetch())
 }

crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs

@@ -15,6 +15,44 @@ use super::{
 use crate::db::DatabaseAuthority;
 use crate::shard_runtime::shard::ShardRuntimeEvictionBlocker;
 
+/// The store-close reserve is 12 s, so this leaves most of it for the
+/// shutdown checkpoints themselves.
+const SHUTDOWN_HOLD_DRAIN_LIMIT: Duration = Duration::from_secs(3);
+const SHUTDOWN_HOLD_DRAIN_POLL: Duration = Duration::from_millis(10);
+
+/// What still keeps one mounted runtime from closing at shutdown.
+struct RuntimeHold {
+    blockers: Vec<ShardRuntimeEvictionBlocker>,
+    profile_pins: usize,
+    graph_leases: usize,
+}
+
+impl RuntimeHold {
+    fn of(
+        state: &super::RegistryState,
+        key: &StoreRuntimeKey,
+        owner: &StoreRuntimeOwnerAttachment,
+    ) -> Self {
+        let mut blockers = owner
+            .runtime()
+            .eviction_eligibility(Duration::ZERO)
+            .blockers;
+        blockers.retain(|blocker| !matches!(blocker, ShardRuntimeEvictionBlocker::PinnedProfile));
+        Self {
+            blockers,
+            profile_pins: state.profile_pin_tokens.get(key).map_or(0, BTreeSet::len),
+            graph_leases: state
+                .graph_publications
+                .get(key)
+                .map_or(0, |retained| retained.lease_tokens.len()),
+        }
+    }
+
+    fn is_idle(&self) -> bool {
+        self.blockers.is_empty() && self.profile_pins == 0 && self.graph_leases == 0
+    }
+}
+
 /// Proof that one exact runtime reached `Closed` after all physical `SQLite`
 /// handles joined and before its registry entry was removed.
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -119,11 +157,16 @@ impl StoreRuntimeRegistry {
     /// checkpoint. The daemon process exits with this registry reachable, so
     /// no destructor closes these attachments otherwise. In-flight opens are
     /// cancelled and joined first, so none publishes or keeps a physical
-    /// handle after the scan. Held runtimes stay mounted and are logged with
-    /// their blockers. Each reserved runtime owns its own writer and readers,
-    /// so they close concurrently and the shutdown waits for the slowest
-    /// checkpoint rather than their sum. Returns the number of runtimes
-    /// closed.
+    /// handle after the scan. Each reserved runtime owns its own writer and
+    /// readers, so they close concurrently and the shutdown waits for the
+    /// slowest checkpoint rather than their sum.
+    ///
+    /// A held runtime is scanned again until its holders let go, for at most
+    /// [`SHUTDOWN_HOLD_DRAIN_LIMIT`]: owners joined before store close can
+    /// still be unwinding the tasks that hold their leases, and under load
+    /// that tail outlasts the owner join (#4138). A runtime still held at the
+    /// limit stays mounted and is logged with its blockers. Returns the number
+    /// of runtimes closed.
     #[tracing::instrument(
         name = "runtime_core.registry.close_idle_for_shutdown",
         level = "trace",
@@ -135,34 +178,55 @@ impl StoreRuntimeRegistry {
                 operation: "close idle registered runtimes for shutdown",
             })?;
         self.cancel_and_join_opens_for_shutdown(&cx).await?;
-        let (reservations, reserve_failure) = {
+        let deadline = std::time::Instant::now() + SHUTDOWN_HOLD_DRAIN_LIMIT;
+        let mut closed = 0;
+        loop {
+            let final_pass = std::time::Instant::now() >= deadline;
+            let (batch, held) = self.close_idle_batch(final_pass).await?;
+            closed += batch;
+            if !held || final_pass {
+                return Ok(closed);
+            }
+            if crate::native_runtime::sleep_until(
+                std::time::Instant::now() + SHUTDOWN_HOLD_DRAIN_POLL,
+            )
+            .await
+            .is_err()
+            {
+                return self
+                    .close_idle_batch(true)
+                    .await
+                    .map(|(batch, _)| closed + batch);
+            }
+        }
+    }
+
+    /// Closes every runtime idle right now and reports whether any stays
+    /// held. Only the final pass logs the held ones.
+    async fn close_idle_batch(
+        &self,
+        final_pass: bool,
+    ) -> Result<(usize, bool), StoreRuntimeRegistryFailure> {
+        let (reservations, held, reserve_failure) = {
             let mut state = self.lock_state();
             let mut idle = Vec::new();
+            let mut held = false;
             for (key, entry) in &state.entries {
                 let RegistryEntry::Ready(ready) = entry else {
                     continue;
                 };
-                let mut blockers = ready
-                    .owner
-                    .runtime()
-                    .eviction_eligibility(Duration::ZERO)
-                    .blockers;
-                blockers.retain(|blocker| {
-                    !matches!(blocker, ShardRuntimeEvictionBlocker::PinnedProfile)
-                });
-                let profile_pins = state.profile_pin_tokens.get(key).map_or(0, BTreeSet::len);
-                let graph_leases = state
-                    .graph_publications
-                    .get(key)
-                    .map_or(0, |retained| retained.lease_tokens.len());
-                if blockers.is_empty() && profile_pins == 0 && graph_leases == 0 {
+                let hold = RuntimeHold::of(&state, key, &ready.owner);
+                if hold.is_idle() {
                     idle.push(key.clone());
-                } else {
+                    continue;
+                }
+                held = true;
+                if final_pass {
                     tracing::warn!(
                         path = %ready.owner.locator().path().display(),
-                        ?blockers,
-                        profile_pins,
-                        graph_leases,
+                        blockers = ?hold.blockers,
+                        profile_pins = hold.profile_pins,
+                        graph_leases = hold.graph_leases,
                         "store runtime still held at shutdown; its WAL is not truncated"
                     );
                 }
@@ -178,7 +242,7 @@ impl StoreRuntimeRegistry {
                     }
                 }
             }
-            (reservations, reserve_failure)
+            (reservations, held, reserve_failure)
         };
         let closed = reservations.len();
         let mut first_failure = reserve_failure;
@@ -209,7 +273,7 @@ impl StoreRuntimeRegistry {
                 first_failure.get_or_insert(failure);
             }
         }
-        first_failure.map_or(Ok(closed), Err)
+        first_failure.map_or(Ok((closed, held)), Err)
     }
 
     #[tracing::instrument(name = "runtime_core.registry.close_path", level = "trace", skip_all)]
@@ -896,6 +960,28 @@ mod tests {
         drop(profile);
     }
 
+    /// A lease its owner releases moments after store close begins must not
+    /// leave the store's WAL behind (#4138).
+    #[asupersync::test]
+    async fn shutdown_close_waits_for_a_lease_released_while_it_runs() {
+        let temporary = tempfile::tempdir().unwrap();
+        let path = temporary.path().join("runtime.db");
+        let (registry, profile, code, _authority) = mount_code_runtime(path.clone()).await;
+        let binding = code.binding().clone();
+        let release = std::thread::spawn(move || {
+            std::thread::sleep(Duration::from_millis(50));
+            drop(code);
+        });
+
+        assert_eq!(registry.close_idle_for_shutdown().await.unwrap(), 1);
+        assert!(matches!(
+            registry.lookup(&binding),
+            StoreRuntimeLookup::Missing { .. }
+        ));
+        release.join().unwrap();
+        drop(profile);
+    }
+
     #[asupersync::test]
     async fn refused_shutdown_close_still_settles_its_reserved_runtime() {
         let temporary = tempfile::tempdir().unwrap();

docs/plans/storage/README.md

@@ -638,6 +638,10 @@ changes a file that holds memory.
 11. Graph engine gate. Measure SQLite code packs against sealed Grafeo on this
     repository (activation RSS, traversal p50 and p95). Keep whichever wins; delete
     the other.
+12. Growth acceptance. Run a daemon on a fresh profile for 24 hours under a
+    recorded ingest and code-index load, and read `telemetry.storage_usage`. Proof:
+    every class stays within its budget, and the WAL of every file is at or under
+    its soft limit plus one batch at each sample.
 
 ## What this supersedes
 
@@ -674,13 +678,61 @@ changes a file that holds memory.
   in step 8. The additive-migration rule for memory ships with the first change to a
   memory table (the conversion in step 8), not as machinery with no migration to
   run. The class contract is unchanged.
-- Step 3 (2026-10-16, branch `refactor/storage-worktree-liveness`). The first
-  growth source is closed: orphan-store liveness no longer counts the shared git
-  common directory, so a deleted worktree's store becomes collectable while its
-  repository lives. Still open in step 3: a daemon-run collector for unregistered
-  `proj_*` directories, collecting a store's non-memory files when memory rows pin
-  it, per-tick response-handle sweeps, `hook_analytics.jsonl` rotation, and
-  age-pruned idempotency rows.
+- Step 3 (2026-10-16, done). Four PRs stopped the growth this step names:
+  - #4120. Orphan-store liveness no longer counts the shared git common
+    directory, so a deleted worktree's store becomes collectable while its
+    repository lives.
+  - #4126. The maintenance tick runs one bounded page (8 directories, 5 s) of the
+    unregistered-directory sweep, which only Doctor previewed before. The sweep
+    still refuses a directory that holds memory rows or a payload it cannot
+    inspect. Re-registering such a directory from a self-describing `project.json`
+    needs the step 8 layout, so until then it stays whole.
+  - #4128 (the #4099 lane). Every tick sweeps expired response handles, and a
+    fully imported `hook_analytics.jsonl` over 64 MiB is truncated under its
+    append lock.
+  - #4129. Each commit retires the idempotency rows of an incarnation (a daemon
+    generation) whose checkpoint on that shard has not moved for 7 days. The
+    committing incarnation is never aged out. This departs from the literal "7 d
+    of age". Pruning a live writer's rows by age needs an index on
+    `committed_at_micros`. That index changes the exact final schema shape, and
+    stores that hold memory have no migration path before step 8. Retiring
+    exited incarnations ends the stranding the design names, and the age rule
+    arrives with `journal.db`.
+  Collecting the non-memory files of a store whose memory rows pin it moves to
+  step 8, where memory gets its own file.
+- Step 5 (2026-10-16, in progress). Four decisions:
+  - The journal shape is "service inside the job". `ExactSqlHandle::transact`
+    runs one whole write transaction on the writer thread as a job (#4133). A
+    Work or Workflow operation runs its existing synchronous contracts decision
+    inside that job, over a storage scoped to the job's transaction, and that
+    storage holds no `ExactSqlHandle`. Effects outside storage (Git, providers,
+    other stores) stay outside the job.
+  - Shipped so far: handoff grants (#4133), pause and resume of a run (#4139),
+    and the git-index store, whose actor thread is deleted (#4140).
+  - `transact` has no operation-id ledger yet. The ledger is new schema on stores
+    with no migration path, so it ships with `journal.db` in step 8. Until then
+    each domain keeps its own idempotency.
+  - Reads use `query_async` and preview/apply use `on_blocking_pool`. Both are
+    blocking-pool hops that step 7 replaces.
+- #4138 (2026-10-16). Shutdown store close waits up to 3 s for leases to drain
+  before deciding a store's WAL cannot be truncated. Under load, owners joined
+  before store close could still be unwinding the tasks that held their leases,
+  and `graceful_stop_truncates_profile_and_project_store_wals` failed. This is an
+  interim remedy; the step 7 executor owns close.
+- Growth sources from #4099 still open, each with the step that closes it:
+  1. LCM payloads that no summary covers yet: step 10's evidence lever.
+  2. Session databases of unmounted projects get no retention: step 10, whose
+     retention covers every project in the catalog.
+  3. Observation projection history (active and unavailable skeleton rows):
+     step 8 makes it derived and step 10 trims it behind the evidence window.
+  4. The projection disposition ledger: step 8. Dispositions become derived rows
+     that cascade with evidence.
+  5. Generation staging release: step 9 (`scratch/`).
+  6. The 512 MiB response-handle cap on top of #4128's TTL sweep: step 10.
+  7. Hook events into `telemetry.hook_events`: step 8.
+  8. Closing a store's lane when its file is lost, instead of a daemon restart:
+     step 7's unlink probe. #4130 is the interim restart remedy.
+  9. A 24-hour acceptance run on a fresh profile showing bounded growth: step 12.
 - Step 4 merged into step 8 (2026-10-16). A dead-letter valve on today's global
   queue was built and then dropped: the projection code deliberately keeps corrupt
   projection authority (ghost or divergent provenance, divergent workflow state) a
@@ -708,5 +760,7 @@ changes a file that holds memory.
 
 ## Next implementation step
 
-The rest of step 3: a daemon-run collector for unregistered project directories
-that never deletes durable data, on top of the #4099 measurement lane.
+The rest of step 5, slice by slice: placement and the leaf Work ports, the
+native-integration and remote-replay actor threads, then the attempt chain together
+with run control and Workflow. After those, delete `off_native_thread` for Work and
+the native wait counters.

docs/plans/storage/decisions.tsv

@@ -32,3 +32,14 @@ ts	phase	decision	why	evidence	result
 2026-10-16T13:00:00Z	implement	step 3a: orphan-store liveness stops counting the shared git common dir	first #4099 growth source: deleted worktrees' stores stayed Live forever	PR 4120; maintenance unit_test sweep_plans_a_deleted_worktrees_store_while_its_repository_lives (mutation fails)	107 passed
 2026-10-16T13:30:00Z	implement	built a dead-letter valve on today's global projection queue, then dropped it	three observation_collision_tests deliberately keep corrupt provenance a hard error; skipping launders corrupt derived state; the design's answer is a derived rebuild, which needs the step-8 partition	global-db unit_test: 3 collision tests failed against the valve; parked patch in scratchpad	step 4 merged into step 8
 2026-10-16T13:35:00Z	implement	architect Phase E check after the dropped valve: one deviation, the sketch stands	Phase E needs a pattern of friction; this was one place where today's layout lacks the derived file the design relies on	README Implementation reconciliation	no re-sketch
+2026-10-16T15:00:00Z	implement	step 3b: the maintenance cold-store page runs the unregistered-directory sweep with apply on, one bounded page per tick	only Doctor previewed it; the sweep already refuses durable memory and unverifiable payloads; re-registration from project.json needs the step 8 layout	PR 4126; the_maintenance_cold_store_page_collects_an_unregistered_project_dir (2 mutations killed)	maintenance unit_test 109 passed
+2026-10-16T15:05:00Z	implement	folded the separate 4099-orphans attempt as nothing	its test duplicates 4120 plus the existing relink and alias tests	/fast/tmp/4099-orphans diff	worktree safe to remove
+2026-10-16T15:10:00Z	implement	left the response-handle sweep and hook_analytics rotation to the 4099 lane	shipped there as PR 4128	coordinator message	not duplicated
+2026-10-16T15:30:00Z	implement	step 3c: retire idempotency rows of incarnations whose shard checkpoint is quiet for 7 d; never age out the live incarnation	an age index changes the exact final shape of stores holding memory, which have no migration path; incarnation is the daemon generation, so restarts strand rows	PR 4129; 3 mutations killed	deviation from the literal 7 d age prune, recorded in README
+2026-10-16T15:40:00Z	implement	kept step 4 folded into step 8	no new evidence against the 13:30 call; 4107 fixed the freeze; contract and sanitization errors already reject; transient errors already back off	observation_projection/rebuild.rs project_observation_once	kept
+2026-10-16T16:10:00Z	implement	step 5 ships as slices; 5a is the transact primitive plus handoff grants	census: 25 sync traits and 78 methods; handoff is 4 methods, the only Work-adjacent path parking native threads, and already idempotent	PR 4133	3 mutations killed
+2026-10-16T16:10:00Z	implement	transact carries no operation-id ledger in step 5	the ledger is new schema on stores with no migration path; per-domain idempotency already exists; the ledger ships with journal.db	PR 4133 body	deviation recorded in README
+2026-10-16T16:40:00Z	implement	step 5 shape: run the existing sync contracts decision inside one writer job over a job-scoped storage (hybrid B), not async ports	one atomic load-decide-update per operation and no parked thread; async ports keep multi-trip transactions; effects outside storage are hoisted out of the job	design probe of dispatch_work_application arms; PR 4139	pause and resume shipped
+2026-10-16T17:30:00Z	implement	5c: GitIndexTransactionStore async, actor thread deleted, preview GC on a NativeTask; kept on_blocking_pool for preview and apply	those futures still run sync Git and a block_on configuration read and must not poll on a foreign executor thread	PR 4140; 6 mutations killed	first of three actor threads gone
+2026-10-16T18:30:00Z	implement	graceful_stop WAL failure diagnosed as a master shutdown race, not storage PRs	clean master fails 8/8 under concurrent load; the same merge commit passes and fails by time; the daemon log shows client leases still held at store close	#4138; load repro script	fixed by a bounded lease drain in registry close
+2026-10-16T18:45:00Z	implement	#4138 fix lives in the registry close path, not the barred lifecycle file: close idle runtimes at once and rescan held ones for up to 3 s	the store-close reserve is 12 s; idle runtimes must keep closing concurrently (attachment test)	load repro: fix 24/24 pass while master fails; disabling the rescan fails 16/16	interim until the step 7 executor owns close

tests/support/isolated_profile.rs

@@ -174,6 +174,9 @@ pub fn apply_hermetic_child_env(command: &mut Command, home: &Path) {
         // Host discovery may invoke gh; its telemetry must not write a device
         // identity into the fixture home or send fixture activity upstream.
         .env("GH_TELEMETRY", "0")
+        // The advisory version check would otherwise call the GitHub API from
+        // every spawned daemon (tracedecay_dashboard_api::cloud).
+        .env("TRACEDECAY_NO_RELEASE_CHECK", "1")
         .env("PATH", hermetic_path::<&Path>(&[]));
 }