From 17580ca845cf4ececb5102d878dc1620ddc4f92c Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 15:01:08 +0000 Subject: [PATCH 1/2] fix(runtime-core): let shutdown close wait for draining leases --- crates/tracedecay-dashboard-api/src/cloud.rs | 7 + .../src/shard_runtime/registry/close.rs | 138 ++++++++++++++---- tests/support/isolated_profile.rs | 3 + 3 files changed, 122 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-dashboard-api/src/cloud.rs b/crates/tracedecay-dashboard-api/src/cloud.rs index 8e2e031582..598e6e4ab3 100644 --- a/crates/tracedecay-dashboard-api/src/cloud.rs +++ b/crates/tracedecay-dashboard-api/src/cloud.rs @@ -185,6 +185,13 @@ pub fn flush_pending(amount: u64) -> Option { FLUSH_PENDING.get().and_then(|flush| flush(amount)) } +/// Set by test harnesses so no spawned CLI or daemon asks GitHub for the +/// latest release; the advisory version check then has no answer. +pub const NO_RELEASE_CHECK_ENV: &str = "TRACEDECAY_NO_RELEASE_CHECK"; + pub fn fetch_latest_version() -> Option> { + if std::env::var_os(NO_RELEASE_CHECK_ENV).is_some() { + return None; + } FETCH_LATEST_VERSION.get().map(|fetch| fetch()) } diff --git a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs index 6bae0e805b..00f36d4230 100644 --- a/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs +++ b/crates/tracedecay-runtime-core/src/shard_runtime/registry/close.rs @@ -15,6 +15,44 @@ use super::{ use crate::db::DatabaseAuthority; use crate::shard_runtime::shard::ShardRuntimeEvictionBlocker; +/// The store-close reserve is 12 s, so this leaves most of it for the +/// shutdown checkpoints themselves. +const SHUTDOWN_HOLD_DRAIN_LIMIT: Duration = Duration::from_secs(3); +const SHUTDOWN_HOLD_DRAIN_POLL: Duration = Duration::from_millis(10); + +/// What still keeps one mounted runtime from closing at shutdown. +struct RuntimeHold { + blockers: Vec, + profile_pins: usize, + graph_leases: usize, +} + +impl RuntimeHold { + fn of( + state: &super::RegistryState, + key: &StoreRuntimeKey, + owner: &StoreRuntimeOwnerAttachment, + ) -> Self { + let mut blockers = owner + .runtime() + .eviction_eligibility(Duration::ZERO) + .blockers; + blockers.retain(|blocker| !matches!(blocker, ShardRuntimeEvictionBlocker::PinnedProfile)); + Self { + blockers, + profile_pins: state.profile_pin_tokens.get(key).map_or(0, BTreeSet::len), + graph_leases: state + .graph_publications + .get(key) + .map_or(0, |retained| retained.lease_tokens.len()), + } + } + + fn is_idle(&self) -> bool { + self.blockers.is_empty() && self.profile_pins == 0 && self.graph_leases == 0 + } +} + /// Proof that one exact runtime reached `Closed` after all physical `SQLite` /// handles joined and before its registry entry was removed. #[derive(Clone, Debug, PartialEq, Eq)] @@ -119,11 +157,16 @@ impl StoreRuntimeRegistry { /// checkpoint. The daemon process exits with this registry reachable, so /// no destructor closes these attachments otherwise. In-flight opens are /// cancelled and joined first, so none publishes or keeps a physical - /// handle after the scan. Held runtimes stay mounted and are logged with - /// their blockers. Each reserved runtime owns its own writer and readers, - /// so they close concurrently and the shutdown waits for the slowest - /// checkpoint rather than their sum. Returns the number of runtimes - /// closed. + /// handle after the scan. Each reserved runtime owns its own writer and + /// readers, so they close concurrently and the shutdown waits for the + /// slowest checkpoint rather than their sum. + /// + /// A held runtime is scanned again until its holders let go, for at most + /// [`SHUTDOWN_HOLD_DRAIN_LIMIT`]: owners joined before store close can + /// still be unwinding the tasks that hold their leases, and under load + /// that tail outlasts the owner join (#4138). A runtime still held at the + /// limit stays mounted and is logged with its blockers. Returns the number + /// of runtimes closed. #[tracing::instrument( name = "runtime_core.registry.close_idle_for_shutdown", level = "trace", @@ -135,34 +178,55 @@ impl StoreRuntimeRegistry { operation: "close idle registered runtimes for shutdown", })?; self.cancel_and_join_opens_for_shutdown(&cx).await?; - let (reservations, reserve_failure) = { + let deadline = std::time::Instant::now() + SHUTDOWN_HOLD_DRAIN_LIMIT; + let mut closed = 0; + loop { + let final_pass = std::time::Instant::now() >= deadline; + let (batch, held) = self.close_idle_batch(final_pass).await?; + closed += batch; + if !held || final_pass { + return Ok(closed); + } + if crate::native_runtime::sleep_until( + std::time::Instant::now() + SHUTDOWN_HOLD_DRAIN_POLL, + ) + .await + .is_err() + { + return self + .close_idle_batch(true) + .await + .map(|(batch, _)| closed + batch); + } + } + } + + /// Closes every runtime idle right now and reports whether any stays + /// held. Only the final pass logs the held ones. + async fn close_idle_batch( + &self, + final_pass: bool, + ) -> Result<(usize, bool), StoreRuntimeRegistryFailure> { + let (reservations, held, reserve_failure) = { let mut state = self.lock_state(); let mut idle = Vec::new(); + let mut held = false; for (key, entry) in &state.entries { let RegistryEntry::Ready(ready) = entry else { continue; }; - let mut blockers = ready - .owner - .runtime() - .eviction_eligibility(Duration::ZERO) - .blockers; - blockers.retain(|blocker| { - !matches!(blocker, ShardRuntimeEvictionBlocker::PinnedProfile) - }); - let profile_pins = state.profile_pin_tokens.get(key).map_or(0, BTreeSet::len); - let graph_leases = state - .graph_publications - .get(key) - .map_or(0, |retained| retained.lease_tokens.len()); - if blockers.is_empty() && profile_pins == 0 && graph_leases == 0 { + let hold = RuntimeHold::of(&state, key, &ready.owner); + if hold.is_idle() { idle.push(key.clone()); - } else { + continue; + } + held = true; + if final_pass { tracing::warn!( path = %ready.owner.locator().path().display(), - ?blockers, - profile_pins, - graph_leases, + blockers = ?hold.blockers, + profile_pins = hold.profile_pins, + graph_leases = hold.graph_leases, "store runtime still held at shutdown; its WAL is not truncated" ); } @@ -178,7 +242,7 @@ impl StoreRuntimeRegistry { } } } - (reservations, reserve_failure) + (reservations, held, reserve_failure) }; let closed = reservations.len(); let mut first_failure = reserve_failure; @@ -209,7 +273,7 @@ impl StoreRuntimeRegistry { first_failure.get_or_insert(failure); } } - first_failure.map_or(Ok(closed), Err) + first_failure.map_or(Ok((closed, held)), Err) } #[tracing::instrument(name = "runtime_core.registry.close_path", level = "trace", skip_all)] @@ -896,6 +960,28 @@ mod tests { drop(profile); } + /// A lease its owner releases moments after store close begins must not + /// leave the store's WAL behind (#4138). + #[asupersync::test] + async fn shutdown_close_waits_for_a_lease_released_while_it_runs() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("runtime.db"); + let (registry, profile, code, _authority) = mount_code_runtime(path.clone()).await; + let binding = code.binding().clone(); + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + drop(code); + }); + + assert_eq!(registry.close_idle_for_shutdown().await.unwrap(), 1); + assert!(matches!( + registry.lookup(&binding), + StoreRuntimeLookup::Missing { .. } + )); + release.join().unwrap(); + drop(profile); + } + #[asupersync::test] async fn refused_shutdown_close_still_settles_its_reserved_runtime() { let temporary = tempfile::tempdir().unwrap(); diff --git a/tests/support/isolated_profile.rs b/tests/support/isolated_profile.rs index c7f86e4f62..82a24a026e 100644 --- a/tests/support/isolated_profile.rs +++ b/tests/support/isolated_profile.rs @@ -174,6 +174,9 @@ pub fn apply_hermetic_child_env(command: &mut Command, home: &Path) { // Host discovery may invoke gh; its telemetry must not write a device // identity into the fixture home or send fixture activity upstream. .env("GH_TELEMETRY", "0") + // The advisory version check would otherwise call the GitHub API from + // every spawned daemon (tracedecay_dashboard_api::cloud). + .env("TRACEDECAY_NO_RELEASE_CHECK", "1") .env("PATH", hermetic_path::<&Path>(&[])); } From bdd758fdcd9db67629b29214a501c0ee78daf956 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 16 Oct 2026 15:02:05 +0000 Subject: [PATCH 2/2] docs(storage): reconcile steps 3 and 5 and the open growth sources --- docs/plans/storage/README.md | 72 ++++++++++++++++++++++++++++---- docs/plans/storage/decisions.tsv | 11 +++++ 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/docs/plans/storage/README.md b/docs/plans/storage/README.md index 5d9da45dfd..a094ba09c0 100644 --- a/docs/plans/storage/README.md +++ b/docs/plans/storage/README.md @@ -638,6 +638,10 @@ changes a file that holds memory. 11. Graph engine gate. Measure SQLite code packs against sealed Grafeo on this repository (activation RSS, traversal p50 and p95). Keep whichever wins; delete the other. +12. Growth acceptance. Run a daemon on a fresh profile for 24 hours under a + recorded ingest and code-index load, and read `telemetry.storage_usage`. Proof: + every class stays within its budget, and the WAL of every file is at or under + its soft limit plus one batch at each sample. ## What this supersedes @@ -674,13 +678,61 @@ changes a file that holds memory. in step 8. The additive-migration rule for memory ships with the first change to a memory table (the conversion in step 8), not as machinery with no migration to run. The class contract is unchanged. -- Step 3 (2026-10-16, branch `refactor/storage-worktree-liveness`). The first - growth source is closed: orphan-store liveness no longer counts the shared git - common directory, so a deleted worktree's store becomes collectable while its - repository lives. Still open in step 3: a daemon-run collector for unregistered - `proj_*` directories, collecting a store's non-memory files when memory rows pin - it, per-tick response-handle sweeps, `hook_analytics.jsonl` rotation, and - age-pruned idempotency rows. +- Step 3 (2026-10-16, done). Four PRs stopped the growth this step names: + - #4120. Orphan-store liveness no longer counts the shared git common + directory, so a deleted worktree's store becomes collectable while its + repository lives. + - #4126. The maintenance tick runs one bounded page (8 directories, 5 s) of the + unregistered-directory sweep, which only Doctor previewed before. The sweep + still refuses a directory that holds memory rows or a payload it cannot + inspect. Re-registering such a directory from a self-describing `project.json` + needs the step 8 layout, so until then it stays whole. + - #4128 (the #4099 lane). Every tick sweeps expired response handles, and a + fully imported `hook_analytics.jsonl` over 64 MiB is truncated under its + append lock. + - #4129. Each commit retires the idempotency rows of an incarnation (a daemon + generation) whose checkpoint on that shard has not moved for 7 days. The + committing incarnation is never aged out. This departs from the literal "7 d + of age". Pruning a live writer's rows by age needs an index on + `committed_at_micros`. That index changes the exact final schema shape, and + stores that hold memory have no migration path before step 8. Retiring + exited incarnations ends the stranding the design names, and the age rule + arrives with `journal.db`. + Collecting the non-memory files of a store whose memory rows pin it moves to + step 8, where memory gets its own file. +- Step 5 (2026-10-16, in progress). Four decisions: + - The journal shape is "service inside the job". `ExactSqlHandle::transact` + runs one whole write transaction on the writer thread as a job (#4133). A + Work or Workflow operation runs its existing synchronous contracts decision + inside that job, over a storage scoped to the job's transaction, and that + storage holds no `ExactSqlHandle`. Effects outside storage (Git, providers, + other stores) stay outside the job. + - Shipped so far: handoff grants (#4133), pause and resume of a run (#4139), + and the git-index store, whose actor thread is deleted (#4140). + - `transact` has no operation-id ledger yet. The ledger is new schema on stores + with no migration path, so it ships with `journal.db` in step 8. Until then + each domain keeps its own idempotency. + - Reads use `query_async` and preview/apply use `on_blocking_pool`. Both are + blocking-pool hops that step 7 replaces. +- #4138 (2026-10-16). Shutdown store close waits up to 3 s for leases to drain + before deciding a store's WAL cannot be truncated. Under load, owners joined + before store close could still be unwinding the tasks that held their leases, + and `graceful_stop_truncates_profile_and_project_store_wals` failed. This is an + interim remedy; the step 7 executor owns close. +- Growth sources from #4099 still open, each with the step that closes it: + 1. LCM payloads that no summary covers yet: step 10's evidence lever. + 2. Session databases of unmounted projects get no retention: step 10, whose + retention covers every project in the catalog. + 3. Observation projection history (active and unavailable skeleton rows): + step 8 makes it derived and step 10 trims it behind the evidence window. + 4. The projection disposition ledger: step 8. Dispositions become derived rows + that cascade with evidence. + 5. Generation staging release: step 9 (`scratch/`). + 6. The 512 MiB response-handle cap on top of #4128's TTL sweep: step 10. + 7. Hook events into `telemetry.hook_events`: step 8. + 8. Closing a store's lane when its file is lost, instead of a daemon restart: + step 7's unlink probe. #4130 is the interim restart remedy. + 9. A 24-hour acceptance run on a fresh profile showing bounded growth: step 12. - Step 4 merged into step 8 (2026-10-16). A dead-letter valve on today's global queue was built and then dropped: the projection code deliberately keeps corrupt projection authority (ghost or divergent provenance, divergent workflow state) a @@ -708,5 +760,7 @@ changes a file that holds memory. ## Next implementation step -The rest of step 3: a daemon-run collector for unregistered project directories -that never deletes durable data, on top of the #4099 measurement lane. +The rest of step 5, slice by slice: placement and the leaf Work ports, the +native-integration and remote-replay actor threads, then the attempt chain together +with run control and Workflow. After those, delete `off_native_thread` for Work and +the native wait counters. diff --git a/docs/plans/storage/decisions.tsv b/docs/plans/storage/decisions.tsv index a6c7ffe533..9e3acaeda0 100644 --- a/docs/plans/storage/decisions.tsv +++ b/docs/plans/storage/decisions.tsv @@ -32,3 +32,14 @@ ts phase decision why evidence result 2026-10-16T13:00:00Z implement step 3a: orphan-store liveness stops counting the shared git common dir first #4099 growth source: deleted worktrees' stores stayed Live forever PR 4120; maintenance unit_test sweep_plans_a_deleted_worktrees_store_while_its_repository_lives (mutation fails) 107 passed 2026-10-16T13:30:00Z implement built a dead-letter valve on today's global projection queue, then dropped it three observation_collision_tests deliberately keep corrupt provenance a hard error; skipping launders corrupt derived state; the design's answer is a derived rebuild, which needs the step-8 partition global-db unit_test: 3 collision tests failed against the valve; parked patch in scratchpad step 4 merged into step 8 2026-10-16T13:35:00Z implement architect Phase E check after the dropped valve: one deviation, the sketch stands Phase E needs a pattern of friction; this was one place where today's layout lacks the derived file the design relies on README Implementation reconciliation no re-sketch +2026-10-16T15:00:00Z implement step 3b: the maintenance cold-store page runs the unregistered-directory sweep with apply on, one bounded page per tick only Doctor previewed it; the sweep already refuses durable memory and unverifiable payloads; re-registration from project.json needs the step 8 layout PR 4126; the_maintenance_cold_store_page_collects_an_unregistered_project_dir (2 mutations killed) maintenance unit_test 109 passed +2026-10-16T15:05:00Z implement folded the separate 4099-orphans attempt as nothing its test duplicates 4120 plus the existing relink and alias tests /fast/tmp/4099-orphans diff worktree safe to remove +2026-10-16T15:10:00Z implement left the response-handle sweep and hook_analytics rotation to the 4099 lane shipped there as PR 4128 coordinator message not duplicated +2026-10-16T15:30:00Z implement step 3c: retire idempotency rows of incarnations whose shard checkpoint is quiet for 7 d; never age out the live incarnation an age index changes the exact final shape of stores holding memory, which have no migration path; incarnation is the daemon generation, so restarts strand rows PR 4129; 3 mutations killed deviation from the literal 7 d age prune, recorded in README +2026-10-16T15:40:00Z implement kept step 4 folded into step 8 no new evidence against the 13:30 call; 4107 fixed the freeze; contract and sanitization errors already reject; transient errors already back off observation_projection/rebuild.rs project_observation_once kept +2026-10-16T16:10:00Z implement step 5 ships as slices; 5a is the transact primitive plus handoff grants census: 25 sync traits and 78 methods; handoff is 4 methods, the only Work-adjacent path parking native threads, and already idempotent PR 4133 3 mutations killed +2026-10-16T16:10:00Z implement transact carries no operation-id ledger in step 5 the ledger is new schema on stores with no migration path; per-domain idempotency already exists; the ledger ships with journal.db PR 4133 body deviation recorded in README +2026-10-16T16:40:00Z implement step 5 shape: run the existing sync contracts decision inside one writer job over a job-scoped storage (hybrid B), not async ports one atomic load-decide-update per operation and no parked thread; async ports keep multi-trip transactions; effects outside storage are hoisted out of the job design probe of dispatch_work_application arms; PR 4139 pause and resume shipped +2026-10-16T17:30:00Z implement 5c: GitIndexTransactionStore async, actor thread deleted, preview GC on a NativeTask; kept on_blocking_pool for preview and apply those futures still run sync Git and a block_on configuration read and must not poll on a foreign executor thread PR 4140; 6 mutations killed first of three actor threads gone +2026-10-16T18:30:00Z implement graceful_stop WAL failure diagnosed as a master shutdown race, not storage PRs clean master fails 8/8 under concurrent load; the same merge commit passes and fails by time; the daemon log shows client leases still held at store close #4138; load repro script fixed by a bounded lease drain in registry close +2026-10-16T18:45:00Z implement #4138 fix lives in the registry close path, not the barred lifecycle file: close idle runtimes at once and rescan held ones for up to 3 s the store-close reserve is 12 s; idle runtimes must keep closing concurrently (attachment test) load repro: fix 24/24 pass while master fails; disabling the rescan fails 16/16 interim until the step 7 executor owns close