From aebe3f54b73b686af639d337c22683badac38f03 Mon Sep 17 00:00:00 2001 From: scgopi Date: Mon, 28 Sep 2026 14:16:22 -0700 Subject: [PATCH 1/3] Test that finished remote loops come back after a reboot Signed-off-by: scgopi --- .../Tests/RemoteSessionResumeTests.swift | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index 9687860e..4268ed6b 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -467,4 +467,141 @@ struct RemoteSessionResumeTests { #expect(started.value.isEmpty) } + + // MARK: - Finished loops across a remote reboot + + @Test + func theSweepRestoresFinishedLoopsTheRebootKilledAndLeavesThemFinished() async { + // A finished unattended loop whose pane was still open dialed "waiting for graphcoded" + // forever after a codespace restart: the pane leaves every unattended loop to the + // daemon, and the sweep skipped every resolved one. + let started = LockIsolated<[LoopNode]>([]) + let restored = LockIsolated<[LoopNode]>([]) + let succeeded = LoopNode( + title: "Done", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .succeeded) + let stopped = LoopNode( + title: "Poll", loopType: .timeBased, triggerPrompt: "/loop 1h Check", state: .stopped) + var missingCLI = LoopNode( + title: "NoCLI", loopType: .goalBased, goal: GoalSpec(summary: "ship"), state: .stopped) + missingCLI.launchFailure = LaunchFailure(executable: "claude", backend: .claudeCode) + let turn = LoopNode( + title: "Read", loopType: .turnBased, checkDescription: "Sound?", state: .succeeded) + let running = goalNode() + let store = GraphStore( + graph: LoopGraph( + scope: LoopGraphScope(projectPath: location.projectPath, name: "widget"), + nodes: [succeeded, stopped, missingCLI, turn, running]), + onEnsureSession: { node, _ in started.withValue { $0.append(node) } }, + onRestoreRebootedSessions: { nodes, _ in restored.withValue { $0 += nodes } }) + + await store.ensureUnattendedSessionsAlive() + try? await Task.sleep(for: .milliseconds(200)) + + #expect(started.value.map(\.id) == [running.id]) + #expect(Set(restored.value.map(\.id)) == [succeeded.id, stopped.id]) + for copy in restored.value { + let prompt = copy.sessionPrompt(forProjectPath: location.projectPath) ?? "" + #expect(!prompt.contains("tests pass")) + #expect(!prompt.contains("/loop")) + } + let states = await store.graph.nodes.map(\.state) + #expect(states == [.succeeded, .stopped, .stopped, .succeeded, .running]) + } + + @Test + func theRestoreOnlyRunsWhenTheBootChangedAndResumesQuietly() throws { + let node = LoopNode( + title: "Done", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .succeeded) + let quiet = GraphStore.rebootRestoreCopy(of: node) + let invocation = try #require( + ZmxSessionLauncher.remoteEnsureInvocation( + forNode: quiet, at: location, onlyAfterReboot: true)) + let script = try #require(invocation.last) + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + #expect(script.contains(RemoteBootMarker.captureFragment)) + #expect(script.contains("cat \(RemoteBootMarker.markerExpression(forSessionName: name))")) + #expect(script.contains("[ \"$gc_boot\" != \"$gc_last\" ]")) + #expect(script.contains("'--resume'")) + #expect(!script.contains("tests pass")) + let gate = try #require(script.range(of: "[ \"$gc_boot\" != \"$gc_last\" ]")) + let run = try #require(script.range(of: "'run'")) + #expect(gate.lowerBound < run.lowerBound) + } + + @Test + func everyDaemonEnsureRecordsTheBootItSawTheSessionIn() throws { + // The marker is what tells a pane, and now the sweep, that a missing session died + // with the machine. Only a pane attach wrote it, so a session the daemon started and + // no pane ever joined had no marker, and one the daemon restored kept a stale one. + let node = goalNode() + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let invocation = try #require( + ZmxSessionLauncher.remoteEnsureInvocation(forNode: node, at: location)) + let script = try #require(invocation.last) + let write = RemoteBootMarker.writeFragment(forSessionName: name) + #expect(script.components(separatedBy: write).count == 3) + } + + @Test + func aDaemonKillForgetsTheBootSoTheSessionIsNotRestored() throws { + // Ended on purpose (a finished loop freed, a stop, a delete): after a later reboot + // the sweep must not bring it back, and a pane must read "ended", not "rebooted". + let node = goalNode() + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let script = try #require( + ZmxSessionLauncher.remoteKillInvocation(forNode: node, at: location).last) + #expect(script.contains("rm -f \(RemoteBootMarker.markerExpression(forSessionName: name))")) + } + + @Test + func theRebootProbeNamesOnlyMissingSessionsFromAnEarlierBoot() throws { + let home = FileManager.default.temporaryDirectory + .appendingPathComponent("reboot-probe-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: home) } + let bin = home.appendingPathComponent("bin", isDirectory: true) + let boots = home.appendingPathComponent(".graphcode/boots", isDirectory: true) + try FileManager.default.createDirectory(at: bin, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: boots, withIntermediateDirectories: true) + let zmx = bin.appendingPathComponent("zmx") + try """ + #!/bin/sh + printf ' name=alive\\tpid=1\\tclients=0\\n name=husk\\tpid=2\\tended=5\\texit_code=0\\n' + """.write(to: zmx, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: zmx.path) + let bootProbe = Process() + bootProbe.executableURL = URL(fileURLWithPath: "/bin/sh") + bootProbe.arguments = ["-c", RemoteBootMarker.captureFragment + "; printf %s \"$gc_boot\""] + let bootPipe = Pipe() + bootProbe.standardOutput = bootPipe + try bootProbe.run() + bootProbe.waitUntilExit() + let boot = String(decoding: bootPipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + try #require(!boot.isEmpty) + for (name, marker) in [ + ("rebooted", "an-earlier-boot"), ("sameboot", boot), ("alive", "an-earlier-boot"), + ("husk", "an-earlier-boot"), + ] { + try marker.write( + to: boots.appendingPathComponent(name), atomically: true, encoding: .utf8) + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + ZmxSessionLauncher.rebootProbeScript( + forSessionNames: ["rebooted", "sameboot", "alive", "husk", "unmarked"]), + ] + process.environment = ["HOME": home.path, "PATH": bin.path + ":/usr/bin:/bin:/usr/sbin"] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + process.waitUntilExit() + let output = String(decoding: pipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + + #expect(process.terminationStatus == 0) + #expect(ZmxSessionLauncher.parseRebootProbe(output) == ["rebooted", "husk"]) + } } From 8a3b5ad7306019bac7e09508b7a251dc01b2c8d5 Mon Sep 17 00:00:00 2001 From: scgopi Date: Mon, 28 Sep 2026 14:26:15 -0700 Subject: [PATCH 2/3] Restore finished remote loops a host reboot killed A finished goal or time loop whose pane was open when its codespace restarted dialed 'waiting for graphcoded' forever: the pane leaves every unattended loop to the daemon, and the remote sweep skipped every resolved node. The sweep now probes each host once for finished loops whose session is missing and was last seen in an earlier boot, and brings those back as their banked conversation, with no task, poller, heartbeat or state change. The daemon's ensure records the boot marker too, and a daemon kill clears it, so a session ended on purpose is never restored. Signed-off-by: scgopi --- GraphcodeKit/Sources/GraphStore.swift | 36 ++++++++- GraphcodeKit/Sources/ProjectRegistry.swift | 5 ++ .../Sources/Sessions/CLISessionBackend.swift | 5 ++ .../Sources/Sessions/ZmxSessionLauncher.swift | 79 +++++++++++++++++-- .../Tests/RemoteSessionResumeTests.swift | 24 ++++-- 5 files changed, 134 insertions(+), 15 deletions(-) diff --git a/GraphcodeKit/Sources/GraphStore.swift b/GraphcodeKit/Sources/GraphStore.swift index 6f815762..109e3602 100644 --- a/GraphcodeKit/Sources/GraphStore.swift +++ b/GraphcodeKit/Sources/GraphStore.swift @@ -52,6 +52,7 @@ public actor GraphStore { private let onGraphEvent: (@Sendable (DaemonEvent) -> [UUID: DaemonWireEnvelope])? private let onConnectionFailure: (@Sendable (UUID) -> Void)? private let onEnsureSession: (@Sendable (LoopNode, String?) -> Void)? + private let onRestoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? private let onTerminateSession: (@Sendable (LoopNode, String?) -> Void)? /// Kills a loop's session and, for an unattended loop, relaunches it on the same /// transcript. Awaited, unlike the two above: the answer is whether the old session @@ -321,6 +322,7 @@ public actor GraphStore { onGraphEvent: (@Sendable (DaemonEvent) -> [UUID: DaemonWireEnvelope])? = nil, onConnectionFailure: (@Sendable (UUID) -> Void)? = nil, onEnsureSession: (@Sendable (LoopNode, String?) -> Void)? = nil, + onRestoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? = nil, onFindMissingProvider: (@Sendable (LoopNode, String?) async -> LaunchFailure?)? = nil, onTerminateSession: (@Sendable (LoopNode, String?) -> Void)? = nil, onRestartSession: (@Sendable (LoopNode, String?) async -> Bool)? = nil, @@ -364,6 +366,7 @@ public actor GraphStore { self.onGraphEvent = onGraphEvent self.onConnectionFailure = onConnectionFailure self.onEnsureSession = onEnsureSession + self.onRestoreRebootedSessions = onRestoreRebootedSessions self.onFindMissingProvider = onFindMissingProvider self.onTerminateSession = onTerminateSession self.onRestartSession = onRestartSession @@ -4474,16 +4477,43 @@ public actor GraphStore { /// sweep would restart each poller's interval and a goal polled less often than the /// sweep would never fire at all. The pollers are already running; they are in-memory /// and a remote reboot doesn't touch them. - /// - **Resolved nodes are skipped whatever their loop type.** The load-time version - /// restarts a `.stopped` time-based node, which is defensible once at boot and wrong - /// every minute: a human who stopped a remote loop would watch it come back. + /// - **Resolved nodes are never ensured, whatever their loop type.** The load-time + /// version restarts a `.stopped` time-based node, which is defensible once at boot and + /// wrong every minute: a human who stopped a remote loop would watch it come back. + /// + /// A resolved node's session is still brought back when the host's reboot killed it + /// (`onRestoreRebootedSessions`): a pane leaves every unattended loop to this daemon, + /// so a finished loop's open pane otherwise dialed "waiting for graphcoded" forever. The + /// restore is the conversation only (`rebootRestoreCopy`): no task, no poller or + /// heartbeat, no state change. Attended loops are not here — their pane restores them. public func ensureUnattendedSessionsAlive() async { for node in graph.nodes where node.runsUnattended && !node.isResolved { ensureSession(node) } + let finished = graph.nodes.filter { + $0.runsUnattended && $0.isResolved && $0.launchFailure == nil + } + if let onRestoreRebootedSessions, !finished.isEmpty { + let path = graph.project.path + let copies = finished.map(Self.rebootRestoreCopy) + Task.detached { await onRestoreRebootedSessions(copies, path) } + } await broadcastIfTemplatesRefreshed() } + /// A finished loop as its reboot restore launches it: the banked conversation resumed, + /// or, with nothing banked, a session opening on this note instead of the loop's task — + /// the same shape `resumeResolvedSession` gives a finished loop a human opens. + static func rebootRestoreCopy(of node: LoopNode) -> LoopNode { + var quiet = node + quiet.loopType = .sketch + quiet.attachments = [] + quiet.firstInstruction = + "[graphcode] The machine this loop runs on restarted, and its earlier conversation " + + "could not be resumed. This loop has finished; wait for the human's question." + return quiet + } + // MARK: - Broadcast private func broadcast() async { diff --git a/GraphcodeKit/Sources/ProjectRegistry.swift b/GraphcodeKit/Sources/ProjectRegistry.swift index 7bc62cb4..dfaba26d 100644 --- a/GraphcodeKit/Sources/ProjectRegistry.swift +++ b/GraphcodeKit/Sources/ProjectRegistry.swift @@ -59,6 +59,7 @@ public actor ProjectRegistry { /// one named project — see `sidebarSubscribers`. private var sidebarConnections: Set = [] private let ensureSession: (@Sendable (LoopNode, String?) -> Void)? + private let restoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? private let terminateSession: (@Sendable (LoopNode, String?) -> Void)? private let restartSession: (@Sendable (LoopNode, String?) async -> Bool)? private let startQuickChat: @@ -121,6 +122,8 @@ public actor ProjectRegistry { platformPaths: any PlatformPaths = CurrentPlatformPaths.value, replayStore: DaemonReplayStore = DaemonReplayStore(), ensureSession: (@Sendable (LoopNode, String?) -> Void)? = CLISessionBackend.ensureSession, + restoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? = + CLISessionBackend.restoreRebootedSessions, terminateSession: (@Sendable (LoopNode, String?) -> Void)? = CLISessionBackend.terminateSession, restartSession: (@Sendable (LoopNode, String?) async -> Bool)? = @@ -163,6 +166,7 @@ public actor ProjectRegistry { quickChatStore = QuickChatStore(baseDirectory: persistenceDirectory) self.replayStore = replayStore self.ensureSession = ensureSession + self.restoreRebootedSessions = restoreRebootedSessions self.terminateSession = terminateSession self.restartSession = restartSession self.evaluatePredicate = evaluatePredicate @@ -1011,6 +1015,7 @@ public actor ProjectRegistry { }, onConnectionFailure: onConnectionFailure, onEnsureSession: ensureSession, + onRestoreRebootedSessions: restoreRebootedSessions, onFindMissingProvider: { node, path in await ProviderPath.missingProvider(for: node, projectPath: path) }, diff --git a/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift b/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift index 273a5580..f51b4d35 100644 --- a/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift +++ b/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift @@ -287,6 +287,11 @@ extension CLISessionBackend { Task.detached { await backend(for: node).launch(node, path) } } + public static let restoreRebootedSessions: @Sendable ([LoopNode], String) async -> Void = { + nodes, path in + await ZmxSessionLauncher.restoreRebootedRemote(nodes, projectPath: path) + } + public static let terminateSession: @Sendable (LoopNode, String?) -> Void = { node, path in Task.detached { await backend(for: node).terminate(node, path) } } diff --git a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift index 258ce3ad..24d4c7c7 100644 --- a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift +++ b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift @@ -1635,7 +1635,7 @@ public enum ZmxSessionLauncher { static func remoteEnsureInvocation( forNode node: LoopNode, at location: RemoteProjectLocation, settings: GraphcodeSettings = GraphcodeSettingsStore.load(), - bridgeState: RemoteBridgeWireState? = nil + bridgeState: RemoteBridgeWireState? = nil, onlyAfterReboot: Bool = false ) -> [String]? { let shedPrompt = ShedPromptReport() guard @@ -1710,13 +1710,25 @@ public enum ZmxSessionLauncher { let launch = agentLabelCommand(zmxPath: "zmx", forNode: node) .map { "\(create) && { \($0) || true; }" } ?? create + // The boot this session is alive in, recorded by the daemon as well as by a pane + // attach (`RemoteBootMarker`): it is how a pane, and `rebootProbeScript`, tell a + // session that died with the machine from one that ended. + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let markerWrite = RemoteBootMarker.writeFragment(forSessionName: name) + var missing = trustSeed + hooksWrite + "{ \(launch); } && { \(markerWrite); }" + if onlyAfterReboot { + let marker = RemoteBootMarker.markerExpression(forSessionName: name) + missing = + "\(RemoteBootMarker.captureFragment); gc_last=$(cat \(marker) 2>/dev/null); " + + "if [ -n \"$gc_boot\" ] && [ -n \"$gc_last\" ] && [ \"$gc_boot\" != \"$gc_last\" ]; " + + "then \(missing); fi" + } let script = "cd \(RemoteProjectLocation.shellQuoted(location.remotePath)) && { " + deliveryFragment( delivery, ifSessionMissing: check, bridgeStateGeneration: bridgeState.map(\.generation)) - + "\(check) >/dev/null 2>&1\(bank) || \(repair){ " + trustSeed + hooksWrite - + "\(launch); }; }" + + "\(check) >/dev/null 2>&1\(bank) && { \(markerWrite); } || \(repair){ \(missing); }; }" return location.sshInvocation(remoteCommand: location.remoteLoginShellCommand(script)) } @@ -2228,7 +2240,12 @@ public enum ZmxSessionLauncher { static func remoteKillInvocation( forNode node: LoopNode, at location: RemoteProjectLocation ) -> [String] { - let script = quotedCommand(["zmx"] + killArguments(forNode: node)) + // A session ended on purpose must not read as one a reboot killed, to the pane or to + // `rebootProbeScript`. + let marker = RemoteBootMarker.markerExpression( + forSessionName: SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName) + let script = + "rm -f \(marker); " + quotedCommand(["zmx"] + killArguments(forNode: node)) return location.sshInvocation(remoteCommand: location.remoteLoginShellCommand(script)) } @@ -2236,7 +2253,9 @@ public enum ZmxSessionLauncher { _ = await runRemoteRetrying(remoteKillInvocation(forNode: node, at: location)) } - private static func startRemote(_ node: LoopNode, at location: RemoteProjectLocation) async { + private static func startRemote( + _ node: LoopNode, at location: RemoteProjectLocation, onlyAfterReboot: Bool = false + ) async { // A codespace that is down is redialed on the shared schedule, not on every sweep. guard await CodespaceDialBreaker.shared.permits(location) else { return } // A dial already in flight for this node is doing this job; a second one racing it @@ -2294,7 +2313,7 @@ public enum ZmxSessionLauncher { // as the local path: no UI here, the node's state stays honest, opening the loop // retries. if let ensure = remoteEnsureInvocation( - forNode: node, at: location, bridgeState: bridgeState + forNode: node, at: location, bridgeState: bridgeState, onlyAfterReboot: onlyAfterReboot ) { if await runRemoteRetrying(ensure) { await CodespaceDialBreaker.shared.record(location, reached: true) @@ -2304,6 +2323,54 @@ public enum ZmxSessionLauncher { await RemoteEnsureGate.shared.end(node.id, token: lease) } + /// Brings back the sessions of finished loops that a reboot of their remote host killed + /// (`GraphStore.ensureUnattendedSessionsAlive`). One probe dial per host names the + /// sessions that are missing *and* were last seen alive in an earlier boot; only those + /// are dialed again, each behind the same boot gate, so a finished loop costs nothing + /// per sweep — a codespace dial spends the human's API quota (issue #480). + /// + /// `nodes` are already the quiet copies the store made (`GraphStore.rebootRestoreCopy`): + /// the create resumes the banked conversation, or opens on a note, never on the task. + static func restoreRebootedRemote(_ nodes: [LoopNode], projectPath: String) async { + guard !nodes.isEmpty, let location = RemoteProjectLocation.parse(projectPath: projectPath) + else { return } + let name = { (node: LoopNode) in + SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + } + let probe = location.sshInvocation( + remoteCommand: location.remoteLoginShellCommand( + rebootProbeScript(forSessionNames: nodes.map(name)))) + let (succeeded, output) = await collectRemoteOutput(probe, location: location) + guard succeeded else { return } + let rebooted = parseRebootProbe(output) + for node in nodes where rebooted.contains(name(node)) { + await startRemote(node, at: location, onlyAfterReboot: true) + } + } + + /// Prints `rebooted ` for each session that is not running and whose boot marker + /// names a boot other than this one — the pane's reboot verdict, made for many sessions + /// in one shell. A host that cannot answer (no boot ID, `zmx` not up yet) prints nothing. + static func rebootProbeScript(forSessionNames names: [String]) -> String { + let quotedNames = names.map(RemoteProjectLocation.shellQuoted).joined(separator: " ") + return "\(RemoteBootMarker.captureFragment); [ -n \"$gc_boot\" ] || exit 0; " + + "gc_ls=$(zmx ls 2>/dev/null) || exit 0; gc_tab=$(printf '\\t'); " + + "for gc_n in \(quotedNames); do " + + "gc_last=$(cat \"$HOME/.graphcode/boots/$gc_n\" 2>/dev/null); " + + "[ -n \"$gc_last\" ] && [ \"$gc_last\" != \"$gc_boot\" ] || continue; " + + "printf '%s\\n' \"$gc_ls\" | grep -v -e \"${gc_tab}ended=\" -e \"${gc_tab}err=\" " + + "| grep -q \"name=$gc_n$gc_tab\" || printf 'rebooted %s\\n' \"$gc_n\"; done; exit 0" + } + + static func parseRebootProbe(_ output: String) -> Set { + Set( + output.split(whereSeparator: \.isNewline).compactMap { line in + let fields = line.split(whereSeparator: \.isWhitespace) + guard fields.count == 2, fields[0] == "rebooted" else { return nil } + return String(fields[1]) + }) + } + static func start(_ node: LoopNode, projectPath: String? = nil) async { if let projectPath, let remote = RemoteProjectLocation.parse(projectPath: projectPath) { await startRemote(node, at: remote) diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index 4268ed6b..0ce3a1dc 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -467,8 +467,20 @@ struct RemoteSessionResumeTests { #expect(started.value.isEmpty) } +} + +/// A finished unattended loop across a remote reboot: its session comes back as the +/// conversation it was, never as another pass at the task. +@Suite +struct RemoteRebootRestoreTests { + private let location = RemoteProjectLocation( + user: "dev", host: "codespace", port: 2222, remotePath: "/workspaces/widget") - // MARK: - Finished loops across a remote reboot + private func goalNode() -> LoopNode { + LoopNode( + title: "Fix", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .running) + } @Test func theSweepRestoresFinishedLoopsTheRebootKilledAndLeavesThemFinished() async { @@ -540,7 +552,8 @@ struct RemoteSessionResumeTests { let invocation = try #require( ZmxSessionLauncher.remoteEnsureInvocation(forNode: node, at: location)) let script = try #require(invocation.last) - let write = RemoteBootMarker.writeFragment(forSessionName: name) + // Quote-free, so the login shell's re-quoting of the script cannot hide it. + let write = ">\(RemoteBootMarker.markerExpression(forSessionName: name))" #expect(script.components(separatedBy: write).count == 3) } @@ -565,10 +578,9 @@ struct RemoteSessionResumeTests { try FileManager.default.createDirectory(at: bin, withIntermediateDirectories: true) try FileManager.default.createDirectory(at: boots, withIntermediateDirectories: true) let zmx = bin.appendingPathComponent("zmx") - try """ - #!/bin/sh - printf ' name=alive\\tpid=1\\tclients=0\\n name=husk\\tpid=2\\tended=5\\texit_code=0\\n' - """.write(to: zmx, atomically: true, encoding: .utf8) + let listing = + " name=alive\\tpid=1\\tclients=0\\n name=husk\\tpid=2\\tended=5\\texit_code=0\\n" + try "#!/bin/sh\nprintf '\(listing)'\n".write(to: zmx, atomically: true, encoding: .utf8) try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: zmx.path) let bootProbe = Process() bootProbe.executableURL = URL(fileURLWithPath: "/bin/sh") From 3903423302dae22d93fd3816d3bcfd8db316e076 Mon Sep 17 00:00:00 2001 From: scgopi Date: Mon, 28 Sep 2026 14:53:56 -0700 Subject: [PATCH 3/3] Probe for rebooted finished loops only after a pane redials The reboot probe ran on every liveness sweep once a project had a finished loop, so an idle codespace went from no dials to one gh run a minute. A remote pane now touches a per-host stamp before each redial, and the sweep probes a host only when a stamp is newer than its last answered probe: a healthy host costs nothing, and an outage is bounded by the pane's own Signed-off-by: scgopi #480 schedule. --- .../Sources/Domain/SSHReconnectLoop.swift | 17 ++++-- .../Sources/Sessions/ZmxSessionLauncher.swift | 57 ++++++++++++++++++- .../Ghostty/GhosttyTerminalView+Remote.swift | 11 +++- .../Tests/RemoteSessionResumeTests.swift | 36 ++++++++++++ 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift index d65fb821..3dc68e2a 100644 --- a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift +++ b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift @@ -22,7 +22,12 @@ import Foundation public enum SSHReconnectLoop { public static let maxDelaySeconds = 15 - public static func script(connect: String, reconnect: String) -> String { + /// `redialStamp` is touched before every redial: it is how `graphcoded` learns, for + /// free, that a host may have rebooted under a loop it restores + /// (`ZmxSessionLauncher.restoreRebootedRemote`). + public static func script( + connect: String, reconnect: String, redialStamp: String? = nil + ) -> String { let passExit = "; gc_rc=$?; [ \"$gc_rc\" -ne 255 ] && exit \"$gc_rc\"" return "trap 'exit 130' INT; " + connect + passExit + "; " @@ -31,7 +36,7 @@ public enum SSHReconnectLoop { + #"Press Ctrl-C to stop. ──\033[0m\r\n' "$gc_rc" "$gc_delay"; "# + "sleep \"$gc_delay\"; gc_delay=$((gc_delay * 2)); " + "[ \"$gc_delay\" -gt \(maxDelaySeconds) ] && gc_delay=\(maxDelaySeconds); " - + reconnect + passExit + "; done" + + touching(redialStamp) + reconnect + passExit + "; done" } /// A Codespace surface's loop: the same dials and exit handling, retried on @@ -52,7 +57,7 @@ public enum SSHReconnectLoop { /// genuinely ended nonzero, which converges: the redial reattaches a live session, and /// a gone one takes the reconnect script's session-ended branch to a clean exit 0. public static func codespaceScript( - connect: String, reconnect: String, pauseMarker: String, + connect: String, reconnect: String, pauseMarker: String, redialStamp: String? = nil, schedule: CodespaceDialSchedule = .standard, upAfter: Int = 330 ) -> String { let marker = quoted(pauseMarker) @@ -100,7 +105,11 @@ public enum SSHReconnectLoop { + #"printf '\033[1;33m── Connection failed (exit %s). Retrying in %ss. "# + #"Press Ctrl-C to stop. ──\033[0m\r\n' "$gc_rc" "$gc_wait"; "# + "gc_wait_or_ask \"$gc_wait\" && { \(restart); }; fi; " - + "gc_t=$(date +%s); " + reconnect + passExit + clock + "; done" + + "gc_t=$(date +%s); " + touching(redialStamp) + reconnect + passExit + clock + "; done" + } + + private static func touching(_ stamp: String?) -> String { + stamp.map { "touch \(quoted($0)) 2>/dev/null; " } ?? "" } /// `RemoteProjectLocation.shellQuoted`, repeated because this file also builds in the diff --git a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift index 24d4c7c7..fadf019c 100644 --- a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift +++ b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift @@ -2326,14 +2326,22 @@ public enum ZmxSessionLauncher { /// Brings back the sessions of finished loops that a reboot of their remote host killed /// (`GraphStore.ensureUnattendedSessionsAlive`). One probe dial per host names the /// sessions that are missing *and* were last seen alive in an earlier boot; only those - /// are dialed again, each behind the same boot gate, so a finished loop costs nothing - /// per sweep — a codespace dial spends the human's API quota (issue #480). + /// are dialed again, each behind the same boot gate. + /// + /// The probe itself runs only when a pane of that host has redialed since the last + /// probe that answered (`redialStamp`): the one thing left dialing a finished loop's + /// host is its pane, and a healthy host has no pane redialing, so the sweep spends + /// nothing — a codespace dial spends the human's API quota (issue #480). /// /// `nodes` are already the quiet copies the store made (`GraphStore.rebootRestoreCopy`): /// the create resumes the banked conversation, or opens on a note, never on the task. - static func restoreRebootedRemote(_ nodes: [LoopNode], projectPath: String) async { + static func restoreRebootedRemote( + _ nodes: [LoopNode], projectPath: String, gate: RebootProbeGate = .shared + ) async { guard !nodes.isEmpty, let location = RemoteProjectLocation.parse(projectPath: projectPath) else { return } + let asked = Date() + guard await gate.panesRedialed(location) else { return } let name = { (node: LoopNode) in SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName } @@ -2342,12 +2350,55 @@ public enum ZmxSessionLauncher { rebootProbeScript(forSessionNames: nodes.map(name)))) let (succeeded, output) = await collectRemoteOutput(probe, location: location) guard succeeded else { return } + await gate.probed(location, at: asked) let rebooted = parseRebootProbe(output) for node in nodes where rebooted.contains(name(node)) { await startRemote(node, at: location, onlyAfterReboot: true) } } + /// Touched by a remote pane's reconnect loop before every redial + /// (`SSHReconnectLoop`), and read by `RebootProbeGate`. Per host, not per loop: one + /// probe answers for every loop on it. + public static func redialStamp(for location: RemoteProjectLocation) -> URL { + SupportDirectory.url.appendingPathComponent("remote-redials", isDirectory: true) + .appendingPathComponent("\(location.host).redial") + } + + /// Whether a host's panes have redialed since its last answered probe — the only + /// state `restoreRebootedRemote` keeps. Stamps from before this daemon started count + /// once, so a pane left waiting across a daemon restart is still answered. + actor RebootProbeGate { + static let shared = RebootProbeGate() + + private let stampFor: @Sendable (RemoteProjectLocation) -> URL + private var probedAt: [String: Date] = [:] + + init( + stampFor: @escaping @Sendable (RemoteProjectLocation) -> URL = { + ZmxSessionLauncher.redialStamp(for: $0) + } + ) { + self.stampFor = stampFor + } + + func panesRedialed(_ location: RemoteProjectLocation) -> Bool { + guard + let touched = + (try? FileManager.default.attributesOfItem( + atPath: stampFor(location).path))?[.modificationDate] as? Date + else { return false } + guard let since = probedAt[location.host] else { return true } + return touched > since + } + + /// Recorded only for a probe that answered: one that failed leaves the redial + /// pending, so the host is probed once it is back even if every pane has paused. + func probed(_ location: RemoteProjectLocation, at date: Date) { + probedAt[location.host] = date + } + } + /// Prints `rebooted ` for each session that is not running and whose boot marker /// names a boot other than this one — the pane's reboot verdict, made for many sessions /// in one shell. A host that cannot answer (no boot ID, `zmx` not up yet) prints nothing. diff --git a/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift b/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift index 31448076..800df370 100644 --- a/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift +++ b/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift @@ -45,14 +45,21 @@ extension GhosttyTerminalView { remoteCommand: location.remoteLoginShellCommand(script), interactive: true) let reconnect = location.sshCommandLine( remoteCommand: location.remoteLoginShellCommand(reconnectScript), interactive: true) + let stamp = ZmxSessionLauncher.redialStamp(for: location) + try? FileManager.default.createDirectory( + at: stamp.deletingLastPathComponent(), withIntermediateDirectories: true) guard location.isCodespace else { - return ["/bin/sh", "-c", SSHReconnectLoop.script(connect: connect, reconnect: reconnect)] + return [ + "/bin/sh", "-c", + SSHReconnectLoop.script(connect: connect, reconnect: reconnect, redialStamp: stamp.path), + ] } return [ "/bin/sh", "-c", SSHReconnectLoop.codespaceScript( connect: connect, reconnect: reconnect, - pauseMarker: CodespaceDialBreaker.reconnectMarker(for: location).path), + pauseMarker: CodespaceDialBreaker.reconnectMarker(for: location).path, + redialStamp: stamp.path), ] } diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index 0ce3a1dc..984b8f27 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -616,4 +616,40 @@ struct RemoteRebootRestoreTests { #expect(process.terminationStatus == 0) #expect(ZmxSessionLauncher.parseRebootProbe(output) == ["rebooted", "husk"]) } + + @Test + func aHealthyHostIsNeverProbed() async throws { + // The probe is a dial, and on a codespace a dial spends the human's API quota. Only + // a pane redialing its host is worth one; a host nobody is redialing costs nothing. + let stamp = FileManager.default.temporaryDirectory + .appendingPathComponent("redial-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: stamp) } + let gate = ZmxSessionLauncher.RebootProbeGate(stampFor: { _ in stamp }) + + #expect(await !gate.panesRedialed(location)) + + FileManager.default.createFile(atPath: stamp.path, contents: nil) + #expect(await gate.panesRedialed(location)) + + await gate.probed(location, at: Date().addingTimeInterval(1)) + #expect(await !gate.panesRedialed(location)) + + try FileManager.default.setAttributes( + [.modificationDate: Date().addingTimeInterval(5)], ofItemAtPath: stamp.path) + #expect(await gate.panesRedialed(location)) + } + + @Test(arguments: [false, true]) + func aPaneStampsItsHostBeforeEveryRedial(codespace: Bool) throws { + let script = + codespace + ? SSHReconnectLoop.codespaceScript( + connect: "CONNECT", reconnect: "RECONNECT", pauseMarker: "/tmp/p", + redialStamp: "/tmp/stamp") + : SSHReconnectLoop.script( + connect: "CONNECT", reconnect: "RECONNECT", redialStamp: "/tmp/stamp") + let touch = try #require(script.range(of: "touch '/tmp/stamp' 2>/dev/null; RECONNECT")) + let connect = try #require(script.range(of: "CONNECT")) + #expect(connect.upperBound <= touch.lowerBound) + } }