diff --git a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift index d65fb821..3dc68e2a 100644 --- a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift +++ b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift @@ -22,7 +22,12 @@ import Foundation public enum SSHReconnectLoop { public static let maxDelaySeconds = 15 - public static func script(connect: String, reconnect: String) -> String { + /// `redialStamp` is touched before every redial: it is how `graphcoded` learns, for + /// free, that a host may have rebooted under a loop it restores + /// (`ZmxSessionLauncher.restoreRebootedRemote`). + public static func script( + connect: String, reconnect: String, redialStamp: String? = nil + ) -> String { let passExit = "; gc_rc=$?; [ \"$gc_rc\" -ne 255 ] && exit \"$gc_rc\"" return "trap 'exit 130' INT; " + connect + passExit + "; " @@ -31,7 +36,7 @@ public enum SSHReconnectLoop { + #"Press Ctrl-C to stop. ──\033[0m\r\n' "$gc_rc" "$gc_delay"; "# + "sleep \"$gc_delay\"; gc_delay=$((gc_delay * 2)); " + "[ \"$gc_delay\" -gt \(maxDelaySeconds) ] && gc_delay=\(maxDelaySeconds); " - + reconnect + passExit + "; done" + + touching(redialStamp) + reconnect + passExit + "; done" } /// A Codespace surface's loop: the same dials and exit handling, retried on @@ -52,7 +57,7 @@ public enum SSHReconnectLoop { /// genuinely ended nonzero, which converges: the redial reattaches a live session, and /// a gone one takes the reconnect script's session-ended branch to a clean exit 0. public static func codespaceScript( - connect: String, reconnect: String, pauseMarker: String, + connect: String, reconnect: String, pauseMarker: String, redialStamp: String? = nil, schedule: CodespaceDialSchedule = .standard, upAfter: Int = 330 ) -> String { let marker = quoted(pauseMarker) @@ -100,7 +105,11 @@ public enum SSHReconnectLoop { + #"printf '\033[1;33m── Connection failed (exit %s). Retrying in %ss. "# + #"Press Ctrl-C to stop. ──\033[0m\r\n' "$gc_rc" "$gc_wait"; "# + "gc_wait_or_ask \"$gc_wait\" && { \(restart); }; fi; " - + "gc_t=$(date +%s); " + reconnect + passExit + clock + "; done" + + "gc_t=$(date +%s); " + touching(redialStamp) + reconnect + passExit + clock + "; done" + } + + private static func touching(_ stamp: String?) -> String { + stamp.map { "touch \(quoted($0)) 2>/dev/null; " } ?? "" } /// `RemoteProjectLocation.shellQuoted`, repeated because this file also builds in the diff --git a/GraphcodeKit/Sources/GraphStore.swift b/GraphcodeKit/Sources/GraphStore.swift index 6f815762..109e3602 100644 --- a/GraphcodeKit/Sources/GraphStore.swift +++ b/GraphcodeKit/Sources/GraphStore.swift @@ -52,6 +52,7 @@ public actor GraphStore { private let onGraphEvent: (@Sendable (DaemonEvent) -> [UUID: DaemonWireEnvelope])? private let onConnectionFailure: (@Sendable (UUID) -> Void)? private let onEnsureSession: (@Sendable (LoopNode, String?) -> Void)? + private let onRestoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? private let onTerminateSession: (@Sendable (LoopNode, String?) -> Void)? /// Kills a loop's session and, for an unattended loop, relaunches it on the same /// transcript. Awaited, unlike the two above: the answer is whether the old session @@ -321,6 +322,7 @@ public actor GraphStore { onGraphEvent: (@Sendable (DaemonEvent) -> [UUID: DaemonWireEnvelope])? = nil, onConnectionFailure: (@Sendable (UUID) -> Void)? = nil, onEnsureSession: (@Sendable (LoopNode, String?) -> Void)? = nil, + onRestoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? = nil, onFindMissingProvider: (@Sendable (LoopNode, String?) async -> LaunchFailure?)? = nil, onTerminateSession: (@Sendable (LoopNode, String?) -> Void)? = nil, onRestartSession: (@Sendable (LoopNode, String?) async -> Bool)? = nil, @@ -364,6 +366,7 @@ public actor GraphStore { self.onGraphEvent = onGraphEvent self.onConnectionFailure = onConnectionFailure self.onEnsureSession = onEnsureSession + self.onRestoreRebootedSessions = onRestoreRebootedSessions self.onFindMissingProvider = onFindMissingProvider self.onTerminateSession = onTerminateSession self.onRestartSession = onRestartSession @@ -4474,16 +4477,43 @@ public actor GraphStore { /// sweep would restart each poller's interval and a goal polled less often than the /// sweep would never fire at all. The pollers are already running; they are in-memory /// and a remote reboot doesn't touch them. - /// - **Resolved nodes are skipped whatever their loop type.** The load-time version - /// restarts a `.stopped` time-based node, which is defensible once at boot and wrong - /// every minute: a human who stopped a remote loop would watch it come back. + /// - **Resolved nodes are never ensured, whatever their loop type.** The load-time + /// version restarts a `.stopped` time-based node, which is defensible once at boot and + /// wrong every minute: a human who stopped a remote loop would watch it come back. + /// + /// A resolved node's session is still brought back when the host's reboot killed it + /// (`onRestoreRebootedSessions`): a pane leaves every unattended loop to this daemon, + /// so a finished loop's open pane otherwise dialed "waiting for graphcoded" forever. The + /// restore is the conversation only (`rebootRestoreCopy`): no task, no poller or + /// heartbeat, no state change. Attended loops are not here — their pane restores them. public func ensureUnattendedSessionsAlive() async { for node in graph.nodes where node.runsUnattended && !node.isResolved { ensureSession(node) } + let finished = graph.nodes.filter { + $0.runsUnattended && $0.isResolved && $0.launchFailure == nil + } + if let onRestoreRebootedSessions, !finished.isEmpty { + let path = graph.project.path + let copies = finished.map(Self.rebootRestoreCopy) + Task.detached { await onRestoreRebootedSessions(copies, path) } + } await broadcastIfTemplatesRefreshed() } + /// A finished loop as its reboot restore launches it: the banked conversation resumed, + /// or, with nothing banked, a session opening on this note instead of the loop's task — + /// the same shape `resumeResolvedSession` gives a finished loop a human opens. + static func rebootRestoreCopy(of node: LoopNode) -> LoopNode { + var quiet = node + quiet.loopType = .sketch + quiet.attachments = [] + quiet.firstInstruction = + "[graphcode] The machine this loop runs on restarted, and its earlier conversation " + + "could not be resumed. This loop has finished; wait for the human's question." + return quiet + } + // MARK: - Broadcast private func broadcast() async { diff --git a/GraphcodeKit/Sources/ProjectRegistry.swift b/GraphcodeKit/Sources/ProjectRegistry.swift index 7bc62cb4..dfaba26d 100644 --- a/GraphcodeKit/Sources/ProjectRegistry.swift +++ b/GraphcodeKit/Sources/ProjectRegistry.swift @@ -59,6 +59,7 @@ public actor ProjectRegistry { /// one named project — see `sidebarSubscribers`. private var sidebarConnections: Set = [] private let ensureSession: (@Sendable (LoopNode, String?) -> Void)? + private let restoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? private let terminateSession: (@Sendable (LoopNode, String?) -> Void)? private let restartSession: (@Sendable (LoopNode, String?) async -> Bool)? private let startQuickChat: @@ -121,6 +122,8 @@ public actor ProjectRegistry { platformPaths: any PlatformPaths = CurrentPlatformPaths.value, replayStore: DaemonReplayStore = DaemonReplayStore(), ensureSession: (@Sendable (LoopNode, String?) -> Void)? = CLISessionBackend.ensureSession, + restoreRebootedSessions: (@Sendable ([LoopNode], String) async -> Void)? = + CLISessionBackend.restoreRebootedSessions, terminateSession: (@Sendable (LoopNode, String?) -> Void)? = CLISessionBackend.terminateSession, restartSession: (@Sendable (LoopNode, String?) async -> Bool)? = @@ -163,6 +166,7 @@ public actor ProjectRegistry { quickChatStore = QuickChatStore(baseDirectory: persistenceDirectory) self.replayStore = replayStore self.ensureSession = ensureSession + self.restoreRebootedSessions = restoreRebootedSessions self.terminateSession = terminateSession self.restartSession = restartSession self.evaluatePredicate = evaluatePredicate @@ -1011,6 +1015,7 @@ public actor ProjectRegistry { }, onConnectionFailure: onConnectionFailure, onEnsureSession: ensureSession, + onRestoreRebootedSessions: restoreRebootedSessions, onFindMissingProvider: { node, path in await ProviderPath.missingProvider(for: node, projectPath: path) }, diff --git a/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift b/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift index 273a5580..f51b4d35 100644 --- a/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift +++ b/GraphcodeKit/Sources/Sessions/CLISessionBackend.swift @@ -287,6 +287,11 @@ extension CLISessionBackend { Task.detached { await backend(for: node).launch(node, path) } } + public static let restoreRebootedSessions: @Sendable ([LoopNode], String) async -> Void = { + nodes, path in + await ZmxSessionLauncher.restoreRebootedRemote(nodes, projectPath: path) + } + public static let terminateSession: @Sendable (LoopNode, String?) -> Void = { node, path in Task.detached { await backend(for: node).terminate(node, path) } } diff --git a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift index 258ce3ad..fadf019c 100644 --- a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift +++ b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift @@ -1635,7 +1635,7 @@ public enum ZmxSessionLauncher { static func remoteEnsureInvocation( forNode node: LoopNode, at location: RemoteProjectLocation, settings: GraphcodeSettings = GraphcodeSettingsStore.load(), - bridgeState: RemoteBridgeWireState? = nil + bridgeState: RemoteBridgeWireState? = nil, onlyAfterReboot: Bool = false ) -> [String]? { let shedPrompt = ShedPromptReport() guard @@ -1710,13 +1710,25 @@ public enum ZmxSessionLauncher { let launch = agentLabelCommand(zmxPath: "zmx", forNode: node) .map { "\(create) && { \($0) || true; }" } ?? create + // The boot this session is alive in, recorded by the daemon as well as by a pane + // attach (`RemoteBootMarker`): it is how a pane, and `rebootProbeScript`, tell a + // session that died with the machine from one that ended. + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let markerWrite = RemoteBootMarker.writeFragment(forSessionName: name) + var missing = trustSeed + hooksWrite + "{ \(launch); } && { \(markerWrite); }" + if onlyAfterReboot { + let marker = RemoteBootMarker.markerExpression(forSessionName: name) + missing = + "\(RemoteBootMarker.captureFragment); gc_last=$(cat \(marker) 2>/dev/null); " + + "if [ -n \"$gc_boot\" ] && [ -n \"$gc_last\" ] && [ \"$gc_boot\" != \"$gc_last\" ]; " + + "then \(missing); fi" + } let script = "cd \(RemoteProjectLocation.shellQuoted(location.remotePath)) && { " + deliveryFragment( delivery, ifSessionMissing: check, bridgeStateGeneration: bridgeState.map(\.generation)) - + "\(check) >/dev/null 2>&1\(bank) || \(repair){ " + trustSeed + hooksWrite - + "\(launch); }; }" + + "\(check) >/dev/null 2>&1\(bank) && { \(markerWrite); } || \(repair){ \(missing); }; }" return location.sshInvocation(remoteCommand: location.remoteLoginShellCommand(script)) } @@ -2228,7 +2240,12 @@ public enum ZmxSessionLauncher { static func remoteKillInvocation( forNode node: LoopNode, at location: RemoteProjectLocation ) -> [String] { - let script = quotedCommand(["zmx"] + killArguments(forNode: node)) + // A session ended on purpose must not read as one a reboot killed, to the pane or to + // `rebootProbeScript`. + let marker = RemoteBootMarker.markerExpression( + forSessionName: SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName) + let script = + "rm -f \(marker); " + quotedCommand(["zmx"] + killArguments(forNode: node)) return location.sshInvocation(remoteCommand: location.remoteLoginShellCommand(script)) } @@ -2236,7 +2253,9 @@ public enum ZmxSessionLauncher { _ = await runRemoteRetrying(remoteKillInvocation(forNode: node, at: location)) } - private static func startRemote(_ node: LoopNode, at location: RemoteProjectLocation) async { + private static func startRemote( + _ node: LoopNode, at location: RemoteProjectLocation, onlyAfterReboot: Bool = false + ) async { // A codespace that is down is redialed on the shared schedule, not on every sweep. guard await CodespaceDialBreaker.shared.permits(location) else { return } // A dial already in flight for this node is doing this job; a second one racing it @@ -2294,7 +2313,7 @@ public enum ZmxSessionLauncher { // as the local path: no UI here, the node's state stays honest, opening the loop // retries. if let ensure = remoteEnsureInvocation( - forNode: node, at: location, bridgeState: bridgeState + forNode: node, at: location, bridgeState: bridgeState, onlyAfterReboot: onlyAfterReboot ) { if await runRemoteRetrying(ensure) { await CodespaceDialBreaker.shared.record(location, reached: true) @@ -2304,6 +2323,105 @@ public enum ZmxSessionLauncher { await RemoteEnsureGate.shared.end(node.id, token: lease) } + /// Brings back the sessions of finished loops that a reboot of their remote host killed + /// (`GraphStore.ensureUnattendedSessionsAlive`). One probe dial per host names the + /// sessions that are missing *and* were last seen alive in an earlier boot; only those + /// are dialed again, each behind the same boot gate. + /// + /// The probe itself runs only when a pane of that host has redialed since the last + /// probe that answered (`redialStamp`): the one thing left dialing a finished loop's + /// host is its pane, and a healthy host has no pane redialing, so the sweep spends + /// nothing — a codespace dial spends the human's API quota (issue #480). + /// + /// `nodes` are already the quiet copies the store made (`GraphStore.rebootRestoreCopy`): + /// the create resumes the banked conversation, or opens on a note, never on the task. + static func restoreRebootedRemote( + _ nodes: [LoopNode], projectPath: String, gate: RebootProbeGate = .shared + ) async { + guard !nodes.isEmpty, let location = RemoteProjectLocation.parse(projectPath: projectPath) + else { return } + let asked = Date() + guard await gate.panesRedialed(location) else { return } + let name = { (node: LoopNode) in + SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + } + let probe = location.sshInvocation( + remoteCommand: location.remoteLoginShellCommand( + rebootProbeScript(forSessionNames: nodes.map(name)))) + let (succeeded, output) = await collectRemoteOutput(probe, location: location) + guard succeeded else { return } + await gate.probed(location, at: asked) + let rebooted = parseRebootProbe(output) + for node in nodes where rebooted.contains(name(node)) { + await startRemote(node, at: location, onlyAfterReboot: true) + } + } + + /// Touched by a remote pane's reconnect loop before every redial + /// (`SSHReconnectLoop`), and read by `RebootProbeGate`. Per host, not per loop: one + /// probe answers for every loop on it. + public static func redialStamp(for location: RemoteProjectLocation) -> URL { + SupportDirectory.url.appendingPathComponent("remote-redials", isDirectory: true) + .appendingPathComponent("\(location.host).redial") + } + + /// Whether a host's panes have redialed since its last answered probe — the only + /// state `restoreRebootedRemote` keeps. Stamps from before this daemon started count + /// once, so a pane left waiting across a daemon restart is still answered. + actor RebootProbeGate { + static let shared = RebootProbeGate() + + private let stampFor: @Sendable (RemoteProjectLocation) -> URL + private var probedAt: [String: Date] = [:] + + init( + stampFor: @escaping @Sendable (RemoteProjectLocation) -> URL = { + ZmxSessionLauncher.redialStamp(for: $0) + } + ) { + self.stampFor = stampFor + } + + func panesRedialed(_ location: RemoteProjectLocation) -> Bool { + guard + let touched = + (try? FileManager.default.attributesOfItem( + atPath: stampFor(location).path))?[.modificationDate] as? Date + else { return false } + guard let since = probedAt[location.host] else { return true } + return touched > since + } + + /// Recorded only for a probe that answered: one that failed leaves the redial + /// pending, so the host is probed once it is back even if every pane has paused. + func probed(_ location: RemoteProjectLocation, at date: Date) { + probedAt[location.host] = date + } + } + + /// Prints `rebooted ` for each session that is not running and whose boot marker + /// names a boot other than this one — the pane's reboot verdict, made for many sessions + /// in one shell. A host that cannot answer (no boot ID, `zmx` not up yet) prints nothing. + static func rebootProbeScript(forSessionNames names: [String]) -> String { + let quotedNames = names.map(RemoteProjectLocation.shellQuoted).joined(separator: " ") + return "\(RemoteBootMarker.captureFragment); [ -n \"$gc_boot\" ] || exit 0; " + + "gc_ls=$(zmx ls 2>/dev/null) || exit 0; gc_tab=$(printf '\\t'); " + + "for gc_n in \(quotedNames); do " + + "gc_last=$(cat \"$HOME/.graphcode/boots/$gc_n\" 2>/dev/null); " + + "[ -n \"$gc_last\" ] && [ \"$gc_last\" != \"$gc_boot\" ] || continue; " + + "printf '%s\\n' \"$gc_ls\" | grep -v -e \"${gc_tab}ended=\" -e \"${gc_tab}err=\" " + + "| grep -q \"name=$gc_n$gc_tab\" || printf 'rebooted %s\\n' \"$gc_n\"; done; exit 0" + } + + static func parseRebootProbe(_ output: String) -> Set { + Set( + output.split(whereSeparator: \.isNewline).compactMap { line in + let fields = line.split(whereSeparator: \.isWhitespace) + guard fields.count == 2, fields[0] == "rebooted" else { return nil } + return String(fields[1]) + }) + } + static func start(_ node: LoopNode, projectPath: String? = nil) async { if let projectPath, let remote = RemoteProjectLocation.parse(projectPath: projectPath) { await startRemote(node, at: remote) diff --git a/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift b/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift index 31448076..800df370 100644 --- a/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift +++ b/graphcode/Sources/Infrastructure/Ghostty/GhosttyTerminalView+Remote.swift @@ -45,14 +45,21 @@ extension GhosttyTerminalView { remoteCommand: location.remoteLoginShellCommand(script), interactive: true) let reconnect = location.sshCommandLine( remoteCommand: location.remoteLoginShellCommand(reconnectScript), interactive: true) + let stamp = ZmxSessionLauncher.redialStamp(for: location) + try? FileManager.default.createDirectory( + at: stamp.deletingLastPathComponent(), withIntermediateDirectories: true) guard location.isCodespace else { - return ["/bin/sh", "-c", SSHReconnectLoop.script(connect: connect, reconnect: reconnect)] + return [ + "/bin/sh", "-c", + SSHReconnectLoop.script(connect: connect, reconnect: reconnect, redialStamp: stamp.path), + ] } return [ "/bin/sh", "-c", SSHReconnectLoop.codespaceScript( connect: connect, reconnect: reconnect, - pauseMarker: CodespaceDialBreaker.reconnectMarker(for: location).path), + pauseMarker: CodespaceDialBreaker.reconnectMarker(for: location).path, + redialStamp: stamp.path), ] } diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index 9687860e..984b8f27 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -468,3 +468,188 @@ struct RemoteSessionResumeTests { #expect(started.value.isEmpty) } } + +/// A finished unattended loop across a remote reboot: its session comes back as the +/// conversation it was, never as another pass at the task. +@Suite +struct RemoteRebootRestoreTests { + private let location = RemoteProjectLocation( + user: "dev", host: "codespace", port: 2222, remotePath: "/workspaces/widget") + + private func goalNode() -> LoopNode { + LoopNode( + title: "Fix", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .running) + } + + @Test + func theSweepRestoresFinishedLoopsTheRebootKilledAndLeavesThemFinished() async { + // A finished unattended loop whose pane was still open dialed "waiting for graphcoded" + // forever after a codespace restart: the pane leaves every unattended loop to the + // daemon, and the sweep skipped every resolved one. + let started = LockIsolated<[LoopNode]>([]) + let restored = LockIsolated<[LoopNode]>([]) + let succeeded = LoopNode( + title: "Done", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .succeeded) + let stopped = LoopNode( + title: "Poll", loopType: .timeBased, triggerPrompt: "/loop 1h Check", state: .stopped) + var missingCLI = LoopNode( + title: "NoCLI", loopType: .goalBased, goal: GoalSpec(summary: "ship"), state: .stopped) + missingCLI.launchFailure = LaunchFailure(executable: "claude", backend: .claudeCode) + let turn = LoopNode( + title: "Read", loopType: .turnBased, checkDescription: "Sound?", state: .succeeded) + let running = goalNode() + let store = GraphStore( + graph: LoopGraph( + scope: LoopGraphScope(projectPath: location.projectPath, name: "widget"), + nodes: [succeeded, stopped, missingCLI, turn, running]), + onEnsureSession: { node, _ in started.withValue { $0.append(node) } }, + onRestoreRebootedSessions: { nodes, _ in restored.withValue { $0 += nodes } }) + + await store.ensureUnattendedSessionsAlive() + try? await Task.sleep(for: .milliseconds(200)) + + #expect(started.value.map(\.id) == [running.id]) + #expect(Set(restored.value.map(\.id)) == [succeeded.id, stopped.id]) + for copy in restored.value { + let prompt = copy.sessionPrompt(forProjectPath: location.projectPath) ?? "" + #expect(!prompt.contains("tests pass")) + #expect(!prompt.contains("/loop")) + } + let states = await store.graph.nodes.map(\.state) + #expect(states == [.succeeded, .stopped, .stopped, .succeeded, .running]) + } + + @Test + func theRestoreOnlyRunsWhenTheBootChangedAndResumesQuietly() throws { + let node = LoopNode( + title: "Done", loopType: .goalBased, goal: GoalSpec(summary: "tests pass"), + state: .succeeded) + let quiet = GraphStore.rebootRestoreCopy(of: node) + let invocation = try #require( + ZmxSessionLauncher.remoteEnsureInvocation( + forNode: quiet, at: location, onlyAfterReboot: true)) + let script = try #require(invocation.last) + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + #expect(script.contains(RemoteBootMarker.captureFragment)) + #expect(script.contains("cat \(RemoteBootMarker.markerExpression(forSessionName: name))")) + #expect(script.contains("[ \"$gc_boot\" != \"$gc_last\" ]")) + #expect(script.contains("'--resume'")) + #expect(!script.contains("tests pass")) + let gate = try #require(script.range(of: "[ \"$gc_boot\" != \"$gc_last\" ]")) + let run = try #require(script.range(of: "'run'")) + #expect(gate.lowerBound < run.lowerBound) + } + + @Test + func everyDaemonEnsureRecordsTheBootItSawTheSessionIn() throws { + // The marker is what tells a pane, and now the sweep, that a missing session died + // with the machine. Only a pane attach wrote it, so a session the daemon started and + // no pane ever joined had no marker, and one the daemon restored kept a stale one. + let node = goalNode() + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let invocation = try #require( + ZmxSessionLauncher.remoteEnsureInvocation(forNode: node, at: location)) + let script = try #require(invocation.last) + // Quote-free, so the login shell's re-quoting of the script cannot hide it. + let write = ">\(RemoteBootMarker.markerExpression(forSessionName: name))" + #expect(script.components(separatedBy: write).count == 3) + } + + @Test + func aDaemonKillForgetsTheBootSoTheSessionIsNotRestored() throws { + // Ended on purpose (a finished loop freed, a stop, a delete): after a later reboot + // the sweep must not bring it back, and a pane must read "ended", not "rebooted". + let node = goalNode() + let name = SurfaceRef(id: node.id, launchesClaudeCode: true).zmxSessionName + let script = try #require( + ZmxSessionLauncher.remoteKillInvocation(forNode: node, at: location).last) + #expect(script.contains("rm -f \(RemoteBootMarker.markerExpression(forSessionName: name))")) + } + + @Test + func theRebootProbeNamesOnlyMissingSessionsFromAnEarlierBoot() throws { + let home = FileManager.default.temporaryDirectory + .appendingPathComponent("reboot-probe-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: home) } + let bin = home.appendingPathComponent("bin", isDirectory: true) + let boots = home.appendingPathComponent(".graphcode/boots", isDirectory: true) + try FileManager.default.createDirectory(at: bin, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: boots, withIntermediateDirectories: true) + let zmx = bin.appendingPathComponent("zmx") + let listing = + " name=alive\\tpid=1\\tclients=0\\n name=husk\\tpid=2\\tended=5\\texit_code=0\\n" + try "#!/bin/sh\nprintf '\(listing)'\n".write(to: zmx, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: zmx.path) + let bootProbe = Process() + bootProbe.executableURL = URL(fileURLWithPath: "/bin/sh") + bootProbe.arguments = ["-c", RemoteBootMarker.captureFragment + "; printf %s \"$gc_boot\""] + let bootPipe = Pipe() + bootProbe.standardOutput = bootPipe + try bootProbe.run() + bootProbe.waitUntilExit() + let boot = String(decoding: bootPipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + try #require(!boot.isEmpty) + for (name, marker) in [ + ("rebooted", "an-earlier-boot"), ("sameboot", boot), ("alive", "an-earlier-boot"), + ("husk", "an-earlier-boot"), + ] { + try marker.write( + to: boots.appendingPathComponent(name), atomically: true, encoding: .utf8) + } + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = [ + "-c", + ZmxSessionLauncher.rebootProbeScript( + forSessionNames: ["rebooted", "sameboot", "alive", "husk", "unmarked"]), + ] + process.environment = ["HOME": home.path, "PATH": bin.path + ":/usr/bin:/bin:/usr/sbin"] + let pipe = Pipe() + process.standardOutput = pipe + try process.run() + process.waitUntilExit() + let output = String(decoding: pipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self) + + #expect(process.terminationStatus == 0) + #expect(ZmxSessionLauncher.parseRebootProbe(output) == ["rebooted", "husk"]) + } + + @Test + func aHealthyHostIsNeverProbed() async throws { + // The probe is a dial, and on a codespace a dial spends the human's API quota. Only + // a pane redialing its host is worth one; a host nobody is redialing costs nothing. + let stamp = FileManager.default.temporaryDirectory + .appendingPathComponent("redial-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(at: stamp) } + let gate = ZmxSessionLauncher.RebootProbeGate(stampFor: { _ in stamp }) + + #expect(await !gate.panesRedialed(location)) + + FileManager.default.createFile(atPath: stamp.path, contents: nil) + #expect(await gate.panesRedialed(location)) + + await gate.probed(location, at: Date().addingTimeInterval(1)) + #expect(await !gate.panesRedialed(location)) + + try FileManager.default.setAttributes( + [.modificationDate: Date().addingTimeInterval(5)], ofItemAtPath: stamp.path) + #expect(await gate.panesRedialed(location)) + } + + @Test(arguments: [false, true]) + func aPaneStampsItsHostBeforeEveryRedial(codespace: Bool) throws { + let script = + codespace + ? SSHReconnectLoop.codespaceScript( + connect: "CONNECT", reconnect: "RECONNECT", pauseMarker: "/tmp/p", + redialStamp: "/tmp/stamp") + : SSHReconnectLoop.script( + connect: "CONNECT", reconnect: "RECONNECT", redialStamp: "/tmp/stamp") + let touch = try #require(script.range(of: "touch '/tmp/stamp' 2>/dev/null; RECONNECT")) + let connect = try #require(script.range(of: "CONNECT")) + #expect(connect.upperBound <= touch.lowerBound) + } +}