RAL-179: a Chrome segfault at start gets the one stated retry; a teardown crash is said

docs/roadmap.md

@@ -1405,3 +1405,14 @@ principles named were durable, accurate and reactive. Found in rally 0x7 by plus
   roadmap with the default driver and named it a conflict resolved, where a checkout with the attributes named
   none. Done when: each merge is remerged with its first parent's `.gitattributes` (`--attr-source`), the output
   says which attributes it used, and a test checks a checkout without the attributes gets the head's answer.
+- [ ] **RAL-179: A Chrome segfault at start gets the one stated retry; a crash at teardown is said.** From the
+  lamplighter's third crash on Murph (2026-11-14 05:29:59, SIGSEGV in `/app/extra/chrome` through zypak's libsystemd
+  bus call, during ci-local's check of 90de3d3, which passed and logged nothing of it), assigned by the watchman.
+  Done when: in `tests/browser/sign-in.mjs`, a segfault at start (139, or SIGSEGV) gets RAL-160's one retry, said
+  on standard error and kept as `retried_after_abort` with Chrome's standard error, and a second crash fails, so a
+  Chrome that crashes every time still fails; a crash as Chrome is closed, seen in the launcher's exit, is said on
+  standard error with the end of what Chrome said, and each Chrome core `coredumpctl` lists since the run began is
+  said with its time, pid and signal, as this run's when the pid was one of its processes and otherwise as perhaps a
+  run alongside; observed only, never changing the result; the browser tests carry those lines to their own
+  standard error, so ci-local's log has them; with tests for a segfault retried, a second failing, and a teardown
+  crash with a stand-in coredumpctl.

tests/browser/sign-in.mjs

@@ -59,8 +59,10 @@ const chromeArgs = [
 const STDERR_KEPT = 4096;
 let browser = null;
 let stderrTail = "";
-// An abort at start that was tried once more: its status and what Chrome said.
+// A crash at start that was tried once more: its status and what Chrome said.
 let firstAbort = null;
+// When this run began, to the second, for coredumpctl's --since (RAL-179).
+const startedAt = Math.floor(Date.now() / 1000);
 
 function launch() {
   stderrTail = "";
@@ -111,20 +113,31 @@ function finish(code, value) {
     if (code !== 0 && exitSeen && said.exit === undefined) said.exit = exitSeen;
     console.log(JSON.stringify(said));
   }
+  // This run's Chrome processes as it ends, by their profile, so a core can be told as this run's.
+  const ours = ourPids();
   let cleaned = false;
-  const done = () => {
+  const done = (closed) => {
     if (cleaned) return;
     cleaned = true;
     reap();
     for (const dir of [profile, sandboxed]) {
       if (dir) try { rmSync(dir, { recursive: true, force: true }); } catch { /* still held */ }
     }
+    // RAL-179: Chrome's exit as it was closed came through the launcher; a crash there is said,
+    // and its core waited for a moment, since systemd writes it after the process has gone.
+    const crashed = closed && isCrash(closed.code, closed.signal);
+    if (crashed) {
+      process.stderr.write(`sign-in.mjs: Chrome ${crashWord(closed)} at teardown `
+        + `(${closed.signal || closed.code}) at ${exitFacts(browser).at}, after the run had its answer; `
+        + `the result stands; the end of what it said: ${JSON.stringify(stderrTail.slice(-600))} (RAL-179)\n`);
+    }
+    sayCores(ours, crashed ? 3000 : 0);
     process.exit(code);
   };
   if (!browser || !browserEndpoint || browser.exitCode !== null) return done();
   // The launcher exits once Chrome has closed, so remove the profile only then, or Chrome writes
   // it back while shutting down; give up waiting after a few seconds.
-  browser.once("exit", done);
+  browser.once("exit", (exitCode, exitSignal) => done({ code: exitCode, signal: exitSignal }));
   setTimeout(done, 5000);
   try {
     const closer = new WebSocket(browserEndpoint);
@@ -144,7 +157,47 @@ const fail = (value) => {
   finish(1, value);
   return new Promise(() => {});
 };
+// RAL-160, RAL-179: a crash at start is an abort (134, SIGABRT) or a segfault (139, SIGSEGV), both
+// seen inside flatpak's zypak spawn helper on Murph. Under flatpak the launcher exits with 128 plus
+// the signal; run directly, Chrome dies by the signal itself.
 const isAbort = (code, signal) => code === 134 || signal === "SIGABRT";
+const isCrash = (code, signal) => isAbort(code, signal) || code === 139 || signal === "SIGSEGV";
+const crashWord = ({ code, signal }) => (isAbort(code, signal) ? "aborted" : "crashed");
+
+// RAL-179: a Chrome that crashes as it's closed, after the run has its answer, fails nothing and
+// was seen nowhere (the lamplighter found one only in coredumpctl, 2026-11-14 05:29:59). So, at the
+// end, this run's crashed Chrome is said on standard error: from the launcher's exit, if Chrome's
+// own exit came through it; and from coredumpctl, each Chrome core dumped since this run began,
+// said as this run's when its pid was one of this run's processes, and otherwise as perhaps a run
+// alongside. Observed only: it never changes the run's result.
+const coredumpctl = process.env.RALLY_BROWSER_COREDUMPCTL || "coredumpctl";
+function chromeCores() {
+  const out = spawnSync(coredumpctl, ["list", "--json=short", "--no-pager", "-q", `--since=@${startedAt}`], {
+    encoding: "utf8", timeout: 3000,
+  });
+  if (out.status !== 0 || !out.stdout) return [];
+  try {
+    return JSON.parse(out.stdout).filter((core) => /chrom/.test(basename(String(core.exe || ""))));
+  } catch { return []; }
+}
+const signalNames = { 6: "SIGABRT", 11: "SIGSEGV", 4: "SIGILL", 7: "SIGBUS", 5: "SIGTRAP" };
+function sayCores(ours, wait) {
+  let cores = chromeCores();
+  for (let waited = 0; !cores.length && waited < wait; waited += 250) {
+    pause(250);
+    cores = chromeCores();
+  }
+  for (const core of cores) {
+    const at = new Date(Number(core.time) / 1000).toISOString().replace(/\.\d+Z$/, "Z");
+    const whose = ours.has(Number(core.pid))
+      ? "one of this run's processes"
+      : "not one of this run's processes, so perhaps a run alongside";
+    process.stderr.write(`sign-in.mjs: a Chrome core was dumped at ${at}, pid ${core.pid} (${whose}), `
+      + `${signalNames[core.sig] || "signal " + core.sig}, ${core.exe} (RAL-179)\n`);
+  }
+}
+const ourPids = () => new Set(spawnSync("pgrep", ["-f", "--", profileName]).stdout.toString()
+  .split(/\s+/).filter(Boolean).map(Number));
 
 // What the machine looked like when Chrome exited, so a failure can be tied to its own records
 // (the lamplighter, RAL-160): the time to the second, which coredumpctl lists a crash by; the pid
@@ -236,24 +289,26 @@ async function startOnce() {
 }
 
 // RAL-160: an abort at start (exit 134, or SIGABRT, before the code page has loaded) is tried once
-// more, and said: on standard error now, and as `retried_after_abort` in what's printed. A second
-// abort, or any other exit, fails with what Chrome said each time. Nothing more is papered over.
+// more, and said: on standard error now, and as `retried_after_abort` in what's printed. RAL-179:
+// a segfault at start (139, or SIGSEGV) as well. Once only: a second crash, or any other exit,
+// fails with what Chrome said each time, so a Chrome that crashes every time still fails.
+// Nothing more is papered over.
 const exitStatus = ({ code, signal }) => signal || code;
 let page;
 try {
   page = await startOnce();
 } catch (error) {
   if (!error.exit) throw error;
   const first = { status: exitStatus(error.exit), ...exitSeen, chrome_stderr: stderrTail };
-  if (!isAbort(error.exit.code, error.exit.signal)) {
+  if (!isCrash(error.exit.code, error.exit.signal)) {
     await fail({ error: `the browser exited with ${first.status} before the page loaded` });
   }
   reap();
   for (const dir of [profile, sandboxed]) {
     if (dir) try { rmSync(dir, { recursive: true, force: true }); } catch { /* still held */ }
   }
   firstAbort = first;
-  process.stderr.write(`sign-in.mjs: Chrome aborted at start (${first.status}) at ${first.at}, pid ${first.pid} `
+  process.stderr.write(`sign-in.mjs: Chrome ${crashWord(error.exit)} at start (${first.status}) at ${first.at}, pid ${first.pid} `
     + `(${first.pid_is}), load ${first.loadavg}, document portal mounted: ${first.doc_portal_mounted}; `
     + `trying once more (RAL-160). What it said:\n${first.chrome_stderr}\n`);
   try {
@@ -262,9 +317,10 @@ try {
     if (!again.exit) throw again;
     const status = exitStatus(again.exit);
     await fail({
-      error: isAbort(again.exit.code, again.exit.signal)
-        ? `Chrome aborted at start twice (${first.status}, then ${status})`
-        : `the browser exited with ${status} before the page loaded, after an abort at start`,
+      error: isCrash(again.exit.code, again.exit.signal)
+        ? `Chrome ${isAbort(error.exit.code, error.exit.signal) && isAbort(again.exit.code, again.exit.signal)
+          ? "aborted" : "crashed"} at start twice (${first.status}, then ${status})`
+        : `the browser exited with ${status} before the page loaded, after a crash at start`,
     });
   }
 }

tests/web.rs

@@ -1114,6 +1114,18 @@ fn run_browser(
     probe: Option<&str>,
     schemes: Option<&str>,
     window: Option<&str>,
+) -> (bool, Value, String) {
+    run_browser_env(viewer, chrome, probe, schemes, window, &[])
+}
+
+/// [`run_browser`], with variables set for the script, such as a stand-in coredumpctl (RAL-179).
+fn run_browser_env(
+    viewer: &Viewer,
+    chrome: &str,
+    probe: Option<&str>,
+    schemes: Option<&str>,
+    window: Option<&str>,
+    env: &[(&str, &str)],
 ) -> (bool, Value, String) {
     let url = format!("http://{}{}", viewer.address, login(viewer));
     let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/browser/sign-in.mjs");
@@ -1133,6 +1145,7 @@ fn run_browser(
     if let Some(window) = window {
         command.env("RALLY_BROWSER_WINDOW", window);
     }
+    command.envs(env.iter().copied());
     let output = command
         .output()
         .expect("node is needed to drive the browser");
@@ -1155,6 +1168,12 @@ fn run_browser(
             first["chrome_stderr"].as_str().unwrap_or("nothing")
         );
     }
+    // RAL-179: a crash at teardown, and any Chrome core dumped during the run, reach the test's
+    // own standard error, so ci-local's log carries them even when the test passes.
+    for line in stderr.lines().filter(|line| line.contains("(RAL-179)")) {
+        use std::io::Write;
+        let _ = writeln!(std::io::stderr(), "browser test: {line}");
+    }
     (output.status.success(), seen, stderr)
 }
 
@@ -1225,6 +1244,90 @@ fn an_abort_at_start_is_tried_once_more_and_said() {
     nothing_of_the_browser_survives(&seen);
 }
 
+/// RAL-179: a segfault at start (139, as flatpak's launcher gives Chrome's SIGSEGV) gets the same
+/// one stated retry; a second fails, so a Chrome that crashes every time still fails the test.
+#[test]
+fn a_segfault_at_start_is_tried_once_more_and_a_second_fails() {
+    let Some(chrome) = browser() else { return };
+    let viewer = start();
+    let dir = tempfile::tempdir().unwrap();
+    let once = chrome_that_fails(dir.path(), 1, 139, &chrome);
+    let (ok, seen, stderr) = run_browser(&viewer, &once, None, None, None);
+    assert!(ok, "{seen}\n{stderr}");
+    assert_eq!(seen["retried_after_abort"]["status"], 139, "{seen}");
+    assert!(
+        stderr.contains("Chrome crashed at start (139) at ") && stderr.contains("trying once more"),
+        "{stderr}"
+    );
+    assert_eq!(seen["after"]["status"], 200, "the retry signed in: {seen}");
+    nothing_of_the_browser_survives(&seen);
+
+    let other = tempfile::tempdir().unwrap();
+    let always = chrome_that_fails(other.path(), 99, 139, &chrome);
+    let (ok, seen, _) = run_browser(&viewer, &always, None, None, None);
+    assert!(!ok, "{seen}");
+    assert_eq!(
+        seen["error"], "Chrome crashed at start twice (139, then 139)",
+        "{seen}"
+    );
+    assert_eq!(
+        std::fs::read_to_string(other.path().join("runs"))
+            .unwrap()
+            .trim(),
+        "2",
+        "tried once more, and only once"
+    );
+    nothing_of_the_browser_survives(&seen);
+}
+
+/// RAL-179: Chrome crashing as it's closed, after the run has its answer, is said on standard
+/// error, with each Chrome core coredumpctl lists since the run began; the result stands.
+#[test]
+fn a_crash_at_teardown_is_said_and_the_result_stands() {
+    use std::os::unix::fs::PermissionsExt;
+    let Some(chrome) = browser() else { return };
+    let viewer = start();
+    let dir = tempfile::tempdir().unwrap();
+    // Runs the real browser, then exits as a launcher does whose Chrome segfaulted on closing.
+    let wrapper = dir.path().join("crashes-on-close");
+    std::fs::write(&wrapper, "#!/bin/sh\n\"$@\"\nexit 139\n").unwrap();
+    std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap();
+    // A stand-in coredumpctl: one Chrome core of another process, and a core that isn't Chrome's.
+    let cores = dir.path().join("coredumpctl");
+    std::fs::write(
+        &cores,
+        "#!/bin/sh\necho '[{\"time\":1794652199536351,\"pid\":1,\"sig\":11,\"exe\":\"/app/extra/chrome\"},\
+         {\"time\":1794652199536351,\"pid\":2,\"sig\":11,\"exe\":\"/usr/bin/node\"}]'\n",
+    )
+    .unwrap();
+    std::fs::set_permissions(&cores, std::fs::Permissions::from_mode(0o755)).unwrap();
+    let command = format!("{} {chrome}", wrapper.display());
+    let (ok, seen, stderr) = run_browser_env(
+        &viewer,
+        &command,
+        None,
+        None,
+        None,
+        &[("RALLY_BROWSER_COREDUMPCTL", cores.to_str().unwrap())],
+    );
+    assert!(ok, "the run's answer stands: {seen}\n{stderr}");
+    assert_eq!(seen["after"]["status"], 200, "{seen}");
+    assert!(
+        stderr.contains("Chrome crashed at teardown (139) at ")
+            && stderr.contains("the result stands; the end of what it said: \""),
+        "{stderr}"
+    );
+    assert!(
+        stderr.contains(
+            "a Chrome core was dumped at 2026-11-14T10:29:59Z, pid 1 (not one of this run's \
+             processes, so perhaps a run alongside), SIGSEGV, /app/extra/chrome (RAL-179)"
+        ),
+        "{stderr}"
+    );
+    assert!(!stderr.contains("/usr/bin/node"), "{stderr}");
+    nothing_of_the_browser_survives(&seen);
+}
+
 /// RAL-160: a second abort fails, with what Chrome said both times; and an exit at start that
 /// isn't an abort isn't tried again, and says what Chrome said.
 #[test]