From 774070aa78f339ec82c37b0bb6cee894ea473615 Mon Sep 17 00:00:00 2001 From: theBlanketBaron Date: Sat, 14 Nov 2026 05:50:26 -0500 Subject: [PATCH] RAL-179: a Chrome segfault at start gets the one stated retry; a teardown crash is said From the lamplighter's third crash on Murph (SIGSEGV through zypak's bus call, at teardown, seen only in coredumpctl). A 139/SIGSEGV at start is retried once like RAL-160's abort, and a second fails. A crash as Chrome is closed, and each Chrome core coredumpctl lists since the run began, is said on standard error and carried to the test's; observed only, the result stands. Co-Authored-By: Claude Opus 5.5 --- docs/roadmap.md | 11 ++++ tests/browser/sign-in.mjs | 76 ++++++++++++++++++++++++---- tests/web.rs | 103 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 10 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 5923b6f..3f66c63 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1394,3 +1394,14 @@ principles named were durable, accurate and reactive. Found in rally 0x7 by plus roadmap with the default driver and named it a conflict resolved, where a checkout with the attributes named none. Done when: each merge is remerged with its first parent's `.gitattributes` (`--attr-source`), the output says which attributes it used, and a test checks a checkout without the attributes gets the head's answer. +- [ ] **RAL-179: A Chrome segfault at start gets the one stated retry; a crash at teardown is said.** From the + lamplighter's third crash on Murph (2026-11-14 05:29:59, SIGSEGV in `/app/extra/chrome` through zypak's libsystemd + bus call, during ci-local's check of 90de3d3, which passed and logged nothing of it), assigned by the watchman. + Done when: in `tests/browser/sign-in.mjs`, a segfault at start (139, or SIGSEGV) gets RAL-160's one retry, said + on standard error and kept as `retried_after_abort` with Chrome's standard error, and a second crash fails, so a + Chrome that crashes every time still fails; a crash as Chrome is closed, seen in the launcher's exit, is said on + standard error with the end of what Chrome said, and each Chrome core `coredumpctl` lists since the run began is + said with its time, pid and signal, as this run's when the pid was one of its processes and otherwise as perhaps a + run alongside; observed only, never changing the result; the browser tests carry those lines to their own + standard error, so ci-local's log has them; with tests for a segfault retried, a second failing, and a teardown + crash with a stand-in coredumpctl. diff --git a/tests/browser/sign-in.mjs b/tests/browser/sign-in.mjs index 63625f6..340a337 100644 --- a/tests/browser/sign-in.mjs +++ b/tests/browser/sign-in.mjs @@ -59,8 +59,10 @@ const chromeArgs = [ const STDERR_KEPT = 4096; let browser = null; let stderrTail = ""; -// An abort at start that was tried once more: its status and what Chrome said. +// A crash at start that was tried once more: its status and what Chrome said. let firstAbort = null; +// When this run began, to the second, for coredumpctl's --since (RAL-179). +const startedAt = Math.floor(Date.now() / 1000); function launch() { stderrTail = ""; @@ -111,20 +113,31 @@ function finish(code, value) { if (code !== 0 && exitSeen && said.exit === undefined) said.exit = exitSeen; console.log(JSON.stringify(said)); } + // This run's Chrome processes as it ends, by their profile, so a core can be told as this run's. + const ours = ourPids(); let cleaned = false; - const done = () => { + const done = (closed) => { if (cleaned) return; cleaned = true; reap(); for (const dir of [profile, sandboxed]) { if (dir) try { rmSync(dir, { recursive: true, force: true }); } catch { /* still held */ } } + // RAL-179: Chrome's exit as it was closed came through the launcher; a crash there is said, + // and its core waited for a moment, since systemd writes it after the process has gone. + const crashed = closed && isCrash(closed.code, closed.signal); + if (crashed) { + process.stderr.write(`sign-in.mjs: Chrome ${crashWord(closed)} at teardown ` + + `(${closed.signal || closed.code}) at ${exitFacts(browser).at}, after the run had its answer; ` + + `the result stands; the end of what it said: ${JSON.stringify(stderrTail.slice(-600))} (RAL-179)\n`); + } + sayCores(ours, crashed ? 3000 : 0); process.exit(code); }; if (!browser || !browserEndpoint || browser.exitCode !== null) return done(); // The launcher exits once Chrome has closed, so remove the profile only then, or Chrome writes // it back while shutting down; give up waiting after a few seconds. - browser.once("exit", done); + browser.once("exit", (exitCode, exitSignal) => done({ code: exitCode, signal: exitSignal })); setTimeout(done, 5000); try { const closer = new WebSocket(browserEndpoint); @@ -144,7 +157,47 @@ const fail = (value) => { finish(1, value); return new Promise(() => {}); }; +// RAL-160, RAL-179: a crash at start is an abort (134, SIGABRT) or a segfault (139, SIGSEGV), both +// seen inside flatpak's zypak spawn helper on Murph. Under flatpak the launcher exits with 128 plus +// the signal; run directly, Chrome dies by the signal itself. const isAbort = (code, signal) => code === 134 || signal === "SIGABRT"; +const isCrash = (code, signal) => isAbort(code, signal) || code === 139 || signal === "SIGSEGV"; +const crashWord = ({ code, signal }) => (isAbort(code, signal) ? "aborted" : "crashed"); + +// RAL-179: a Chrome that crashes as it's closed, after the run has its answer, fails nothing and +// was seen nowhere (the lamplighter found one only in coredumpctl, 2026-11-14 05:29:59). So, at the +// end, this run's crashed Chrome is said on standard error: from the launcher's exit, if Chrome's +// own exit came through it; and from coredumpctl, each Chrome core dumped since this run began, +// said as this run's when its pid was one of this run's processes, and otherwise as perhaps a run +// alongside. Observed only: it never changes the run's result. +const coredumpctl = process.env.RALLY_BROWSER_COREDUMPCTL || "coredumpctl"; +function chromeCores() { + const out = spawnSync(coredumpctl, ["list", "--json=short", "--no-pager", "-q", `--since=@${startedAt}`], { + encoding: "utf8", timeout: 3000, + }); + if (out.status !== 0 || !out.stdout) return []; + try { + return JSON.parse(out.stdout).filter((core) => /chrom/.test(basename(String(core.exe || "")))); + } catch { return []; } +} +const signalNames = { 6: "SIGABRT", 11: "SIGSEGV", 4: "SIGILL", 7: "SIGBUS", 5: "SIGTRAP" }; +function sayCores(ours, wait) { + let cores = chromeCores(); + for (let waited = 0; !cores.length && waited < wait; waited += 250) { + pause(250); + cores = chromeCores(); + } + for (const core of cores) { + const at = new Date(Number(core.time) / 1000).toISOString().replace(/\.\d+Z$/, "Z"); + const whose = ours.has(Number(core.pid)) + ? "one of this run's processes" + : "not one of this run's processes, so perhaps a run alongside"; + process.stderr.write(`sign-in.mjs: a Chrome core was dumped at ${at}, pid ${core.pid} (${whose}), ` + + `${signalNames[core.sig] || "signal " + core.sig}, ${core.exe} (RAL-179)\n`); + } +} +const ourPids = () => new Set(spawnSync("pgrep", ["-f", "--", profileName]).stdout.toString() + .split(/\s+/).filter(Boolean).map(Number)); // What the machine looked like when Chrome exited, so a failure can be tied to its own records // (the lamplighter, RAL-160): the time to the second, which coredumpctl lists a crash by; the pid @@ -236,8 +289,10 @@ async function startOnce() { } // RAL-160: an abort at start (exit 134, or SIGABRT, before the code page has loaded) is tried once -// more, and said: on standard error now, and as `retried_after_abort` in what's printed. A second -// abort, or any other exit, fails with what Chrome said each time. Nothing more is papered over. +// more, and said: on standard error now, and as `retried_after_abort` in what's printed. RAL-179: +// a segfault at start (139, or SIGSEGV) as well. Once only: a second crash, or any other exit, +// fails with what Chrome said each time, so a Chrome that crashes every time still fails. +// Nothing more is papered over. const exitStatus = ({ code, signal }) => signal || code; let page; try { @@ -245,7 +300,7 @@ try { } catch (error) { if (!error.exit) throw error; const first = { status: exitStatus(error.exit), ...exitSeen, chrome_stderr: stderrTail }; - if (!isAbort(error.exit.code, error.exit.signal)) { + if (!isCrash(error.exit.code, error.exit.signal)) { await fail({ error: `the browser exited with ${first.status} before the page loaded` }); } reap(); @@ -253,7 +308,7 @@ try { if (dir) try { rmSync(dir, { recursive: true, force: true }); } catch { /* still held */ } } firstAbort = first; - process.stderr.write(`sign-in.mjs: Chrome aborted at start (${first.status}) at ${first.at}, pid ${first.pid} ` + process.stderr.write(`sign-in.mjs: Chrome ${crashWord(error.exit)} at start (${first.status}) at ${first.at}, pid ${first.pid} ` + `(${first.pid_is}), load ${first.loadavg}, document portal mounted: ${first.doc_portal_mounted}; ` + `trying once more (RAL-160). What it said:\n${first.chrome_stderr}\n`); try { @@ -262,9 +317,10 @@ try { if (!again.exit) throw again; const status = exitStatus(again.exit); await fail({ - error: isAbort(again.exit.code, again.exit.signal) - ? `Chrome aborted at start twice (${first.status}, then ${status})` - : `the browser exited with ${status} before the page loaded, after an abort at start`, + error: isCrash(again.exit.code, again.exit.signal) + ? `Chrome ${isAbort(error.exit.code, error.exit.signal) && isAbort(again.exit.code, again.exit.signal) + ? "aborted" : "crashed"} at start twice (${first.status}, then ${status})` + : `the browser exited with ${status} before the page loaded, after a crash at start`, }); } } diff --git a/tests/web.rs b/tests/web.rs index cc2a620..af1c923 100644 --- a/tests/web.rs +++ b/tests/web.rs @@ -1114,6 +1114,18 @@ fn run_browser( probe: Option<&str>, schemes: Option<&str>, window: Option<&str>, +) -> (bool, Value, String) { + run_browser_env(viewer, chrome, probe, schemes, window, &[]) +} + +/// [`run_browser`], with variables set for the script, such as a stand-in coredumpctl (RAL-179). +fn run_browser_env( + viewer: &Viewer, + chrome: &str, + probe: Option<&str>, + schemes: Option<&str>, + window: Option<&str>, + env: &[(&str, &str)], ) -> (bool, Value, String) { let url = format!("http://{}{}", viewer.address, login(viewer)); let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/browser/sign-in.mjs"); @@ -1133,6 +1145,7 @@ fn run_browser( if let Some(window) = window { command.env("RALLY_BROWSER_WINDOW", window); } + command.envs(env.iter().copied()); let output = command .output() .expect("node is needed to drive the browser"); @@ -1155,6 +1168,12 @@ fn run_browser( first["chrome_stderr"].as_str().unwrap_or("nothing") ); } + // RAL-179: a crash at teardown, and any Chrome core dumped during the run, reach the test's + // own standard error, so ci-local's log carries them even when the test passes. + for line in stderr.lines().filter(|line| line.contains("(RAL-179)")) { + use std::io::Write; + let _ = writeln!(std::io::stderr(), "browser test: {line}"); + } (output.status.success(), seen, stderr) } @@ -1225,6 +1244,90 @@ fn an_abort_at_start_is_tried_once_more_and_said() { nothing_of_the_browser_survives(&seen); } +/// RAL-179: a segfault at start (139, as flatpak's launcher gives Chrome's SIGSEGV) gets the same +/// one stated retry; a second fails, so a Chrome that crashes every time still fails the test. +#[test] +fn a_segfault_at_start_is_tried_once_more_and_a_second_fails() { + let Some(chrome) = browser() else { return }; + let viewer = start(); + let dir = tempfile::tempdir().unwrap(); + let once = chrome_that_fails(dir.path(), 1, 139, &chrome); + let (ok, seen, stderr) = run_browser(&viewer, &once, None, None, None); + assert!(ok, "{seen}\n{stderr}"); + assert_eq!(seen["retried_after_abort"]["status"], 139, "{seen}"); + assert!( + stderr.contains("Chrome crashed at start (139) at ") && stderr.contains("trying once more"), + "{stderr}" + ); + assert_eq!(seen["after"]["status"], 200, "the retry signed in: {seen}"); + nothing_of_the_browser_survives(&seen); + + let other = tempfile::tempdir().unwrap(); + let always = chrome_that_fails(other.path(), 99, 139, &chrome); + let (ok, seen, _) = run_browser(&viewer, &always, None, None, None); + assert!(!ok, "{seen}"); + assert_eq!( + seen["error"], "Chrome crashed at start twice (139, then 139)", + "{seen}" + ); + assert_eq!( + std::fs::read_to_string(other.path().join("runs")) + .unwrap() + .trim(), + "2", + "tried once more, and only once" + ); + nothing_of_the_browser_survives(&seen); +} + +/// RAL-179: Chrome crashing as it's closed, after the run has its answer, is said on standard +/// error, with each Chrome core coredumpctl lists since the run began; the result stands. +#[test] +fn a_crash_at_teardown_is_said_and_the_result_stands() { + use std::os::unix::fs::PermissionsExt; + let Some(chrome) = browser() else { return }; + let viewer = start(); + let dir = tempfile::tempdir().unwrap(); + // Runs the real browser, then exits as a launcher does whose Chrome segfaulted on closing. + let wrapper = dir.path().join("crashes-on-close"); + std::fs::write(&wrapper, "#!/bin/sh\n\"$@\"\nexit 139\n").unwrap(); + std::fs::set_permissions(&wrapper, std::fs::Permissions::from_mode(0o755)).unwrap(); + // A stand-in coredumpctl: one Chrome core of another process, and a core that isn't Chrome's. + let cores = dir.path().join("coredumpctl"); + std::fs::write( + &cores, + "#!/bin/sh\necho '[{\"time\":1794652199536351,\"pid\":1,\"sig\":11,\"exe\":\"/app/extra/chrome\"},\ + {\"time\":1794652199536351,\"pid\":2,\"sig\":11,\"exe\":\"/usr/bin/node\"}]'\n", + ) + .unwrap(); + std::fs::set_permissions(&cores, std::fs::Permissions::from_mode(0o755)).unwrap(); + let command = format!("{} {chrome}", wrapper.display()); + let (ok, seen, stderr) = run_browser_env( + &viewer, + &command, + None, + None, + None, + &[("RALLY_BROWSER_COREDUMPCTL", cores.to_str().unwrap())], + ); + assert!(ok, "the run's answer stands: {seen}\n{stderr}"); + assert_eq!(seen["after"]["status"], 200, "{seen}"); + assert!( + stderr.contains("Chrome crashed at teardown (139) at ") + && stderr.contains("the result stands; the end of what it said: \""), + "{stderr}" + ); + assert!( + stderr.contains( + "a Chrome core was dumped at 2026-11-14T10:29:59Z, pid 1 (not one of this run's \ + processes, so perhaps a run alongside), SIGSEGV, /app/extra/chrome (RAL-179)" + ), + "{stderr}" + ); + assert!(!stderr.contains("/usr/bin/node"), "{stderr}"); + nothing_of_the_browser_survives(&seen); +} + /// RAL-160: a second abort fails, with what Chrome said both times; and an exit at start that /// isn't an abort isn't tried again, and says what Chrome said. #[test]