Skip to content

Commit c287d77

Browse files
committed
Retry on transient CDP disconnects
Add a --retries option to the run command and implement retry logic in TestRunner to re-run tests when failures are caused by transient CDP disconnects (CdpDisconnectedException or MotusTargetClosedException, including when wrapped). Emit a [RETRY] notice for retried attempts. Make ExecuteTestAsync return failure info so retries can inspect exception chains. Make CdpTransport mark a connection as disconnected so SendRawAsync fails fast after the receive loop observes a disconnect. Improve coverage handling: enable/disable Debugger on sessions and switch to Debugger.getScriptSource for retrieving script source; add corresponding CDP JSON types and BiDi translations. Improve CoverageReporterFactory errors and add tests covering reporter validation, retry behaviour, and transport fast-fail.
1 parent 150ad8b commit c287d77

10 files changed

Lines changed: 444 additions & 32 deletions

File tree

src/Motus.Cli/Commands/RunCommand.cs

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ public static Command Build()
3030
Description = "Enable coverage reporting: console | html:<dir> | cobertura:<path>. Repeat the flag for multiple formats (e.g. --coverage console --coverage html:./out).",
3131
Arity = ArgumentArity.ZeroOrMore,
3232
};
33+
var retriesOpt = new Option<int>("--retries")
34+
{
35+
Description = "Re-run a failing test up to N additional times when the failure is a transient CDP disconnect. Default: 0 (no retries).",
36+
DefaultValueFactory = _ => 0,
37+
};
3338

3439
var cmd = new Command("run", "Discover and execute tests from assemblies")
3540
{
@@ -41,6 +46,7 @@ public static Command Build()
4146
a11yOpt,
4247
perfBudgetOpt,
4348
coverageOpt,
49+
retriesOpt,
4450
};
4551

4652
cmd.SetAction(async (parseResult, ct) =>
@@ -54,6 +60,7 @@ public static Command Build()
5460
var perfBudget = parseResult.GetValue(perfBudgetOpt);
5561
var coverageSpecs = parseResult.GetValue(coverageOpt);
5662
var coverageRequested = parseResult.GetResult(coverageOpt) is not null;
63+
var retries = parseResult.GetValue(retriesOpt);
5764

5865
if (a11yMode is not null)
5966
{
@@ -88,15 +95,27 @@ public static Command Build()
8895
: int.Parse(workersSpec);
8996

9097
var reporter = ReporterFactory.Create(reporterSpecs);
91-
var coverageReporters = coverageRequested
92-
? CoverageReporterFactory.Create(coverageSpecs)
93-
: null;
98+
99+
IReadOnlyList<Motus.Abstractions.ICoverageReporter>? coverageReporters = null;
100+
if (coverageRequested)
101+
{
102+
try
103+
{
104+
coverageReporters = CoverageReporterFactory.Create(coverageSpecs);
105+
}
106+
catch (ArgumentException ex)
107+
{
108+
// Surface bad --coverage specs as a clean error, not a stack trace.
109+
Console.Error.WriteLine($"Error: {ex.Message}");
110+
return 1;
111+
}
112+
}
94113

95114
var discovery = new TestDiscovery();
96115
var tests = discovery.Discover(assemblies, filter);
97116

98117
var runner = new TestRunner(workers);
99-
var runResult = await runner.RunAsync(tests, reporter, a11yMode, perfBudget, coverageReporters);
118+
var runResult = await runner.RunAsync(tests, reporter, a11yMode, perfBudget, coverageReporters, retries);
100119

101120
return (runResult.Failed > 0 || runResult.CoverageThresholdsFailed) ? 1 : 0;
102121
});

src/Motus.Cli/Services/Reporters/CoverageReporterFactory.cs

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ public static IReadOnlyList<ICoverageReporter> Create(IReadOnlyList<string>? spe
2020
return result;
2121
}
2222

23+
private const string SupportedFormats =
24+
"Supported formats: console | html:<dir> | cobertura:<path>";
25+
2326
private static ICoverageReporter CreateSingle(string spec)
2427
{
2528
var colonIdx = spec.IndexOf(':');
@@ -28,18 +31,33 @@ private static ICoverageReporter CreateSingle(string spec)
2831
return spec.ToLowerInvariant() switch
2932
{
3033
"console" => new CoverageConsoleReporter(),
31-
_ => throw new ArgumentException($"Unknown coverage format: {spec}"),
34+
"html" => throw new ArgumentException(
35+
"Coverage format 'html' requires an output directory. " +
36+
"Use --coverage html:<dir> (e.g. --coverage html:./coverage)."),
37+
"cobertura" => throw new ArgumentException(
38+
"Coverage format 'cobertura' requires an output file path. " +
39+
"Use --coverage cobertura:<path> (e.g. --coverage cobertura:./coverage.xml)."),
40+
_ => throw new ArgumentException(
41+
$"Unknown coverage format '{spec}'. {SupportedFormats}."),
3242
};
3343
}
3444

3545
var format = spec[..colonIdx].ToLowerInvariant();
3646
var path = spec[(colonIdx + 1)..];
3747

48+
if (string.IsNullOrWhiteSpace(path))
49+
throw new ArgumentException(
50+
$"Coverage format '{format}' was given an empty target. " +
51+
$"Use --coverage {format}:<{(format == "html" ? "dir" : "path")}>.");
52+
3853
return format switch
3954
{
4055
"html" => new CoverageHtmlReporter(path),
4156
"cobertura" => new CoberturaReporter(path),
42-
_ => throw new ArgumentException($"Unknown coverage format: {format}"),
57+
"console" => throw new ArgumentException(
58+
"Coverage format 'console' does not take a target. Use --coverage console."),
59+
_ => throw new ArgumentException(
60+
$"Unknown coverage format '{format}'. {SupportedFormats}."),
4361
};
4462
}
4563
}

src/Motus.Cli/Services/TestRunner.cs

Lines changed: 79 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,11 @@ public async Task<TestRunResult> RunAsync(
3131
IReporter reporter,
3232
string? a11yMode,
3333
bool enforcePerfBudget,
34-
IReadOnlyList<ICoverageReporter>? coverageReporters)
34+
IReadOnlyList<ICoverageReporter>? coverageReporters,
35+
int maxRetries = 0)
3536
{
37+
if (maxRetries < 0)
38+
maxRetries = 0;
3639
var suiteName = tests.Count > 0
3740
? tests[0].TestClass.Assembly.GetName().Name ?? "Motus Tests"
3841
: "Motus Tests";
@@ -78,20 +81,42 @@ public async Task<TestRunResult> RunAsync(
7881
var testInfo = new TestInfo(test.FullName, suiteName);
7982
await reporter.OnTestStartAsync(testInfo);
8083

81-
AccessibilityViolationSink.Begin();
82-
PerformanceMetricsSink.Begin();
83-
CoverageSink.Begin();
84-
8584
CliTestResult result;
8685

8786
if (failedAssemblies.Contains(test.TestClass.Assembly))
8887
{
88+
AccessibilityViolationSink.Begin();
89+
PerformanceMetricsSink.Begin();
90+
CoverageSink.Begin();
8991
result = new CliTestResult(test.FullName, false, TimeSpan.Zero,
9092
"AssemblyInitialize failed", null);
9193
}
9294
else
9395
{
94-
result = await ExecuteTestAsync(test);
96+
var totalAttempts = maxRetries + 1;
97+
var attempt = 0;
98+
Exception? failure;
99+
while (true)
100+
{
101+
attempt++;
102+
103+
AccessibilityViolationSink.Begin();
104+
PerformanceMetricsSink.Begin();
105+
CoverageSink.Begin();
106+
107+
(result, failure) = await ExecuteTestAsync(test);
108+
109+
if (result.Passed || attempt >= totalAttempts || !IsTransientCdpFailure(failure))
110+
break;
111+
112+
// Discard sink data captured during the failed attempt; a clean
113+
// Begin() runs at the top of the next iteration.
114+
AccessibilityViolationSink.End();
115+
PerformanceMetricsSink.End();
116+
CoverageSink.End();
117+
118+
WriteRetryNotice(test.FullName, attempt + 1, totalAttempts);
119+
}
95120
}
96121

97122
var violations = AccessibilityViolationSink.End();
@@ -274,7 +299,47 @@ public async Task<TestRunResult> RunAsync(
274299
return runResult;
275300
}
276301

277-
private async Task<CliTestResult> ExecuteTestAsync(DiscoveredTest test)
302+
/// <summary>
303+
/// Emits a [RETRY] notice in the same shape (and color, when stderr is a TTY)
304+
/// as the [PASS]/[FAIL] lines emitted by ConsoleReporter, so the retry log
305+
/// reads as part of the test stream instead of free-form debug output.
306+
/// </summary>
307+
private static void WriteRetryNotice(string testName, int attempt, int totalAttempts)
308+
{
309+
const string Yellow = "\x1b[33m";
310+
const string Gray = "\x1b[90m";
311+
const string Reset = "\x1b[0m";
312+
313+
var useColor = !Console.IsErrorRedirected;
314+
if (useColor)
315+
{
316+
Console.Error.WriteLine(
317+
$" {Yellow}[RETRY]{Reset} {testName} {Gray}(CDP disconnect, attempt {attempt}/{totalAttempts}){Reset}");
318+
}
319+
else
320+
{
321+
Console.Error.WriteLine(
322+
$" [RETRY] {testName} (CDP disconnect, attempt {attempt}/{totalAttempts})");
323+
}
324+
}
325+
326+
/// <summary>
327+
/// Returns true when the exception chain contains a transient CDP failure that
328+
/// is safe to recover from by re-running the entire test (which rebuilds a fresh
329+
/// browser context and WebSocket). Used to drive --retries.
330+
/// </summary>
331+
private static bool IsTransientCdpFailure(Exception? ex)
332+
{
333+
while (ex is not null)
334+
{
335+
if (ex is CdpDisconnectedException || ex is Abstractions.MotusTargetClosedException)
336+
return true;
337+
ex = ex.InnerException;
338+
}
339+
return false;
340+
}
341+
342+
private async Task<(CliTestResult Result, Exception? Failure)> ExecuteTestAsync(DiscoveredTest test)
278343
{
279344
var testSw = Stopwatch.StartNew();
280345
object? instance = null;
@@ -299,17 +364,21 @@ private async Task<CliTestResult> ExecuteTestAsync(DiscoveredTest test)
299364
await task;
300365

301366
testSw.Stop();
302-
return new CliTestResult(test.FullName, true, testSw.Elapsed, null, null);
367+
return (new CliTestResult(test.FullName, true, testSw.Elapsed, null, null), null);
303368
}
304369
catch (TargetInvocationException ex) when (ex.InnerException is not null)
305370
{
306371
testSw.Stop();
307-
return new CliTestResult(test.FullName, false, testSw.Elapsed, ex.InnerException.Message, ex.InnerException.StackTrace);
372+
return (
373+
new CliTestResult(test.FullName, false, testSw.Elapsed, ex.InnerException.Message, ex.InnerException.StackTrace),
374+
ex.InnerException);
308375
}
309376
catch (Exception ex)
310377
{
311378
testSw.Stop();
312-
return new CliTestResult(test.FullName, false, testSw.Elapsed, ex.Message, ex.StackTrace);
379+
return (
380+
new CliTestResult(test.FullName, false, testSw.Elapsed, ex.Message, ex.StackTrace),
381+
ex);
313382
}
314383
finally
315384
{

src/Motus/Coverage/CoverageCollector.cs

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,16 @@ await session.SendAsync(
6060
CdpJsonContext.Default.ProfilerEnableResult,
6161
CancellationToken.None).ConfigureAwait(false);
6262

63+
// Debugger.enable is required for Debugger.getScriptSource at teardown.
64+
// Without it, getScriptSource returns no source and every file ends up
65+
// 0/0 lines.
66+
await session.SendAsync(
67+
"Debugger.enable",
68+
new DebuggerEnableParams(),
69+
CdpJsonContext.Default.DebuggerEnableParams,
70+
CdpJsonContext.Default.DebuggerEnableResult,
71+
CancellationToken.None).ConfigureAwait(false);
72+
6373
await session.SendAsync(
6474
"Profiler.startPreciseCoverage",
6575
new ProfilerStartPreciseCoverageParams(
@@ -147,10 +157,10 @@ public async Task OnPageClosedAsync(IPage page)
147157
try
148158
{
149159
var src = await session.SendAsync(
150-
"Profiler.getScriptSource",
151-
new ProfilerGetScriptSourceParams(s.ScriptId),
152-
CdpJsonContext.Default.ProfilerGetScriptSourceParams,
153-
CdpJsonContext.Default.ProfilerGetScriptSourceResult,
160+
"Debugger.getScriptSource",
161+
new DebuggerGetScriptSourceParams(s.ScriptId),
162+
CdpJsonContext.Default.DebuggerGetScriptSourceParams,
163+
CdpJsonContext.Default.DebuggerGetScriptSourceResult,
154164
CancellationToken.None).ConfigureAwait(false);
155165
source = src.ScriptSource ?? string.Empty;
156166
}
@@ -189,6 +199,18 @@ await session.SendAsync(
189199
{
190200
// Best-effort: stopping the profiler is not critical
191201
}
202+
203+
try
204+
{
205+
await session.SendAsync(
206+
"Debugger.disable",
207+
CdpJsonContext.Default.DebuggerDisableResult,
208+
CancellationToken.None).ConfigureAwait(false);
209+
}
210+
catch
211+
{
212+
// Best-effort: disabling the debugger is not critical
213+
}
192214
}
193215
catch (Exception ex)
194216
{

src/Motus/Protocol/CdpJsonContext.cs

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,12 @@ namespace Motus;
221221
[JsonSerializable(typeof(ProfilerScriptCoverage))]
222222
[JsonSerializable(typeof(ProfilerFunctionCoverage))]
223223
[JsonSerializable(typeof(ProfilerCoverageRange))]
224-
[JsonSerializable(typeof(ProfilerGetScriptSourceParams))]
225-
[JsonSerializable(typeof(ProfilerGetScriptSourceResult))]
224+
// --- Debugger domain ---
225+
[JsonSerializable(typeof(DebuggerEnableParams))]
226+
[JsonSerializable(typeof(DebuggerEnableResult))]
227+
[JsonSerializable(typeof(DebuggerDisableResult))]
228+
[JsonSerializable(typeof(DebuggerGetScriptSourceParams))]
229+
[JsonSerializable(typeof(DebuggerGetScriptSourceResult))]
226230
// --- Source map document ---
227231
[JsonSerializable(typeof(SourceMapJsonDto))]
228232
// --- DOMDebugger domain ---
@@ -783,9 +787,23 @@ internal sealed record ProfilerTakePreciseCoverageResult(
783787

784788
internal sealed record ProfilerStopPreciseCoverageResult();
785789

786-
internal sealed record ProfilerGetScriptSourceParams(string ScriptId);
790+
// ============================================================================
791+
// Debugger domain
792+
// ============================================================================
793+
//
794+
// Debugger.getScriptSource is what fetches V8 script source for line-level
795+
// coverage stats. Profiler.getScriptSource does not exist in CDP; Debugger
796+
// owns script-source retrieval and must be enabled on the session first.
797+
798+
internal sealed record DebuggerEnableParams(double? MaxScriptsCacheSize = null);
799+
800+
internal sealed record DebuggerEnableResult(string? DebuggerId = null);
801+
802+
internal sealed record DebuggerDisableResult();
803+
804+
internal sealed record DebuggerGetScriptSourceParams(string ScriptId);
787805

788-
internal sealed record ProfilerGetScriptSourceResult(string ScriptSource);
806+
internal sealed record DebuggerGetScriptSourceResult(string ScriptSource, string? Bytecode = null);
789807

790808
// ============================================================================
791809
// Page domain (screenshot with clip)

src/Motus/Transport/BiDi/BiDiTranslationRegistry.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,9 @@ private static Dictionary<string, IBiDiTranslation> BuildTranslations()
7676
new NoOpTranslation("Profiler.startPreciseCoverage", "session.status"),
7777
new NoOpTranslation("Profiler.takePreciseCoverage", "session.status"),
7878
new NoOpTranslation("Profiler.stopPreciseCoverage", "session.status"),
79-
new NoOpTranslation("Profiler.getScriptSource", "session.status"),
79+
new NoOpTranslation("Debugger.enable", "session.status"),
80+
new NoOpTranslation("Debugger.disable", "session.status"),
81+
new NoOpTranslation("Debugger.getScriptSource", "session.status"),
8082
new NoOpTranslation("CSS.startRuleUsageTracking", "session.status"),
8183
new NoOpTranslation("CSS.stopRuleUsageTracking", "session.status"),
8284
new NoOpTranslation("CSS.getStyleSheetText", "session.status"),

0 commit comments

Comments
 (0)