Gate runtime scenario retry attempts across demo policy and release checks

Author: codex

.kiro/specs/multimodal-agents/progress.md

@@ -131,6 +131,8 @@
 100. Synced fast-retry workflow docs and alignment tests: updated `README.md` + `docs/challenge-demo-runbook.md`, extended `tests/unit/release-script-alias-alignment.test.ts`, and added `tests/unit/demo-fast-retry-docs-alignment.test.ts` to keep alias and documentation consistent.
 101. Expanded transient scenario retry coverage in `scripts/demo-e2e.ps1`: `runtime.lifecycle.endpoints` and `runtime.metrics.endpoints` now use bounded transient retries (`ScenarioRetryMaxAttempts`/`ScenarioRetryBackoffMs`) in addition to operator/visual flows, reducing false-negative demo failures from short-lived service/network blips.
 102. Updated alignment guard `tests/unit/demo-scenario-retry-alignment.test.ts` to enforce retry configuration across operator/ui/runtime flaky scenarios and prevent future drift in demo retry policy wiring.
+103. Added runtime scenario attempt KPIs to demo summary (`runtimeLifecycleScenarioAttempts`, `runtimeMetricsScenarioAttempts`) in `scripts/demo-e2e.ps1`, so runtime retry behavior is visible in judge-facing artifacts and release logs.
+104. Extended scenario retry gates across policy/release/docs/tests: `scripts/demo-e2e-policy-check.mjs` and `scripts/release-readiness.ps1` now enforce runtime attempt bounds (`1..options.scenarioRetryMaxAttempts`), runbook evidence was updated, and unit coverage expanded in `tests/unit/demo-e2e-policy-check.test.ts`, `tests/unit/release-readiness.test.ts`, `tests/unit/demo-scenario-retry-alignment.test.ts`, and `tests/unit/runbook-release-alignment.test.ts`.
 
 ## Current Focus Queue
 

docs/challenge-demo-runbook.md

@@ -135,6 +135,8 @@ The release gate (`scripts/release-readiness.ps1`) hard-fails when these evidenc
   - strict final run (`npm run verify:release:strict`) enforces `kpi.scenarioRetriesUsedCount = 0`
   - `kpi.uiVisualTestingScenarioAttempts <= options.scenarioRetryMaxAttempts`
   - `kpi.operatorConsoleActionsScenarioAttempts <= options.scenarioRetryMaxAttempts`
+  - `kpi.runtimeLifecycleScenarioAttempts <= options.scenarioRetryMaxAttempts`
+  - `kpi.runtimeMetricsScenarioAttempts <= options.scenarioRetryMaxAttempts`
   - `kpi.scenarioRetryableFailuresTotal >= 0`
 - Perf-load anti-drift (from `artifacts/perf-load/policy-check.json`):
   - required check items include:

scripts/demo-e2e-policy-check.mjs

@@ -832,6 +832,26 @@ async function main() {
     kpis.operatorConsoleActionsScenarioAttempts,
     "1..options.scenarioRetryMaxAttempts",
   );
+  const runtimeLifecycleScenarioAttempts = toNumber(kpis.runtimeLifecycleScenarioAttempts);
+  addCheck(
+    "kpi.runtimeLifecycleScenarioAttempts",
+    Number.isFinite(runtimeLifecycleScenarioAttempts) &&
+      runtimeLifecycleScenarioAttempts >= 1 &&
+      Number.isFinite(scenarioRetryMaxAttempts) &&
+      runtimeLifecycleScenarioAttempts <= scenarioRetryMaxAttempts,
+    kpis.runtimeLifecycleScenarioAttempts,
+    "1..options.scenarioRetryMaxAttempts",
+  );
+  const runtimeMetricsScenarioAttempts = toNumber(kpis.runtimeMetricsScenarioAttempts);
+  addCheck(
+    "kpi.runtimeMetricsScenarioAttempts",
+    Number.isFinite(runtimeMetricsScenarioAttempts) &&
+      runtimeMetricsScenarioAttempts >= 1 &&
+      Number.isFinite(scenarioRetryMaxAttempts) &&
+      runtimeMetricsScenarioAttempts <= scenarioRetryMaxAttempts,
+    kpis.runtimeMetricsScenarioAttempts,
+    "1..options.scenarioRetryMaxAttempts",
+  );
   const scenarioRetryableFailuresTotal = toNumber(kpis.scenarioRetryableFailuresTotal);
   addCheck(
     "kpi.scenarioRetryableFailuresTotal",

scripts/demo-e2e.ps1

@@ -2693,6 +2693,8 @@ $runtimeLifecycleData = Get-ScenarioData -Name "runtime.lifecycle.endpoints"
 $runtimeMetricsData = Get-ScenarioData -Name "runtime.metrics.endpoints"
 $uiVisualTestingScenario = @($script:ScenarioResults | Where-Object { $_.name -eq "ui.visual_testing" } | Select-Object -First 1)
 $operatorActionsScenario = @($script:ScenarioResults | Where-Object { $_.name -eq "operator.console.actions" } | Select-Object -First 1)
+$runtimeLifecycleScenario = @($script:ScenarioResults | Where-Object { $_.name -eq "runtime.lifecycle.endpoints" } | Select-Object -First 1)
+$runtimeMetricsScenario = @($script:ScenarioResults | Where-Object { $_.name -eq "runtime.metrics.endpoints" } | Select-Object -First 1)
 $scenarioRetriedSet = @($script:ScenarioResults | Where-Object { [bool]$_.retried })
 $scenarioRetryableFailuresTotal = @($script:ScenarioResults | ForEach-Object { [int]$_.retryableFailureCount } | Measure-Object -Sum).Sum
 $uiExecutorService = $script:ServiceStatuses | Where-Object { $_.name -eq "ui-executor" } | Select-Object -First 1
@@ -2846,6 +2848,8 @@ $summary = [ordered]@{
     scenarioRetryableFailuresTotal = [int]$scenarioRetryableFailuresTotal
     uiVisualTestingScenarioAttempts = if ($uiVisualTestingScenario.Count -gt 0) { [int]$uiVisualTestingScenario[0].attempts } else { $null }
     operatorConsoleActionsScenarioAttempts = if ($operatorActionsScenario.Count -gt 0) { [int]$operatorActionsScenario[0].attempts } else { $null }
+    runtimeLifecycleScenarioAttempts = if ($runtimeLifecycleScenario.Count -gt 0) { [int]$runtimeLifecycleScenario[0].attempts } else { $null }
+    runtimeMetricsScenarioAttempts = if ($runtimeMetricsScenario.Count -gt 0) { [int]$runtimeMetricsScenario[0].attempts } else { $null }
     delegatedRoute = if ($null -ne $delegationData) { $delegationData.delegatedRoute } else { $null }
     assistiveRouterMode = if ($null -ne $delegationData) { $delegationData.routingMode } else { $null }
     assistiveRouterReason = if ($null -ne $delegationData) { $delegationData.routingReason } else { $null }

scripts/release-readiness.ps1

@@ -450,6 +450,34 @@ if ((-not $SkipDemoE2E) -and (Test-Path $SummaryPath)) {
     )
   }
 
+  $runtimeLifecycleScenarioAttempts = To-NumberOrNaN $summary.kpis.runtimeLifecycleScenarioAttempts
+  if (
+    [double]::IsNaN($runtimeLifecycleScenarioAttempts) -or
+    $runtimeLifecycleScenarioAttempts -lt 1 -or
+    $runtimeLifecycleScenarioAttempts -gt $scenarioRetryMaxAttempts
+  ) {
+    Fail (
+      "Critical KPI check failed: kpi.runtimeLifecycleScenarioAttempts expected 1.." +
+      $summary.options.scenarioRetryMaxAttempts +
+      ", actual " +
+      $summary.kpis.runtimeLifecycleScenarioAttempts
+    )
+  }
+
+  $runtimeMetricsScenarioAttempts = To-NumberOrNaN $summary.kpis.runtimeMetricsScenarioAttempts
+  if (
+    [double]::IsNaN($runtimeMetricsScenarioAttempts) -or
+    $runtimeMetricsScenarioAttempts -lt 1 -or
+    $runtimeMetricsScenarioAttempts -gt $scenarioRetryMaxAttempts
+  ) {
+    Fail (
+      "Critical KPI check failed: kpi.runtimeMetricsScenarioAttempts expected 1.." +
+      $summary.options.scenarioRetryMaxAttempts +
+      ", actual " +
+      $summary.kpis.runtimeMetricsScenarioAttempts
+    )
+  }
+
   $scenarioRetryableFailuresTotal = To-NumberOrNaN $summary.kpis.scenarioRetryableFailuresTotal
   if ([double]::IsNaN($scenarioRetryableFailuresTotal) -or $scenarioRetryableFailuresTotal -lt 0) {
     Fail ("Critical KPI check failed: kpi.scenarioRetryableFailuresTotal expected >= 0, actual " + $summary.kpis.scenarioRetryableFailuresTotal)
@@ -687,20 +715,26 @@ if ((-not $SkipDemoE2E) -and (Test-Path $SummaryPath)) {
   $scenarioRetryableFailuresTotal = $summary.kpis.scenarioRetryableFailuresTotal
   $uiVisualAttempts = $summary.kpis.uiVisualTestingScenarioAttempts
   $operatorActionsAttempts = $summary.kpis.operatorConsoleActionsScenarioAttempts
+  $runtimeLifecycleAttempts = $summary.kpis.runtimeLifecycleScenarioAttempts
+  $runtimeMetricsAttempts = $summary.kpis.runtimeMetricsScenarioAttempts
   if (
     $null -ne $scenarioRetryAttempts -or
     $null -ne $scenarioRetryBackoff -or
     $null -ne $scenarioRetriesUsedCount -or
     $null -ne $uiVisualAttempts -or
-    $null -ne $operatorActionsAttempts
+    $null -ne $operatorActionsAttempts -or
+    $null -ne $runtimeLifecycleAttempts -or
+    $null -ne $runtimeMetricsAttempts
   ) {
     Write-Host (
       "demo.scenario.retry: max_attempts=" + $scenarioRetryAttempts +
       ", backoff_ms=" + $scenarioRetryBackoff +
       ", retries_used=" + $scenarioRetriesUsedCount +
       ", retryable_failures=" + $scenarioRetryableFailuresTotal +
       ", ui.visual_testing_attempts=" + $uiVisualAttempts +
-      ", operator.console.actions_attempts=" + $operatorActionsAttempts
+      ", operator.console.actions_attempts=" + $operatorActionsAttempts +
+      ", runtime.lifecycle.endpoints_attempts=" + $runtimeLifecycleAttempts +
+      ", runtime.metrics.endpoints_attempts=" + $runtimeMetricsAttempts
     )
   }
   $gatewayRoundTrip = $summary.kpis.gatewayWsRoundTripMs

tests/unit/demo-e2e-policy-check.test.ts

@@ -147,6 +147,8 @@ function createPassingSummary(overrides?: {
     scenarioRetriesUsedCount: 0,
     uiVisualTestingScenarioAttempts: 1,
     operatorConsoleActionsScenarioAttempts: 1,
+    runtimeLifecycleScenarioAttempts: 1,
+    runtimeMetricsScenarioAttempts: 1,
     scenarioRetryableFailuresTotal: 0,
     sandboxPolicyValidated: true,
     visualTestingStatus: "passed",
@@ -240,7 +242,7 @@ test("demo-e2e policy check passes with baseline passing summary", () => {
   const result = runPolicyCheck(createPassingSummary());
   assert.equal(result.exitCode, 0, JSON.stringify(result.payload));
   assert.equal(result.payload.ok, true);
-  assert.equal(result.payload.checks, 160);
+  assert.equal(result.payload.checks, 162);
 });
 
 test("demo-e2e policy check fails when assistant activity lifecycle KPI is missing", () => {
@@ -455,6 +457,44 @@ test("demo-e2e policy check fails when ui visual scenario attempts exceed config
   assert.ok(violations.some((item) => item.includes("kpi.uiVisualTestingScenarioAttempts")));
 });
 
+test("demo-e2e policy check fails when runtime lifecycle scenario attempts exceed configured retry max", () => {
+  const result = runPolicyCheck(
+    createPassingSummary({
+      kpis: {
+        runtimeLifecycleScenarioAttempts: 3,
+      },
+      options: {
+        scenarioRetryMaxAttempts: 2,
+      },
+    }),
+  );
+  assert.equal(result.exitCode, 1);
+  assert.equal(result.payload.ok, false);
+  const details = result.payload.details as Record<string, unknown>;
+  assert.ok(Array.isArray(details?.violations));
+  const violations = details.violations as string[];
+  assert.ok(violations.some((item) => item.includes("kpi.runtimeLifecycleScenarioAttempts")));
+});
+
+test("demo-e2e policy check fails when runtime metrics scenario attempts exceed configured retry max", () => {
+  const result = runPolicyCheck(
+    createPassingSummary({
+      kpis: {
+        runtimeMetricsScenarioAttempts: 3,
+      },
+      options: {
+        scenarioRetryMaxAttempts: 2,
+      },
+    }),
+  );
+  assert.equal(result.exitCode, 1);
+  assert.equal(result.payload.ok, false);
+  const details = result.payload.details as Record<string, unknown>;
+  assert.ok(Array.isArray(details?.violations));
+  const violations = details.violations as string[];
+  assert.ok(violations.some((item) => item.includes("kpi.runtimeMetricsScenarioAttempts")));
+});
+
 test("demo-e2e policy check fails when operator.device_nodes.lifecycle scenario is missing", () => {
   const summary = createPassingSummary();
   summary.scenarios = (summary.scenarios as Array<Record<string, unknown>>).filter(

tests/unit/demo-scenario-retry-alignment.test.ts

@@ -54,4 +54,6 @@ test("demo-e2e applies transient retry to flaky operator/ui/runtime scenarios",
   assert.match(source, /scenarioRetriesUsedCount/);
   assert.match(source, /uiVisualTestingScenarioAttempts/);
   assert.match(source, /operatorConsoleActionsScenarioAttempts/);
+  assert.match(source, /runtimeLifecycleScenarioAttempts/);
+  assert.match(source, /runtimeMetricsScenarioAttempts/);
 });

tests/unit/release-readiness.test.ts

@@ -41,6 +41,8 @@ function createPassingSummary(
     scenarioRetriesUsedCount: number | string;
     uiVisualTestingScenarioAttempts: number | string;
     operatorConsoleActionsScenarioAttempts: number | string;
+    runtimeLifecycleScenarioAttempts: number | string;
+    runtimeMetricsScenarioAttempts: number | string;
     scenarioRetryableFailuresTotal: number | string;
     analyticsSplitTargetsValidated: boolean | string;
     analyticsBigQueryConfigValidated: boolean | string;
@@ -173,6 +175,12 @@ function createPassingSummary(
       operatorConsoleActionsScenarioAttempts: hasOverride("operatorConsoleActionsScenarioAttempts")
         ? overrides.operatorConsoleActionsScenarioAttempts
         : 1,
+      runtimeLifecycleScenarioAttempts: hasOverride("runtimeLifecycleScenarioAttempts")
+        ? overrides.runtimeLifecycleScenarioAttempts
+        : 1,
+      runtimeMetricsScenarioAttempts: hasOverride("runtimeMetricsScenarioAttempts")
+        ? overrides.runtimeMetricsScenarioAttempts
+        : 1,
       scenarioRetryableFailuresTotal: hasOverride("scenarioRetryableFailuresTotal")
         ? overrides.scenarioRetryableFailuresTotal
         : 0,
@@ -680,6 +688,38 @@ test(
   },
 );
 
+test(
+  "release-readiness fails when runtime lifecycle scenario attempts exceed configured retry max",
+  { skip: skipIfNoPowerShell },
+  () => {
+    const result = runReleaseReadiness(
+      createPassingSummary({
+        scenarioRetryMaxAttempts: "2",
+        runtimeLifecycleScenarioAttempts: "3",
+      }),
+    );
+    assert.equal(result.exitCode, 1);
+    const output = `${result.stderr}\n${result.stdout}`;
+    assert.match(output, /kpi\.runtimeLifecycleScenarioAttempts expected 1\.\.2, actual 3/i);
+  },
+);
+
+test(
+  "release-readiness fails when runtime metrics scenario attempts exceed configured retry max",
+  { skip: skipIfNoPowerShell },
+  () => {
+    const result = runReleaseReadiness(
+      createPassingSummary({
+        scenarioRetryMaxAttempts: "2",
+        runtimeMetricsScenarioAttempts: "3",
+      }),
+    );
+    assert.equal(result.exitCode, 1);
+    const output = `${result.stderr}\n${result.stdout}`;
+    assert.match(output, /kpi\.runtimeMetricsScenarioAttempts expected 1\.\.2, actual 3/i);
+  },
+);
+
 test(
   "release-readiness fails when analytics split targets KPI is not validated",
   { skip: skipIfNoPowerShell },

tests/unit/runbook-release-alignment.test.ts

@@ -66,6 +66,8 @@ test("runbook documents release perf artifact-only mode and critical evidence ke
     "`kpi.scenarioRetriesUsedCount = 0`",
     "`kpi.uiVisualTestingScenarioAttempts <= options.scenarioRetryMaxAttempts`",
     "`kpi.operatorConsoleActionsScenarioAttempts <= options.scenarioRetryMaxAttempts`",
+    "`kpi.runtimeLifecycleScenarioAttempts <= options.scenarioRetryMaxAttempts`",
+    "`kpi.runtimeMetricsScenarioAttempts <= options.scenarioRetryMaxAttempts`",
     "`kpi.scenarioRetryableFailuresTotal >= 0`",
     "`workload.live.p95`",
     "`workload.ui.p95`",