diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07b2ebe..c5c0055 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,7 @@ jobs: node --test backend-shutdown.test.cjs node --test ui-entry.test.cjs node --test candidate-profile.test.cjs + node --test ../../frontend/desktop-reference/candidate-soak-policy.test.cjs ../../frontend/desktop-reference/candidate-cycle.test.cjs npx electron --version - name: Build and stage packaged backend run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 621fce7..df31231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ### Added +- Supervised 5..30 minute candidate UI soak with bounded scratch storage, RSS, + active processes, logs and deadlines; interface-return and traffic checks. + A local 903-second run reconciled 48,771 packets and an observed VPN interface + cycle. Memory growth and periodic probe overhead remain profiling gates. +- Interface API preflight reuses the provider's fresh enumeration snapshot, + avoiding duplicate discovery and mutation of provider-owned payloads. - Isolated Windows candidate builder with fresh packaged backend, short artifact paths, separate metadata-only profile and bounded build/live-test storage. A local 122-second packaged UI run reconciled 10,189 accepted/acknowledged/stored diff --git a/backend/app/services/sniffer_service.py b/backend/app/services/sniffer_service.py index 42b31af..4094692 100644 --- a/backend/app/services/sniffer_service.py +++ b/backend/app/services/sniffer_service.py @@ -716,8 +716,9 @@ def flow_service(self) -> FlowService: def capture_interfaces(self) -> dict[str, Any]: payload = self._capture_provider.list_interfaces() - payload["preflight"] = self.capture_preflight() - return payload + from_snapshot = getattr(self._capture_provider, 'preflight_for_interfaces', None) + preflight = from_snapshot(payload).to_dict() if callable(from_snapshot) else self.capture_preflight() + return {**payload, 'preflight': preflight} def capture_preflight(self) -> dict[str, Any]: return self._capture_provider.preflight().to_dict() diff --git a/core/capture/system_provider.py b/core/capture/system_provider.py index dc5b4e5..465a721 100644 --- a/core/capture/system_provider.py +++ b/core/capture/system_provider.py @@ -328,9 +328,11 @@ def resolve_interface(self, candidate: str | None) -> str | None: return self._resolve_func(candidate) def preflight(self) -> CapturePreflightReport: + return self.preflight_for_interfaces(self.list_interfaces()) + + def preflight_for_interfaces(self, interfaces: dict[str, Any]) -> CapturePreflightReport: os_name = str(self._os_name_getter() or "").strip().lower() or "unknown" supported = os_name in SUPPORTED_CAPTURE_SYSTEMS - interfaces = self.list_interfaces() if self._use_subprocess_interface_discovery: scapy_ok, scapy_detail = self._scapy_status_from_interfaces_payload(interfaces) else: diff --git a/docs/DESKTOP_CANDIDATE_SOAK.md b/docs/DESKTOP_CANDIDATE_SOAK.md new file mode 100644 index 0000000..7c31747 --- /dev/null +++ b/docs/DESKTOP_CANDIDATE_SOAK.md @@ -0,0 +1,126 @@ +# Supervised Candidate Soak + +This is a 5..30 minute live validation tier before multi-hour acceptance. +It is not a production monitor or an unlimited-duration background job. + +```powershell +python scripts/qa/candidate_soak.py --executable '' --duration 900 --require-vpn-cycle +``` + +Use the existing independently packaged candidate; no installed app is replaced. +The supervisor checks its build manifest, creates a new scratch workspace and +launches the actual Electron UI with its isolated backend and metadata profile. +Capture is real and must be authorized on the host/network. The test itself does +not change VPN, routes or firewall rules and does not generate network traffic. + +## Bounds + +- Requested capture: 300..1800 seconds, using monotonic clocks. The supervisor + deadline includes another 120 seconds for launch, UI checks and shutdown. +- Sampled scratch storage: 512 MiB, including temporary profile, SQLite, journal + and worker log. Worker log has an additional 2 MiB limit. +- Sampled aggregate process RSS: 2 GiB; this sums shared pages across processes + and is not unique physical-memory consumption. +- Minimum free disk headroom: 2 GiB during execution, 3 GiB before starting. +- At most 64 concurrently retained live process identities. Exited helpers and + their CPU baselines are retired rather than exhausting a lifetime-count budget. + Only the owned test tree and observed live descendants can be terminated. + +These are sampled watchdogs, not OS-enforced quotas; brief overshoot is possible. +On a guard failure the run fails, even if its earlier counters looked healthy. +The fresh test workspace is disposable and removed after process cleanup; never +use this cleanup policy on an installed recovery journal. + +The short test still rejects durations over 120 seconds unless the supervisor's +matching scratch directory, nonce and bounded policy are present. The long test +extends only its own metadata retention to requested duration plus five minutes, +so the final retained SQLite count can be compared with acknowledgements without +expected retention expiry invalidating the test. Storage remains watchdog-bounded. + +## Evidence + +Only numeric summaries, fixed stage/status labels and the active UI page are +printed. No domain, IP, interface identity, token, raw bytes or traffic screenshot +is exported. Resource samples are retained in a bounded 128-sample tail. CPU is +sampled across observed processes and reported relative to one CPU core; it may +exceed 100%. API p95 is the bounded last 256 requests, not a whole-run benchmark. + +Success requires real acknowledged traffic, zero final pending journal records, +matching accepted/acknowledged/SQLite packet counts, no reported history failure +or drop, no renderer error, working selection/investigation/Auto-follow, normal +acknowledged shutdown and temporary-data removal. + +`--require-vpn-cycle` additionally requires observing departure from the first +traffic-bearing interface, return to that same interface, and increasing confirmed +traffic after return. Interface identity stays in worker memory only. Dynamic +profiling counters cannot impersonate a route change. This checks an observed +interface cycle, not an unchanged-interface PPP filter change, public exit-country +policy, or complete validation of every OS/VPN transition. + +A missing driver counter remains unknown, not zero loss. A 15-minute success +does not establish several-hour stability or exactly-once stateful analysis. + +## Initial Findings + +The first 900-second attempt was stopped by its supervisor at 433.73 seconds: +the initial guard incorrectly counted 65 lifetime process identities rather than +concurrent live processes. It is a failed run, not an application crash or a +completed soak. Its last minute sample had 8,892 accepted/acknowledged records, +zero pending, two observed interface changes with return and further confirmed +traffic, no renderer error and no reported driver buffer drop. Scratch peak was +30,762,289 bytes and was removed. Final Stop/SQLite reconciliation was not reached. + +Review also found duplicate interface discovery inside one `/interfaces` response: +the service enumerated interfaces, then provider preflight enumerated them again. +`SystemCaptureProvider.preflight_for_interfaces` now reuses the same fresh snapshot; +other providers retain their existing fallback. The service no longer mutates the +provider-owned payload when adding preflight. This avoids the second enumeration +without a new stale cache or a slower route-refresh interval. Focused regressions +cover single-snapshot readiness and compatibility. + +## Completed Local Run: 2026-11-12 + +The fresh package from `08c68198cb6dce7e69800ff92cdc44f1e0706a9d` completed +the corrected supervised run. Capture and final UI checks took 903 seconds; +supervisor startup, checks and cleanup took 911.03 seconds in total. + +| Observation | Measured | +| --- | ---: | +| Accepted / acknowledged / retained SQLite rows | 48,771 / 48,771 / 48,771 | +| Final / sampled peak journal pending | 0 / 3,866 | +| Interface switches | 2 | +| Departure, return and further confirmed traffic | Observed | +| History failed / dropped writes | 0 / 0 | +| Reported driver buffer drops | 0; final coverage complete | +| Renderer errors | 0 | +| Bounded API tail p95 | 22.11 ms | +| Selection through investigation findings | 354 ms | +| Stop | 675 ms | +| Sampled scratch peak | 75,444,505 bytes | +| Sampled aggregate process RSS peak | 1,134.74 MiB | +| Sampled aggregate CPU, one-core basis | 28.12% | +| Sampled active process peak | 10 | +| Lifetime observed process identities | 126 | +| Profile and supervisor workspace removed | Yes | + +The active-process guard stayed enabled. Most later identities were short-lived +capture-backend probes (117 sampled identities), not 126 simultaneously live +processes. Interface enumeration is now shared within one API response, but these +periodic backend probes still exist; their CPU cost needs separate attribution. + +Aggregate RSS for samples with nine active processes increased from 950.77 MiB +at 154.2 seconds to 1,049.82 MiB at 894.5 seconds (99.05 MiB). Shared pages are +counted more than once; this is neither unique memory nor proof of a leak. +Do not extrapolate a flat memory curve or several-hour stability from this run. +Memory ownership, retained state/caches and probe overhead are follow-up gates. + +The user also exercised paused Auto-follow during capture. The harness returned +to Live Monitor for its final selection and resumed follow before Stop. Its +inspection check confirms finding-panel navigation, not exhaustive protocol +decoding or absence of every handled HTTP error. No real packet screenshot or +raw traffic report was exported, and the installed app remained unchanged. + +Windows CI additionally exposed a test-isolation race: build-guard unit tests +mocked the shared `time.monotonic`, allowing other background threads to consume +their finite fake-clock sequence. The mocks now replace only the candidate +module's dependency references, leaving global clocks/process APIs unchanged. diff --git a/docs/README.md b/docs/README.md index 62a07b3..d14e408 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ describe their own release, not the current branch or installed desktop overlay. - [Isolated journal/history probe and local evidence](JOURNAL_HISTORY_PROBE.md) - [Standard frontend live-follow continuity](LIVE_FOLLOW_VALIDATION.md) - [Isolated Windows desktop candidate](DESKTOP_CANDIDATE.md) +- [Supervised candidate soak](DESKTOP_CANDIDATE_SOAK.md) - [Screenshot provenance](SCREENSHOTS.md) - [Repository and GitHub maintenance](REPOSITORY_MAINTENANCE.md) diff --git a/frontend/desktop-reference/candidate-cycle.cjs b/frontend/desktop-reference/candidate-cycle.cjs new file mode 100644 index 0000000..35a4898 --- /dev/null +++ b/frontend/desktop-reference/candidate-cycle.cjs @@ -0,0 +1,14 @@ +function observeInterfaceCycle(previous, capture) { + const next = { ...previous }; + if (!capture.selected_iface || !(capture.ipc_accepted > 0)) return next; + const identity = JSON.stringify(capture.selected_iface); + if (!next.initial) next.initial = identity; + else if (identity !== next.initial) next.left = true; + else if (next.left && !next.returned) { + next.returned = true; + next.acknowledgedAtReturn = capture.journal_acknowledged_records; + } + if (next.returned && capture.journal_acknowledged_records > next.acknowledgedAtReturn) next.trafficAfterReturn = true; + return next; +} +module.exports = { observeInterfaceCycle }; diff --git a/frontend/desktop-reference/candidate-cycle.test.cjs b/frontend/desktop-reference/candidate-cycle.test.cjs new file mode 100644 index 0000000..2c2a8b6 --- /dev/null +++ b/frontend/desktop-reference/candidate-cycle.test.cjs @@ -0,0 +1,25 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { observeInterfaceCycle: observe } = require('./candidate-cycle.cjs'); +const sample = (selected_iface, count) => ({ selected_iface, ipc_accepted: count, journal_acknowledged_records: count }); +test('idle or absent interface cannot establish a baseline', () => { + assert.deepEqual(observe({}, sample('A', 0)), {}); + assert.deepEqual(observe({}, sample(null, 10)), {}); +}); +test('profiling counters cannot impersonate an interface change', () => { + const baseline = observe({}, sample('A', 1)); + const next = observe(baseline, { ...sample('A', 20), capture_profile: { seen_packets: 20 } }); + assert.equal(next.left, undefined); + assert.equal(baseline.left, undefined); +}); +test('departure alone is not a completed return', () => { + const state = observe(observe({}, sample('A', 1)), sample('B', 2)); + assert.equal(state.left, true); + assert.equal(state.returned, undefined); +}); +test('return requires subsequent confirmed traffic', () => { + const returned = observe(observe(observe({}, sample('A', 1)), sample('B', 2)), sample('A', 3)); + assert.equal(returned.returned, true); + assert.equal(returned.trafficAfterReturn, undefined); + assert.equal(observe(returned, sample('A', 4)).trafficAfterReturn, true); +}); diff --git a/frontend/desktop-reference/candidate-soak-policy.cjs b/frontend/desktop-reference/candidate-soak-policy.cjs new file mode 100644 index 0000000..3dd6c91 --- /dev/null +++ b/frontend/desktop-reference/candidate-soak-policy.cjs @@ -0,0 +1,14 @@ +const path = require('node:path'); + +function validateSoakPolicy({ root, cwd, grant, nonce, duration }) { + if (!path.isAbsolute(root) || path.resolve(root) !== path.resolve(cwd) || !path.basename(root).startsWith('netbot-candidate-soak-')) { + throw new Error('Long capture requires its supervised scratch workspace'); + } + if (grant.version !== 1 || !nonce || nonce !== grant.nonce || grant.durationSeconds !== duration || + !Number.isInteger(duration) || duration < 300 || duration > 1800 || grant.maxProfileBytes !== 512 * 1024**2 || + typeof grant.requireVpnCycle !== 'boolean') { + throw new Error('Invalid supervised soak policy'); + } + return grant; +} +module.exports = { validateSoakPolicy }; diff --git a/frontend/desktop-reference/candidate-soak-policy.test.cjs b/frontend/desktop-reference/candidate-soak-policy.test.cjs new file mode 100644 index 0000000..646d171 --- /dev/null +++ b/frontend/desktop-reference/candidate-soak-policy.test.cjs @@ -0,0 +1,22 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { validateSoakPolicy } = require('./candidate-soak-policy.cjs'); + +const root = path.resolve('netbot-candidate-soak-fixture'); +const grant = { version: 1, nonce: 'fixture-nonce', durationSeconds: 900, maxProfileBytes: 512 * 1024**2, requireVpnCycle: true }; +const fixture = () => ({ root, cwd: root, grant: { ...grant }, nonce: grant.nonce, duration: 900 }); +test('explicit supervisor grant permits only the bounded longer mode', () => { + assert.equal(validateSoakPolicy(fixture()).durationSeconds, 900); +}); +test('ordinary or mismatched workspaces cannot enable long capture', () => { + for (const change of [{ root: 'relative' }, { cwd: path.dirname(root) }, { root: path.resolve('ordinary'), cwd: path.resolve('ordinary') }]) { + assert.throws(() => validateSoakPolicy({ ...fixture(), ...change })); + } +}); +test('duration, nonce and storage limits cannot silently expand', () => { + for (const change of [{ nonce: 'wrong' }, { duration: 7200 }, { duration: 120 }, + { grant: { ...grant, maxProfileBytes: 1024**3 } }, { grant: { ...grant, requireVpnCycle: 'yes' } }]) { + assert.throws(() => validateSoakPolicy({ ...fixture(), ...change })); + } +}); diff --git a/frontend/desktop-reference/test-candidate-capture.mjs b/frontend/desktop-reference/test-candidate-capture.mjs index a1df2cc..db3bec7 100644 --- a/frontend/desktop-reference/test-candidate-capture.mjs +++ b/frontend/desktop-reference/test-candidate-capture.mjs @@ -6,14 +6,25 @@ import path from 'node:path'; import os from 'node:os'; import net from 'node:net'; import { execFileSync } from 'node:child_process'; +import { validateSoakPolicy } from './candidate-soak-policy.cjs'; +import { observeInterfaceCycle } from './candidate-cycle.cjs'; const executable = process.env.NETBOT_CANDIDATE_EXE; const python = process.env.NETBOT_SMOKE_PYTHON_BIN; const duration = Number(process.env.NETBOT_CANDIDATE_DURATION || 60); assert.ok(executable && python, 'Explicit candidate executable and report Python are required'); -assert.ok(duration >= 15 && duration <= 120, 'Live test must last 15..120 seconds'); +let soak; +if (process.env.NETBOT_CANDIDATE_SOAK_ROOT) { + const root = await fs.realpath(process.env.NETBOT_CANDIDATE_SOAK_ROOT); + soak = validateSoakPolicy({ root, cwd: await fs.realpath(process.cwd()), + grant: JSON.parse(await fs.readFile(path.join(root, 'policy.json'), 'utf8')), + nonce: process.env.NETBOT_CANDIDATE_SOAK_NONCE, duration }); +} else { + assert.ok(duration >= 15 && duration <= 120, 'Live test must last 15..120 seconds'); +} await fs.access(executable); -const profile = await fs.mkdtemp(path.join(os.tmpdir(), 'netbot-candidate-smoke-')); +const profileParent = soak ? await fs.realpath(process.env.NETBOT_CANDIDATE_SOAK_ROOT) : path.resolve(os.tmpdir()); +const profile = await fs.mkdtemp(path.join(profileParent, 'netbot-candidate-smoke-')); const listener = net.createServer(); await new Promise(resolve => listener.listen(0, '127.0.0.1', resolve)); const port = listener.address().port; @@ -36,52 +47,62 @@ try { env }); ownerPid = app.process().pid; const window = await app.firstWindow(); - const errors = []; - window.on('pageerror', error => errors.push(error.name)); + let rendererErrors = 0; + window.on('pageerror', () => { rendererErrors++; }); await window.getByRole('heading', { name: 'Command Center', exact: true }).waitFor({ timeout: 45000 }); const runtime = await window.evaluate(() => window.netbotproDesktop); const apiBase = new URL(runtime.apiBase); assert.equal(apiBase.hostname, '127.0.0.1'); assert.notEqual(apiBase.port, String(port), 'Candidate reused the deliberately occupied inherited API port'); const request = async (endpoint, options = {}) => { - const requestedAt = Date.now(); + const requestedAt = performance.now(); const result = await fetch(`${runtime.apiBase}${endpoint}`, { ...options, headers: { 'X-NetBot-Token': runtime.localToken, ...options.headers }, signal: AbortSignal.timeout(10000) }); assert.ok(result.ok, `Candidate API status ${result.status}`); const body = await result.json(); - requestDurations.push(Date.now() - requestedAt); + requestDurations.push(performance.now() - requestedAt); if (requestDurations.length > 256) requestDurations.shift(); return body; }; + if (soak) await request('/settings', { method: 'PUT', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ persist_logs: true, retention_minutes: Math.ceil(duration / 60) + 5 }) }); await window.locator('nav.app-nav').getByRole('button', { name: 'Live Monitor', exact: true }).click(); const bar = window.locator('header[aria-label="Capture status"]'); await bar.getByRole('button', { name: 'Start', exact: true }).click({ timeout: 20000 }); await window.waitForFunction(() => { const stop = document.querySelector('.soc-stop'); return stop && !stop.disabled; }, null, { timeout: 20000 }); console.log(JSON.stringify({ stage: 'capture_started', durationSeconds: duration, isolatedCandidate: true })); - const started = Date.now(); + const started = performance.now(); let pendingPeak = 0, interfaceChanges = 0, previousInterface, last, nextSample = 0; - while (Date.now() - started < duration * 1000) { + let cycle = {}; + while (performance.now() - started < duration * 1000) { last = await request('/sniffer/state'); assert.equal(last.running, true, 'Capture unexpectedly stopped'); assert.ok(!last.ipc_callback_errors, 'Capture callback failure'); + assert.ok(!last.capture_error, 'Capture reported an error'); + assert.ok(!last.capture_callback_errors, 'Driver callback failure'); + assert.ok(!last.driver_stats?.buffer_dropped_reported, 'Driver reported drops during capture'); pendingPeak = Math.max(pendingPeak, Number(last.journal_pending_records || 0)); if (previousInterface && last.selected_iface && previousInterface !== last.selected_iface) interfaceChanges++; previousInterface = last.selected_iface || previousInterface; - if (Date.now() - started >= nextSample) { + const hadBaseline = !!cycle.initial; + cycle = observeInterfaceCycle(cycle, last); + if (!hadBaseline && cycle.initial) console.log(JSON.stringify({ stage: 'route_baseline_ready', seconds: Math.round((performance.now() - started) / 1000) })); + if (performance.now() - started >= nextSample) { const history = await request('/packets?limit=25&offset=0&only_remote=false'); const remote = await request('/packets?limit=1&offset=0&only_remote=true'); const ui = await window.evaluate(() => ({ activePage: document.querySelector('nav [aria-current="page"]')?.dataset.navId, follow: document.querySelector('.lm-follow input')?.checked, streamLabel: document.querySelector('.app-nav-footer small:last-child')?.textContent })); - console.log(JSON.stringify({ stage: 'capture_sample', seconds: Math.round((Date.now() - started) / 1000), + console.log(JSON.stringify({ stage: 'capture_sample', seconds: Math.round((performance.now() - started) / 1000), accepted: last.ipc_accepted, acknowledged: last.journal_acknowledged_records, pending: last.journal_pending_records, historyTotal: history.total, historyItems: history.items?.length, remoteTotal: remote.total, ui, websocketClients: last.observability?.event_bus?.subscribers, - uiRows: await window.locator('.lm-packets tbody tr[tabindex]').count(), rendererErrors: errors.length, + uiRows: await window.locator('.lm-packets tbody tr[tabindex]').count(), rendererErrors, interfaceChanges, routeSwitches: last.route_switches, captureError: !!last.capture_error, + leftInitialInterface: !!cycle.left, returnedToInitialInterface: !!cycle.returned, trafficAfterReturn: !!cycle.trafficAfterReturn, routeWarning: !!last.route_warning, health: last.health, controlSampleAgeSeconds: last.control_sample_age_seconds, driverBufferDroppedReported: last.driver_stats?.buffer_dropped_reported ?? null })); - nextSample += 15000; + nextSample += soak ? 60000 : 15000; if (ui.activePage === 'monitor' && history.total > 0 && await window.locator('.lm-packets tbody tr[tabindex]').count() === 0) { await window.bringToFront(); await new Promise(resolve => setTimeout(resolve, 500)); @@ -89,23 +110,23 @@ try { } } peakBytes = Math.max(peakBytes, await bytes(profile)); - assert.ok(peakBytes < 128 * 1024**2, 'Candidate storage budget exceeded'); + assert.ok(peakBytes < (soak?.maxProfileBytes || 128 * 1024**2), 'Candidate storage budget exceeded'); await new Promise(resolve => setTimeout(resolve, 1000)); } await window.locator('nav.app-nav').getByRole('button', { name: 'Live Monitor', exact: true }).click(); await window.getByRole('heading', { name: 'Live Monitor', exact: true }).waitFor(); const panel = window.locator('.lm-packets'); - const selectionStarted = Date.now(); + const selectionStarted = performance.now(); await panel.locator('tbody tr[tabindex]').first().locator('td').first().click({ timeout: 10000 }); await window.getByRole('button', { name: 'Open in Investigate', exact: true }).click(); await window.getByRole('heading', { name: 'Key Findings', exact: true }).waitFor({ timeout: 15000 }); - const selectionMilliseconds = Date.now() - selectionStarted; + const selectionMilliseconds = Math.round(performance.now() - selectionStarted); await window.locator('nav.app-nav').getByRole('button', { name: 'Live Monitor', exact: true }).click(); const follow = panel.getByRole('switch', { name: 'Auto-follow' }); if (await follow.isChecked()) await follow.click(); await follow.click(); await window.waitForFunction(() => document.querySelector('.lm-follow input')?.checked); - const stoppedAt = Date.now(); + const stoppedAt = performance.now(); await bar.getByRole('button', { name: 'Stop', exact: true }).click(); await window.waitForFunction(() => { const start = document.querySelector('.soc-start'); return start && !start.disabled; }); let final; @@ -117,15 +138,16 @@ try { assert.equal(final.running, false, 'Capture did not stop'); const capture = final.observability?.capture || final; const persistence = final.observability?.persistence || {}; - report = { live: true, packagedCandidate: true, apiPortIsolated: true, durationSeconds: Math.round((Date.now() - started) / 1000), + report = { live: true, packagedCandidate: true, apiPortIsolated: true, durationSeconds: Math.round((performance.now() - started) / 1000), accepted: capture.ipc_accepted, acknowledged: capture.journal_acknowledged_records, pendingFinal: capture.journal_pending_records, pendingPeak, interfaceChanges, routeSwitches: capture.route_switches, + leftInitialInterface: !!cycle.left, returnedToInitialInterface: !!cycle.returned, trafficAfterReturn: !!cycle.trafficAfterReturn, historyFailed: persistence.failed_writes, historyDropped: persistence.dropped_writes, driverBufferDroppedReported: capture.driver_stats?.buffer_dropped_reported ?? null, driverCoverageComplete: capture.driver_stats?.coverage_complete ?? null, - stopMilliseconds: Date.now() - stoppedAt, selectionMilliseconds, + stopMilliseconds: Math.round(performance.now() - stoppedAt), selectionMilliseconds, sampledApiP95Milliseconds: requestDurations.toSorted((a, b) => a - b)[Math.max(0, Math.ceil(requestDurations.length * .95) - 1)], - rendererErrors: errors.length, profilePeakBytes: peakBytes }; + rendererErrors, profilePeakBytes: peakBytes }; console.log(JSON.stringify({ stage: 'capture_stopped', ...report })); assert.ok(report.acknowledged > 0, 'No history-confirmed traffic'); assert.equal(report.accepted, report.acknowledged, 'Capture acknowledgement mismatch'); @@ -147,9 +169,10 @@ try { catch { if (ownerPid) execFileSync('taskkill', ['/PID', String(ownerPid), '/T', '/F'], { windowsHide: true, stdio: 'ignore', timeout: 10000 }); } finally { clearTimeout(timer); } } - assert.equal(path.dirname(profile), path.resolve(os.tmpdir())); + assert.equal(path.dirname(profile), profileParent); assert.ok(path.basename(profile).startsWith('netbot-candidate-smoke-')); await fs.rm(profile, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 }); await new Promise(resolve => listener.close(resolve)); } console.log(JSON.stringify({ ...report, temporaryProfileRemoved: true })); +if (soak?.requireVpnCycle) assert.ok(report.trafficAfterReturn, 'Complete VPN binding return with subsequent confirmed traffic was not observed'); diff --git a/scripts/qa/candidate_soak.py b/scripts/qa/candidate_soak.py new file mode 100644 index 0000000..56e5cdf --- /dev/null +++ b/scripts/qa/candidate_soak.py @@ -0,0 +1,205 @@ +"""Supervise a 5..30 minute packaged live UI test with disposable bounded storage.""" +from __future__ import annotations + +import argparse +from collections import deque +import json +import os +from pathlib import Path +import secrets +import re +import shutil +import subprocess +import sys +import tempfile +import time + +import psutil + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from scripts.qa.journal_history_probe import workspace_bytes, stop_owned_tree + +MIB = 1024**2 + + +def budget_reason(*, elapsed, duration, storage, rss, free, children, log_bytes): + if elapsed > duration + 120: + return 'soak_deadline_exceeded' + if storage > 512 * MIB: + return 'soak_storage_budget_exceeded' + if rss > 2048 * MIB: + return 'soak_memory_budget_exceeded' + if free < 2048 * MIB: + return 'soak_disk_headroom_exhausted' + if children > 64: + return 'soak_process_budget_exceeded' + if log_bytes > 2 * MIB: + return 'soak_log_budget_exceeded' + return None + + +def read_reports(log): + if not log.exists(): + return [] + result = [] + with log.open('rb') as stream: + data = stream.read(2 * MIB) + for line in data.decode('utf-8', errors='replace').splitlines(): + try: + row = json.loads(line) + if isinstance(row, dict) and (row.get('stage') in {'capture_started', 'route_baseline_ready', 'capture_sample', 'capture_stopped'} or row.get('temporaryProfileRemoved') is True): + result.append(row) + except ValueError: + pass + return result + + +def retire_exited(observed, previous_cpu): + for key, item in list(observed.items()): + try: + alive = item.is_running() + except psutil.NoSuchProcess: + alive = False + if not alive: + observed.pop(key, None) + previous_cpu.pop(key, None) + + +def run_soak(executable, duration=900, require_vpn_cycle=False): + if not isinstance(duration, int) or not 300 <= duration <= 1800: + raise ValueError('Soak duration must be 300..1800 seconds') + if os.name != 'nt': + raise RuntimeError('Packaged candidate soak requires Windows') + executable = Path(executable).resolve() + if not executable.is_file(): + raise ValueError('Explicit candidate executable required') + manifest = json.loads((executable.parents[2] / 'candidate.json').read_text(encoding='utf-8')) + if Path(manifest.get('executable', '')).resolve() != executable or not re.fullmatch('[0-9a-f]{40}', manifest.get('revision', '')): + raise ValueError('Candidate build manifest does not match the executable') + node = shutil.which('node') + if not node or shutil.disk_usage(tempfile.gettempdir()).free < 3 * 1024**3: + raise RuntimeError('Node and at least 3 GiB free scratch space are required') + with tempfile.TemporaryDirectory(prefix='netbot-candidate-soak-') as temporary: + folder = Path(temporary).resolve() + nonce = secrets.token_hex(24) + policy = {'version': 1, 'nonce': nonce, 'durationSeconds': duration, + 'maxProfileBytes': 512 * MIB, 'requireVpnCycle': bool(require_vpn_cycle)} + (folder / 'policy.json').write_text(json.dumps(policy), encoding='utf-8') + env = {**os.environ, 'NETBOT_CANDIDATE_EXE': str(executable), + 'NETBOT_SMOKE_PYTHON_BIN': sys.executable, 'NETBOT_CANDIDATE_DURATION': str(duration), + 'NETBOT_CANDIDATE_SOAK_ROOT': str(folder), 'NETBOT_CANDIDATE_SOAK_NONCE': nonce} + log = folder / 'worker.log' + started = time.monotonic() + peak_storage = peak_rss = 0 + previous_cpu = {} + total_cpu = 0.0 + observed = {} + observed_total = active_peak = 0 + roles = {} + series = deque(maxlen=128) + reported = 0 + reason = None + with log.open('wb') as output: + process = subprocess.Popen([node, str(ROOT / 'frontend/desktop-reference/test-candidate-capture.mjs')], + cwd=folder, env=env, stdout=output, stderr=output) + try: + while process.poll() is None: + elapsed = time.monotonic() - started + rss = 0 + try: + owner = psutil.Process(process.pid) + items = [owner, *owner.children(recursive=True)] + for item in items: + try: + key = (item.pid, item.create_time()) + if key not in observed: + observed_total += 1 + command = item.cmdline() + name = item.name().lower() + role = ('interface_discovery' if '--capture-discovery-json' in command else + 'capture_probe' if '--capture-backend-probe' in command else + 'electron' if name == 'netbotpro candidate.exe' else + 'backend' if name == 'netbotpro-backend.exe' else + 'node' if name == 'node.exe' else 'other') + roles[role] = roles.get(role, 0) + 1 + observed[key] = item + except psutil.NoSuchProcess: + pass + retire_exited(observed, previous_cpu) + for key, item in list(observed.items()): + try: + rss += item.memory_info().rss + times = item.cpu_times() + cpu = times.user + times.system + total_cpu += max(0, cpu - previous_cpu.get(key, cpu)) + previous_cpu[key] = cpu + except psutil.NoSuchProcess: + pass + except psutil.NoSuchProcess: + items = [] + storage = workspace_bytes(folder) + active_peak = max(active_peak, len(observed)) + peak_storage, peak_rss = max(peak_storage, storage), max(peak_rss, rss) + reason = budget_reason(elapsed=elapsed, duration=duration, storage=storage, rss=rss, + free=shutil.disk_usage(folder).free, children=len(observed), log_bytes=log.stat().st_size) + if reason: + break + rows = read_reports(log) + for row in rows[reported:]: + print(json.dumps(row), flush=True) + reported = len(rows) + if not series or elapsed - series[-1]['seconds'] >= 30: + sample = {'stage': 'resource_sample', 'seconds': round(elapsed, 1), + 'aggregate_process_rss_mb': round(rss / MIB, 2), 'temporary_mb': round(storage / MIB, 2), + 'active_processes': len(observed), 'observed_processes_total': observed_total, + 'sampled_process_roles': dict(roles)} + series.append(sample) + print(json.dumps(sample), flush=True) + time.sleep(1) + finally: + if process.poll() is None: + stop_owned_tree(process) + for item in observed.values(): + try: + if item.is_running(): + item.kill() + except psutil.NoSuchProcess: + pass + psutil.wait_procs(list(observed.values()), timeout=5) + process.wait(timeout=5) + rows = read_reports(log) + for row in rows[reported:]: + print(json.dumps(row), flush=True) + completed = next((row for row in reversed(rows) if row.get('temporaryProfileRemoved')), None) + elapsed = time.monotonic() - started + if reason is None: + reason = ('candidate_worker_failed' if process.returncode else 'missing_candidate_report' if completed is None + else 'soak_completed_too_early' if elapsed < duration else None) + report = {'ok': reason is None, + 'reason': reason, + 'candidate_source_revision': manifest['revision'], + 'requested_seconds': duration, 'supervisor_seconds': round(elapsed, 2), + 'temporary_peak_bytes': peak_storage, 'aggregate_process_peak_rss_mb': round(peak_rss / MIB, 2), + 'sampled_aggregate_cpu_one_core_percent': round(100 * total_cpu / elapsed, 2), + 'active_processes_peak': active_peak, 'observed_processes_total': observed_total, + 'sampled_process_roles': roles, 'candidate': completed, + 'resource_samples': list(series)} + report['temporary_workspace_removed'] = not folder.exists() + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--executable', required=True) + parser.add_argument('--duration', type=int, default=900) + parser.add_argument('--require-vpn-cycle', action='store_true') + args = parser.parse_args() + report = run_soak(args.executable, args.duration, args.require_vpn_cycle) + print(json.dumps(report), flush=True) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_candidate_soak.py b/tests/test_candidate_soak.py new file mode 100644 index 0000000..cfa20f2 --- /dev/null +++ b/tests/test_candidate_soak.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock + +from scripts.qa.candidate_soak import MIB, budget_reason, read_reports, run_soak, retire_exited + + +class CandidateSoakTests(unittest.TestCase): + def test_completed_helpers_do_not_exhaust_active_process_budget(self): + dead = Mock() + dead.is_running.return_value = False + alive = Mock() + alive.is_running.return_value = True + observed = {('dead', 1): dead, ('live', 2): alive} + cpu = {('dead', 1): 4, ('live', 2): 2} + retire_exited(observed, cpu) + self.assertEqual(observed, {('live', 2): alive}) + self.assertEqual(cpu, {('live', 2): 2}) + + def test_normal_sample_stays_within_budget(self): + self.assertIsNone(budget_reason(elapsed=10, duration=900, storage=MIB, rss=MIB, free=4096*MIB, children=5, log_bytes=100)) + + def test_each_resource_guard_fails_independently(self): + base = dict(elapsed=10, duration=900, storage=MIB, rss=MIB, free=4096*MIB, children=5, log_bytes=100) + for change, expected in [({'elapsed': 1021}, 'deadline'), ({'storage': 513*MIB}, 'storage'), + ({'rss': 2049*MIB}, 'memory'), ({'free': MIB}, 'headroom'), + ({'children': 65}, 'process'), ({'log_bytes': 3*MIB}, 'log')]: + with self.subTest(expected=expected): + self.assertIn(expected, budget_reason(**{**base, **change})) + + def test_log_reader_ignores_non_reports(self): + with tempfile.TemporaryDirectory() as folder: + log = Path(folder) / 'worker.log' + log.write_text('not json\n' + json.dumps({'nonce': 'not-a-report'}) + '\n' + json.dumps({'stage': 'capture_sample', 'accepted': 10})) + self.assertEqual(read_reports(log), [{'stage': 'capture_sample', 'accepted': 10}]) + + def test_long_mode_rejects_unbounded_durations_before_launch(self): + for duration in (0, 120, 1801, 14400, 900.5): + with self.subTest(duration=duration), self.assertRaises(ValueError): + run_soak('not-launched', duration) diff --git a/tests/test_capture_provider.py b/tests/test_capture_provider.py index ad92b56..33d2092 100644 --- a/tests/test_capture_provider.py +++ b/tests/test_capture_provider.py @@ -17,6 +17,18 @@ def detail(self, os_name: str) -> str: class CaptureProviderTests(unittest.TestCase): + def test_snapshot_preflight_does_not_rediscover_interfaces(self): + def unexpected_discovery(): + raise AssertionError('snapshot preflight must not enumerate twice') + provider = SystemCaptureProvider(interfaces_func=unexpected_discovery, + privilege_checker=_PrivilegeChecker(elevated=True), os_name_getter=lambda: 'linux', + scapy_checker=lambda: (True, 'ok')) + snapshot = {'items': [{'value': 'fixture'}], 'recommended': 'fixture', 'source': 'live'} + report = provider.preflight_for_interfaces(snapshot).to_dict() + self.assertTrue(report['ready']) + self.assertEqual(report['recommended_interface'], 'fixture') + self.assertEqual(snapshot, {'items': [{'value': 'fixture'}], 'recommended': 'fixture', 'source': 'live'}) + def test_capture_backend_warmup_is_opt_in(self): with patch("core.capture.system_provider.ensure_capture_backend") as ensure_capture_backend: SystemCaptureProvider(os_name_getter=lambda: "windows") diff --git a/tests/test_desktop_candidate_builder.py b/tests/test_desktop_candidate_builder.py index a151646..f6a13cf 100644 --- a/tests/test_desktop_candidate_builder.py +++ b/tests/test_desktop_candidate_builder.py @@ -1,5 +1,6 @@ from pathlib import Path import tempfile +import time from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -10,27 +11,30 @@ class CandidateBuildGuardTests(unittest.TestCase): def test_repeated_builds_cannot_accumulate_unbounded_artifacts(self): with patch.object(candidate, 'folder_bytes', return_value=2 * candidate.GIB + 1), \ - patch.object(candidate.shutil, 'disk_usage', return_value=SimpleNamespace(free=10 * candidate.GIB)): + patch.object(candidate, 'shutil', SimpleNamespace(disk_usage=lambda _: SimpleNamespace(free=10 * candidate.GIB))): with self.assertRaisesRegex(RuntimeError, 'cache_budget_exceeded'): candidate.check_build_admission(Path.cwd()) def test_admission_requires_free_disk_headroom(self): - with patch.object(candidate.shutil, 'disk_usage', return_value=SimpleNamespace(free=candidate.GIB)): + with patch.object(candidate, 'shutil', SimpleNamespace(disk_usage=lambda _: SimpleNamespace(free=candidate.GIB))): with self.assertRaisesRegex(RuntimeError, '5 GiB'): candidate.check_build_admission(Path.cwd()) def run_guard(self, process, *, size=0, clock=(0, 1)): owner = Mock() owner.children.return_value = [] + real_clock = time.monotonic + process_api = SimpleNamespace(Process=Mock(return_value=owner), wait_procs=Mock(), NoSuchProcess=candidate.psutil.NoSuchProcess) + subprocess_api = SimpleNamespace(Popen=Mock(return_value=process), STDOUT=candidate.subprocess.STDOUT) + clock_api = SimpleNamespace(monotonic=Mock(side_effect=clock), sleep=Mock()) with tempfile.TemporaryDirectory(prefix='netbot-candidate-unit-') as folder: - with patch.object(candidate.subprocess, 'Popen', return_value=process), \ - patch.object(candidate.psutil, 'Process', return_value=owner), \ - patch.object(candidate.psutil, 'wait_procs'), \ + with patch.object(candidate, 'subprocess', subprocess_api), \ + patch.object(candidate, 'psutil', process_api), \ patch.object(candidate, 'folder_bytes', return_value=size), \ - patch.object(candidate.shutil, 'disk_usage', return_value=SimpleNamespace(free=10 * candidate.GIB)), \ - patch.object(candidate.time, 'monotonic', side_effect=clock), \ - patch.object(candidate.time, 'sleep'): + patch.object(candidate, 'shutil', SimpleNamespace(disk_usage=lambda _: SimpleNamespace(free=10 * candidate.GIB))), \ + patch.object(candidate, 'time', clock_api): try: + self.assertIs(time.monotonic, real_clock) candidate.guarded_run(['fixture-command'], Path(folder), Path(folder)) finally: self.owner = owner diff --git a/tests/test_sniffer_service.py b/tests/test_sniffer_service.py index af00ce7..c5050f0 100644 --- a/tests/test_sniffer_service.py +++ b/tests/test_sniffer_service.py @@ -173,6 +173,23 @@ def test_capture_interfaces_include_preflight(self): self.assertEqual(payload["recommended"], "eth0") self.assertEqual(payload["preflight"]["provider"], "fake") + def test_capture_interfaces_reuses_snapshot_when_provider_supports_it(self): + provider = _FakeCaptureProvider() + original = provider.list_interfaces() + seen = [] + provider.list_interfaces = lambda: original + report = provider.preflight() + provider.preflight_for_interfaces = lambda payload: (seen.append(payload), report)[1] + provider.preflight = lambda: self.fail('snapshot provider must not run a second preflight discovery') + service = SnifferService(EventBus(), capture_provider=provider) + try: + payload = service.capture_interfaces() + self.assertIs(seen[0], original) + self.assertNotIn('preflight', original) + self.assertTrue(payload['preflight']['ready']) + finally: + service.close() + def test_start_fails_fast_when_preflight_is_not_ready(self): service = SnifferService( EventBus(), capture_provider=_UnavailableCaptureProvider()