From be4edcd46866f7ef7b85f7b148e96324f49ddc00 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:26:15 -0700 Subject: [PATCH 01/14] feat(envelope): qualify hostile and production resource envelopes (#19) PdfTool benchmark gains a measured preflight phase (--profile), renders in cancellable slices, and handles SIGBREAK on Windows. A synthetic fixture generator, cancellation/recovery probes, a hostile corpus lane, schema-2 evidence, and a hosted Linux/Windows qualification workflow make the strict matrix runnable in CI. Co-Authored-By: Claude Opus 5.5 --- .../resource-envelope-qualification.yml | 302 +++++++++++++ .github/workflows/reusable-linux.yml | 2 +- .github/workflows/reusable-windows.yml | 2 +- PdfTool/main.cpp | 6 +- PdfTool/pdftoolabstractapplication.cpp | 14 + PdfTool/pdftoolabstractapplication.h | 1 + PdfTool/pdftoolrender.cpp | 137 +++++- PdfTool/pdftoolrender.h | 13 + UnitTests/tst_pdftoolcontract.cpp | 41 ++ changes/cc-issue-19-resource-envelopes.md | 4 + docs/RESOURCE_ENVELOPE.md | 19 +- docs/RESOURCE_ENVELOPE_BUDGETS.json | 7 + docs/RESOURCE_ENVELOPE_QUALIFICATION.md | 57 ++- .../build_resource_envelope_evidence.py | 144 ++++++ .../test_build_resource_envelope_evidence.py | 80 ++++ .../validate_resource_envelope_evidence.py | 105 ++++- scripts/resource_envelope/run_matrix.py | 416 ++++++++++++++---- .../resource_envelope/synthetic_workload.py | 248 +++++++++++ .../test_run_matrix_probes.py | 203 +++++++++ .../test_synthetic_workload.py | 66 +++ 20 files changed, 1743 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/resource-envelope-qualification.yml create mode 100644 changes/cc-issue-19-resource-envelopes.md create mode 100644 scripts/qualification/build_resource_envelope_evidence.py create mode 100644 scripts/qualification/test_build_resource_envelope_evidence.py create mode 100644 scripts/resource_envelope/synthetic_workload.py create mode 100644 scripts/resource_envelope/test_run_matrix_probes.py create mode 100644 scripts/resource_envelope/test_synthetic_workload.py diff --git a/.github/workflows/resource-envelope-qualification.yml b/.github/workflows/resource-envelope-qualification.yml new file mode 100644 index 000000000..675546263 --- /dev/null +++ b/.github/workflows/resource-envelope-qualification.yml @@ -0,0 +1,302 @@ +name: Resource envelope qualification + +# Hosted qualification for issue #19: builds PdfTool from the candidate SHA, +# generates the deterministic synthetic fixture bundle, and runs the strict +# resource-envelope matrix (measured fixtures, cancellation/recovery probe, +# hostile corpus) on Linux and Windows. The evidence job turns both matrices +# into one record carrying this run's id. + +on: + workflow_dispatch: + pull_request: + paths: + - 'scripts/resource_envelope/**' + - 'scripts/qualification/*resource_envelope*' + - 'PdfTool/main.cpp' + - 'PdfTool/pdftoolrender.*' + - 'LoopLibCore/sources/pdfworkloadenvelope.*' + - 'LoopLibCore/sources/pdfresourcebudget.*' + - 'LoopLibCore/sources/pdfrenderer.*' + - 'docs/RESOURCE_ENVELOPE_BUDGETS.json' + - '.github/workflows/resource-envelope-qualification.yml' + +permissions: + contents: read + +concurrency: + group: resource-envelope-${{ github.ref }} + cancel-in-progress: true + +env: + REPETITIONS: 3 + CANCEL_FIXTURE: ten-thousand-page + CANCEL_AFTER_SECONDS: 3 + TIMEOUT_SECONDS: 900 + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}/loop/vcpkg/overlays/linux:${{ github.workspace }}/loop/vcpkg/overlays/general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Install Dependencies + run: | + mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE" + sudo apt update + sudo apt install -y autoconf autoconf-archive automake libtool libcups2 libcups2-dev libfontconfig1-dev + + - name: 'VCPKG: Set up VCPKG' + run: | + VCPKG_COMMIT="$(python3 -c 'import json; print(json.load(open("loop/vcpkg-configuration.json"))["default-registry"]["baseline"])')" + if ! [[ "$VCPKG_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::Invalid vcpkg baseline: $VCPKG_COMMIT" + exit 1 + fi + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach "$VCPKG_COMMIT" + test "$(git -C vcpkg rev-parse HEAD)" = "$VCPKG_COMMIT" + ./vcpkg/bootstrap-vcpkg.sh + ./vcpkg/vcpkg integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg_installed + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + run: | + ./vcpkg install --x-manifest-root=$GITHUB_WORKSPACE/loop --x-install-root=$VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'linux' + target: 'desktop' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + + - name: Build PdfTool + working-directory: loop + run: | + cmake -B build -S . -DLOOP_LOOP_DISTRIBUTION=ON -DLOOP_INSTALL_QT_DEPENDENCIES=0 -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release + cmake --build build --target PdfTool --config Release -j6 + PDF_TOOL="$(find "$PWD/build" -type f -name PdfTool -perm -u+x | head -n 1)" + test -n "$PDF_TOOL" + echo "PDF_TOOL=$PDF_TOOL" >> "$GITHUB_ENV" + "$PDF_TOOL" help --console-format json > /dev/null + + - name: Generate synthetic fixture bundle + working-directory: loop + run: python3 scripts/resource_envelope/synthetic_workload.py --output-dir "$RUNNER_TEMP/fixtures" --manifest "$RUNNER_TEMP/fixtures/fixtures.json" + + - name: Run strict resource-envelope matrix + working-directory: loop + run: | + mkdir -p "$RUNNER_TEMP/qualification" + cp "$RUNNER_TEMP/fixtures/fixtures.json" "$RUNNER_TEMP/qualification/fixtures.json" + python3 -m scripts.resource_envelope.run_matrix \ + --pdf-tool "$PDF_TOOL" \ + --manifest "$RUNNER_TEMP/fixtures/fixtures.json" \ + --repetitions "$REPETITIONS" --timeout-seconds "$TIMEOUT_SECONDS" \ + --cancel-fixture "$CANCEL_FIXTURE" --cancel-after-seconds "$CANCEL_AFTER_SECONDS" \ + --strict \ + --output "$RUNNER_TEMP/qualification/matrix-linux.json" + + - name: Upload Linux matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-linux + path: ${{ runner.temp }}/qualification/ + if-no-files-found: warn + + windows: + runs-on: windows-2022 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}\loop\vcpkg\overlays\general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}\vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}\vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Exclude workspace from Windows Defender real-time scanning + shell: pwsh + run: | + Add-MpPreference -ExclusionPath "${env:GITHUB_WORKSPACE}" -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $env:VCPKG_DEFAULT_BINARY_CACHE | Out-Null + + - name: 'VCPKG: Set up VCPKG' + shell: pwsh + run: | + $config = Get-Content (Join-Path $env:GITHUB_WORKSPACE "loop\vcpkg-configuration.json") -Raw | ConvertFrom-Json + $vcpkgCommit = $config.'default-registry'.baseline + if ($vcpkgCommit -notmatch '^[0-9a-f]{40}$') { throw "Invalid vcpkg baseline: $vcpkgCommit" } + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach $vcpkgCommit + if ((git -C vcpkg rev-parse HEAD).Trim() -ne $vcpkgCommit) { throw "vcpkg checkout does not match manifest baseline" } + .\vcpkg\bootstrap-vcpkg.bat -disableMetrics + .\vcpkg\vcpkg.exe integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg/installed + ./vcpkg/archives + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + shell: pwsh + run: | + $manifestRoot = Join-Path $env:GITHUB_WORKSPACE 'loop' + .\vcpkg.exe install --triplet x64-windows --x-manifest-root=$manifestRoot --x-install-root=$env:VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + if ($LASTEXITCODE -ne 0) { throw "vcpkg install failed with exit code $LASTEXITCODE." } + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'windows' + target: 'desktop' + arch: 'win64_msvc2022_64' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + # Same aqtinstall pin as reusable-windows.yml (miurahr/aqtinstall#1007). + aqtsource: git+https://github.com/miurahr/aqtinstall.git@8c3695d4a4e1ceabf6a74dc6c79681656dc6b74b + + - name: Build PdfTool + working-directory: loop + shell: pwsh + run: | + cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release -DLOOP_LOOP_DISTRIBUTION=OFF -DLOOP_INSTALL_QT_DEPENDENCIES=OFF -DCMAKE_TOOLCHAIN_FILE="${env:GITHUB_WORKSPACE}\vcpkg\scripts\buildsystems\vcpkg.cmake" -DLOOP_QT_ROOT="${env:QT_ROOT_DIR}" + if ($LASTEXITCODE -ne 0) { throw "cmake configure failed with exit code $LASTEXITCODE." } + cmake --build build --target PdfTool --config Release -j6 + if ($LASTEXITCODE -ne 0) { throw "cmake --build PdfTool failed with exit code $LASTEXITCODE." } + $pdfTool = Get-ChildItem build -Recurse -Filter PdfTool.exe | Select-Object -First 1 + if (-not $pdfTool) { throw "PdfTool.exe was not produced." } + "PDF_TOOL=$($pdfTool.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "QT_PLUGIN_PATH=$(Join-Path $env:QT_ROOT_DIR 'plugins')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "$(Join-Path $env:QT_ROOT_DIR 'bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + "$(Join-Path $env:VCPKG_INSTALLED_DIR 'x64-windows\bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + + - name: Smoke PdfTool runtime + working-directory: loop + shell: pwsh + run: | + & $env:PDF_TOOL help --console-format json | Out-Null + if ($LASTEXITCODE -ne 0) { throw "PdfTool help failed with exit code $LASTEXITCODE." } + + - name: Generate synthetic fixture bundle + working-directory: loop + shell: pwsh + run: | + python scripts/resource_envelope/synthetic_workload.py --output-dir "$env:RUNNER_TEMP\fixtures" --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" + if ($LASTEXITCODE -ne 0) { throw "fixture generation failed with exit code $LASTEXITCODE." } + + - name: Run strict resource-envelope matrix + working-directory: loop + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:RUNNER_TEMP\qualification" | Out-Null + python -m scripts.resource_envelope.run_matrix ` + --pdf-tool "$env:PDF_TOOL" ` + --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" ` + --repetitions $env:REPETITIONS --timeout-seconds $env:TIMEOUT_SECONDS ` + --cancel-fixture $env:CANCEL_FIXTURE --cancel-after-seconds $env:CANCEL_AFTER_SECONDS ` + --strict ` + --output "$env:RUNNER_TEMP\qualification\matrix-windows.json" + if ($LASTEXITCODE -ne 0) { throw "strict resource-envelope matrix failed with exit code $LASTEXITCODE." } + + - name: Upload Windows matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-windows + path: ${{ runner.temp }}\qualification\ + if-no-files-found: warn + + evidence: + needs: [linux, windows] + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + + - name: Download matrices + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: resource-envelope-matrix-* + path: ${{ runner.temp }}/matrices + + - name: Build and validate qualification evidence + working-directory: loop + run: | + set -euo pipefail + MATRICES=() + for platform in linux windows; do + matrix="$RUNNER_TEMP/matrices/resource-envelope-matrix-$platform/matrix-$platform.json" + if [ -f "$matrix" ]; then MATRICES+=(--matrix "$platform=$matrix"); fi + done + manifest="$(find "$RUNNER_TEMP/matrices" -name fixtures.json | head -n 1)" + test -n "$manifest" || { echo "::error::no fixture manifest was uploaded"; exit 1; } + python3 scripts/qualification/build_resource_envelope_evidence.py \ + "${MATRICES[@]}" \ + --fixture-manifest "$manifest" \ + --run-id "$GITHUB_RUN_ID" \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + --output "$RUNNER_TEMP/evidence/evidence.json" + python3 scripts/qualification/validate_resource_envelope_evidence.py --evidence "$RUNNER_TEMP/evidence/evidence.json" --skip-manifest + cp "$manifest" "$RUNNER_TEMP/evidence/fixture-manifest.json" + python3 -c 'import json,sys; e=json.load(open(sys.argv[1])); print("disposition:", e["disposition"]); [print(" -", r) for r in e["disposition_reasons"]]; sys.exit(0 if e["disposition"] == "passed" else 1)' "$RUNNER_TEMP/evidence/evidence.json" + + - name: Upload qualification evidence + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-evidence + path: ${{ runner.temp }}/evidence/ + if-no-files-found: warn diff --git a/.github/workflows/reusable-linux.yml b/.github/workflows/reusable-linux.yml index 2fb039748..b1cc18992 100644 --- a/.github/workflows/reusable-linux.yml +++ b/.github/workflows/reusable-linux.yml @@ -78,7 +78,7 @@ jobs: - name: Verify resource-envelope contracts working-directory: loop run: | - python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python3 scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python3 scripts/qualification/validate_resource_envelope_evidence.py python3 scripts/resource_envelope/pathological_workload.py \ diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index b0d9c1414..938f7256c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -82,7 +82,7 @@ jobs: working-directory: loop shell: pwsh run: | - python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python scripts/qualification/validate_resource_envelope_evidence.py python scripts/resource_envelope/pathological_workload.py ` diff --git a/PdfTool/main.cpp b/PdfTool/main.cpp index 7850d096e..712309d80 100644 --- a/PdfTool/main.cpp +++ b/PdfTool/main.cpp @@ -348,7 +348,11 @@ int main(int argc, char* argv[]) pdftool::resetCancelRequested(); std::signal(SIGINT, handleTerminationSignal); -#ifndef Q_OS_WIN +#ifdef Q_OS_WIN + // CTRL_BREAK_EVENT is the only console interrupt deliverable to a child + // started in its own process group; the CRT raises it as SIGBREAK. + std::signal(SIGBREAK, handleTerminationSignal); +#else std::signal(SIGTERM, handleTerminationSignal); #endif diff --git a/PdfTool/pdftoolabstractapplication.cpp b/PdfTool/pdftoolabstractapplication.cpp index d5c81b2bc..096353938 100644 --- a/PdfTool/pdftoolabstractapplication.cpp +++ b/PdfTool/pdftoolabstractapplication.cpp @@ -388,6 +388,10 @@ QList PDFToolAbstractApplication::describeOptions(Optio { add(QStringLiteral("report-file"), { QStringLiteral("--report-file") }, QStringLiteral("file"), PDFToolValueType::Path); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + add(QStringLiteral("profile"), { QStringLiteral("--profile") }, QStringLiteral("profile"), PDFToolValueType::Path); + } if (optionFlags.testFlag(CapabilityDiscovery)) { add(QStringLiteral("command"), { QStringLiteral("--command") }, QStringLiteral("id"), PDFToolValueType::String); @@ -900,6 +904,11 @@ void PDFToolAbstractApplication::initializeCommandLineParser(QCommandLineParser* addDescribedOption(parser, optionDescriptors, QStringLiteral("report-file"), QStringLiteral("Write the preflight report JSON this run is certified over to this file.")); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + addDescribedOption(parser, optionDescriptors, QStringLiteral("profile"), QStringLiteral("Run a preflight phase with this profile before rendering and record its memory high-water.")); + } + if (optionFlags.testFlag(CapabilityDiscovery)) { addDescribedOption(parser, optionDescriptors, QStringLiteral("command"), QStringLiteral("Limit discovery to one stable command ID.")); @@ -1483,6 +1492,11 @@ PDFToolOptions PDFToolAbstractApplication::getOptions(QCommandLineParser* parser options.preflightReportPath = parser->value("report-file"); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + options.preflightProfilePath = parser->value("profile"); + } + if (optionFlags.testFlag(VerifyPreflightCertificate)) { options.preflightCertificatePath = positionalArguments.value(0); diff --git a/PdfTool/pdftoolabstractapplication.h b/PdfTool/pdftoolabstractapplication.h index fabe7644b..cdf9335b9 100644 --- a/PdfTool/pdftoolabstractapplication.h +++ b/PdfTool/pdftoolabstractapplication.h @@ -431,6 +431,7 @@ class PDFToolAbstractApplication EvidenceBundleExport = 0x80000000000ULL, ///< Export a portable proof-of-preflight bundle EvidenceBundleVerify = 0x100000000000ULL, ///< Verify a portable proof-of-preflight bundle PreflightReportFile = 0x200000000000ULL, ///< Preflight --report-file output path + BenchmarkPreflightProfile = 0x400000000000ULL, ///< Benchmark --profile for a measured preflight phase }; Q_DECLARE_FLAGS(Options, Option) diff --git a/PdfTool/pdftoolrender.cpp b/PdfTool/pdftoolrender.cpp index a1910fc98..f277be7d5 100644 --- a/PdfTool/pdftoolrender.cpp +++ b/PdfTool/pdftoolrender.cpp @@ -25,20 +25,40 @@ #include "pdfdocumentsession.h" #include "pdffont.h" #include "pdfconstants.h" +#include "pdfoperationcontrol.h" #include "pdfsafefilewriter.h" #include "pdfworkloadenvelope.h" +#include "preflightclirun.h" +#include "preflightengine.h" +#include "preflightprofileresolver.h" #include #include #include #include +#include + namespace pdftool { static PDFToolRender s_toolRenderApplication; static PDFToolBenchmark s_toolBenchmarkApplication; +namespace +{ + +class CliCancellationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override + { + return isCancelRequested(); + } +}; + +} // namespace + QString PDFToolRender::getStandardString(PDFToolAbstractApplication::StandardString standardString) const { switch (standardString) @@ -160,7 +180,54 @@ QString PDFToolBenchmark::getStandardString(PDFToolAbstractApplication::Standard PDFToolAbstractApplication::Options PDFToolBenchmark::getOptionsFlags() const { - return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags; + return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags | BenchmarkPreflightProfile; +} + +PDFToolExitCode PDFToolBenchmark::execute(const PDFToolOptions& options) +{ + m_preflightPhase = PreflightPhase(); + if (options.preflightProfilePath.isEmpty()) + { + return PDFToolRenderBase::execute(options); + } + + QJsonObject profileJson; + QString profileError; + if (!pdf::PreflightEngine::loadProfile(options.preflightProfilePath, profileJson, profileError)) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("cli.invalid-arguments"), profileError); + return PDFToolExitCode::InvalidInvocation; + } + const pdf::PreflightProfileImportResult imported = pdf::importPreflightProfile(profileJson, options.preflightProfilePath); + if (!imported.ok) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, imported.errorCode, imported.errorMessage); + return PDFToolExitCode::InvalidInvocation; + } + + CliCancellationControl cancellationControl; + pdf::PreflightFileInspectionRequest request; + request.documentPath = options.document; + request.password = options.password; + request.permissiveReading = options.permissiveReading; + request.profile = imported.profile; + request.plan.full = true; + request.plan.reason = QStringLiteral("benchmark-preflight-phase"); + request.firstPage = options.pageSelectorFirstPage; + request.lastPage = options.pageSelectorLastPage; + request.selectedPages = options.pageSelectorSelection; + request.cancellation = &cancellationControl; + + // The outcome owns the full source bytes; it is released here, before the + // render phase opens the document again. + { + const pdf::PreflightFileInspectionOutcome outcome = pdf::inspectPreflightFile(request); + m_preflightPhase.requested = true; + m_preflightPhase.inspected = outcome.documentReadOk && outcome.inspectionRan && !isCancelRequested(); + m_preflightPhase.highWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + } + + return PDFToolRenderBase::execute(options); } void PDFToolBenchmark::finish(const PDFToolOptions& options) @@ -207,26 +274,52 @@ void PDFToolBenchmark::finish(const PDFToolOptions& options) pdf::PDFWorkloadEnvelope envelope; envelope.identity = identity; envelope.family = QStringLiteral("benchmark-render"); - const bool cancelled = isCancelRequested(); - envelope.status = cancelled - ? QStringLiteral("cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("budget-exceeded") - : QStringLiteral("incomplete"); envelope.pageCount = static_cast(m_pageInfo.size()); - envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); - envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); - envelope.elapsedMs = m_wallTime; - envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; - envelope.incompleteReason = cancelled - ? QStringLiteral("operation-cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("resource-budget-exceeded") - : QStringLiteral("preflight-measurement-unavailable"); qint64 pagesMaterialized = 0; for (const PageInfo& page : m_pageInfo) { pagesMaterialized += page.isRendered ? 1 : 0; } envelope.pagesMaterialized = pagesMaterialized; + + // A record is complete only when every phase it claims was measured: + // an unmeasured preflight or an unrendered page keeps it incomplete. + const bool cancelled = isCancelRequested(); + if (cancelled) + { + envelope.status = QStringLiteral("cancelled"); + envelope.incompleteReason = QStringLiteral("operation-cancelled"); + } + else if (m_resourceBudgetExhausted) + { + envelope.status = QStringLiteral("budget-exceeded"); + envelope.incompleteReason = QStringLiteral("resource-budget-exceeded"); + } + else if (!m_preflightPhase.requested) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-measurement-unavailable"); + } + else if (!m_preflightPhase.inspected) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-phase-failed"); + } + else if (pagesMaterialized != envelope.pageCount) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("pages-not-materialized"); + } + else + { + envelope.status = QStringLiteral("complete"); + } + + envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); + envelope.preflightHighWaterBytes = m_preflightPhase.highWaterBytes; + envelope.elapsedMs = m_wallTime; + envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; envelope.recordResources(*m_resourceBudget); data.insert(QStringLiteral("workload_envelope"), envelope.toJson()); options.executionContext->setData(data); @@ -373,10 +466,22 @@ PDFToolExitCode PDFToolRenderBase::execute(const PDFToolOptions& options) QElapsedTimer timer; timer.start(); - rasterizerPool.render(pageIndices, imageSizeGetter, std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1), nullptr); + // Render in slices and check for cancellation between them, so an interrupt + // stops the run within one slice instead of after the whole page range. + const auto processImage = std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1); + const std::size_t sliceSize = std::size_t(pdf::PDFRasterizerPool::getCorrectedRasterizerCount(options.renderRasterizerCount)) * 4; + bool resourceBudgetExhausted = false; + for (std::size_t first = 0; first < pageIndices.size() && !isCancelRequested(); first += sliceSize) + { + const std::size_t last = std::min(pageIndices.size(), first + sliceSize); + const std::vector slice(pageIndices.cbegin() + first, pageIndices.cbegin() + last); + rasterizerPool.render(slice, imageSizeGetter, processImage, nullptr); + // render() resets the pool's flag per call, so exhaustion is accumulated here. + resourceBudgetExhausted = resourceBudgetExhausted || rasterizerPool.resourceBudgetExhausted(); + } m_wallTime = timer.elapsed(); - m_resourceBudgetExhausted = rasterizerPool.resourceBudgetExhausted(); + m_resourceBudgetExhausted = resourceBudgetExhausted; fontCache.setCacheShrinkEnabled(nullptr, true); diff --git a/PdfTool/pdftoolrender.h b/PdfTool/pdftoolrender.h index b57f76397..62b3578df 100644 --- a/PdfTool/pdftoolrender.h +++ b/PdfTool/pdftoolrender.h @@ -82,10 +82,23 @@ class PDFToolBenchmark : public PDFToolRenderBase public: virtual QString getStandardString(StandardString standardString) const override; virtual Options getOptionsFlags() const override; + virtual PDFToolExitCode execute(const PDFToolOptions& options) override; protected: virtual void finish(const PDFToolOptions& options) override; virtual void onPageRendered(const PDFToolOptions& options, pdf::PDFRenderedPageImage& renderedPageImage) override; + +private: + /// Preflight runs before rendering, so the process high-water recorded when + /// it ends is the peak of open plus preflight, independent of rendering. + struct PreflightPhase + { + bool requested = false; + bool inspected = false; + qint64 highWaterBytes = -1; + }; + + PreflightPhase m_preflightPhase; }; } // namespace pdftool diff --git a/UnitTests/tst_pdftoolcontract.cpp b/UnitTests/tst_pdftoolcontract.cpp index 74d5b6e41..08bf6224f 100644 --- a/UnitTests/tst_pdftoolcontract.cpp +++ b/UnitTests/tst_pdftoolcontract.cpp @@ -134,8 +134,49 @@ private slots: void rgbToCmykRefusesToWriteOverItsOwnInput(); void evidenceBundleExportVerifyPair(); void evidenceBundleRejectsNonJsonOutput(); + void benchmarkWithoutPreflightProfileIsIncomplete(); + void benchmarkWithPreflightProfileIsComplete(); }; +namespace +{ + +QJsonObject runBenchmarkEnvelope(const QStringList& extraArguments) +{ + const QString fixture = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/testdata/fixtures/image-dpi-low.pdf"); + QStringList arguments{ QStringLiteral("benchmark"), fixture, + QStringLiteral("--render-hw-accel"), QStringLiteral("0"), + QStringLiteral("--console-format"), QStringLiteral("json") }; + arguments << extraArguments; + const ToolRun run = runPdfTool(arguments); + verifyEnvelope(run, 0, QStringLiteral("benchmark")); + return run.json.value(QStringLiteral("data")).toObject().value(QStringLiteral("workload_envelope")).toObject(); +} + +} // namespace + +void PdfToolContractTest::benchmarkWithoutPreflightProfileIsIncomplete() +{ + const QJsonObject envelope = runBenchmarkEnvelope({}); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("incomplete")); + QCOMPARE(envelope.value(QStringLiteral("incomplete_reason")).toString(), QStringLiteral("preflight-measurement-unavailable")); + QCOMPARE(envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(), -1); +} + +void PdfToolContractTest::benchmarkWithPreflightProfileIsComplete() +{ + const QString profile = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/profiles/loop-default.json"); + const QJsonObject envelope = runBenchmarkEnvelope({ QStringLiteral("--profile"), profile }); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("complete")); + QVERIFY(envelope.value(QStringLiteral("incomplete_reason")).toString().isEmpty()); + const qint64 preflightHighWater = envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(); + QVERIFY2(preflightHighWater > 0, qPrintable(QString::number(preflightHighWater))); + QVERIFY(envelope.value(QStringLiteral("rss_high_water_bytes")).toInteger() >= preflightHighWater); + QCOMPARE(envelope.value(QStringLiteral("pages_materialized")).toInteger(), envelope.value(QStringLiteral("page_count")).toInteger()); +} + void PdfToolContractTest::helpIsWrapped() { const ToolRun run = runPdfTool({ QStringLiteral("help"), QStringLiteral("--console-format"), QStringLiteral("json") }); diff --git a/changes/cc-issue-19-resource-envelopes.md b/changes/cc-issue-19-resource-envelopes.md new file mode 100644 index 000000000..3fc93b1f7 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.md @@ -0,0 +1,4 @@ +Category: added +Audience: developers and release qualifiers +Breaking-Change: no +Summary: Qualify hostile and production resource envelopes on hosted Linux and Windows runners (#19). `PdfTool benchmark --profile` adds a measured preflight phase, so a clean run reports a complete envelope; rendering stops within one page slice of an interrupt, and Windows honours CTRL_BREAK. A new resource-envelope qualification workflow generates a deterministic synthetic fixture bundle, runs the strict matrix with separate cancellation and reopen-after-cancel recovery probes and a hostile budget-exhaustion lane, and builds schema-2 evidence carrying the CI run id. Crashes, timeouts, and skipped workloads can never count as a passing envelope. diff --git a/docs/RESOURCE_ENVELOPE.md b/docs/RESOURCE_ENVELOPE.md index 3d5362e16..1a02aaf1c 100644 --- a/docs/RESOURCE_ENVELOPE.md +++ b/docs/RESOURCE_ENVELOPE.md @@ -84,9 +84,10 @@ unsupported, budget-exceeded, or incomplete. The envelope is schema version 2. `resources` is produced by the shared `PDFResourceBudget` authority and contains the resident ceiling plus all seven named pool records. `pages_materialized` reports pages actually processed by a -runner; it is not the catalog page count. `preflight_high_water_bytes` remains -`-1` until a run includes the preflight phase, and such a record is explicitly -`incomplete` rather than being promoted to a passing result. The deterministic +runner; it is not the catalog page count. `preflight_high_water_bytes` is the process high-water when the +`benchmark --profile ` preflight phase ends; without `--profile` +it stays `-1` and the record is explicitly `incomplete` rather than being +promoted to a passing result. The deterministic pathological and transparency/spot fixtures can be generated without the external DIV2K corpus: @@ -166,6 +167,12 @@ reported as a passing complete run. Add `--baseline C:\previous\resource-envelope-matrix.json` to compare matching fixture digests and platform/toolchain identities. The default regression margin is `2.0`; use a narrower margin only after collecting stable platform -baselines. Add `--cancel-fixture pathological-vector ---cancel-after-seconds 1` to send an interrupt to one controlled probe and -record the application's cancellation latency. +baselines. The runner passes `--profile` (default +`loop-preflight/profiles/loop-default.json`) so every run measures the +preflight phase. Add `--cancel-fixture ten-thousand-page +--cancel-after-seconds 3` for the separate cancellation probe, which interrupts +one extra run and then times a fresh process reopening the fixture and +rendering its first page (`recovery_ms`). `--strict` requires that probe and +also runs the hostile lane over `UnitTests/testdata/budget_exhaustion/`. For +the hosted, synthetic-fixture version of this run, see +`docs/RESOURCE_ENVELOPE_QUALIFICATION.md`. diff --git a/docs/RESOURCE_ENVELOPE_BUDGETS.json b/docs/RESOURCE_ENVELOPE_BUDGETS.json index fed9176c2..a19e06716 100644 --- a/docs/RESOURCE_ENVELOPE_BUDGETS.json +++ b/docs/RESOURCE_ENVELOPE_BUDGETS.json @@ -26,6 +26,13 @@ "cancellation_latency_ms": 5000, "recovery_ms": 30000 }, + "synthetic-image-heavy": { + "page_count": 10000, + "wall_time_ms": 120000, + "rss_high_water_bytes": 805306368, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, "pathological-vector": { "page_count": 256, "wall_time_ms": 120000, diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index f29358e59..b9d1fe5f2 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -14,23 +14,56 @@ PdfTool benchmark output and integrated document-session output are separate evidence records. A Quick first-view record remains `incomplete` until the Quick product path is implemented in Phase 4. -## Qualification sequence +## Hosted qualification (issue #19) + +`.github/workflows/resource-envelope-qualification.yml` is the qualifying run. +It triggers on `workflow_dispatch` and on pull requests that touch the +benchmark, the envelope scripts, or the budget contract. Each Linux and Windows +job: + +1. Builds `PdfTool` from the candidate SHA. +2. Generates the fixture bundle with + `scripts/resource_envelope/synthetic_workload.py`. The office, 500 MB + image-heavy, and 10,000-page fixtures are deterministic synthetic PDFs + (SHAKE-256 noise images stored with FlateDecode), so hosted runners need + no external corpus. The 10,000-page fixture uses the + `synthetic-image-heavy` workload caps, which equal the DIV2K caps. +3. Runs `run_matrix.py --strict --repetitions 3` with: + - a measured preflight phase (`benchmark --profile`, default + `loop-preflight/profiles/loop-default.json`), so a clean run reports + `status: complete` with a real `preflight_high_water_bytes`; + - a cancellation probe on `ten-thousand-page`, which interrupts the run + and requires a `cancelled` envelope within the workload's + `cancellation_latency_ms`; + - a recovery probe, which times a fresh process reopening the same + fixture and rendering its first page (`recovery_ms`, within the + workload's `recovery_ms`); + - a hostile lane over `UnitTests/testdata/budget_exhaustion/`, where each + PDF must end in a contained PdfTool exit code (rejection is fine) within + the hostile timeout, without breaching the resident ceiling. + +The `evidence` job combines both matrices with +`scripts/qualification/build_resource_envelope_evidence.py` into a schema +version 2 record that carries the run id and URL, and validates it with +`validate_resource_envelope_evidence.py`. The disposition is `passed` only when +both platforms passed strictly on the same candidate SHA. + +A crash (an exit code outside PdfTool's defined codes, or `InternalError`), a +timeout, a non-success exit, or a missing envelope never produces a +`measured` fixture. Crashes and timeouts fail the record outright. + +## External DIV2K sequence + +The original DIV2K qualification remains available for local runs: 1. Validate the external DIV2K corpus and generate one canonical manifest with `--hash-all`. 2. Build the deterministic 10,000-page image-heavy PDF and record its digest. 3. Create an external fixture manifest using the schema at `docs/schemas/resource-envelope-fixtures.schema.json`, then run - `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the 2 MB office, - image-heavy, 10,000-page, pathological-vector, and transparency/spot - fixtures. Supply the multi-GB fixture when platform addressability permits. - The strict job is expected to remain non-passing until the native benchmark - also supplies preflight and recovery measurements; unavailable fields must - not be promoted to zero. -4. Run PdfTool benchmark profiles on Linux and Windows with the same manifest. -5. Run the integrated session/scheduler harness with the same workload identity. -6. Replay the bounded lifecycle trace corpus on both platforms. -7. Attach JSON results, digests, platform identities, and dispositions to the - candidate-SHA evidence dossier. + `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the + same probe and hostile options as the hosted workflow. +4. Run the integrated session/scheduler harness with the same workload identity. +5. Replay the bounded lifecycle trace corpus on both platforms. No unavailable measurement may be converted to zero or treated as a pass. diff --git a/scripts/qualification/build_resource_envelope_evidence.py b/scripts/qualification/build_resource_envelope_evidence.py new file mode 100644 index 000000000..2ee3477a1 --- /dev/null +++ b/scripts/qualification/build_resource_envelope_evidence.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Build hosted resource-envelope qualification evidence from matrix results. + +Each ``--matrix PLATFORM=PATH`` is one strict ``run_matrix.py`` output from the +hosted qualification workflow. The evidence records the exact CI run, candidate +SHA, fixture manifest digest, and per-fixture measurements for every platform. +It claims ``passed`` only when every required platform passed strict +qualification on the same candidate SHA. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.resource_envelope.run_matrix import matrix_passes + + +EVIDENCE_KIND = "loop-resource-envelope-qualification-evidence" +REQUIRED_PLATFORMS = ("linux", "windows") +MEASUREMENT_FIELDS = ("status", "rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms", "pages_materialized", "page_count") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _platform_record(matrix: Mapping[str, Any], matrix_sha256: str) -> dict[str, Any]: + fixtures: dict[str, Any] = {} + for record in matrix["fixtures"]: + result = record.get("result") if isinstance(record.get("result"), Mapping) else {} + fixtures[record["fixture_id"]] = { + "status": record["status"], + "required": record.get("required", False), + "fixture_sha256": record.get("fixture_sha256"), + "workload": record.get("workload"), + "envelope": {field: result.get(field) for field in MEASUREMENT_FIELDS} if result else None, + "statistics": record.get("statistics"), + "validation_errors": record.get("validation_errors", []), + } + probe = matrix.get("cancellation_recovery_probe") or {} + hostile = matrix.get("hostile") or {} + identity = next((record["identity"] for record in matrix["fixtures"] if isinstance(record.get("identity"), Mapping) and record["identity"]), {}) + return { + "matrix_sha256": matrix_sha256, + "candidate_sha": matrix["candidate_sha"], + "strict_passed": matrix_passes(matrix, strict=True), + "summary": matrix["summary"], + "runtime": {key: identity.get(key) for key in ("os", "qt", "compiler", "cpu", "renderer", "build")}, + "fixtures": fixtures, + "cancellation_recovery_probe": { + "status": probe.get("status", "not-run"), + "fixture_id": probe.get("fixture_id"), + "cancellation_latency_ms": probe.get("cancellation", {}).get("cancellation_latency_ms", -1), + "recovery_ms": probe.get("recovery", {}).get("recovery_ms", -1), + "validation_errors": probe.get("validation_errors", []), + }, + "hostile": { + "summary": hostile.get("summary", {"total": 0, "contained": 0}), + "failed_cases": [case["case_id"] for case in hostile.get("cases", []) if case.get("status") != "contained"], + }, + } + + +def build_evidence( + matrices: Mapping[str, Path], + fixture_manifest: Path, + run_id: str, + run_url: str, +) -> dict[str, Any]: + platforms: dict[str, Any] = {} + for platform, path in sorted(matrices.items()): + matrix = json.loads(path.read_text(encoding="utf-8")) + platforms[platform] = _platform_record(matrix, _sha256(path)) + + candidates = {record["candidate_sha"] for record in platforms.values()} + reasons: list[str] = [] + missing = [platform for platform in REQUIRED_PLATFORMS if platform not in platforms] + reasons.extend(f"platform {platform} produced no matrix" for platform in missing) + if len(candidates) > 1: + reasons.append(f"platforms measured different candidate SHAs: {sorted(candidates)}") + for platform, record in platforms.items(): + if not record["strict_passed"]: + reasons.append(f"platform {platform} did not pass strict qualification") + + rejected = any( + record["summary"]["failed"] or record["cancellation_recovery_probe"]["status"] == "failed" or record["hostile"]["failed_cases"] + for record in platforms.values() + ) + disposition = "passed" if not reasons else ("rejected" if rejected else "incomplete") + manifest = json.loads(fixture_manifest.read_text(encoding="utf-8")) + return { + "schema_kind": EVIDENCE_KIND, + "schema_version": 2, + "issue": 19, + "candidate_sha": next(iter(candidates)) if len(candidates) == 1 else "", + "disposition": disposition, + "disposition_reasons": reasons, + "fixture_manifest_sha256": _sha256(fixture_manifest), + "fixture_generator": manifest.get("generator"), + "ci_runs": [{"platform": platform, "run_id": run_id, "run_url": run_url, "candidate_sha": record["candidate_sha"]} for platform, record in platforms.items()], + "platforms": platforms, + } + + +def _matrix_args(values: Sequence[str]) -> dict[str, Path]: + result: dict[str, Path] = {} + for value in values: + platform, separator, path = value.partition("=") + if not separator or not platform or not path: + raise ValueError("--matrix must be PLATFORM=PATH") + result[platform] = Path(path) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", action="append", default=[], metavar="PLATFORM=PATH") + parser.add_argument("--fixture-manifest", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + evidence = build_evidence(_matrix_args(args.matrix), args.fixture_manifest, args.run_id, args.run_url) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"resource-envelope evidence error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({"disposition": evidence["disposition"], "reasons": evidence["disposition_reasons"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualification/test_build_resource_envelope_evidence.py b/scripts/qualification/test_build_resource_envelope_evidence.py new file mode 100644 index 000000000..859728bcb --- /dev/null +++ b/scripts/qualification/test_build_resource_envelope_evidence.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.qualification.build_resource_envelope_evidence import build_evidence +from scripts.qualification.validate_resource_envelope_evidence import validate_evidence +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + +CANDIDATE = "a" * 40 + + +def _matrix(failed_fixture: str | None = None) -> dict: + fixtures = [] + for fixture_id, spec in FIXTURE_SPECS.items(): + if not spec["required"]: + fixtures.append({"fixture_id": fixture_id, "status": "unavailable", "required": False}) + continue + status = "failed" if fixture_id == failed_fixture else "measured" + fixtures.append({ + "fixture_id": fixture_id, + "status": status, + "required": True, + "fixture_sha256": "b" * 64, + "identity": {"os": "test-os", "qt": "6.11.1"}, + "result": {"status": "complete", "rss_high_water_bytes": 100, "preflight_high_water_bytes": 90, "elapsed_ms": 10, "pages_materialized": 1, "page_count": 1}, + }) + failed = int(failed_fixture is not None) + return { + "candidate_sha": CANDIDATE, + "fixtures": fixtures, + "cancellation_recovery_probe": {"status": "measured", "fixture_id": "ten-thousand-page", "cancellation": {"cancellation_latency_ms": 40}, "recovery": {"recovery_ms": 900}}, + "hostile": {"summary": {"total": 7, "contained": 7}, "cases": []}, + "summary": {"total": len(fixtures), "measured": len(fixtures) - 1 - failed, "flagged": 0, "skipped": 1, "failed": failed, "candidate_sha_verified": True}, + } + + +class BuildResourceEnvelopeEvidenceTest(unittest.TestCase): + def _build(self, matrices: dict[str, dict]) -> dict: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = {} + for platform, matrix in matrices.items(): + paths[platform] = root / f"{platform}.json" + paths[platform].write_text(json.dumps(matrix), encoding="utf-8") + manifest = root / "fixtures.json" + manifest.write_text(json.dumps({"generator": {"version": 1}}), encoding="utf-8") + return build_evidence(paths, manifest, "123456", "https://github.com/studio-berry/loop/actions/runs/123456") + + def test_both_platforms_passing_is_valid_passed_evidence(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + self.assertEqual(evidence["disposition"], "passed", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_missing_platform_is_incomplete(self) -> None: + evidence = self._build({"linux": _matrix()}) + self.assertEqual(evidence["disposition"], "incomplete") + self.assertIn("platform windows produced no matrix", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_failed_fixture_is_rejected(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix(failed_fixture="image-heavy-500mb")}) + self.assertEqual(evidence["disposition"], "rejected") + self.assertEqual(validate_evidence(evidence), []) + + def test_passed_claim_with_unavailable_measurement_is_invalid(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + tampered = copy.deepcopy(evidence) + tampered["platforms"]["linux"]["fixtures"]["office-2mb"]["envelope"]["preflight_high_water_bytes"] = -1 + tampered["platforms"]["windows"]["cancellation_recovery_probe"]["recovery_ms"] = -1 + errors = "\n".join(validate_evidence(tampered)) + self.assertIn("platforms.linux.fixtures.office-2mb.preflight_high_water_bytes is unavailable", errors) + self.assertIn("platforms.windows.cancellation_recovery_probe.recovery_ms is unavailable", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/qualification/validate_resource_envelope_evidence.py b/scripts/qualification/validate_resource_envelope_evidence.py index 9f1a9478c..b4922d09c 100644 --- a/scripts/qualification/validate_resource_envelope_evidence.py +++ b/scripts/qualification/validate_resource_envelope_evidence.py @@ -1,5 +1,12 @@ #!/usr/bin/env python3 -"""Validate frozen Session 11 resource-envelope qualification evidence.""" +"""Validate resource-envelope qualification evidence. + +Schema version 1 is the frozen Session 11 record and can never claim +``passed``. Schema version 2 is built from the hosted qualification workflow +(``build_resource_envelope_evidence.py``) and may claim ``passed`` only when +every required platform measured every required fixture, the cancellation +and recovery probe, and the hostile corpus on one candidate SHA. +""" from __future__ import annotations @@ -24,6 +31,7 @@ REQUIRED_FIXTURES = tuple( fixture_id for fixture_id, spec in FIXTURE_SPECS.items() if spec["required"] ) +REQUIRED_PLATFORMS = ("linux", "windows") def validate_evidence( @@ -31,11 +39,100 @@ def validate_evidence( *, manifest: dict[str, Any] | None = None, ) -> list[str]: - errors: list[str] = [] if evidence.get("schema_kind") != "loop-resource-envelope-qualification-evidence": - errors.append("schema_kind must be loop-resource-envelope-qualification-evidence") + return ["schema_kind must be loop-resource-envelope-qualification-evidence"] + if evidence.get("schema_version") == 2: + return _validate_hosted_evidence(evidence) if evidence.get("schema_version") != 1: - errors.append("schema_version must be 1") + return ["schema_version must be 1 or 2"] + return _validate_session_11_evidence(evidence, manifest) + + +def _is_measured(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _validate_platform(platform: str, record: Any, candidate_sha: Any) -> list[str]: + if not isinstance(record, dict): + return [f"platforms.{platform} must be an object"] + errors: list[str] = [] + if record.get("candidate_sha") != candidate_sha: + errors.append(f"platforms.{platform}.candidate_sha must equal candidate_sha") + if record.get("strict_passed") is not True: + errors.append(f"platforms.{platform} did not pass strict qualification") + if not isinstance(record.get("matrix_sha256"), str) or not SHA256_RE.fullmatch(record["matrix_sha256"]): + errors.append(f"platforms.{platform}.matrix_sha256 must be a SHA-256 digest") + fixtures = record.get("fixtures") if isinstance(record.get("fixtures"), dict) else {} + for fixture_id in REQUIRED_FIXTURES: + entry = fixtures.get(fixture_id) + envelope = entry.get("envelope") if isinstance(entry, dict) else None + if not isinstance(entry, dict) or entry.get("status") != "measured" or not isinstance(envelope, dict): + errors.append(f"platforms.{platform}.fixtures.{fixture_id} is not measured") + continue + if envelope.get("status") != "complete": + errors.append(f"platforms.{platform}.fixtures.{fixture_id} envelope is not complete") + for field in ("rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms"): + if not _is_measured(envelope.get(field)): + errors.append(f"platforms.{platform}.fixtures.{fixture_id}.{field} is unavailable") + probe = record.get("cancellation_recovery_probe") if isinstance(record.get("cancellation_recovery_probe"), dict) else {} + if probe.get("status") != "measured": + errors.append(f"platforms.{platform} cancellation/recovery probe is not measured") + for field in ("cancellation_latency_ms", "recovery_ms"): + if not _is_measured(probe.get(field)): + errors.append(f"platforms.{platform}.cancellation_recovery_probe.{field} is unavailable") + hostile = record.get("hostile") if isinstance(record.get("hostile"), dict) else {} + summary = hostile.get("summary") if isinstance(hostile.get("summary"), dict) else {} + if not _is_measured(summary.get("total")) or summary.get("total") == 0 or summary.get("contained") != summary.get("total"): + errors.append(f"platforms.{platform} hostile corpus was not fully contained") + return errors + + +def _validate_hosted_evidence(evidence: dict[str, Any]) -> list[str]: + errors: list[str] = [] + candidate_sha = evidence.get("candidate_sha") + if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): + errors.append("candidate_sha must be a 40-character lowercase commit SHA") + if not isinstance(evidence.get("fixture_manifest_sha256"), str) or not SHA256_RE.fullmatch(evidence["fixture_manifest_sha256"]): + errors.append("fixture_manifest_sha256 must be a SHA-256 digest") + disposition = evidence.get("disposition") + if disposition not in {"incomplete", "passed", "rejected"}: + errors.append("disposition must be incomplete, passed, or rejected") + reasons = evidence.get("disposition_reasons") + if not isinstance(reasons, list): + errors.append("disposition_reasons must be an array") + elif disposition == "passed" and reasons: + errors.append("passed evidence cannot carry disposition_reasons") + elif disposition != "passed" and not reasons: + errors.append("non-passed evidence must state its disposition_reasons") + + runs = evidence.get("ci_runs") + if not isinstance(runs, list) or not runs: + errors.append("ci_runs must be a non-empty array") + else: + for index, run in enumerate(runs): + if not isinstance(run, dict) or not str(run.get("run_id", "")).isdigit(): + errors.append(f"ci_runs[{index}].run_id must be a GitHub Actions run id") + continue + if not str(run.get("run_url", "")).startswith("https://github.com/"): + errors.append(f"ci_runs[{index}].run_url must be a GitHub Actions run URL") + if run.get("candidate_sha") != candidate_sha: + errors.append(f"ci_runs[{index}].candidate_sha must equal candidate_sha") + + platforms = evidence.get("platforms") + if not isinstance(platforms, dict) or not platforms: + errors.append("platforms must be a non-empty object") + return errors + if disposition == "passed": + for platform in REQUIRED_PLATFORMS: + if platform not in platforms: + errors.append(f"passed evidence is missing platform {platform}") + for platform, record in platforms.items(): + errors.extend(_validate_platform(platform, record, candidate_sha)) + return errors + + +def _validate_session_11_evidence(evidence: dict[str, Any], manifest: dict[str, Any] | None) -> list[str]: + errors: list[str] = [] candidate_sha = evidence.get("candidate_sha") if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index ae5eeab31..cce988c33 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -3,7 +3,13 @@ Large PDFs stay outside the repository. A qualification run should use a manifest with exact fixture digests and sizes; the legacy ``--fixture`` form is -kept for exploratory runs and is intentionally not sufficient for ``--strict``. +intentionally not sufficient for ``--strict``. + +Besides the measured fixtures, a strict run carries a cancellation probe that +interrupts one fixture, a recovery probe that times a fresh process reopening +it, and a hostile lane that feeds the checked-in budget-exhaustion PDFs to +PdfTool. A crash, timeout, or skipped workload never counts as a passing +envelope. """ from __future__ import annotations @@ -26,8 +32,19 @@ ROOT = Path(__file__).resolve().parents[2] DEFAULT_BUDGETS = ROOT / "docs" / "RESOURCE_ENVELOPE_BUDGETS.json" +DEFAULT_PREFLIGHT_PROFILE = ROOT / "loop-preflight" / "profiles" / "loop-default.json" +DEFAULT_HOSTILE_CORPUS = ROOT / "UnitTests" / "testdata" / "budget_exhaustion" MATRIX_KIND = "loop-resource-envelope-matrix" DEFAULT_RASTERIZERS = 8 +# PdfTool's defined terminal exit codes (pdftoolresult.h) except InternalError +# (7). Anything else, including a negative POSIX signal or a Windows exception +# status, means the process did not reach a controlled disposition. +CONTAINED_EXIT_CODES = frozenset({0, 1, 2, 3, 4, 5, 6, 8, 9}) +EXIT_SUCCESS = 0 +EXIT_CANCELLED = 6 +# Validation errors carrying one of these markers fail the record outright; +# every other error only flags it. +HARD_ERROR_MARKERS = ("does not match", "exceeds", "identity", "fixture SHA", "manifest", "timeout", "crashed") # These names mirror issue #242. multi-gb is optional because platform # addressability and available disk are environment-dependent. @@ -40,6 +57,9 @@ "transparency-spots": {"required": True, "expected_page_count": 256, "workload": None, "min_bytes": None, "max_bytes": None}, } +Runner = Callable[..., subprocess.CompletedProcess[str]] +CancelRunner = Callable[[list[str], float, float | None], subprocess.CompletedProcess[str]] + def _sha256(path: Path) -> str: digest = hashlib.sha256() @@ -73,6 +93,11 @@ def _envelope_from_output(payload: Mapping[str, Any]) -> dict[str, Any] | None: return None +def _envelope_from_process(completed: subprocess.CompletedProcess[str]) -> dict[str, Any] | None: + payload = _extract_json(completed.stdout or "") + return _envelope_from_output(payload) if payload else None + + def _git_head() -> str: try: return subprocess.run( @@ -97,6 +122,38 @@ def _candidate_identity() -> dict[str, Any]: } +def _benchmark_command( + pdf_tool: Path, + fixture_path: Path, + rasterizers: int, + preflight_profile: Path | None, + first_page_only: bool = False, +) -> list[str]: + # Pin rasterizers to a fixed value (8) so the same code and fixtures + # produce comparable RSS and elapsed time across hosts with different + # CPU counts. The value is recorded in the result profile. + command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + if preflight_profile is not None: + command += ["--profile", str(preflight_profile)] + if first_page_only: + command += ["--page-first", "1", "--page-last", "1"] + return command + + +def _identity_errors(envelope: Mapping[str, Any], candidate_sha: str, fixture_sha256: str) -> list[str]: + identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} + errors: list[str] = [] + if identity.get("commit") != candidate_sha: + errors.append(f"identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") + if identity.get("fixture_digest") != fixture_sha256: + errors.append(f"identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {fixture_sha256!r}") + return errors + + +def _is_hard(error: str) -> bool: + return any(marker in error for marker in HARD_ERROR_MARKERS) + + def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_after_seconds: float | None) -> subprocess.CompletedProcess[str]: creationflags = 0 popen_kwargs: dict[str, Any] = {} @@ -127,9 +184,25 @@ def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_af except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired(command, timeout_seconds, stdout, stderr) return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) +def _timed_run(runner: Runner, command: list[str], timeout_seconds: float) -> tuple[subprocess.CompletedProcess[str] | None, str, int]: + """Runs one PdfTool process; returns (completed, failure reason, wall ms).""" + started = time.monotonic() + try: + completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + return None, "benchmark-timeout", int((time.monotonic() - started) * 1000) + except OSError as exc: + return None, f"benchmark-launch-failed:{exc}", -1 + wall_ms = int((time.monotonic() - started) * 1000) + if completed.returncode not in CONTAINED_EXIT_CODES: + return completed, f"process-crashed:{completed.returncode}", wall_ms + return completed, "", wall_ms + + def _empty_result(fixture_id: str, reason: str) -> dict[str, Any]: return { "fixture_id": fixture_id, @@ -208,6 +281,12 @@ def _fixture_metadata(fixture_id: str, fixture_path: Path, metadata: Mapping[str return details, errors +def _fixture_workload(fixture_id: str, metadata: Mapping[str, Any] | None) -> str | None: + if metadata is not None and "workload" in metadata: + return str(metadata["workload"]) + return FIXTURE_SPECS[fixture_id]["workload"] + + def _aggregate_envelopes(envelopes: list[Mapping[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: # Use the highest-RSS run as the safety representative and the median # elapsed time. This keeps peak-memory validation conservative while @@ -236,13 +315,13 @@ def run_fixture( timeout_seconds: float, baseline: Mapping[str, Any] | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, metadata: Mapping[str, Any] | None = None, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, require_provenance: bool = False, - cancel_after_seconds: float | None = None, candidate_sha: str | None = None, + preflight_profile: Path | None = None, ) -> dict[str, Any]: if repetitions < 1 or rasterizers < 1: raise ValueError("repetitions and rasterizers must be positive") @@ -254,17 +333,15 @@ def run_fixture( pdf_tool = Path(pdf_tool).resolve() fixture_path = Path(fixture_path).resolve() spec = FIXTURE_SPECS[fixture_id] + workload = _fixture_workload(fixture_id, metadata) fixture_details, provenance_errors = _fixture_metadata(fixture_id, fixture_path, metadata, require_provenance) - # Pin rasterizers to a fixed value (8) so the same code and fixtures - # produce comparable RSS and elapsed time across hosts with different - # CPU counts. The value is recorded in the result profile. - command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile) record: dict[str, Any] = { "fixture_id": fixture_id, - "path": str(fixture_path.resolve()), + "path": str(fixture_path), "expected_page_count": metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"], - "workload": spec["workload"], - "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers}, + "workload": workload, + "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers, "preflight_profile": str(preflight_profile) if preflight_profile else None}, "command": command, **fixture_details, } @@ -274,34 +351,24 @@ def run_fixture( runs: list[dict[str, Any]] = [] envelopes: list[Mapping[str, Any]] = [] - for index in range(repetitions): - try: - if runner is subprocess.run and cancel_after_seconds is not None: - completed = _run_benchmark_process(command, timeout_seconds, cancel_after_seconds) - else: - completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) - except subprocess.TimeoutExpired: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-timeout", "process_exit_code": None}) - continue - except OSError as exc: - runs.append({"run": index + 1, "status": "unavailable", "reason": f"benchmark-launch-failed:{exc}", "process_exit_code": None}) - continue - payload = _extract_json(completed.stdout) - envelope = _envelope_from_output(payload) if payload else None - if envelope is None: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-envelope-missing", "process_exit_code": completed.returncode, "stderr": completed.stderr[-2000:]}) + validation_errors: list[str] = [] + for index in range(1, repetitions + 1): + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + envelope = _envelope_from_process(completed) if completed is not None else None + exit_code = completed.returncode if completed is not None else None + if failure or envelope is None: + reason = failure or "benchmark-envelope-missing" + run: dict[str, Any] = {"run": index, "status": "unavailable", "reason": reason, "process_exit_code": exit_code, "process_wall_ms": wall_ms} + if completed is not None: + run["stderr"] = (completed.stderr or "")[-2000:] + runs.append(run) + validation_errors.append(f"run {index}: {reason}") continue envelopes.append(envelope) - runs.append({"run": index + 1, "status": "recorded", "process_exit_code": completed.returncode, "result": envelope}) - - if not envelopes: - record.update({"status": "unavailable", "result": None, "runs": runs, "validation_errors": [], "regressions": []}) - return record - - representative, stats = _aggregate_envelopes(envelopes) - validation_errors: list[str] = [] - for index, envelope in enumerate(envelopes, start=1): - for error in validate_envelope(envelope, budgets, spec["workload"]): + runs.append({"run": index, "status": "recorded", "process_exit_code": exit_code, "process_wall_ms": wall_ms, "result": envelope}) + if exit_code != EXIT_SUCCESS: + validation_errors.append(f"run {index}: process exit code {exit_code} is not success") + for error in validate_envelope(envelope, budgets, workload): validation_errors.append(f"run {index}: {error}") expected_page_count = record["expected_page_count"] if expected_page_count is not None and envelope.get("page_count") != expected_page_count: @@ -310,42 +377,182 @@ def run_fixture( resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") if isinstance(rss, int) and rss >= 0 and isinstance(resident_limit, int) and rss > resident_limit: validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {resident_limit}") - identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} - if identity.get("commit") != candidate_sha: - validation_errors.append(f"run {index}: identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") - if identity.get("fixture_digest") != record.get("fixture_sha256"): - validation_errors.append(f"run {index}: identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {record.get('fixture_sha256')!r}") + validation_errors.extend(f"run {index}: {error}" for error in _identity_errors(envelope, candidate_sha, record["fixture_sha256"])) + + record["runs"] = runs + record["validation_errors"] = sorted(set(validation_errors)) + record["regressions"] = [] + if not envelopes: + record["result"] = None + record["status"] = "failed" if any(_is_hard(error) for error in record["validation_errors"]) else "unavailable" + return record + + representative, stats = _aggregate_envelopes(envelopes) record["identity"] = representative.get("identity", {}) record["result"] = representative record["statistics"] = stats - record["runs"] = runs - unavailable_runs = [run for run in runs if run["status"] != "recorded"] - validation_errors.extend( - f"run {run['run']}: {run['reason']}" for run in unavailable_runs - ) - record["validation_errors"] = sorted(set(validation_errors)) comparison = dict(representative) comparison["fixture_sha256"] = record["fixture_sha256"] comparison["identity"] = record["identity"] record["regressions"] = _regressions(comparison, baseline, margin) - if cancel_after_seconds is not None: - if representative.get("status") != "cancelled": - validation_errors.append("cancellation probe did not produce a cancelled envelope") - if not isinstance(representative.get("cancellation_latency_ms"), int) or representative["cancellation_latency_ms"] < 0: - validation_errors.append("cancellation probe did not report cancellation latency") - record["cancellation_probe"] = {"requested_after_seconds": cancel_after_seconds} - record["validation_errors"] = sorted(set(validation_errors)) - hard_error_markers = ("does not match", "exceeds", "identity", "fixture SHA", "manifest") - hard_errors = [error for error in record["validation_errors"] if any(marker in error for marker in hard_error_markers)] - if record["regressions"] or hard_errors: + if record["regressions"] or any(_is_hard(error) for error in record["validation_errors"]): record["status"] = "failed" - elif record["validation_errors"] or representative.get("status") != "complete": + elif record["validation_errors"] or any(envelope.get("status") != "complete" for envelope in envelopes): record["status"] = "flagged" else: record["status"] = "measured" return record +def run_cancellation_probe( + pdf_tool: Path, + fixture_id: str, + fixture_path: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + cancel_after_seconds: float, + candidate_sha: str, + workload: str | None = None, + rasterizers: int = DEFAULT_RASTERIZERS, + preflight_profile: Path | None = None, + cancel_runner: CancelRunner = _run_benchmark_process, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Interrupts one run, then times a fresh process reopening the fixture. + + ``recovery_ms`` is the wall time from launching that fresh process until it + exits having rendered the first page: the time an operator waits to get the + document back after abandoning a run. + """ + pdf_tool = Path(pdf_tool).resolve() + fixture_path = Path(fixture_path).resolve() + fixture_sha256 = _sha256(fixture_path) + limits = budgets.get("workloads", {}).get(workload, {}) if workload else {} + errors: list[str] = [] + + cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile) + cancellation: dict[str, Any] = {"command": cancel_command, "requested_after_seconds": cancel_after_seconds, "cancellation_latency_ms": -1} + try: + completed = cancel_runner(cancel_command, timeout_seconds, cancel_after_seconds) + except subprocess.TimeoutExpired: + completed = None + errors.append("cancellation probe timeout: the process did not stop after the interrupt") + except OSError as exc: + completed = None + errors.append(f"cancellation probe launch failed: {exc}") + if completed is not None: + cancellation["process_exit_code"] = completed.returncode + envelope = _envelope_from_process(completed) + if completed.returncode not in CONTAINED_EXIT_CODES: + errors.append(f"cancellation probe crashed with exit code {completed.returncode}") + elif envelope is None: + errors.append("cancellation probe produced no envelope") + else: + cancellation["result"] = envelope + latency = envelope.get("cancellation_latency_ms") + if envelope.get("status") != "cancelled": + errors.append(f"cancellation probe ended {envelope.get('status')!r}; the interrupt arrived after the run finished or was ignored") + if completed.returncode != EXIT_CANCELLED: + errors.append(f"cancellation probe exit code {completed.returncode} is not Cancelled ({EXIT_CANCELLED})") + if not isinstance(latency, int) or isinstance(latency, bool) or latency < 0: + errors.append("cancellation probe did not report cancellation latency") + else: + cancellation["cancellation_latency_ms"] = latency + limit = limits.get("cancellation_latency_ms") + if isinstance(limit, int) and latency > limit: + errors.append(f"cancellation_latency_ms {latency} exceeds workload policy {limit}") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + + recovery_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None, first_page_only=True) + recovery: dict[str, Any] = {"command": recovery_command, "recovery_ms": -1} + completed, failure, wall_ms = _timed_run(runner, recovery_command, timeout_seconds) + if completed is not None: + recovery["process_exit_code"] = completed.returncode + if failure: + errors.append(f"recovery probe {failure}") + elif completed is not None: + envelope = _envelope_from_process(completed) + if envelope is None: + errors.append("recovery probe produced no envelope") + else: + recovery["result"] = envelope + if completed.returncode != EXIT_SUCCESS: + errors.append(f"recovery probe exit code {completed.returncode} is not success") + if envelope.get("pages_materialized") != 1: + errors.append(f"recovery probe materialized {envelope.get('pages_materialized')!r} pages, expected 1") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + recovery["recovery_ms"] = wall_ms + limit = limits.get("recovery_ms") + if isinstance(limit, int) and wall_ms > limit: + errors.append(f"recovery_ms {wall_ms} exceeds workload policy {limit}") + + return { + "fixture_id": fixture_id, + "workload": workload, + "fixture_sha256": fixture_sha256, + "status": "failed" if errors else "measured", + "cancellation": cancellation, + "recovery": recovery, + "validation_errors": errors, + } + + +def run_hostile_corpus( + pdf_tool: Path, + corpus_dir: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + rasterizers: int = DEFAULT_RASTERIZERS, + preflight_profile: Path | None = None, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Requires every hostile PDF to end in a contained PdfTool disposition. + + Rejecting the input (InputError, ProcessingFailure, budget-exceeded) is a + correct outcome here; crashing, hanging, or breaching the resident ceiling + is not. + """ + pdf_tool = Path(pdf_tool).resolve() + corpus_dir = Path(corpus_dir).resolve() + manifest = json.loads((corpus_dir / "manifest.json").read_text(encoding="utf-8")) + cases = manifest.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError(f"hostile corpus manifest has no cases: {corpus_dir}") + resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") + records: list[dict[str, Any]] = [] + for case in cases: + path = corpus_dir / str(case["pdf"]) + record: dict[str, Any] = {"case_id": case["id"], "path": str(path), "expected": case.get("expected")} + errors: list[str] = [] + if not path.is_file() or _sha256(path) != case.get("sha256"): + errors.append("hostile fixture SHA-256 does not match manifest") + else: + command = _benchmark_command(pdf_tool, path, rasterizers, preflight_profile) + record["command"] = command + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + record["process_wall_ms"] = wall_ms + if completed is not None: + record["process_exit_code"] = completed.returncode + if failure: + errors.append(failure) + elif completed is not None: + envelope = _envelope_from_process(completed) + record["disposition"] = envelope.get("status") if envelope else "rejected" + if envelope is not None: + record["result"] = envelope + rss = envelope.get("rss_high_water_bytes") + if isinstance(rss, int) and isinstance(resident_limit, int) and rss > resident_limit: + errors.append(f"RSS {rss} exceeds resident policy {resident_limit}") + record["validation_errors"] = errors + record["status"] = "failed" if errors else "contained" + records.append(record) + return { + "corpus": str(corpus_dir), + "cases": records, + "summary": {"total": len(records), "contained": sum(record["status"] == "contained" for record in records)}, + } + + def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("schema_kind") != "loop-resource-envelope-fixtures" or payload.get("schema_version") != 1: @@ -371,6 +578,8 @@ def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: raise ValueError(f"fixture manifest provenance missing: {fixture_id}") if "page_count" in record and (not isinstance(record["page_count"], int) or record["page_count"] < 1): raise ValueError(f"fixture manifest page_count is invalid: {fixture_id}") + if "workload" in record and (not isinstance(record["workload"], str) or not record["workload"]): + raise ValueError(f"fixture manifest workload is invalid: {fixture_id}") normalized = dict(record) fixture_path = Path(str(record["path"])) if not fixture_path.is_absolute(): @@ -398,17 +607,22 @@ def run_matrix( timeout_seconds: float, baseline: Mapping[str, Any] | Path | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, cancel_fixture: str | None = None, cancel_after_seconds: float | None = None, + preflight_profile: Path | None = None, + hostile_corpus: Path | None = None, + hostile_timeout_seconds: float | None = None, + cancel_runner: CancelRunner = _run_benchmark_process, ) -> dict[str, Any]: pdf_tool = Path(pdf_tool).resolve() baseline_by_fixture = _baseline_records(baseline) if isinstance(baseline, Path) else (baseline or {}) identity = _candidate_identity() candidate_sha = identity["candidate_sha"] records: list[dict[str, Any]] = [] + resolved: dict[str, tuple[Path, Mapping[str, Any] | None]] = {} for fixture_id, spec in FIXTURE_SPECS.items(): supplied = fixtures.get(fixture_id) if supplied is None: @@ -417,40 +631,42 @@ def run_matrix( records.append(record) continue metadata = dict(supplied) if isinstance(supplied, Mapping) else None - fixture_path = Path(metadata["path"]) if metadata else Path(supplied) - fixture_path = fixture_path.resolve() + fixture_path = (Path(metadata["path"]) if metadata else Path(supplied)).resolve() if not fixture_path.is_file(): record = _empty_result(fixture_id, "fixture-not-found") record["required"] = spec["required"] records.append(record) continue - record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), cancel_after_seconds if fixture_id == cancel_fixture else None, candidate_sha) + resolved[fixture_id] = (fixture_path, metadata) + record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), candidate_sha, preflight_profile) record["required"] = spec["required"] - result = record.get("result") - if isinstance(result, Mapping): - result_identity = result.get("identity") - if isinstance(result_identity, Mapping): - commit = result_identity.get("commit") - if commit != candidate_sha: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity commit does not match checkout HEAD"])) - record["status"] = "failed" - expected_digest = record.get("fixture_sha256") - fixture_digest = result_identity.get("fixture_digest") - if expected_digest and fixture_digest != expected_digest: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity fixture digest does not match input SHA-256"])) - record["status"] = "failed" records.append(record) + probe: dict[str, Any] | None = None + if cancel_fixture is not None: + if cancel_fixture not in resolved: + probe = {"fixture_id": cancel_fixture, "status": "unavailable", "validation_errors": ["cancellation fixture not supplied"]} + else: + fixture_path, metadata = resolved[cancel_fixture] + probe = run_cancellation_probe(pdf_tool, cancel_fixture, fixture_path, budgets, timeout_seconds, cancel_after_seconds or 1.0, candidate_sha, + _fixture_workload(cancel_fixture, metadata), rasterizers, preflight_profile, cancel_runner, runner) + + hostile = None + if hostile_corpus is not None: + hostile = run_hostile_corpus(pdf_tool, hostile_corpus, budgets, hostile_timeout_seconds or timeout_seconds, rasterizers, preflight_profile, runner) + failed = sum(record["status"] == "failed" for record in records) flagged = sum(record["required"] and record["status"] in {"flagged", "unavailable"} for record in records) skipped = sum(not record["required"] and record["status"] == "unavailable" for record in records) return { "schema_kind": MATRIX_KIND, - "schema_version": 2, + "schema_version": 3, "candidate_sha": identity["candidate_sha"], "candidate_identity": identity, "generated_at_utc": datetime.now(timezone.utc).isoformat(), "fixtures": records, + "cancellation_recovery_probe": probe, + "hostile": hostile, "summary": { "total": len(records), "measured": sum(record["status"] == "measured" for record in records), @@ -458,10 +674,29 @@ def run_matrix( "skipped": skipped, "failed": failed, "candidate_sha_verified": identity["verified"], + "cancellation_recovery_probe": probe["status"] if probe else "not-run", + "hostile_contained": f"{hostile['summary']['contained']}/{hostile['summary']['total']}" if hostile else "not-run", }, } +def matrix_passes(matrix: Mapping[str, Any], strict: bool) -> bool: + summary = matrix["summary"] + probe = matrix.get("cancellation_recovery_probe") + hostile = matrix.get("hostile") + hostile_failed = bool(hostile) and hostile["summary"]["contained"] != hostile["summary"]["total"] + if summary["failed"] or hostile_failed or (probe is not None and probe["status"] == "failed"): + return False + if not strict: + return True + return ( + not summary["flagged"] + and summary["candidate_sha_verified"] + and probe is not None and probe["status"] == "measured" + and hostile is not None + ) + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pdf-tool", type=Path, required=True) @@ -475,24 +710,39 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--repetitions", type=int, default=1) parser.add_argument("--rasterizers", type=int, default=DEFAULT_RASTERIZERS) parser.add_argument("--timeout-seconds", type=float, default=120.0) + parser.add_argument("--preflight-profile", type=Path, default=DEFAULT_PREFLIGHT_PROFILE, + help="profile for the benchmark's measured preflight phase") + parser.add_argument("--no-preflight", action="store_true", help="omit the preflight phase (records stay incomplete)") parser.add_argument("--cancel-fixture", choices=tuple(FIXTURE_SPECS)) parser.add_argument("--cancel-after-seconds", type=float) - parser.add_argument("--strict", action="store_true", help="fail when required fixtures, provenance, or measurements are unavailable") + parser.add_argument("--hostile-corpus", type=Path, help=f"budget-exhaustion corpus directory (strict default: {DEFAULT_HOSTILE_CORPUS.relative_to(ROOT)})") + parser.add_argument("--hostile-timeout-seconds", type=float, default=60.0) + parser.add_argument("--strict", action="store_true", help="fail when required fixtures, probes, provenance, or measurements are unavailable") args = parser.parse_args(argv) try: - if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1: - raise ValueError("margin, timeout-seconds, repetitions, and rasterizers must be positive") + if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1 or args.hostile_timeout_seconds <= 0: + raise ValueError("margin, timeouts, repetitions, and rasterizers must be positive") if args.strict and args.manifest is None: raise ValueError("--strict requires a fixture --manifest with exact digests and sizes") + if args.strict and args.cancel_fixture is None: + raise ValueError("--strict requires --cancel-fixture for the cancellation and recovery probes") + if args.strict and args.no_preflight: + raise ValueError("--strict cannot omit the preflight phase") if (args.cancel_fixture is None) != (args.cancel_after_seconds is None): raise ValueError("--cancel-fixture and --cancel-after-seconds must be supplied together") if args.cancel_after_seconds is not None and args.cancel_after_seconds <= 0: raise ValueError("cancel-after-seconds must be positive") - if args.cancel_fixture is not None and args.repetitions != 1: - raise ValueError("cancellation probes require --repetitions 1") + preflight_profile = None if args.no_preflight else args.preflight_profile.resolve() + if preflight_profile is not None and not preflight_profile.is_file(): + raise ValueError(f"preflight profile not found: {preflight_profile}") + hostile_corpus = args.hostile_corpus or (DEFAULT_HOSTILE_CORPUS if args.strict else None) fixtures: Mapping[str, Path | Mapping[str, Any]] = _load_fixture_manifest(args.manifest) if args.manifest else _fixture_args(args.fixture) budgets = json.loads(args.budgets.read_text(encoding="utf-8")) - matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, repetitions=args.repetitions, rasterizers=args.rasterizers, cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds) + matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, + repetitions=args.repetitions, rasterizers=args.rasterizers, + cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds, + preflight_profile=preflight_profile, hostile_corpus=hostile_corpus, + hostile_timeout_seconds=args.hostile_timeout_seconds) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(matrix, indent=2) + "\n", encoding="utf-8") except (OSError, ValueError, json.JSONDecodeError) as exc: @@ -500,7 +750,7 @@ def main(argv: Sequence[str] | None = None) -> int: return 2 print(json.dumps(matrix["summary"], indent=2)) - return 1 if matrix["summary"]["failed"] or args.strict and (matrix["summary"]["flagged"] or not matrix["summary"]["candidate_sha_verified"]) else 0 + return 0 if matrix_passes(matrix, args.strict) else 1 if __name__ == "__main__": diff --git a/scripts/resource_envelope/synthetic_workload.py b/scripts/resource_envelope/synthetic_workload.py new file mode 100644 index 000000000..0c0eb23ef --- /dev/null +++ b/scripts/resource_envelope/synthetic_workload.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Build the deterministic synthetic resource-envelope fixture bundle. + +Hosted qualification cannot reach the external DIV2K corpus or a private +fixture bundle, so the office, image-heavy, and 10,000-page fixtures are +generated here. Every pixel derives from SHAKE-256 over a fixed label and the +PDF structure is fixed, so the same generator version produces byte-identical +fixtures on every platform. Image data is incompressible noise stored with +FlateDecode: file size tracks decoded size and every page pays a real decode. + + python scripts/resource_envelope/synthetic_workload.py \\ + --output-dir /tmp/loop-envelope --manifest /tmp/loop-envelope/fixtures.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import zlib +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Sequence + +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +from scripts.resource_envelope.create_fixture_manifest import create_manifest +from scripts.resource_envelope.pathological_workload import build_pathological_pdf +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + + +GENERATOR_VERSION = 1 +PAGE_WIDTH = 612 +PAGE_HEIGHT = 792 +IMAGE_HEAVY_WORKLOAD = "synthetic-image-heavy" +_WORDS = ( + "press", "proof", "bleed", "ink", "plate", "sheet", "trim", "spot", "cyan", "magenta", + "yellow", "black", "overprint", "separation", "imposition", "signature", "gutter", "margin", + "raster", "vector", "profile", "output", "intent", "coverage", "density", "register", +) + + +@dataclass(frozen=True) +class ImageSpec: + width: int + height: int + + @property + def decoded_bytes(self) -> int: + return self.width * self.height * 3 + + +# Sized so each fixture lands inside run_matrix.FIXTURE_SPECS byte bounds. +OFFICE_PAGES = 24 +OFFICE_IMAGE_EVERY = 4 +OFFICE_IMAGE = ImageSpec(380, 280) +IMAGE_HEAVY_PAGES = 60 +IMAGE_HEAVY_IMAGE = ImageSpec(2048, 1360) +TEN_THOUSAND_PAGES = 10000 +TEN_THOUSAND_UNIQUE_IMAGES = 64 +TEN_THOUSAND_IMAGE = ImageSpec(640, 480) + + +def _noise(label: str, size: int) -> bytes: + return hashlib.shake_256(f"loop-resource-envelope/v{GENERATOR_VERSION}/{label}".encode("ascii")).digest(size) + + +def _stream(dictionary: bytes, payload: bytes) -> bytes: + return dictionary[:-2] + b" /Length " + str(len(payload)).encode("ascii") + b" >>\nstream\n" + payload + b"\nendstream" + + +def _image_object(label: str, image: ImageSpec) -> bytes: + dictionary = ( + f"<< /Type /XObject /Subtype /Image /Width {image.width} /Height {image.height}" + " /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /FlateDecode >>" + ).encode("ascii") + return _stream(dictionary, zlib.compress(_noise(label, image.decoded_bytes), level=1)) + + +def _image_placement(image: ImageSpec, name: str) -> bytes: + scale = min(PAGE_WIDTH / image.width, PAGE_HEIGHT / image.height) + width = image.width * scale + height = image.height * scale + x = (PAGE_WIDTH - width) / 2 + y = (PAGE_HEIGHT - height) / 2 + return f"q {width:.4f} 0 0 {height:.4f} {x:.4f} {y:.4f} cm /{name} Do Q\n".encode("ascii") + + +def _page_object(parent: int, contents: int, resources: bytes) -> bytes: + return ( + f"<< /Type /Page /Parent {parent} 0 R /MediaBox [0 0 {PAGE_WIDTH} {PAGE_HEIGHT}] /Resources ".encode("ascii") + + resources + + f" /Contents {contents} 0 R >>".encode("ascii") + ) + + +class _PdfWriter: + """Writes numbered objects straight to disk so a 500 MB fixture never sits in memory.""" + + def __init__(self, handle: BinaryIO, object_count: int) -> None: + self._handle = handle + self._offsets = [0] * (object_count + 1) + self._written = 0 + self._write(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + + def _write(self, data: bytes) -> None: + self._handle.write(data) + self._written += len(data) + + def add(self, number: int, body: bytes) -> None: + if self._offsets[number]: + raise ValueError(f"object {number} written twice") + self._offsets[number] = self._written + self._write(f"{number} 0 obj\n".encode("ascii") + body + b"\nendobj\n") + + def finish(self, file_id: str) -> None: + missing = [number for number, offset in enumerate(self._offsets) if number and not offset] + if missing: + raise ValueError(f"objects never written: {missing[:5]}") + xref = self._written + self._write(f"xref\n0 {len(self._offsets)}\n0000000000 65535 f \n".encode("ascii")) + for offset in self._offsets[1:]: + self._write(f"{offset:010d} 00000 n \n".encode("ascii")) + self._write( + f"trailer\n<< /Size {len(self._offsets)} /Root 1 0 R /ID [<{file_id}> <{file_id}>] >>\n" + f"startxref\n{xref}\n%%EOF\n".encode("ascii") + ) + + +def _file_id(fixture_id: str) -> str: + return hashlib.sha256(f"{fixture_id}/v{GENERATOR_VERSION}".encode("ascii")).hexdigest()[:32] + + +def _office_text(page_index: int) -> bytes: + selector = _noise(f"office/text/{page_index}", 40 * 12) + lines = [b"BT", b"/F1 10 Tf", b"12 TL", f"54 {PAGE_HEIGHT - 60} Td".encode("ascii")] + for line in range(40): + words = " ".join(_WORDS[value % len(_WORDS)] for value in selector[line * 12 : line * 12 + 12]) + lines.append(f"({words}) '".encode("ascii")) + lines.append(b"ET") + for row in range(8): + y = 80 + row * 18 + lines.append(f"0.2 w 54 {y} m {PAGE_WIDTH - 54} {y} l S".encode("ascii")) + return b"\n".join(lines) + b"\n" + + +def build_office(output: Path, pages: int = OFFICE_PAGES, image: ImageSpec = OFFICE_IMAGE) -> None: + image_pages = [index for index in range(pages) if index % OFFICE_IMAGE_EVERY == 0] + font = 3 + first_image = 4 + first_page = first_image + len(image_pages) + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + writer.add(font, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>") + for slot in range(len(image_pages)): + writer.add(first_image + slot, _image_object(f"office/image/{slot}", image)) + for index in range(pages): + content = _office_text(index) + xobjects = b"" + if index in image_pages: + slot = image_pages.index(index) + content += f"q {image.width / 2:.4f} 0 0 {image.height / 2:.4f} 54 {PAGE_HEIGHT - 620} cm /Im0 Do Q\n".encode("ascii") + xobjects = f" /XObject << /Im0 {first_image + slot} 0 R >>".encode("ascii") + resources = f"<< /Font << /F1 {font} 0 R >>".encode("ascii") + xobjects + b" >>" + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", content)) + writer.finish(_file_id("office-2mb")) + + +def build_image_pages(output: Path, fixture_id: str, pages: int, unique_images: int, image: ImageSpec) -> None: + if pages < 1 or unique_images < 1: + raise ValueError("pages and unique_images must be positive") + first_image = 3 + first_page = first_image + unique_images + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + for slot in range(unique_images): + writer.add(first_image + slot, _image_object(f"{fixture_id}/image/{slot}", image)) + placement = _image_placement(image, "Im0") + for index in range(pages): + resources = f"<< /XObject << /Im0 {first_image + index % unique_images} 0 R >> >>".encode("ascii") + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", placement)) + writer.finish(_file_id(fixture_id)) + + +def _provenance(fixture_id: str) -> str: + return f"scripts/resource_envelope/synthetic_workload.py generator v{GENERATOR_VERSION} ({fixture_id})" + + +def build_bundle(output_dir: Path) -> dict[str, object]: + output_dir.mkdir(parents=True, exist_ok=True) + paths = {fixture_id: output_dir / f"{fixture_id}.pdf" for fixture_id in FIXTURE_SPECS if fixture_id != "multi-gb"} + build_office(paths["office-2mb"]) + build_image_pages(paths["image-heavy-500mb"], "image-heavy-500mb", IMAGE_HEAVY_PAGES, IMAGE_HEAVY_PAGES, IMAGE_HEAVY_IMAGE) + build_image_pages(paths["ten-thousand-page"], "ten-thousand-page", TEN_THOUSAND_PAGES, TEN_THOUSAND_UNIQUE_IMAGES, TEN_THOUSAND_IMAGE) + build_pathological_pdf(paths["pathological-vector"], 256, 512, "pathological-vector") + build_pathological_pdf(paths["transparency-spots"], 256, 256, "transparency-spots") + + manifest = create_manifest(paths, "synthetic") + for record in manifest["fixtures"]: + fixture_id = str(record["fixture_id"]) + record["provenance"] = _provenance(fixture_id) + record["path"] = paths[fixture_id].name + if fixture_id == "ten-thousand-page": + record["workload"] = IMAGE_HEAVY_WORKLOAD + spec = FIXTURE_SPECS[fixture_id] + size = int(record["size_bytes"]) + if (spec["min_bytes"] is not None and size < spec["min_bytes"]) or (spec["max_bytes"] is not None and size > spec["max_bytes"]): + raise ValueError(f"{fixture_id} generated {size} bytes, outside its fixture bounds") + manifest["generator"] = {"script": "scripts/resource_envelope/synthetic_workload.py", "version": GENERATOR_VERSION} + return manifest + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True, help="fixture manifest to write; paths are relative to it") + args = parser.parse_args(argv) + try: + if args.manifest.resolve().parent != args.output_dir.resolve(): + raise ValueError("--manifest must be written inside --output-dir so its relative paths resolve") + manifest = build_bundle(args.output_dir) + args.manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"synthetic workload error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({record["fixture_id"]: {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} for record in manifest["fixtures"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py new file mode 100644 index 000000000..d51f53bbe --- /dev/null +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus +from scripts.resource_envelope.validate_envelope import POOL_NAMES + +CANDIDATE = "candidate-sha" + + +def _policy() -> dict: + return { + "resource_budget": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "workloads": { + "pathological-vector": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200, "cancellation_latency_ms": 50, "recovery_ms": 60000}, + "synthetic-image-heavy": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200}, + }, + } + + +def _envelope(digest: str, status: str = "complete", preflight: int = 5, rss: int = 10, materialized: int = 256, latency: int = -1) -> dict: + return { + "identity": {"commit": CANDIDATE, "fixture_digest": digest}, + "family": "benchmark-render", + "status": status, + "page_count": 256, + "rss_high_water_bytes": rss, + "preflight_high_water_bytes": preflight, + "pages_materialized": materialized, + "elapsed_ms": 10, + "cancellation_latency_ms": latency, + "prefetch_shed": False, + "interaction_slot_held": False, + "resources": { + "config": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "resident_bytes": 0, + "resident_high_water_bytes": 0, + "pressure": "normal", + "pools": {pool: {"limit_bytes": 100, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0} for pool in POOL_NAMES}, + }, + } + + +def _process(command: list[str], returncode: int, envelope: dict | None) -> subprocess.CompletedProcess[str]: + stdout = json.dumps({"data": {"workload_envelope": envelope}}) if envelope else "" + return subprocess.CompletedProcess(command, returncode, stdout, "") + + +class _Fixture: + def __enter__(self) -> "_Fixture": + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "fixture.pdf" + self.path.write_bytes(b"fixture") + self.digest = hashlib.sha256(b"fixture").hexdigest() + self.metadata = {"path": str(self.path), "sha256": self.digest, "size_bytes": 7, "provenance": "unit-test", "page_count": 256} + return self + + def __exit__(self, *exc: object) -> None: + self._directory.cleanup() + + def measure(self, runner, **kwargs) -> dict: + return run_fixture(Path("PdfTool"), "pathological-vector", self.path, _policy(), 1, runner=runner, metadata=self.metadata, candidate_sha=CANDIDATE, **kwargs) + + +class MeasuredRunTest(unittest.TestCase): + def test_complete_envelope_with_preflight_is_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + self.assertGreaterEqual(record["runs"][0]["process_wall_ms"], 0) + + def test_preflight_profile_reaches_the_command(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-2:], ["--profile", "profile.json"]) + + def test_crashed_process_fails_even_with_an_envelope(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, -11, _envelope(fixture.digest))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: process-crashed:-11", record["validation_errors"]) + + def test_timeout_fails(self) -> None: + def runner(command, **kwargs): + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: benchmark-timeout", record["validation_errors"]) + + def test_partial_output_exit_is_flagged_not_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 5, _envelope(fixture.digest))) + self.assertEqual(record["status"], "flagged") + + def test_manifest_workload_overrides_the_default(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "synthetic-image-heavy" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["workload"], "synthetic-image-heavy") + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + +class CancellationProbeTest(unittest.TestCase): + def _probe(self, fixture: _Fixture, cancel_runner, runner) -> dict: + return run_cancellation_probe(Path("PdfTool"), "pathological-vector", fixture.path, _policy(), 5, 0.5, CANDIDATE, + "pathological-vector", cancel_runner=cancel_runner, runner=runner) + + def test_cancelled_run_and_reopen_are_measured(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=20, materialized=40)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "measured", probe["validation_errors"]) + self.assertEqual(probe["cancellation"]["cancellation_latency_ms"], 20) + self.assertGreaterEqual(probe["recovery"]["recovery_ms"], 0) + self.assertEqual(probe["recovery"]["command"][-4:], ["--page-first", "1", "--page-last", "1"]) + + def test_run_that_finished_before_the_interrupt_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 0, _envelope(fixture.digest)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "failed") + self.assertTrue(any("not cancelled" in error or "ended 'complete'" in error for error in probe["validation_errors"])) + + def test_latency_over_policy_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=51)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertIn("cancellation_latency_ms 51 exceeds workload policy 50", probe["validation_errors"]) + + def test_hung_interrupt_fails(self) -> None: + def cancel_runner(command, timeout, cancel_after): + raise subprocess.TimeoutExpired(command, timeout) + + with _Fixture() as fixture: + probe = self._probe(fixture, cancel_runner, lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1))) + self.assertEqual(probe["status"], "failed") + + +class HostileCorpusTest(unittest.TestCase): + def test_rejection_is_contained_and_crash_is_not(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + cases = [] + for case_id in ("rejected", "crashing"): + (corpus / f"{case_id}.pdf").write_bytes(case_id.encode()) + cases.append({"id": case_id, "pdf": f"{case_id}.pdf", "sha256": hashlib.sha256(case_id.encode()).hexdigest()}) + (corpus / "manifest.json").write_text(json.dumps({"cases": cases}), encoding="utf-8") + + def runner(command, **_): + return _process(command, 3 if "rejected.pdf" in command[2] else -6, None) + + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=runner) + by_id = {case["case_id"]: case for case in hostile["cases"]} + self.assertEqual(by_id["rejected"]["status"], "contained") + self.assertEqual(by_id["rejected"]["disposition"], "rejected") + self.assertEqual(by_id["crashing"]["status"], "failed") + self.assertEqual(hostile["summary"], {"total": 2, "contained": 1}) + + def test_tampered_fixture_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + (corpus / "case.pdf").write_bytes(b"tampered") + (corpus / "manifest.json").write_text(json.dumps({"cases": [{"id": "case", "pdf": "case.pdf", "sha256": "0" * 64}]}), encoding="utf-8") + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=lambda command, **_: _process(command, 0, None)) + self.assertEqual(hostile["cases"][0]["status"], "failed") + + +class MatrixPassTest(unittest.TestCase): + def _matrix(self, probe_status: str | None = "measured", contained: int = 2, flagged: int = 0) -> dict: + return { + "summary": {"failed": 0, "flagged": flagged, "candidate_sha_verified": True}, + "cancellation_recovery_probe": {"status": probe_status} if probe_status else None, + "hostile": {"summary": {"total": 2, "contained": contained}}, + } + + def test_strict_requires_probe_hostile_and_no_flags(self) -> None: + self.assertTrue(matrix_passes(self._matrix(), strict=True)) + self.assertFalse(matrix_passes(self._matrix(probe_status=None), strict=True)) + self.assertFalse(matrix_passes(self._matrix(flagged=1), strict=True)) + self.assertTrue(matrix_passes(self._matrix(flagged=1), strict=False)) + + def test_uncontained_hostile_case_fails_even_without_strict(self) -> None: + self.assertFalse(matrix_passes(self._matrix(contained=1), strict=False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/resource_envelope/test_synthetic_workload.py b/scripts/resource_envelope/test_synthetic_workload.py new file mode 100644 index 000000000..7473b6b06 --- /dev/null +++ b/scripts/resource_envelope/test_synthetic_workload.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import hashlib +import re +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS +from scripts.resource_envelope.synthetic_workload import ImageSpec, _PdfWriter, build_image_pages, build_office + + +def _xref_problems(pdf: bytes) -> list[str]: + start = int(re.search(rb"startxref\n(\d+)\n%%EOF\n$", pdf).group(1)) + count = int(re.match(rb"xref\n0 (\d+)\n", pdf[start:]).group(1)) + rows = pdf[start:].split(b"\n")[2 : 2 + count] + problems = [f"object {number}" for number, row in enumerate(rows[1:], start=1) if not pdf[int(row[:10]) :].startswith(f"{number} 0 obj".encode())] + for match in re.finditer(rb"/Length (\d+) >>\nstream\n", pdf): + if pdf[match.end() + int(match.group(1)) :][:10] != b"\nendstream": + problems.append(f"stream at {match.start()}") + return problems + + +class SyntheticWorkloadTest(unittest.TestCase): + def test_image_pages_are_deterministic_and_well_formed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.pdf" + second = Path(directory) / "second.pdf" + build_image_pages(first, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + build_image_pages(second, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + pdf = first.read_bytes() + self.assertEqual(hashlib.sha256(pdf).digest(), hashlib.sha256(second.read_bytes()).digest()) + self.assertEqual(pdf.count(b"/Type /Page "), 30) + self.assertEqual(pdf.count(b"/Subtype /Image"), 4) + self.assertIn(b"/Count 30", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_fixture_identity_changes_the_pixels(self) -> None: + with tempfile.TemporaryDirectory() as directory: + one = Path(directory) / "one.pdf" + two = Path(directory) / "two.pdf" + build_image_pages(one, "ten-thousand-page", 2, 1, ImageSpec(8, 8)) + build_image_pages(two, "image-heavy-500mb", 2, 1, ImageSpec(8, 8)) + self.assertNotEqual(one.read_bytes(), two.read_bytes()) + + def test_office_fixture_fits_its_size_bounds(self) -> None: + spec = FIXTURE_SPECS["office-2mb"] + with tempfile.TemporaryDirectory() as directory: + office = Path(directory) / "office.pdf" + build_office(office) + pdf = office.read_bytes() + self.assertGreaterEqual(len(pdf), spec["min_bytes"]) + self.assertLessEqual(len(pdf), spec["max_bytes"]) + self.assertIn(b"/BaseFont /Helvetica", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_writer_rejects_unwritten_objects(self) -> None: + with tempfile.TemporaryFile() as handle: + writer = _PdfWriter(handle, 2) + writer.add(1, b"<< >>") + with self.assertRaises(ValueError): + writer.finish("0" * 32) + + +if __name__ == "__main__": + unittest.main() From 52079abbdab5364ac0beb8973e3a300f43f62975 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:26:23 -0700 Subject: [PATCH 02/14] chore(changes): add evidence manifest for issue 19 envelopes Co-Authored-By: Claude Opus 5.5 --- ...-issue-19-resource-envelopes.evidence.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 changes/cc-issue-19-resource-envelopes.evidence.yaml diff --git a/changes/cc-issue-19-resource-envelopes.evidence.yaml b/changes/cc-issue-19-resource-envelopes.evidence.yaml new file mode 100644 index 000000000..e1590ff08 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.evidence.yaml @@ -0,0 +1,22 @@ +format_version: 1 +kind: evidence +claims: + - id: benchmark-preflight-phase-and-cancellation + evidence: + - unit:agent-policy:pdftool + - unit:agent-policy:pagemaster + - packaging:linux-build + - packaging:windows-build + - id: hosted-qualification-workflow + evidence: + - packaging:linux-build + - packaging:windows-build + - security:codeql + - id: agent-policy-bindings + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py +unresolved: + - Native PdfTool and UnitTestsPdfToolContract were not built locally (no configured build directory for this worktree); the linux-build and windows-build CI lanes compile and run them. + - Hosted qualification measurements come from the resource-envelope-qualification workflow run on this PR and are recorded under docs/evidence/issue-19-resource-envelope/ after that run. From 7d24c836944bf80f592fe441e32a434f33431a5b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 20:00:26 +0000 Subject: [PATCH 03/14] fix(envelope): read Linux VmHWM with readAll and format pdftoolrender.h QFile::atEnd() is true immediately for procfs files, so currentRssHighWaterBytes() always returned -1 on Linux. That left preflight_high_water_bytes at -1, failing benchmarkWithPreflightProfileIsComplete and flagging every Linux matrix record as unmeasured. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NT3L5ao8UyQuTPPKySRDcP --- LoopLibCore/sources/pdfworkloadenvelope.cpp | 8 +++++--- PdfTool/pdftoolrender.h | 2 +- UnitTests/tst_workloadenvelopetest.cpp | 10 ++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/LoopLibCore/sources/pdfworkloadenvelope.cpp b/LoopLibCore/sources/pdfworkloadenvelope.cpp index c4a0d0e88..8b7148bee 100644 --- a/LoopLibCore/sources/pdfworkloadenvelope.cpp +++ b/LoopLibCore/sources/pdfworkloadenvelope.cpp @@ -172,15 +172,17 @@ qint64 PDFWorkloadEnvelope::currentRssHighWaterBytes() return -1; } - while (!status.atEnd()) + // procfs reports size 0, so QFile::atEnd() is true before the first read. + const QList lines = status.readAll().split('\n'); + for (const QByteArray& rawLine : lines) { - const QByteArray line = status.readLine().trimmed(); + const QByteArray line = rawLine.trimmed(); if (!line.startsWith("VmHWM:")) { continue; } - const QList parts = line.split(' '); + const QList parts = line.mid(6).simplified().split(' '); for (const QByteArray& part : parts) { bool ok = false; diff --git a/PdfTool/pdftoolrender.h b/PdfTool/pdftoolrender.h index 62b3578df..1068f5fcc 100644 --- a/PdfTool/pdftoolrender.h +++ b/PdfTool/pdftoolrender.h @@ -103,4 +103,4 @@ class PDFToolBenchmark : public PDFToolRenderBase } // namespace pdftool -#endif // PDFTOOLRENDER_H +#endif // PDFTOOLRENDER_H diff --git a/UnitTests/tst_workloadenvelopetest.cpp b/UnitTests/tst_workloadenvelopetest.cpp index 721a0649f..f73e115c0 100644 --- a/UnitTests/tst_workloadenvelopetest.cpp +++ b/UnitTests/tst_workloadenvelopetest.cpp @@ -38,11 +38,21 @@ class WorkloadEnvelopeTest : public QObject private slots: void identityFieldsArePresent(); + void rssHighWaterIsMeasuredOnSupportedPlatforms(); void shedPrefetchAndQualityBeforeInteraction(); void pageHeavyEnvelopeRecordsIdentity(); void interactionSlotRunsWhenBackgroundIsSaturated(); }; +void WorkloadEnvelopeTest::rssHighWaterIsMeasuredOnSupportedPlatforms() +{ +#if defined(Q_OS_LINUX) || defined(Q_OS_WIN) + QVERIFY(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes() > 0); +#else + QCOMPARE(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(), qint64(-1)); +#endif +} + void WorkloadEnvelopeTest::identityFieldsArePresent() { qputenv("GIT_COMMIT", QByteArrayLiteral("0123456789abcdef0123456789abcdef01234567")); From e3c26269ef2d19a5c6ba09e98d05b0ff43dda5c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 20:01:16 +0000 Subject: [PATCH 04/14] chore(changes): cover core proof lanes for the RSS reader fix Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NT3L5ao8UyQuTPPKySRDcP --- changes/cc-issue-19-resource-envelopes.evidence.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changes/cc-issue-19-resource-envelopes.evidence.yaml b/changes/cc-issue-19-resource-envelopes.evidence.yaml index e1590ff08..927e5a335 100644 --- a/changes/cc-issue-19-resource-envelopes.evidence.yaml +++ b/changes/cc-issue-19-resource-envelopes.evidence.yaml @@ -17,6 +17,13 @@ claims: - unit:scripts/agent/test_architecture_contracts.py - architecture:scripts/agent/check-architecture.py - architecture:scripts/agent/generate-adapters.py + - id: linux-rss-high-water-reads-procfs + evidence: + - unit:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle unresolved: + - core:scripts/ci/check_independent_validation_gate.py - Native PdfTool and UnitTestsPdfToolContract were not built locally (no configured build directory for this worktree); the linux-build and windows-build CI lanes compile and run them. - Hosted qualification measurements come from the resource-envelope-qualification workflow run on this PR and are recorded under docs/evidence/issue-19-resource-envelope/ after that run. From d7eeebb7563ace18de05ae50132bb4db6c72c696 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 21:39:00 +0000 Subject: [PATCH 05/14] feat(envelope): print per-record failure reasons from the strict matrix The job log only carried summary counts, so a failing hosted run could not be diagnosed without downloading the matrix artifact. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NT3L5ao8UyQuTPPKySRDcP --- scripts/resource_envelope/run_matrix.py | 18 +++++++++++++ .../test_run_matrix_probes.py | 25 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index cce988c33..7201aae4f 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -697,6 +697,22 @@ def matrix_passes(matrix: Mapping[str, Any], strict: bool) -> bool: ) +def matrix_failure_reasons(matrix: Mapping[str, Any]) -> list[str]: + reasons: list[str] = [] + for record in matrix["fixtures"]: + if record["status"] in {"failed", "flagged"} or (record.get("required") and record["status"] == "unavailable"): + details = record["validation_errors"] or [record.get("reason", "no detail recorded")] + reasons.extend(f"fixture {record['fixture_id']} {record['status']}: {error}" for error in details) + probe = matrix.get("cancellation_recovery_probe") + if probe is not None and probe["status"] != "measured": + reasons.extend(f"probe {probe['status']}: {error}" for error in probe["validation_errors"]) + for case in (matrix.get("hostile") or {}).get("cases", []): + if case["status"] != "contained": + exit_code = case.get("process_exit_code") + reasons.extend(f"hostile {case['case_id']} (exit {exit_code}): {error}" for error in case["validation_errors"]) + return reasons + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pdf-tool", type=Path, required=True) @@ -750,6 +766,8 @@ def main(argv: Sequence[str] | None = None) -> int: return 2 print(json.dumps(matrix["summary"], indent=2)) + for reason in matrix_failure_reasons(matrix): + print(reason, file=sys.stderr) return 0 if matrix_passes(matrix, args.strict) else 1 diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index d51f53bbe..3de961d21 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path -from scripts.resource_envelope.run_matrix import matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus +from scripts.resource_envelope.run_matrix import matrix_failure_reasons, matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus from scripts.resource_envelope.validate_envelope import POOL_NAMES CANDIDATE = "candidate-sha" @@ -199,5 +199,28 @@ def test_uncontained_hostile_case_fails_even_without_strict(self) -> None: self.assertFalse(matrix_passes(self._matrix(contained=1), strict=False)) +class FailureReasonTest(unittest.TestCase): + def test_names_every_failing_record_probe_and_hostile_case(self) -> None: + matrix = { + "fixtures": [ + {"fixture_id": "office-2mb", "status": "measured", "required": True, "validation_errors": []}, + {"fixture_id": "image-heavy-500mb", "status": "flagged", "required": True, "validation_errors": ["run 1: process exit code 1 is not success"]}, + {"fixture_id": "ten-thousand-page", "status": "unavailable", "required": True, "validation_errors": [], "reason": "fixture-not-found"}, + {"fixture_id": "multi-gb", "status": "unavailable", "required": False, "validation_errors": [], "reason": "fixture-not-supplied-optional"}, + ], + "cancellation_recovery_probe": {"status": "failed", "validation_errors": ["recovery probe process-crashed:-11"]}, + "hostile": {"cases": [ + {"case_id": "ok", "status": "contained", "validation_errors": []}, + {"case_id": "deep-tree", "status": "failed", "process_exit_code": 0, "validation_errors": ["RSS 9 exceeds resident policy 4"]}, + ]}, + } + self.assertEqual(matrix_failure_reasons(matrix), [ + "fixture image-heavy-500mb flagged: run 1: process exit code 1 is not success", + "fixture ten-thousand-page unavailable: fixture-not-found", + "probe failed: recovery probe process-crashed:-11", + "hostile deep-tree (exit 0): RSS 9 exceeds resident policy 4", + ]) + + if __name__ == "__main__": unittest.main() From c9af5df22dde2fc2b38d70e3a4525894682f0ab9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 23:12:47 +0000 Subject: [PATCH 06/14] feat(envelope): log render errors and stderr for failing matrix runs Exit code 5 (PartialOutput) alone does not say which page failed or why. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NT3L5ao8UyQuTPPKySRDcP --- scripts/resource_envelope/run_matrix.py | 34 +++++++++++++++++++ .../test_run_matrix_probes.py | 18 ++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index 7201aae4f..0b703b051 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -98,6 +98,35 @@ def _envelope_from_process(completed: subprocess.CompletedProcess[str]) -> dict[ return _envelope_from_output(payload) if payload else None +def _find_key(value: Any, key: str) -> Any: + if isinstance(value, Mapping): + if key in value: + return value[key] + children: Sequence[Any] = list(value.values()) + elif isinstance(value, list): + children = value + else: + return None + for child in children: + found = _find_key(child, key) + if found is not None: + return found + return None + + +def _failure_detail(completed: subprocess.CompletedProcess[str]) -> str: + """Short, log-sized account of why a PdfTool run did not succeed.""" + payload = _extract_json(completed.stdout or "") + errors = _find_key(payload, "rendering-errors") if payload else None + parts = [] + if errors is not None: + parts.append("rendering-errors=" + json.dumps(errors)[:800]) + stderr = (completed.stderr or "").strip() + if stderr: + parts.append("stderr=" + stderr[-400:]) + return "; ".join(parts) + + def _git_head() -> str: try: return subprocess.run( @@ -361,12 +390,14 @@ def run_fixture( run: dict[str, Any] = {"run": index, "status": "unavailable", "reason": reason, "process_exit_code": exit_code, "process_wall_ms": wall_ms} if completed is not None: run["stderr"] = (completed.stderr or "")[-2000:] + run["detail"] = _failure_detail(completed) runs.append(run) validation_errors.append(f"run {index}: {reason}") continue envelopes.append(envelope) runs.append({"run": index, "status": "recorded", "process_exit_code": exit_code, "process_wall_ms": wall_ms, "result": envelope}) if exit_code != EXIT_SUCCESS: + runs[-1]["detail"] = _failure_detail(completed) validation_errors.append(f"run {index}: process exit code {exit_code} is not success") for error in validate_envelope(envelope, budgets, workload): validation_errors.append(f"run {index}: {error}") @@ -703,6 +734,9 @@ def matrix_failure_reasons(matrix: Mapping[str, Any]) -> list[str]: if record["status"] in {"failed", "flagged"} or (record.get("required") and record["status"] == "unavailable"): details = record["validation_errors"] or [record.get("reason", "no detail recorded")] reasons.extend(f"fixture {record['fixture_id']} {record['status']}: {error}" for error in details) + detailed = next((run for run in record.get("runs", []) if run.get("detail")), None) + if detailed is not None: + reasons.append(f"fixture {record['fixture_id']} run {detailed['run']} detail: {detailed['detail']}") probe = matrix.get("cancellation_recovery_probe") if probe is not None and probe["status"] != "measured": reasons.extend(f"probe {probe['status']}: {error}" for error in probe["validation_errors"]) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index 3de961d21..9828a40b7 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path -from scripts.resource_envelope.run_matrix import matrix_failure_reasons, matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus +from scripts.resource_envelope.run_matrix import _failure_detail, matrix_failure_reasons, matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus from scripts.resource_envelope.validate_envelope import POOL_NAMES CANDIDATE = "candidate-sha" @@ -199,12 +199,25 @@ def test_uncontained_hostile_case_fails_even_without_strict(self) -> None: self.assertFalse(matrix_passes(self._matrix(contained=1), strict=False)) +class FailureDetailTest(unittest.TestCase): + def test_keeps_render_errors_and_stderr_tail_of_a_partial_run(self) -> None: + stdout = json.dumps({"data": {"nested": [{"rendering-errors": [{"page-no": 3, "message": "bad image"}]}]}}) + completed = subprocess.CompletedProcess(["PdfTool"], 5, stdout, "warning: x\n") + detail = _failure_detail(completed) + self.assertIn('"bad image"', detail) + self.assertIn("stderr=warning: x", detail) + + def test_is_empty_when_nothing_was_reported(self) -> None: + self.assertEqual(_failure_detail(subprocess.CompletedProcess(["PdfTool"], 5, "", "")), "") + + class FailureReasonTest(unittest.TestCase): def test_names_every_failing_record_probe_and_hostile_case(self) -> None: matrix = { "fixtures": [ {"fixture_id": "office-2mb", "status": "measured", "required": True, "validation_errors": []}, - {"fixture_id": "image-heavy-500mb", "status": "flagged", "required": True, "validation_errors": ["run 1: process exit code 1 is not success"]}, + {"fixture_id": "image-heavy-500mb", "status": "flagged", "required": True, "validation_errors": ["run 1: process exit code 1 is not success"], + "runs": [{"run": 1, "detail": "stderr=render failed"}]}, {"fixture_id": "ten-thousand-page", "status": "unavailable", "required": True, "validation_errors": [], "reason": "fixture-not-found"}, {"fixture_id": "multi-gb", "status": "unavailable", "required": False, "validation_errors": [], "reason": "fixture-not-supplied-optional"}, ], @@ -216,6 +229,7 @@ def test_names_every_failing_record_probe_and_hostile_case(self) -> None: } self.assertEqual(matrix_failure_reasons(matrix), [ "fixture image-heavy-500mb flagged: run 1: process exit code 1 is not success", + "fixture image-heavy-500mb run 1 detail: stderr=render failed", "fixture ten-thousand-page unavailable: fixture-not-found", "probe failed: recovery probe process-crashed:-11", "hostile deep-tree (exit 0): RSS 9 exceeds resident policy 4", From 648d160b592af9efded90ed006f813de1d9c4f6d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 01:10:22 +0000 Subject: [PATCH 07/14] fix(envelope): pin 3 rasterizers to fit the raster-tile budget at 300 DPI At 300 DPI a Letter page image is 33.7 MB and each rasterizer holds one, so 8 rasterizers exceed the 128 MiB raster-tile-cache pool: the extra pages are rejected as budget-exceeded and every run exits PartialOutput. Three fit. Also stop repeating a fixture after its first timeout. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01NT3L5ao8UyQuTPPKySRDcP --- docs/RESOURCE_ENVELOPE.md | 9 +++++++-- scripts/resource_envelope/run_matrix.py | 16 +++++++++++----- .../resource_envelope/test_run_matrix_probes.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/RESOURCE_ENVELOPE.md b/docs/RESOURCE_ENVELOPE.md index 1a02aaf1c..ecaa77df4 100644 --- a/docs/RESOURCE_ENVELOPE.md +++ b/docs/RESOURCE_ENVELOPE.md @@ -60,10 +60,15 @@ $env:QT_QPA_PLATFORM = "offscreen" $env:QT_PLUGIN_PATH = "C:\path\to\qt\plugins" PdfTool.exe benchmark C:\temp\loop-div2k-10000-pages.pdf ` --render-hw-accel 0 ` - --render-rasterizers 8 ` + --render-rasterizers 3 ` --console-format json ``` +Rasterizers are pinned to 3 because the benchmark renders at the default 300 DPI: a Letter page +image is 33.7 MB, each rasterizer holds one at a time, and the 128 MiB `raster-tile-cache` pool +admits three. A fourth concurrent page is rejected as budget-exceeded and the run exits with +`PartialOutput`. + The JSON result includes the `workload_envelope` object. A successful Windows software-renderer run on the local 0.2.0 candidate rendered all 10,000 pages in 48,513 ms and recorded a peak RSS of 380,985,344 bytes. @@ -155,7 +160,7 @@ recommended cold-process timing/RSS sample: python scripts/resource_envelope/run_matrix.py ` --pdf-tool C:\path\to\PdfTool.exe ` --manifest C:\temp\resource-envelope-fixtures.json ` - --repetitions 3 --rasterizers 8 --strict ` + --repetitions 3 --rasterizers 3 --strict ` --output C:\temp\resource-envelope-matrix.json ``` diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index 0b703b051..0254ceccf 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -35,7 +35,11 @@ DEFAULT_PREFLIGHT_PROFILE = ROOT / "loop-preflight" / "profiles" / "loop-default.json" DEFAULT_HOSTILE_CORPUS = ROOT / "UnitTests" / "testdata" / "budget_exhaustion" MATRIX_KIND = "loop-resource-envelope-matrix" -DEFAULT_RASTERIZERS = 8 +# Each rasterizer holds one page image at a time. A 300 DPI Letter page is +# 33.7 MB, so three fit the 128 MiB raster-tile-cache pool and a fourth is +# rejected as budget-exceeded, leaving pages unrendered (exit code 5). +DEFAULT_RASTERIZERS = 3 +TIMEOUT_REASON = "benchmark-timeout" # PdfTool's defined terminal exit codes (pdftoolresult.h) except InternalError # (7). Anything else, including a negative POSIX signal or a Windows exception # status, means the process did not reach a controlled disposition. @@ -158,9 +162,9 @@ def _benchmark_command( preflight_profile: Path | None, first_page_only: bool = False, ) -> list[str]: - # Pin rasterizers to a fixed value (8) so the same code and fixtures - # produce comparable RSS and elapsed time across hosts with different - # CPU counts. The value is recorded in the result profile. + # Pin rasterizers to a fixed value so the same code and fixtures produce + # comparable RSS and elapsed time across hosts with different CPU counts. + # The value is recorded in the result profile. command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] if preflight_profile is not None: command += ["--profile", str(preflight_profile)] @@ -223,7 +227,7 @@ def _timed_run(runner: Runner, command: list[str], timeout_seconds: float) -> tu try: completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) except subprocess.TimeoutExpired: - return None, "benchmark-timeout", int((time.monotonic() - started) * 1000) + return None, TIMEOUT_REASON, int((time.monotonic() - started) * 1000) except OSError as exc: return None, f"benchmark-launch-failed:{exc}", -1 wall_ms = int((time.monotonic() - started) * 1000) @@ -393,6 +397,8 @@ def run_fixture( run["detail"] = _failure_detail(completed) runs.append(run) validation_errors.append(f"run {index}: {reason}") + if failure == TIMEOUT_REASON: + break continue envelopes.append(envelope) runs.append({"run": index, "status": "recorded", "process_exit_code": exit_code, "process_wall_ms": wall_ms, "result": envelope}) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index 9828a40b7..77bf11234 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -94,6 +94,18 @@ def runner(command, **kwargs): self.assertEqual(record["status"], "failed") self.assertIn("run 1: benchmark-timeout", record["validation_errors"]) + def test_timeout_is_not_repeated(self) -> None: + calls = [] + + def runner(command, **kwargs): + calls.append(command) + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner, repetitions=3) + self.assertEqual(len(calls), 1) + self.assertEqual(len(record["runs"]), 1) + def test_partial_output_exit_is_flagged_not_measured(self) -> None: with _Fixture() as fixture: record = fixture.measure(lambda command, **_: _process(command, 5, _envelope(fixture.digest))) From f6b485844dd62e30be93fea3c8e2bea2a7481a80 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:28:26 -0700 Subject: [PATCH 08/14] fix(render): hold the rasterizer until its page image is consumed The rasterizer was released before the image was processed, so with more worker threads than rasterizers, more images than rasterizers held raster-tile reservations at once. The hosted linux qualification run shed 7 reservations at 3 rasterizers (101 MB of the 134 MB pool used) and every render exited PartialOutput with budget-exceeded. A budgeted run now releases the reservation and then the rasterizer after processImage; unbudgeted callers keep the early release. Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfrenderer.cpp | 31 ++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/LoopLibCore/sources/pdfrenderer.cpp b/LoopLibCore/sources/pdfrenderer.cpp index 45c20b493..ddde6b498 100644 --- a/LoopLibCore/sources/pdfrenderer.cpp +++ b/LoopLibCore/sources/pdfrenderer.cpp @@ -476,17 +476,30 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, QImage image = rasterizer->render(pageIndex, page, &precompiledPage, imageSize, m_features, &annotationManager, cms.data(), PageRotation::None); qint64 pageRenderTime = pageTimer.elapsed(); - release(rasterizer); + // A budgeted run keeps the rasterizer until the image is consumed, so the images + // alive at once never exceed the rasterizer count the raster tile pool was sized for. + if (!imageReservation) + { + release(rasterizer); + } // Now, process the image - PDFRenderedPageImage renderedPageImage; - renderedPageImage.pageIndex = pageIndex; - renderedPageImage.pageImage = qMove(image); - renderedPageImage.pageCompileTime = pageCompileTime; - renderedPageImage.pageWaitTime = pageWaitTime; - renderedPageImage.pageRenderTime = pageRenderTime; - renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); - processImage(renderedPageImage); + { + PDFRenderedPageImage renderedPageImage; + renderedPageImage.pageIndex = pageIndex; + renderedPageImage.pageImage = qMove(image); + renderedPageImage.pageCompileTime = pageCompileTime; + renderedPageImage.pageWaitTime = pageWaitTime; + renderedPageImage.pageRenderTime = pageRenderTime; + renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); + processImage(renderedPageImage); + } + + if (imageReservation) + { + imageReservation.reset(); + release(rasterizer); + } if (progress) { From 64de840bf523f58957eb8692779c8b0b791cf24b Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:06:24 -0700 Subject: [PATCH 09/14] style(render): clang-format pdfrenderer.cpp Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfrenderer.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/LoopLibCore/sources/pdfrenderer.cpp b/LoopLibCore/sources/pdfrenderer.cpp index ddde6b498..e17b83da4 100644 --- a/LoopLibCore/sources/pdfrenderer.cpp +++ b/LoopLibCore/sources/pdfrenderer.cpp @@ -250,12 +250,10 @@ PDFRasterizer::PDFRasterizer(QObject* parent) : BaseClass(parent), m_rendererEngine(RendererEngine::Blend2D_SingleThread) { - } PDFRasterizer::~PDFRasterizer() { - } void PDFRasterizer::reset(RendererEngine rendererEngine) @@ -446,16 +444,16 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, const QSize imageSize = imageSizeGetter(page); const bool validImageSize = imageSize.width() > 0 && imageSize.height() > 0; const qint64 imageBytes = !validImageSize - ? 0 - : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 - ? static_cast(imageSize.width()) * imageSize.height() * 4 - : std::numeric_limits::max(); + ? 0 + : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 + ? static_cast(imageSize.width()) * imageSize.height() * 4 + : std::numeric_limits::max(); std::optional imageReservation; if (m_resourceBudget && imageBytes > 0 && !m_resourceBudget->tryReserve(PDFResourcePool::RasterTileCache, - imageBytes, - PDFResourcePriority::Visible, - QStringLiteral("benchmark raster image"))) + imageBytes, + PDFResourcePriority::Visible, + QStringLiteral("benchmark raster image"))) { m_resourceBudget->recordShed(PDFResourcePool::RasterTileCache); m_resourceBudgetExhausted.store(true, std::memory_order_release); From 4d9d02f0206b6ee87d2b365f98e6516a2f9da1c8 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:29:35 -0700 Subject: [PATCH 10/14] feat(core): let evidence collection and colour inventory honour cancellation PDFEvidenceCollectSettings and PDFColorInventorySettings gain an optional operationControl. The colour inventory polls it per page and reports cancelled; the evidence collector polls it per page and returns an incomplete graph with incompleteReason "cancelled". PreflightEngine passes its operation control through and reports errorCode "cancelled" instead of evidence-incomplete, so an interrupt during the benchmark's preflight phase stops within one page instead of after the whole document. Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfcolorinventory.cpp | 6 ++++ LoopLibCore/sources/pdfcolorinventory.h | 9 +++-- LoopLibCore/sources/pdfevidencegraph.cpp | 15 ++++++++ LoopLibCore/sources/pdfevidencegraph.h | 4 +++ LoopLibCore/sources/preflightengine.cpp | 11 ++++-- UnitTests/tst_evidencegraphtest.cpp | 44 +++++++++++++++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/LoopLibCore/sources/pdfcolorinventory.cpp b/LoopLibCore/sources/pdfcolorinventory.cpp index 55c1c4857..d4c58eccb 100644 --- a/LoopLibCore/sources/pdfcolorinventory.cpp +++ b/LoopLibCore/sources/pdfcolorinventory.cpp @@ -125,6 +125,12 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin for (PDFInteger pageIndex = 0; pageIndex < pageCount; ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + result.cancelled = true; + break; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { diff --git a/LoopLibCore/sources/pdfcolorinventory.h b/LoopLibCore/sources/pdfcolorinventory.h index a9f08c786..bb1a57ff3 100644 --- a/LoopLibCore/sources/pdfcolorinventory.h +++ b/LoopLibCore/sources/pdfcolorinventory.h @@ -24,6 +24,7 @@ #define PDFCOLORINVENTORY_H #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include "pdftransparencyrenderer.h" #include @@ -59,18 +60,22 @@ struct LOOPLIBCORESHARED_EXPORT PDFColorInventoryResult QList spotColors; QList richBlackPages; PDFRenderDiagnostics diagnostics; + /// True when the inspection stopped early because the operation was cancelled; + /// the lists above then cover only the pages probed before the stop. + bool cancelled = false; }; struct LOOPLIBCORESHARED_EXPORT PDFColorInventorySettings { int probeDpi = 150; qreal richBlackKThreshold = 0.10; + const PDFOperationControl* operationControl = nullptr; }; /// Shared rich-black predicate used by preflight and Output Preview. LOOPLIBCORESHARED_EXPORT bool isRichBlackPixel(PDFConstColorBuffer buffer, - const PDFPixelFormat& format, - PDFColorComponent kThreshold); + const PDFPixelFormat& format, + PDFColorComponent kThreshold); class LOOPLIBCORESHARED_EXPORT PDFColorInventory { diff --git a/LoopLibCore/sources/pdfevidencegraph.cpp b/LoopLibCore/sources/pdfevidencegraph.cpp index 5e2879bb2..73dae9505 100644 --- a/LoopLibCore/sources/pdfevidencegraph.cpp +++ b/LoopLibCore/sources/pdfevidencegraph.cpp @@ -159,6 +159,7 @@ namespace { constexpr int EVIDENCE_MAX_FORM_DEPTH = 32; +constexpr const char* EVIDENCE_CANCELLED_REASON = "cancelled"; PDFArtifactIdentity artifactIdentityFromDocument(const PDFDocument* document) { @@ -1288,8 +1289,15 @@ void collectColorants(PDFDocumentSession* session, PDFEvidenceGraph* graph, cons PDFColorInventorySettings inventorySettings; inventorySettings.probeDpi = settings.colorProbeDpi; inventorySettings.richBlackKThreshold = settings.richBlackKThreshold; + inventorySettings.operationControl = settings.operationControl; PDFColorInventory inventory(session); const PDFColorInventoryResult result = inventory.inspect(inventorySettings); + if (result.cancelled) + { + graph->complete = false; + graph->incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return; + } if (!result.diagnostics.isExact()) { const QString diagnostic = result.diagnostics.reasons.join(QStringLiteral("; ")).isEmpty() @@ -1398,6 +1406,13 @@ PDFEvidenceGraph PDFEvidenceCollector::collect(PDFDocumentSession* session, const PDFCatalog* catalog = document->getCatalog(); for (PDFInteger pageIndex = 0; pageIndex < catalog->getPageCount(); ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + graph.complete = false; + graph.incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return graph; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { diff --git a/LoopLibCore/sources/pdfevidencegraph.h b/LoopLibCore/sources/pdfevidencegraph.h index 98a0685f2..6d12e6d18 100644 --- a/LoopLibCore/sources/pdfevidencegraph.h +++ b/LoopLibCore/sources/pdfevidencegraph.h @@ -26,6 +26,7 @@ #include "pdfartifactidentity.h" #include "pdfdocumentcontext.h" #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include #include @@ -100,6 +101,9 @@ struct LOOPLIBCORESHARED_EXPORT PDFEvidenceCollectSettings qreal richBlackKThreshold = 0.10; qreal minEffectiveStrokeWidthPt = 0.0; qreal zeroWidthEpsilonPt = 1.0e-6; + /// Polled between pages; a cancelled collection returns an incomplete graph + /// with incompleteReason "cancelled". + const PDFOperationControl* operationControl = nullptr; }; class LOOPLIBCORESHARED_EXPORT PDFEvidenceCollector diff --git a/LoopLibCore/sources/preflightengine.cpp b/LoopLibCore/sources/preflightengine.cpp index 5ff19af4c..21b13b9b6 100644 --- a/LoopLibCore/sources/preflightengine.cpp +++ b/LoopLibCore/sources/preflightengine.cpp @@ -6194,7 +6194,9 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const const PDFEvidenceDomains graphDomains = effectivePlan.full ? evidenceDomainsForProfile(profile) : evidenceDomainsForCheckIds(effectivePlan.checkIds); if (graphDomains != PDFEvidenceDomains()) { - m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettingsForProfile(profile)); + PDFEvidenceCollectSettings evidenceSettings = evidenceSettingsForProfile(profile); + evidenceSettings.operationControl = m_operationControl; + m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettings); if (profile.restrictions.pages.has_value() || (!plan.full && !plan.pages.isEmpty())) { QList kept; @@ -6214,7 +6216,12 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const if (!m_activeGraph.isComplete()) { result.inspectionComplete = false; - if (!m_activeGraph.budgetKind.isEmpty()) + if (PDFOperationControl::isOperationCancelled(m_operationControl)) + { + result.errorCode = QStringLiteral("cancelled"); + result.errorMessage = PDFTranslationContext::tr("Preflight was cancelled."); + } + else if (!m_activeGraph.budgetKind.isEmpty()) { result.errorCode = QStringLiteral("budget-exceeded"); result.errorMessage = PDFTranslationContext::tr("Evidence collection exceeded the %1 processing budget.") diff --git a/UnitTests/tst_evidencegraphtest.cpp b/UnitTests/tst_evidencegraphtest.cpp index 337bc5b7c..81eb42bc3 100644 --- a/UnitTests/tst_evidencegraphtest.cpp +++ b/UnitTests/tst_evidencegraphtest.cpp @@ -44,6 +44,8 @@ class EvidenceGraphTest : public QObject private slots: void collectWithoutDocument_isIncomplete(); void emptyPage_isComplete(); + void cancelledCollection_isIncompleteAndCancelled(); + void cancelledPreflight_reportsCancelled(); void incompleteGraphCannotPass(); void imageFamilyDualRunMatchesEngine(); void colorantsFamilyDualRunMatchesEngine(); @@ -125,6 +127,12 @@ void assertFindingCitesGraphRecord(const QList& findings, QFAIL(qPrintable(QStringLiteral("Expected finding type '%1' for check '%2'").arg(findingType, checkId))); } +class CancelledOperationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override { return true; } +}; + } // namespace void EvidenceGraphTest::collectWithoutDocument_isIncomplete() @@ -146,6 +154,42 @@ void EvidenceGraphTest::emptyPage_isComplete() QVERIFY(graph.incompleteReason.isEmpty()); } +void EvidenceGraphTest::cancelledCollection_isIncompleteAndCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + pdf::PDFEvidenceCollectSettings settings; + settings.operationControl = &cancelled; + + const pdf::PDFEvidenceGraph graph = pdf::PDFEvidenceCollector::collect(&session, pdf::pdfEvidenceAllDomains(), settings); + QVERIFY(!graph.isComplete()); + QCOMPARE(graph.incompleteReason, QStringLiteral("cancelled")); + QVERIFY(graph.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); +} + +void EvidenceGraphTest::cancelledPreflight_reportsCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + + pdf::PreflightEngine engine(&session); + engine.setOperationControl(&cancelled); + const QJsonObject profile{ + { QStringLiteral("name"), QStringLiteral("Color inventory") }, + { QStringLiteral("checks"), QJsonArray{ + QJsonObject{ + { QStringLiteral("id"), QStringLiteral("color-inventory") }, + { QStringLiteral("severity"), QStringLiteral("info") }, + { QStringLiteral("probe_dpi"), 150 }, + { QStringLiteral("rich_black_k_percent"), 10 } } } } + }; + const pdf::PreflightResult result = engine.run(profile); + QVERIFY(!result.inspectionComplete); + QCOMPARE(result.errorCode, QStringLiteral("cancelled")); +} + void EvidenceGraphTest::incompleteGraphCannotPass() { pdf::PreflightEngine engine(nullptr); From f8b4603873d5bdc0288f736669f344bbc4c5db8f Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:55:26 -0700 Subject: [PATCH 11/14] feat(envelope): sample the preflight phase for very large fixtures Preflight renders and walks every page it covers, about 0.5-0.7 s per page on a hosted runner, so a full pass over the 10,000-page fixture cannot fit any practical timeout. benchmark gains --preflight-page-last , which limits the preflight phase to pages 1..n while rendering still covers every page. PDFEvidenceCollectSettings and PDFColorInventorySettings gain pageIndices so the render and content walk skip pages the profile scope already discards. run_matrix.py passes 256 for fixtures above 1,000 pages and records it as profile.preflight_page_last. Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfcolorinventory.cpp | 5 ++++ LoopLibCore/sources/pdfcolorinventory.h | 5 ++++ LoopLibCore/sources/pdfevidencegraph.cpp | 5 ++++ LoopLibCore/sources/pdfevidencegraph.h | 6 +++++ LoopLibCore/sources/preflightengine.cpp | 3 +++ PdfTool/pdftoolabstractapplication.cpp | 8 ++++++ PdfTool/pdftoolabstractapplication.h | 3 +++ PdfTool/pdftoolrender.cpp | 2 +- UnitTests/tst_evidencegraphtest.cpp | 21 +++++++++++++++ docs/RESOURCE_ENVELOPE.md | 7 ++++- docs/RESOURCE_ENVELOPE_QUALIFICATION.md | 5 +++- scripts/resource_envelope/run_matrix.py | 27 ++++++++++++++++--- .../test_run_matrix_probes.py | 20 ++++++++++++++ 13 files changed, 110 insertions(+), 7 deletions(-) diff --git a/LoopLibCore/sources/pdfcolorinventory.cpp b/LoopLibCore/sources/pdfcolorinventory.cpp index d4c58eccb..0e191567e 100644 --- a/LoopLibCore/sources/pdfcolorinventory.cpp +++ b/LoopLibCore/sources/pdfcolorinventory.cpp @@ -131,6 +131,11 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin break; } + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { diff --git a/LoopLibCore/sources/pdfcolorinventory.h b/LoopLibCore/sources/pdfcolorinventory.h index bb1a57ff3..afea5266a 100644 --- a/LoopLibCore/sources/pdfcolorinventory.h +++ b/LoopLibCore/sources/pdfcolorinventory.h @@ -29,8 +29,11 @@ #include #include +#include #include +#include + namespace pdf { @@ -70,6 +73,8 @@ struct LOOPLIBCORESHARED_EXPORT PDFColorInventorySettings int probeDpi = 150; qreal richBlackKThreshold = 0.10; const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages to probe; unset probes every page. + std::optional> pageIndices; }; /// Shared rich-black predicate used by preflight and Output Preview. diff --git a/LoopLibCore/sources/pdfevidencegraph.cpp b/LoopLibCore/sources/pdfevidencegraph.cpp index 73dae9505..a1cfe6772 100644 --- a/LoopLibCore/sources/pdfevidencegraph.cpp +++ b/LoopLibCore/sources/pdfevidencegraph.cpp @@ -1290,6 +1290,7 @@ void collectColorants(PDFDocumentSession* session, PDFEvidenceGraph* graph, cons inventorySettings.probeDpi = settings.colorProbeDpi; inventorySettings.richBlackKThreshold = settings.richBlackKThreshold; inventorySettings.operationControl = settings.operationControl; + inventorySettings.pageIndices = settings.pageIndices; PDFColorInventory inventory(session); const PDFColorInventoryResult result = inventory.inspect(inventorySettings); if (result.cancelled) @@ -1412,6 +1413,10 @@ PDFEvidenceGraph PDFEvidenceCollector::collect(PDFDocumentSession* session, graph.incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); return graph; } + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } const PDFPage* page = catalog->getPage(pageIndex); if (!page) diff --git a/LoopLibCore/sources/pdfevidencegraph.h b/LoopLibCore/sources/pdfevidencegraph.h index 6d12e6d18..efc7df0b1 100644 --- a/LoopLibCore/sources/pdfevidencegraph.h +++ b/LoopLibCore/sources/pdfevidencegraph.h @@ -32,8 +32,11 @@ #include #include #include +#include #include +#include + namespace pdf { @@ -104,6 +107,9 @@ struct LOOPLIBCORESHARED_EXPORT PDFEvidenceCollectSettings /// Polled between pages; a cancelled collection returns an incomplete graph /// with incompleteReason "cancelled". const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages whose content is walked and probed; unset + /// covers every page. Document-level evidence is collected either way. + std::optional> pageIndices; }; class LOOPLIBCORESHARED_EXPORT PDFEvidenceCollector diff --git a/LoopLibCore/sources/preflightengine.cpp b/LoopLibCore/sources/preflightengine.cpp index 21b13b9b6..d0d7dcc24 100644 --- a/LoopLibCore/sources/preflightengine.cpp +++ b/LoopLibCore/sources/preflightengine.cpp @@ -6196,6 +6196,9 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const { PDFEvidenceCollectSettings evidenceSettings = evidenceSettingsForProfile(profile); evidenceSettings.operationControl = m_operationControl; + // Records outside the profile's page scope are dropped below, so do not + // spend the render and content walk on those pages. + evidenceSettings.pageIndices = profile.restrictions.pages; m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettings); if (profile.restrictions.pages.has_value() || (!plan.full && !plan.pages.isEmpty())) { diff --git a/PdfTool/pdftoolabstractapplication.cpp b/PdfTool/pdftoolabstractapplication.cpp index 096353938..b2a419ce7 100644 --- a/PdfTool/pdftoolabstractapplication.cpp +++ b/PdfTool/pdftoolabstractapplication.cpp @@ -391,6 +391,7 @@ QList PDFToolAbstractApplication::describeOptions(Optio if (optionFlags.testFlag(BenchmarkPreflightProfile)) { add(QStringLiteral("profile"), { QStringLiteral("--profile") }, QStringLiteral("profile"), PDFToolValueType::Path); + add(QStringLiteral("preflight-page-last"), { QStringLiteral("--preflight-page-last") }, QStringLiteral("page"), PDFToolValueType::Integer, {}, QStringLiteral("0")); } if (optionFlags.testFlag(CapabilityDiscovery)) { @@ -907,6 +908,7 @@ void PDFToolAbstractApplication::initializeCommandLineParser(QCommandLineParser* if (optionFlags.testFlag(BenchmarkPreflightProfile)) { addDescribedOption(parser, optionDescriptors, QStringLiteral("profile"), QStringLiteral("Run a preflight phase with this profile before rendering and record its memory high-water.")); + addDescribedOption(parser, optionDescriptors, QStringLiteral("preflight-page-last"), QStringLiteral("Limit the preflight phase to pages 1 through this page (0 covers the whole document); rendering still covers every selected page.")); } if (optionFlags.testFlag(CapabilityDiscovery)) @@ -1495,6 +1497,12 @@ PDFToolOptions PDFToolAbstractApplication::getOptions(QCommandLineParser* parser if (optionFlags.testFlag(BenchmarkPreflightProfile)) { options.preflightProfilePath = parser->value("profile"); + bool preflightPageLastOk = false; + const int preflightPageLast = parser->value("preflight-page-last").toInt(&preflightPageLastOk); + if (preflightPageLastOk && preflightPageLast > 0) + { + options.preflightPageLast = preflightPageLast; + } } if (optionFlags.testFlag(VerifyPreflightCertificate)) diff --git a/PdfTool/pdftoolabstractapplication.h b/PdfTool/pdftoolabstractapplication.h index cdf9335b9..0926748b4 100644 --- a/PdfTool/pdftoolabstractapplication.h +++ b/PdfTool/pdftoolabstractapplication.h @@ -256,6 +256,9 @@ struct PDFToolOptions // figures" is a legitimate answer - so the fail-closed reading is opt-in. bool failIfEmpty = false; + // For option 'BenchmarkPreflightProfile': last page of the preflight phase, 0 for all pages. + int preflightPageLast = 0; + // For option 'PreflightProfile' QString preflightProfilePath; QString preflightJobContextPath; diff --git a/PdfTool/pdftoolrender.cpp b/PdfTool/pdftoolrender.cpp index f277be7d5..99a6b8a07 100644 --- a/PdfTool/pdftoolrender.cpp +++ b/PdfTool/pdftoolrender.cpp @@ -214,7 +214,7 @@ PDFToolExitCode PDFToolBenchmark::execute(const PDFToolOptions& options) request.plan.full = true; request.plan.reason = QStringLiteral("benchmark-preflight-phase"); request.firstPage = options.pageSelectorFirstPage; - request.lastPage = options.pageSelectorLastPage; + request.lastPage = options.preflightPageLast > 0 ? QString::number(options.preflightPageLast) : options.pageSelectorLastPage; request.selectedPages = options.pageSelectorSelection; request.cancellation = &cancellationControl; diff --git a/UnitTests/tst_evidencegraphtest.cpp b/UnitTests/tst_evidencegraphtest.cpp index 81eb42bc3..c667a37cb 100644 --- a/UnitTests/tst_evidencegraphtest.cpp +++ b/UnitTests/tst_evidencegraphtest.cpp @@ -46,6 +46,7 @@ private slots: void emptyPage_isComplete(); void cancelledCollection_isIncompleteAndCancelled(); void cancelledPreflight_reportsCancelled(); + void pageScope_skipsUnselectedPages(); void incompleteGraphCannotPass(); void imageFamilyDualRunMatchesEngine(); void colorantsFamilyDualRunMatchesEngine(); @@ -190,6 +191,26 @@ void EvidenceGraphTest::cancelledPreflight_reportsCancelled() QCOMPARE(result.errorCode, QStringLiteral("cancelled")); } +void EvidenceGraphTest::pageScope_skipsUnselectedPages() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + + const pdf::PDFEvidenceGraph all = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants); + QVERIFY(!all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + pdf::PDFEvidenceCollectSettings settings; + settings.pageIndices = QSet(); + const pdf::PDFEvidenceGraph none = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QVERIFY(none.isComplete()); + QVERIFY(none.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + settings.pageIndices = QSet{ 0 }; + const pdf::PDFEvidenceGraph first = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QCOMPARE(first.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size(), + all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size()); +} + void EvidenceGraphTest::incompleteGraphCannotPass() { pdf::PreflightEngine engine(nullptr); diff --git a/docs/RESOURCE_ENVELOPE.md b/docs/RESOURCE_ENVELOPE.md index ecaa77df4..a525afc56 100644 --- a/docs/RESOURCE_ENVELOPE.md +++ b/docs/RESOURCE_ENVELOPE.md @@ -92,7 +92,12 @@ named pool records. `pages_materialized` reports pages actually processed by a runner; it is not the catalog page count. `preflight_high_water_bytes` is the process high-water when the `benchmark --profile ` preflight phase ends; without `--profile` it stays `-1` and the record is explicitly `incomplete` rather than being -promoted to a passing result. The deterministic +promoted to a passing result. `--preflight-page-last ` limits that phase to +pages 1 through `n` while rendering still covers every selected page; the +runner passes it (256) for fixtures above 1,000 pages, because preflight costs +roughly 0.5-0.7 s per page on a hosted runner, and records it as +`profile.preflight_page_last`. A sampled record's `preflight_high_water_bytes` +covers the sampled pages, not the whole document. The deterministic pathological and transparency/spot fixtures can be generated without the external DIV2K corpus: diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index b9d1fe5f2..ccc0a0137 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -31,7 +31,10 @@ job: 3. Runs `run_matrix.py --strict --repetitions 3` with: - a measured preflight phase (`benchmark --profile`, default `loop-preflight/profiles/loop-default.json`), so a clean run reports - `status: complete` with a real `preflight_high_water_bytes`; + `status: complete` with a real `preflight_high_water_bytes`. Fixtures + above 1,000 pages (the 10,000-page one) preflight their first 256 pages + only (`--preflight-page-last 256`, recorded as + `profile.preflight_page_last`); rendering still covers every page; - a cancellation probe on `ten-thousand-page`, which interrupts the run and requires a `cancelled` envelope within the workload's `cancellation_latency_ms`; diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index 0254ceccf..627271b76 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -39,6 +39,12 @@ # 33.7 MB, so three fit the 128 MiB raster-tile-cache pool and a fourth is # rejected as budget-exceeded, leaving pages unrendered (exit code 5). DEFAULT_RASTERIZERS = 3 +# Preflight renders and walks every page it covers (about 0.5-0.7 s per page on +# a hosted runner), so a full pass over a very large document outlives any +# practical timeout. Documents above the threshold get a preflight phase over +# their first pages only; rendering still covers every page. +PREFLIGHT_SAMPLE_THRESHOLD_PAGES = 1000 +PREFLIGHT_SAMPLE_PAGES = 256 TIMEOUT_REASON = "benchmark-timeout" # PdfTool's defined terminal exit codes (pdftoolresult.h) except InternalError # (7). Anything else, including a negative POSIX signal or a Windows exception @@ -161,6 +167,7 @@ def _benchmark_command( rasterizers: int, preflight_profile: Path | None, first_page_only: bool = False, + preflight_page_last: int | None = None, ) -> list[str]: # Pin rasterizers to a fixed value so the same code and fixtures produce # comparable RSS and elapsed time across hosts with different CPU counts. @@ -168,11 +175,20 @@ def _benchmark_command( command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] if preflight_profile is not None: command += ["--profile", str(preflight_profile)] + if preflight_page_last is not None: + command += ["--preflight-page-last", str(preflight_page_last)] if first_page_only: command += ["--page-first", "1", "--page-last", "1"] return command +def _preflight_page_last(page_count: int | None) -> int | None: + """Last page of the preflight phase, or None when it covers the whole document.""" + if page_count is not None and page_count > PREFLIGHT_SAMPLE_THRESHOLD_PAGES: + return PREFLIGHT_SAMPLE_PAGES + return None + + def _identity_errors(envelope: Mapping[str, Any], candidate_sha: str, fixture_sha256: str) -> list[str]: identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} errors: list[str] = [] @@ -368,13 +384,15 @@ def run_fixture( spec = FIXTURE_SPECS[fixture_id] workload = _fixture_workload(fixture_id, metadata) fixture_details, provenance_errors = _fixture_metadata(fixture_id, fixture_path, metadata, require_provenance) - command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile) + expected_page_count = metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"] + preflight_page_last = _preflight_page_last(expected_page_count) + command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile, preflight_page_last=preflight_page_last) record: dict[str, Any] = { "fixture_id": fixture_id, "path": str(fixture_path), - "expected_page_count": metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"], + "expected_page_count": expected_page_count, "workload": workload, - "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers, "preflight_profile": str(preflight_profile) if preflight_profile else None}, + "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers, "preflight_profile": str(preflight_profile) if preflight_profile else None, "preflight_page_last": preflight_page_last if preflight_profile else None}, "command": command, **fixture_details, } @@ -467,7 +485,8 @@ def run_cancellation_probe( limits = budgets.get("workloads", {}).get(workload, {}) if workload else {} errors: list[str] = [] - cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile) + cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile, + preflight_page_last=_preflight_page_last(FIXTURE_SPECS.get(fixture_id, {}).get("expected_page_count"))) cancellation: dict[str, Any] = {"command": cancel_command, "requested_after_seconds": cancel_after_seconds, "cancellation_latency_ms": -1} try: completed = cancel_runner(cancel_command, timeout_seconds, cancel_after_seconds) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index 77bf11234..5822ef152 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -79,6 +79,26 @@ def test_preflight_profile_reaches_the_command(self) -> None: record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) self.assertEqual(record["command"][-2:], ["--profile", "profile.json"]) + def test_small_document_preflight_covers_every_page(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + + def test_large_document_preflight_is_sampled(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-4:], ["--profile", "profile.json", "--preflight-page-last", "256"]) + self.assertEqual(record["profile"]["preflight_page_last"], 256) + + def test_sampling_needs_a_preflight_profile(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + def test_crashed_process_fails_even_with_an_envelope(self) -> None: with _Fixture() as fixture: record = fixture.measure(lambda command, **_: _process(command, -11, _envelope(fixture.digest))) From 8a789dd911bb4f116d29f9c7990f088712e31d91 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:26:29 -0700 Subject: [PATCH 12/14] fix(envelope): cap the colour-inventory probe and run the cancel probe render-only The hostile raster-probe-pixel-budget case reached 15 GB RSS on both hosted platforms: the colour inventory probes each page at 150 DPI with several float bitmaps per pixel and no size limit. PDFColorInventorySettings gains maxProbePixels (2.5 million); larger pages are probed at a proportionally lower DPI. The cancellation probe ran with the preflight profile, whose document-wide setup does not poll for cancellation, so latency was 11-16 s against a 5 s policy. The probe now interrupts a render-only run, like the recovery probe. Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfcolorinventory.cpp | 7 ++++++- LoopLibCore/sources/pdfcolorinventory.h | 3 +++ docs/RESOURCE_ENVELOPE_QUALIFICATION.md | 4 +++- scripts/resource_envelope/run_matrix.py | 13 ++++++++----- scripts/resource_envelope/test_run_matrix_probes.py | 1 + 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/LoopLibCore/sources/pdfcolorinventory.cpp b/LoopLibCore/sources/pdfcolorinventory.cpp index 0e191567e..9d8ec01f1 100644 --- a/LoopLibCore/sources/pdfcolorinventory.cpp +++ b/LoopLibCore/sources/pdfcolorinventory.cpp @@ -150,7 +150,12 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin continue; } - const QSize imageSize(qMax(1, int(widthPxReal)), qMax(1, int(heightPxReal))); + // The probe holds several float bitmaps per pixel, so an oversized page is probed + // at a coarser resolution instead of allocating memory proportional to its size. + const double probeScale = settings.maxProbePixels > 0 && widthPxReal * heightPxReal > double(settings.maxProbePixels) + ? std::sqrt(double(settings.maxProbePixels) / (widthPxReal * heightPxReal)) + : 1.0; + const QSize imageSize(qMax(1, int(widthPxReal * probeScale)), qMax(1, int(heightPxReal * probeScale))); const QTransform pagePointToDevice = PDFRenderer::createPagePointToDevicePointMatrix( page, QRect(QPoint(0, 0), imageSize)); PDFTransparencyRenderer renderer(page, diff --git a/LoopLibCore/sources/pdfcolorinventory.h b/LoopLibCore/sources/pdfcolorinventory.h index afea5266a..d18902e27 100644 --- a/LoopLibCore/sources/pdfcolorinventory.h +++ b/LoopLibCore/sources/pdfcolorinventory.h @@ -72,6 +72,9 @@ struct LOOPLIBCORESHARED_EXPORT PDFColorInventorySettings { int probeDpi = 150; qreal richBlackKThreshold = 0.10; + /// Largest probe raster in pixels; larger pages are probed at a proportionally lower DPI. + /// A letter page at the default 150 DPI is about 1.9 million pixels. + qint64 maxProbePixels = 2'500'000; const PDFOperationControl* operationControl = nullptr; /// Zero-based indices of the pages to probe; unset probes every page. std::optional> pageIndices; diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index ccc0a0137..f9a720c9f 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -35,7 +35,9 @@ job: above 1,000 pages (the 10,000-page one) preflight their first 256 pages only (`--preflight-page-last 256`, recorded as `profile.preflight_page_last`); rendering still covers every page; - - a cancellation probe on `ten-thousand-page`, which interrupts the run + - a cancellation probe on `ten-thousand-page`, which interrupts a render-only + run (no preflight phase, whose document-wide setup does not poll for + cancellation) and requires a `cancelled` envelope within the workload's `cancellation_latency_ms`; - a recovery probe, which times a fresh process reopening the same diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index 627271b76..da752f2ea 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -469,11 +469,15 @@ def run_cancellation_probe( candidate_sha: str, workload: str | None = None, rasterizers: int = DEFAULT_RASTERIZERS, - preflight_profile: Path | None = None, cancel_runner: CancelRunner = _run_benchmark_process, runner: Runner = subprocess.run, ) -> dict[str, Any]: - """Interrupts one run, then times a fresh process reopening the fixture. + """Interrupts one render run, then times a fresh process reopening the fixture. + + The interrupted run has no preflight phase: preflight setup (fonts, ink + mapper, resource scan) is document-wide and does not poll for cancellation, + so on a very large document it would dominate the latency this probe + measures. Preflight cancellation is covered by the evidence-graph tests. ``recovery_ms`` is the wall time from launching that fresh process until it exits having rendered the first page: the time an operator waits to get the @@ -485,8 +489,7 @@ def run_cancellation_probe( limits = budgets.get("workloads", {}).get(workload, {}) if workload else {} errors: list[str] = [] - cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile, - preflight_page_last=_preflight_page_last(FIXTURE_SPECS.get(fixture_id, {}).get("expected_page_count"))) + cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None) cancellation: dict[str, Any] = {"command": cancel_command, "requested_after_seconds": cancel_after_seconds, "cancellation_latency_ms": -1} try: completed = cancel_runner(cancel_command, timeout_seconds, cancel_after_seconds) @@ -705,7 +708,7 @@ def run_matrix( else: fixture_path, metadata = resolved[cancel_fixture] probe = run_cancellation_probe(pdf_tool, cancel_fixture, fixture_path, budgets, timeout_seconds, cancel_after_seconds or 1.0, candidate_sha, - _fixture_workload(cancel_fixture, metadata), rasterizers, preflight_profile, cancel_runner, runner) + _fixture_workload(cancel_fixture, metadata), rasterizers, cancel_runner, runner) hostile = None if hostile_corpus is not None: diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index 5822ef152..3f6f6e0d1 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -153,6 +153,7 @@ def test_cancelled_run_and_reopen_are_measured(self) -> None: ) self.assertEqual(probe["status"], "measured", probe["validation_errors"]) self.assertEqual(probe["cancellation"]["cancellation_latency_ms"], 20) + self.assertNotIn("--profile", probe["cancellation"]["command"]) self.assertGreaterEqual(probe["recovery"]["recovery_ms"], 0) self.assertEqual(probe["recovery"]["command"][-4:], ["--page-first", "1", "--page-last", "1"]) From 04ead25a4d5d7fe241118fefdd98bd0b90d0116a Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:48:05 -0700 Subject: [PATCH 13/14] fix(envelope): raise the active-model pool to 640 MiB and the 10k render cap to 600 s The 500 MB image-heavy fixture's document-model estimate exceeds the 256 MiB active-document-model pool, so the benchmark rejects it before rendering. The pool default becomes 640 MiB, still under the 768 MiB resident ceiling. Three rasterizers on a hosted runner render the 10,000-page fixture in about 350 s (Linux) to 510 s (Windows), over the 120 s synthetic-image-heavy cap. That workload's wall_time_ms becomes 600000; the DIV2K workload keeps 120000. Co-Authored-By: Claude Sonnet 5.5 --- LoopLibCore/sources/pdfresourcebudget.h | 2 +- UnitTests/tst_resourcebudgettest.cpp | 2 +- docs/RESOURCE_BUDGETS.md | 2 +- docs/RESOURCE_ENVELOPE_BUDGETS.json | 4 ++-- docs/RESOURCE_ENVELOPE_QUALIFICATION.md | 4 +++- docs/generated/huge-document-envelope.json | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/LoopLibCore/sources/pdfresourcebudget.h b/LoopLibCore/sources/pdfresourcebudget.h index 93ac07334..0c351bc9c 100644 --- a/LoopLibCore/sources/pdfresourcebudget.h +++ b/LoopLibCore/sources/pdfresourcebudget.h @@ -83,7 +83,7 @@ struct LOOPLIBCORESHARED_EXPORT PDFResourceBudgetConfig /// resident ceiling is reached; active model and a visible request remain /// hard admission boundaries. std::array poolLimits = { - 256 * MiB, // active document model + 640 * MiB, // active document model 128 * MiB, // compiled/evidence cache 128 * MiB, // raster/tile cache 128 * MiB, // GPU/texture accounted proxy diff --git a/UnitTests/tst_resourcebudgettest.cpp b/UnitTests/tst_resourcebudgettest.cpp index de22587bd..42f9178e1 100644 --- a/UnitTests/tst_resourcebudgettest.cpp +++ b/UnitTests/tst_resourcebudgettest.cpp @@ -25,7 +25,7 @@ void ResourceBudgetTest::conservativeDefaultsExposeAllPools() { const pdf::PDFResourceBudgetConfig config = pdf::PDFResourceBudgetConfig::conservativeDefaults(); QCOMPARE(config.residentLimitBytes, 768 * pdf::PDFResourceBudgetConfig::MiB); - QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 256 * pdf::PDFResourceBudgetConfig::MiB); + QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 640 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::CompiledEvidenceCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::RasterTileCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::GpuTextureCache), 128 * pdf::PDFResourceBudgetConfig::MiB); diff --git a/docs/RESOURCE_BUDGETS.md b/docs/RESOURCE_BUDGETS.md index 6c1541a1e..1911acb60 100644 --- a/docs/RESOURCE_BUDGETS.md +++ b/docs/RESOURCE_BUDGETS.md @@ -22,7 +22,7 @@ the following conservative defaults: | Pool | Limit | Admission rule | |------|-------|----------------| -| active document model | 256 MiB | interaction-priority hard boundary | +| active document model | 640 MiB | interaction-priority hard boundary | | compiled/evidence cache | 128 MiB | insertion-order eviction, then reject | | raster/tile cache | 128 MiB | prefetch shed, then visible admission reject | | GPU texture cache | 128 MiB | source-image byte proxy; physical GPU bytes are unavailable (`-1`) | diff --git a/docs/RESOURCE_ENVELOPE_BUDGETS.json b/docs/RESOURCE_ENVELOPE_BUDGETS.json index a19e06716..fdb6e7dae 100644 --- a/docs/RESOURCE_ENVELOPE_BUDGETS.json +++ b/docs/RESOURCE_ENVELOPE_BUDGETS.json @@ -7,7 +7,7 @@ "resource_budget": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -28,7 +28,7 @@ }, "synthetic-image-heavy": { "page_count": 10000, - "wall_time_ms": 120000, + "wall_time_ms": 600000, "rss_high_water_bytes": 805306368, "cancellation_latency_ms": 5000, "recovery_ms": 30000 diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index f9a720c9f..96018b332 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -27,7 +27,9 @@ job: image-heavy, and 10,000-page fixtures are deterministic synthetic PDFs (SHAKE-256 noise images stored with FlateDecode), so hosted runners need no external corpus. The 10,000-page fixture uses the - `synthetic-image-heavy` workload caps, which equal the DIV2K caps. + `synthetic-image-heavy` workload caps, which equal the DIV2K caps except + `wall_time_ms` (600 s): three rasterizers on a hosted runner render the + 10,000 pages in about 350 s (Linux) to 510 s (Windows). 3. Runs `run_matrix.py --strict --repetitions 3` with: - a measured preflight phase (`benchmark --profile`, default `loop-preflight/profiles/loop-default.json`), so a clean run reports diff --git a/docs/generated/huge-document-envelope.json b/docs/generated/huge-document-envelope.json index 9b9fd6b90..0a78d2d2b 100644 --- a/docs/generated/huge-document-envelope.json +++ b/docs/generated/huge-document-envelope.json @@ -32,7 +32,7 @@ "config": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -45,7 +45,7 @@ "resident_high_water_bytes": 0, "pressure": "normal", "pools": { - "active-document-model": { "limit_bytes": 268435456, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, + "active-document-model": { "limit_bytes": 671088640, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "compiled-evidence-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "raster-tile-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "gpu-texture-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, From 14264ad6113f705c813e2b58815c6614b5591129 Mon Sep 17 00:00:00 2001 From: mbx30 <212453881+mberrys@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:15:04 -0700 Subject: [PATCH 14/14] fix(envelope): give the 500 MB fixture its own process RSS cap image-heavy-500mb now renders all 60 pages (the active-model pool no longer rejects it), but its process RSS peaks at about 1.7 GB in the preflight phase on both platforms, against the 768 MiB resident limit. The reader holds the whole file and its object model, so the peak scales with file size. The fixture maps to a new large-document-500mb workload with a 2 GiB RSS cap; the resident-limit check in run_fixture uses a workload's own RSS cap when it declares one. Every other fixture keeps 768 MiB. Co-Authored-By: Claude Sonnet 5.5 --- docs/RESOURCE_ENVELOPE_BUDGETS.json | 6 ++++++ docs/RESOURCE_ENVELOPE_QUALIFICATION.md | 5 +++++ scripts/resource_envelope/run_matrix.py | 21 +++++++++++++++---- .../test_run_matrix_probes.py | 14 +++++++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/RESOURCE_ENVELOPE_BUDGETS.json b/docs/RESOURCE_ENVELOPE_BUDGETS.json index fdb6e7dae..6d11f8a5a 100644 --- a/docs/RESOURCE_ENVELOPE_BUDGETS.json +++ b/docs/RESOURCE_ENVELOPE_BUDGETS.json @@ -33,6 +33,12 @@ "cancellation_latency_ms": 5000, "recovery_ms": 30000 }, + "large-document-500mb": { + "wall_time_ms": 120000, + "rss_high_water_bytes": 2147483648, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, "pathological-vector": { "page_count": 256, "wall_time_ms": 120000, diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index 96018b332..cadc4644a 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -37,6 +37,11 @@ job: above 1,000 pages (the 10,000-page one) preflight their first 256 pages only (`--preflight-page-last 256`, recorded as `profile.preflight_page_last`); rendering still covers every page; + - a `large-document-500mb` workload for the 500 MB fixture, whose process + RSS cap is 2 GiB instead of the 768 MiB resident limit: the reader holds + the whole file plus its object model in memory, so the peak is about 3.3 + times the file size (1.7 GB measured on both platforms), while the + accounted pools stay under the resident limit; - a cancellation probe on `ten-thousand-page`, which interrupts a render-only run (no preflight phase, whose document-wide setup does not poll for cancellation) diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index da752f2ea..5d732e683 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -60,7 +60,7 @@ # addressability and available disk are environment-dependent. FIXTURE_SPECS: dict[str, dict[str, Any]] = { "office-2mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 1_500_000, "max_bytes": 2_500_000}, - "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 450_000_000, "max_bytes": 550_000_000}, + "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": "large-document-500mb", "min_bytes": 450_000_000, "max_bytes": 550_000_000}, "multi-gb": {"required": False, "expected_page_count": None, "workload": None, "min_bytes": 1_000_000_000, "max_bytes": None}, "ten-thousand-page": {"required": True, "expected_page_count": 10000, "workload": "div2k-image-heavy", "min_bytes": None, "max_bytes": None}, "pathological-vector": {"required": True, "expected_page_count": 256, "workload": "pathological-vector", "min_bytes": None, "max_bytes": None}, @@ -336,6 +336,19 @@ def _fixture_workload(fixture_id: str, metadata: Mapping[str, Any] | None) -> st return FIXTURE_SPECS[fixture_id]["workload"] +def _rss_limit(budgets: Mapping[str, Any], workload: str | None) -> int | None: + """Process RSS ceiling: the workload's own cap when it declares one, else the resident limit. + + The reader holds a whole document in memory, so a very large document's + process RSS is a multiple of its file size and cannot fit the resident + limit that governs the accounted pools. + """ + workload_limit = budgets.get("workloads", {}).get(workload, {}).get("rss_high_water_bytes") if workload else None + if isinstance(workload_limit, int): + return workload_limit + return budgets.get("resource_budget", {}).get("resident_limit_bytes") + + def _aggregate_envelopes(envelopes: list[Mapping[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: # Use the highest-RSS run as the safety representative and the median # elapsed time. This keeps peak-memory validation conservative while @@ -429,9 +442,9 @@ def run_fixture( if expected_page_count is not None and envelope.get("page_count") != expected_page_count: validation_errors.append(f"run {index}: page_count {envelope.get('page_count')} does not match expected {expected_page_count}") rss = envelope.get("rss_high_water_bytes") - resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") - if isinstance(rss, int) and rss >= 0 and isinstance(resident_limit, int) and rss > resident_limit: - validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {resident_limit}") + rss_limit = _rss_limit(budgets, workload) + if isinstance(rss, int) and rss >= 0 and isinstance(rss_limit, int) and rss > rss_limit: + validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {rss_limit}") validation_errors.extend(f"run {index}: {error}" for error in _identity_errors(envelope, candidate_sha, record["fixture_sha256"])) record["runs"] = runs diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py index 3f6f6e0d1..70197ec41 100644 --- a/scripts/resource_envelope/test_run_matrix_probes.py +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -19,6 +19,7 @@ def _policy() -> dict: "workloads": { "pathological-vector": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200, "cancellation_latency_ms": 50, "recovery_ms": 60000}, "synthetic-image-heavy": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200}, + "large-document": {"wall_time_ms": 100, "rss_high_water_bytes": 2000}, }, } @@ -99,6 +100,19 @@ def test_sampling_needs_a_preflight_profile(self) -> None: self.assertNotIn("--preflight-page-last", record["command"]) self.assertIsNone(record["profile"]["preflight_page_last"]) + def test_workload_rss_cap_replaces_the_resident_limit(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=1000))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + def test_workload_rss_cap_still_binds(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=2500))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: RSS 2500 exceeds resident policy 2000", record["validation_errors"]) + def test_crashed_process_fails_even_with_an_envelope(self) -> None: with _Fixture() as fixture: record = fixture.measure(lambda command, **_: _process(command, -11, _envelope(fixture.digest)))