diff --git a/.github/workflows/resource-envelope-qualification.yml b/.github/workflows/resource-envelope-qualification.yml new file mode 100644 index 000000000..675546263 --- /dev/null +++ b/.github/workflows/resource-envelope-qualification.yml @@ -0,0 +1,302 @@ +name: Resource envelope qualification + +# Hosted qualification for issue #19: builds PdfTool from the candidate SHA, +# generates the deterministic synthetic fixture bundle, and runs the strict +# resource-envelope matrix (measured fixtures, cancellation/recovery probe, +# hostile corpus) on Linux and Windows. The evidence job turns both matrices +# into one record carrying this run's id. + +on: + workflow_dispatch: + pull_request: + paths: + - 'scripts/resource_envelope/**' + - 'scripts/qualification/*resource_envelope*' + - 'PdfTool/main.cpp' + - 'PdfTool/pdftoolrender.*' + - 'LoopLibCore/sources/pdfworkloadenvelope.*' + - 'LoopLibCore/sources/pdfresourcebudget.*' + - 'LoopLibCore/sources/pdfrenderer.*' + - 'docs/RESOURCE_ENVELOPE_BUDGETS.json' + - '.github/workflows/resource-envelope-qualification.yml' + +permissions: + contents: read + +concurrency: + group: resource-envelope-${{ github.ref }} + cancel-in-progress: true + +env: + REPETITIONS: 3 + CANCEL_FIXTURE: ten-thousand-page + CANCEL_AFTER_SECONDS: 3 + TIMEOUT_SECONDS: 900 + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}/loop/vcpkg/overlays/linux:${{ github.workspace }}/loop/vcpkg/overlays/general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/.vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Install Dependencies + run: | + mkdir -p "$VCPKG_DEFAULT_BINARY_CACHE" + sudo apt update + sudo apt install -y autoconf autoconf-archive automake libtool libcups2 libcups2-dev libfontconfig1-dev + + - name: 'VCPKG: Set up VCPKG' + run: | + VCPKG_COMMIT="$(python3 -c 'import json; print(json.load(open("loop/vcpkg-configuration.json"))["default-registry"]["baseline"])')" + if ! [[ "$VCPKG_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::Invalid vcpkg baseline: $VCPKG_COMMIT" + exit 1 + fi + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach "$VCPKG_COMMIT" + test "$(git -C vcpkg rev-parse HEAD)" = "$VCPKG_COMMIT" + ./vcpkg/bootstrap-vcpkg.sh + ./vcpkg/vcpkg integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg_installed + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + run: | + ./vcpkg install --x-manifest-root=$GITHUB_WORKSPACE/loop --x-install-root=$VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'linux' + target: 'desktop' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + + - name: Build PdfTool + working-directory: loop + run: | + cmake -B build -S . -DLOOP_LOOP_DISTRIBUTION=ON -DLOOP_INSTALL_QT_DEPENDENCIES=0 -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release + cmake --build build --target PdfTool --config Release -j6 + PDF_TOOL="$(find "$PWD/build" -type f -name PdfTool -perm -u+x | head -n 1)" + test -n "$PDF_TOOL" + echo "PDF_TOOL=$PDF_TOOL" >> "$GITHUB_ENV" + "$PDF_TOOL" help --console-format json > /dev/null + + - name: Generate synthetic fixture bundle + working-directory: loop + run: python3 scripts/resource_envelope/synthetic_workload.py --output-dir "$RUNNER_TEMP/fixtures" --manifest "$RUNNER_TEMP/fixtures/fixtures.json" + + - name: Run strict resource-envelope matrix + working-directory: loop + run: | + mkdir -p "$RUNNER_TEMP/qualification" + cp "$RUNNER_TEMP/fixtures/fixtures.json" "$RUNNER_TEMP/qualification/fixtures.json" + python3 -m scripts.resource_envelope.run_matrix \ + --pdf-tool "$PDF_TOOL" \ + --manifest "$RUNNER_TEMP/fixtures/fixtures.json" \ + --repetitions "$REPETITIONS" --timeout-seconds "$TIMEOUT_SECONDS" \ + --cancel-fixture "$CANCEL_FIXTURE" --cancel-after-seconds "$CANCEL_AFTER_SECONDS" \ + --strict \ + --output "$RUNNER_TEMP/qualification/matrix-linux.json" + + - name: Upload Linux matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-linux + path: ${{ runner.temp }}/qualification/ + if-no-files-found: warn + + windows: + runs-on: windows-2022 + timeout-minutes: 180 + env: + VCPKG_OVERLAY_PORTS: ${{ github.workspace }}\loop\vcpkg\overlays\general + VCPKG_INSTALLED_DIR: ${{ github.workspace }}\vcpkg_installed + VCPKG_ROOT: ${{ github.workspace }}\vcpkg + VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg-binary-cache + QT_QPA_PLATFORM: offscreen + SENTRY_DSN: off + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + fetch-depth: 0 + + - name: Exclude workspace from Windows Defender real-time scanning + shell: pwsh + run: | + Add-MpPreference -ExclusionPath "${env:GITHUB_WORKSPACE}" -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $env:VCPKG_DEFAULT_BINARY_CACHE | Out-Null + + - name: 'VCPKG: Set up VCPKG' + shell: pwsh + run: | + $config = Get-Content (Join-Path $env:GITHUB_WORKSPACE "loop\vcpkg-configuration.json") -Raw | ConvertFrom-Json + $vcpkgCommit = $config.'default-registry'.baseline + if ($vcpkgCommit -notmatch '^[0-9a-f]{40}$') { throw "Invalid vcpkg baseline: $vcpkgCommit" } + git clone https://github.com/microsoft/vcpkg.git vcpkg + git -C vcpkg checkout --detach $vcpkgCommit + if ((git -C vcpkg rev-parse HEAD).Trim() -ne $vcpkgCommit) { throw "vcpkg checkout does not match manifest baseline" } + .\vcpkg\bootstrap-vcpkg.bat -disableMetrics + .\vcpkg\vcpkg.exe integrate install + + - name: 'VCPKG: Cache vcpkg dependencies' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: | + ./vcpkg/downloads + ./vcpkg/packages + ./vcpkg/installed + ./vcpkg/archives + key: ${{ runner.os }}-vcpkg-v2-${{ hashFiles('**/vcpkg.json', '**/vcpkg-configuration.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg-v2- + + - name: 'VCPKG: Install project dependencies' + working-directory: vcpkg + shell: pwsh + run: | + $manifestRoot = Join-Path $env:GITHUB_WORKSPACE 'loop' + .\vcpkg.exe install --triplet x64-windows --x-manifest-root=$manifestRoot --x-install-root=$env:VCPKG_INSTALLED_DIR --clean-buildtrees-after-build --clean-packages-after-build + if ($LASTEXITCODE -ne 0) { throw "vcpkg install failed with exit code $LASTEXITCODE." } + + - name: Install Qt + uses: jurplel/install-qt-action@48d3ad6db93f3627c8ee7a0454bc6f3744f7e730 # v4.3.1 + with: + version: '6.11.1' + host: 'windows' + target: 'desktop' + arch: 'win64_msvc2022_64' + dir: '${{ github.workspace }}/qt/' + install-deps: 'true' + modules: 'qtspeech qtmultimedia' + cache: 'true' + cache-key-prefix: ${{ runner.os }}-qt-6111 + # Same aqtinstall pin as reusable-windows.yml (miurahr/aqtinstall#1007). + aqtsource: git+https://github.com/miurahr/aqtinstall.git@8c3695d4a4e1ceabf6a74dc6c79681656dc6b74b + + - name: Build PdfTool + working-directory: loop + shell: pwsh + run: | + cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DCMAKE_VCPKG_BUILD_TYPE=Release -DLOOP_LOOP_DISTRIBUTION=OFF -DLOOP_INSTALL_QT_DEPENDENCIES=OFF -DCMAKE_TOOLCHAIN_FILE="${env:GITHUB_WORKSPACE}\vcpkg\scripts\buildsystems\vcpkg.cmake" -DLOOP_QT_ROOT="${env:QT_ROOT_DIR}" + if ($LASTEXITCODE -ne 0) { throw "cmake configure failed with exit code $LASTEXITCODE." } + cmake --build build --target PdfTool --config Release -j6 + if ($LASTEXITCODE -ne 0) { throw "cmake --build PdfTool failed with exit code $LASTEXITCODE." } + $pdfTool = Get-ChildItem build -Recurse -Filter PdfTool.exe | Select-Object -First 1 + if (-not $pdfTool) { throw "PdfTool.exe was not produced." } + "PDF_TOOL=$($pdfTool.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "QT_PLUGIN_PATH=$(Join-Path $env:QT_ROOT_DIR 'plugins')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "$(Join-Path $env:QT_ROOT_DIR 'bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + "$(Join-Path $env:VCPKG_INSTALLED_DIR 'x64-windows\bin')" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + + - name: Smoke PdfTool runtime + working-directory: loop + shell: pwsh + run: | + & $env:PDF_TOOL help --console-format json | Out-Null + if ($LASTEXITCODE -ne 0) { throw "PdfTool help failed with exit code $LASTEXITCODE." } + + - name: Generate synthetic fixture bundle + working-directory: loop + shell: pwsh + run: | + python scripts/resource_envelope/synthetic_workload.py --output-dir "$env:RUNNER_TEMP\fixtures" --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" + if ($LASTEXITCODE -ne 0) { throw "fixture generation failed with exit code $LASTEXITCODE." } + + - name: Run strict resource-envelope matrix + working-directory: loop + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:RUNNER_TEMP\qualification" | Out-Null + python -m scripts.resource_envelope.run_matrix ` + --pdf-tool "$env:PDF_TOOL" ` + --manifest "$env:RUNNER_TEMP\fixtures\fixtures.json" ` + --repetitions $env:REPETITIONS --timeout-seconds $env:TIMEOUT_SECONDS ` + --cancel-fixture $env:CANCEL_FIXTURE --cancel-after-seconds $env:CANCEL_AFTER_SECONDS ` + --strict ` + --output "$env:RUNNER_TEMP\qualification\matrix-windows.json" + if ($LASTEXITCODE -ne 0) { throw "strict resource-envelope matrix failed with exit code $LASTEXITCODE." } + + - name: Upload Windows matrix + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-matrix-windows + path: ${{ runner.temp }}\qualification\ + if-no-files-found: warn + + evidence: + needs: [linux, windows] + if: ${{ always() }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + path: loop + + - name: Download matrices + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: resource-envelope-matrix-* + path: ${{ runner.temp }}/matrices + + - name: Build and validate qualification evidence + working-directory: loop + run: | + set -euo pipefail + MATRICES=() + for platform in linux windows; do + matrix="$RUNNER_TEMP/matrices/resource-envelope-matrix-$platform/matrix-$platform.json" + if [ -f "$matrix" ]; then MATRICES+=(--matrix "$platform=$matrix"); fi + done + manifest="$(find "$RUNNER_TEMP/matrices" -name fixtures.json | head -n 1)" + test -n "$manifest" || { echo "::error::no fixture manifest was uploaded"; exit 1; } + python3 scripts/qualification/build_resource_envelope_evidence.py \ + "${MATRICES[@]}" \ + --fixture-manifest "$manifest" \ + --run-id "$GITHUB_RUN_ID" \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + --output "$RUNNER_TEMP/evidence/evidence.json" + python3 scripts/qualification/validate_resource_envelope_evidence.py --evidence "$RUNNER_TEMP/evidence/evidence.json" --skip-manifest + cp "$manifest" "$RUNNER_TEMP/evidence/fixture-manifest.json" + python3 -c 'import json,sys; e=json.load(open(sys.argv[1])); print("disposition:", e["disposition"]); [print(" -", r) for r in e["disposition_reasons"]]; sys.exit(0 if e["disposition"] == "passed" else 1)' "$RUNNER_TEMP/evidence/evidence.json" + + - name: Upload qualification evidence + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: resource-envelope-evidence + path: ${{ runner.temp }}/evidence/ + if-no-files-found: warn diff --git a/.github/workflows/reusable-linux.yml b/.github/workflows/reusable-linux.yml index 2fb039748..b1cc18992 100644 --- a/.github/workflows/reusable-linux.yml +++ b/.github/workflows/reusable-linux.yml @@ -78,7 +78,7 @@ jobs: - name: Verify resource-envelope contracts working-directory: loop run: | - python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python3 -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python3 scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python3 scripts/qualification/validate_resource_envelope_evidence.py python3 scripts/resource_envelope/pathological_workload.py \ diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index b0d9c1414..938f7256c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -82,7 +82,7 @@ jobs: working-directory: loop shell: pwsh run: | - python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence -v + python -m unittest scripts.resource_envelope.test_validate_envelope scripts.resource_envelope.test_pathological_workload scripts.resource_envelope.test_run_matrix scripts.resource_envelope.test_run_matrix_probes scripts.resource_envelope.test_synthetic_workload scripts.resource_envelope.test_budget_exhaustion_corpus scripts.qualification.test_validate_resource_envelope_evidence scripts.qualification.test_build_resource_envelope_evidence -v python scripts/resource_envelope/validate_envelope.py docs/generated/huge-document-envelope.json python scripts/qualification/validate_resource_envelope_evidence.py python scripts/resource_envelope/pathological_workload.py ` diff --git a/LoopLibCore/sources/pdfcolorinventory.cpp b/LoopLibCore/sources/pdfcolorinventory.cpp index 55c1c4857..9d8ec01f1 100644 --- a/LoopLibCore/sources/pdfcolorinventory.cpp +++ b/LoopLibCore/sources/pdfcolorinventory.cpp @@ -125,6 +125,17 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin for (PDFInteger pageIndex = 0; pageIndex < pageCount; ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + result.cancelled = true; + break; + } + + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { @@ -139,7 +150,12 @@ PDFColorInventoryResult PDFColorInventory::inspect(const PDFColorInventorySettin continue; } - const QSize imageSize(qMax(1, int(widthPxReal)), qMax(1, int(heightPxReal))); + // The probe holds several float bitmaps per pixel, so an oversized page is probed + // at a coarser resolution instead of allocating memory proportional to its size. + const double probeScale = settings.maxProbePixels > 0 && widthPxReal * heightPxReal > double(settings.maxProbePixels) + ? std::sqrt(double(settings.maxProbePixels) / (widthPxReal * heightPxReal)) + : 1.0; + const QSize imageSize(qMax(1, int(widthPxReal * probeScale)), qMax(1, int(heightPxReal * probeScale))); const QTransform pagePointToDevice = PDFRenderer::createPagePointToDevicePointMatrix( page, QRect(QPoint(0, 0), imageSize)); PDFTransparencyRenderer renderer(page, diff --git a/LoopLibCore/sources/pdfcolorinventory.h b/LoopLibCore/sources/pdfcolorinventory.h index a9f08c786..d18902e27 100644 --- a/LoopLibCore/sources/pdfcolorinventory.h +++ b/LoopLibCore/sources/pdfcolorinventory.h @@ -24,12 +24,16 @@ #define PDFCOLORINVENTORY_H #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include "pdftransparencyrenderer.h" #include #include +#include #include +#include + namespace pdf { @@ -59,18 +63,27 @@ struct LOOPLIBCORESHARED_EXPORT PDFColorInventoryResult QList spotColors; QList richBlackPages; PDFRenderDiagnostics diagnostics; + /// True when the inspection stopped early because the operation was cancelled; + /// the lists above then cover only the pages probed before the stop. + bool cancelled = false; }; struct LOOPLIBCORESHARED_EXPORT PDFColorInventorySettings { int probeDpi = 150; qreal richBlackKThreshold = 0.10; + /// Largest probe raster in pixels; larger pages are probed at a proportionally lower DPI. + /// A letter page at the default 150 DPI is about 1.9 million pixels. + qint64 maxProbePixels = 2'500'000; + const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages to probe; unset probes every page. + std::optional> pageIndices; }; /// Shared rich-black predicate used by preflight and Output Preview. LOOPLIBCORESHARED_EXPORT bool isRichBlackPixel(PDFConstColorBuffer buffer, - const PDFPixelFormat& format, - PDFColorComponent kThreshold); + const PDFPixelFormat& format, + PDFColorComponent kThreshold); class LOOPLIBCORESHARED_EXPORT PDFColorInventory { diff --git a/LoopLibCore/sources/pdfevidencegraph.cpp b/LoopLibCore/sources/pdfevidencegraph.cpp index 5e2879bb2..a1cfe6772 100644 --- a/LoopLibCore/sources/pdfevidencegraph.cpp +++ b/LoopLibCore/sources/pdfevidencegraph.cpp @@ -159,6 +159,7 @@ namespace { constexpr int EVIDENCE_MAX_FORM_DEPTH = 32; +constexpr const char* EVIDENCE_CANCELLED_REASON = "cancelled"; PDFArtifactIdentity artifactIdentityFromDocument(const PDFDocument* document) { @@ -1288,8 +1289,16 @@ void collectColorants(PDFDocumentSession* session, PDFEvidenceGraph* graph, cons PDFColorInventorySettings inventorySettings; inventorySettings.probeDpi = settings.colorProbeDpi; inventorySettings.richBlackKThreshold = settings.richBlackKThreshold; + inventorySettings.operationControl = settings.operationControl; + inventorySettings.pageIndices = settings.pageIndices; PDFColorInventory inventory(session); const PDFColorInventoryResult result = inventory.inspect(inventorySettings); + if (result.cancelled) + { + graph->complete = false; + graph->incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return; + } if (!result.diagnostics.isExact()) { const QString diagnostic = result.diagnostics.reasons.join(QStringLiteral("; ")).isEmpty() @@ -1398,6 +1407,17 @@ PDFEvidenceGraph PDFEvidenceCollector::collect(PDFDocumentSession* session, const PDFCatalog* catalog = document->getCatalog(); for (PDFInteger pageIndex = 0; pageIndex < catalog->getPageCount(); ++pageIndex) { + if (PDFOperationControl::isOperationCancelled(settings.operationControl)) + { + graph.complete = false; + graph.incompleteReason = QString::fromLatin1(EVIDENCE_CANCELLED_REASON); + return graph; + } + if (settings.pageIndices && !settings.pageIndices->contains(int(pageIndex))) + { + continue; + } + const PDFPage* page = catalog->getPage(pageIndex); if (!page) { diff --git a/LoopLibCore/sources/pdfevidencegraph.h b/LoopLibCore/sources/pdfevidencegraph.h index 98a0685f2..efc7df0b1 100644 --- a/LoopLibCore/sources/pdfevidencegraph.h +++ b/LoopLibCore/sources/pdfevidencegraph.h @@ -26,13 +26,17 @@ #include "pdfartifactidentity.h" #include "pdfdocumentcontext.h" #include "pdfglobal.h" +#include "pdfoperationcontrol.h" #include #include #include #include +#include #include +#include + namespace pdf { @@ -100,6 +104,12 @@ struct LOOPLIBCORESHARED_EXPORT PDFEvidenceCollectSettings qreal richBlackKThreshold = 0.10; qreal minEffectiveStrokeWidthPt = 0.0; qreal zeroWidthEpsilonPt = 1.0e-6; + /// Polled between pages; a cancelled collection returns an incomplete graph + /// with incompleteReason "cancelled". + const PDFOperationControl* operationControl = nullptr; + /// Zero-based indices of the pages whose content is walked and probed; unset + /// covers every page. Document-level evidence is collected either way. + std::optional> pageIndices; }; class LOOPLIBCORESHARED_EXPORT PDFEvidenceCollector diff --git a/LoopLibCore/sources/pdfrenderer.cpp b/LoopLibCore/sources/pdfrenderer.cpp index 45c20b493..e17b83da4 100644 --- a/LoopLibCore/sources/pdfrenderer.cpp +++ b/LoopLibCore/sources/pdfrenderer.cpp @@ -250,12 +250,10 @@ PDFRasterizer::PDFRasterizer(QObject* parent) : BaseClass(parent), m_rendererEngine(RendererEngine::Blend2D_SingleThread) { - } PDFRasterizer::~PDFRasterizer() { - } void PDFRasterizer::reset(RendererEngine rendererEngine) @@ -446,16 +444,16 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, const QSize imageSize = imageSizeGetter(page); const bool validImageSize = imageSize.width() > 0 && imageSize.height() > 0; const qint64 imageBytes = !validImageSize - ? 0 - : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 - ? static_cast(imageSize.width()) * imageSize.height() * 4 - : std::numeric_limits::max(); + ? 0 + : static_cast(imageSize.width()) <= std::numeric_limits::max() / imageSize.height() / 4 + ? static_cast(imageSize.width()) * imageSize.height() * 4 + : std::numeric_limits::max(); std::optional imageReservation; if (m_resourceBudget && imageBytes > 0 && !m_resourceBudget->tryReserve(PDFResourcePool::RasterTileCache, - imageBytes, - PDFResourcePriority::Visible, - QStringLiteral("benchmark raster image"))) + imageBytes, + PDFResourcePriority::Visible, + QStringLiteral("benchmark raster image"))) { m_resourceBudget->recordShed(PDFResourcePool::RasterTileCache); m_resourceBudgetExhausted.store(true, std::memory_order_release); @@ -476,17 +474,30 @@ void PDFRasterizerPool::render(const std::vector& pageIndices, QImage image = rasterizer->render(pageIndex, page, &precompiledPage, imageSize, m_features, &annotationManager, cms.data(), PageRotation::None); qint64 pageRenderTime = pageTimer.elapsed(); - release(rasterizer); + // A budgeted run keeps the rasterizer until the image is consumed, so the images + // alive at once never exceed the rasterizer count the raster tile pool was sized for. + if (!imageReservation) + { + release(rasterizer); + } // Now, process the image - PDFRenderedPageImage renderedPageImage; - renderedPageImage.pageIndex = pageIndex; - renderedPageImage.pageImage = qMove(image); - renderedPageImage.pageCompileTime = pageCompileTime; - renderedPageImage.pageWaitTime = pageWaitTime; - renderedPageImage.pageRenderTime = pageRenderTime; - renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); - processImage(renderedPageImage); + { + PDFRenderedPageImage renderedPageImage; + renderedPageImage.pageIndex = pageIndex; + renderedPageImage.pageImage = qMove(image); + renderedPageImage.pageCompileTime = pageCompileTime; + renderedPageImage.pageWaitTime = pageWaitTime; + renderedPageImage.pageRenderTime = pageRenderTime; + renderedPageImage.pageTotalTime = totalPageTimer.elapsed(); + processImage(renderedPageImage); + } + + if (imageReservation) + { + imageReservation.reset(); + release(rasterizer); + } if (progress) { diff --git a/LoopLibCore/sources/pdfresourcebudget.h b/LoopLibCore/sources/pdfresourcebudget.h index 93ac07334..0c351bc9c 100644 --- a/LoopLibCore/sources/pdfresourcebudget.h +++ b/LoopLibCore/sources/pdfresourcebudget.h @@ -83,7 +83,7 @@ struct LOOPLIBCORESHARED_EXPORT PDFResourceBudgetConfig /// resident ceiling is reached; active model and a visible request remain /// hard admission boundaries. std::array poolLimits = { - 256 * MiB, // active document model + 640 * MiB, // active document model 128 * MiB, // compiled/evidence cache 128 * MiB, // raster/tile cache 128 * MiB, // GPU/texture accounted proxy diff --git a/LoopLibCore/sources/pdfworkloadenvelope.cpp b/LoopLibCore/sources/pdfworkloadenvelope.cpp index c4a0d0e88..8b7148bee 100644 --- a/LoopLibCore/sources/pdfworkloadenvelope.cpp +++ b/LoopLibCore/sources/pdfworkloadenvelope.cpp @@ -172,15 +172,17 @@ qint64 PDFWorkloadEnvelope::currentRssHighWaterBytes() return -1; } - while (!status.atEnd()) + // procfs reports size 0, so QFile::atEnd() is true before the first read. + const QList lines = status.readAll().split('\n'); + for (const QByteArray& rawLine : lines) { - const QByteArray line = status.readLine().trimmed(); + const QByteArray line = rawLine.trimmed(); if (!line.startsWith("VmHWM:")) { continue; } - const QList parts = line.split(' '); + const QList parts = line.mid(6).simplified().split(' '); for (const QByteArray& part : parts) { bool ok = false; diff --git a/LoopLibCore/sources/preflightengine.cpp b/LoopLibCore/sources/preflightengine.cpp index 5ff19af4c..d0d7dcc24 100644 --- a/LoopLibCore/sources/preflightengine.cpp +++ b/LoopLibCore/sources/preflightengine.cpp @@ -6194,7 +6194,12 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const const PDFEvidenceDomains graphDomains = effectivePlan.full ? evidenceDomainsForProfile(profile) : evidenceDomainsForCheckIds(effectivePlan.checkIds); if (graphDomains != PDFEvidenceDomains()) { - m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettingsForProfile(profile)); + PDFEvidenceCollectSettings evidenceSettings = evidenceSettingsForProfile(profile); + evidenceSettings.operationControl = m_operationControl; + // Records outside the profile's page scope are dropped below, so do not + // spend the render and content walk on those pages. + evidenceSettings.pageIndices = profile.restrictions.pages; + m_activeGraph = PDFEvidenceCollector::collect(m_session, graphDomains, evidenceSettings); if (profile.restrictions.pages.has_value() || (!plan.full && !plan.pages.isEmpty())) { QList kept; @@ -6214,7 +6219,12 @@ PreflightResult PreflightEngine::run(const PreflightProfileData& profile, const if (!m_activeGraph.isComplete()) { result.inspectionComplete = false; - if (!m_activeGraph.budgetKind.isEmpty()) + if (PDFOperationControl::isOperationCancelled(m_operationControl)) + { + result.errorCode = QStringLiteral("cancelled"); + result.errorMessage = PDFTranslationContext::tr("Preflight was cancelled."); + } + else if (!m_activeGraph.budgetKind.isEmpty()) { result.errorCode = QStringLiteral("budget-exceeded"); result.errorMessage = PDFTranslationContext::tr("Evidence collection exceeded the %1 processing budget.") diff --git a/PdfTool/main.cpp b/PdfTool/main.cpp index 7850d096e..712309d80 100644 --- a/PdfTool/main.cpp +++ b/PdfTool/main.cpp @@ -348,7 +348,11 @@ int main(int argc, char* argv[]) pdftool::resetCancelRequested(); std::signal(SIGINT, handleTerminationSignal); -#ifndef Q_OS_WIN +#ifdef Q_OS_WIN + // CTRL_BREAK_EVENT is the only console interrupt deliverable to a child + // started in its own process group; the CRT raises it as SIGBREAK. + std::signal(SIGBREAK, handleTerminationSignal); +#else std::signal(SIGTERM, handleTerminationSignal); #endif diff --git a/PdfTool/pdftoolabstractapplication.cpp b/PdfTool/pdftoolabstractapplication.cpp index d5c81b2bc..b2a419ce7 100644 --- a/PdfTool/pdftoolabstractapplication.cpp +++ b/PdfTool/pdftoolabstractapplication.cpp @@ -388,6 +388,11 @@ QList PDFToolAbstractApplication::describeOptions(Optio { add(QStringLiteral("report-file"), { QStringLiteral("--report-file") }, QStringLiteral("file"), PDFToolValueType::Path); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + add(QStringLiteral("profile"), { QStringLiteral("--profile") }, QStringLiteral("profile"), PDFToolValueType::Path); + add(QStringLiteral("preflight-page-last"), { QStringLiteral("--preflight-page-last") }, QStringLiteral("page"), PDFToolValueType::Integer, {}, QStringLiteral("0")); + } if (optionFlags.testFlag(CapabilityDiscovery)) { add(QStringLiteral("command"), { QStringLiteral("--command") }, QStringLiteral("id"), PDFToolValueType::String); @@ -900,6 +905,12 @@ void PDFToolAbstractApplication::initializeCommandLineParser(QCommandLineParser* addDescribedOption(parser, optionDescriptors, QStringLiteral("report-file"), QStringLiteral("Write the preflight report JSON this run is certified over to this file.")); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + addDescribedOption(parser, optionDescriptors, QStringLiteral("profile"), QStringLiteral("Run a preflight phase with this profile before rendering and record its memory high-water.")); + addDescribedOption(parser, optionDescriptors, QStringLiteral("preflight-page-last"), QStringLiteral("Limit the preflight phase to pages 1 through this page (0 covers the whole document); rendering still covers every selected page.")); + } + if (optionFlags.testFlag(CapabilityDiscovery)) { addDescribedOption(parser, optionDescriptors, QStringLiteral("command"), QStringLiteral("Limit discovery to one stable command ID.")); @@ -1483,6 +1494,17 @@ PDFToolOptions PDFToolAbstractApplication::getOptions(QCommandLineParser* parser options.preflightReportPath = parser->value("report-file"); } + if (optionFlags.testFlag(BenchmarkPreflightProfile)) + { + options.preflightProfilePath = parser->value("profile"); + bool preflightPageLastOk = false; + const int preflightPageLast = parser->value("preflight-page-last").toInt(&preflightPageLastOk); + if (preflightPageLastOk && preflightPageLast > 0) + { + options.preflightPageLast = preflightPageLast; + } + } + if (optionFlags.testFlag(VerifyPreflightCertificate)) { options.preflightCertificatePath = positionalArguments.value(0); diff --git a/PdfTool/pdftoolabstractapplication.h b/PdfTool/pdftoolabstractapplication.h index fabe7644b..0926748b4 100644 --- a/PdfTool/pdftoolabstractapplication.h +++ b/PdfTool/pdftoolabstractapplication.h @@ -256,6 +256,9 @@ struct PDFToolOptions // figures" is a legitimate answer - so the fail-closed reading is opt-in. bool failIfEmpty = false; + // For option 'BenchmarkPreflightProfile': last page of the preflight phase, 0 for all pages. + int preflightPageLast = 0; + // For option 'PreflightProfile' QString preflightProfilePath; QString preflightJobContextPath; @@ -431,6 +434,7 @@ class PDFToolAbstractApplication EvidenceBundleExport = 0x80000000000ULL, ///< Export a portable proof-of-preflight bundle EvidenceBundleVerify = 0x100000000000ULL, ///< Verify a portable proof-of-preflight bundle PreflightReportFile = 0x200000000000ULL, ///< Preflight --report-file output path + BenchmarkPreflightProfile = 0x400000000000ULL, ///< Benchmark --profile for a measured preflight phase }; Q_DECLARE_FLAGS(Options, Option) diff --git a/PdfTool/pdftoolrender.cpp b/PdfTool/pdftoolrender.cpp index a1910fc98..99a6b8a07 100644 --- a/PdfTool/pdftoolrender.cpp +++ b/PdfTool/pdftoolrender.cpp @@ -25,20 +25,40 @@ #include "pdfdocumentsession.h" #include "pdffont.h" #include "pdfconstants.h" +#include "pdfoperationcontrol.h" #include "pdfsafefilewriter.h" #include "pdfworkloadenvelope.h" +#include "preflightclirun.h" +#include "preflightengine.h" +#include "preflightprofileresolver.h" #include #include #include #include +#include + namespace pdftool { static PDFToolRender s_toolRenderApplication; static PDFToolBenchmark s_toolBenchmarkApplication; +namespace +{ + +class CliCancellationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override + { + return isCancelRequested(); + } +}; + +} // namespace + QString PDFToolRender::getStandardString(PDFToolAbstractApplication::StandardString standardString) const { switch (standardString) @@ -160,7 +180,54 @@ QString PDFToolBenchmark::getStandardString(PDFToolAbstractApplication::Standard PDFToolAbstractApplication::Options PDFToolBenchmark::getOptionsFlags() const { - return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags; + return ConsoleFormat | OpenDocument | PageSelector | ImageExportSettingsResolution | ColorManagementSystem | RenderFlags | BenchmarkPreflightProfile; +} + +PDFToolExitCode PDFToolBenchmark::execute(const PDFToolOptions& options) +{ + m_preflightPhase = PreflightPhase(); + if (options.preflightProfilePath.isEmpty()) + { + return PDFToolRenderBase::execute(options); + } + + QJsonObject profileJson; + QString profileError; + if (!pdf::PreflightEngine::loadProfile(options.preflightProfilePath, profileJson, profileError)) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, QStringLiteral("cli.invalid-arguments"), profileError); + return PDFToolExitCode::InvalidInvocation; + } + const pdf::PreflightProfileImportResult imported = pdf::importPreflightProfile(profileJson, options.preflightProfilePath); + if (!imported.ok) + { + reportDiagnostic(options, PDFToolDiagnosticSeverity::Error, imported.errorCode, imported.errorMessage); + return PDFToolExitCode::InvalidInvocation; + } + + CliCancellationControl cancellationControl; + pdf::PreflightFileInspectionRequest request; + request.documentPath = options.document; + request.password = options.password; + request.permissiveReading = options.permissiveReading; + request.profile = imported.profile; + request.plan.full = true; + request.plan.reason = QStringLiteral("benchmark-preflight-phase"); + request.firstPage = options.pageSelectorFirstPage; + request.lastPage = options.preflightPageLast > 0 ? QString::number(options.preflightPageLast) : options.pageSelectorLastPage; + request.selectedPages = options.pageSelectorSelection; + request.cancellation = &cancellationControl; + + // The outcome owns the full source bytes; it is released here, before the + // render phase opens the document again. + { + const pdf::PreflightFileInspectionOutcome outcome = pdf::inspectPreflightFile(request); + m_preflightPhase.requested = true; + m_preflightPhase.inspected = outcome.documentReadOk && outcome.inspectionRan && !isCancelRequested(); + m_preflightPhase.highWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + } + + return PDFToolRenderBase::execute(options); } void PDFToolBenchmark::finish(const PDFToolOptions& options) @@ -207,26 +274,52 @@ void PDFToolBenchmark::finish(const PDFToolOptions& options) pdf::PDFWorkloadEnvelope envelope; envelope.identity = identity; envelope.family = QStringLiteral("benchmark-render"); - const bool cancelled = isCancelRequested(); - envelope.status = cancelled - ? QStringLiteral("cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("budget-exceeded") - : QStringLiteral("incomplete"); envelope.pageCount = static_cast(m_pageInfo.size()); - envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); - envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); - envelope.elapsedMs = m_wallTime; - envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; - envelope.incompleteReason = cancelled - ? QStringLiteral("operation-cancelled") - : m_resourceBudgetExhausted ? QStringLiteral("resource-budget-exceeded") - : QStringLiteral("preflight-measurement-unavailable"); qint64 pagesMaterialized = 0; for (const PageInfo& page : m_pageInfo) { pagesMaterialized += page.isRendered ? 1 : 0; } envelope.pagesMaterialized = pagesMaterialized; + + // A record is complete only when every phase it claims was measured: + // an unmeasured preflight or an unrendered page keeps it incomplete. + const bool cancelled = isCancelRequested(); + if (cancelled) + { + envelope.status = QStringLiteral("cancelled"); + envelope.incompleteReason = QStringLiteral("operation-cancelled"); + } + else if (m_resourceBudgetExhausted) + { + envelope.status = QStringLiteral("budget-exceeded"); + envelope.incompleteReason = QStringLiteral("resource-budget-exceeded"); + } + else if (!m_preflightPhase.requested) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-measurement-unavailable"); + } + else if (!m_preflightPhase.inspected) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("preflight-phase-failed"); + } + else if (pagesMaterialized != envelope.pageCount) + { + envelope.status = QStringLiteral("incomplete"); + envelope.incompleteReason = QStringLiteral("pages-not-materialized"); + } + else + { + envelope.status = QStringLiteral("complete"); + } + + envelope.rssHighWaterBytes = pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(); + envelope.processCommitHighWaterBytes = pdf::PDFWorkloadEnvelope::currentProcessCommitHighWaterBytes(); + envelope.preflightHighWaterBytes = m_preflightPhase.highWaterBytes; + envelope.elapsedMs = m_wallTime; + envelope.cancellationLatencyMs = cancelled ? cancellationLatencyMs() : -1; envelope.recordResources(*m_resourceBudget); data.insert(QStringLiteral("workload_envelope"), envelope.toJson()); options.executionContext->setData(data); @@ -373,10 +466,22 @@ PDFToolExitCode PDFToolRenderBase::execute(const PDFToolOptions& options) QElapsedTimer timer; timer.start(); - rasterizerPool.render(pageIndices, imageSizeGetter, std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1), nullptr); + // Render in slices and check for cancellation between them, so an interrupt + // stops the run within one slice instead of after the whole page range. + const auto processImage = std::bind(&PDFToolRenderBase::onPageRendered, this, options, std::placeholders::_1); + const std::size_t sliceSize = std::size_t(pdf::PDFRasterizerPool::getCorrectedRasterizerCount(options.renderRasterizerCount)) * 4; + bool resourceBudgetExhausted = false; + for (std::size_t first = 0; first < pageIndices.size() && !isCancelRequested(); first += sliceSize) + { + const std::size_t last = std::min(pageIndices.size(), first + sliceSize); + const std::vector slice(pageIndices.cbegin() + first, pageIndices.cbegin() + last); + rasterizerPool.render(slice, imageSizeGetter, processImage, nullptr); + // render() resets the pool's flag per call, so exhaustion is accumulated here. + resourceBudgetExhausted = resourceBudgetExhausted || rasterizerPool.resourceBudgetExhausted(); + } m_wallTime = timer.elapsed(); - m_resourceBudgetExhausted = rasterizerPool.resourceBudgetExhausted(); + m_resourceBudgetExhausted = resourceBudgetExhausted; fontCache.setCacheShrinkEnabled(nullptr, true); diff --git a/PdfTool/pdftoolrender.h b/PdfTool/pdftoolrender.h index b57f76397..1068f5fcc 100644 --- a/PdfTool/pdftoolrender.h +++ b/PdfTool/pdftoolrender.h @@ -82,12 +82,25 @@ class PDFToolBenchmark : public PDFToolRenderBase public: virtual QString getStandardString(StandardString standardString) const override; virtual Options getOptionsFlags() const override; + virtual PDFToolExitCode execute(const PDFToolOptions& options) override; protected: virtual void finish(const PDFToolOptions& options) override; virtual void onPageRendered(const PDFToolOptions& options, pdf::PDFRenderedPageImage& renderedPageImage) override; + +private: + /// Preflight runs before rendering, so the process high-water recorded when + /// it ends is the peak of open plus preflight, independent of rendering. + struct PreflightPhase + { + bool requested = false; + bool inspected = false; + qint64 highWaterBytes = -1; + }; + + PreflightPhase m_preflightPhase; }; } // namespace pdftool -#endif // PDFTOOLRENDER_H +#endif // PDFTOOLRENDER_H diff --git a/UnitTests/tst_evidencegraphtest.cpp b/UnitTests/tst_evidencegraphtest.cpp index 337bc5b7c..c667a37cb 100644 --- a/UnitTests/tst_evidencegraphtest.cpp +++ b/UnitTests/tst_evidencegraphtest.cpp @@ -44,6 +44,9 @@ class EvidenceGraphTest : public QObject private slots: void collectWithoutDocument_isIncomplete(); void emptyPage_isComplete(); + void cancelledCollection_isIncompleteAndCancelled(); + void cancelledPreflight_reportsCancelled(); + void pageScope_skipsUnselectedPages(); void incompleteGraphCannotPass(); void imageFamilyDualRunMatchesEngine(); void colorantsFamilyDualRunMatchesEngine(); @@ -125,6 +128,12 @@ void assertFindingCitesGraphRecord(const QList& findings, QFAIL(qPrintable(QStringLiteral("Expected finding type '%1' for check '%2'").arg(findingType, checkId))); } +class CancelledOperationControl final : public pdf::PDFOperationControl +{ +public: + bool isOperationCancelled() const override { return true; } +}; + } // namespace void EvidenceGraphTest::collectWithoutDocument_isIncomplete() @@ -146,6 +155,62 @@ void EvidenceGraphTest::emptyPage_isComplete() QVERIFY(graph.incompleteReason.isEmpty()); } +void EvidenceGraphTest::cancelledCollection_isIncompleteAndCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + pdf::PDFEvidenceCollectSettings settings; + settings.operationControl = &cancelled; + + const pdf::PDFEvidenceGraph graph = pdf::PDFEvidenceCollector::collect(&session, pdf::pdfEvidenceAllDomains(), settings); + QVERIFY(!graph.isComplete()); + QCOMPARE(graph.incompleteReason, QStringLiteral("cancelled")); + QVERIFY(graph.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); +} + +void EvidenceGraphTest::cancelledPreflight_reportsCancelled() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + const CancelledOperationControl cancelled; + + pdf::PreflightEngine engine(&session); + engine.setOperationControl(&cancelled); + const QJsonObject profile{ + { QStringLiteral("name"), QStringLiteral("Color inventory") }, + { QStringLiteral("checks"), QJsonArray{ + QJsonObject{ + { QStringLiteral("id"), QStringLiteral("color-inventory") }, + { QStringLiteral("severity"), QStringLiteral("info") }, + { QStringLiteral("probe_dpi"), 150 }, + { QStringLiteral("rich_black_k_percent"), 10 } } } } + }; + const pdf::PreflightResult result = engine.run(profile); + QVERIFY(!result.inspectionComplete); + QCOMPARE(result.errorCode, QStringLiteral("cancelled")); +} + +void EvidenceGraphTest::pageScope_skipsUnselectedPages() +{ + pdf::PDFDocument document = loadFixtureDocument("rich-black.pdf"); + pdf::PDFDocumentSession session(&document); + + const pdf::PDFEvidenceGraph all = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants); + QVERIFY(!all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + pdf::PDFEvidenceCollectSettings settings; + settings.pageIndices = QSet(); + const pdf::PDFEvidenceGraph none = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QVERIFY(none.isComplete()); + QVERIFY(none.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).isEmpty()); + + settings.pageIndices = QSet{ 0 }; + const pdf::PDFEvidenceGraph first = pdf::PDFEvidenceCollector::collect(&session, pdf::PDFEvidenceDomain::Colorants, settings); + QCOMPARE(first.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size(), + all.recordsForTarget(pdf::PDFEvidenceDomain::Colorants, QStringLiteral("rich-black")).size()); +} + void EvidenceGraphTest::incompleteGraphCannotPass() { pdf::PreflightEngine engine(nullptr); diff --git a/UnitTests/tst_pdftoolcontract.cpp b/UnitTests/tst_pdftoolcontract.cpp index 74d5b6e41..08bf6224f 100644 --- a/UnitTests/tst_pdftoolcontract.cpp +++ b/UnitTests/tst_pdftoolcontract.cpp @@ -134,8 +134,49 @@ private slots: void rgbToCmykRefusesToWriteOverItsOwnInput(); void evidenceBundleExportVerifyPair(); void evidenceBundleRejectsNonJsonOutput(); + void benchmarkWithoutPreflightProfileIsIncomplete(); + void benchmarkWithPreflightProfileIsComplete(); }; +namespace +{ + +QJsonObject runBenchmarkEnvelope(const QStringList& extraArguments) +{ + const QString fixture = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/testdata/fixtures/image-dpi-low.pdf"); + QStringList arguments{ QStringLiteral("benchmark"), fixture, + QStringLiteral("--render-hw-accel"), QStringLiteral("0"), + QStringLiteral("--console-format"), QStringLiteral("json") }; + arguments << extraArguments; + const ToolRun run = runPdfTool(arguments); + verifyEnvelope(run, 0, QStringLiteral("benchmark")); + return run.json.value(QStringLiteral("data")).toObject().value(QStringLiteral("workload_envelope")).toObject(); +} + +} // namespace + +void PdfToolContractTest::benchmarkWithoutPreflightProfileIsIncomplete() +{ + const QJsonObject envelope = runBenchmarkEnvelope({}); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("incomplete")); + QCOMPARE(envelope.value(QStringLiteral("incomplete_reason")).toString(), QStringLiteral("preflight-measurement-unavailable")); + QCOMPARE(envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(), -1); +} + +void PdfToolContractTest::benchmarkWithPreflightProfileIsComplete() +{ + const QString profile = QStringLiteral(LOOP_PREFLIGHT_SOURCE_DIR "/profiles/loop-default.json"); + const QJsonObject envelope = runBenchmarkEnvelope({ QStringLiteral("--profile"), profile }); + QVERIFY(!envelope.isEmpty()); + QCOMPARE(envelope.value(QStringLiteral("status")).toString(), QStringLiteral("complete")); + QVERIFY(envelope.value(QStringLiteral("incomplete_reason")).toString().isEmpty()); + const qint64 preflightHighWater = envelope.value(QStringLiteral("preflight_high_water_bytes")).toInteger(); + QVERIFY2(preflightHighWater > 0, qPrintable(QString::number(preflightHighWater))); + QVERIFY(envelope.value(QStringLiteral("rss_high_water_bytes")).toInteger() >= preflightHighWater); + QCOMPARE(envelope.value(QStringLiteral("pages_materialized")).toInteger(), envelope.value(QStringLiteral("page_count")).toInteger()); +} + void PdfToolContractTest::helpIsWrapped() { const ToolRun run = runPdfTool({ QStringLiteral("help"), QStringLiteral("--console-format"), QStringLiteral("json") }); diff --git a/UnitTests/tst_resourcebudgettest.cpp b/UnitTests/tst_resourcebudgettest.cpp index de22587bd..42f9178e1 100644 --- a/UnitTests/tst_resourcebudgettest.cpp +++ b/UnitTests/tst_resourcebudgettest.cpp @@ -25,7 +25,7 @@ void ResourceBudgetTest::conservativeDefaultsExposeAllPools() { const pdf::PDFResourceBudgetConfig config = pdf::PDFResourceBudgetConfig::conservativeDefaults(); QCOMPARE(config.residentLimitBytes, 768 * pdf::PDFResourceBudgetConfig::MiB); - QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 256 * pdf::PDFResourceBudgetConfig::MiB); + QCOMPARE(config.limit(pdf::PDFResourcePool::ActiveDocumentModel), 640 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::CompiledEvidenceCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::RasterTileCache), 128 * pdf::PDFResourceBudgetConfig::MiB); QCOMPARE(config.limit(pdf::PDFResourcePool::GpuTextureCache), 128 * pdf::PDFResourceBudgetConfig::MiB); diff --git a/UnitTests/tst_workloadenvelopetest.cpp b/UnitTests/tst_workloadenvelopetest.cpp index 721a0649f..f73e115c0 100644 --- a/UnitTests/tst_workloadenvelopetest.cpp +++ b/UnitTests/tst_workloadenvelopetest.cpp @@ -38,11 +38,21 @@ class WorkloadEnvelopeTest : public QObject private slots: void identityFieldsArePresent(); + void rssHighWaterIsMeasuredOnSupportedPlatforms(); void shedPrefetchAndQualityBeforeInteraction(); void pageHeavyEnvelopeRecordsIdentity(); void interactionSlotRunsWhenBackgroundIsSaturated(); }; +void WorkloadEnvelopeTest::rssHighWaterIsMeasuredOnSupportedPlatforms() +{ +#if defined(Q_OS_LINUX) || defined(Q_OS_WIN) + QVERIFY(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes() > 0); +#else + QCOMPARE(pdf::PDFWorkloadEnvelope::currentRssHighWaterBytes(), qint64(-1)); +#endif +} + void WorkloadEnvelopeTest::identityFieldsArePresent() { qputenv("GIT_COMMIT", QByteArrayLiteral("0123456789abcdef0123456789abcdef01234567")); diff --git a/changes/cc-issue-19-resource-envelopes.evidence.yaml b/changes/cc-issue-19-resource-envelopes.evidence.yaml new file mode 100644 index 000000000..927e5a335 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.evidence.yaml @@ -0,0 +1,29 @@ +format_version: 1 +kind: evidence +claims: + - id: benchmark-preflight-phase-and-cancellation + evidence: + - unit:agent-policy:pdftool + - unit:agent-policy:pagemaster + - packaging:linux-build + - packaging:windows-build + - id: hosted-qualification-workflow + evidence: + - packaging:linux-build + - packaging:windows-build + - security:codeql + - id: agent-policy-bindings + evidence: + - unit:scripts/agent/test_architecture_contracts.py + - architecture:scripts/agent/check-architecture.py + - architecture:scripts/agent/generate-adapters.py + - id: linux-rss-high-water-reads-procfs + evidence: + - unit:agent-policy:core + - architecture:docs/generated/architecture-catalog.json + - security:scripts/ci/check_source_integrity.py + - differential:UnitTestsConversionOracle +unresolved: + - core:scripts/ci/check_independent_validation_gate.py + - Native PdfTool and UnitTestsPdfToolContract were not built locally (no configured build directory for this worktree); the linux-build and windows-build CI lanes compile and run them. + - Hosted qualification measurements come from the resource-envelope-qualification workflow run on this PR and are recorded under docs/evidence/issue-19-resource-envelope/ after that run. diff --git a/changes/cc-issue-19-resource-envelopes.md b/changes/cc-issue-19-resource-envelopes.md new file mode 100644 index 000000000..3fc93b1f7 --- /dev/null +++ b/changes/cc-issue-19-resource-envelopes.md @@ -0,0 +1,4 @@ +Category: added +Audience: developers and release qualifiers +Breaking-Change: no +Summary: Qualify hostile and production resource envelopes on hosted Linux and Windows runners (#19). `PdfTool benchmark --profile` adds a measured preflight phase, so a clean run reports a complete envelope; rendering stops within one page slice of an interrupt, and Windows honours CTRL_BREAK. A new resource-envelope qualification workflow generates a deterministic synthetic fixture bundle, runs the strict matrix with separate cancellation and reopen-after-cancel recovery probes and a hostile budget-exhaustion lane, and builds schema-2 evidence carrying the CI run id. Crashes, timeouts, and skipped workloads can never count as a passing envelope. diff --git a/docs/RESOURCE_BUDGETS.md b/docs/RESOURCE_BUDGETS.md index 6c1541a1e..1911acb60 100644 --- a/docs/RESOURCE_BUDGETS.md +++ b/docs/RESOURCE_BUDGETS.md @@ -22,7 +22,7 @@ the following conservative defaults: | Pool | Limit | Admission rule | |------|-------|----------------| -| active document model | 256 MiB | interaction-priority hard boundary | +| active document model | 640 MiB | interaction-priority hard boundary | | compiled/evidence cache | 128 MiB | insertion-order eviction, then reject | | raster/tile cache | 128 MiB | prefetch shed, then visible admission reject | | GPU texture cache | 128 MiB | source-image byte proxy; physical GPU bytes are unavailable (`-1`) | diff --git a/docs/RESOURCE_ENVELOPE.md b/docs/RESOURCE_ENVELOPE.md index 3d5362e16..a525afc56 100644 --- a/docs/RESOURCE_ENVELOPE.md +++ b/docs/RESOURCE_ENVELOPE.md @@ -60,10 +60,15 @@ $env:QT_QPA_PLATFORM = "offscreen" $env:QT_PLUGIN_PATH = "C:\path\to\qt\plugins" PdfTool.exe benchmark C:\temp\loop-div2k-10000-pages.pdf ` --render-hw-accel 0 ` - --render-rasterizers 8 ` + --render-rasterizers 3 ` --console-format json ``` +Rasterizers are pinned to 3 because the benchmark renders at the default 300 DPI: a Letter page +image is 33.7 MB, each rasterizer holds one at a time, and the 128 MiB `raster-tile-cache` pool +admits three. A fourth concurrent page is rejected as budget-exceeded and the run exits with +`PartialOutput`. + The JSON result includes the `workload_envelope` object. A successful Windows software-renderer run on the local 0.2.0 candidate rendered all 10,000 pages in 48,513 ms and recorded a peak RSS of 380,985,344 bytes. @@ -84,9 +89,15 @@ unsupported, budget-exceeded, or incomplete. The envelope is schema version 2. `resources` is produced by the shared `PDFResourceBudget` authority and contains the resident ceiling plus all seven named pool records. `pages_materialized` reports pages actually processed by a -runner; it is not the catalog page count. `preflight_high_water_bytes` remains -`-1` until a run includes the preflight phase, and such a record is explicitly -`incomplete` rather than being promoted to a passing result. The deterministic +runner; it is not the catalog page count. `preflight_high_water_bytes` is the process high-water when the +`benchmark --profile ` preflight phase ends; without `--profile` +it stays `-1` and the record is explicitly `incomplete` rather than being +promoted to a passing result. `--preflight-page-last ` limits that phase to +pages 1 through `n` while rendering still covers every selected page; the +runner passes it (256) for fixtures above 1,000 pages, because preflight costs +roughly 0.5-0.7 s per page on a hosted runner, and records it as +`profile.preflight_page_last`. A sampled record's `preflight_high_water_bytes` +covers the sampled pages, not the whole document. The deterministic pathological and transparency/spot fixtures can be generated without the external DIV2K corpus: @@ -154,7 +165,7 @@ recommended cold-process timing/RSS sample: python scripts/resource_envelope/run_matrix.py ` --pdf-tool C:\path\to\PdfTool.exe ` --manifest C:\temp\resource-envelope-fixtures.json ` - --repetitions 3 --rasterizers 8 --strict ` + --repetitions 3 --rasterizers 3 --strict ` --output C:\temp\resource-envelope-matrix.json ``` @@ -166,6 +177,12 @@ reported as a passing complete run. Add `--baseline C:\previous\resource-envelope-matrix.json` to compare matching fixture digests and platform/toolchain identities. The default regression margin is `2.0`; use a narrower margin only after collecting stable platform -baselines. Add `--cancel-fixture pathological-vector ---cancel-after-seconds 1` to send an interrupt to one controlled probe and -record the application's cancellation latency. +baselines. The runner passes `--profile` (default +`loop-preflight/profiles/loop-default.json`) so every run measures the +preflight phase. Add `--cancel-fixture ten-thousand-page +--cancel-after-seconds 3` for the separate cancellation probe, which interrupts +one extra run and then times a fresh process reopening the fixture and +rendering its first page (`recovery_ms`). `--strict` requires that probe and +also runs the hostile lane over `UnitTests/testdata/budget_exhaustion/`. For +the hosted, synthetic-fixture version of this run, see +`docs/RESOURCE_ENVELOPE_QUALIFICATION.md`. diff --git a/docs/RESOURCE_ENVELOPE_BUDGETS.json b/docs/RESOURCE_ENVELOPE_BUDGETS.json index fed9176c2..6d11f8a5a 100644 --- a/docs/RESOURCE_ENVELOPE_BUDGETS.json +++ b/docs/RESOURCE_ENVELOPE_BUDGETS.json @@ -7,7 +7,7 @@ "resource_budget": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -26,6 +26,19 @@ "cancellation_latency_ms": 5000, "recovery_ms": 30000 }, + "synthetic-image-heavy": { + "page_count": 10000, + "wall_time_ms": 600000, + "rss_high_water_bytes": 805306368, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, + "large-document-500mb": { + "wall_time_ms": 120000, + "rss_high_water_bytes": 2147483648, + "cancellation_latency_ms": 5000, + "recovery_ms": 30000 + }, "pathological-vector": { "page_count": 256, "wall_time_ms": 120000, diff --git a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md index f29358e59..cadc4644a 100644 --- a/docs/RESOURCE_ENVELOPE_QUALIFICATION.md +++ b/docs/RESOURCE_ENVELOPE_QUALIFICATION.md @@ -14,23 +14,68 @@ PdfTool benchmark output and integrated document-session output are separate evidence records. A Quick first-view record remains `incomplete` until the Quick product path is implemented in Phase 4. -## Qualification sequence +## Hosted qualification (issue #19) + +`.github/workflows/resource-envelope-qualification.yml` is the qualifying run. +It triggers on `workflow_dispatch` and on pull requests that touch the +benchmark, the envelope scripts, or the budget contract. Each Linux and Windows +job: + +1. Builds `PdfTool` from the candidate SHA. +2. Generates the fixture bundle with + `scripts/resource_envelope/synthetic_workload.py`. The office, 500 MB + image-heavy, and 10,000-page fixtures are deterministic synthetic PDFs + (SHAKE-256 noise images stored with FlateDecode), so hosted runners need + no external corpus. The 10,000-page fixture uses the + `synthetic-image-heavy` workload caps, which equal the DIV2K caps except + `wall_time_ms` (600 s): three rasterizers on a hosted runner render the + 10,000 pages in about 350 s (Linux) to 510 s (Windows). +3. Runs `run_matrix.py --strict --repetitions 3` with: + - a measured preflight phase (`benchmark --profile`, default + `loop-preflight/profiles/loop-default.json`), so a clean run reports + `status: complete` with a real `preflight_high_water_bytes`. Fixtures + above 1,000 pages (the 10,000-page one) preflight their first 256 pages + only (`--preflight-page-last 256`, recorded as + `profile.preflight_page_last`); rendering still covers every page; + - a `large-document-500mb` workload for the 500 MB fixture, whose process + RSS cap is 2 GiB instead of the 768 MiB resident limit: the reader holds + the whole file plus its object model in memory, so the peak is about 3.3 + times the file size (1.7 GB measured on both platforms), while the + accounted pools stay under the resident limit; + - a cancellation probe on `ten-thousand-page`, which interrupts a render-only + run (no preflight phase, whose document-wide setup does not poll for + cancellation) + and requires a `cancelled` envelope within the workload's + `cancellation_latency_ms`; + - a recovery probe, which times a fresh process reopening the same + fixture and rendering its first page (`recovery_ms`, within the + workload's `recovery_ms`); + - a hostile lane over `UnitTests/testdata/budget_exhaustion/`, where each + PDF must end in a contained PdfTool exit code (rejection is fine) within + the hostile timeout, without breaching the resident ceiling. + +The `evidence` job combines both matrices with +`scripts/qualification/build_resource_envelope_evidence.py` into a schema +version 2 record that carries the run id and URL, and validates it with +`validate_resource_envelope_evidence.py`. The disposition is `passed` only when +both platforms passed strictly on the same candidate SHA. + +A crash (an exit code outside PdfTool's defined codes, or `InternalError`), a +timeout, a non-success exit, or a missing envelope never produces a +`measured` fixture. Crashes and timeouts fail the record outright. + +## External DIV2K sequence + +The original DIV2K qualification remains available for local runs: 1. Validate the external DIV2K corpus and generate one canonical manifest with `--hash-all`. 2. Build the deterministic 10,000-page image-heavy PDF and record its digest. 3. Create an external fixture manifest using the schema at `docs/schemas/resource-envelope-fixtures.schema.json`, then run - `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the 2 MB office, - image-heavy, 10,000-page, pathological-vector, and transparency/spot - fixtures. Supply the multi-GB fixture when platform addressability permits. - The strict job is expected to remain non-passing until the native benchmark - also supplies preflight and recovery measurements; unavailable fields must - not be promoted to zero. -4. Run PdfTool benchmark profiles on Linux and Windows with the same manifest. -5. Run the integrated session/scheduler harness with the same workload identity. -6. Replay the bounded lifecycle trace corpus on both platforms. -7. Attach JSON results, digests, platform identities, and dispositions to the - candidate-SHA evidence dossier. + `scripts/resource_envelope/run_matrix.py --manifest ... --strict` with the + same probe and hostile options as the hosted workflow. +4. Run the integrated session/scheduler harness with the same workload identity. +5. Replay the bounded lifecycle trace corpus on both platforms. No unavailable measurement may be converted to zero or treated as a pass. diff --git a/docs/generated/huge-document-envelope.json b/docs/generated/huge-document-envelope.json index 9b9fd6b90..0a78d2d2b 100644 --- a/docs/generated/huge-document-envelope.json +++ b/docs/generated/huge-document-envelope.json @@ -32,7 +32,7 @@ "config": { "resident_limit_bytes": 805306368, "pool_limits_bytes": { - "active-document-model": 268435456, + "active-document-model": 671088640, "compiled-evidence-cache": 134217728, "raster-tile-cache": 134217728, "gpu-texture-cache": 134217728, @@ -45,7 +45,7 @@ "resident_high_water_bytes": 0, "pressure": "normal", "pools": { - "active-document-model": { "limit_bytes": 268435456, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, + "active-document-model": { "limit_bytes": 671088640, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "compiled-evidence-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "raster-tile-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, "gpu-texture-cache": { "limit_bytes": 134217728, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0 }, diff --git a/scripts/qualification/build_resource_envelope_evidence.py b/scripts/qualification/build_resource_envelope_evidence.py new file mode 100644 index 000000000..2ee3477a1 --- /dev/null +++ b/scripts/qualification/build_resource_envelope_evidence.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Build hosted resource-envelope qualification evidence from matrix results. + +Each ``--matrix PLATFORM=PATH`` is one strict ``run_matrix.py`` output from the +hosted qualification workflow. The evidence records the exact CI run, candidate +SHA, fixture manifest digest, and per-fixture measurements for every platform. +It claims ``passed`` only when every required platform passed strict +qualification on the same candidate SHA. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.resource_envelope.run_matrix import matrix_passes + + +EVIDENCE_KIND = "loop-resource-envelope-qualification-evidence" +REQUIRED_PLATFORMS = ("linux", "windows") +MEASUREMENT_FIELDS = ("status", "rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms", "pages_materialized", "page_count") + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _platform_record(matrix: Mapping[str, Any], matrix_sha256: str) -> dict[str, Any]: + fixtures: dict[str, Any] = {} + for record in matrix["fixtures"]: + result = record.get("result") if isinstance(record.get("result"), Mapping) else {} + fixtures[record["fixture_id"]] = { + "status": record["status"], + "required": record.get("required", False), + "fixture_sha256": record.get("fixture_sha256"), + "workload": record.get("workload"), + "envelope": {field: result.get(field) for field in MEASUREMENT_FIELDS} if result else None, + "statistics": record.get("statistics"), + "validation_errors": record.get("validation_errors", []), + } + probe = matrix.get("cancellation_recovery_probe") or {} + hostile = matrix.get("hostile") or {} + identity = next((record["identity"] for record in matrix["fixtures"] if isinstance(record.get("identity"), Mapping) and record["identity"]), {}) + return { + "matrix_sha256": matrix_sha256, + "candidate_sha": matrix["candidate_sha"], + "strict_passed": matrix_passes(matrix, strict=True), + "summary": matrix["summary"], + "runtime": {key: identity.get(key) for key in ("os", "qt", "compiler", "cpu", "renderer", "build")}, + "fixtures": fixtures, + "cancellation_recovery_probe": { + "status": probe.get("status", "not-run"), + "fixture_id": probe.get("fixture_id"), + "cancellation_latency_ms": probe.get("cancellation", {}).get("cancellation_latency_ms", -1), + "recovery_ms": probe.get("recovery", {}).get("recovery_ms", -1), + "validation_errors": probe.get("validation_errors", []), + }, + "hostile": { + "summary": hostile.get("summary", {"total": 0, "contained": 0}), + "failed_cases": [case["case_id"] for case in hostile.get("cases", []) if case.get("status") != "contained"], + }, + } + + +def build_evidence( + matrices: Mapping[str, Path], + fixture_manifest: Path, + run_id: str, + run_url: str, +) -> dict[str, Any]: + platforms: dict[str, Any] = {} + for platform, path in sorted(matrices.items()): + matrix = json.loads(path.read_text(encoding="utf-8")) + platforms[platform] = _platform_record(matrix, _sha256(path)) + + candidates = {record["candidate_sha"] for record in platforms.values()} + reasons: list[str] = [] + missing = [platform for platform in REQUIRED_PLATFORMS if platform not in platforms] + reasons.extend(f"platform {platform} produced no matrix" for platform in missing) + if len(candidates) > 1: + reasons.append(f"platforms measured different candidate SHAs: {sorted(candidates)}") + for platform, record in platforms.items(): + if not record["strict_passed"]: + reasons.append(f"platform {platform} did not pass strict qualification") + + rejected = any( + record["summary"]["failed"] or record["cancellation_recovery_probe"]["status"] == "failed" or record["hostile"]["failed_cases"] + for record in platforms.values() + ) + disposition = "passed" if not reasons else ("rejected" if rejected else "incomplete") + manifest = json.loads(fixture_manifest.read_text(encoding="utf-8")) + return { + "schema_kind": EVIDENCE_KIND, + "schema_version": 2, + "issue": 19, + "candidate_sha": next(iter(candidates)) if len(candidates) == 1 else "", + "disposition": disposition, + "disposition_reasons": reasons, + "fixture_manifest_sha256": _sha256(fixture_manifest), + "fixture_generator": manifest.get("generator"), + "ci_runs": [{"platform": platform, "run_id": run_id, "run_url": run_url, "candidate_sha": record["candidate_sha"]} for platform, record in platforms.items()], + "platforms": platforms, + } + + +def _matrix_args(values: Sequence[str]) -> dict[str, Path]: + result: dict[str, Path] = {} + for value in values: + platform, separator, path = value.partition("=") + if not separator or not platform or not path: + raise ValueError("--matrix must be PLATFORM=PATH") + result[platform] = Path(path) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--matrix", action="append", default=[], metavar="PLATFORM=PATH") + parser.add_argument("--fixture-manifest", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + evidence = build_evidence(_matrix_args(args.matrix), args.fixture_manifest, args.run_id, args.run_url) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(f"resource-envelope evidence error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({"disposition": evidence["disposition"], "reasons": evidence["disposition_reasons"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qualification/test_build_resource_envelope_evidence.py b/scripts/qualification/test_build_resource_envelope_evidence.py new file mode 100644 index 000000000..859728bcb --- /dev/null +++ b/scripts/qualification/test_build_resource_envelope_evidence.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import copy +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.qualification.build_resource_envelope_evidence import build_evidence +from scripts.qualification.validate_resource_envelope_evidence import validate_evidence +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + +CANDIDATE = "a" * 40 + + +def _matrix(failed_fixture: str | None = None) -> dict: + fixtures = [] + for fixture_id, spec in FIXTURE_SPECS.items(): + if not spec["required"]: + fixtures.append({"fixture_id": fixture_id, "status": "unavailable", "required": False}) + continue + status = "failed" if fixture_id == failed_fixture else "measured" + fixtures.append({ + "fixture_id": fixture_id, + "status": status, + "required": True, + "fixture_sha256": "b" * 64, + "identity": {"os": "test-os", "qt": "6.11.1"}, + "result": {"status": "complete", "rss_high_water_bytes": 100, "preflight_high_water_bytes": 90, "elapsed_ms": 10, "pages_materialized": 1, "page_count": 1}, + }) + failed = int(failed_fixture is not None) + return { + "candidate_sha": CANDIDATE, + "fixtures": fixtures, + "cancellation_recovery_probe": {"status": "measured", "fixture_id": "ten-thousand-page", "cancellation": {"cancellation_latency_ms": 40}, "recovery": {"recovery_ms": 900}}, + "hostile": {"summary": {"total": 7, "contained": 7}, "cases": []}, + "summary": {"total": len(fixtures), "measured": len(fixtures) - 1 - failed, "flagged": 0, "skipped": 1, "failed": failed, "candidate_sha_verified": True}, + } + + +class BuildResourceEnvelopeEvidenceTest(unittest.TestCase): + def _build(self, matrices: dict[str, dict]) -> dict: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = {} + for platform, matrix in matrices.items(): + paths[platform] = root / f"{platform}.json" + paths[platform].write_text(json.dumps(matrix), encoding="utf-8") + manifest = root / "fixtures.json" + manifest.write_text(json.dumps({"generator": {"version": 1}}), encoding="utf-8") + return build_evidence(paths, manifest, "123456", "https://github.com/studio-berry/loop/actions/runs/123456") + + def test_both_platforms_passing_is_valid_passed_evidence(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + self.assertEqual(evidence["disposition"], "passed", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_missing_platform_is_incomplete(self) -> None: + evidence = self._build({"linux": _matrix()}) + self.assertEqual(evidence["disposition"], "incomplete") + self.assertIn("platform windows produced no matrix", evidence["disposition_reasons"]) + self.assertEqual(validate_evidence(evidence), []) + + def test_failed_fixture_is_rejected(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix(failed_fixture="image-heavy-500mb")}) + self.assertEqual(evidence["disposition"], "rejected") + self.assertEqual(validate_evidence(evidence), []) + + def test_passed_claim_with_unavailable_measurement_is_invalid(self) -> None: + evidence = self._build({"linux": _matrix(), "windows": _matrix()}) + tampered = copy.deepcopy(evidence) + tampered["platforms"]["linux"]["fixtures"]["office-2mb"]["envelope"]["preflight_high_water_bytes"] = -1 + tampered["platforms"]["windows"]["cancellation_recovery_probe"]["recovery_ms"] = -1 + errors = "\n".join(validate_evidence(tampered)) + self.assertIn("platforms.linux.fixtures.office-2mb.preflight_high_water_bytes is unavailable", errors) + self.assertIn("platforms.windows.cancellation_recovery_probe.recovery_ms is unavailable", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/qualification/validate_resource_envelope_evidence.py b/scripts/qualification/validate_resource_envelope_evidence.py index 9f1a9478c..b4922d09c 100644 --- a/scripts/qualification/validate_resource_envelope_evidence.py +++ b/scripts/qualification/validate_resource_envelope_evidence.py @@ -1,5 +1,12 @@ #!/usr/bin/env python3 -"""Validate frozen Session 11 resource-envelope qualification evidence.""" +"""Validate resource-envelope qualification evidence. + +Schema version 1 is the frozen Session 11 record and can never claim +``passed``. Schema version 2 is built from the hosted qualification workflow +(``build_resource_envelope_evidence.py``) and may claim ``passed`` only when +every required platform measured every required fixture, the cancellation +and recovery probe, and the hostile corpus on one candidate SHA. +""" from __future__ import annotations @@ -24,6 +31,7 @@ REQUIRED_FIXTURES = tuple( fixture_id for fixture_id, spec in FIXTURE_SPECS.items() if spec["required"] ) +REQUIRED_PLATFORMS = ("linux", "windows") def validate_evidence( @@ -31,11 +39,100 @@ def validate_evidence( *, manifest: dict[str, Any] | None = None, ) -> list[str]: - errors: list[str] = [] if evidence.get("schema_kind") != "loop-resource-envelope-qualification-evidence": - errors.append("schema_kind must be loop-resource-envelope-qualification-evidence") + return ["schema_kind must be loop-resource-envelope-qualification-evidence"] + if evidence.get("schema_version") == 2: + return _validate_hosted_evidence(evidence) if evidence.get("schema_version") != 1: - errors.append("schema_version must be 1") + return ["schema_version must be 1 or 2"] + return _validate_session_11_evidence(evidence, manifest) + + +def _is_measured(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def _validate_platform(platform: str, record: Any, candidate_sha: Any) -> list[str]: + if not isinstance(record, dict): + return [f"platforms.{platform} must be an object"] + errors: list[str] = [] + if record.get("candidate_sha") != candidate_sha: + errors.append(f"platforms.{platform}.candidate_sha must equal candidate_sha") + if record.get("strict_passed") is not True: + errors.append(f"platforms.{platform} did not pass strict qualification") + if not isinstance(record.get("matrix_sha256"), str) or not SHA256_RE.fullmatch(record["matrix_sha256"]): + errors.append(f"platforms.{platform}.matrix_sha256 must be a SHA-256 digest") + fixtures = record.get("fixtures") if isinstance(record.get("fixtures"), dict) else {} + for fixture_id in REQUIRED_FIXTURES: + entry = fixtures.get(fixture_id) + envelope = entry.get("envelope") if isinstance(entry, dict) else None + if not isinstance(entry, dict) or entry.get("status") != "measured" or not isinstance(envelope, dict): + errors.append(f"platforms.{platform}.fixtures.{fixture_id} is not measured") + continue + if envelope.get("status") != "complete": + errors.append(f"platforms.{platform}.fixtures.{fixture_id} envelope is not complete") + for field in ("rss_high_water_bytes", "preflight_high_water_bytes", "elapsed_ms"): + if not _is_measured(envelope.get(field)): + errors.append(f"platforms.{platform}.fixtures.{fixture_id}.{field} is unavailable") + probe = record.get("cancellation_recovery_probe") if isinstance(record.get("cancellation_recovery_probe"), dict) else {} + if probe.get("status") != "measured": + errors.append(f"platforms.{platform} cancellation/recovery probe is not measured") + for field in ("cancellation_latency_ms", "recovery_ms"): + if not _is_measured(probe.get(field)): + errors.append(f"platforms.{platform}.cancellation_recovery_probe.{field} is unavailable") + hostile = record.get("hostile") if isinstance(record.get("hostile"), dict) else {} + summary = hostile.get("summary") if isinstance(hostile.get("summary"), dict) else {} + if not _is_measured(summary.get("total")) or summary.get("total") == 0 or summary.get("contained") != summary.get("total"): + errors.append(f"platforms.{platform} hostile corpus was not fully contained") + return errors + + +def _validate_hosted_evidence(evidence: dict[str, Any]) -> list[str]: + errors: list[str] = [] + candidate_sha = evidence.get("candidate_sha") + if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): + errors.append("candidate_sha must be a 40-character lowercase commit SHA") + if not isinstance(evidence.get("fixture_manifest_sha256"), str) or not SHA256_RE.fullmatch(evidence["fixture_manifest_sha256"]): + errors.append("fixture_manifest_sha256 must be a SHA-256 digest") + disposition = evidence.get("disposition") + if disposition not in {"incomplete", "passed", "rejected"}: + errors.append("disposition must be incomplete, passed, or rejected") + reasons = evidence.get("disposition_reasons") + if not isinstance(reasons, list): + errors.append("disposition_reasons must be an array") + elif disposition == "passed" and reasons: + errors.append("passed evidence cannot carry disposition_reasons") + elif disposition != "passed" and not reasons: + errors.append("non-passed evidence must state its disposition_reasons") + + runs = evidence.get("ci_runs") + if not isinstance(runs, list) or not runs: + errors.append("ci_runs must be a non-empty array") + else: + for index, run in enumerate(runs): + if not isinstance(run, dict) or not str(run.get("run_id", "")).isdigit(): + errors.append(f"ci_runs[{index}].run_id must be a GitHub Actions run id") + continue + if not str(run.get("run_url", "")).startswith("https://github.com/"): + errors.append(f"ci_runs[{index}].run_url must be a GitHub Actions run URL") + if run.get("candidate_sha") != candidate_sha: + errors.append(f"ci_runs[{index}].candidate_sha must equal candidate_sha") + + platforms = evidence.get("platforms") + if not isinstance(platforms, dict) or not platforms: + errors.append("platforms must be a non-empty object") + return errors + if disposition == "passed": + for platform in REQUIRED_PLATFORMS: + if platform not in platforms: + errors.append(f"passed evidence is missing platform {platform}") + for platform, record in platforms.items(): + errors.extend(_validate_platform(platform, record, candidate_sha)) + return errors + + +def _validate_session_11_evidence(evidence: dict[str, Any], manifest: dict[str, Any] | None) -> list[str]: + errors: list[str] = [] candidate_sha = evidence.get("candidate_sha") if not isinstance(candidate_sha, str) or not SHA_RE.fullmatch(candidate_sha): diff --git a/scripts/resource_envelope/run_matrix.py b/scripts/resource_envelope/run_matrix.py index ae5eeab31..5d732e683 100644 --- a/scripts/resource_envelope/run_matrix.py +++ b/scripts/resource_envelope/run_matrix.py @@ -3,7 +3,13 @@ Large PDFs stay outside the repository. A qualification run should use a manifest with exact fixture digests and sizes; the legacy ``--fixture`` form is -kept for exploratory runs and is intentionally not sufficient for ``--strict``. +intentionally not sufficient for ``--strict``. + +Besides the measured fixtures, a strict run carries a cancellation probe that +interrupts one fixture, a recovery probe that times a fresh process reopening +it, and a hostile lane that feeds the checked-in budget-exhaustion PDFs to +PdfTool. A crash, timeout, or skipped workload never counts as a passing +envelope. """ from __future__ import annotations @@ -26,20 +32,44 @@ ROOT = Path(__file__).resolve().parents[2] DEFAULT_BUDGETS = ROOT / "docs" / "RESOURCE_ENVELOPE_BUDGETS.json" +DEFAULT_PREFLIGHT_PROFILE = ROOT / "loop-preflight" / "profiles" / "loop-default.json" +DEFAULT_HOSTILE_CORPUS = ROOT / "UnitTests" / "testdata" / "budget_exhaustion" MATRIX_KIND = "loop-resource-envelope-matrix" -DEFAULT_RASTERIZERS = 8 +# Each rasterizer holds one page image at a time. A 300 DPI Letter page is +# 33.7 MB, so three fit the 128 MiB raster-tile-cache pool and a fourth is +# rejected as budget-exceeded, leaving pages unrendered (exit code 5). +DEFAULT_RASTERIZERS = 3 +# Preflight renders and walks every page it covers (about 0.5-0.7 s per page on +# a hosted runner), so a full pass over a very large document outlives any +# practical timeout. Documents above the threshold get a preflight phase over +# their first pages only; rendering still covers every page. +PREFLIGHT_SAMPLE_THRESHOLD_PAGES = 1000 +PREFLIGHT_SAMPLE_PAGES = 256 +TIMEOUT_REASON = "benchmark-timeout" +# PdfTool's defined terminal exit codes (pdftoolresult.h) except InternalError +# (7). Anything else, including a negative POSIX signal or a Windows exception +# status, means the process did not reach a controlled disposition. +CONTAINED_EXIT_CODES = frozenset({0, 1, 2, 3, 4, 5, 6, 8, 9}) +EXIT_SUCCESS = 0 +EXIT_CANCELLED = 6 +# Validation errors carrying one of these markers fail the record outright; +# every other error only flags it. +HARD_ERROR_MARKERS = ("does not match", "exceeds", "identity", "fixture SHA", "manifest", "timeout", "crashed") # These names mirror issue #242. multi-gb is optional because platform # addressability and available disk are environment-dependent. FIXTURE_SPECS: dict[str, dict[str, Any]] = { "office-2mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 1_500_000, "max_bytes": 2_500_000}, - "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": None, "min_bytes": 450_000_000, "max_bytes": 550_000_000}, + "image-heavy-500mb": {"required": True, "expected_page_count": None, "workload": "large-document-500mb", "min_bytes": 450_000_000, "max_bytes": 550_000_000}, "multi-gb": {"required": False, "expected_page_count": None, "workload": None, "min_bytes": 1_000_000_000, "max_bytes": None}, "ten-thousand-page": {"required": True, "expected_page_count": 10000, "workload": "div2k-image-heavy", "min_bytes": None, "max_bytes": None}, "pathological-vector": {"required": True, "expected_page_count": 256, "workload": "pathological-vector", "min_bytes": None, "max_bytes": None}, "transparency-spots": {"required": True, "expected_page_count": 256, "workload": None, "min_bytes": None, "max_bytes": None}, } +Runner = Callable[..., subprocess.CompletedProcess[str]] +CancelRunner = Callable[[list[str], float, float | None], subprocess.CompletedProcess[str]] + def _sha256(path: Path) -> str: digest = hashlib.sha256() @@ -73,6 +103,40 @@ def _envelope_from_output(payload: Mapping[str, Any]) -> dict[str, Any] | None: return None +def _envelope_from_process(completed: subprocess.CompletedProcess[str]) -> dict[str, Any] | None: + payload = _extract_json(completed.stdout or "") + return _envelope_from_output(payload) if payload else None + + +def _find_key(value: Any, key: str) -> Any: + if isinstance(value, Mapping): + if key in value: + return value[key] + children: Sequence[Any] = list(value.values()) + elif isinstance(value, list): + children = value + else: + return None + for child in children: + found = _find_key(child, key) + if found is not None: + return found + return None + + +def _failure_detail(completed: subprocess.CompletedProcess[str]) -> str: + """Short, log-sized account of why a PdfTool run did not succeed.""" + payload = _extract_json(completed.stdout or "") + errors = _find_key(payload, "rendering-errors") if payload else None + parts = [] + if errors is not None: + parts.append("rendering-errors=" + json.dumps(errors)[:800]) + stderr = (completed.stderr or "").strip() + if stderr: + parts.append("stderr=" + stderr[-400:]) + return "; ".join(parts) + + def _git_head() -> str: try: return subprocess.run( @@ -97,6 +161,48 @@ def _candidate_identity() -> dict[str, Any]: } +def _benchmark_command( + pdf_tool: Path, + fixture_path: Path, + rasterizers: int, + preflight_profile: Path | None, + first_page_only: bool = False, + preflight_page_last: int | None = None, +) -> list[str]: + # Pin rasterizers to a fixed value so the same code and fixtures produce + # comparable RSS and elapsed time across hosts with different CPU counts. + # The value is recorded in the result profile. + command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + if preflight_profile is not None: + command += ["--profile", str(preflight_profile)] + if preflight_page_last is not None: + command += ["--preflight-page-last", str(preflight_page_last)] + if first_page_only: + command += ["--page-first", "1", "--page-last", "1"] + return command + + +def _preflight_page_last(page_count: int | None) -> int | None: + """Last page of the preflight phase, or None when it covers the whole document.""" + if page_count is not None and page_count > PREFLIGHT_SAMPLE_THRESHOLD_PAGES: + return PREFLIGHT_SAMPLE_PAGES + return None + + +def _identity_errors(envelope: Mapping[str, Any], candidate_sha: str, fixture_sha256: str) -> list[str]: + identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} + errors: list[str] = [] + if identity.get("commit") != candidate_sha: + errors.append(f"identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") + if identity.get("fixture_digest") != fixture_sha256: + errors.append(f"identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {fixture_sha256!r}") + return errors + + +def _is_hard(error: str) -> bool: + return any(marker in error for marker in HARD_ERROR_MARKERS) + + def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_after_seconds: float | None) -> subprocess.CompletedProcess[str]: creationflags = 0 popen_kwargs: dict[str, Any] = {} @@ -127,9 +233,25 @@ def _run_benchmark_process(command: list[str], timeout_seconds: float, cancel_af except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired(command, timeout_seconds, stdout, stderr) return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) +def _timed_run(runner: Runner, command: list[str], timeout_seconds: float) -> tuple[subprocess.CompletedProcess[str] | None, str, int]: + """Runs one PdfTool process; returns (completed, failure reason, wall ms).""" + started = time.monotonic() + try: + completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + return None, TIMEOUT_REASON, int((time.monotonic() - started) * 1000) + except OSError as exc: + return None, f"benchmark-launch-failed:{exc}", -1 + wall_ms = int((time.monotonic() - started) * 1000) + if completed.returncode not in CONTAINED_EXIT_CODES: + return completed, f"process-crashed:{completed.returncode}", wall_ms + return completed, "", wall_ms + + def _empty_result(fixture_id: str, reason: str) -> dict[str, Any]: return { "fixture_id": fixture_id, @@ -208,6 +330,25 @@ def _fixture_metadata(fixture_id: str, fixture_path: Path, metadata: Mapping[str return details, errors +def _fixture_workload(fixture_id: str, metadata: Mapping[str, Any] | None) -> str | None: + if metadata is not None and "workload" in metadata: + return str(metadata["workload"]) + return FIXTURE_SPECS[fixture_id]["workload"] + + +def _rss_limit(budgets: Mapping[str, Any], workload: str | None) -> int | None: + """Process RSS ceiling: the workload's own cap when it declares one, else the resident limit. + + The reader holds a whole document in memory, so a very large document's + process RSS is a multiple of its file size and cannot fit the resident + limit that governs the accounted pools. + """ + workload_limit = budgets.get("workloads", {}).get(workload, {}).get("rss_high_water_bytes") if workload else None + if isinstance(workload_limit, int): + return workload_limit + return budgets.get("resource_budget", {}).get("resident_limit_bytes") + + def _aggregate_envelopes(envelopes: list[Mapping[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: # Use the highest-RSS run as the safety representative and the median # elapsed time. This keeps peak-memory validation conservative while @@ -236,13 +377,13 @@ def run_fixture( timeout_seconds: float, baseline: Mapping[str, Any] | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, metadata: Mapping[str, Any] | None = None, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, require_provenance: bool = False, - cancel_after_seconds: float | None = None, candidate_sha: str | None = None, + preflight_profile: Path | None = None, ) -> dict[str, Any]: if repetitions < 1 or rasterizers < 1: raise ValueError("repetitions and rasterizers must be positive") @@ -254,17 +395,17 @@ def run_fixture( pdf_tool = Path(pdf_tool).resolve() fixture_path = Path(fixture_path).resolve() spec = FIXTURE_SPECS[fixture_id] + workload = _fixture_workload(fixture_id, metadata) fixture_details, provenance_errors = _fixture_metadata(fixture_id, fixture_path, metadata, require_provenance) - # Pin rasterizers to a fixed value (8) so the same code and fixtures - # produce comparable RSS and elapsed time across hosts with different - # CPU counts. The value is recorded in the result profile. - command = [str(pdf_tool), "benchmark", str(fixture_path), "--render-hw-accel", "0", "--render-rasterizers", str(rasterizers), "--console-format", "json"] + expected_page_count = metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"] + preflight_page_last = _preflight_page_last(expected_page_count) + command = _benchmark_command(pdf_tool, fixture_path, rasterizers, preflight_profile, preflight_page_last=preflight_page_last) record: dict[str, Any] = { "fixture_id": fixture_id, - "path": str(fixture_path.resolve()), - "expected_page_count": metadata.get("page_count", spec["expected_page_count"]) if metadata else spec["expected_page_count"], - "workload": spec["workload"], - "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers}, + "path": str(fixture_path), + "expected_page_count": expected_page_count, + "workload": workload, + "profile": {"render_hw_accel": False, "render_rasterizers": rasterizers, "preflight_profile": str(preflight_profile) if preflight_profile else None, "preflight_page_last": preflight_page_last if preflight_profile else None}, "command": command, **fixture_details, } @@ -274,78 +415,216 @@ def run_fixture( runs: list[dict[str, Any]] = [] envelopes: list[Mapping[str, Any]] = [] - for index in range(repetitions): - try: - if runner is subprocess.run and cancel_after_seconds is not None: - completed = _run_benchmark_process(command, timeout_seconds, cancel_after_seconds) - else: - completed = runner(command, cwd=ROOT, check=False, capture_output=True, text=True, timeout=timeout_seconds) - except subprocess.TimeoutExpired: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-timeout", "process_exit_code": None}) - continue - except OSError as exc: - runs.append({"run": index + 1, "status": "unavailable", "reason": f"benchmark-launch-failed:{exc}", "process_exit_code": None}) - continue - payload = _extract_json(completed.stdout) - envelope = _envelope_from_output(payload) if payload else None - if envelope is None: - runs.append({"run": index + 1, "status": "unavailable", "reason": "benchmark-envelope-missing", "process_exit_code": completed.returncode, "stderr": completed.stderr[-2000:]}) + validation_errors: list[str] = [] + for index in range(1, repetitions + 1): + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + envelope = _envelope_from_process(completed) if completed is not None else None + exit_code = completed.returncode if completed is not None else None + if failure or envelope is None: + reason = failure or "benchmark-envelope-missing" + run: dict[str, Any] = {"run": index, "status": "unavailable", "reason": reason, "process_exit_code": exit_code, "process_wall_ms": wall_ms} + if completed is not None: + run["stderr"] = (completed.stderr or "")[-2000:] + run["detail"] = _failure_detail(completed) + runs.append(run) + validation_errors.append(f"run {index}: {reason}") + if failure == TIMEOUT_REASON: + break continue envelopes.append(envelope) - runs.append({"run": index + 1, "status": "recorded", "process_exit_code": completed.returncode, "result": envelope}) - - if not envelopes: - record.update({"status": "unavailable", "result": None, "runs": runs, "validation_errors": [], "regressions": []}) - return record - - representative, stats = _aggregate_envelopes(envelopes) - validation_errors: list[str] = [] - for index, envelope in enumerate(envelopes, start=1): - for error in validate_envelope(envelope, budgets, spec["workload"]): + runs.append({"run": index, "status": "recorded", "process_exit_code": exit_code, "process_wall_ms": wall_ms, "result": envelope}) + if exit_code != EXIT_SUCCESS: + runs[-1]["detail"] = _failure_detail(completed) + validation_errors.append(f"run {index}: process exit code {exit_code} is not success") + for error in validate_envelope(envelope, budgets, workload): validation_errors.append(f"run {index}: {error}") expected_page_count = record["expected_page_count"] if expected_page_count is not None and envelope.get("page_count") != expected_page_count: validation_errors.append(f"run {index}: page_count {envelope.get('page_count')} does not match expected {expected_page_count}") rss = envelope.get("rss_high_water_bytes") - resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") - if isinstance(rss, int) and rss >= 0 and isinstance(resident_limit, int) and rss > resident_limit: - validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {resident_limit}") - identity = envelope.get("identity") if isinstance(envelope.get("identity"), Mapping) else {} - if identity.get("commit") != candidate_sha: - validation_errors.append(f"run {index}: identity.commit {identity.get('commit')!r} does not match candidate {candidate_sha!r}") - if identity.get("fixture_digest") != record.get("fixture_sha256"): - validation_errors.append(f"run {index}: identity.fixture_digest {identity.get('fixture_digest')!r} does not match input {record.get('fixture_sha256')!r}") + rss_limit = _rss_limit(budgets, workload) + if isinstance(rss, int) and rss >= 0 and isinstance(rss_limit, int) and rss > rss_limit: + validation_errors.append(f"run {index}: RSS {rss} exceeds resident policy {rss_limit}") + validation_errors.extend(f"run {index}: {error}" for error in _identity_errors(envelope, candidate_sha, record["fixture_sha256"])) + + record["runs"] = runs + record["validation_errors"] = sorted(set(validation_errors)) + record["regressions"] = [] + if not envelopes: + record["result"] = None + record["status"] = "failed" if any(_is_hard(error) for error in record["validation_errors"]) else "unavailable" + return record + + representative, stats = _aggregate_envelopes(envelopes) record["identity"] = representative.get("identity", {}) record["result"] = representative record["statistics"] = stats - record["runs"] = runs - unavailable_runs = [run for run in runs if run["status"] != "recorded"] - validation_errors.extend( - f"run {run['run']}: {run['reason']}" for run in unavailable_runs - ) - record["validation_errors"] = sorted(set(validation_errors)) comparison = dict(representative) comparison["fixture_sha256"] = record["fixture_sha256"] comparison["identity"] = record["identity"] record["regressions"] = _regressions(comparison, baseline, margin) - if cancel_after_seconds is not None: - if representative.get("status") != "cancelled": - validation_errors.append("cancellation probe did not produce a cancelled envelope") - if not isinstance(representative.get("cancellation_latency_ms"), int) or representative["cancellation_latency_ms"] < 0: - validation_errors.append("cancellation probe did not report cancellation latency") - record["cancellation_probe"] = {"requested_after_seconds": cancel_after_seconds} - record["validation_errors"] = sorted(set(validation_errors)) - hard_error_markers = ("does not match", "exceeds", "identity", "fixture SHA", "manifest") - hard_errors = [error for error in record["validation_errors"] if any(marker in error for marker in hard_error_markers)] - if record["regressions"] or hard_errors: + if record["regressions"] or any(_is_hard(error) for error in record["validation_errors"]): record["status"] = "failed" - elif record["validation_errors"] or representative.get("status") != "complete": + elif record["validation_errors"] or any(envelope.get("status") != "complete" for envelope in envelopes): record["status"] = "flagged" else: record["status"] = "measured" return record +def run_cancellation_probe( + pdf_tool: Path, + fixture_id: str, + fixture_path: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + cancel_after_seconds: float, + candidate_sha: str, + workload: str | None = None, + rasterizers: int = DEFAULT_RASTERIZERS, + cancel_runner: CancelRunner = _run_benchmark_process, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Interrupts one render run, then times a fresh process reopening the fixture. + + The interrupted run has no preflight phase: preflight setup (fonts, ink + mapper, resource scan) is document-wide and does not poll for cancellation, + so on a very large document it would dominate the latency this probe + measures. Preflight cancellation is covered by the evidence-graph tests. + + ``recovery_ms`` is the wall time from launching that fresh process until it + exits having rendered the first page: the time an operator waits to get the + document back after abandoning a run. + """ + pdf_tool = Path(pdf_tool).resolve() + fixture_path = Path(fixture_path).resolve() + fixture_sha256 = _sha256(fixture_path) + limits = budgets.get("workloads", {}).get(workload, {}) if workload else {} + errors: list[str] = [] + + cancel_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None) + cancellation: dict[str, Any] = {"command": cancel_command, "requested_after_seconds": cancel_after_seconds, "cancellation_latency_ms": -1} + try: + completed = cancel_runner(cancel_command, timeout_seconds, cancel_after_seconds) + except subprocess.TimeoutExpired: + completed = None + errors.append("cancellation probe timeout: the process did not stop after the interrupt") + except OSError as exc: + completed = None + errors.append(f"cancellation probe launch failed: {exc}") + if completed is not None: + cancellation["process_exit_code"] = completed.returncode + envelope = _envelope_from_process(completed) + if completed.returncode not in CONTAINED_EXIT_CODES: + errors.append(f"cancellation probe crashed with exit code {completed.returncode}") + elif envelope is None: + errors.append("cancellation probe produced no envelope") + else: + cancellation["result"] = envelope + latency = envelope.get("cancellation_latency_ms") + if envelope.get("status") != "cancelled": + errors.append(f"cancellation probe ended {envelope.get('status')!r}; the interrupt arrived after the run finished or was ignored") + if completed.returncode != EXIT_CANCELLED: + errors.append(f"cancellation probe exit code {completed.returncode} is not Cancelled ({EXIT_CANCELLED})") + if not isinstance(latency, int) or isinstance(latency, bool) or latency < 0: + errors.append("cancellation probe did not report cancellation latency") + else: + cancellation["cancellation_latency_ms"] = latency + limit = limits.get("cancellation_latency_ms") + if isinstance(limit, int) and latency > limit: + errors.append(f"cancellation_latency_ms {latency} exceeds workload policy {limit}") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + + recovery_command = _benchmark_command(pdf_tool, fixture_path, rasterizers, None, first_page_only=True) + recovery: dict[str, Any] = {"command": recovery_command, "recovery_ms": -1} + completed, failure, wall_ms = _timed_run(runner, recovery_command, timeout_seconds) + if completed is not None: + recovery["process_exit_code"] = completed.returncode + if failure: + errors.append(f"recovery probe {failure}") + elif completed is not None: + envelope = _envelope_from_process(completed) + if envelope is None: + errors.append("recovery probe produced no envelope") + else: + recovery["result"] = envelope + if completed.returncode != EXIT_SUCCESS: + errors.append(f"recovery probe exit code {completed.returncode} is not success") + if envelope.get("pages_materialized") != 1: + errors.append(f"recovery probe materialized {envelope.get('pages_materialized')!r} pages, expected 1") + errors.extend(_identity_errors(envelope, candidate_sha, fixture_sha256)) + recovery["recovery_ms"] = wall_ms + limit = limits.get("recovery_ms") + if isinstance(limit, int) and wall_ms > limit: + errors.append(f"recovery_ms {wall_ms} exceeds workload policy {limit}") + + return { + "fixture_id": fixture_id, + "workload": workload, + "fixture_sha256": fixture_sha256, + "status": "failed" if errors else "measured", + "cancellation": cancellation, + "recovery": recovery, + "validation_errors": errors, + } + + +def run_hostile_corpus( + pdf_tool: Path, + corpus_dir: Path, + budgets: Mapping[str, Any], + timeout_seconds: float, + rasterizers: int = DEFAULT_RASTERIZERS, + preflight_profile: Path | None = None, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Requires every hostile PDF to end in a contained PdfTool disposition. + + Rejecting the input (InputError, ProcessingFailure, budget-exceeded) is a + correct outcome here; crashing, hanging, or breaching the resident ceiling + is not. + """ + pdf_tool = Path(pdf_tool).resolve() + corpus_dir = Path(corpus_dir).resolve() + manifest = json.loads((corpus_dir / "manifest.json").read_text(encoding="utf-8")) + cases = manifest.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError(f"hostile corpus manifest has no cases: {corpus_dir}") + resident_limit = budgets.get("resource_budget", {}).get("resident_limit_bytes") + records: list[dict[str, Any]] = [] + for case in cases: + path = corpus_dir / str(case["pdf"]) + record: dict[str, Any] = {"case_id": case["id"], "path": str(path), "expected": case.get("expected")} + errors: list[str] = [] + if not path.is_file() or _sha256(path) != case.get("sha256"): + errors.append("hostile fixture SHA-256 does not match manifest") + else: + command = _benchmark_command(pdf_tool, path, rasterizers, preflight_profile) + record["command"] = command + completed, failure, wall_ms = _timed_run(runner, command, timeout_seconds) + record["process_wall_ms"] = wall_ms + if completed is not None: + record["process_exit_code"] = completed.returncode + if failure: + errors.append(failure) + elif completed is not None: + envelope = _envelope_from_process(completed) + record["disposition"] = envelope.get("status") if envelope else "rejected" + if envelope is not None: + record["result"] = envelope + rss = envelope.get("rss_high_water_bytes") + if isinstance(rss, int) and isinstance(resident_limit, int) and rss > resident_limit: + errors.append(f"RSS {rss} exceeds resident policy {resident_limit}") + record["validation_errors"] = errors + record["status"] = "failed" if errors else "contained" + records.append(record) + return { + "corpus": str(corpus_dir), + "cases": records, + "summary": {"total": len(records), "contained": sum(record["status"] == "contained" for record in records)}, + } + + def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("schema_kind") != "loop-resource-envelope-fixtures" or payload.get("schema_version") != 1: @@ -371,6 +650,8 @@ def _load_fixture_manifest(path: Path) -> dict[str, dict[str, Any]]: raise ValueError(f"fixture manifest provenance missing: {fixture_id}") if "page_count" in record and (not isinstance(record["page_count"], int) or record["page_count"] < 1): raise ValueError(f"fixture manifest page_count is invalid: {fixture_id}") + if "workload" in record and (not isinstance(record["workload"], str) or not record["workload"]): + raise ValueError(f"fixture manifest workload is invalid: {fixture_id}") normalized = dict(record) fixture_path = Path(str(record["path"])) if not fixture_path.is_absolute(): @@ -398,17 +679,22 @@ def run_matrix( timeout_seconds: float, baseline: Mapping[str, Any] | Path | None = None, margin: float = 2.0, - runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + runner: Runner = subprocess.run, repetitions: int = 1, rasterizers: int = DEFAULT_RASTERIZERS, cancel_fixture: str | None = None, cancel_after_seconds: float | None = None, + preflight_profile: Path | None = None, + hostile_corpus: Path | None = None, + hostile_timeout_seconds: float | None = None, + cancel_runner: CancelRunner = _run_benchmark_process, ) -> dict[str, Any]: pdf_tool = Path(pdf_tool).resolve() baseline_by_fixture = _baseline_records(baseline) if isinstance(baseline, Path) else (baseline or {}) identity = _candidate_identity() candidate_sha = identity["candidate_sha"] records: list[dict[str, Any]] = [] + resolved: dict[str, tuple[Path, Mapping[str, Any] | None]] = {} for fixture_id, spec in FIXTURE_SPECS.items(): supplied = fixtures.get(fixture_id) if supplied is None: @@ -417,40 +703,42 @@ def run_matrix( records.append(record) continue metadata = dict(supplied) if isinstance(supplied, Mapping) else None - fixture_path = Path(metadata["path"]) if metadata else Path(supplied) - fixture_path = fixture_path.resolve() + fixture_path = (Path(metadata["path"]) if metadata else Path(supplied)).resolve() if not fixture_path.is_file(): record = _empty_result(fixture_id, "fixture-not-found") record["required"] = spec["required"] records.append(record) continue - record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), cancel_after_seconds if fixture_id == cancel_fixture else None, candidate_sha) + resolved[fixture_id] = (fixture_path, metadata) + record = run_fixture(pdf_tool, fixture_id, fixture_path, budgets, timeout_seconds, baseline_by_fixture.get(fixture_id), margin, runner, metadata, repetitions, rasterizers, bool(metadata), candidate_sha, preflight_profile) record["required"] = spec["required"] - result = record.get("result") - if isinstance(result, Mapping): - result_identity = result.get("identity") - if isinstance(result_identity, Mapping): - commit = result_identity.get("commit") - if commit != candidate_sha: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity commit does not match checkout HEAD"])) - record["status"] = "failed" - expected_digest = record.get("fixture_sha256") - fixture_digest = result_identity.get("fixture_digest") - if expected_digest and fixture_digest != expected_digest: - record["validation_errors"] = sorted(set(record.get("validation_errors", []) + ["PdfTool identity fixture digest does not match input SHA-256"])) - record["status"] = "failed" records.append(record) + probe: dict[str, Any] | None = None + if cancel_fixture is not None: + if cancel_fixture not in resolved: + probe = {"fixture_id": cancel_fixture, "status": "unavailable", "validation_errors": ["cancellation fixture not supplied"]} + else: + fixture_path, metadata = resolved[cancel_fixture] + probe = run_cancellation_probe(pdf_tool, cancel_fixture, fixture_path, budgets, timeout_seconds, cancel_after_seconds or 1.0, candidate_sha, + _fixture_workload(cancel_fixture, metadata), rasterizers, cancel_runner, runner) + + hostile = None + if hostile_corpus is not None: + hostile = run_hostile_corpus(pdf_tool, hostile_corpus, budgets, hostile_timeout_seconds or timeout_seconds, rasterizers, preflight_profile, runner) + failed = sum(record["status"] == "failed" for record in records) flagged = sum(record["required"] and record["status"] in {"flagged", "unavailable"} for record in records) skipped = sum(not record["required"] and record["status"] == "unavailable" for record in records) return { "schema_kind": MATRIX_KIND, - "schema_version": 2, + "schema_version": 3, "candidate_sha": identity["candidate_sha"], "candidate_identity": identity, "generated_at_utc": datetime.now(timezone.utc).isoformat(), "fixtures": records, + "cancellation_recovery_probe": probe, + "hostile": hostile, "summary": { "total": len(records), "measured": sum(record["status"] == "measured" for record in records), @@ -458,10 +746,48 @@ def run_matrix( "skipped": skipped, "failed": failed, "candidate_sha_verified": identity["verified"], + "cancellation_recovery_probe": probe["status"] if probe else "not-run", + "hostile_contained": f"{hostile['summary']['contained']}/{hostile['summary']['total']}" if hostile else "not-run", }, } +def matrix_passes(matrix: Mapping[str, Any], strict: bool) -> bool: + summary = matrix["summary"] + probe = matrix.get("cancellation_recovery_probe") + hostile = matrix.get("hostile") + hostile_failed = bool(hostile) and hostile["summary"]["contained"] != hostile["summary"]["total"] + if summary["failed"] or hostile_failed or (probe is not None and probe["status"] == "failed"): + return False + if not strict: + return True + return ( + not summary["flagged"] + and summary["candidate_sha_verified"] + and probe is not None and probe["status"] == "measured" + and hostile is not None + ) + + +def matrix_failure_reasons(matrix: Mapping[str, Any]) -> list[str]: + reasons: list[str] = [] + for record in matrix["fixtures"]: + if record["status"] in {"failed", "flagged"} or (record.get("required") and record["status"] == "unavailable"): + details = record["validation_errors"] or [record.get("reason", "no detail recorded")] + reasons.extend(f"fixture {record['fixture_id']} {record['status']}: {error}" for error in details) + detailed = next((run for run in record.get("runs", []) if run.get("detail")), None) + if detailed is not None: + reasons.append(f"fixture {record['fixture_id']} run {detailed['run']} detail: {detailed['detail']}") + probe = matrix.get("cancellation_recovery_probe") + if probe is not None and probe["status"] != "measured": + reasons.extend(f"probe {probe['status']}: {error}" for error in probe["validation_errors"]) + for case in (matrix.get("hostile") or {}).get("cases", []): + if case["status"] != "contained": + exit_code = case.get("process_exit_code") + reasons.extend(f"hostile {case['case_id']} (exit {exit_code}): {error}" for error in case["validation_errors"]) + return reasons + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pdf-tool", type=Path, required=True) @@ -475,24 +801,39 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--repetitions", type=int, default=1) parser.add_argument("--rasterizers", type=int, default=DEFAULT_RASTERIZERS) parser.add_argument("--timeout-seconds", type=float, default=120.0) + parser.add_argument("--preflight-profile", type=Path, default=DEFAULT_PREFLIGHT_PROFILE, + help="profile for the benchmark's measured preflight phase") + parser.add_argument("--no-preflight", action="store_true", help="omit the preflight phase (records stay incomplete)") parser.add_argument("--cancel-fixture", choices=tuple(FIXTURE_SPECS)) parser.add_argument("--cancel-after-seconds", type=float) - parser.add_argument("--strict", action="store_true", help="fail when required fixtures, provenance, or measurements are unavailable") + parser.add_argument("--hostile-corpus", type=Path, help=f"budget-exhaustion corpus directory (strict default: {DEFAULT_HOSTILE_CORPUS.relative_to(ROOT)})") + parser.add_argument("--hostile-timeout-seconds", type=float, default=60.0) + parser.add_argument("--strict", action="store_true", help="fail when required fixtures, probes, provenance, or measurements are unavailable") args = parser.parse_args(argv) try: - if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1: - raise ValueError("margin, timeout-seconds, repetitions, and rasterizers must be positive") + if args.margin <= 0 or args.timeout_seconds <= 0 or args.repetitions < 1 or args.rasterizers < 1 or args.hostile_timeout_seconds <= 0: + raise ValueError("margin, timeouts, repetitions, and rasterizers must be positive") if args.strict and args.manifest is None: raise ValueError("--strict requires a fixture --manifest with exact digests and sizes") + if args.strict and args.cancel_fixture is None: + raise ValueError("--strict requires --cancel-fixture for the cancellation and recovery probes") + if args.strict and args.no_preflight: + raise ValueError("--strict cannot omit the preflight phase") if (args.cancel_fixture is None) != (args.cancel_after_seconds is None): raise ValueError("--cancel-fixture and --cancel-after-seconds must be supplied together") if args.cancel_after_seconds is not None and args.cancel_after_seconds <= 0: raise ValueError("cancel-after-seconds must be positive") - if args.cancel_fixture is not None and args.repetitions != 1: - raise ValueError("cancellation probes require --repetitions 1") + preflight_profile = None if args.no_preflight else args.preflight_profile.resolve() + if preflight_profile is not None and not preflight_profile.is_file(): + raise ValueError(f"preflight profile not found: {preflight_profile}") + hostile_corpus = args.hostile_corpus or (DEFAULT_HOSTILE_CORPUS if args.strict else None) fixtures: Mapping[str, Path | Mapping[str, Any]] = _load_fixture_manifest(args.manifest) if args.manifest else _fixture_args(args.fixture) budgets = json.loads(args.budgets.read_text(encoding="utf-8")) - matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, repetitions=args.repetitions, rasterizers=args.rasterizers, cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds) + matrix = run_matrix(args.pdf_tool, fixtures, budgets, args.timeout_seconds, args.baseline, args.margin, + repetitions=args.repetitions, rasterizers=args.rasterizers, + cancel_fixture=args.cancel_fixture, cancel_after_seconds=args.cancel_after_seconds, + preflight_profile=preflight_profile, hostile_corpus=hostile_corpus, + hostile_timeout_seconds=args.hostile_timeout_seconds) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(matrix, indent=2) + "\n", encoding="utf-8") except (OSError, ValueError, json.JSONDecodeError) as exc: @@ -500,7 +841,9 @@ def main(argv: Sequence[str] | None = None) -> int: return 2 print(json.dumps(matrix["summary"], indent=2)) - return 1 if matrix["summary"]["failed"] or args.strict and (matrix["summary"]["flagged"] or not matrix["summary"]["candidate_sha_verified"]) else 0 + for reason in matrix_failure_reasons(matrix): + print(reason, file=sys.stderr) + return 0 if matrix_passes(matrix, args.strict) else 1 if __name__ == "__main__": diff --git a/scripts/resource_envelope/synthetic_workload.py b/scripts/resource_envelope/synthetic_workload.py new file mode 100644 index 000000000..0c0eb23ef --- /dev/null +++ b/scripts/resource_envelope/synthetic_workload.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Build the deterministic synthetic resource-envelope fixture bundle. + +Hosted qualification cannot reach the external DIV2K corpus or a private +fixture bundle, so the office, image-heavy, and 10,000-page fixtures are +generated here. Every pixel derives from SHAKE-256 over a fixed label and the +PDF structure is fixed, so the same generator version produces byte-identical +fixtures on every platform. Image data is incompressible noise stored with +FlateDecode: file size tracks decoded size and every page pays a real decode. + + python scripts/resource_envelope/synthetic_workload.py \\ + --output-dir /tmp/loop-envelope --manifest /tmp/loop-envelope/fixtures.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import zlib +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Sequence + +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +from scripts.resource_envelope.create_fixture_manifest import create_manifest +from scripts.resource_envelope.pathological_workload import build_pathological_pdf +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS + + +GENERATOR_VERSION = 1 +PAGE_WIDTH = 612 +PAGE_HEIGHT = 792 +IMAGE_HEAVY_WORKLOAD = "synthetic-image-heavy" +_WORDS = ( + "press", "proof", "bleed", "ink", "plate", "sheet", "trim", "spot", "cyan", "magenta", + "yellow", "black", "overprint", "separation", "imposition", "signature", "gutter", "margin", + "raster", "vector", "profile", "output", "intent", "coverage", "density", "register", +) + + +@dataclass(frozen=True) +class ImageSpec: + width: int + height: int + + @property + def decoded_bytes(self) -> int: + return self.width * self.height * 3 + + +# Sized so each fixture lands inside run_matrix.FIXTURE_SPECS byte bounds. +OFFICE_PAGES = 24 +OFFICE_IMAGE_EVERY = 4 +OFFICE_IMAGE = ImageSpec(380, 280) +IMAGE_HEAVY_PAGES = 60 +IMAGE_HEAVY_IMAGE = ImageSpec(2048, 1360) +TEN_THOUSAND_PAGES = 10000 +TEN_THOUSAND_UNIQUE_IMAGES = 64 +TEN_THOUSAND_IMAGE = ImageSpec(640, 480) + + +def _noise(label: str, size: int) -> bytes: + return hashlib.shake_256(f"loop-resource-envelope/v{GENERATOR_VERSION}/{label}".encode("ascii")).digest(size) + + +def _stream(dictionary: bytes, payload: bytes) -> bytes: + return dictionary[:-2] + b" /Length " + str(len(payload)).encode("ascii") + b" >>\nstream\n" + payload + b"\nendstream" + + +def _image_object(label: str, image: ImageSpec) -> bytes: + dictionary = ( + f"<< /Type /XObject /Subtype /Image /Width {image.width} /Height {image.height}" + " /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /FlateDecode >>" + ).encode("ascii") + return _stream(dictionary, zlib.compress(_noise(label, image.decoded_bytes), level=1)) + + +def _image_placement(image: ImageSpec, name: str) -> bytes: + scale = min(PAGE_WIDTH / image.width, PAGE_HEIGHT / image.height) + width = image.width * scale + height = image.height * scale + x = (PAGE_WIDTH - width) / 2 + y = (PAGE_HEIGHT - height) / 2 + return f"q {width:.4f} 0 0 {height:.4f} {x:.4f} {y:.4f} cm /{name} Do Q\n".encode("ascii") + + +def _page_object(parent: int, contents: int, resources: bytes) -> bytes: + return ( + f"<< /Type /Page /Parent {parent} 0 R /MediaBox [0 0 {PAGE_WIDTH} {PAGE_HEIGHT}] /Resources ".encode("ascii") + + resources + + f" /Contents {contents} 0 R >>".encode("ascii") + ) + + +class _PdfWriter: + """Writes numbered objects straight to disk so a 500 MB fixture never sits in memory.""" + + def __init__(self, handle: BinaryIO, object_count: int) -> None: + self._handle = handle + self._offsets = [0] * (object_count + 1) + self._written = 0 + self._write(b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + + def _write(self, data: bytes) -> None: + self._handle.write(data) + self._written += len(data) + + def add(self, number: int, body: bytes) -> None: + if self._offsets[number]: + raise ValueError(f"object {number} written twice") + self._offsets[number] = self._written + self._write(f"{number} 0 obj\n".encode("ascii") + body + b"\nendobj\n") + + def finish(self, file_id: str) -> None: + missing = [number for number, offset in enumerate(self._offsets) if number and not offset] + if missing: + raise ValueError(f"objects never written: {missing[:5]}") + xref = self._written + self._write(f"xref\n0 {len(self._offsets)}\n0000000000 65535 f \n".encode("ascii")) + for offset in self._offsets[1:]: + self._write(f"{offset:010d} 00000 n \n".encode("ascii")) + self._write( + f"trailer\n<< /Size {len(self._offsets)} /Root 1 0 R /ID [<{file_id}> <{file_id}>] >>\n" + f"startxref\n{xref}\n%%EOF\n".encode("ascii") + ) + + +def _file_id(fixture_id: str) -> str: + return hashlib.sha256(f"{fixture_id}/v{GENERATOR_VERSION}".encode("ascii")).hexdigest()[:32] + + +def _office_text(page_index: int) -> bytes: + selector = _noise(f"office/text/{page_index}", 40 * 12) + lines = [b"BT", b"/F1 10 Tf", b"12 TL", f"54 {PAGE_HEIGHT - 60} Td".encode("ascii")] + for line in range(40): + words = " ".join(_WORDS[value % len(_WORDS)] for value in selector[line * 12 : line * 12 + 12]) + lines.append(f"({words}) '".encode("ascii")) + lines.append(b"ET") + for row in range(8): + y = 80 + row * 18 + lines.append(f"0.2 w 54 {y} m {PAGE_WIDTH - 54} {y} l S".encode("ascii")) + return b"\n".join(lines) + b"\n" + + +def build_office(output: Path, pages: int = OFFICE_PAGES, image: ImageSpec = OFFICE_IMAGE) -> None: + image_pages = [index for index in range(pages) if index % OFFICE_IMAGE_EVERY == 0] + font = 3 + first_image = 4 + first_page = first_image + len(image_pages) + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + writer.add(font, b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>") + for slot in range(len(image_pages)): + writer.add(first_image + slot, _image_object(f"office/image/{slot}", image)) + for index in range(pages): + content = _office_text(index) + xobjects = b"" + if index in image_pages: + slot = image_pages.index(index) + content += f"q {image.width / 2:.4f} 0 0 {image.height / 2:.4f} 54 {PAGE_HEIGHT - 620} cm /Im0 Do Q\n".encode("ascii") + xobjects = f" /XObject << /Im0 {first_image + slot} 0 R >>".encode("ascii") + resources = f"<< /Font << /F1 {font} 0 R >>".encode("ascii") + xobjects + b" >>" + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", content)) + writer.finish(_file_id("office-2mb")) + + +def build_image_pages(output: Path, fixture_id: str, pages: int, unique_images: int, image: ImageSpec) -> None: + if pages < 1 or unique_images < 1: + raise ValueError("pages and unique_images must be positive") + first_image = 3 + first_page = first_image + unique_images + object_count = first_page + 2 * pages - 1 + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as handle: + writer = _PdfWriter(handle, object_count) + writer.add(1, b"<< /Type /Catalog /Pages 2 0 R >>") + kids = " ".join(f"{first_page + 2 * index} 0 R" for index in range(pages)) + writer.add(2, f"<< /Type /Pages /Kids [{kids}] /Count {pages} >>".encode("ascii")) + for slot in range(unique_images): + writer.add(first_image + slot, _image_object(f"{fixture_id}/image/{slot}", image)) + placement = _image_placement(image, "Im0") + for index in range(pages): + resources = f"<< /XObject << /Im0 {first_image + index % unique_images} 0 R >> >>".encode("ascii") + page_object = first_page + 2 * index + writer.add(page_object, _page_object(2, page_object + 1, resources)) + writer.add(page_object + 1, _stream(b"<< >>", placement)) + writer.finish(_file_id(fixture_id)) + + +def _provenance(fixture_id: str) -> str: + return f"scripts/resource_envelope/synthetic_workload.py generator v{GENERATOR_VERSION} ({fixture_id})" + + +def build_bundle(output_dir: Path) -> dict[str, object]: + output_dir.mkdir(parents=True, exist_ok=True) + paths = {fixture_id: output_dir / f"{fixture_id}.pdf" for fixture_id in FIXTURE_SPECS if fixture_id != "multi-gb"} + build_office(paths["office-2mb"]) + build_image_pages(paths["image-heavy-500mb"], "image-heavy-500mb", IMAGE_HEAVY_PAGES, IMAGE_HEAVY_PAGES, IMAGE_HEAVY_IMAGE) + build_image_pages(paths["ten-thousand-page"], "ten-thousand-page", TEN_THOUSAND_PAGES, TEN_THOUSAND_UNIQUE_IMAGES, TEN_THOUSAND_IMAGE) + build_pathological_pdf(paths["pathological-vector"], 256, 512, "pathological-vector") + build_pathological_pdf(paths["transparency-spots"], 256, 256, "transparency-spots") + + manifest = create_manifest(paths, "synthetic") + for record in manifest["fixtures"]: + fixture_id = str(record["fixture_id"]) + record["provenance"] = _provenance(fixture_id) + record["path"] = paths[fixture_id].name + if fixture_id == "ten-thousand-page": + record["workload"] = IMAGE_HEAVY_WORKLOAD + spec = FIXTURE_SPECS[fixture_id] + size = int(record["size_bytes"]) + if (spec["min_bytes"] is not None and size < spec["min_bytes"]) or (spec["max_bytes"] is not None and size > spec["max_bytes"]): + raise ValueError(f"{fixture_id} generated {size} bytes, outside its fixture bounds") + manifest["generator"] = {"script": "scripts/resource_envelope/synthetic_workload.py", "version": GENERATOR_VERSION} + return manifest + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True, help="fixture manifest to write; paths are relative to it") + args = parser.parse_args(argv) + try: + if args.manifest.resolve().parent != args.output_dir.resolve(): + raise ValueError("--manifest must be written inside --output-dir so its relative paths resolve") + manifest = build_bundle(args.output_dir) + args.manifest.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + except (OSError, ValueError) as exc: + print(f"synthetic workload error: {exc}", file=sys.stderr) + return 2 + print(json.dumps({record["fixture_id"]: {"sha256": record["sha256"], "size_bytes": record["size_bytes"]} for record in manifest["fixtures"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resource_envelope/test_run_matrix_probes.py b/scripts/resource_envelope/test_run_matrix_probes.py new file mode 100644 index 000000000..70197ec41 --- /dev/null +++ b/scripts/resource_envelope/test_run_matrix_probes.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import _failure_detail, matrix_failure_reasons, matrix_passes, run_cancellation_probe, run_fixture, run_hostile_corpus +from scripts.resource_envelope.validate_envelope import POOL_NAMES + +CANDIDATE = "candidate-sha" + + +def _policy() -> dict: + return { + "resource_budget": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "workloads": { + "pathological-vector": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200, "cancellation_latency_ms": 50, "recovery_ms": 60000}, + "synthetic-image-heavy": {"page_count": 256, "wall_time_ms": 100, "rss_high_water_bytes": 200}, + "large-document": {"wall_time_ms": 100, "rss_high_water_bytes": 2000}, + }, + } + + +def _envelope(digest: str, status: str = "complete", preflight: int = 5, rss: int = 10, materialized: int = 256, latency: int = -1) -> dict: + return { + "identity": {"commit": CANDIDATE, "fixture_digest": digest}, + "family": "benchmark-render", + "status": status, + "page_count": 256, + "rss_high_water_bytes": rss, + "preflight_high_water_bytes": preflight, + "pages_materialized": materialized, + "elapsed_ms": 10, + "cancellation_latency_ms": latency, + "prefetch_shed": False, + "interaction_slot_held": False, + "resources": { + "config": {"resident_limit_bytes": 200, "pool_limits_bytes": {pool: 100 for pool in POOL_NAMES}}, + "resident_bytes": 0, + "resident_high_water_bytes": 0, + "pressure": "normal", + "pools": {pool: {"limit_bytes": 100, "current_bytes": 0, "high_water_bytes": 0, "evictions": 0, "shed": 0} for pool in POOL_NAMES}, + }, + } + + +def _process(command: list[str], returncode: int, envelope: dict | None) -> subprocess.CompletedProcess[str]: + stdout = json.dumps({"data": {"workload_envelope": envelope}}) if envelope else "" + return subprocess.CompletedProcess(command, returncode, stdout, "") + + +class _Fixture: + def __enter__(self) -> "_Fixture": + self._directory = tempfile.TemporaryDirectory() + self.path = Path(self._directory.name) / "fixture.pdf" + self.path.write_bytes(b"fixture") + self.digest = hashlib.sha256(b"fixture").hexdigest() + self.metadata = {"path": str(self.path), "sha256": self.digest, "size_bytes": 7, "provenance": "unit-test", "page_count": 256} + return self + + def __exit__(self, *exc: object) -> None: + self._directory.cleanup() + + def measure(self, runner, **kwargs) -> dict: + return run_fixture(Path("PdfTool"), "pathological-vector", self.path, _policy(), 1, runner=runner, metadata=self.metadata, candidate_sha=CANDIDATE, **kwargs) + + +class MeasuredRunTest(unittest.TestCase): + def test_complete_envelope_with_preflight_is_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + self.assertGreaterEqual(record["runs"][0]["process_wall_ms"], 0) + + def test_preflight_profile_reaches_the_command(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-2:], ["--profile", "profile.json"]) + + def test_small_document_preflight_covers_every_page(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + + def test_large_document_preflight_is_sampled(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest)), preflight_profile=Path("profile.json")) + self.assertEqual(record["command"][-4:], ["--profile", "profile.json", "--preflight-page-last", "256"]) + self.assertEqual(record["profile"]["preflight_page_last"], 256) + + def test_sampling_needs_a_preflight_profile(self) -> None: + with _Fixture() as fixture: + fixture.metadata["page_count"] = 10000 + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertNotIn("--preflight-page-last", record["command"]) + self.assertIsNone(record["profile"]["preflight_page_last"]) + + def test_workload_rss_cap_replaces_the_resident_limit(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=1000))) + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + def test_workload_rss_cap_still_binds(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "large-document" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest, rss=2500))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: RSS 2500 exceeds resident policy 2000", record["validation_errors"]) + + def test_crashed_process_fails_even_with_an_envelope(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, -11, _envelope(fixture.digest))) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: process-crashed:-11", record["validation_errors"]) + + def test_timeout_fails(self) -> None: + def runner(command, **kwargs): + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner) + self.assertEqual(record["status"], "failed") + self.assertIn("run 1: benchmark-timeout", record["validation_errors"]) + + def test_timeout_is_not_repeated(self) -> None: + calls = [] + + def runner(command, **kwargs): + calls.append(command) + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + + with _Fixture() as fixture: + record = fixture.measure(runner, repetitions=3) + self.assertEqual(len(calls), 1) + self.assertEqual(len(record["runs"]), 1) + + def test_partial_output_exit_is_flagged_not_measured(self) -> None: + with _Fixture() as fixture: + record = fixture.measure(lambda command, **_: _process(command, 5, _envelope(fixture.digest))) + self.assertEqual(record["status"], "flagged") + + def test_manifest_workload_overrides_the_default(self) -> None: + with _Fixture() as fixture: + fixture.metadata["workload"] = "synthetic-image-heavy" + record = fixture.measure(lambda command, **_: _process(command, 0, _envelope(fixture.digest))) + self.assertEqual(record["workload"], "synthetic-image-heavy") + self.assertEqual(record["status"], "measured", record["validation_errors"]) + + +class CancellationProbeTest(unittest.TestCase): + def _probe(self, fixture: _Fixture, cancel_runner, runner) -> dict: + return run_cancellation_probe(Path("PdfTool"), "pathological-vector", fixture.path, _policy(), 5, 0.5, CANDIDATE, + "pathological-vector", cancel_runner=cancel_runner, runner=runner) + + def test_cancelled_run_and_reopen_are_measured(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=20, materialized=40)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "measured", probe["validation_errors"]) + self.assertEqual(probe["cancellation"]["cancellation_latency_ms"], 20) + self.assertNotIn("--profile", probe["cancellation"]["command"]) + self.assertGreaterEqual(probe["recovery"]["recovery_ms"], 0) + self.assertEqual(probe["recovery"]["command"][-4:], ["--page-first", "1", "--page-last", "1"]) + + def test_run_that_finished_before_the_interrupt_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 0, _envelope(fixture.digest)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertEqual(probe["status"], "failed") + self.assertTrue(any("not cancelled" in error or "ended 'complete'" in error for error in probe["validation_errors"])) + + def test_latency_over_policy_fails(self) -> None: + with _Fixture() as fixture: + probe = self._probe( + fixture, + lambda command, timeout, cancel_after: _process(command, 6, _envelope(fixture.digest, status="cancelled", latency=51)), + lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1)), + ) + self.assertIn("cancellation_latency_ms 51 exceeds workload policy 50", probe["validation_errors"]) + + def test_hung_interrupt_fails(self) -> None: + def cancel_runner(command, timeout, cancel_after): + raise subprocess.TimeoutExpired(command, timeout) + + with _Fixture() as fixture: + probe = self._probe(fixture, cancel_runner, lambda command, **_: _process(command, 0, _envelope(fixture.digest, status="incomplete", preflight=-1, materialized=1))) + self.assertEqual(probe["status"], "failed") + + +class HostileCorpusTest(unittest.TestCase): + def test_rejection_is_contained_and_crash_is_not(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + cases = [] + for case_id in ("rejected", "crashing"): + (corpus / f"{case_id}.pdf").write_bytes(case_id.encode()) + cases.append({"id": case_id, "pdf": f"{case_id}.pdf", "sha256": hashlib.sha256(case_id.encode()).hexdigest()}) + (corpus / "manifest.json").write_text(json.dumps({"cases": cases}), encoding="utf-8") + + def runner(command, **_): + return _process(command, 3 if "rejected.pdf" in command[2] else -6, None) + + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=runner) + by_id = {case["case_id"]: case for case in hostile["cases"]} + self.assertEqual(by_id["rejected"]["status"], "contained") + self.assertEqual(by_id["rejected"]["disposition"], "rejected") + self.assertEqual(by_id["crashing"]["status"], "failed") + self.assertEqual(hostile["summary"], {"total": 2, "contained": 1}) + + def test_tampered_fixture_fails(self) -> None: + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) + (corpus / "case.pdf").write_bytes(b"tampered") + (corpus / "manifest.json").write_text(json.dumps({"cases": [{"id": "case", "pdf": "case.pdf", "sha256": "0" * 64}]}), encoding="utf-8") + hostile = run_hostile_corpus(Path("PdfTool"), corpus, _policy(), 5, runner=lambda command, **_: _process(command, 0, None)) + self.assertEqual(hostile["cases"][0]["status"], "failed") + + +class MatrixPassTest(unittest.TestCase): + def _matrix(self, probe_status: str | None = "measured", contained: int = 2, flagged: int = 0) -> dict: + return { + "summary": {"failed": 0, "flagged": flagged, "candidate_sha_verified": True}, + "cancellation_recovery_probe": {"status": probe_status} if probe_status else None, + "hostile": {"summary": {"total": 2, "contained": contained}}, + } + + def test_strict_requires_probe_hostile_and_no_flags(self) -> None: + self.assertTrue(matrix_passes(self._matrix(), strict=True)) + self.assertFalse(matrix_passes(self._matrix(probe_status=None), strict=True)) + self.assertFalse(matrix_passes(self._matrix(flagged=1), strict=True)) + self.assertTrue(matrix_passes(self._matrix(flagged=1), strict=False)) + + def test_uncontained_hostile_case_fails_even_without_strict(self) -> None: + self.assertFalse(matrix_passes(self._matrix(contained=1), strict=False)) + + +class FailureDetailTest(unittest.TestCase): + def test_keeps_render_errors_and_stderr_tail_of_a_partial_run(self) -> None: + stdout = json.dumps({"data": {"nested": [{"rendering-errors": [{"page-no": 3, "message": "bad image"}]}]}}) + completed = subprocess.CompletedProcess(["PdfTool"], 5, stdout, "warning: x\n") + detail = _failure_detail(completed) + self.assertIn('"bad image"', detail) + self.assertIn("stderr=warning: x", detail) + + def test_is_empty_when_nothing_was_reported(self) -> None: + self.assertEqual(_failure_detail(subprocess.CompletedProcess(["PdfTool"], 5, "", "")), "") + + +class FailureReasonTest(unittest.TestCase): + def test_names_every_failing_record_probe_and_hostile_case(self) -> None: + matrix = { + "fixtures": [ + {"fixture_id": "office-2mb", "status": "measured", "required": True, "validation_errors": []}, + {"fixture_id": "image-heavy-500mb", "status": "flagged", "required": True, "validation_errors": ["run 1: process exit code 1 is not success"], + "runs": [{"run": 1, "detail": "stderr=render failed"}]}, + {"fixture_id": "ten-thousand-page", "status": "unavailable", "required": True, "validation_errors": [], "reason": "fixture-not-found"}, + {"fixture_id": "multi-gb", "status": "unavailable", "required": False, "validation_errors": [], "reason": "fixture-not-supplied-optional"}, + ], + "cancellation_recovery_probe": {"status": "failed", "validation_errors": ["recovery probe process-crashed:-11"]}, + "hostile": {"cases": [ + {"case_id": "ok", "status": "contained", "validation_errors": []}, + {"case_id": "deep-tree", "status": "failed", "process_exit_code": 0, "validation_errors": ["RSS 9 exceeds resident policy 4"]}, + ]}, + } + self.assertEqual(matrix_failure_reasons(matrix), [ + "fixture image-heavy-500mb flagged: run 1: process exit code 1 is not success", + "fixture image-heavy-500mb run 1 detail: stderr=render failed", + "fixture ten-thousand-page unavailable: fixture-not-found", + "probe failed: recovery probe process-crashed:-11", + "hostile deep-tree (exit 0): RSS 9 exceeds resident policy 4", + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/resource_envelope/test_synthetic_workload.py b/scripts/resource_envelope/test_synthetic_workload.py new file mode 100644 index 000000000..7473b6b06 --- /dev/null +++ b/scripts/resource_envelope/test_synthetic_workload.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import hashlib +import re +import tempfile +import unittest +from pathlib import Path + +from scripts.resource_envelope.run_matrix import FIXTURE_SPECS +from scripts.resource_envelope.synthetic_workload import ImageSpec, _PdfWriter, build_image_pages, build_office + + +def _xref_problems(pdf: bytes) -> list[str]: + start = int(re.search(rb"startxref\n(\d+)\n%%EOF\n$", pdf).group(1)) + count = int(re.match(rb"xref\n0 (\d+)\n", pdf[start:]).group(1)) + rows = pdf[start:].split(b"\n")[2 : 2 + count] + problems = [f"object {number}" for number, row in enumerate(rows[1:], start=1) if not pdf[int(row[:10]) :].startswith(f"{number} 0 obj".encode())] + for match in re.finditer(rb"/Length (\d+) >>\nstream\n", pdf): + if pdf[match.end() + int(match.group(1)) :][:10] != b"\nendstream": + problems.append(f"stream at {match.start()}") + return problems + + +class SyntheticWorkloadTest(unittest.TestCase): + def test_image_pages_are_deterministic_and_well_formed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.pdf" + second = Path(directory) / "second.pdf" + build_image_pages(first, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + build_image_pages(second, "ten-thousand-page", 30, 4, ImageSpec(16, 12)) + pdf = first.read_bytes() + self.assertEqual(hashlib.sha256(pdf).digest(), hashlib.sha256(second.read_bytes()).digest()) + self.assertEqual(pdf.count(b"/Type /Page "), 30) + self.assertEqual(pdf.count(b"/Subtype /Image"), 4) + self.assertIn(b"/Count 30", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_fixture_identity_changes_the_pixels(self) -> None: + with tempfile.TemporaryDirectory() as directory: + one = Path(directory) / "one.pdf" + two = Path(directory) / "two.pdf" + build_image_pages(one, "ten-thousand-page", 2, 1, ImageSpec(8, 8)) + build_image_pages(two, "image-heavy-500mb", 2, 1, ImageSpec(8, 8)) + self.assertNotEqual(one.read_bytes(), two.read_bytes()) + + def test_office_fixture_fits_its_size_bounds(self) -> None: + spec = FIXTURE_SPECS["office-2mb"] + with tempfile.TemporaryDirectory() as directory: + office = Path(directory) / "office.pdf" + build_office(office) + pdf = office.read_bytes() + self.assertGreaterEqual(len(pdf), spec["min_bytes"]) + self.assertLessEqual(len(pdf), spec["max_bytes"]) + self.assertIn(b"/BaseFont /Helvetica", pdf) + self.assertEqual(_xref_problems(pdf), []) + + def test_writer_rejects_unwritten_objects(self) -> None: + with tempfile.TemporaryFile() as handle: + writer = _PdfWriter(handle, 2) + writer.add(1, b"<< >>") + with self.assertRaises(ValueError): + writer.finish("0" * 32) + + +if __name__ == "__main__": + unittest.main()