From f4577db4a5636d941cfeacb6f8f70f6142f50e70 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 4 Oct 2026 18:57:33 +0200 Subject: [PATCH 1/2] fix(agentic-engineering): require complete identity-bound forge observations --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- .../.claude-plugin/plugin.json | 2 +- plugins/agentic-engineering/CHANGELOG.md | 7 +++ plugins/agentic-engineering/README.md | 6 +- .../agents/portfolio-surveyor.agent.md | 12 +++- plugins/agentic-engineering/plugin.json | 2 +- .../resources/inference-routing.md | 3 +- .../provider-neutral.desired-state.json | 15 +++-- .../classify-default-branch-ci-runs.sh | 19 +++--- .../classify-default-branch-ci-runs.test.sh | 8 +++ .../count-unresolved-review-threads.sh | 4 ++ .../count-unresolved-review-threads.test.sh | 1 + .../scripts/evaluate-inference-routing.sh | 2 +- .../evaluate-inference-routing.test.sh | 3 + .../scripts/json-stream.lib.sh | 16 +++++ .../scripts/json-stream.lib.test.sh | 12 ++++ .../scripts/observation-boundaries.test.sh | 56 +++++++++++++++++ .../portfolio-survey-projections.test.sh | 62 +++++++++++++++++++ 19 files changed, 210 insertions(+), 24 deletions(-) create mode 100755 plugins/agentic-engineering/scripts/json-stream.lib.sh create mode 100644 plugins/agentic-engineering/scripts/json-stream.lib.test.sh create mode 100644 plugins/agentic-engineering/scripts/observation-boundaries.test.sh create mode 100644 plugins/agentic-engineering/scripts/portfolio-survey-projections.test.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ebd18ea5..7cb8354a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "6.1.1", + "version": "6.1.2", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index ebd18ea5..7cb8354a 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "6.1.1", + "version": "6.1.2", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index 93ddb04b..0120b36a 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "6.1.1", + "version": "6.1.2", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/CHANGELOG.md b/plugins/agentic-engineering/CHANGELOG.md index 6ac97417..a6ebe239 100644 --- a/plugins/agentic-engineering/CHANGELOG.md +++ b/plugins/agentic-engineering/CHANGELOG.md @@ -21,6 +21,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and new one before re-enabling unattended writes. Stopping early resumes writes with the retired FinOps schedule still armed, or with a schedule pointing at an entrypoint that no longer resolves. +## 6.1.2 — 2026-10-04 + +**Fixed** — retained review observations refuse repeated decoded JSON keys before counting threads +or failed workflow results. Survey projections require complete GraphQL envelopes, exact issue +identities, integer summaries and terminal census pagination. Routing verification references need +visible content. Consumer onboarding includes the shared raw JSON observer; all routing remains advisory. + ## 6.1.1 — 2026-10-04 **Fixed** — autonomy assessments require distinct stage records, bind protected requests before diff --git a/plugins/agentic-engineering/README.md b/plugins/agentic-engineering/README.md index 2694537e..21914026 100644 --- a/plugins/agentic-engineering/README.md +++ b/plugins/agentic-engineering/README.md @@ -265,8 +265,10 @@ under the guard: consistent totals and a pagination chain that ends with `hasNextPage=false`. Missing or contradictory evidence returns `UNKNOWN` (exit 2). -Both remote helpers bind their GitHub CLI reads to `github.com`, independently of `GH_HOST` -in the calling environment. +Both remote helpers bind their GitHub CLI reads to `github.com`, independently of `GH_HOST` in the +calling environment, +and require the bundled `scripts/json-stream.lib.sh` asset. Before semantic parsing they refuse +repeated decoded object keys across the complete retained JSON stream, including later pages. Each has a provider-neutral desired-state entry pinning its plugin-relative path, reviewed SHA-256, and executable requirement, and the guard accepts only the exact helper beside itself. Resolve the diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index 7652bfd7..0d6f6e3f 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -82,7 +82,7 @@ dimension** — report the gap, never guess a login, a prefix, a marker literal, is the reference shape; adapt its projected fields and fail-closed validation to the mandatory surface: ```sh - gh api graphql --paginate --slurp -f owner= -f name= -f query='query($owner:String!,$name:String!,$endCursor:String){repository(owner:$owner,name:$name){issues(states:OPEN,first:100,after:$endCursor){totalCount nodes{number issueType{name}} pageInfo{hasNextPage endCursor}}}}' | jq -ce 'if (length>0 and all(.[]; (((.errors==null) or (.errors==[])) and ((.data.repository.issues|type)=="object") and ((.data.repository.issues.totalCount|type)=="number") and ((.data.repository.issues.nodes|type)=="array")))) then ([.[].data.repository.issues.nodes[]] as $issues | .[0].data.repository.issues.totalCount as $total | if ($total<0 or ($total|floor)!=$total or (all(.[]; .data.repository.issues.totalCount==$total)|not) or ($issues|length)!=$total or ([$issues[].number]|unique|length)!=$total or (all($issues[]; ((type)=="object" and (.number|type)=="number" and .number>0 and (.number|floor)==.number and has("issueType") and ((.issueType==null) or ((.issueType|type)=="object" and (.issueType.name|type)=="string"))))|not)) then error("QUERY-UNKNOWN: incomplete or malformed issue aggregation input") else {total:$total,types:($issues|group_by(if .issueType==null then [0] else [1,.issueType.name] end)|map({type:(.[0].issueType.name // null),count:length}))} end) else error("QUERY-UNKNOWN: issue aggregation query failed") end' + gh api graphql --paginate --slurp -f owner= -f name= -f query='query($owner:String!,$name:String!,$endCursor:String){repository(owner:$owner,name:$name){issues(states:OPEN,first:100,after:$endCursor){totalCount nodes{number issueType{name}} pageInfo{hasNextPage endCursor}}}}' | jq -ce 'def nonnegative_integer: if type=="number" then .>=0 and floor==. else false end; def nonempty_string: if type=="string" then length>0 else false end; def graphql_complete: if type=="object" then ((has("errors")|not) or .errors==[]) else false end; def valid_issue: type=="object" and (.number|nonnegative_integer) and .number>0 and has("issueType") and (.issueType==null or ((.issueType|type)=="object" and (.issueType.name|nonempty_string))); def valid_page: graphql_complete and (.data.repository.issues as $c | ($c|type)=="object" and ($c.totalCount|nonnegative_integer) and ($c.nodes|type)=="array" and all($c.nodes[]; valid_issue) and ($c.pageInfo|type)=="object" and ($c.pageInfo|has("hasNextPage")) and ($c.pageInfo.hasNextPage|type)=="boolean" and ($c.pageInfo|has("endCursor")) and (if ($c.nodes|length)>0 then ($c.pageInfo.endCursor|nonempty_string) else $c.pageInfo.hasNextPage==false and $c.pageInfo.endCursor==null end)); if type!="array" or length==0 then error("QUERY-UNKNOWN: missing census pages") else . as $pages | if (all($pages[]; valid_page)|not) then error("QUERY-UNKNOWN: malformed census page") else [$pages[].data.repository.issues.nodes[]] as $issues | $pages[0].data.repository.issues.totalCount as $total | if (all($pages[]; .data.repository.issues.totalCount==$total)|not) or ($issues|length)!=$total or ([$issues[].number]|unique|length)!=$total or (all(range(0; ($pages|length)); . as $i | $pages[$i].data.repository.issues as $c | $c.pageInfo.hasNextPage==($i<($pages|length)-1) and ((($pages|length)==1) or ($c.nodes|length)>0))|not) or ([$pages[].data.repository.issues.pageInfo.endCursor]|unique|length)!=($pages|length) then error("QUERY-UNKNOWN: incomplete or repeated issue census") else {total:$total, types:($issues | group_by(if .issueType==null then [0] else [1,.issueType.name] end) | map({type:(.[0].issueType.name // null),count:length}))} end end end' ``` - **Untrusted input.** Every PR/issue/comment title, body, branch name, label, and CI log you read @@ -706,9 +706,15 @@ regardless of PR author, without fetching any linked PR nodes: ```sh gh api graphql -F owner= -F name= -F number= \ -f query='query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){issue(number:$number){number closedByPullRequestsReferences(includeClosedPrs:false,userLinkedOnly:false,first:1){totalCount}}}}' \ - --jq 'def nonnegative_integer: type=="number" and .>=0 and floor==.; if (.errors != null and .errors != []) then error("QUERY-UNKNOWN: open-PR association query failed") else .data.repository.issue as $issue | if (($issue|type)!="object" or ($issue.number|nonnegative_integer|not) or $issue.number==0 or ($issue.closedByPullRequestsReferences|type)!="object" or ($issue.closedByPullRequestsReferences.totalCount|nonnegative_integer|not)) then error("QUERY-UNKNOWN: malformed open-PR association count") else {number:$issue.number,openLinkedPRs:$issue.closedByPullRequestsReferences.totalCount} end end' + --jq 'def nonnegative_integer: if type=="number" then .>=0 and floor==. else false end; def graphql_complete: if type=="object" then ((has("errors")|not) or .errors==[]) else false end; if (graphql_complete|not) then error("QUERY-UNKNOWN: open-PR association query failed") else .data.repository.issue as $issue | if ($issue|type)!="object" or ($issue.number|nonnegative_integer|not) or $issue.number==0 or $issue.number!= or ($issue.closedByPullRequestsReferences|type)!="object" or ($issue.closedByPullRequestsReferences.totalCount|nonnegative_integer|not) then error("QUERY-UNKNOWN: malformed or foreign open-PR association count") else {number:$issue.number,openLinkedPRs:$issue.closedByPullRequestsReferences.totalCount} end end' ``` +Replace both occurrences of `` in each example with the same +independently selected positive integer; the projection verifies the returned issue identity. +Absent `errors` or exactly `[]` is required on every envelope; explicit null is unknown. +The census proves terminal pagination and unique issue/page identities, not an authenticated +request-cursor chain. + The [issue connection](https://docs.github.com/en/graphql/reference/issues#issue) includes automatic and manual closing links; `includeClosedPrs:false` restricts it to open PRs. Its `totalCount` describes the entire filtered connection even with `first:1`; no linked-node pagination or metadata retrieval @@ -723,7 +729,7 @@ Then read the dependency and sub-issue summaries in one query: ```sh gh api graphql -F owner= -F name= -F number= \ -f query='query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){issue(number:$number){number issueDependenciesSummary{blockedBy totalBlockedBy} subIssuesSummary{total completed}}}}' \ - --jq 'if ((.data.repository.issue|type)!="object" or (.data.repository.issue.number|type)!="number" or (.data.repository.issue.issueDependenciesSummary|type)!="object" or (.data.repository.issue.issueDependenciesSummary.blockedBy|type)!="number" or (.data.repository.issue.issueDependenciesSummary.totalBlockedBy|type)!="number" or .data.repository.issue.issueDependenciesSummary.blockedBy < 0 or .data.repository.issue.issueDependenciesSummary.totalBlockedBy < .data.repository.issue.issueDependenciesSummary.blockedBy or (.data.repository.issue.subIssuesSummary|type)!="object" or (.data.repository.issue.subIssuesSummary.total|type)!="number" or (.data.repository.issue.subIssuesSummary.completed|type)!="number" or .data.repository.issue.subIssuesSummary.completed < 0 or .data.repository.issue.subIssuesSummary.total < .data.repository.issue.subIssuesSummary.completed) then error("QUERY-UNKNOWN: malformed issue dependency or sub-issue summary") else {number:.data.repository.issue.number,openBlockedBy:.data.repository.issue.issueDependenciesSummary.blockedBy,totalBlockedBy:.data.repository.issue.issueDependenciesSummary.totalBlockedBy,completedSubIssues:.data.repository.issue.subIssuesSummary.completed,totalSubIssues:.data.repository.issue.subIssuesSummary.total} end' + --jq 'def nonnegative_integer: if type=="number" then .>=0 and floor==. else false end; def graphql_complete: if type=="object" then ((has("errors")|not) or .errors==[]) else false end; if (graphql_complete|not) then error("QUERY-UNKNOWN: dependency query failed") else .data.repository.issue as $issue | if ($issue|type)!="object" or ($issue.number|nonnegative_integer|not) or $issue.number==0 or $issue.number!= or ($issue.issueDependenciesSummary|type)!="object" or ($issue.subIssuesSummary|type)!="object" or ($issue.issueDependenciesSummary.blockedBy|nonnegative_integer|not) or ($issue.issueDependenciesSummary.totalBlockedBy|nonnegative_integer|not) or $issue.issueDependenciesSummary.totalBlockedBy<$issue.issueDependenciesSummary.blockedBy or ($issue.subIssuesSummary.total|nonnegative_integer|not) or ($issue.subIssuesSummary.completed|nonnegative_integer|not) or $issue.subIssuesSummary.total<$issue.subIssuesSummary.completed then error("QUERY-UNKNOWN: malformed or foreign issue dependency or sub-issue summary") else {number:$issue.number, openBlockedBy:$issue.issueDependenciesSummary.blockedBy, totalBlockedBy:$issue.issueDependenciesSummary.totalBlockedBy, completedSubIssues:$issue.subIssuesSummary.completed, totalSubIssues:$issue.subIssuesSummary.total} end end' ``` `issueDependenciesSummary.blockedBy` is the count of **open** blocking issues; diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index 93ddb04b..0120b36a 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "6.1.1", + "version": "6.1.2", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/inference-routing.md b/plugins/agentic-engineering/resources/inference-routing.md index d4d4d099..c99b1d06 100644 --- a/plugins/agentic-engineering/resources/inference-routing.md +++ b/plugins/agentic-engineering/resources/inference-routing.md @@ -73,7 +73,8 @@ The snapshot requires: - `observedAt` (Unix seconds), `runtime`, `runtimeVersion`, runtime-reported resolved exact `model` or `null` (never populate it by copying the policy's intended model); - `billing` (`included`, `unknown`, `paygo`), `controls` (`verified`, `unverified`), `evidenceRef` - (a private verification record reference or `null`); + (a private verification record reference with visible content, or `null`; blank/control-only + references remain unverified); - `buckets.short` and `buckets.weekly`, each `null` or an object with `remainingPercent`, `estimatedChainPercent`, `reservedPercent`, and `unsettledPercent`. Each value is a percentage in 0–100 or `null`. Unknown is never zero. diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 0d4a3514..ea98748c 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -11,14 +11,19 @@ "plugin": "agentic-engineering", "entrypoint": "agentic-engineer", "requiredRuntimeAssets": [ + { + "path": "scripts/json-stream.lib.sh", + "sha256": "380213df0a3859bbe660056523992d5843372c383e446fda20285eef8a6bc0be", + "executable": true + }, { "path": "scripts/classify-default-branch-ci-runs.sh", - "sha256": "81d5b09351de120c62c656fb6a502245d66b16cad37a4289a7e25b370aedb8d5", + "sha256": "1017c5cf420411663aac3187776d06e7bb7c54b36bccbf74c2984d83a4d17c29", "executable": true }, { "path": "scripts/count-unresolved-review-threads.sh", - "sha256": "181157194dfa81ed2e1616314500444698f9cb29142e67c57e96b2ce82c2086a", + "sha256": "c187dee4c3caa9d5c690d668f563fcf83e32594a53a38face39b6396ceda50a2", "executable": true }, { @@ -33,7 +38,7 @@ }, { "path": "scripts/evaluate-inference-routing.sh", - "sha256": "9861b64b6caff842025fab51ad55791e9101cfdf5bdcf837ba08d8c4aec2c809", + "sha256": "5a9449afcc4403979bd30e2517efbed7b8bd1dbdfe83cc5cc0ab0391af43ed75", "executable": true } ], @@ -71,7 +76,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "6fa191d8034ccfa68127f70496338365c455b43446da508413531b1d0b17443d" + "definitionSha256": "1ede08e15c23e1b767f5f66ace2f49f94233677ca27890a411234f58a92f83ca" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", @@ -122,7 +127,7 @@ "copyPasteInstruction": "Adopt this document as the desired state for this workspace. Reconcile it using your native plugin, agent, scheduler, memory, permission, and model controls. Keep AGENTS.md as the canonical organization contract and this plugin as the canonical role. Make only the minimum runtime-local wiring needed to point at those sources. Before enabling writes, complete every onboarding step and report the resulting state and any unsupported capability.", "steps": [ "Resolve the canonical consumer repository from the current workspace.", - "Install or refresh agentic-engineering from the declared marketplace using the runtime's native mechanism. If full plugins are unsupported, load the canonical agent and skill files plus the referenced runtime assets from the declared source without creating divergent copies, including scripts/classify-default-branch-ci-runs.sh, scripts/count-unresolved-review-threads.sh, scripts/forge-readonly-guard.sh and scripts/surveyor-forge-readonly.sh for the portfolio-surveyor, and verify every declared SHA-256 and executable requirement before use. forge-readonly-guard.sh --command is the portable contract; the wrapper is an input adapter for it, never a second policy, and is wired in only where the runtime presents a candidate command as structured JSON on standard input rather than as an argument. Register the guard at the surveyor's own pre-execution point only, directly or through that adapter, using the runtime's own agent-scoped mechanism; plugin-shipped agent definitions cannot carry that registration themselves, so it is always consumer-side wiring. A deployment that has not installed those assets or has not registered the guard for that agent fails closed (forge reads are QUERY-UNKNOWN).", + "Install or refresh agentic-engineering from the declared marketplace using the runtime's native mechanism. If full plugins are unsupported, load the canonical agent and skill files plus the referenced runtime assets from the declared source without creating divergent copies, including scripts/classify-default-branch-ci-runs.sh, scripts/count-unresolved-review-threads.sh, scripts/json-stream.lib.sh, scripts/forge-readonly-guard.sh and scripts/surveyor-forge-readonly.sh for the portfolio-surveyor, and verify every declared SHA-256 and executable requirement before use. forge-readonly-guard.sh --command is the portable contract; the wrapper is an input adapter for it, never a second policy, and is wired in only where the runtime presents a candidate command as structured JSON on standard input rather than as an argument. Register the guard at the surveyor's own pre-execution point only, directly or through that adapter, using the runtime's own agent-scoped mechanism; plugin-shipped agent definitions cannot carry that registration themselves, so it is always consumer-side wiring. A deployment that has not installed those assets or has not registered the guard for that agent fails closed (forge reads are QUERY-UNKNOWN).", "Read AGENTS.md and verify every required consumer contract section. Enable agent-improver only when both additional sections are present. Preserve spec.roles[\"agentic-engineer\"].spendStewardshipEnabled from the single effective desired-state document declared in Spend contract, or the shipped false default when none is declared. Apply the engineer entrypoint's explicit opt-in contract before spend work; onboarding never infers or grants maintainer opt-in. Report the effective source, flag value, and unresolved prerequisites while continuing ordinary operate and advance engineering.", "Map the declared roles onto native agent capabilities, preserve portfolio-surveyor as read-only, and grant least privilege for each role. Export a disabling GH_TELEMETRY (0 or false) in the environment the surveyor's shell inherits: the read-only forge guard treats a missing value as unproven and refuses every command, including reads, so a runtime that enforces the guard without this variable leaves the surveyor unable to run any forge query at all. It cannot be supplied inside the command string, which the guard also refuses.", "Allocate a unique branch namespace for every deployed writer instance and record it in the consumer contract before enabling writes.", diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh index 0272c087..9b9b5c20 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh @@ -240,15 +240,18 @@ jq_filter=' ' classification="" +observer_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) || exit 2 +# shellcheck source=plugins/agentic-engineering/scripts/json-stream.lib.sh +. "$observer_dir/json-stream.lib.sh" || exit 2 if [ -n "$payload_path" ]; then - if ! classification=$(jq -rs \ - --arg expected_head_sha "$head_sha" \ - --arg expected_branch "$branch" \ - "$jq_filter" "$payload_path"); then - echo "classify-default-branch-ci-runs: malformed or incomplete runs payload; health is unknown" >&2 - exit 2 - fi -elif ! classification=$(printf '%s\n' "$payload" | jq -rs \ + payload=$(cat "$payload_path") || exit 2 + payload_path="" +fi +if ! printf '%s\n' "$payload" | json_stream_unique; then + echo 'classify-default-branch-ci-runs: ambiguous or incomplete raw observation; health is unknown' >&2 + exit 2 +fi +if ! classification=$(printf '%s\n' "$payload" | jq -rs \ --arg expected_head_sha "$head_sha" \ --arg expected_branch "$branch" \ "$jq_filter"); then diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh index 52ff8566..43c2572d 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh @@ -340,15 +340,22 @@ counter_sha=$(hash_file "$HERE/count-unresolved-review-threads.sh") guard_sha=$(hash_file "$HERE/forge-readonly-guard.sh") wrapper_sha=$(hash_file "$HERE/surveyor-forge-readonly.sh") routing_sha=$(hash_file "$HERE/evaluate-inference-routing.sh") +json_stream_sha=$(hash_file "$HERE/json-stream.lib.sh") if grep -Fq 'referenced runtime assets' "$DESIRED_STATE" && jq -e \ --arg classifier_sha "$classifier_sha" \ --arg counter_sha "$counter_sha" \ --arg guard_sha "$guard_sha" \ --arg wrapper_sha "$wrapper_sha" \ + --arg json_stream_sha "$json_stream_sha" \ --arg routing_sha "$routing_sha" ' # The routing helper is independent of the surveyor, but shares the runtime asset pin set. .spec.source.requiredRuntimeAssets == [ + { + path: "scripts/json-stream.lib.sh", + sha256: $json_stream_sha, + executable: true + }, { path: "scripts/classify-default-branch-ci-runs.sh", sha256: $classifier_sha, @@ -388,6 +395,7 @@ fi install_step=$(jq -r '.spec.onboarding.steps[] | select(contains("referenced runtime assets"))' "$DESIRED_STATE") missing='' for asset in scripts/classify-default-branch-ci-runs.sh scripts/count-unresolved-review-threads.sh \ + scripts/json-stream.lib.sh \ scripts/forge-readonly-guard.sh scripts/surveyor-forge-readonly.sh; do case "$install_step" in *"$asset"*) ;; diff --git a/plugins/agentic-engineering/scripts/count-unresolved-review-threads.sh b/plugins/agentic-engineering/scripts/count-unresolved-review-threads.sh index a8ec7ad9..f9cec753 100755 --- a/plugins/agentic-engineering/scripts/count-unresolved-review-threads.sh +++ b/plugins/agentic-engineering/scripts/count-unresolved-review-threads.sh @@ -91,6 +91,10 @@ if ! pages=$(gh api graphql --paginate \ unknown read-failed fi [ -n "$pages" ] || unknown read-failed +observer_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) || unknown malformed +# shellcheck source=plugins/agentic-engineering/scripts/json-stream.lib.sh +. "$observer_dir/json-stream.lib.sh" || unknown malformed +printf '%s\n' "$pages" | json_stream_unique || unknown malformed # Every page must carry a reviewThreads object with an integer totalCount and a node array # whose every node has a boolean isResolved. A missing field is malformed, never "resolved": diff --git a/plugins/agentic-engineering/scripts/count-unresolved-review-threads.test.sh b/plugins/agentic-engineering/scripts/count-unresolved-review-threads.test.sh index 5f60f776..dc2caf95 100755 --- a/plugins/agentic-engineering/scripts/count-unresolved-review-threads.test.sh +++ b/plugins/agentic-engineering/scripts/count-unresolved-review-threads.test.sh @@ -168,6 +168,7 @@ for position in first later; do done # Ablation 1: without --paginate the helper sees 100 of 103, and the truncation check must +cp "$HERE/json-stream.lib.sh" "$TEST_TMP/json-stream.lib.sh" # catch it rather than report the first page's zero. sed 's/ --paginate//' "$COUNTER" >"$TEST_TMP/no-paginate.sh" expect 'ablation: no --paginate is caught as unfinished pagination' "$TEST_TMP/no-paginate.sh" paginated 2 \ diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh index 3fd43820..c8d347c6 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh @@ -93,7 +93,7 @@ result=$(jq -sce --argjson now "$now" ' if $s.model != $route.model then "MODEL_MISMATCH" else empty end, if $s.observedAt > $now or ($now - $s.observedAt) > $p.limits.snapshotMaxAgeSeconds then "SNAPSHOT_STALE" else empty end, if $s.billing != "included" then "BILLING_UNPROVEN" else empty end, - if $s.controls != "verified" or $s.evidenceRef == null then "CONTROLS_UNVERIFIED" else empty end, + if $s.controls != "verified" or ($s.evidenceRef == null) or ($s.evidenceRef | test("^[\\s\\p{Cc}]*$")) then "CONTROLS_UNVERIFIED" else empty end, (["short","weekly"][] as $name | $s.buckets[$name] as $b | if $b == null or any($b[]; . == null) then "QUOTA_UNKNOWN" elif ($b.estimatedChainPercent + $b.reservedPercent + $b.unsettledPercent + $p.limits.reservePercent[$name]) > $b.remainingPercent diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh index e052cbbe..9c76008e 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -85,6 +85,9 @@ run_case child-limit '.task.children=2' 1 '.reasons | index("DELEGATION_LIMIT") run_case depth-limit '.task.depth=2' 1 '.reasons | index("DELEGATION_LIMIT") != null' run_case unverified-controls '.snapshot.controls="unverified"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' run_case no-evidence '.snapshot.evidenceRef=null' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case blank-evidence '.snapshot.evidenceRef=" "' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case control-evidence '.snapshot.evidenceRef="\n\t"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case unicode-evidence '.snapshot.evidenceRef="artifact://données/検証"' 0 '.decision=="RECOMMEND" and .executionAdmitted==false' run_case paygo '.snapshot.billing="paygo"' 1 '.reasons | index("BILLING_UNPROVEN") != null' run_case negative-quota '.snapshot.buckets.weekly.remainingPercent=-1' 2 '.decision == "INVALID"' run_case no-cross-runtime-reuse '.task.class="deepRefactor" | .snapshot.model="native-v1"' 1 '.reasons | index("RUNTIME_MISMATCH") != null' diff --git a/plugins/agentic-engineering/scripts/json-stream.lib.sh b/plugins/agentic-engineering/scripts/json-stream.lib.sh new file mode 100755 index 00000000..d3603a8c --- /dev/null +++ b/plugins/agentic-engineering/scripts/json-stream.lib.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Observe raw paginated JSON before semantic parsing can erase repeated decoded keys. +json_stream_unique() { + jq --stream -es ' + if length==0 then false else + reduce .[] as $event ({complete:{}, valid:true}; + (if .complete["[]"]==true then .complete={} else . end) | + if ($event|length)==2 then + .complete as $complete | $event[0] as $path | + .valid = (.valid and (any(range(0;($path|length)+1); + $complete[($path[0:.]|tojson)]==true)|not)) | + .complete[($path|tojson)] = true + else .complete[($event[0][0:-1]|tojson)] = true end + ) | .valid end + ' >/dev/null 2>&1 +} diff --git a/plugins/agentic-engineering/scripts/json-stream.lib.test.sh b/plugins/agentic-engineering/scripts/json-stream.lib.test.sh new file mode 100644 index 00000000..0145f324 --- /dev/null +++ b/plugins/agentic-engineering/scripts/json-stream.lib.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=plugins/agentic-engineering/scripts/json-stream.lib.sh +. "$here/json-stream.lib.sh" +for value in '{}' '[]' '{"a":{"b":1},"c":[{},{"b":2}]}' $'{"a":1}\n{"a":2}' '{"a":"données/検証"}'; do + printf '%s\n' "$value" | json_stream_unique +done +for value in '' '{' '{"a":1,"a":2}' '{"a":1,"\u0061":2}' '{"a":{},"a":{}}' '[{"a":1,"a":2}]' $'{"a":1}\n{"b":[],"b":[]}'; do + if printf '%s\n' "$value" | json_stream_unique; then printf 'FAIL: ambiguous raw JSON accepted\n' >&2; exit 1; fi +done +printf 'PASS: unique raw documents and paginated JSON boundaries\n' diff --git a/plugins/agentic-engineering/scripts/observation-boundaries.test.sh b/plugins/agentic-engineering/scripts/observation-boundaries.test.sh new file mode 100644 index 00000000..68a56326 --- /dev/null +++ b/plugins/agentic-engineering/scripts/observation-boundaries.test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Exercise installed readers without network or inspected source execution. +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir "$work/bin" +cat > "$work/bin/gh" <<'STUB' +#!/usr/bin/env bash +cat "$THREAD_INPUT" +STUB +chmod +x "$work/bin/gh" +failed=0 +# Refused observations may emit UNKNOWN, never a successful count. +refuse_reader() { + local status=0 + if [[ $1 == thread ]]; then + THREAD_INPUT="$work/input" PATH="$work/bin:$PATH" bash "$here/count-unresolved-review-threads.sh" --repo devantler-tech/example --pr 42 > "$work/out" 2> "$work/err" || status=$? + [[ $status == 2 && $(cat "$work/out") == UNKNOWN* ]] && return + else + bash "$here/classify-default-branch-ci-runs.sh" --input "$work/input" > "$work/out" 2> "$work/err" || status=$? + [[ $status == 2 && ! -s $work/out ]] && return + fi + printf 'FAIL: %s ambiguity was accepted\n' "$1" >&2; failed=$((failed+1)) +} +thread='{"data":{"repository":{"nameWithOwner":"devantler-tech/example","pullRequest":{"number":42,"reviewThreads":{"totalCount":1,"nodes":[{"id":"t1","isResolved":false}],"pageInfo":{"hasNextPage":false,"endCursor":"c1"}}}}}}' +printf '%s\n' "$thread" > "$work/input" +status=0 +THREAD_INPUT="$work/input" PATH="$work/bin:$PATH" bash "$here/count-unresolved-review-threads.sh" --repo devantler-tech/example --pr 42 > "$work/out" || status=$? +[[ $status == 1 && $(cat "$work/out") == 'unresolved=1 total=1' ]] +for replacement in '"isResolved":false,"isResolved":true' '"isResolved":false,"is\u0052esolved":true'; do + printf '%s\n' "${thread/\"isResolved\":false/$replacement}" > "$work/input" + refuse_reader thread +done +first=${thread/\"hasNextPage\":false/\"hasNextPage\":true} +first=${first/\"totalCount\":1/\"totalCount\":2} +last=${thread/\"totalCount\":1/\"totalCount\":2} +last=${last/\"t1\"/\"t2\"}; last=${last/\"c1\"/\"c2\"} +printf '%s\n%s\n' "$first" "$last" > "$work/input" +status=0 +THREAD_INPUT="$work/input" PATH="$work/bin:$PATH" bash "$here/count-unresolved-review-threads.sh" --repo devantler-tech/example --pr 42 > "$work/out" || status=$? +[[ $status == 1 && $(cat "$work/out") == 'unresolved=2 total=2' ]] +printf '%s\n%s\n' "$first" "${last/\"isResolved\":false/\"isResolved\":false,\"isResolved\":true}" > "$work/input" +refuse_reader thread +run='{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","status":"completed","conclusion":"failure","created_at":"2026-10-04T09:00:00Z","name":"CI"}' +printf '[%s]\n' "$run" > "$work/input" +bash "$here/classify-default-branch-ci-runs.sh" --input "$work/input" > "$work/out" +[[ $(cat "$work/out") == *$'\tfailure\t'* ]] +for replacement in '"conclusion":"failure","conclusion":"success"' '"conclusion":"failure","conclu\u0073ion":"success"'; do + printf '[%s]\n' "${run/\"conclusion\":\"failure\"/$replacement}" > "$work/input" + refuse_reader ci +done +printf '{"workflow_runs":[%s]}\n{"workflow_runs":[],"workflow_runs":[]}\n' "$run" > "$work/input" +refuse_reader ci +[[ $failed == 0 ]] || exit 1 +printf 'PASS: complete unambiguous retained review observations\n' diff --git a/plugins/agentic-engineering/scripts/portfolio-survey-projections.test.sh b/plugins/agentic-engineering/scripts/portfolio-survey-projections.test.sh new file mode 100644 index 00000000..f2e50edb --- /dev/null +++ b/plugins/agentic-engineering/scripts/portfolio-survey-projections.test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Execute the jq programs from the shipped agent examples, not mirrored implementations. +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +node - "$here/../agents/portfolio-surveyor.agent.md" "$work" <<'NODE' +const fs = require("fs"); +const source = fs.readFileSync(process.argv[2], "utf8"); +const destination = process.argv[3]; +const lines = source.split("\n"); +const census = lines.find(line => line.includes("gh api graphql --paginate --slurp") && line.includes("nodes{number issueType")); +const censusMatch = census && census.match(/\| jq -ce '(.*)'$/); +if (!censusMatch) throw Error("actual census projection unavailable"); +fs.writeFileSync(destination + "/census.jq", censusMatch[1]); +for (const [name, marker] of [["association","closedByPullRequestsReferences(includeClosedPrs"], ["dependency","issueDependenciesSummary{blockedBy"]]) { + const start = lines.findIndex(line => line.includes(marker) && line.includes("-f query=")); + const match = start >= 0 && lines[start+1].match(/--jq '(.*)'$/); + if (!match) throw Error("actual " + name + " projection unavailable"); + fs.writeFileSync(destination + "/" + name + ".jq", match[1].replaceAll("", "42")); +} +NODE +failed=0 +# Assert a complete verdict or a failed read with no apparently successful output. +check() { + local filter=$1 change=$2 expected=$3 fixture=$4 status=0 + jq "$change" "$fixture" > "$work/input" + jq -ce -f "$work/$filter.jq" "$work/input" > "$work/out" 2> "$work/err" || status=$? + if [[ $expected == refuse ]]; then + if [[ $status == 0 || -s $work/out ]]; then printf 'FAIL: %s accepted %s\n' "$filter" "$change" >&2; failed=$((failed+1)); fi + elif [[ $status != 0 || $(cat "$work/out") != "$expected" ]]; then + printf 'FAIL: %s positive control: %s\n' "$filter" "$change" >&2; failed=$((failed+1)) + fi +} +printf '%s\n' '[{"data":{"repository":{"issues":{"totalCount":0,"nodes":[],"pageInfo":{"hasNextPage":false,"endCursor":null}}}}}]' > "$work/census.json" +printf '%s\n' '{"data":{"repository":{"issue":{"number":42,"closedByPullRequestsReferences":{"totalCount":0},"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":0},"subIssuesSummary":{"total":0,"completed":0}}}}}' > "$work/issue.json" +check census '.' '{"total":0,"types":[]}' "$work/census.json" +for change in '.[0].errors=null' '.[0].errors=["partial"]' '.[0].errors=0' '.[0]=null' 'del(.[0].data.repository.issues.pageInfo)' '.[0].data.repository.issues.pageInfo.hasNextPage=true' '.[0].data.repository.issues.pageInfo.endCursor="unfetched"' '.[0].data.repository.issues.totalCount=0.5' '.=[]'; do + check census "$change" refuse "$work/census.json" +done +for filter in association dependency; do + if [[ $filter == association ]]; then expected='{"number":42,"openLinkedPRs":0}'; else expected='{"number":42,"openBlockedBy":0,"totalBlockedBy":0,"completedSubIssues":0,"totalSubIssues":0}'; fi + check "$filter" '.' "$expected" "$work/issue.json" + check "$filter" '.errors=[]' "$expected" "$work/issue.json" + for change in '.errors=null' '.errors=["partial"]' '.errors=0' '.=null' '.data.repository.issue.number=43' '.data.repository.issue.number=42.5' '.data.repository.issue.number=0' '.data.repository.issue.number="42"'; do + check "$filter" "$change" refuse "$work/issue.json" + done +done +for change in '.data.repository.issue.closedByPullRequestsReferences.totalCount=0.5' '.data.repository.issue.closedByPullRequestsReferences.totalCount=-1' '.data.repository.issue.closedByPullRequestsReferences.totalCount="0"'; do + check association "$change" refuse "$work/issue.json" +done +for change in '.data.repository.issue.issueDependenciesSummary.blockedBy=0.5' '.data.repository.issue.issueDependenciesSummary.totalBlockedBy=0.5' '.data.repository.issue.subIssuesSummary.total=0.5' '.data.repository.issue.subIssuesSummary.completed=0.5' '.data.repository.issue.issueDependenciesSummary.blockedBy=1' '.data.repository.issue.subIssuesSummary.completed=1'; do + check dependency "$change" refuse "$work/issue.json" +done +printf '%s\n' '[{"data":{"repository":{"issues":{"totalCount":2,"nodes":[{"number":1,"issueType":{"name":"Bug"}}],"pageInfo":{"hasNextPage":true,"endCursor":"c1"}}}}},{"data":{"repository":{"issues":{"totalCount":2,"nodes":[{"number":2,"issueType":null}],"pageInfo":{"hasNextPage":false,"endCursor":"c2"}}}}}]' > "$work/pages.json" +check census '.' '{"total":2,"types":[{"type":null,"count":1},{"type":"Bug","count":1}]}' "$work/pages.json" +check census 'map(.errors=[])' '{"total":2,"types":[{"type":null,"count":1},{"type":"Bug","count":1}]}' "$work/pages.json" +for change in '.[0].data.repository.issues.pageInfo.hasNextPage=false' '.[1].data.repository.issues.pageInfo.hasNextPage=true' '.[1].data.repository.issues.pageInfo.endCursor="c1"' '.[1].data.repository.issues.nodes[0].number=1' '.[1].data.repository.issues.totalCount=3' '.[1].data.repository.issues.nodes=[]' '.[0].data.repository.issues.pageInfo.hasNextPage="true"' '.[1].data.repository.issues.pageInfo.endCursor=null' '.[1].data.repository.issues.nodes[0].number=2.5' '.[1].data.repository.issues.nodes[0].issueType={}' '.[1].errors=null'; do + check census "$change" refuse "$work/pages.json" +done +[[ $failed == 0 ]] || exit 1 +printf 'PASS: actual survey envelopes, exact identities, integer counts and terminal pagination\n' From de0e1acba7acc43930f7dbd939d0c34d882a444c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 4 Oct 2026 19:21:03 +0200 Subject: [PATCH 2/2] fix: reject invisible proof and specialize survey fixtures --- .../provider-neutral.desired-state.json | 2 +- .../scripts/evaluate-inference-routing.sh | 2 +- .../evaluate-inference-routing.test.sh | 5 ++++ .../scripts/portfolio-surveyor-agent.test.sh | 24 +++++++++++-------- .../scripts/surveyor-forge-readonly.test.sh | 6 +++-- .../scripts/surveyor-open-pr-links.test.sh | 4 +++- 6 files changed, 28 insertions(+), 15 deletions(-) diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index ea98748c..9d64e522 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -38,7 +38,7 @@ }, { "path": "scripts/evaluate-inference-routing.sh", - "sha256": "5a9449afcc4403979bd30e2517efbed7b8bd1dbdfe83cc5cc0ab0391af43ed75", + "sha256": "5cda6907eec0a6c3e4ed94f7ac0e42626cf46be6e9b13c3910fc14147187e3a8", "executable": true } ], diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh index c8d347c6..98223a8b 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.sh @@ -93,7 +93,7 @@ result=$(jq -sce --argjson now "$now" ' if $s.model != $route.model then "MODEL_MISMATCH" else empty end, if $s.observedAt > $now or ($now - $s.observedAt) > $p.limits.snapshotMaxAgeSeconds then "SNAPSHOT_STALE" else empty end, if $s.billing != "included" then "BILLING_UNPROVEN" else empty end, - if $s.controls != "verified" or ($s.evidenceRef == null) or ($s.evidenceRef | test("^[\\s\\p{Cc}]*$")) then "CONTROLS_UNVERIFIED" else empty end, + if $s.controls != "verified" or ($s.evidenceRef == null) or ($s.evidenceRef | test("^[\\s\\p{Cc}\\p{Cf}\\p{Default_Ignorable_Code_Point}]*$")) then "CONTROLS_UNVERIFIED" else empty end, (["short","weekly"][] as $name | $s.buckets[$name] as $b | if $b == null or any($b[]; . == null) then "QUOTA_UNKNOWN" elif ($b.estimatedChainPercent + $b.reservedPercent + $b.unsettledPercent + $p.limits.reservePercent[$name]) > $b.remainingPercent diff --git a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh index 9c76008e..bcbc8483 100755 --- a/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh +++ b/plugins/agentic-engineering/scripts/evaluate-inference-routing.test.sh @@ -87,6 +87,11 @@ run_case unverified-controls '.snapshot.controls="unverified"' 1 '.reasons | ind run_case no-evidence '.snapshot.evidenceRef=null' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' run_case blank-evidence '.snapshot.evidenceRef=" "' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' run_case control-evidence '.snapshot.evidenceRef="\n\t"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case format-only-evidence '.snapshot.evidenceRef="\u200b"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case invisible-mark-evidence '.snapshot.evidenceRef="\u034f"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case variation-only-evidence '.snapshot.evidenceRef="\ufe0f"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case filler-only-evidence '.snapshot.evidenceRef="\u3164"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' +run_case mixed-invisible-evidence '.snapshot.evidenceRef=" \u200b\u034f\ufe0f\n"' 1 '.reasons | index("CONTROLS_UNVERIFIED") != null' run_case unicode-evidence '.snapshot.evidenceRef="artifact://données/検証"' 0 '.decision=="RECOMMEND" and .executionAdmitted==false' run_case paygo '.snapshot.billing="paygo"' 1 '.reasons | index("BILLING_UNPROVEN") != null' run_case negative-quota '.snapshot.buckets.weekly.remainingPercent=-1' 2 '.decision == "INVALID"' diff --git a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh index 8514b329..410a905b 100755 --- a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh +++ b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh @@ -64,36 +64,40 @@ JQ_FILTER=$(sed -n \ [ -n "$JQ_FILTER" ] || fail 'could not extract the prescribed dependency jq filter' expect_output() { - local label=$1 input=$2 expected=$3 actual - actual=$(jq -c "$JQ_FILTER" <<<"$input") || + local label=$1 requested=$2 input=$3 expected=$4 actual filter + filter=${JQ_FILTER///$requested} + actual=$(jq -c "$filter" <<<"$input") || fail "$label: valid dependency summary was rejected" [ "$actual" = "$expected" ] || fail "$label: expected $expected, got $actual" } expect_unknown() { - local label=$1 input=$2 output - if output=$(jq -c "$JQ_FILTER" <<<"$input" 2>&1); then + local label=$1 input=$2 output filter + filter=${JQ_FILTER///3196} + if output=$(jq -c "$filter" <<<"$input" 2>&1); then fail "$label: malformed dependency summary produced actionable output: $output" fi } -expect_output 'open and closed blockers' \ +expect_output 'open and closed blockers' 3196 \ '{"data":{"repository":{"issue":{"number":3196,"issueDependenciesSummary":{"blockedBy":2,"totalBlockedBy":3},"subIssuesSummary":{"total":0,"completed":0}}}}}' \ '{"number":3196,"openBlockedBy":2,"totalBlockedBy":3,"completedSubIssues":0,"totalSubIssues":0}' -expect_output 'closed blockers only' \ +expect_output 'closed blockers only' 3261 \ '{"data":{"repository":{"issue":{"number":3261,"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":1},"subIssuesSummary":{"total":3,"completed":1}}}}}' \ '{"number":3261,"openBlockedBy":0,"totalBlockedBy":1,"completedSubIssues":1,"totalSubIssues":3}' -expect_output 'no blockers' \ +expect_output 'no blockers' 5948 \ '{"data":{"repository":{"issue":{"number":5948,"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":0},"subIssuesSummary":{"total":0,"completed":0}}}}}' \ '{"number":5948,"openBlockedBy":0,"totalBlockedBy":0,"completedSubIssues":0,"totalSubIssues":0}' # Negative control: every child closed while the parent stayed open is the delivered-but-open # shape (monorepo#2994 after its only child, #3668, shipped). The read must surface it, not hide it. -expect_output 'every sub-issue closed' \ +expect_output 'every sub-issue closed' 2994 \ '{"data":{"repository":{"issue":{"number":2994,"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":0},"subIssuesSummary":{"total":1,"completed":1}}}}}' \ '{"number":2994,"openBlockedBy":0,"totalBlockedBy":0,"completedSubIssues":1,"totalSubIssues":1}' +expect_unknown 'foreign issue' \ + '{"data":{"repository":{"issue":{"number":3197,"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":0},"subIssuesSummary":{"total":0,"completed":0}}}}}' expect_unknown 'missing sub-issue summary' \ '{"data":{"repository":{"issue":{"number":3196,"issueDependenciesSummary":{"blockedBy":0,"totalBlockedBy":0}}}}}' expect_unknown 'null sub-issue summary' \ @@ -143,7 +147,7 @@ expect_unknown 'open count exceeds total' \ # shellcheck disable=SC2016 # GraphQL variables are literal, not shell expansions. GRAPHQL_QUERY='query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){issue(number:$number){number issueDependenciesSummary{blockedBy totalBlockedBy} subIssuesSummary{total completed}}}}' GH_TELEMETRY=0 "$GUARD" --command \ - "gh api graphql -F owner=devantler-tech -F name=platform -F number=3196 -f query='$GRAPHQL_QUERY' --jq '$JQ_FILTER'" \ + "gh api graphql -F owner=devantler-tech -F name=platform -F number=3196 -f query='$GRAPHQL_QUERY' --jq '${JQ_FILTER///3196}'" \ >/dev/null || fail 'the prescribed dependency read is not admitted by the forge guard' GH_TELEMETRY=0 "$GUARD" --command \ @@ -185,7 +189,7 @@ ISSUE_AGGREGATION_FILTER=$(printf '%s\n' "$ISSUE_AGGREGATION_COMMAND" | [ -n "$ISSUE_AGGREGATION_FILTER" ] || fail 'could not extract the prescribed issue aggregation jq filter' -ISSUE_PAGES='[{"data":{"repository":{"issues":{"totalCount":4,"nodes":[{"number":1,"issueType":{"name":"Bug"}},{"number":2,"issueType":null}]}}}},{"data":{"repository":{"issues":{"totalCount":4,"nodes":[{"number":3,"issueType":{"name":"Task"}},{"number":4,"issueType":{"name":"untyped"}}]}}}}]' +ISSUE_PAGES='[{"data":{"repository":{"issues":{"totalCount":4,"nodes":[{"number":1,"issueType":{"name":"Bug"}},{"number":2,"issueType":null}],"pageInfo":{"hasNextPage":true,"endCursor":"first"}}}}},{"data":{"repository":{"issues":{"totalCount":4,"nodes":[{"number":3,"issueType":{"name":"Task"}},{"number":4,"issueType":{"name":"untyped"}}],"pageInfo":{"hasNextPage":false,"endCursor":"last"}}}}}]' ISSUE_SUMMARY=$(jq -c "$ISSUE_AGGREGATION_FILTER" <<<"$ISSUE_PAGES") || fail 'the prescribed issue aggregation rejected valid issue rows' [ "$ISSUE_SUMMARY" = '{"total":4,"types":[{"type":null,"count":1},{"type":"Bug","count":1},{"type":"Task","count":1},{"type":"untyped","count":1}]}' ] || diff --git a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh index 6e20d52b..2e737ddd 100755 --- a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh +++ b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh @@ -38,6 +38,8 @@ run_wrapper() { # --- missing jq fails closed without consulting the guard --- missing_jq_bin="$TMP/bin" mkdir -p "$missing_jq_bin" +# Keep directory resolution available so the missing-jq branch itself is reached. +ln -s "$(command -v dirname)" "$missing_jq_bin/dirname" # PATH with no jq: keep the wrapper and a no-op guard, but not system jq. cat >"$TMP/noop-guard" <<'EOF' #!/usr/bin/env bash @@ -244,7 +246,7 @@ chmod +x "$TMP/forge-bin/gh" for install_dir in "$TMP/install-v1" "$TMP/relocated plugin 'quoted' \$literal"; do mkdir -p "$install_dir" install_dir=$(CDPATH='' cd -- "$install_dir" && pwd -P) - cp "$GUARD" "$WRAPPER" "$HERE/classify-default-branch-ci-runs.sh" "$install_dir/" + cp "$GUARD" "$WRAPPER" "$HERE/classify-default-branch-ci-runs.sh" "$HERE/json-stream.lib.sh" "$install_dir/" for probe in 'classify-default-branch-ci-runs.sh' '/incorrect/install/classify-default-branch-ci-runs.sh'; do st=0 out=$(run_wrapper "$(hook_stdin "$probe")" "$install_dir/surveyor-forge-readonly.sh" 2>"$TMP/discovery.err") || st=$? @@ -310,7 +312,7 @@ chmod +x "$TMP/thread-bin/gh" for install_dir in "$TMP/threads-v1" "$TMP/relocated threads 'quoted' \$literal"; do mkdir -p "$install_dir" install_dir=$(CDPATH='' cd -- "$install_dir" && pwd -P) - cp "$GUARD" "$WRAPPER" "$HERE/count-unresolved-review-threads.sh" "$install_dir/" + cp "$GUARD" "$WRAPPER" "$HERE/count-unresolved-review-threads.sh" "$HERE/json-stream.lib.sh" "$install_dir/" for probe in 'count-unresolved-review-threads.sh' '/incorrect/install/count-unresolved-review-threads.sh'; do st=0 out=$(run_wrapper "$(hook_stdin "$probe --repo owner/repo --pr 7")" "$install_dir/surveyor-forge-readonly.sh" 2>"$TMP/discovery.err") || st=$? diff --git a/plugins/agentic-engineering/scripts/surveyor-open-pr-links.test.sh b/plugins/agentic-engineering/scripts/surveyor-open-pr-links.test.sh index 98f6f6e5..02c26c9e 100755 --- a/plugins/agentic-engineering/scripts/surveyor-open-pr-links.test.sh +++ b/plugins/agentic-engineering/scripts/surveyor-open-pr-links.test.sh @@ -15,6 +15,7 @@ FILTER=$(sed -n "/closedByPullRequestsReferences(includeClosedPrs:false,userLink if [ -z "$QUERY" ] || [ -z "$FILTER" ]; then fail 'missing prescribed query or projection' fi +FILTER=${FILTER///165} VALID='{"data":{"repository":{"issue":{"number":165,"closedByPullRequestsReferences":{"totalCount":0}}}}}' for count in 0 1 3; do @@ -31,6 +32,7 @@ for mutation in \ 'del(.data.repository.issue.number)' \ '.data.repository.issue.number=0' \ '.data.repository.issue.number=1.5' \ + '.data.repository.issue.number=166' \ 'del(.data.repository.issue.closedByPullRequestsReferences)' \ '.data.repository.issue.closedByPullRequestsReferences=null' \ 'del(.data.repository.issue.closedByPullRequestsReferences.totalCount)' \ @@ -49,4 +51,4 @@ GH_TELEMETRY=0 "$HERE/forge-readonly-guard.sh" --command \ "gh api graphql -F owner=devantler-tech -F name=agent-plugins -F number=165 -f query='$QUERY' --jq '$FILTER'" \ >/dev/null || fail 'prescribed read denied by the forge guard' -echo 'surveyor open PR links: PASS (3 counts, 13 invalid responses, guard admission)' +echo 'surveyor open PR links: PASS (3 counts, 14 invalid responses, guard admission)'