From 935d9b510220e019cc7a96870158eb4d0d21f351 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 15:47:50 -0400 Subject: [PATCH 01/12] test(ts): add selective integration-test script --- test-infra/scripts/run-selective-ts.sh | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 test-infra/scripts/run-selective-ts.sh diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh new file mode 100755 index 0000000000..e29ed70d84 --- /dev/null +++ b/test-infra/scripts/run-selective-ts.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Selective TypeScript integration testing. +# Runs only the integ specs whose module graph depends on changed files. +# Falls back to the full suite on structural changes; skips when no TS source changed. +# Shared by local dev (npm run test:integ:selective) and CI. + +# --- Resolve repo root so the script is callable from anywhere --- +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +# --- Determine the base ref to diff against --- +# CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the +# closest of {origin/main, main, master, */main} — mirrors get-diff.sh. +BASE="${SELECTIVE_BASE_REF:-}" +if [[ -z "$BASE" ]]; then + candidates=() + for ref in main master; do + git rev-parse --verify "$ref" &>/dev/null && candidates+=("$ref") + for remote_ref in $(git for-each-ref --format='%(refname:short)' "refs/remotes/*/$ref" 2>/dev/null); do + candidates+=("$remote_ref") + done + done + if [[ ${#candidates[@]} -eq 0 ]]; then + echo "WARNING: no base branch found; running full integration suite." >&2 + npm run test:integ:all + exit $? + fi + BASE="${candidates[0]}" + best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999) + for ref in "${candidates[@]:1}"; do + d=$(git rev-list --count "$ref"..HEAD 2>/dev/null || echo 999999) + if [[ "$d" -lt "$best" ]]; then BASE="$ref"; best="$d"; fi + done +fi + +# --- Compute changed files --- +# merge-base diff INCLUDING uncommitted working-tree changes, so the local +# inner loop tests what you just edited. (Two-arg form: base..working-tree.) +MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" +CHANGED="$(git diff --name-only "$MERGE_BASE")" + +if [[ -z "$CHANGED" ]]; then + echo "No changes detected vs $BASE — skipping integration tests." + exit 0 +fi + +# DRY_RUN prints the chosen branch and exits before invoking any test command. +# Used by verification scenarios so they never trigger live AWS integ runs. +DRY_RUN="${SELECTIVE_DRY_RUN:-}" + +# --- Branch 1: structural fallback --- +STRUCTURAL='^package\.json$|^package-lock\.json$|^strands-ts/package\.json$|^strands-ts/tsconfig.*\.json$|^strands-ts/vitest\.config\.ts$|^strands-ts/test/integ/__fixtures__/|^\.github/workflows/typescript-' +if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then + echo "Structural change detected — running full integration suite." + [[ -n "$DRY_RUN" ]] && exit 0 + npm run test:integ:all + exit $? +fi + +# --- Branch 2: no TS source changed --- +TS_SOURCE="$(echo "$CHANGED" | grep -E '^(strands-ts|strands-wasm|wit)/' || true)" +if [[ -z "$TS_SOURCE" ]]; then + echo "No strands-ts source changes — skipping integration tests." + exit 0 +fi + +# --- Branch 3: selective --- +# Pass changed source files to Vitest's module-graph tracer, scoped to both +# integ projects. vitest related exits 0 with "No test files found" when none +# depend on the changes — a valid skip. +echo "Selective run for changed files:" +echo "$TS_SOURCE" | sed 's/^/ /' +[[ -n "$DRY_RUN" ]] && exit 0 +# shellcheck disable=SC2046 # intentional word-splitting of the file list +( cd strands-ts && npx vitest related $(echo "$TS_SOURCE" | sed 's#^strands-ts/##') \ + --project integ-node --project integ-browser --run ) From 08bb749b342da7041e48ede2cc19b6233cd1f460 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 16:03:48 -0400 Subject: [PATCH 02/12] fix(ts): strip strands-wasm/wit path prefixes in selective run Co-Authored-By: Claude Opus 4.8 --- test-infra/scripts/run-selective-ts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index e29ed70d84..dd0612aafe 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -74,5 +74,5 @@ echo "Selective run for changed files:" echo "$TS_SOURCE" | sed 's/^/ /' [[ -n "$DRY_RUN" ]] && exit 0 # shellcheck disable=SC2046 # intentional word-splitting of the file list -( cd strands-ts && npx vitest related $(echo "$TS_SOURCE" | sed 's#^strands-ts/##') \ +( cd strands-ts && npx vitest related $(echo "$TS_SOURCE" | sed -E 's#^(strands-ts|strands-wasm|wit)/##') \ --project integ-node --project integ-browser --run ) From 952c54ccfae7a1723508d1095fb6d8fc6a2f1d7a Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 16:07:51 -0400 Subject: [PATCH 03/12] test(ts): add test:integ:selective npm script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index c1d49251c1..fd6ca1218b 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:all:coverage": "npm run test:all:coverage -w strands-ts", "test:integ": "npm run test:integ -w strands-ts", "test:integ:all": "npm run test:integ:all -w strands-ts", + "test:integ:selective": "bash ./test-infra/scripts/run-selective-ts.sh", "test:browser:install": "npm run test:browser:install -w strands-ts", "test:package": "npm run test:package -w strands-ts", "lint": "npm run lint -w strands-ts", From cfba53b96d2e1c2d18bc473c4a0d79d4be0bc909 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 16:12:34 -0400 Subject: [PATCH 04/12] ci(ts): use selective integration testing in PR workflow --- .github/workflows/typescript-integration-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/typescript-integration-test.yml b/.github/workflows/typescript-integration-test.yml index 716f8aa4d5..025ac834db 100644 --- a/.github/workflows/typescript-integration-test.yml +++ b/.github/workflows/typescript-integration-test.yml @@ -51,6 +51,7 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} # Pull the commit from the forked repo + fetch-depth: 0 # full history so selective testing can diff against the base persist-credentials: false # Don't persist credentials for subsequent actions allow-unsafe-pr-checkout: true # opt back into fork-PR checkout, blocked by default in checkout@v7 @@ -69,7 +70,9 @@ jobs: run: npm run build - name: Run integration tests - run: npm run test:integ:all + env: + SELECTIVE_BASE_REF: ${{ github.event.pull_request.base.sha }} + run: npm run test:integ:selective - name: Upload test artifacts if: always() From 596b6506fa102ad378e236bba599ba9c6fa7571e Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 17:28:30 -0400 Subject: [PATCH 05/12] docs(ts): document selective integration testing --- CONTRIBUTING.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d496f3f90..ff4da8f813 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,22 @@ npm run lint # lint npm run type-check # type checking ``` +#### Running Selective Integration Tests Locally + +From the repository root, run only the integration tests relevant to your +changes (computed relative to `main`, including uncommitted edits): + +```bash +npm run test:integ:selective +``` + +This uses Vitest's module graph to run only the `integ-node` and +`integ-browser` specs that depend on the source files you changed. If you +alter a structural file (`package.json`, `package-lock.json`, a `strands-ts` +`tsconfig`, `vitest.config.ts`, a shared integration fixture under +`test/integ/__fixtures__/`, or a TypeScript CI workflow), the full +integration suite runs automatically. + ### Documentation Site The documentation site uses Astro with the Starlight theme. From d1e5e131a3f364350afaae2338191ec4a3247c89 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 23:15:17 -0400 Subject: [PATCH 06/12] fix(ts): fall back to full suite when base ref is unresolvable Co-Authored-By: Claude Opus 4.8 --- test-infra/scripts/run-selective-ts.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index dd0612aafe..2555be42a4 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -39,7 +39,11 @@ fi # merge-base diff INCLUDING uncommitted working-tree changes, so the local # inner loop tests what you just edited. (Two-arg form: base..working-tree.) MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" -CHANGED="$(git diff --name-only "$MERGE_BASE")" +CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { + echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 + npm run test:integ:all + exit $? +} if [[ -z "$CHANGED" ]]; then echo "No changes detected vs $BASE — skipping integration tests." From 210266d170727befd5d5f1faa3b5725006563bb2 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Tue, 23 Jun 2026 23:46:44 -0400 Subject: [PATCH 07/12] fix(ts): close selective-test fail-safe gaps from review - structural set now catches nested tsconfigs, __resources__ (?url assets), strandly/, and the orchestration script itself (all invisible to the graph) - DRY_RUN guards the early full-suite fallbacks so verification never hits AWS - word-split-safe file array (portable to bash 3.2) - merge_group falls back to merge_group.base_sha, not origin/main --- .../workflows/typescript-integration-test.yml | 6 +++- test-infra/scripts/run-selective-ts.sh | 33 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/workflows/typescript-integration-test.yml b/.github/workflows/typescript-integration-test.yml index 025ac834db..1393a42132 100644 --- a/.github/workflows/typescript-integration-test.yml +++ b/.github/workflows/typescript-integration-test.yml @@ -71,7 +71,11 @@ jobs: - name: Run integration tests env: - SELECTIVE_BASE_REF: ${{ github.event.pull_request.base.sha }} + # PR events expose the base SHA directly; merge_group exposes it as + # merge_group.base_sha. Falling back to the merge-queue base avoids + # diffing against origin/main (which would miss interactions with + # earlier-enqueued PRs and could narrow the test set unsafely). + SELECTIVE_BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} run: npm run test:integ:selective - name: Upload test artifacts diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 2555be42a4..2918b26e92 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -10,6 +10,11 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$ROOT" +# DRY_RUN prints the chosen branch and exits before invoking any test command. +# Used by verification scenarios so they never trigger live AWS integ runs. +# Defined up top so it also short-circuits the early full-suite fallbacks below. +DRY_RUN="${SELECTIVE_DRY_RUN:-}" + # --- Determine the base ref to diff against --- # CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the # closest of {origin/main, main, master, */main} — mirrors get-diff.sh. @@ -24,6 +29,7 @@ if [[ -z "$BASE" ]]; then done if [[ ${#candidates[@]} -eq 0 ]]; then echo "WARNING: no base branch found; running full integration suite." >&2 + [[ -n "$DRY_RUN" ]] && exit 0 npm run test:integ:all exit $? fi @@ -41,6 +47,7 @@ fi MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 + [[ -n "$DRY_RUN" ]] && exit 0 npm run test:integ:all exit $? } @@ -50,12 +57,18 @@ if [[ -z "$CHANGED" ]]; then exit 0 fi -# DRY_RUN prints the chosen branch and exits before invoking any test command. -# Used by verification scenarios so they never trigger live AWS integ runs. -DRY_RUN="${SELECTIVE_DRY_RUN:-}" - # --- Branch 1: structural fallback --- -STRUCTURAL='^package\.json$|^package-lock\.json$|^strands-ts/package\.json$|^strands-ts/tsconfig.*\.json$|^strands-ts/vitest\.config\.ts$|^strands-ts/test/integ/__fixtures__/|^\.github/workflows/typescript-' +# Anything here forces the FULL suite. The graph tracer cannot see these as +# dependencies of a test, so they must fail safe: +# - dependency manifests / lockfiles +# - any tsconfig under strands-ts (nested too: src/, test/integ/ define the $/sdk alias) +# - the vitest config +# - shared test fixtures AND binary resources imported via Vite `?url` +# (the module graph does not traverse `?url` edges in reverse) +# - strandly/ (workspace member that CI triggers on but the graph can't trace) +# - this orchestration script itself +# - the TypeScript CI workflows +STRUCTURAL='^package\.json$|^package-lock\.json$|^strands-ts/package\.json$|^strands-ts/(.*/)?tsconfig.*\.json$|^strands-ts/vitest\.config\.ts$|^strands-ts/test/integ/__fixtures__/|^strands-ts/test/integ/__resources__/|^strandly/|^test-infra/scripts/run-selective-ts\.sh$|^\.github/workflows/typescript-' if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then echo "Structural change detected — running full integration suite." [[ -n "$DRY_RUN" ]] && exit 0 @@ -77,6 +90,12 @@ fi echo "Selective run for changed files:" echo "$TS_SOURCE" | sed 's/^/ /' [[ -n "$DRY_RUN" ]] && exit 0 -# shellcheck disable=SC2046 # intentional word-splitting of the file list -( cd strands-ts && npx vitest related $(echo "$TS_SOURCE" | sed -E 's#^(strands-ts|strands-wasm|wit)/##') \ +# Collect paths into an array (relative to strands-ts/) so filenames with +# spaces survive. while-read keeps this portable to macOS bash 3.2 (mapfile +# is bash 4+). TS_SOURCE is guaranteed non-empty by the Branch 2 check above. +files=() +while IFS= read -r f; do + [[ -n "$f" ]] && files+=("$f") +done < <(echo "$TS_SOURCE" | sed -E 's#^(strands-ts|strands-wasm|wit)/##') +( cd strands-ts && npx vitest related "${files[@]}" \ --project integ-node --project integ-browser --run ) From 6750edca6c87c64b11007f2d9f38118ec8cac8f6 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Wed, 24 Jun 2026 00:26:04 -0400 Subject: [PATCH 08/12] fix(ts): trace package-specifier imports and include untracked files - alias @strands-agents/sdk to ./src in both integ projects so vitest related traces src changes to tests importing via the package specifier (which otherwise resolves through exports to dist/ and was silently skipped) - append untracked files to the changed set so brand-new local source files select their covering tests --- strands-ts/vitest.config.ts | 8 ++++++++ test-infra/scripts/run-selective-ts.sh | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/strands-ts/vitest.config.ts b/strands-ts/vitest.config.ts index 9aaac4a48a..a076bbc260 100644 --- a/strands-ts/vitest.config.ts +++ b/strands-ts/vitest.config.ts @@ -69,6 +69,10 @@ export default defineConfig({ alias: { '$/sdk': path.resolve(__dirname, './src'), '$/vended': path.resolve(__dirname, './src/vended-tools'), + // Resolve the package specifier to source so `vitest related` can + // trace src changes to integ tests that import via '@strands-agents/sdk' + // (which otherwise resolves through package exports to dist/). + '@strands-agents/sdk': path.resolve(__dirname, './src'), }, include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.node.ts'], name: { label: 'integ-node', color: 'magenta' }, @@ -89,6 +93,10 @@ export default defineConfig({ alias: { '$/sdk': path.resolve(__dirname, './src'), '$/vended': path.resolve(__dirname, './src/vended-tools'), + // Resolve the package specifier to source so `vitest related` can + // trace src changes to integ tests that import via '@strands-agents/sdk' + // (which otherwise resolves through package exports to dist/). + '@strands-agents/sdk': path.resolve(__dirname, './src'), }, include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.browser.ts'], name: { label: 'integ-browser', color: 'yellow' }, diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 2918b26e92..0333a6f606 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -42,8 +42,12 @@ if [[ -z "$BASE" ]]; then fi # --- Compute changed files --- -# merge-base diff INCLUDING uncommitted working-tree changes, so the local -# inner loop tests what you just edited. (Two-arg form: base..working-tree.) +# Diff the working tree against the merge-base so the local inner loop tests +# what you just edited, including uncommitted edits. git diff only reports +# tracked files, so untracked (new, not-yet-added) files are appended via +# ls-files --others so brand-new source files select their covering tests too. +# --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no +# local edits, so this reduces to the committed diff. MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 @@ -51,6 +55,8 @@ CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { npm run test:integ:all exit $? } +UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)" +CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)" if [[ -z "$CHANGED" ]]; then echo "No changes detected vs $BASE — skipping integration tests." From 0910064a50797fc1f6483776465fc7a5950d7862 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Wed, 24 Jun 2026 00:29:44 -0400 Subject: [PATCH 09/12] test(ts): add classification regression test for selective runner Adds a SELECTIVE_CHANGED_FILES test seam and a pure-bash test that feeds synthetic changed-file lists through the structural/skip/selective classifier with no git state or network. Locks in the structural fallback set so a future regex edit cannot silently start skipping tests. --- test-infra/scripts/run-selective-ts.sh | 85 +++++++++++++----------- test-infra/scripts/test/classify.test.sh | 75 +++++++++++++++++++++ 2 files changed, 122 insertions(+), 38 deletions(-) create mode 100755 test-infra/scripts/test/classify.test.sh diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 0333a6f606..182bde916c 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -15,51 +15,60 @@ cd "$ROOT" # Defined up top so it also short-circuits the early full-suite fallbacks below. DRY_RUN="${SELECTIVE_DRY_RUN:-}" -# --- Determine the base ref to diff against --- -# CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the -# closest of {origin/main, main, master, */main} — mirrors get-diff.sh. -BASE="${SELECTIVE_BASE_REF:-}" -if [[ -z "$BASE" ]]; then - candidates=() - for ref in main master; do - git rev-parse --verify "$ref" &>/dev/null && candidates+=("$ref") - for remote_ref in $(git for-each-ref --format='%(refname:short)' "refs/remotes/*/$ref" 2>/dev/null); do - candidates+=("$remote_ref") +# --- Compute changed files --- +# Test seam: when SELECTIVE_CHANGED_FILES is defined (even empty) it overrides +# the git-derived change set, so the classification logic below can be exercised +# in isolation with no git state or network. ${VAR+x} detects "is it set", +# which lets a test assert the empty-set (skip) case distinctly from "unset". +# See test/classify.test.sh. +if [[ -n "${SELECTIVE_CHANGED_FILES+x}" ]]; then + CHANGED="$(printf '%s' "$SELECTIVE_CHANGED_FILES" | grep -v '^$' || true)" +else + # --- Determine the base ref to diff against --- + # CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the + # closest of {origin/main, main, master, */main} — mirrors get-diff.sh. + BASE="${SELECTIVE_BASE_REF:-}" + if [[ -z "$BASE" ]]; then + candidates=() + for ref in main master; do + git rev-parse --verify "$ref" &>/dev/null && candidates+=("$ref") + for remote_ref in $(git for-each-ref --format='%(refname:short)' "refs/remotes/*/$ref" 2>/dev/null); do + candidates+=("$remote_ref") + done + done + if [[ ${#candidates[@]} -eq 0 ]]; then + echo "WARNING: no base branch found; running full integration suite." >&2 + [[ -n "$DRY_RUN" ]] && exit 0 + npm run test:integ:all + exit $? + fi + BASE="${candidates[0]}" + best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999) + for ref in "${candidates[@]:1}"; do + d=$(git rev-list --count "$ref"..HEAD 2>/dev/null || echo 999999) + if [[ "$d" -lt "$best" ]]; then BASE="$ref"; best="$d"; fi done - done - if [[ ${#candidates[@]} -eq 0 ]]; then - echo "WARNING: no base branch found; running full integration suite." >&2 + fi + + # Diff the working tree against the merge-base so the local inner loop tests + # what you just edited, including uncommitted edits. git diff only reports + # tracked files, so untracked (new, not-yet-added) files are appended via + # ls-files --others so brand-new source files select their covering tests too. + # --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no + # local edits, so this reduces to the committed diff. + MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" + CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { + echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 [[ -n "$DRY_RUN" ]] && exit 0 npm run test:integ:all exit $? - fi - BASE="${candidates[0]}" - best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999) - for ref in "${candidates[@]:1}"; do - d=$(git rev-list --count "$ref"..HEAD 2>/dev/null || echo 999999) - if [[ "$d" -lt "$best" ]]; then BASE="$ref"; best="$d"; fi - done + } + UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)" + CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)" fi -# --- Compute changed files --- -# Diff the working tree against the merge-base so the local inner loop tests -# what you just edited, including uncommitted edits. git diff only reports -# tracked files, so untracked (new, not-yet-added) files are appended via -# ls-files --others so brand-new source files select their covering tests too. -# --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no -# local edits, so this reduces to the committed diff. -MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" -CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { - echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 - [[ -n "$DRY_RUN" ]] && exit 0 - npm run test:integ:all - exit $? -} -UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)" -CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)" - if [[ -z "$CHANGED" ]]; then - echo "No changes detected vs $BASE — skipping integration tests." + echo "No changes detected vs ${BASE:-base} — skipping integration tests." exit 0 fi diff --git a/test-infra/scripts/test/classify.test.sh b/test-infra/scripts/test/classify.test.sh new file mode 100755 index 0000000000..f6bbfbcc5b --- /dev/null +++ b/test-infra/scripts/test/classify.test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Regression test for run-selective-ts.sh change classification. +# +# Feeds synthetic changed-file lists through the SELECTIVE_CHANGED_FILES test +# seam with SELECTIVE_DRY_RUN=1, so the script classifies into structural / +# skip / selective and exits before invoking git or vitest. No git state, no +# network, no live AWS. Run: bash test-infra/scripts/test/classify.test.sh + +SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/run-selective-ts.sh" + +pass=0 +fail=0 + +# expect_branch +# is one of: structural | skip | selective +expect_branch() { + local expected="$1" desc="$2" + shift 2 + local changed + changed="$(printf '%s\n' "$@")" + + local out + out="$(SELECTIVE_DRY_RUN=1 SELECTIVE_CHANGED_FILES="$changed" bash "$SCRIPT" 2>&1)" + + local actual + case "$out" in + *"Structural change detected"*) actual="structural" ;; + *"No strands-ts source changes"*) actual="skip" ;; + *"No changes detected"*) actual="skip" ;; + *"Selective run for changed files"*) actual="selective" ;; + *) actual="unknown" ;; + esac + + if [[ "$actual" == "$expected" ]]; then + pass=$((pass + 1)) + echo "ok - $desc ($actual)" + else + fail=$((fail + 1)) + echo "FAIL - $desc: expected $expected, got $actual" + echo " output: $out" + fi +} + +# --- Structural: must force the full suite --- +expect_branch structural "root package.json" "package.json" +expect_branch structural "root lockfile" "package-lock.json" +expect_branch structural "strands-ts package.json" "strands-ts/package.json" +expect_branch structural "top-level tsconfig" "strands-ts/tsconfig.json" +expect_branch structural "nested src tsconfig" "strands-ts/src/tsconfig.json" +expect_branch structural "nested integ tsconfig" "strands-ts/test/integ/tsconfig.json" +expect_branch structural "vitest config" "strands-ts/vitest.config.ts" +expect_branch structural "shared integ fixture" "strands-ts/test/integ/__fixtures__/_setup-global.ts" +expect_branch structural "binary integ resource" "strands-ts/test/integ/__resources__/yellow.png" +expect_branch structural "strandly member" "strandly/src/cli.ts" +expect_branch structural "the orchestration script itself" "test-infra/scripts/run-selective-ts.sh" +expect_branch structural "typescript CI workflow" ".github/workflows/typescript-ts-test.yml" +expect_branch structural "structural mixed with source" "README.md" "strands-ts/src/agent/agent.ts" "package.json" + +# --- Skip: no strands-ts source touched --- +expect_branch skip "docs only" "site/docs/index.md" +expect_branch skip "root readme" "README.md" +expect_branch skip "python only" "strands-py/src/strands/agent.py" +expect_branch skip "empty change set" "" + +# --- Selective: traceable source change --- +expect_branch selective "single src file" "strands-ts/src/agent/agent.ts" +expect_branch selective "src plus unrelated docs" "strands-ts/src/agent/agent.ts" "site/docs/x.md" +expect_branch selective "strands-wasm source" "strands-wasm/src/lib.rs" +expect_branch selective "wit interface" "wit/world.wit" + +echo "---" +echo "passed: $pass, failed: $fail" +[[ "$fail" -eq 0 ]] From 6d13dc147ba3867c0ddf6ffccd070accebf87041 Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Wed, 24 Jun 2026 01:17:15 -0400 Subject: [PATCH 10/12] refactor(ts): readable structural pattern list, drop dead wasm/wit refs - structural fallback expressed as a commented pattern array instead of a 213-char single-line regex; each trigger is reviewable in isolation - remove strands-wasm/ and wit/ handling (those dirs were removed upstream), so the source filter and prefix strip only deal with strands-ts/ --- test-infra/scripts/run-selective-ts.sh | 31 ++++++++++++++---------- test-infra/scripts/test/classify.test.sh | 3 +-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 182bde916c..80debcc9a2 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -73,17 +73,22 @@ if [[ -z "$CHANGED" ]]; then fi # --- Branch 1: structural fallback --- -# Anything here forces the FULL suite. The graph tracer cannot see these as -# dependencies of a test, so they must fail safe: -# - dependency manifests / lockfiles -# - any tsconfig under strands-ts (nested too: src/, test/integ/ define the $/sdk alias) -# - the vitest config -# - shared test fixtures AND binary resources imported via Vite `?url` -# (the module graph does not traverse `?url` edges in reverse) -# - strandly/ (workspace member that CI triggers on but the graph can't trace) -# - this orchestration script itself -# - the TypeScript CI workflows -STRUCTURAL='^package\.json$|^package-lock\.json$|^strands-ts/package\.json$|^strands-ts/(.*/)?tsconfig.*\.json$|^strands-ts/vitest\.config\.ts$|^strands-ts/test/integ/__fixtures__/|^strands-ts/test/integ/__resources__/|^strandly/|^test-infra/scripts/run-selective-ts\.sh$|^\.github/workflows/typescript-' +# These paths force the FULL suite: the module-graph tracer cannot see them as +# dependencies of a test, so a change to any of them must fail safe rather than +# risk a narrowed (or empty) selective run. +STRUCTURAL_PATTERNS=( + '^package\.json$' # root dependency manifest + '^package-lock\.json$' # root lockfile + '^strands-ts/package\.json$' # workspace manifest + '^strands-ts/(.*/)?tsconfig.*\.json$' # any tsconfig (src/ + test/integ/ define the $/sdk alias) + '^strands-ts/vitest\.config\.ts$' # vitest config (projects, aliases) + '^strands-ts/test/integ/__fixtures__/' # shared integ setup/fixtures + '^strands-ts/test/integ/__resources__/' # binary assets imported via Vite `?url` (not traced in reverse) + '^strandly/' # workspace member CI triggers on but the graph cannot trace + '^test-infra/scripts/run-selective-ts\.sh$' # this orchestration script + '^\.github/workflows/typescript-' # TypeScript CI workflows +) +STRUCTURAL="$(IFS='|'; echo "${STRUCTURAL_PATTERNS[*]}")" if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then echo "Structural change detected — running full integration suite." [[ -n "$DRY_RUN" ]] && exit 0 @@ -92,7 +97,7 @@ if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then fi # --- Branch 2: no TS source changed --- -TS_SOURCE="$(echo "$CHANGED" | grep -E '^(strands-ts|strands-wasm|wit)/' || true)" +TS_SOURCE="$(echo "$CHANGED" | grep -E '^strands-ts/' || true)" if [[ -z "$TS_SOURCE" ]]; then echo "No strands-ts source changes — skipping integration tests." exit 0 @@ -111,6 +116,6 @@ echo "$TS_SOURCE" | sed 's/^/ /' files=() while IFS= read -r f; do [[ -n "$f" ]] && files+=("$f") -done < <(echo "$TS_SOURCE" | sed -E 's#^(strands-ts|strands-wasm|wit)/##') +done < <(echo "$TS_SOURCE" | sed -E 's#^strands-ts/##') ( cd strands-ts && npx vitest related "${files[@]}" \ --project integ-node --project integ-browser --run ) diff --git a/test-infra/scripts/test/classify.test.sh b/test-infra/scripts/test/classify.test.sh index f6bbfbcc5b..53b9eb77b6 100755 --- a/test-infra/scripts/test/classify.test.sh +++ b/test-infra/scripts/test/classify.test.sh @@ -67,8 +67,7 @@ expect_branch skip "empty change set" "" # --- Selective: traceable source change --- expect_branch selective "single src file" "strands-ts/src/agent/agent.ts" expect_branch selective "src plus unrelated docs" "strands-ts/src/agent/agent.ts" "site/docs/x.md" -expect_branch selective "strands-wasm source" "strands-wasm/src/lib.rs" -expect_branch selective "wit interface" "wit/world.wit" +expect_branch selective "nested src file" "strands-ts/src/vended-tools/bash/types.ts" echo "---" echo "passed: $pass, failed: $fail" From 07743c95b99d7b1415ccac3fa4815131ba0a66bb Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Wed, 24 Jun 2026 01:26:09 -0400 Subject: [PATCH 11/12] refactor(ts): extract run_full_suite helper, dedupe alias comment - collapse the three identical full-suite fallback blocks (structural branch, no-base-found, cannot-diff) into a single run_full_suite helper so the fallback behaviour has one definition; status line now goes to stderr - replace the duplicated @strands-agents/sdk alias comment in the integ-browser project with a back-reference to integ-node No behaviour change: classify.test.sh passes 20/20; both git-path fallbacks verified to print + exit 0 under dry-run. --- strands-ts/vitest.config.ts | 4 +--- test-infra/scripts/run-selective-ts.sh | 29 +++++++++++++------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/strands-ts/vitest.config.ts b/strands-ts/vitest.config.ts index a076bbc260..49421c3191 100644 --- a/strands-ts/vitest.config.ts +++ b/strands-ts/vitest.config.ts @@ -93,9 +93,7 @@ export default defineConfig({ alias: { '$/sdk': path.resolve(__dirname, './src'), '$/vended': path.resolve(__dirname, './src/vended-tools'), - // Resolve the package specifier to source so `vitest related` can - // trace src changes to integ tests that import via '@strands-agents/sdk' - // (which otherwise resolves through package exports to dist/). + // Same package-specifier alias as integ-node above (see comment there). '@strands-agents/sdk': path.resolve(__dirname, './src'), }, include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.browser.ts'], diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 80debcc9a2..89d709017a 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -15,6 +15,17 @@ cd "$ROOT" # Defined up top so it also short-circuits the early full-suite fallbacks below. DRY_RUN="${SELECTIVE_DRY_RUN:-}" +# Run the entire integration suite. Used by the structural branch and by every +# fail-safe fallback, so the "what does a full run mean" decision lives in one +# place. Honours DRY_RUN (print intent, run nothing) and exits with the suite's +# status so a test failure surfaces as a non-zero script exit. +run_full_suite() { + echo "$1" >&2 + [[ -n "$DRY_RUN" ]] && exit 0 + npm run test:integ:all + exit $? +} + # --- Compute changed files --- # Test seam: when SELECTIVE_CHANGED_FILES is defined (even empty) it overrides # the git-derived change set, so the classification logic below can be exercised @@ -37,10 +48,7 @@ else done done if [[ ${#candidates[@]} -eq 0 ]]; then - echo "WARNING: no base branch found; running full integration suite." >&2 - [[ -n "$DRY_RUN" ]] && exit 0 - npm run test:integ:all - exit $? + run_full_suite "WARNING: no base branch found; running full integration suite." fi BASE="${candidates[0]}" best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999) @@ -57,12 +65,8 @@ else # --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no # local edits, so this reduces to the committed diff. MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" - CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || { - echo "WARNING: cannot diff against $MERGE_BASE; running full integration suite." >&2 - [[ -n "$DRY_RUN" ]] && exit 0 - npm run test:integ:all - exit $? - } + CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || \ + run_full_suite "WARNING: cannot diff against $MERGE_BASE; running full integration suite." UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)" CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)" fi @@ -90,10 +94,7 @@ STRUCTURAL_PATTERNS=( ) STRUCTURAL="$(IFS='|'; echo "${STRUCTURAL_PATTERNS[*]}")" if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then - echo "Structural change detected — running full integration suite." - [[ -n "$DRY_RUN" ]] && exit 0 - npm run test:integ:all - exit $? + run_full_suite "Structural change detected — running full integration suite." fi # --- Branch 2: no TS source changed --- From 42f26efe1f79c869b9e6374b23b348d25afefa8f Mon Sep 17 00:00:00 2001 From: Jonathan Segev Date: Wed, 24 Jun 2026 09:28:56 -0400 Subject: [PATCH 12/12] docs(ts): pin the vitest related exit-0-on-empty assumption Document that selective testing's fail-safe invariant depends on vitest v4 exiting 0 (not non-zero) when no spec covers a change, so a future major bump that changes this is caught at review rather than silently breaking selection. --- test-infra/scripts/run-selective-ts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh index 89d709017a..402bf35bf7 100755 --- a/test-infra/scripts/run-selective-ts.sh +++ b/test-infra/scripts/run-selective-ts.sh @@ -108,6 +108,12 @@ fi # Pass changed source files to Vitest's module-graph tracer, scoped to both # integ projects. vitest related exits 0 with "No test files found" when none # depend on the changes — a valid skip. +# +# LOAD-BEARING ASSUMPTION (vitest v4, pinned in strands-ts/package.json): the +# "no covering spec" case exits 0, not non-zero. The whole "never skip a test +# that should run" invariant relies on this — a non-zero here would turn safe +# skips into red CI. If a future vitest major changes this, this branch must be +# revisited (and the version floor in package.json bumped deliberately). echo "Selective run for changed files:" echo "$TS_SOURCE" | sed 's/^/ /' [[ -n "$DRY_RUN" ]] && exit 0