test(ts): change-based selective integration testing

.github/workflows/typescript-integration-test.yml

@@ -51,6 +51,7 @@ jobs:
         uses: actions/checkout@v7
         with:
           ref: ${{ github.event.pull_request.head.sha }} # Pull the commit from the forked repo
+          fetch-depth: 0  # full history so selective testing can diff against the base
           persist-credentials: false  # Don't persist credentials for subsequent actions
           allow-unsafe-pr-checkout: true  # opt back into fork-PR checkout, blocked by default in checkout@v7
 
@@ -69,7 +70,13 @@ jobs:
         run: npm run build
 
       - name: Run integration tests
-        run: npm run test:integ:all
+        env:
+          # PR events expose the base SHA directly; merge_group exposes it as
+          # merge_group.base_sha. Falling back to the merge-queue base avoids
+          # diffing against origin/main (which would miss interactions with
+          # earlier-enqueued PRs and could narrow the test set unsafely).
+          SELECTIVE_BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+        run: npm run test:integ:selective
 
       - name: Upload test artifacts
         if: always()

CONTRIBUTING.md

@@ -155,6 +155,22 @@ npm run lint        # lint
 npm run type-check  # type checking
 ```
 
+#### Running Selective Integration Tests Locally
+
+From the repository root, run only the integration tests relevant to your
+changes (computed relative to `main`, including uncommitted edits):
+
+```bash
+npm run test:integ:selective
+```
+
+This uses Vitest's module graph to run only the `integ-node` and
+`integ-browser` specs that depend on the source files you changed. If you
+alter a structural file (`package.json`, `package-lock.json`, a `strands-ts`
+`tsconfig`, `vitest.config.ts`, a shared integration fixture under
+`test/integ/__fixtures__/`, or a TypeScript CI workflow), the full
+integration suite runs automatically.
+
 ### Documentation Site
 
 The documentation site uses Astro with the Starlight theme.

package.json

@@ -20,6 +20,7 @@
     "test:all:coverage": "npm run test:all:coverage -w strands-ts",
     "test:integ": "npm run test:integ -w strands-ts",
     "test:integ:all": "npm run test:integ:all -w strands-ts",
+    "test:integ:selective": "bash ./test-infra/scripts/run-selective-ts.sh",
     "test:browser:install": "npm run test:browser:install -w strands-ts",
     "test:package": "npm run test:package -w strands-ts",
     "lint": "npm run lint -w strands-ts",

strands-ts/vitest.config.ts

@@ -69,6 +69,10 @@ export default defineConfig({
           alias: {
             '$/sdk': path.resolve(__dirname, './src'),
             '$/vended': path.resolve(__dirname, './src/vended-tools'),
+            // Resolve the package specifier to source so `vitest related` can
+            // trace src changes to integ tests that import via '@strands-agents/sdk'
+            // (which otherwise resolves through package exports to dist/).
+            '@strands-agents/sdk': path.resolve(__dirname, './src'),
           },
           include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.node.ts'],
           name: { label: 'integ-node', color: 'magenta' },
@@ -89,6 +93,8 @@ export default defineConfig({
           alias: {
             '$/sdk': path.resolve(__dirname, './src'),
             '$/vended': path.resolve(__dirname, './src/vended-tools'),
+            // Same package-specifier alias as integ-node above (see comment there).
+            '@strands-agents/sdk': path.resolve(__dirname, './src'),
           },
           include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.browser.ts'],
           name: { label: 'integ-browser', color: 'yellow' },

test-infra/scripts/run-selective-ts.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Selective TypeScript integration testing.
+# Runs only the integ specs whose module graph depends on changed files.
+# Falls back to the full suite on structural changes; skips when no TS source changed.
+# Shared by local dev (npm run test:integ:selective) and CI.
+
+# --- Resolve repo root so the script is callable from anywhere ---
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT"
+
+# DRY_RUN prints the chosen branch and exits before invoking any test command.
+# Used by verification scenarios so they never trigger live AWS integ runs.
+# Defined up top so it also short-circuits the early full-suite fallbacks below.
+DRY_RUN="${SELECTIVE_DRY_RUN:-}"
+
+# Run the entire integration suite. Used by the structural branch and by every
+# fail-safe fallback, so the "what does a full run mean" decision lives in one
+# place. Honours DRY_RUN (print intent, run nothing) and exits with the suite's
+# status so a test failure surfaces as a non-zero script exit.
+run_full_suite() {
+  echo "$1" >&2
+  [[ -n "$DRY_RUN" ]] && exit 0
+  npm run test:integ:all
+  exit $?
+}
+
+# --- Compute changed files ---
+# Test seam: when SELECTIVE_CHANGED_FILES is defined (even empty) it overrides
+# the git-derived change set, so the classification logic below can be exercised
+# in isolation with no git state or network. ${VAR+x} detects "is it set",
+# which lets a test assert the empty-set (skip) case distinctly from "unset".
+# See test/classify.test.sh.
+if [[ -n "${SELECTIVE_CHANGED_FILES+x}" ]]; then
+  CHANGED="$(printf '%s' "$SELECTIVE_CHANGED_FILES" | grep -v '^$' || true)"
+else
+  # --- Determine the base ref to diff against ---
+  # CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the
+  # closest of {origin/main, main, master, */main} — mirrors get-diff.sh.
+  BASE="${SELECTIVE_BASE_REF:-}"
+  if [[ -z "$BASE" ]]; then
+    candidates=()
+    for ref in main master; do
+      git rev-parse --verify "$ref" &>/dev/null && candidates+=("$ref")
+      for remote_ref in $(git for-each-ref --format='%(refname:short)' "refs/remotes/*/$ref" 2>/dev/null); do
+        candidates+=("$remote_ref")
+      done
+    done
+    if [[ ${#candidates[@]} -eq 0 ]]; then
+      run_full_suite "WARNING: no base branch found; running full integration suite."
+    fi
+    BASE="${candidates[0]}"
+    best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999)
+    for ref in "${candidates[@]:1}"; do
+      d=$(git rev-list --count "$ref"..HEAD 2>/dev/null || echo 999999)
+      if [[ "$d" -lt "$best" ]]; then BASE="$ref"; best="$d"; fi
+    done
+  fi
+
+  # Diff the working tree against the merge-base so the local inner loop tests
+  # what you just edited, including uncommitted edits. git diff only reports
+  # tracked files, so untracked (new, not-yet-added) files are appended via
+  # ls-files --others so brand-new source files select their covering tests too.
+  # --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no
+  # local edits, so this reduces to the committed diff.
+  MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")"
+  CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || \
+    run_full_suite "WARNING: cannot diff against $MERGE_BASE; running full integration suite."
+  UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)"
+  CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)"
+fi
+
+if [[ -z "$CHANGED" ]]; then
+  echo "No changes detected vs ${BASE:-base} — skipping integration tests."
+  exit 0
+fi
+
+# --- Branch 1: structural fallback ---
+# These paths force the FULL suite: the module-graph tracer cannot see them as
+# dependencies of a test, so a change to any of them must fail safe rather than
+# risk a narrowed (or empty) selective run.
+STRUCTURAL_PATTERNS=(
+  '^package\.json$'                            # root dependency manifest
+  '^package-lock\.json$'                       # root lockfile
+  '^strands-ts/package\.json$'                 # workspace manifest
+  '^strands-ts/(.*/)?tsconfig.*\.json$'        # any tsconfig (src/ + test/integ/ define the $/sdk alias)
+  '^strands-ts/vitest\.config\.ts$'            # vitest config (projects, aliases)
+  '^strands-ts/test/integ/__fixtures__/'       # shared integ setup/fixtures
+  '^strands-ts/test/integ/__resources__/'      # binary assets imported via Vite `?url` (not traced in reverse)
+  '^strandly/'                                 # workspace member CI triggers on but the graph cannot trace
+  '^test-infra/scripts/run-selective-ts\.sh$'  # this orchestration script
+  '^\.github/workflows/typescript-'            # TypeScript CI workflows
+)
+STRUCTURAL="$(IFS='|'; echo "${STRUCTURAL_PATTERNS[*]}")"
+if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then
+  run_full_suite "Structural change detected — running full integration suite."
+fi
+
+# --- Branch 2: no TS source changed ---
+TS_SOURCE="$(echo "$CHANGED" | grep -E '^strands-ts/' || true)"
+if [[ -z "$TS_SOURCE" ]]; then
+  echo "No strands-ts source changes — skipping integration tests."
+  exit 0
+fi
+
+# --- Branch 3: selective ---
+# Pass changed source files to Vitest's module-graph tracer, scoped to both
+# integ projects. vitest related exits 0 with "No test files found" when none
+# depend on the changes — a valid skip.
+#
+# LOAD-BEARING ASSUMPTION (vitest v4, pinned in strands-ts/package.json): the
+# "no covering spec" case exits 0, not non-zero. The whole "never skip a test
+# that should run" invariant relies on this — a non-zero here would turn safe
+# skips into red CI. If a future vitest major changes this, this branch must be
+# revisited (and the version floor in package.json bumped deliberately).
+echo "Selective run for changed files:"
+echo "$TS_SOURCE" | sed 's/^/  /'
+[[ -n "$DRY_RUN" ]] && exit 0
+# Collect paths into an array (relative to strands-ts/) so filenames with
+# spaces survive. while-read keeps this portable to macOS bash 3.2 (mapfile
+# is bash 4+). TS_SOURCE is guaranteed non-empty by the Branch 2 check above.
+files=()
+while IFS= read -r f; do
+  [[ -n "$f" ]] && files+=("$f")
+done < <(echo "$TS_SOURCE" | sed -E 's#^strands-ts/##')
+( cd strands-ts && npx vitest related "${files[@]}" \
+    --project integ-node --project integ-browser --run )

test-infra/scripts/test/classify.test.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Regression test for run-selective-ts.sh change classification.
+#
+# Feeds synthetic changed-file lists through the SELECTIVE_CHANGED_FILES test
+# seam with SELECTIVE_DRY_RUN=1, so the script classifies into structural /
+# skip / selective and exits before invoking git or vitest. No git state, no
+# network, no live AWS. Run: bash test-infra/scripts/test/classify.test.sh
+
+SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/run-selective-ts.sh"
+
+pass=0
+fail=0
+
+# expect_branch <expected> <description> <changed-files...>
+# <expected> is one of: structural | skip | selective
+expect_branch() {
+  local expected="$1" desc="$2"
+  shift 2
+  local changed
+  changed="$(printf '%s\n' "$@")"
+
+  local out
+  out="$(SELECTIVE_DRY_RUN=1 SELECTIVE_CHANGED_FILES="$changed" bash "$SCRIPT" 2>&1)"
+
+  local actual
+  case "$out" in
+    *"Structural change detected"*) actual="structural" ;;
+    *"No strands-ts source changes"*) actual="skip" ;;
+    *"No changes detected"*) actual="skip" ;;
+    *"Selective run for changed files"*) actual="selective" ;;
+    *) actual="unknown" ;;
+  esac
+
+  if [[ "$actual" == "$expected" ]]; then
+    pass=$((pass + 1))
+    echo "ok   - $desc ($actual)"
+  else
+    fail=$((fail + 1))
+    echo "FAIL - $desc: expected $expected, got $actual"
+    echo "       output: $out"
+  fi
+}
+
+# --- Structural: must force the full suite ---
+expect_branch structural "root package.json" "package.json"
+expect_branch structural "root lockfile" "package-lock.json"
+expect_branch structural "strands-ts package.json" "strands-ts/package.json"
+expect_branch structural "top-level tsconfig" "strands-ts/tsconfig.json"
+expect_branch structural "nested src tsconfig" "strands-ts/src/tsconfig.json"
+expect_branch structural "nested integ tsconfig" "strands-ts/test/integ/tsconfig.json"
+expect_branch structural "vitest config" "strands-ts/vitest.config.ts"
+expect_branch structural "shared integ fixture" "strands-ts/test/integ/__fixtures__/_setup-global.ts"
+expect_branch structural "binary integ resource" "strands-ts/test/integ/__resources__/yellow.png"
+expect_branch structural "strandly member" "strandly/src/cli.ts"
+expect_branch structural "the orchestration script itself" "test-infra/scripts/run-selective-ts.sh"
+expect_branch structural "typescript CI workflow" ".github/workflows/typescript-ts-test.yml"
+expect_branch structural "structural mixed with source" "README.md" "strands-ts/src/agent/agent.ts" "package.json"
+
+# --- Skip: no strands-ts source touched ---
+expect_branch skip "docs only" "site/docs/index.md"
+expect_branch skip "root readme" "README.md"
+expect_branch skip "python only" "strands-py/src/strands/agent.py"
+expect_branch skip "empty change set" ""
+
+# --- Selective: traceable source change ---
+expect_branch selective "single src file" "strands-ts/src/agent/agent.ts"
+expect_branch selective "src plus unrelated docs" "strands-ts/src/agent/agent.ts" "site/docs/x.md"
+expect_branch selective "nested src file" "strands-ts/src/vended-tools/bash/types.ts"
+
+echo "---"
+echo "passed: $pass, failed: $fail"
+[[ "$fail" -eq 0 ]]