diff --git a/.github/workflows/typescript-integration-test.yml b/.github/workflows/typescript-integration-test.yml index 716f8aa4d5..1393a42132 100644 --- a/.github/workflows/typescript-integration-test.yml +++ b/.github/workflows/typescript-integration-test.yml @@ -51,6 +51,7 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.head.sha }} # Pull the commit from the forked repo + fetch-depth: 0 # full history so selective testing can diff against the base persist-credentials: false # Don't persist credentials for subsequent actions allow-unsafe-pr-checkout: true # opt back into fork-PR checkout, blocked by default in checkout@v7 @@ -69,7 +70,13 @@ jobs: run: npm run build - name: Run integration tests - run: npm run test:integ:all + env: + # PR events expose the base SHA directly; merge_group exposes it as + # merge_group.base_sha. Falling back to the merge-queue base avoids + # diffing against origin/main (which would miss interactions with + # earlier-enqueued PRs and could narrow the test set unsafely). + SELECTIVE_BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + run: npm run test:integ:selective - name: Upload test artifacts if: always() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d496f3f90..ff4da8f813 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,22 @@ npm run lint # lint npm run type-check # type checking ``` +#### Running Selective Integration Tests Locally + +From the repository root, run only the integration tests relevant to your +changes (computed relative to `main`, including uncommitted edits): + +```bash +npm run test:integ:selective +``` + +This uses Vitest's module graph to run only the `integ-node` and +`integ-browser` specs that depend on the source files you changed. If you +alter a structural file (`package.json`, `package-lock.json`, a `strands-ts` +`tsconfig`, `vitest.config.ts`, a shared integration fixture under +`test/integ/__fixtures__/`, or a TypeScript CI workflow), the full +integration suite runs automatically. + ### Documentation Site The documentation site uses Astro with the Starlight theme. diff --git a/package.json b/package.json index c1d49251c1..fd6ca1218b 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:all:coverage": "npm run test:all:coverage -w strands-ts", "test:integ": "npm run test:integ -w strands-ts", "test:integ:all": "npm run test:integ:all -w strands-ts", + "test:integ:selective": "bash ./test-infra/scripts/run-selective-ts.sh", "test:browser:install": "npm run test:browser:install -w strands-ts", "test:package": "npm run test:package -w strands-ts", "lint": "npm run lint -w strands-ts", diff --git a/strands-ts/vitest.config.ts b/strands-ts/vitest.config.ts index 9aaac4a48a..49421c3191 100644 --- a/strands-ts/vitest.config.ts +++ b/strands-ts/vitest.config.ts @@ -69,6 +69,10 @@ export default defineConfig({ alias: { '$/sdk': path.resolve(__dirname, './src'), '$/vended': path.resolve(__dirname, './src/vended-tools'), + // Resolve the package specifier to source so `vitest related` can + // trace src changes to integ tests that import via '@strands-agents/sdk' + // (which otherwise resolves through package exports to dist/). + '@strands-agents/sdk': path.resolve(__dirname, './src'), }, include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.node.ts'], name: { label: 'integ-node', color: 'magenta' }, @@ -89,6 +93,8 @@ export default defineConfig({ alias: { '$/sdk': path.resolve(__dirname, './src'), '$/vended': path.resolve(__dirname, './src/vended-tools'), + // Same package-specifier alias as integ-node above (see comment there). + '@strands-agents/sdk': path.resolve(__dirname, './src'), }, include: ['test/integ/**/*.test.ts', 'test/integ/**/*.test.browser.ts'], name: { label: 'integ-browser', color: 'yellow' }, diff --git a/test-infra/scripts/run-selective-ts.sh b/test-infra/scripts/run-selective-ts.sh new file mode 100755 index 0000000000..402bf35bf7 --- /dev/null +++ b/test-infra/scripts/run-selective-ts.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Selective TypeScript integration testing. +# Runs only the integ specs whose module graph depends on changed files. +# Falls back to the full suite on structural changes; skips when no TS source changed. +# Shared by local dev (npm run test:integ:selective) and CI. + +# --- Resolve repo root so the script is callable from anywhere --- +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +# DRY_RUN prints the chosen branch and exits before invoking any test command. +# Used by verification scenarios so they never trigger live AWS integ runs. +# Defined up top so it also short-circuits the early full-suite fallbacks below. +DRY_RUN="${SELECTIVE_DRY_RUN:-}" + +# Run the entire integration suite. Used by the structural branch and by every +# fail-safe fallback, so the "what does a full run mean" decision lives in one +# place. Honours DRY_RUN (print intent, run nothing) and exits with the suite's +# status so a test failure surfaces as a non-zero script exit. +run_full_suite() { + echo "$1" >&2 + [[ -n "$DRY_RUN" ]] && exit 0 + npm run test:integ:all + exit $? +} + +# --- Compute changed files --- +# Test seam: when SELECTIVE_CHANGED_FILES is defined (even empty) it overrides +# the git-derived change set, so the classification logic below can be exercised +# in isolation with no git state or network. ${VAR+x} detects "is it set", +# which lets a test assert the empty-set (skip) case distinctly from "unset". +# See test/classify.test.sh. +if [[ -n "${SELECTIVE_CHANGED_FILES+x}" ]]; then + CHANGED="$(printf '%s' "$SELECTIVE_CHANGED_FILES" | grep -v '^$' || true)" +else + # --- Determine the base ref to diff against --- + # CI passes SELECTIVE_BASE_REF (the PR base SHA). Locally, discover the + # closest of {origin/main, main, master, */main} — mirrors get-diff.sh. + BASE="${SELECTIVE_BASE_REF:-}" + if [[ -z "$BASE" ]]; then + candidates=() + for ref in main master; do + git rev-parse --verify "$ref" &>/dev/null && candidates+=("$ref") + for remote_ref in $(git for-each-ref --format='%(refname:short)' "refs/remotes/*/$ref" 2>/dev/null); do + candidates+=("$remote_ref") + done + done + if [[ ${#candidates[@]} -eq 0 ]]; then + run_full_suite "WARNING: no base branch found; running full integration suite." + fi + BASE="${candidates[0]}" + best=$(git rev-list --count "$BASE"..HEAD 2>/dev/null || echo 999999) + for ref in "${candidates[@]:1}"; do + d=$(git rev-list --count "$ref"..HEAD 2>/dev/null || echo 999999) + if [[ "$d" -lt "$best" ]]; then BASE="$ref"; best="$d"; fi + done + fi + + # Diff the working tree against the merge-base so the local inner loop tests + # what you just edited, including uncommitted edits. git diff only reports + # tracked files, so untracked (new, not-yet-added) files are appended via + # ls-files --others so brand-new source files select their covering tests too. + # --exclude-standard honours .gitignore. On CI (HEAD == base SHA) there are no + # local edits, so this reduces to the committed diff. + MERGE_BASE="$(git merge-base "$BASE" HEAD 2>/dev/null || echo "$BASE")" + CHANGED="$(git diff --name-only "$MERGE_BASE" 2>/dev/null)" || \ + run_full_suite "WARNING: cannot diff against $MERGE_BASE; running full integration suite." + UNTRACKED="$(git ls-files --others --exclude-standard 2>/dev/null || true)" + CHANGED="$(printf '%s\n%s' "$CHANGED" "$UNTRACKED" | grep -v '^$' || true)" +fi + +if [[ -z "$CHANGED" ]]; then + echo "No changes detected vs ${BASE:-base} — skipping integration tests." + exit 0 +fi + +# --- Branch 1: structural fallback --- +# These paths force the FULL suite: the module-graph tracer cannot see them as +# dependencies of a test, so a change to any of them must fail safe rather than +# risk a narrowed (or empty) selective run. +STRUCTURAL_PATTERNS=( + '^package\.json$' # root dependency manifest + '^package-lock\.json$' # root lockfile + '^strands-ts/package\.json$' # workspace manifest + '^strands-ts/(.*/)?tsconfig.*\.json$' # any tsconfig (src/ + test/integ/ define the $/sdk alias) + '^strands-ts/vitest\.config\.ts$' # vitest config (projects, aliases) + '^strands-ts/test/integ/__fixtures__/' # shared integ setup/fixtures + '^strands-ts/test/integ/__resources__/' # binary assets imported via Vite `?url` (not traced in reverse) + '^strandly/' # workspace member CI triggers on but the graph cannot trace + '^test-infra/scripts/run-selective-ts\.sh$' # this orchestration script + '^\.github/workflows/typescript-' # TypeScript CI workflows +) +STRUCTURAL="$(IFS='|'; echo "${STRUCTURAL_PATTERNS[*]}")" +if echo "$CHANGED" | grep -qE "$STRUCTURAL"; then + run_full_suite "Structural change detected — running full integration suite." +fi + +# --- Branch 2: no TS source changed --- +TS_SOURCE="$(echo "$CHANGED" | grep -E '^strands-ts/' || true)" +if [[ -z "$TS_SOURCE" ]]; then + echo "No strands-ts source changes — skipping integration tests." + exit 0 +fi + +# --- Branch 3: selective --- +# Pass changed source files to Vitest's module-graph tracer, scoped to both +# integ projects. vitest related exits 0 with "No test files found" when none +# depend on the changes — a valid skip. +# +# LOAD-BEARING ASSUMPTION (vitest v4, pinned in strands-ts/package.json): the +# "no covering spec" case exits 0, not non-zero. The whole "never skip a test +# that should run" invariant relies on this — a non-zero here would turn safe +# skips into red CI. If a future vitest major changes this, this branch must be +# revisited (and the version floor in package.json bumped deliberately). +echo "Selective run for changed files:" +echo "$TS_SOURCE" | sed 's/^/ /' +[[ -n "$DRY_RUN" ]] && exit 0 +# Collect paths into an array (relative to strands-ts/) so filenames with +# spaces survive. while-read keeps this portable to macOS bash 3.2 (mapfile +# is bash 4+). TS_SOURCE is guaranteed non-empty by the Branch 2 check above. +files=() +while IFS= read -r f; do + [[ -n "$f" ]] && files+=("$f") +done < <(echo "$TS_SOURCE" | sed -E 's#^strands-ts/##') +( cd strands-ts && npx vitest related "${files[@]}" \ + --project integ-node --project integ-browser --run ) diff --git a/test-infra/scripts/test/classify.test.sh b/test-infra/scripts/test/classify.test.sh new file mode 100755 index 0000000000..53b9eb77b6 --- /dev/null +++ b/test-infra/scripts/test/classify.test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Regression test for run-selective-ts.sh change classification. +# +# Feeds synthetic changed-file lists through the SELECTIVE_CHANGED_FILES test +# seam with SELECTIVE_DRY_RUN=1, so the script classifies into structural / +# skip / selective and exits before invoking git or vitest. No git state, no +# network, no live AWS. Run: bash test-infra/scripts/test/classify.test.sh + +SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/run-selective-ts.sh" + +pass=0 +fail=0 + +# expect_branch +# is one of: structural | skip | selective +expect_branch() { + local expected="$1" desc="$2" + shift 2 + local changed + changed="$(printf '%s\n' "$@")" + + local out + out="$(SELECTIVE_DRY_RUN=1 SELECTIVE_CHANGED_FILES="$changed" bash "$SCRIPT" 2>&1)" + + local actual + case "$out" in + *"Structural change detected"*) actual="structural" ;; + *"No strands-ts source changes"*) actual="skip" ;; + *"No changes detected"*) actual="skip" ;; + *"Selective run for changed files"*) actual="selective" ;; + *) actual="unknown" ;; + esac + + if [[ "$actual" == "$expected" ]]; then + pass=$((pass + 1)) + echo "ok - $desc ($actual)" + else + fail=$((fail + 1)) + echo "FAIL - $desc: expected $expected, got $actual" + echo " output: $out" + fi +} + +# --- Structural: must force the full suite --- +expect_branch structural "root package.json" "package.json" +expect_branch structural "root lockfile" "package-lock.json" +expect_branch structural "strands-ts package.json" "strands-ts/package.json" +expect_branch structural "top-level tsconfig" "strands-ts/tsconfig.json" +expect_branch structural "nested src tsconfig" "strands-ts/src/tsconfig.json" +expect_branch structural "nested integ tsconfig" "strands-ts/test/integ/tsconfig.json" +expect_branch structural "vitest config" "strands-ts/vitest.config.ts" +expect_branch structural "shared integ fixture" "strands-ts/test/integ/__fixtures__/_setup-global.ts" +expect_branch structural "binary integ resource" "strands-ts/test/integ/__resources__/yellow.png" +expect_branch structural "strandly member" "strandly/src/cli.ts" +expect_branch structural "the orchestration script itself" "test-infra/scripts/run-selective-ts.sh" +expect_branch structural "typescript CI workflow" ".github/workflows/typescript-ts-test.yml" +expect_branch structural "structural mixed with source" "README.md" "strands-ts/src/agent/agent.ts" "package.json" + +# --- Skip: no strands-ts source touched --- +expect_branch skip "docs only" "site/docs/index.md" +expect_branch skip "root readme" "README.md" +expect_branch skip "python only" "strands-py/src/strands/agent.py" +expect_branch skip "empty change set" "" + +# --- Selective: traceable source change --- +expect_branch selective "single src file" "strands-ts/src/agent/agent.ts" +expect_branch selective "src plus unrelated docs" "strands-ts/src/agent/agent.ts" "site/docs/x.md" +expect_branch selective "nested src file" "strands-ts/src/vended-tools/bash/types.ts" + +echo "---" +echo "passed: $pass, failed: $fail" +[[ "$fail" -eq 0 ]]