diff --git a/.github/workflows/bats-testing.yml b/.github/workflows/bats-testing.yml new file mode 100644 index 00000000..d7c58ac7 --- /dev/null +++ b/.github/workflows/bats-testing.yml @@ -0,0 +1,42 @@ +name: bats testing + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + bats: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: ${{ vars.NODE_VERSION }} + + - name: Cache pnpm dependencies + id: cache-pnpm-dependencies + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 + with: + path: node_modules + key: ${{ runner.os }}-dependencies-node-${{ vars.NODE_VERSION }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-dependencies-node-${{ vars.NODE_VERSION }}- + + - name: Install pnpm + run: npm install -g pnpm@10.6.5 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Bats shell coverage + run: make test-bats BATS_FORMATTER=tap diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 544cc7c4..d4747a67 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,17 @@ If you find an issue to work on, you are welcome to open a PR with a fix. 2. Create a working branch and start with your changes! +#### Maintain Makefile shell coverage + +If your change adds or updates a Makefile target, keep the shell-coverage inventory in +sync: + +- Update `tests/bats/make-target-coverage.tsv` so every Makefile target is marked as + either Bats-covered or already covered by a pull request workflow. +- If the target is not already exercised by CI, add or update the relevant test in + `tests/bats/`. +- Run `make test-bats`. + ### Commit your update Commit the changes once you are happy with them. diff --git a/Makefile b/Makefile index e35f6afb..00ee4188 100755 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ STORYBOOK_BIN = $(BIN_DIR)/storybook JEST_BIN = $(BIN_DIR)/jest SERVE_BIN = $(BIN_DIR)/serve PLAYWRIGHT_BIN = $(BIN_DIR)/playwright +BATS_BIN = pnpm exec bats NEXT_BUILD = $(NEXT_BIN) build --webpack NEXT_BUILD_CMD = $(NEXT_BUILD) && $(IMG_OPTIMIZE) @@ -95,6 +96,7 @@ UI_MODE_URL = http://$(WEBSITE_DOMAIN):$(PLAYWRIGHT_TEST_PORT) MD_LINT_ARGS = -i CHANGELOG.md -i "test-results/**/*.md" -i "playwright-report/data/**/*.md" -i "node_modules/**/*.md" JEST_FLAGS = --verbose +BATS_FORMATTER ?= pretty NETWORK_NAME = website-network @@ -305,6 +307,13 @@ test-unit-client: ## Run all client-side unit tests using Jest (Next.js env, TES test-unit-server: ## Run server-side unit tests for Apollo using Jest (Node.js env, TEST_ENV=server, target: $(TEST_DIR_APOLLO)) $(UNIT_TESTS) TEST_ENV=server $(JEST_BIN) $(JEST_FLAGS) $(TEST_DIR_APOLLO) +test-bats: ## Run Bats coverage for Makefile shell flows and CI helper scripts + DOCKER_COMPOSE_TEST_FILE=docker-compose.test.yml \ + DOCKER_COMPOSE_DEV_FILE=docker-compose.yml \ + COMMON_HEALTHCHECKS_FILE=common-healthchecks.yml \ + DOCKER_COMPOSE_MEMLEAK_FILE=docker-compose.memory-leak.yml \ + $(BATS_BIN) --formatter $(BATS_FORMATTER) -r tests/bats + test-memory-leak: start-prod ## This command executes memory leaks tests using Memlab library. @echo "🧪 Starting memory leak test environment..." $(DOCKER_COMPOSE) $(DOCKER_COMPOSE_MEMLEAK_FILE) up -d diff --git a/README.md b/README.md index a94b74d2..df273f91 100755 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ Testing make test-unit-all: runs unit tests for both client and server environments make test-unit-client: runs unit tests for the client using Jest make test-unit-server: runs unit tests for the server using Jest + make test-bats: runs the Bats shell regression suite for Makefile targets and CI helper scripts make test-memory-leak: runs memory leak tests using Memlab make load-tests: executes load tests using the K6 library make test-e2e: runs end-to-end tests inside the prod container @@ -183,6 +184,20 @@ Example: CI=1 make start ``` +### Bats Shell Coverage + +Use the Bats suite to validate Makefile shell flows and `scripts/ci` helpers that are +not already exercised by the pull request workflows: + +```bash + make test-bats +``` + +The coverage inventory lives in `tests/bats/make-target-coverage.tsv`. When you add a +new Makefile target, either add Bats coverage for it or document the workflow that +already exercises it in that file. Additional suite-maintenance notes live in +`tests/bats/README.md`. + ### Load Testing with K6 This project includes a dedicated load testing service using K6, configured via a Docker Compose profile. diff --git a/package.json b/package.json index 14f38740..de068448 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@typescript-eslint/parser": "^8.60.0", "babel-jest": "30.2.0", "babel-preset-jest": "30.2.0", + "bats": "^1.13.0", "eslint": "^9.39.4", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^18.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4f0d823..f7402df9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -173,6 +173,9 @@ importers: babel-preset-jest: specifier: 30.2.0 version: 30.2.0(@babel/core@7.29.7) + bats: + specifier: ^1.13.0 + version: 1.13.0 eslint: specifier: ^9.39.4 version: 9.39.4(jiti@2.7.0) @@ -6013,6 +6016,13 @@ packages: } engines: { node: '>=10.0.0' } + bats@1.13.0: + resolution: + { + integrity: sha512-giSYKGTOcPZyJDbfbTtzAedLcNWdjCLbXYU3/MwPnjyvDXzu6Dgw8d2M+8jHhZXSmsCMSQqCp+YBsJ603UO4vQ==, + } + hasBin: true + big.js@5.2.2: resolution: { @@ -18698,6 +18708,8 @@ snapshots: basic-ftp@5.3.1: {} + bats@1.13.0: {} + big.js@5.2.2: {} blessed@0.1.81: {} diff --git a/scripts/ci/batch_pw_load.sh b/scripts/ci/batch_pw_load.sh index fdbf0623..75151f10 100755 --- a/scripts/ci/batch_pw_load.sh +++ b/scripts/ci/batch_pw_load.sh @@ -45,7 +45,6 @@ setup_docker_network() { start_prod_dind() { setup_docker_network - make build-prod docker compose ${COMPOSE_ARGS} up -d --wait prod } run_make_with_prod_dind() { @@ -146,4 +145,4 @@ case "${1:-all}" in *) main "$@" ;; - esac \ No newline at end of file + esac diff --git a/tests/bats/README.md b/tests/bats/README.md new file mode 100644 index 00000000..f49f8089 --- /dev/null +++ b/tests/bats/README.md @@ -0,0 +1,25 @@ +# Bats Suite + +This suite covers Makefile targets and `scripts/ci` helper flows that are not already +exercised by the pull request workflows. + +## Run locally + +```bash +make test-bats +``` + +Use `BATS_FORMATTER=tap` when you want CI-style output: + +```bash +make test-bats BATS_FORMATTER=tap +``` + +## Maintain coverage + +1. Update `tests/bats/make-target-coverage.tsv` whenever a Makefile target is added, + removed, or reclassified. +2. If a target is not already exercised by a PR workflow, add or update the Bats test + that covers its shell behavior. +3. If a workflow already exercises the target, document that workflow in + `tests/bats/make-target-coverage.tsv` instead of duplicating the coverage. diff --git a/tests/bats/ci_scripts.bats b/tests/bats/ci_scripts.bats new file mode 100644 index 00000000..bda08e2e --- /dev/null +++ b/tests/bats/ci_scripts.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats + +load './test_helper.bash' + +setup() { + setup_ci_script_test_env +} + +@test "batch_unit_mutation_lint.sh dispatches each DIND flow through existing make targets" { + local script_path="$PROJECT_ROOT/scripts/ci/batch_unit_mutation_lint.sh" + + run_ci_script "$script_path" test-unit + [ "$status" -eq 0 ] + assert_log_contains 'make build' + assert_log_contains 'make create-temp-dev-container-dind TEMP_CONTAINER_NAME=website-dev-test' + assert_log_contains 'make copy-source-to-container-dind TEMP_CONTAINER_NAME=website-dev-test' + assert_log_contains 'make install-deps-in-container-dind TEMP_CONTAINER_NAME=website-dev-test' + assert_log_contains 'make run-unit-tests-dind TEMP_CONTAINER_NAME=website-dev-test' + + reset_command_log + run_ci_script "$script_path" test-mutation + [ "$status" -eq 0 ] + assert_log_contains 'make run-mutation-tests-dind TEMP_CONTAINER_NAME=website-dev-test' + + reset_command_log + run_ci_script "$script_path" test-lint + [ "$status" -eq 0 ] + assert_log_contains 'make run-eslint-tests-dind TEMP_CONTAINER_NAME=website-dev-lint' + assert_log_contains 'make run-typescript-tests-dind TEMP_CONTAINER_NAME=website-dev-lint' + assert_log_contains 'make run-markdown-lint-tests-dind TEMP_CONTAINER_NAME=website-dev-lint' +} + +@test "batch_pw_load.sh dispatches its E2E, visual, and load flows through make and docker" { + local script_path="$PROJECT_ROOT/scripts/ci/batch_pw_load.sh" + + run_ci_script "$script_path" test-e2e + [ "$status" -eq 0 ] + assert_log_contains 'make start-prod' + assert_log_contains 'make test-e2e' + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml exec -T playwright mkdir -p /app' + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml cp playwright:/app/playwright-report/. playwright-report/' + + reset_command_log + run_ci_script "$script_path" test-visual + [ "$status" -eq 0 ] + assert_log_contains 'make start-prod' + assert_log_contains 'make test-visual' + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml exec -T playwright mkdir -p /app/src/test /app/src/config /app/pages/i18n' + + reset_command_log + run_ci_script "$script_path" test-load + [ "$status" -eq 0 ] + assert_log_contains 'make start-prod' + assert_log_contains 'make build-k6' + assert_log_contains 'make create-k6-helper-container-dind K6_HELPER_NAME=website-k6-helper' + assert_log_contains 'make run-load-tests-dind K6_HELPER_NAME=website-k6-helper' + assert_log_contains 'docker cp src/test/load/. website-k6-helper:/loadTests/' +} + +@test "batch_lhci_leak.sh handles CodeBuild skips and the DIND Lighthouse flows" { + local script_path="$PROJECT_ROOT/scripts/ci/batch_lhci_leak.sh" + + run env \ + -C "$SCRIPT_SANDBOX" \ + PATH="$STUB_BIN_DIR:$PATH" \ + COMMAND_LOG="$COMMAND_LOG" \ + CODEBUILD_BUILD_ID=website:1 \ + "$script_path" test-memory-leak + [ "$status" -eq 0 ] + assert_output_contains 'Memory leak tests: SKIPPED' + + reset_command_log + run_ci_script "$script_path" test-lighthouse-desktop + [ "$status" -eq 0 ] + assert_log_contains 'make start-prod' + assert_log_contains 'make install-chromium-lhci' + assert_log_contains 'make test-chromium' + assert_log_contains 'make lighthouse-desktop-dind' + + reset_command_log + run_ci_script "$script_path" test-lighthouse-mobile + [ "$status" -eq 0 ] + assert_log_contains 'make start-prod' + assert_log_contains 'make install-chromium-lhci' + assert_log_contains 'make test-chromium' + assert_log_contains 'make lighthouse-mobile-dind' +} diff --git a/tests/bats/issue_175_contract.bats b/tests/bats/issue_175_contract.bats new file mode 100644 index 00000000..b09c6b18 --- /dev/null +++ b/tests/bats/issue_175_contract.bats @@ -0,0 +1,95 @@ +#!/usr/bin/env bats + +load './test_helper.bash' + +@test "make target coverage manifest accounts for every Makefile target" { + local manifest_path="$PROJECT_ROOT/tests/bats/make-target-coverage.tsv" + local expected_targets="$BATS_TEST_TMPDIR/expected-targets.txt" + local manifest_targets="$BATS_TEST_TMPDIR/manifest-targets.txt" + + [ -f "$manifest_path" ] + + awk -F: '/^[A-Za-z0-9_.-]+:/{ if ($1 !~ /^\./) print $1 }' \ + "$PROJECT_ROOT/Makefile" \ + | sort -u > "$expected_targets" + + tail -n +2 "$manifest_path" | cut -f1 | sort -u > "$manifest_targets" + + run diff -u "$expected_targets" "$manifest_targets" + [ "$status" -eq 0 ] +} + +@test "every Bats-covered target points at an existing test file" { + local manifest_path="$PROJECT_ROOT/tests/bats/make-target-coverage.tsv" + + [ -f "$manifest_path" ] + + while IFS=$'\t' read -r target coverage evidence details; do + [ -n "$target" ] || continue + [ "$target" != "target" ] || continue + + [ -f "$PROJECT_ROOT/$evidence" ] + + if [ "$coverage" = "bats" ]; then + run awk -v target="$target" ' + BEGIN { found = 0 } + { + for (i = 1; i <= NF; i++) { + if ($i == target) { + found = 1 + } + } + } + END { exit(found ? 0 : 1) } + ' FS='[^A-Za-z0-9_.-]+' "$PROJECT_ROOT/$evidence" + [ "$status" -eq 0 ] + fi + done < "$manifest_path" +} + +@test "CI helper scripts only call existing Makefile targets" { + local expected_targets="$BATS_TEST_TMPDIR/known-targets.txt" + local hard_coded_targets="$BATS_TEST_TMPDIR/script-targets.txt" + local missing_targets="$BATS_TEST_TMPDIR/missing-targets.txt" + + awk -F: '/^[A-Za-z0-9_.-]+:/{ if ($1 !~ /^\./) print $1 }' \ + "$PROJECT_ROOT/Makefile" \ + | sort -u > "$expected_targets" + + grep -RhoE 'make [A-Za-z0-9_.-]+' "$PROJECT_ROOT/scripts/ci" \ + | awk '{ print $2 }' \ + | sort -u > "$hard_coded_targets" + + comm -23 "$hard_coded_targets" "$expected_targets" > "$missing_targets" + + run test ! -s "$missing_targets" + [ "$status" -eq 0 ] +} + +@test "a pull request workflow runs make test-bats with explicit read-only permissions" { + local workflows_with_bats="$BATS_TEST_TMPDIR/workflows-with-bats.txt" + local workflow_path + + grep -RFl 'make test-bats' "$PROJECT_ROOT/.github/workflows" > "$workflows_with_bats" + run test -s "$workflows_with_bats" + [ "$status" -eq 0 ] + + while IFS= read -r workflow_path; do + run grep -F 'pull_request:' "$workflow_path" + [ "$status" -eq 0 ] + + run grep -F 'contents: read' "$workflow_path" + [ "$status" -eq 0 ] + + run grep -F 'contents: write' "$workflow_path" + [ "$status" -ne 0 ] + done < "$workflows_with_bats" +} + +@test "repository docs explain how to run and maintain the Bats suite" { + run grep -F 'make test-bats' "$PROJECT_ROOT/README.md" + [ "$status" -eq 0 ] + + run grep -F 'make-target-coverage.tsv' "$PROJECT_ROOT/CONTRIBUTING.md" + [ "$status" -eq 0 ] +} diff --git a/tests/bats/make-target-coverage.tsv b/tests/bats/make-target-coverage.tsv new file mode 100644 index 00000000..b5986b3a --- /dev/null +++ b/tests/bats/make-target-coverage.tsv @@ -0,0 +1,63 @@ +target coverage evidence details +help bats tests/bats/makefile_targets.bats Validated through the help output contract. +start bats tests/bats/makefile_targets.bats Validated in CI=1 mode with a stubbed Next.js binary. +wait-for-dev bats tests/bats/makefile_targets.bats Validated with a stubbed curl readiness check. +create-temp-dev-container-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker commands and env-var guards. +copy-source-to-container-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker and tar commands. +install-deps-in-container-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker and npm/pnpm commands. +run-unit-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls. +run-mutation-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls. +run-eslint-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls. +run-typescript-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls. +run-markdown-lint-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls. +create-k6-helper-container-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker commands and env-var guards. +build-k6 bats tests/bats/makefile_targets.bats Validated with stubbed Docker Compose build calls. +run-load-tests-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker exec calls and env-var guards. +build ci .github/workflows/mutation-testing.yml Executed through `make test-mutation`, which depends on `build`. +build-analyze bats tests/bats/makefile_targets.bats Validated with stubbed Next.js build commands. +build-out bats tests/bats/makefile_targets.bats Validated in an isolated Makefile sandbox with stubbed Docker commands. +format bats tests/bats/makefile_targets.bats Validated with a stubbed pnpm invocation. +lint-next ci .github/workflows/static-testing.yml Executed through `make lint`, which depends on `lint-next`. +lint-tsc ci .github/workflows/static-testing.yml Executed through `make lint`, which depends on `lint-tsc`. +lint-md ci .github/workflows/static-testing.yml Executed through `make lint`, which depends on `lint-md`. +lint ci .github/workflows/static-testing.yml Executed directly by the static testing workflow. +husky bats tests/bats/makefile_targets.bats Validated with a stubbed pnpm invocation. +storybook-start bats tests/bats/makefile_targets.bats Validated with a stubbed Storybook binary. +storybook-build bats tests/bats/makefile_targets.bats Validated with a stubbed Storybook binary. +test-e2e ci .github/workflows/e2e-testing.yml Executed directly by the Playwright PR workflow. +test-e2e-ui bats tests/bats/makefile_targets.bats Validated with stubbed Docker and Playwright commands. +test-visual ci .github/workflows/visual-testing.yml Executed directly by the visual testing workflow. +test-visual-ui bats tests/bats/makefile_targets.bats Validated with stubbed Docker and Playwright commands. +test-visual-update bats tests/bats/makefile_targets.bats Validated with stubbed Docker and Playwright commands. +create-network ci .github/workflows/e2e-testing.yml Executed through `make test-e2e`, which calls `start-prod` and its `create-network` prerequisite. +start-prod ci .github/workflows/e2e-testing.yml Executed through `make test-e2e` and other PR workflows that depend on `start-prod`. +wait-for-prod bats tests/bats/makefile_targets.bats Validated with a stubbed curl readiness check. +test-unit-all ci .github/workflows/unit-testing.yml Executed directly by the unit testing workflow. +test-unit-client ci .github/workflows/unit-testing.yml Executed through `make test-unit-all`, which depends on `test-unit-client`. +test-unit-server ci .github/workflows/unit-testing.yml Executed through `make test-unit-all`, which depends on `test-unit-server`. +test-bats ci .github/workflows/bats-testing.yml Executed directly by the dedicated Bats PR workflow. +test-memory-leak ci .github/workflows/memory-leak-testing.yml Executed directly by the memory leak workflow. +memory-leak-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker Compose commands. +test-mutation ci .github/workflows/mutation-testing.yml Executed directly by the mutation testing workflow. +wait-for-prod-health ci .github/workflows/load-testing.yml Executed through `make load-tests` and `make load-tests-swagger`, which depend on `wait-for-prod-health`. +visual-direct bats tests/bats/makefile_targets.bats Validated with stubbed Docker and Playwright commands. +e2e-direct bats tests/bats/makefile_targets.bats Validated with stubbed Docker and Playwright commands. +all bats tests/bats/makefile_targets.bats Validated through its `build` prerequisite in the Makefile sandbox. +clean bats tests/bats/makefile_targets.bats Validated through its `down` prerequisite in the Makefile sandbox. +load-tests ci .github/workflows/load-testing.yml Executed directly by the load testing workflow matrix. +load-tests-swagger ci .github/workflows/load-testing.yml Executed directly by the load testing workflow matrix. +lighthouse-desktop ci .github/workflows/performance-testing.yml Executed directly by the performance workflow. +lighthouse-desktop-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker Compose LHCI commands. +lighthouse-mobile ci .github/workflows/performance-testing.yml Executed directly by the performance workflow. +lighthouse-mobile-dind bats tests/bats/makefile_targets.bats Validated with stubbed Docker Compose LHCI commands. +install ci .github/workflows/static-testing.yml Executed directly by the static testing workflow. +install-chromium-lhci bats tests/bats/makefile_targets.bats Validated with stubbed Docker Compose and npm commands. +test-chromium bats tests/bats/makefile_targets.bats Validated with a stubbed Chromium version check. +update bats tests/bats/makefile_targets.bats Validated with a stubbed pnpm invocation. +down bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +sh bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +ps bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +logs bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +new-logs bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +stop bats tests/bats/makefile_targets.bats Validated with a stubbed Docker Compose invocation. +check-node-version bats tests/bats/makefile_targets.bats Validated with a stubbed pnpm/node invocation. diff --git a/tests/bats/makefile_targets.bats b/tests/bats/makefile_targets.bats new file mode 100644 index 00000000..e8f36bee --- /dev/null +++ b/tests/bats/makefile_targets.bats @@ -0,0 +1,266 @@ +#!/usr/bin/env bats + +load './test_helper.bash' + +setup() { + setup_makefile_test_env +} + +@test "help lists the Bats entry point" { + run_make_target help + [ "$status" -eq 0 ] + assert_output_contains 'Usage:' + assert_output_contains 'test-bats' +} + +@test "container-backed helper targets fail fast when their required names are missing" { + local target + local required_var + + while IFS='|' read -r target required_var; do + unset "$required_var" + run_make_target "$target" + [ "$status" -ne 0 ] + assert_output_contains "Error: $required_var is required." + done <<'EOF' +create-temp-dev-container-dind|TEMP_CONTAINER_NAME +copy-source-to-container-dind|TEMP_CONTAINER_NAME +install-deps-in-container-dind|TEMP_CONTAINER_NAME +run-unit-tests-dind|TEMP_CONTAINER_NAME +run-mutation-tests-dind|TEMP_CONTAINER_NAME +run-eslint-tests-dind|TEMP_CONTAINER_NAME +run-typescript-tests-dind|TEMP_CONTAINER_NAME +run-markdown-lint-tests-dind|TEMP_CONTAINER_NAME +create-k6-helper-container-dind|K6_HELPER_NAME +run-load-tests-dind|K6_HELPER_NAME +EOF +} + +@test "DIND dev-container targets shell out through docker with the expected commands" { + reset_command_log + run_make_target create-temp-dev-container-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker rm -f website-dev-test' + assert_log_contains 'docker compose -f docker-compose.yml run -d --name website-dev-test --entrypoint sh dev -lc sleep infinity' + + reset_command_log + run_make_target copy-source-to-container-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc mkdir -p /app' + assert_log_contains 'tar -cf -' + assert_log_contains 'docker exec -i website-dev-test sh -lc tar -xf - -C /app' + + reset_command_log + run_make_target install-deps-in-container-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && npm install -g pnpm && pnpm install --frozen-lockfile' + + reset_command_log + run_make_target run-unit-tests-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && make test-unit-client CI=1' + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && make test-unit-server CI=1' + + reset_command_log + run_make_target run-mutation-tests-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && pnpm stryker run' + + reset_command_log + run_make_target run-eslint-tests-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && make lint-next CI=1' + + reset_command_log + run_make_target run-typescript-tests-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && make lint-tsc CI=1' + + reset_command_log + run_make_target run-markdown-lint-tests-dind TEMP_CONTAINER_NAME=website-dev-test + [ "$status" -eq 0 ] + assert_log_contains 'docker exec website-dev-test sh -lc cd /app && make lint-md CI=1' +} + +@test "K6 and DIND quality targets invoke the expected Docker commands" { + reset_command_log + run_make_target create-k6-helper-container-dind K6_HELPER_NAME=website-k6-helper + [ "$status" -eq 0 ] + assert_log_contains 'docker rm -f website-k6-helper' + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml --profile load run -d --name website-k6-helper --entrypoint sh k6 -lc tail -f /dev/null' + + reset_command_log + run_make_target build-k6 + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml --profile load build k6' + + reset_command_log + run_make_target run-load-tests-dind K6_HELPER_NAME=website-k6-helper + [ "$status" -eq 0 ] + assert_log_contains 'docker exec -w /loadTests website-k6-helper k6 run --summary-trend-stats=avg,min,med,max,p(95),p(99) --out web-dashboard=period=1s&export=/loadTests/results/homepage.html /loadTests/homepage.js' + + reset_command_log + run_make_target memory-leak-dind + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -p memleak -f docker-compose.memory-leak.yml up -d --wait memory-leak' + assert_log_contains 'docker compose -p memleak -f docker-compose.memory-leak.yml exec -T memory-leak rm -rf ./src/test/memory-leak/results' + assert_log_contains 'docker compose -p memleak -f docker-compose.memory-leak.yml exec -T memory-leak sh -lc unset DISPLAY;' + assert_log_contains 'docker compose -p memleak -f docker-compose.memory-leak.yml down' +} + +@test "developer convenience targets call the expected local commands" { + reset_command_log + run_make_target start CI=1 + [ "$status" -eq 0 ] + assert_log_contains 'next dev' + + reset_command_log + run_make_target wait-for-dev + [ "$status" -eq 0 ] + assert_output_contains 'Dev service is up and running!' + assert_log_contains 'curl -s -f http://localhost:3000' + + reset_command_log + run_make_target build-analyze + [ "$status" -eq 0 ] + assert_log_contains 'next build --webpack' + assert_log_contains 'next-export-optimize-images' + + reset_command_log + run_make_target build-out + [ "$status" -eq 0 ] + assert_log_contains 'docker build -t next-build -f Dockerfile --target production .' + assert_log_contains 'docker create next-build' + assert_log_contains 'docker cp fake-container-id:/app/out ./' + assert_log_contains 'docker rm fake-container-id' + + reset_command_log + run_make_target format CI=1 + [ "$status" -eq 0 ] + assert_log_contains 'pnpm ./node_modules/.bin/prettier **/*.{js,jsx,ts,tsx,json,css,scss,md} --write --ignore-path .prettierignore' + + reset_command_log + run_make_target husky + [ "$status" -eq 0 ] + assert_log_contains 'pnpm husky install' + + reset_command_log + run_make_target storybook-start CI=1 + [ "$status" -eq 0 ] + assert_log_contains 'storybook dev -p' + + reset_command_log + run_make_target storybook-build CI=1 + [ "$status" -eq 0 ] + assert_log_contains 'storybook build --output-dir storybook-static-ci' + + reset_command_log + run_make_target check-node-version CI=1 + [ "$status" -eq 0 ] + assert_log_contains 'pnpm exec node checkNodeVersion.js' + + reset_command_log + run_make_target update + [ "$status" -eq 0 ] + assert_log_contains 'pnpm update' +} + +@test "prod-side wrapper targets invoke the expected Docker and Playwright flows" { + reset_command_log + run_make_target wait-for-prod + [ "$status" -eq 0 ] + assert_output_contains 'Prod service is up and running!' + assert_log_contains 'curl -s -f http://localhost:3001' + + reset_command_log + run_make_target test-e2e-ui + [ "$status" -eq 0 ] + assert_log_contains 'docker network ls' + assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml up -d' + assert_log_contains 'docker compose -f docker-compose.test.yml ps' + assert_log_contains 'playwright test ./src/test/e2e' + assert_log_contains '--ui-port=9324 --ui-host=0.0.0.0' + + reset_command_log + run_make_target test-visual-ui + [ "$status" -eq 0 ] + assert_log_contains 'playwright test ./src/test/visual' + assert_log_contains '--ui-port=9324 --ui-host=0.0.0.0' + + reset_command_log + run_make_target test-visual-update + [ "$status" -eq 0 ] + assert_log_contains 'playwright test ./src/test/visual --update-snapshots' + + reset_command_log + run_make_target visual-direct + [ "$status" -eq 0 ] + assert_log_contains 'playwright test ./src/test/visual' + + reset_command_log + run_make_target e2e-direct + [ "$status" -eq 0 ] + assert_log_contains 'playwright test ./src/test/e2e' +} + +@test "maintenance targets shell out through Docker and pnpm as expected" { + reset_command_log + run_make_target lighthouse-desktop-dind + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -f docker-compose.test.yml exec -T -w /app prod lhci autorun --config=lighthouserc.desktop.js' + + reset_command_log + run_make_target lighthouse-mobile-dind + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -f docker-compose.test.yml exec -T -w /app prod lhci autorun --config=lighthouserc.mobile.js' + + reset_command_log + run_make_target install-chromium-lhci + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -f docker-compose.test.yml exec -T prod sh -lc apk add --no-cache chromium chromium-chromedriver && npm install -g @lhci/cli@0.14.0' + + reset_command_log + run_make_target test-chromium + [ "$status" -eq 0 ] + assert_log_contains 'docker compose -f docker-compose.test.yml exec -T prod /usr/bin/chromium-browser --version' + + reset_command_log + run_make_target down + [ "$status" -eq 0 ] + assert_log_contains 'docker compose down --remove-orphans' + + reset_command_log + run_make_target sh + [ "$status" -eq 0 ] + assert_log_contains 'docker compose exec dev sh' + + reset_command_log + run_make_target ps + [ "$status" -eq 0 ] + assert_log_contains 'docker compose ps' + + reset_command_log + run_make_target logs + [ "$status" -eq 0 ] + assert_log_contains 'docker compose logs --follow dev' + + reset_command_log + run_make_target new-logs + [ "$status" -eq 0 ] + assert_log_contains 'docker compose logs --tail=0 --follow dev' + + reset_command_log + run_make_target stop + [ "$status" -eq 0 ] + assert_log_contains 'docker compose stop' + + reset_command_log + run_make_target all + [ "$status" -eq 0 ] + assert_log_contains 'docker compose build' + + reset_command_log + run_make_target clean + [ "$status" -eq 0 ] + assert_log_contains 'docker compose down --remove-orphans' +} diff --git a/tests/bats/test_helper.bash b/tests/bats/test_helper.bash new file mode 100644 index 00000000..4ef39081 --- /dev/null +++ b/tests/bats/test_helper.bash @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +PROJECT_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/../.." >/dev/null 2>&1 && pwd)" + +setup_stub_dir() { + export STUB_BIN_DIR="$BATS_TEST_TMPDIR/bin" + export COMMAND_LOG="$BATS_TEST_TMPDIR/commands.log" + + mkdir -p "$STUB_BIN_DIR" + : > "$COMMAND_LOG" + + export PATH="$STUB_BIN_DIR:$PATH" +} + +reset_command_log() { + : > "$COMMAND_LOG" +} + +create_generic_stub() { + local name="$1" + + cat > "$STUB_BIN_DIR/$name" <<'EOF' +#!/usr/bin/env bash +printf '%s %s\n' "$(basename "$0")" "$*" >> "${COMMAND_LOG:?}" +exit 0 +EOF + + chmod +x "$STUB_BIN_DIR/$name" +} + +create_curl_stub() { + cat > "$STUB_BIN_DIR/curl" <<'EOF' +#!/usr/bin/env bash +printf 'curl %s\n' "$*" >> "${COMMAND_LOG:?}" +exit 0 +EOF + + chmod +x "$STUB_BIN_DIR/curl" +} + +create_docker_stub() { + cat > "$STUB_BIN_DIR/docker" <<'EOF' +#!/usr/bin/env bash +printf 'docker %s\n' "$*" >> "${COMMAND_LOG:?}" + +if [ "$1" = "network" ] && [ "$2" = "ls" ]; then + if [ "${FAKE_DOCKER_NETWORK_EXISTS:-0}" = "1" ]; then + printf '%s\n' "${FAKE_DOCKER_NETWORK_NAME:-website-network}" + fi + exit 0 +fi + +if [ "$1" = "create" ]; then + printf 'fake-container-id\n' + exit 0 +fi + +if [ "$1" = "compose" ]; then + for arg in "$@"; do + if [ "$arg" = "ps" ]; then + printf 'prod (healthy)\n' + exit 0 + fi + done +fi + +if [ ! -t 0 ]; then + cat >/dev/null || true +fi + +exit 0 +EOF + + chmod +x "$STUB_BIN_DIR/docker" +} + +create_make_stub() { + cat > "$STUB_BIN_DIR/make" <<'EOF' +#!/usr/bin/env bash +printf 'make %s\n' "$*" >> "${COMMAND_LOG:?}" + +target="" +for arg in "$@"; do + case "$arg" in + -*|*=*) + ;; + *) + target="$arg" + break + ;; + esac +done + +if [ -n "${FAKE_MAKE_FAIL_TARGET:-}" ] && [ "$target" = "$FAKE_MAKE_FAIL_TARGET" ]; then + exit 1 +fi + +exit 0 +EOF + + chmod +x "$STUB_BIN_DIR/make" +} + +setup_makefile_test_env() { + setup_stub_dir + + create_docker_stub + create_curl_stub + create_generic_stub npm + create_generic_stub pnpm + create_generic_stub tar + create_generic_stub next + create_generic_stub next-export-optimize-images + create_generic_stub eslint + create_generic_stub tsc + create_generic_stub storybook + create_generic_stub jest + create_generic_stub serve + create_generic_stub playwright + create_generic_stub lhci + create_generic_stub node + + export MAKEFILE_SANDBOX="$BATS_TEST_TMPDIR/makefile-sandbox" + mkdir -p "$MAKEFILE_SANDBOX" + cp "$PROJECT_ROOT/Makefile" "$MAKEFILE_SANDBOX/Makefile" + cp "$PROJECT_ROOT/.env" "$MAKEFILE_SANDBOX/.env" +} + +setup_ci_script_test_env() { + setup_stub_dir + + create_docker_stub + create_make_stub + create_generic_stub tar + + export SCRIPT_SANDBOX="$BATS_TEST_TMPDIR/script-sandbox" + mkdir -p "$SCRIPT_SANDBOX" + cp "$PROJECT_ROOT/common-healthchecks.yml" "$SCRIPT_SANDBOX/common-healthchecks.yml" +} + +run_make_target() { + local target="$1" + shift + + run env \ + PATH="$STUB_BIN_DIR:$PATH" \ + COMMAND_LOG="$COMMAND_LOG" \ + make -C "$MAKEFILE_SANDBOX" "$target" BIN_DIR="$STUB_BIN_DIR" "$@" +} + +run_ci_script() { + local script_path="$1" + shift + + run env \ + -C "$SCRIPT_SANDBOX" \ + PATH="$STUB_BIN_DIR:$PATH" \ + COMMAND_LOG="$COMMAND_LOG" \ + "$script_path" "$@" +} + +assert_log_contains() { + local expected="$1" + + if ! grep -F -- "$expected" "$COMMAND_LOG" >/dev/null 2>&1; then + echo "Expected command log to contain: $expected" >&2 + echo "--- command log ---" >&2 + cat "$COMMAND_LOG" >&2 + return 1 + fi +} + +assert_output_contains() { + local expected="$1" + local actual_output="${output-}" + + if [[ "$actual_output" != *"$expected"* ]]; then + echo "Expected output to contain: $expected" >&2 + echo "--- output ---" >&2 + printf '%s\n' "$actual_output" >&2 + return 1 + fi +}