test: add Bats coverage for uncovered Makefile targets

.github/workflows/bats-testing.yml

@@ -0,0 +1,42 @@
+name: bats testing
+
+on:
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  bats:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          persist-credentials: false
+
+      - name: Set up Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
+        with:
+          node-version: ${{ vars.NODE_VERSION }}
+
+      - name: Cache pnpm dependencies
+        id: cache-pnpm-dependencies
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-dependencies-node-${{ vars.NODE_VERSION }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-dependencies-node-${{ vars.NODE_VERSION }}-
+
+      - name: Install pnpm
+        run: npm install -g pnpm@10.6.5
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Bats shell coverage
+        run: make test-bats BATS_FORMATTER=tap

CONTRIBUTING.md

@@ -61,6 +61,17 @@ If you find an issue to work on, you are welcome to open a PR with a fix.
 
 2. Create a working branch and start with your changes!
 
+#### Maintain Makefile shell coverage
+
+If your change adds or updates a Makefile target, keep the shell-coverage inventory in
+sync:
+
+- Update `tests/bats/make-target-coverage.tsv` so every Makefile target is marked as
+  either Bats-covered or already covered by a pull request workflow.
+- If the target is not already exercised by CI, add or update the relevant test in
+  `tests/bats/`.
+- Run `make test-bats`.
+
 ### Commit your update
 
 Commit the changes once you are happy with them.

Makefile

@@ -22,6 +22,7 @@ STORYBOOK_BIN               = $(BIN_DIR)/storybook
 JEST_BIN                    = $(BIN_DIR)/jest
 SERVE_BIN                   = $(BIN_DIR)/serve
 PLAYWRIGHT_BIN              = $(BIN_DIR)/playwright
+BATS_BIN                    = pnpm exec bats
 
 NEXT_BUILD                  = $(NEXT_BIN) build --webpack
 NEXT_BUILD_CMD              = $(NEXT_BUILD) && $(IMG_OPTIMIZE)
@@ -95,6 +96,7 @@ UI_MODE_URL                 = http://$(WEBSITE_DOMAIN):$(PLAYWRIGHT_TEST_PORT)
 MD_LINT_ARGS                = -i CHANGELOG.md -i "test-results/**/*.md" -i "playwright-report/data/**/*.md" -i "node_modules/**/*.md"
 
 JEST_FLAGS                  = --verbose
+BATS_FORMATTER              ?= pretty
 
 NETWORK_NAME                = website-network
 
@@ -305,6 +307,13 @@ test-unit-client: ## Run all client-side unit tests using Jest (Next.js env, TES
 test-unit-server: ## Run server-side unit tests for Apollo using Jest (Node.js env, TEST_ENV=server, target: $(TEST_DIR_APOLLO))
 	$(UNIT_TESTS) TEST_ENV=server $(JEST_BIN) $(JEST_FLAGS) $(TEST_DIR_APOLLO)
 
+test-bats: ## Run Bats coverage for Makefile shell flows and CI helper scripts
+	DOCKER_COMPOSE_TEST_FILE=docker-compose.test.yml \
+	DOCKER_COMPOSE_DEV_FILE=docker-compose.yml \
+	COMMON_HEALTHCHECKS_FILE=common-healthchecks.yml \
+	DOCKER_COMPOSE_MEMLEAK_FILE=docker-compose.memory-leak.yml \
+	$(BATS_BIN) --formatter $(BATS_FORMATTER) -r tests/bats
+
 test-memory-leak: start-prod ## This command executes memory leaks tests using Memlab library.
 	@echo "🧪 Starting memory leak test environment..."
 	$(DOCKER_COMPOSE) $(DOCKER_COMPOSE_MEMLEAK_FILE) up -d

README.md

@@ -106,6 +106,7 @@ Testing
   make test-unit-all: runs unit tests for both client and server environments
   make test-unit-client: runs unit tests for the client using Jest
   make test-unit-server: runs unit tests for the server using Jest
+  make test-bats: runs the Bats shell regression suite for Makefile targets and CI helper scripts
   make test-memory-leak: runs memory leak tests using Memlab
   make load-tests: executes load tests using the K6 library
   make test-e2e: runs end-to-end tests inside the prod container
@@ -183,6 +184,20 @@ Example:
   CI=1 make start
 ```
 
+### Bats Shell Coverage
+
+Use the Bats suite to validate Makefile shell flows and `scripts/ci` helpers that are
+not already exercised by the pull request workflows:
+
+```bash
+  make test-bats
+```
+
+The coverage inventory lives in `tests/bats/make-target-coverage.tsv`. When you add a
+new Makefile target, either add Bats coverage for it or document the workflow that
+already exercises it in that file. Additional suite-maintenance notes live in
+`tests/bats/README.md`.
+
 ### Load Testing with K6
 
 This project includes a dedicated load testing service using K6, configured via a Docker Compose profile.

package.json

@@ -59,6 +59,7 @@
     "@typescript-eslint/parser": "^8.60.0",
     "babel-jest": "30.2.0",
     "babel-preset-jest": "30.2.0",
+    "bats": "^1.13.0",
     "eslint": "^9.39.4",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^18.0.0",

scripts/ci/batch_pw_load.sh

@@ -45,7 +45,6 @@ setup_docker_network() {
 
 start_prod_dind() {
     setup_docker_network
-    make build-prod
     docker compose ${COMPOSE_ARGS} up -d --wait prod
 }
 run_make_with_prod_dind() {
@@ -146,4 +145,4 @@ case "${1:-all}" in
     *)
         main "$@"
         ;;
-    esac
+    esac

tests/bats/README.md

@@ -0,0 +1,25 @@
+# Bats Suite
+
+This suite covers Makefile targets and `scripts/ci` helper flows that are not already
+exercised by the pull request workflows.
+
+## Run locally
+
+```bash
+make test-bats
+```
+
+Use `BATS_FORMATTER=tap` when you want CI-style output:
+
+```bash
+make test-bats BATS_FORMATTER=tap
+```
+
+## Maintain coverage
+
+1. Update `tests/bats/make-target-coverage.tsv` whenever a Makefile target is added,
+   removed, or reclassified.
+2. If a target is not already exercised by a PR workflow, add or update the Bats test
+   that covers its shell behavior.
+3. If a workflow already exercises the target, document that workflow in
+   `tests/bats/make-target-coverage.tsv` instead of duplicating the coverage.

tests/bats/ci_scripts.bats

@@ -0,0 +1,87 @@
+#!/usr/bin/env bats
+
+load './test_helper.bash'
+
+setup() {
+  setup_ci_script_test_env
+}
+
+@test "batch_unit_mutation_lint.sh dispatches each DIND flow through existing make targets" {
+  local script_path="$PROJECT_ROOT/scripts/ci/batch_unit_mutation_lint.sh"
+
+  run_ci_script "$script_path" test-unit
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make build'
+  assert_log_contains 'make create-temp-dev-container-dind TEMP_CONTAINER_NAME=website-dev-test'
+  assert_log_contains 'make copy-source-to-container-dind TEMP_CONTAINER_NAME=website-dev-test'
+  assert_log_contains 'make install-deps-in-container-dind TEMP_CONTAINER_NAME=website-dev-test'
+  assert_log_contains 'make run-unit-tests-dind TEMP_CONTAINER_NAME=website-dev-test'
+
+  reset_command_log
+  run_ci_script "$script_path" test-mutation
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make run-mutation-tests-dind TEMP_CONTAINER_NAME=website-dev-test'
+
+  reset_command_log
+  run_ci_script "$script_path" test-lint
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make run-eslint-tests-dind TEMP_CONTAINER_NAME=website-dev-lint'
+  assert_log_contains 'make run-typescript-tests-dind TEMP_CONTAINER_NAME=website-dev-lint'
+  assert_log_contains 'make run-markdown-lint-tests-dind TEMP_CONTAINER_NAME=website-dev-lint'
+}
+
+@test "batch_pw_load.sh dispatches its E2E, visual, and load flows through make and docker" {
+  local script_path="$PROJECT_ROOT/scripts/ci/batch_pw_load.sh"
+
+  run_ci_script "$script_path" test-e2e
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make start-prod'
+  assert_log_contains 'make test-e2e'
+  assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml exec -T playwright mkdir -p /app'
+  assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml cp playwright:/app/playwright-report/. playwright-report/'
+
+  reset_command_log
+  run_ci_script "$script_path" test-visual
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make start-prod'
+  assert_log_contains 'make test-visual'
+  assert_log_contains 'docker compose -f common-healthchecks.yml -f docker-compose.test.yml exec -T playwright mkdir -p /app/src/test /app/src/config /app/pages/i18n'
+
+  reset_command_log
+  run_ci_script "$script_path" test-load
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make start-prod'
+  assert_log_contains 'make build-k6'
+  assert_log_contains 'make create-k6-helper-container-dind K6_HELPER_NAME=website-k6-helper'
+  assert_log_contains 'make run-load-tests-dind K6_HELPER_NAME=website-k6-helper'
+  assert_log_contains 'docker cp src/test/load/. website-k6-helper:/loadTests/'
+}
+
+@test "batch_lhci_leak.sh handles CodeBuild skips and the DIND Lighthouse flows" {
+  local script_path="$PROJECT_ROOT/scripts/ci/batch_lhci_leak.sh"
+
+  run env \
+    -C "$SCRIPT_SANDBOX" \
+    PATH="$STUB_BIN_DIR:$PATH" \
+    COMMAND_LOG="$COMMAND_LOG" \
+    CODEBUILD_BUILD_ID=website:1 \
+    "$script_path" test-memory-leak
+  [ "$status" -eq 0 ]
+  assert_output_contains 'Memory leak tests: SKIPPED'
+
+  reset_command_log
+  run_ci_script "$script_path" test-lighthouse-desktop
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make start-prod'
+  assert_log_contains 'make install-chromium-lhci'
+  assert_log_contains 'make test-chromium'
+  assert_log_contains 'make lighthouse-desktop-dind'
+
+  reset_command_log
+  run_ci_script "$script_path" test-lighthouse-mobile
+  [ "$status" -eq 0 ]
+  assert_log_contains 'make start-prod'
+  assert_log_contains 'make install-chromium-lhci'
+  assert_log_contains 'make test-chromium'
+  assert_log_contains 'make lighthouse-mobile-dind'
+}

tests/bats/issue_175_contract.bats

@@ -0,0 +1,95 @@
+#!/usr/bin/env bats
+
+load './test_helper.bash'
+
+@test "make target coverage manifest accounts for every Makefile target" {
+  local manifest_path="$PROJECT_ROOT/tests/bats/make-target-coverage.tsv"
+  local expected_targets="$BATS_TEST_TMPDIR/expected-targets.txt"
+  local manifest_targets="$BATS_TEST_TMPDIR/manifest-targets.txt"
+
+  [ -f "$manifest_path" ]
+
+  awk -F: '/^[A-Za-z0-9_.-]+:/{ if ($1 !~ /^\./) print $1 }' \
+    "$PROJECT_ROOT/Makefile" \
+    | sort -u > "$expected_targets"
+
+  tail -n +2 "$manifest_path" | cut -f1 | sort -u > "$manifest_targets"
+
+  run diff -u "$expected_targets" "$manifest_targets"
+  [ "$status" -eq 0 ]
+}
+
+@test "every Bats-covered target points at an existing test file" {
+  local manifest_path="$PROJECT_ROOT/tests/bats/make-target-coverage.tsv"
+
+  [ -f "$manifest_path" ]
+
+  while IFS=$'\t' read -r target coverage evidence details; do
+    [ -n "$target" ] || continue
+    [ "$target" != "target" ] || continue
+
+    [ -f "$PROJECT_ROOT/$evidence" ]
+
+    if [ "$coverage" = "bats" ]; then
+      run awk -v target="$target" '
+        BEGIN { found = 0 }
+        {
+          for (i = 1; i <= NF; i++) {
+            if ($i == target) {
+              found = 1
+            }
+          }
+        }
+        END { exit(found ? 0 : 1) }
+      ' FS='[^A-Za-z0-9_.-]+' "$PROJECT_ROOT/$evidence"
+      [ "$status" -eq 0 ]
+    fi
+  done < "$manifest_path"
+}
+
+@test "CI helper scripts only call existing Makefile targets" {
+  local expected_targets="$BATS_TEST_TMPDIR/known-targets.txt"
+  local hard_coded_targets="$BATS_TEST_TMPDIR/script-targets.txt"
+  local missing_targets="$BATS_TEST_TMPDIR/missing-targets.txt"
+
+  awk -F: '/^[A-Za-z0-9_.-]+:/{ if ($1 !~ /^\./) print $1 }' \
+    "$PROJECT_ROOT/Makefile" \
+    | sort -u > "$expected_targets"
+
+  grep -RhoE 'make [A-Za-z0-9_.-]+' "$PROJECT_ROOT/scripts/ci" \
+    | awk '{ print $2 }' \
+    | sort -u > "$hard_coded_targets"
+
+  comm -23 "$hard_coded_targets" "$expected_targets" > "$missing_targets"
+
+  run test ! -s "$missing_targets"
+  [ "$status" -eq 0 ]
+}
+
+@test "a pull request workflow runs make test-bats with explicit read-only permissions" {
+  local workflows_with_bats="$BATS_TEST_TMPDIR/workflows-with-bats.txt"
+  local workflow_path
+
+  grep -RFl 'make test-bats' "$PROJECT_ROOT/.github/workflows" > "$workflows_with_bats"
+  run test -s "$workflows_with_bats"
+  [ "$status" -eq 0 ]
+
+  while IFS= read -r workflow_path; do
+    run grep -F 'pull_request:' "$workflow_path"
+    [ "$status" -eq 0 ]
+
+    run grep -F 'contents: read' "$workflow_path"
+    [ "$status" -eq 0 ]
+
+    run grep -F 'contents: write' "$workflow_path"
+    [ "$status" -ne 0 ]
+  done < "$workflows_with_bats"
+}
+
+@test "repository docs explain how to run and maintain the Bats suite" {
+  run grep -F 'make test-bats' "$PROJECT_ROOT/README.md"
+  [ "$status" -eq 0 ]
+
+  run grep -F 'make-target-coverage.tsv' "$PROJECT_ROOT/CONTRIBUTING.md"
+  [ "$status" -eq 0 ]
+}