diff --git a/.github/scripts/deploy-docs-test.sh b/.github/scripts/deploy-docs-test.sh new file mode 100755 index 000000000..1f87047db --- /dev/null +++ b/.github/scripts/deploy-docs-test.sh @@ -0,0 +1,510 @@ +#!/usr/bin/env bash +# +# deploy-docs-test.sh — end-to-end test of deploy-docs.sh against a local S3. +# +# AWS_ENDPOINT_URL=http://localhost:5000 .github/scripts/deploy-docs-test.sh [build_dir] +# +# build_dir `npm run build`'s output. The test deploys that real site. With +# no argument it synthesises a small Docusaurus-shaped one. +# +# The endpoint is a moto server (motoserver/moto), which serves S3 and +# CloudFront. build_and_check.yml runs one as a service and this against the +# PR's own build. Locally: +# +# docker run -d -p 127.0.0.1:5000:5000 motoserver/moto:5.2.3 +# +# It replays the deploys that broke production, and the weeks after them, with +# DEPLOY_NOW standing in for the clock. The property under test: no asset a +# live, cached or open page can still reference is ever deleted, and every +# asset nothing can reference any more eventually is. + +set -euo pipefail +export LC_ALL=C +# No `| head` anywhere below: under pipefail, head closing the pipe early fails +# the writer with SIGPIPE, and whether it does depends on timing. sed -n 1p +# reads to the end. + +: "${AWS_ENDPOINT_URL:?point AWS_ENDPOINT_URL at a moto server}" +export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-testing} +export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-testing} +export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1} +export AWS_PAGER= + +here=$(cd "$(dirname "$0")" && pwd) +deploy="$here/deploy-docs.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +# moto differs from S3 in two ways the prune depends on, so a shim in front of +# the aws CLI puts S3's behaviour back for delete-objects: +# - S3 rejects more than 1000 keys in one call; moto takes any number. +# - In quiet mode S3 returns only the keys that failed, so on success the CLI +# prints nothing at all; moto lists every key it deleted. +# Without it, a prune that sent one oversized batch, or choked on an empty +# response, would pass here and fail in production. +REAL_AWS=$(command -v aws) +export REAL_AWS +mkdir -p "$tmp/bin" +# It also logs each call when AWS_CALL_LOG is set, so the test can check the +# order a deploy runs its steps in; and when a call matches AWS_FAIL_MATCH it +# makes the call, then fails it, the way a sync that dies after uploading does. +cat > "$tmp/bin/aws" << 'SHIM' +#!/usr/bin/env bash +set -euo pipefail +if [ -n "${AWS_CALL_LOG:-}" ]; then + echo "$*" >> "$AWS_CALL_LOG" +fi +# CloudFront, as the deploy sees it: the status in CloudFront's own casing +# ("Completed"; moto says "COMPLETED"), "InProgress" for the first +# AWS_INVALIDATION_PENDING polls, and AccessDenied when AWS_INVALIDATION_DENY +# is set. +if [ "${1:-} ${2:-}" = "cloudfront get-invalidation" ]; then + if [ -n "${AWS_INVALIDATION_DENY:-}" ]; then + echo "An error occurred (AccessDenied) when calling the GetInvalidation operation: User is not authorized to perform: cloudfront:GetInvalidation" >&2 + exit 254 + fi + status=$("$REAL_AWS" "$@") + polls=$(($(cat "$AWS_INVALIDATION_POLLS" 2> /dev/null || echo 0) + 1)) + echo "$polls" > "$AWS_INVALIDATION_POLLS" + if [ "$polls" -le "${AWS_INVALIDATION_PENDING:-0}" ]; then + echo InProgress + elif [ "$(printf '%s' "$status" | tr '[:upper:]' '[:lower:]')" = completed ]; then + echo Completed + else + echo "$status" + fi + exit 0 +fi +if [ -n "${AWS_FAIL_MATCH:-}" ] && [[ "$*" =~ $AWS_FAIL_MATCH ]]; then + "$REAL_AWS" "$@" || true + echo "injected failure: $*" >&2 + exit 1 +fi +if [ "${1:-}" != s3api ] || [ "${2:-}" != delete-objects ]; then + exec "$REAL_AWS" "$@" +fi +req= +prev= +for a in "$@"; do + [ "$prev" = --delete ] && req=${a#file://} + prev=$a +done +if [ "$(jq '.Objects | length' "$req")" -gt 1000 ]; then + echo "An error occurred (MalformedXML) when calling the DeleteObjects operation: The XML you provided was not well-formed or did not validate against our published schema" >&2 + exit 254 +fi +out=$("$REAL_AWS" "$@") +if [ "$(jq '.Quiet // false' "$req")" = true ]; then + printf '%s' "$out" | jq -e '(.Errors // []) | length > 0' > /dev/null 2>&1 && + printf '%s\n' "$out" | jq '{Errors}' + exit 0 +fi +printf '%s\n' "$out" +SHIM +chmod +x "$tmp/bin/aws" +# sleep, logged: the invalidation wait is checked by what the deploy slept, not +# by wall-clock time, which a slow runner would make flaky. +REAL_SLEEP=$(command -v sleep) +export REAL_SLEEP +cat > "$tmp/bin/sleep" << 'SHIM' +#!/usr/bin/env bash +if [ -n "${SLEEP_LOG:-}" ]; then + echo "$*" >> "$SLEEP_LOG" +fi +exec "$REAL_SLEEP" "$@" +SHIM +chmod +x "$tmp/bin/sleep" +export PATH="$tmp/bin:$PATH" +export AWS_INVALIDATION_POLLS=$tmp/invalidation-polls +# Enough for deploy 1's one slow poll; short enough that a deploy that never +# sees "Completed" fails this test in seconds, not after 15 minutes. +export DOCS_INVALIDATION_WAIT_SECONDS=30 + +MANIFESTS=docs/_deploy/asset-manifests +ASSET_CACHE='public, max-age=31536000, immutable' +PAGE_CACHE='public, max-age=0, s-maxage=86400, must-revalidate' +STATIC_CACHE='public, max-age=3600, s-maxage=86400' + +failures=0 +pass() { echo " ok $*"; } +fail() { + echo " FAIL $*" + failures=$((failures + 1)) +} +check() { # check + local what=$1 + shift + if "$@"; then pass "$what"; else fail "$what"; fi +} + +for _ in $(seq 30); do + aws s3api list-buckets > /dev/null 2>&1 && break + sleep 1 +done + +# ---------------------------------------------------------------- the sites + +# v1: the build as main.yml stages it, with docs/ at the root. +mkdir -p "$tmp/v1" +if [ "$#" -ge 1 ]; then + cp -R "$1" "$tmp/v1/docs" +else + d="$tmp/v1/docs" + mkdir -p "$d/assets/js" "$d/assets/css" "$d/assets/fonts" "$d/assets/images" \ + "$d/server/installation" "$d/img" + printf '%s\n' '' \ + '' > "$d/index.html" + printf '%s\n' '' \ + '' \ + 'x' > "$d/server/installation/index.html" + echo '# Install — “quoted”' > "$d/server/installation.md" + echo '# Keploy — “home”' > "$d/index.md" + echo '# Keploy — API tests · “récord”' > "$d/llms.txt" + echo '' > "$d/sitemap.xml" + echo '' > "$d/img/logo.svg" + echo '' > "$d/img/old-diagram.svg" + echo 'runtime' > "$d/assets/js/runtime~main.11111111.js" + echo 'body{}' > "$d/assets/css/styles.22222222.css" + echo 'font' > "$d/assets/fonts/inter-33333333.woff2" + echo 'img' > "$d/assets/images/arch-0123456789abcdef0123456789abcdef.webp" + # More than one listing page (1000 keys), so the prune's listing must + # paginate to see them all — and, once v2 renames them, more than one + # delete-objects batch. + for i in $(seq 1100); do + printf 'chunk %s' "$i" > "$d/assets/js/$(printf '%08x' "$i").$(printf '%08x' $((i * 7919))).js" + done +fi +v1=$tmp/v1 + +# Everything under assets/ is served as immutable, so it must all be +# content-hashed. Enforced on the real build too: an un-hashed file here would +# be cached for a year under a name the next deploy reuses. +unhashed=$(cd "$v1" && find docs/assets -type f | sed -E 's#.*/##; s/\.(map|LICENSE\.txt)$//' | + { grep -vE '[.-][0-9a-f]{8,}\.[A-Za-z0-9]+$' || true; }) +check "every file under docs/assets/ is content-hashed${unhashed:+: $unhashed}" test -z "$unhashed" +# deploy-docs.sh sorts files into cache classes by extension, and aws s3 sync +# filters are case-sensitive: a page.HTML would be cached as a static file. +upper=$(cd "$v1" && find docs -path docs/assets -prune -o -type f -print | + { grep -iE '\.(html|xml|json|txt|md)$' || true; } | { grep -vE '\.(html|xml|json|txt|md)$' || true; }) +check "pages and text files use lowercase extensions${upper:+: $upper}" test -z "$upper" + +# v2: the next build, after something like a Docusaurus upgrade — EVERY chunk +# renamed — and with one page, one markdown copy and one image removed and a +# page added. One file of each class goes, so each sync pass's --delete is +# exercised. +cp -R "$v1" "$tmp/v2" +v2=$tmp/v2 +(cd "$v2/docs/assets/js" && for f in *; do mv "$f" "v2-$f"; done) +# ...and its pages point at the renamed chunks, as a real build's would. +find "$v2/docs" -path "$v2/docs/assets" -prune -o -type f -name '*.html' -print0 | + xargs -0 perl -pi -e 's#/docs/assets/js/#/docs/assets/js/v2-#g' +gone_asset=$(cd "$v1" && find docs/assets/js -type f -name '*.js' | sort | sed -n 1p) +new_asset="docs/assets/js/v2-${gone_asset##*/}" +# Picked from the build, so a PR that moves some page cannot break this test. +gone_page=$(cd "$v1" && find docs -path docs/assets -prune -o -type f -name index.html -print | + { grep -vx docs/index.html || true; } | sort | tail -n 1) +gone_md=$(cd "$v1" && find docs -path docs/assets -prune -o -type f -name '*.md' -print | sort | sed -n 1p) +gone_static=$(cd "$v1" && find docs -path docs/assets -prune -o -type f -name '*.svg' -print | sort | sed -n 1p) +rm "$v2/$gone_page" "$v2/$gone_md" "$v2/$gone_static" +new_page=docs/new-page/index.html +mkdir -p "$v2/docs/new-page" +echo 'new' > "$v2/$new_page" + +# v2a: the build deploy 2 ships and then fails partway through. It is v2 plus +# one chunk only it has, loaded by its new page; the re-run ships plain v2. So +# d2's pages go live, are replaced a day later, and the chunk they loaded must +# outlive them by the retention window although d2 never completed. +cp -R "$v2" "$tmp/v2a" +v2a=$tmp/v2a +d2_asset=docs/assets/js/d2only.0badf00d.js +echo 'd2 only' > "$v2a/$d2_asset" +printf '\n' "$d2_asset" > "$v2a/$new_page" + +# ---------------------------------------------------------------- helpers + +bucket="docs-e2e-$$-$RANDOM" +aws s3api create-bucket --bucket "$bucket" > /dev/null +CLOUDFRONT_DISTRIBUTION_ID=$(aws cloudfront create-distribution --origin-domain-name "$bucket.s3.amazonaws.com" \ + --query 'Distribution.Id' --output text) +export CLOUDFRONT_DISTRIBUTION_ID +dist=$CLOUDFRONT_DISTRIBUTION_ID + +keys() { # every key under a prefix, sorted + aws s3api list-objects-v2 --bucket "$bucket" --prefix "${1:-}" \ + --query 'Contents[].Key' --output json | jq -r '.[]?' | sort +} +site_keys() { (cd "$1" && find docs -type f) | sort; } +# The bucket's files, and a site's, outside docs/assets and the manifests. +bucket_pages() { keys docs/ | grep -vE '^docs/(assets|_deploy)/' || true; } +site_pages() { site_keys "$1" | grep -v '^docs/assets/' || true; } +has() { aws s3api head-object --bucket "$bucket" --key "$1" > /dev/null 2>&1; } +lacks() { ! has "$1"; } +all_lack() { for k in "$@"; do lacks "$k" || return 1; done; } +header() { aws s3api head-object --bucket "$bucket" --key "$1" --query "$2" --output text; } +header_is() { [ "$(header "$1" "$2")" = "$3" ]; } +header_matches() { header "$1" "$2" | grep -qE "$3"; } +pages_are() { diff <(site_pages "$1") <(bucket_pages) > /dev/null; } +asset_keys_of() { for s in "$@"; do (cd "$s" && find docs/assets -type f); done; } +assets_are() { # the bucket's assets are exactly those of the given sites + diff <(asset_keys_of "$@" | sort -u) <(keys docs/assets/) > /dev/null +} +assets_are_plus() { # assets_are, plus one key that is not in any site + local extra=$1 + shift + diff <({ asset_keys_of "$@"; echo "$extra"; } | sort -u) <(keys docs/assets/) > /dev/null +} +manifests() { keys "$MANIFESTS/" | sed 's#.*/##' | tr '\n' ' ' | sed 's/ $//'; } +sleeps() { { cat "$tmp/sleeps" 2> /dev/null || true; } | tr '\n' ' ' | sed 's/ $//'; } +# The docs/assets keys the pages under a directory load: same-site URLs only, +# so github.com/keploy/docs/assets/... attachment links are not mistaken for +# ours. +refs() { + grep -rhoE '(src|href|content)="(https://keploy\.io)?/docs/assets/[^"]+' --include='*.html' "$1" | + sed -E 's#^[a-z]+="(https://keploy\.io)?/##; s/[?#].*//' | sort -u +} +refs_resolve() { # every asset the pages under a directory load is in the bucket + local want + want=$(refs "$1" || true) + # An empty set would make this pass vacuously. + [ -n "$want" ] && [ -z "$(comm -23 <(printf '%s\n' "$want") <(keys docs/assets/))" ] +} +live_refs_resolve() { # ...and so does every page the bucket serves right now + rm -rf "$tmp/live" + aws s3 sync "s3://$bucket/docs" "$tmp/live/docs" --exclude '*' --include '*.html' --only-show-errors + refs_resolve "$tmp/live" +} +# Line of the first logged aws call matching a pattern (0 if none). +call_at() { grep -nE -m 1 -- "$1" "$tmp/calls" | cut -d: -f1 || echo 0; } +steps_in_order() { + local assets manifest static txt md pages static_delete invalidate done_marker + assets=$(call_at '^s3 sync [^ ]+/docs/assets s3://') + manifest=$(call_at '^s3 cp [^ ]+ s3://[^ ]+/asset-manifests/[^ ]+\.txt') + static=$(call_at '^s3 sync [^ ]+ s3://[^ ]+ --exclude \*\.html') + txt=$(call_at '^s3 sync .* --include \*\.txt') + md=$(call_at '^s3 sync .* --include \*\.md') + pages=$(call_at '^s3 sync .* --include \*\.html') + static_delete=$(call_at '^s3 sync [^ ]+ s3://[^ ]+ --delete --exclude \*\.html') + invalidate=$(call_at '^cloudfront create-invalidation') + done_marker=$(call_at '^s3 cp - s3://[^ ]+/asset-manifests/[^ ]+\.done') + echo " assets=$assets manifest=$manifest static=$static txt=$txt md=$md pages=$pages" \ + "static-delete=$static_delete invalidate=$invalidate done=$done_marker" + [ "$assets" -gt 0 ] && [ "$assets" -lt "$manifest" ] && [ "$manifest" -lt "$static" ] && + [ "$static" -lt "$pages" ] && [ "$txt" -lt "$pages" ] && [ "$md" -lt "$pages" ] && + [ "$manifest" -lt "$txt" ] && [ "$manifest" -lt "$md" ] && + [ "$pages" -lt "$invalidate" ] && [ "$invalidate" -lt "$static_delete" ] && + [ "$static_delete" -lt "$done_marker" ] +} +invalidations() { + aws cloudfront list-invalidations --distribution-id "$dist" --output json | + jq '.InvalidationList.Items // [] | length' +} +run_deploy() { # run_deploy + echo " -- deploy $3 at DEPLOY_NOW=$2" + : > "$tmp/calls" + rm -f "$AWS_INVALIDATION_POLLS" "$tmp/sleeps" + DEPLOY_NOW=$2 AWS_CALL_LOG=$tmp/calls SLEEP_LOG=$tmp/sleeps "$deploy" "$1" "$bucket" "$3" > "$tmp/out" 2>&1 || { + cat "$tmp/out" + fail "deploy $3 exited non-zero" + return 1 + } + sed 's/^/ | /' "$tmp/out" + # A shell error the script survived is still a bug: it means some value was + # not what the code assumed, and the next change may not be so lucky. + if grep -qE 'integer expression expected|syntax error|unbound variable|command not found|No such file' "$tmp/out"; then + fail "deploy $3 printed a shell error" + fi +} + +DAY=86400 +HOUR=3600 +T0=1790000000 + +# The bucket as the old s3-deploy action left it: every object carrying the +# malformed "max-age=public," header, plus files a newer build no longer has. +aws s3 sync "$v1" "s3://$bucket" --cache-control 'max-age=public,' --only-show-errors +pre_asset=docs/assets/js/pre-manifest.deadbeef.js +echo old | aws s3 cp - "s3://$bucket/$pre_asset" --only-show-errors +orphan_page=docs/retired/index.html +echo old | aws s3 cp - "s3://$bucket/$orphan_page" --only-show-errors +# A CI build is always newer than the bucket it deploys over, and sync uploads +# a file only when it is newer than the object (or a different size). So stamp +# the builds after the seed, as a real build is — or the header checks below +# would pass or fail on the order this test happened to write files in. +sleep 1 +find "$v1" "$v2" "$v2a" -type f -exec touch {} + + +sample_asset=$(cd "$v1" && find docs/assets/js -type f -name '*.js' | sort | tail -n 1) +sample_css=$(cd "$v1" && find docs/assets/css -type f -name '*.css' | sed -n 1p) +sample_font=$(cd "$v1" && find docs/assets -type f -name '*.woff2' | sed -n 1p) +sample_md=$(cd "$v2" && find docs -path docs/assets -prune -o -type f -name '*.md' -print | sort | sed -n 1p) +sample_static=$(cd "$v2" && find docs -path docs/assets -prune -o -type f -name '*.svg' -print | sort | sed -n 1p) + +# ---------------------------------------------------------------- scenarios + +echo "== deploy 1: v1 over the old action's bucket, its invalidation slow to land" +export AWS_INVALIDATION_PENDING=1 +run_deploy "$v1" "$T0" d1 +unset AWS_INVALIDATION_PENDING +check "outside docs/assets the bucket holds exactly v1" pages_are "$v1" +check "a page v1 no longer has is deleted" lacks "$orphan_page" +check "v1's assets are all there, and the pre-manifest one is kept: history is shorter than the window" \ + assets_are_plus "$pre_asset" "$v1" +check "assets are cached as immutable (and the old malformed header is gone)" \ + header_is "$sample_asset" CacheControl "$ASSET_CACHE" +check "pages revalidate on every load" header_is docs/index.html CacheControl "$PAGE_CACHE" +check "the sitemap revalidates too" header_is docs/sitemap.xml CacheControl "$PAGE_CACHE" +check "llms.txt revalidates" header_is docs/llms.txt CacheControl "$PAGE_CACHE" +check "llms.txt is text/plain in UTF-8, not Latin-1 by default" \ + header_is docs/llms.txt ContentType 'text/plain; charset=utf-8' +check "a page's .md copy is text/markdown in UTF-8" \ + header_is "$sample_md" ContentType 'text/markdown; charset=utf-8' +check "a static image is cached for an hour, not revalidated on every load" \ + header_is "$sample_static" CacheControl "$STATIC_CACHE" +check "html is served as text/html" header_matches docs/index.html ContentType '^text/html' +check "js is served as javascript" header_matches "$sample_asset" ContentType 'javascript' +check "css is served as text/css" header_matches "$sample_css" ContentType '^text/css' +check "svg is served as image/svg+xml" header_matches "$sample_static" ContentType '^image/svg\+xml' +if [ -n "$sample_font" ]; then + check "woff2 is served as a font" header_matches "$sample_font" ContentType '^font/woff2' +fi +check "the manifest lists exactly v1's assets" \ + diff <(cd "$v1" && find docs/assets -type f | sort) \ + <(aws s3 cp "s3://$bucket/$MANIFESTS/$T0-d1.txt" - --only-show-errors) +check "the manifest is marked complete" test "$(manifests)" = "$T0-d1.done $T0-d1.txt" +check "CloudFront is invalidated once" test "$(invalidations)" = 1 +check "assets, manifest, static, text, pages last; then invalidation, static deletions and the marker" \ + steps_in_order +check "every asset a live page loads is in the bucket" live_refs_resolve +# Counting polls is not enough: a loop that stopped sleeping between them +# would burn the whole wait in a second and fall to the "not landed" path. +check "it slept 10s between the InProgress poll and the next" test "$(sleeps)" = 10 +check "static deletions waited for CloudFront to report the invalidation Completed" \ + test "$(sed -n "1,$(call_at '^s3 sync [^ ]+ s3://[^ ]+ --delete --exclude \*\.html')p" "$tmp/calls" | + grep -c '^cloudfront get-invalidation')" = 2 + +echo "== deploy 2: v2a an hour later, dying once its pages are up" +if DEPLOY_NOW=$((T0 + HOUR)) AWS_FAIL_MATCH='--include \*\.html' "$deploy" "$v2a" "$bucket" d2 > "$tmp/out" 2>&1; then + fail "a deploy whose page sync fails exits 0" +else + pass "a deploy whose page sync fails exits non-zero" +fi +check "the failure was the injected one, in the page sync" grep -q 'injected failure: s3 sync' "$tmp/out" +check "its manifest is written but not marked complete" \ + test "$(manifests)" = "$T0-d1.done $T0-d1.txt $((T0 + HOUR))-d2.txt" +check "its pages went live" has "$new_page" +check "v1's chunks all survive it, and d2's are all up" assets_are_plus "$pre_asset" "$v1" "$v2a" +check "every page left live loads only assets in the bucket" live_refs_resolve + +echo "== deploy 2, re-run: a day later, succeeding" +run_deploy "$v2" $((T0 + DAY)) d2r +check "every v1 chunk is still served, for v1 pages still open" has "$gone_asset" +check "and d2's chunk, for d2 pages still open" has "$d2_asset" +check "v2's chunks are served" has "$new_asset" +check "outside docs/assets the bucket holds exactly v2" pages_are "$v2" +check "the removed page, markdown copy and image are deleted" \ + all_lack "$gone_page" "$gone_md" "$gone_static" +check "the added page is served" has "$new_page" +check "CloudFront is invalidated again, once" test "$(invalidations)" = 2 +check "every asset a live v2 page loads is in the bucket" live_refs_resolve +check "and every asset a v1 page still open in a tab loads" refs_resolve "$v1" + +echo "== deploy 3: v2, 14 days and 2 hours after deploy 1" +# The window starts at T0+2h. d2 (T0+1h) is before it, but it failed, so it +# cannot be the boundary: that is d1, the newest COMPLETED deploy before the +# window. v1's chunks stay, and so do d2's — its pages were live until d2r, a +# day in, whether or not d2 ever finished. +run_deploy "$v2" $((T0 + 14 * DAY + 2 * HOUR)) d3 +check "v1's and d2's chunks are kept: a failed deploy is not a boundary, but its assets count" \ + assets_are "$v1" "$v2a" +check "the pre-manifest asset is deleted: d1 replaced it before the window began" lacks "$pre_asset" +check "a v1 page open since d2r still loads" refs_resolve "$v1" +check "every manifest from d1 on is kept" \ + test "$(manifests)" = "$T0-d1.done $T0-d1.txt $((T0 + HOUR))-d2.txt $((T0 + DAY))-d2r.done $((T0 + DAY))-d2r.txt $((T0 + 14 * DAY + 2 * HOUR))-d3.done $((T0 + 14 * DAY + 2 * HOUR))-d3.txt" + +echo "== deploy 4: v2, 15 days and 1 hour after deploy 1" +# The window starts at T0+1d+1h: d2r is now the boundary, and it replaced +# every v1 and d2 page more than 14 days ago. +run_deploy "$v2" $((T0 + 15 * DAY + HOUR)) d4 +check "every v1-only and d2-only chunk is deleted — more than one delete batch's worth" assets_are "$v2" +check "every asset a live page loads is still in the bucket" live_refs_resolve +check "the prune reports deleting them all" \ + grep -q "deleted $(comm -23 <(asset_keys_of "$v1" "$v2a" | sort -u) <(asset_keys_of "$v2" | sort) | wc -l | tr -d ' ') unused asset" "$tmp/out" +check "outside docs/assets the bucket holds exactly v2" pages_are "$v2" +check "manifests and markers from before d2r are dropped" \ + test "$(manifests)" = "$((T0 + DAY))-d2r.done $((T0 + DAY))-d2r.txt $((T0 + 14 * DAY + 2 * HOUR))-d3.done $((T0 + 14 * DAY + 2 * HOUR))-d3.txt $((T0 + 15 * DAY + HOUR))-d4.done $((T0 + 15 * DAY + HOUR))-d4.txt" + +echo "== a manifest prefix holding something this script did not write" +stray=docs/assets/js/stray.abcdef12.js +echo stray | aws s3 cp - "s3://$bucket/$stray" --only-show-errors +echo x | aws s3 cp - "s3://$bucket/$MANIFESTS/notes.txt" --only-show-errors +run_deploy "$v2" $((T0 + 60 * DAY)) d5 +check "the prune is skipped rather than guessing" grep -q 'notes.txt is not something this script wrote' "$tmp/out" +check "so an unused asset is left in place" has "$stray" +check "and nothing else moved" assets_are_plus "$stray" "$v2" +aws s3 rm "s3://$bucket/$MANIFESTS/notes.txt" --only-show-errors + +# v3: v2 without one static file, to watch when a dropped static file goes. +cp -R "$v2" "$tmp/v3" +v3=$tmp/v3 +rm "$v3/$sample_static" +find "$v3" -type f -exec touch {} + + +echo "== deploy 6: v3, its invalidation not landing in time" +export AWS_INVALIDATION_PENDING=99 DOCS_INVALIDATION_WAIT_SECONDS=5 +run_deploy "$v3" $((T0 + 61 * DAY)) d6 +unset AWS_INVALIDATION_PENDING +export DOCS_INVALIDATION_WAIT_SECONDS=30 +check "it says the invalidation had not landed" grep -q 'had not landed after 5s' "$tmp/out" +check "and it waited exactly its 5s budget, not a whole 10s poll past it" test "$(sleeps)" = 5 +check "so the static file v3 dropped is left for the next deploy" has "$sample_static" +check "and the deploy is still marked complete" has "$MANIFESTS/$((T0 + 61 * DAY))-d6.done" + +echo "== deploy 7: v3, the deploy user unable to read the invalidation" +if DEPLOY_NOW=$((T0 + 62 * DAY)) AWS_INVALIDATION_DENY=1 "$deploy" "$v3" "$bucket" d7 > "$tmp/out" 2>&1; then + fail "a deploy that cannot read its invalidation exits 0" +else + pass "a deploy that cannot read its invalidation fails, rather than leave dropped files public for good" +fi +check "it names the missing permission" grep -q '::error::.*cloudfront:GetInvalidation' "$tmp/out" +check "the static file v3 dropped is still there" has "$sample_static" +check "but the deploy is marked complete" has "$MANIFESTS/$((T0 + 62 * DAY))-d7.done" +check "and the prune ran before it failed" grep -q 'deploy-docs: 5/5' "$tmp/out" + +echo "== deploy 8: v3, all well again" +run_deploy "$v3" $((T0 + 63 * DAY)) d8 +check "the static file v3 dropped is deleted now" lacks "$sample_static" +check "outside docs/assets the bucket holds exactly v3" pages_are "$v3" + +echo "== failures that must fail the deploy" +if DEPLOY_NOW=$T0 "$deploy" "$v2" "no-such-bucket-$$" x > "$tmp/out" 2>&1; then + fail "a deploy to a bucket that does not exist exits 0" +else + pass "a deploy to a bucket that does not exist fails" +fi +# Refused means refused BEFORE touching the bucket: no aws call at all. A +# check that only looked at the exit code would pass for a script that +# uploaded everything and then fell over. +refused() { # refused + local what=$1 + shift + : > "$tmp/calls" + if AWS_CALL_LOG=$tmp/calls "$@" > "$tmp/out" 2>&1; then + fail "$what: exited 0" + elif [ -s "$tmp/calls" ]; then + fail "$what: but only after calling aws: $(sed -n 1p "$tmp/calls")" + else + pass "$what" + fi +} +refused "a deploy with no CLOUDFRONT_DISTRIBUTION_ID is refused" \ + env -u CLOUDFRONT_DISTRIBUTION_ID "$deploy" "$v2" "$bucket" x +mkdir -p "$tmp/empty/docs" +refused "a build with no docs/assets is refused" "$deploy" "$tmp/empty" "$bucket" x +refused "a deploy_id that cannot name a manifest is refused" "$deploy" "$v2" "$bucket" 'bad id' + +echo +if [ "$failures" -ne 0 ]; then + echo "deploy-docs-test: $failures check(s) FAILED" + exit 1 +fi +echo "deploy-docs-test: all checks passed" diff --git a/.github/scripts/deploy-docs.sh b/.github/scripts/deploy-docs.sh new file mode 100755 index 000000000..9f1f28393 --- /dev/null +++ b/.github/scripts/deploy-docs.sh @@ -0,0 +1,364 @@ +#!/usr/bin/env bash +# +# deploy-docs.sh — publish the docs build to S3 so that no page that is live, +# cached, or open in a tab is ever left pointing at a deleted docs/assets/ file +# — the content-hashed JS and CSS a page needs to hydrate at all. +# +# .github/scripts/deploy-docs.sh +# +# site_dir the site as the bucket serves it: docs/ at its root +# bucket the S3 bucket behind keploy.io/docs +# deploy_id names this deploy's manifest; the commit SHA being shipped +# +# Environment: +# CLOUDFRONT_DISTRIBUTION_ID invalidated (/docs*) once the pages are live. +# Required: without it the edge would serve the +# old pages for up to a day. +# DOCS_ASSET_RETENTION_DAYS how long an asset is kept once the pages that +# used it have been replaced (default 14) +# DOCS_INVALIDATION_WAIT_SECONDS +# how long to wait for the invalidation to land +# before leaving dropped static files for the next +# deploy (default 900) +# DEPLOY_NOW this deploy's time as a Unix epoch (default: now). +# Only deploy-docs-test.sh sets it, to replay weeks +# of deploys in seconds. +# AWS credentials and region as the aws CLI reads them; the test also sets +# AWS_ENDPOINT_URL to point it at a local S3. +# +# Keep this interface backward compatible: re-running a finished main.yml run +# replays that run's own workflow file against the script on today's main. +# +# WHY — keploy/docs, 2026-09-23. The deploy used to be a single `s3-deploy +# --deleteRemoved` pass: upload the new build, then delete everything in the +# bucket the new build did not contain. Docusaurus names every JS and CSS chunk +# by content hash, so each deploy deleted the previous build's chunks the moment +# it ran. Anything still holding the previous HTML then asked for a chunk that +# no longer existed and never hydrated: a tab left open, a cached page, or a +# second deploy running at the same time. Two merges 17 s apart (#912, #913) +# did the last of those to production — /docs/server/installation/ rendered, +# but its macOS tab and every other control on it were dead. +# +# So a deploy runs in this order: +# 1. docs/assets/ — the hashed chunks — uploaded FIRST, cached as immutable, +# and never deleted by this pass. The new pages are not live yet, so no +# page can reference a chunk before it exists. +# 2. This deploy's manifest, the list of assets it shipped, written BEFORE the +# pages. Every asset a live page can reference is then on record, even if +# step 3 fails halfway. +# 3. Everything else: static files uploaded, then text and pages synced with +# --delete, pages last. +# 4. CloudFront invalidated, so the edge serves the new pages. Once that has +# landed — no old page left anywhere to ask for them — static files the +# build dropped are deleted. (Un-hashed static files keep their URL, so +# there is no old copy to retain: a tab still open on a page whose image +# was renamed shows it broken, as it always has.) Then the manifest is +# marked complete. +# 5. Old assets pruned: one is deleted only once every page that used it was +# replaced more than DOCS_ASSET_RETENTION_DAYS ago. +# +# Steps 1-4 fail the deploy on any error. Step 5 is housekeeping: a failed or +# skipped prune only leaves unused files in the bucket, so it warns and exits 0 +# rather than paint a successful deploy red. + +set -euo pipefail +# comm needs both inputs sorted under one collation; pin it. +export LC_ALL=C + +if [ "$#" -ne 3 ]; then + echo "usage: $0 " >&2 + exit 2 +fi +site=${1%/} +bucket=$2 +deploy_id=$3 + +# Docusaurus' content-hashed output, under the site's /docs base URL. Nothing +# un-hashed may live here: it is cached as immutable. (static/ has no assets/ +# directory, which is what keeps it that way; deploy-docs-test.sh checks.) +ASSETS=docs/assets +# Under docs/ with everything else, so the deploy writes nowhere the old one +# did not. +MANIFESTS=docs/_deploy/asset-manifests + +# A hashed name never changes content, so browsers may keep it for a year. +ASSET_CACHE="public, max-age=31536000, immutable" +# Pages, sitemaps and text keep their URL and change with every deploy: +# browsers revalidate each load (a 304 when unchanged). CloudFront keeps them a +# day, and step 4 clears it. +PAGE_CACHE="public, max-age=0, s-maxage=86400, must-revalidate" +# Images, fonts and scripts outside docs/assets also keep their URL but rarely +# change, and a page load pulls in dozens of them. An hour stale is fine; +# a revalidation round trip for each one on every load is not. +STATIC_CACHE="public, max-age=3600, s-maxage=86400" + +retention_days=${DOCS_ASSET_RETENTION_DAYS:-14} +wait_seconds=${DOCS_INVALIDATION_WAIT_SECONDS:-900} +now=${DEPLOY_NOW:-$(date +%s)} +for v in "$retention_days" "$wait_seconds" "$now"; do + case "$v" in '' | *[!0-9]*) + echo "deploy-docs: DOCS_ASSET_RETENTION_DAYS, DOCS_INVALIDATION_WAIT_SECONDS and DEPLOY_NOW must be whole numbers (got '$v')" >&2 + exit 2 + ;; + esac +done +if [ -z "${CLOUDFRONT_DISTRIBUTION_ID:-}" ]; then + echo "deploy-docs: CLOUDFRONT_DISTRIBUTION_ID is required — without an invalidation the edge serves the old pages for up to a day" >&2 + exit 2 +fi +case "$deploy_id" in '' | *[!A-Za-z0-9._-]*) + echo "deploy-docs: deploy_id must be [A-Za-z0-9._-]+ (got '$deploy_id')" >&2 + exit 2 + ;; +esac +if [ ! -d "$site/$ASSETS" ]; then + echo "deploy-docs: $site/$ASSETS does not exist — is $site a docs build with docs/ at its root?" >&2 + exit 2 +fi + +# Checked up front: jq is only needed by the prune, which runs after the pages +# are live, and finding it missing there would be too late to stop anything. +for tool in aws jq; do + command -v "$tool" > /dev/null || { + echo "deploy-docs: $tool is required" >&2 + exit 2 + } +done + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +(cd "$site" && find "$ASSETS" -type f) | sort > "$tmp/shipped" +if [ ! -s "$tmp/shipped" ]; then + echo "deploy-docs: $site/$ASSETS is empty — refusing to deploy a build with no assets" >&2 + exit 2 +fi +manifest="$MANIFESTS/$now-$deploy_id.txt" +done_marker="$MANIFESTS/$now-$deploy_id.done" + +echo "deploy-docs: 1/5 uploading $(wc -l < "$tmp/shipped" | tr -d ' ') assets to $ASSETS/ (nothing deleted)" +aws s3 sync "$site/$ASSETS" "s3://$bucket/$ASSETS" \ + --cache-control "$ASSET_CACHE" --no-progress --only-show-errors + +echo "deploy-docs: 2/5 recording them in $manifest" +aws s3 cp "$tmp/shipped" "s3://$bucket/$manifest" \ + --content-type text/plain --cache-control no-store --only-show-errors + +# sync_class [--delete] +# +# One class of file outside docs/assets. --delete removes what the build no +# longer has, but only among the files the filters select, so the four classes +# below must partition the site between them — each file in exactly one. +sync_class() { + local args=(--cache-control "$1") + if [ -n "$2" ]; then + args+=(--content-type "$2") + fi + shift 2 + # The class filters come first: a later filter wins, so the two excludes + # after them hold whatever the class includes. + aws s3 sync "$site" "s3://$bucket" "$@" \ + --exclude "$ASSETS/*" --exclude "$MANIFESTS/*" \ + "${args[@]}" --no-progress --only-show-errors +} + +STATIC=(--exclude '*.html' --exclude '*.xml' --exclude '*.json' --exclude '*.txt' --exclude '*.md') +echo "deploy-docs: 3/5 uploading static files, then syncing text and pages" +sync_class "$STATIC_CACHE" '' "${STATIC[@]}" +# The CLI labels these text/plain and text/markdown with no charset, and a +# client with no charset decodes text/* as Latin-1 (Python's requests does). +# llms.txt, llms-full.txt and the .md copies of every page exist for exactly +# those readers, and nearly all of them hold non-ASCII. +sync_class "$PAGE_CACHE" 'text/plain; charset=utf-8' --delete --exclude '*' --include '*.txt' +sync_class "$PAGE_CACHE" 'text/markdown; charset=utf-8' --delete --exclude '*' --include '*.md' +# Pages last, so every file a page links to is already there when it goes live. +sync_class "$PAGE_CACHE" '' --delete --exclude '*' --include '*.html' --include '*.xml' --include '*.json' + +# invalidation_landed : 0 once CloudFront reports it complete, 1 if it has +# not within wait_seconds, 2 if CloudFront would not say. Polled rather than +# `aws cloudfront wait`, which cannot tell those last two apart. The status is +# matched case-insensitively: CloudFront says "Completed". +invalidation_landed() { + local status step waited=0 + while :; do + status=$(aws cloudfront get-invalidation --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" \ + --id "$1" --query 'Invalidation.Status' --output text) || return 2 + if [ "$(printf '%s' "$status" | tr '[:upper:]' '[:lower:]')" = completed ]; then + return 0 + fi + if [ "$waited" -ge "$wait_seconds" ]; then + return 1 + fi + step=$((wait_seconds - waited < 10 ? wait_seconds - waited : 10)) + sleep "$step" + waited=$((waited + step)) + done +} + +echo "deploy-docs: 4/5 invalidating /docs* on CloudFront" +invalidation=$(aws cloudfront create-invalidation --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" \ + --paths '/docs*' --query 'Invalidation.Id' --output text) +# A slow invalidation only defers the deletions to the next deploy. One whose +# status cannot be read would defer them on EVERY deploy — a file removed from +# static/, a retracted PDF say, would stay public for good behind a green +# check — so that fails the run, once the rest of the deploy is done. +landed=0 +invalidation_landed "$invalidation" || landed=$? +static_error= +case "$landed" in + 0) + echo "deploy-docs: invalidation $invalidation landed; deleting static files the build dropped" + sync_class "$STATIC_CACHE" '' --delete "${STATIC[@]}" + ;; + 1) + echo "::warning::deploy-docs: invalidation $invalidation had not landed after ${wait_seconds}s; static files this build dropped are left for the next deploy to delete" + ;; + *) + static_error="could not read the status of invalidation $invalidation (the aws error above says why; AccessDenied means the deploy user needs cloudfront:GetInvalidation), so static files this build dropped were NOT deleted. The pages are live." + echo "::error::deploy-docs: $static_error" + ;; +esac +# Every page of this deploy is live. Only a deploy that got this far may stand +# as the prune's boundary (see prune): one that failed in step 3 left the pages +# it did not reach at the deploy before it. +aws s3 cp - "s3://$bucket/$done_marker" --content-type text/plain \ + --cache-control no-store --only-show-errors < /dev/null + +# Keys under a prefix, one per line. JSON rather than text output because the +# CLI merges every page before applying --query only for JSON; text applies it +# page by page. A key holding a space survives too. +list_keys() { + aws s3api list-objects-v2 --bucket "$bucket" --prefix "$1" \ + --query 'Contents[].Key' --output json > "$tmp/list.json" && + jq -r '.[]?' "$tmp/list.json" +} + +skip_prune() { + echo "::warning::deploy-docs: prune skipped, nothing deleted — $*" +} + +# Which assets may still be referenced. The boundary is the newest COMPLETED +# deploy from before the window. Its pages were all live until something +# replaced them, which happened inside the window, so a tab opened on them may +# still be open: its assets stay, and so do those of every deploy after it, +# complete or not — a deploy that failed halfway left some of its pages live. +# Everything before the boundary was wholly replaced by it before the window +# began (give or take the minutes the boundary deploy itself took, against a +# window of days). +prune() { + local cutoff=$((now - retention_days * 86400)) + + if ! list_keys "$MANIFESTS/" > "$tmp/listing"; then + skip_prune "could not list $MANIFESTS/" + return 0 + fi + if ! grep -qxF "$done_marker" "$tmp/listing"; then + skip_prune "the listing of $MANIFESTS/ is missing $done_marker, written moments ago, so it cannot be trusted" + return 0 + fi + + # manifests: " " per manifest; done: the names of the markers; + # entries: " " for both. + local key name epoch boundary=-1 + : > "$tmp/manifests" + : > "$tmp/done" + : > "$tmp/entries" + while IFS= read -r key; do + name=${key#"$MANIFESTS/"} + epoch=${name%%-*} + case "$epoch" in '' | *[!0-9]*) epoch=x ;; esac + [ "$epoch" = x ] || echo "$epoch $key" >> "$tmp/entries" + case "$epoch:$name" in + x:*) ;; + *.txt) + echo "$epoch $key" >> "$tmp/manifests" + continue + ;; + *.done) + echo "$name" >> "$tmp/done" + continue + ;; + esac + skip_prune "$key is not something this script wrote (-.txt or .done); the retained set would be a guess" + return 0 + done < "$tmp/listing" + + while read -r epoch key; do + name=${key#"$MANIFESTS/"} + if [ "$epoch" -lt "$cutoff" ] && [ "$epoch" -gt "$boundary" ] && + grep -qxF "${name%.txt}.done" "$tmp/done"; then + boundary=$epoch + fi + done < "$tmp/manifests" + + if [ "$boundary" -lt 0 ]; then + echo "deploy-docs: 5/5 no completed deploy older than $retention_days days yet; nothing to prune" + return 0 + fi + + : > "$tmp/keep.unsorted" + while read -r epoch key; do + [ "$epoch" -ge "$boundary" ] || continue + if ! aws s3 cp "s3://$bucket/$key" - --only-show-errors >> "$tmp/keep.unsorted"; then + skip_prune "could not read $key" + return 0 + fi + done < "$tmp/manifests" + sort -u "$tmp/keep.unsorted" > "$tmp/keep" + + if ! list_keys "$ASSETS/" | sort -u > "$tmp/live"; then + skip_prune "could not list $ASSETS/" + return 0 + fi + # Everything step 1 uploaded must be in the listing. If it is not, the + # listing is short — truncated, or in a format jq did not read — and every + # asset missing from it would look unused. + comm -23 "$tmp/shipped" "$tmp/live" > "$tmp/unlisted" + if [ -s "$tmp/unlisted" ]; then + skip_prune "the listing of $ASSETS/ is missing assets this deploy uploaded, so it cannot be trusted" + return 0 + fi + + comm -23 "$tmp/live" "$tmp/keep" > "$tmp/stale" + local stale_count deleted=0 + stale_count=$(wc -l < "$tmp/stale" | tr -d ' ') + echo "deploy-docs: 5/5 $(wc -l < "$tmp/live" | tr -d ' ') assets in the bucket, $(wc -l < "$tmp/keep" | tr -d ' ') shipped within $retention_days days; deleting $stale_count" + + # delete-objects takes at most 1000 keys a call. + split -l 1000 "$tmp/stale" "$tmp/batch." + local batch errors + for batch in "$tmp"/batch.*; do + [ -s "$batch" ] || continue + jq -R -s '{Objects: (split("\n") | map(select(length > 0) | {Key: .})), Quiet: true}' "$batch" > "$tmp/delete.json" + if ! aws s3api delete-objects --bucket "$bucket" --delete "file://$tmp/delete.json" --output json > "$tmp/deleted.json"; then + skip_prune "delete-objects failed after $deleted of $stale_count deletions" + return 0 + fi + # In quiet mode S3 lists only the keys that failed, and when none did the + # CLI prints nothing at all; -s reads that as [] rather than an error. + errors=$(jq -s '(.[0].Errors // []) | length' "$tmp/deleted.json") + if [ "$errors" -ne 0 ]; then + echo "::warning::deploy-docs: $errors asset(s) could not be deleted and were left in place:" + jq -r '.Errors[] | " \(.Key): \(.Code) \(.Message)"' "$tmp/deleted.json" + fi + deleted=$((deleted + $(wc -l < "$batch") - errors)) + done + + # Manifests and markers from before the boundary no longer decide anything. + # Walked from the listing, not from the manifests, so a marker whose manifest + # went on an earlier run is removed too. + while read -r epoch key; do + [ "$epoch" -lt "$boundary" ] || continue + aws s3 rm "s3://$bucket/$key" --only-show-errors || + echo "::warning::deploy-docs: could not remove $key; the next deploy will retry" + done < "$tmp/entries" + + echo "deploy-docs: deleted $deleted unused asset(s)" +} + +prune + +if [ -n "$static_error" ]; then + echo "deploy-docs: failing the run: $static_error" >&2 + exit 1 +fi diff --git a/.github/workflows/build_and_check.yml b/.github/workflows/build_and_check.yml index 1b09f48bb..a8ec10725 100644 --- a/.github/workflows/build_and_check.yml +++ b/.github/workflows/build_and_check.yml @@ -10,7 +10,14 @@ on: jobs: deploy-preview: runs-on: ubuntu-latest - + + services: + # A local S3 and CloudFront for the deploy e2e step below. + s3: + image: motoserver/moto:5.2.3 + ports: + - 5000:5000 + steps: - name: Checkout code uses: actions/checkout@v4 @@ -28,3 +35,14 @@ jobs: - name: Verify structured-data graph run: npm run verify-schema + + # Deploys this PR's build with main.yml's deploy script into a local S3, + # then replays the weeks after it: no asset a live or open page still + # loads is deleted, every unused one eventually is, and every file gets + # the right cache header. Built here with npm rather than main.yml's + # yarn, to the same layout. Run against the real build so a change that + # puts an un-hashed file under assets/ fails here, not in production. + - name: Deploy e2e (local S3) + env: + AWS_ENDPOINT_URL: http://localhost:5000 + run: .github/scripts/deploy-docs-test.sh build diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5bf41db0d..def2e72aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,6 +8,28 @@ on: push: branches: [ main ] +# One deploy at a time, and every deploy ships main as it is when its run +# starts (the checkout below uses ref: main, not the commit that triggered the +# run). A push to main queues a run behind the one in progress; a queued run +# that a newer push supersedes is cancelled by GitHub; and re-running an old +# run deploys today's main, not the old commit. So however runs queue, are +# cancelled or are re-run, no deploy ships an older main than the one before +# it, and the last push that starts a run ends up live unless its build fails. +# +# cancel-in-progress is false so a deploy already syncing is never cut off +# half-written. +# +# A re-run replays the workflow file of the ORIGINAL run. Runs from before this +# file used deploy-docs.sh ran the old s3-deploy action, whose delete-removed +# wipes every retained asset and manifest: never re-run one of those; push or +# re-run a newer one instead. +# +# Why deploys must never overlap, and how each one keeps the pages it replaces +# working: .github/scripts/deploy-docs.sh. +concurrency: + group: docs-deploy + cancel-in-progress: false + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" @@ -22,7 +44,10 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - + with: + # main as it is now, not github.sha — see the concurrency comment. + ref: main + - name: Set Node.js 22.x uses: actions/setup-node@v3 with: @@ -43,16 +68,11 @@ jobs: mv build-old/* build/docs - name: Deploy - uses: reggionick/s3-deploy@v3 - with: - folder: build - bucket: ${{ secrets.S3_BUCKET }} - bucket-region: ${{ secrets.S3_BUCKET_REGION }} - dist-id: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} - invalidation: /docs* - delete-removed: true - cache: "public, max-age=86400" - private: true + env: + S3_BUCKET: ${{ secrets.S3_BUCKET }} + AWS_DEFAULT_REGION: ${{ secrets.S3_BUCKET_REGION }} + CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.CLOUDFRONT_DISTRIBUTION_ID }} + run: .github/scripts/deploy-docs.sh build "$S3_BUCKET" "$(git rev-parse HEAD)" - name: Submit docs URLs to IndexNow env: