ParleyMap nightly refresh #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ParleyMap nightly refresh | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "19 3 * * *" | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: parleymap-nightly-refresh | |
| cancel-in-progress: false | |
| jobs: | |
| refresh-and-publish: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Validate current live embedded dataset | |
| run: node scripts/validate-demo-data.mjs | |
| - name: Repair current live dataset | |
| run: node scripts/repair-live-dataset.mjs | |
| - name: Apply roster hygiene before crawler | |
| run: node scripts/roster-hygiene.mjs | |
| - name: Validate repaired live embedded dataset | |
| run: node scripts/validate-demo-data.mjs | |
| - name: Validate crawler source registry | |
| run: | | |
| node -e "const fs=require('fs');const r=JSON.parse(fs.readFileSync('data/source-registry.json','utf8'));if(!Array.isArray(r.officialDomains)||r.officialDomains.length<10)throw new Error('source registry officialDomains missing or too small');if(!Array.isArray(r.sitemaps)||r.sitemaps.length<3)throw new Error('source registry sitemaps missing or too small');console.log('source registry ok');" | |
| - name: Sync crawler input from current index.html | |
| run: | | |
| node - <<'NODE' | |
| const fs = require('fs'); | |
| const html = fs.readFileSync('index.html', 'utf8'); | |
| const open = '<script id="demo-data" type="application/json">'; | |
| const close = '</' + 'script>'; | |
| const start = html.indexOf(open); | |
| if (start === -1) throw new Error('demo-data opening tag not found'); | |
| const jsonStart = start + open.length; | |
| const end = html.indexOf(close, jsonStart); | |
| if (end === -1) throw new Error('demo-data closing tag not found'); | |
| const data = JSON.parse(html.slice(jsonStart, end).trim()); | |
| fs.mkdirSync('data', { recursive: true }); | |
| fs.writeFileSync('data/demo.json', JSON.stringify(data, null, 2) + '\n'); | |
| console.log(JSON.stringify({ | |
| people: data.people?.length ?? null, | |
| roster: data.roster?.length ?? null, | |
| expansionRoster: data.expansionRoster?.length ?? null, | |
| appearances: data.appearances?.length ?? null | |
| }, null, 2)); | |
| NODE | |
| - name: Clear old crawler outputs | |
| run: | | |
| rm -f data/crawler/candidate-appearances.json | |
| rm -f data/crawler/publishable-appearances.json | |
| rm -f data/crawler/quality-approved-appearances.json | |
| rm -f data/crawler/rejected-appearances.json | |
| rm -f data/crawler/crawl-report.json | |
| - name: Run bounded crawler | |
| id: crawler | |
| continue-on-error: true | |
| run: timeout 12m node nightly-refresh.mjs --max-roster 60 --max-per-person 2 --max-pages-per-source 8 --lookback-days 21 | |
| env: | |
| PARLEYMAP_FETCH_TIMEOUT_MS: "3000" | |
| - name: Ensure crawler output files exist | |
| run: | | |
| node - <<'NODE' | |
| const fs = require('fs'); | |
| fs.mkdirSync('data/crawler', { recursive: true }); | |
| if (!fs.existsSync('data/crawler/candidate-appearances.json')) { | |
| fs.writeFileSync('data/crawler/candidate-appearances.json', '[]\n'); | |
| } | |
| if (!fs.existsSync('data/crawler/publishable-appearances.json')) { | |
| fs.writeFileSync('data/crawler/publishable-appearances.json', '[]\n'); | |
| } | |
| if (!fs.existsSync('data/crawler/crawl-report.json')) { | |
| fs.writeFileSync('data/crawler/crawl-report.json', JSON.stringify({ | |
| generatedAt: new Date().toISOString(), | |
| status: 'crawler_failed_or_timed_out', | |
| candidates: 0, | |
| publishable: 0 | |
| }, null, 2) + '\n'); | |
| } | |
| NODE | |
| - name: Guard and publish crawler output | |
| run: node scripts/guarded-publish-from-crawler.mjs | |
| - name: Strict audit crawler output and embedded records | |
| run: node scripts/strict-crawler-audit.mjs | |
| - name: Repair live dataset after publish | |
| run: node scripts/repair-live-dataset.mjs | |
| - name: Apply roster hygiene after publish | |
| run: node scripts/roster-hygiene.mjs | |
| - name: Validate final index.html | |
| run: node scripts/validate-demo-data.mjs | |
| - name: Restore crawler input staging file | |
| run: git checkout -- data/demo.json || true | |
| - name: Build quick run summary | |
| if: always() | |
| run: | | |
| mkdir -p data/diagnostics | |
| { | |
| echo "# ParleyMap nightly refresh" | |
| echo "" | |
| echo "Run: ${GITHUB_RUN_NUMBER}" | |
| echo "Generated: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" | |
| echo "" | |
| echo "## Changed files before commit" | |
| git diff --name-only || true | |
| echo "" | |
| echo "## Publish report" | |
| if [ -f data/diagnostics/publish-report.json ]; then cat data/diagnostics/publish-report.json; else echo "No publish report."; fi | |
| echo "" | |
| echo "## Repair report" | |
| if [ -f data/diagnostics/repair-report.json ]; then cat data/diagnostics/repair-report.json; else echo "No repair report."; fi | |
| echo "" | |
| echo "## Roster hygiene report" | |
| if [ -f data/diagnostics/roster-hygiene-report.json ]; then cat data/diagnostics/roster-hygiene-report.json; else echo "No roster hygiene report."; fi | |
| echo "" | |
| echo "## Strict crawler audit" | |
| if [ -f data/diagnostics/strict-crawler-audit-report.json ]; then cat data/diagnostics/strict-crawler-audit-report.json; else echo "No strict crawler audit report."; fi | |
| } > data/diagnostics/LATEST_RUN_SUMMARY.md | |
| cat data/diagnostics/LATEST_RUN_SUMMARY.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Show files changed before commit | |
| run: git diff --name-only | |
| - name: Commit refreshed website data | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "Refresh ParleyMap guarded data" | |
| file_pattern: "index.html data/crawler/*.json data/diagnostics/*.json data/diagnostics/*.md" |