Skip to content

Commit 108f9d5

Browse files
authored
Add files via upload
1 parent 051f4dd commit 108f9d5

4 files changed

Lines changed: 503 additions & 393 deletions

File tree

README_REBUILD.md

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import fs from 'node:fs';
2+
3+
const INDEX_PATH = 'index.html';
4+
const OPEN = '<script id="demo-data" type="application/json">';
5+
const CLOSE = '</' + 'script>';
6+
7+
function extract() {
8+
const html = fs.readFileSync(INDEX_PATH, 'utf8');
9+
const start = html.indexOf(OPEN);
10+
if (start === -1) throw new Error('demo-data opening tag not found in index.html');
11+
const jsonStart = start + OPEN.length;
12+
const end = html.indexOf(CLOSE, jsonStart);
13+
if (end === -1) throw new Error('demo-data closing tag not found in index.html');
14+
return JSON.parse(html.slice(jsonStart, end).trim());
15+
}
16+
17+
const data = extract();
18+
const requiredTopLevel = [
19+
'meta','categories','people','appearances','roster','topRoster','expansionRoster'
20+
];
21+
for (const key of requiredTopLevel) {
22+
if (!(key in data)) throw new Error(`missing top-level key ${key}`);
23+
}
24+
for (const key of ['categories','people','appearances','roster','expansionRoster']) {
25+
if (!Array.isArray(data[key])) throw new Error(`${key} must be an array`);
26+
}
27+
const minimums = { people: 90, roster: 190, expansionRoster: 100, appearances: 500, categories: 10 };
28+
for (const [key, min] of Object.entries(minimums)) {
29+
if (data[key].length < min) throw new Error(`${key} count too low: ${data[key].length} < ${min}`);
30+
}
31+
const firstAppearance = data.appearances[0] || {};
32+
for (const key of ['id','personId','startsAt','title','location','sourcePack']) {
33+
if (!(key in firstAppearance)) throw new Error(`appearance schema missing ${key}`);
34+
}
35+
console.log(JSON.stringify({
36+
status: 'validation_passed',
37+
people: data.people.length,
38+
roster: data.roster.length,
39+
expansionRoster: data.expansionRoster.length,
40+
appearances: data.appearances.length,
41+
categories: data.categories.length,
42+
version: data.meta?.version || null,
43+
lastDataUpdate: data.meta?.lastDataUpdate || null
44+
}, null, 2));

nightly-refresh.yml

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ concurrency:
1313
cancel-in-progress: false
1414

1515
jobs:
16-
nightly:
16+
refresh-and-publish:
1717
runs-on: ubuntu-latest
18-
timeout-minutes: 25
18+
timeout-minutes: 20
1919
steps:
20-
- name: Checkout repository with history
20+
- name: Checkout repository
2121
uses: actions/checkout@v4
2222
with:
2323
ref: main
@@ -28,34 +28,46 @@ jobs:
2828
with:
2929
node-version: "20"
3030

31-
- name: Run safe nightly canonical refresh
31+
- name: Validate current embedded dataset
32+
run: node scripts/validate-demo-data.mjs
33+
34+
- name: Repair canonical dataset, anchors, events, AdSense surface
35+
run: node scripts/repair-live-dataset.mjs
36+
37+
- name: Validate repaired embedded dataset
38+
run: node scripts/validate-demo-data.mjs
39+
40+
- name: Run safe crawler placeholder
3241
run: node nightly-refresh.mjs
3342

34-
- name: Append summary
43+
- name: Show repair summary
3544
if: always()
3645
run: |
3746
if [ -f data/diagnostics/LATEST_RUN_SUMMARY.md ]; then
3847
cat data/diagnostics/LATEST_RUN_SUMMARY.md >> "$GITHUB_STEP_SUMMARY"
48+
cat data/diagnostics/LATEST_RUN_SUMMARY.md
3949
fi
50+
git diff --name-only || true
4051
41-
- name: Show changed files
42-
if: always()
43-
run: git diff --name-only || true
44-
45-
- name: Upload nightly diagnostics
52+
- name: Upload diagnostics
4653
if: always()
4754
uses: actions/upload-artifact@v4
4855
with:
49-
name: parleymap-nightly-canonical-files
56+
name: parleymap-nightly-diagnostics
5057
path: |
51-
index.html
52-
ads.txt
53-
data/demo.json
5458
data/diagnostics/*.json
5559
data/diagnostics/*.md
60+
data/crawler/*.json
61+
ads.txt
62+
privacy.html
63+
impressum.html
64+
about.html
65+
contact.html
66+
methodology.html
67+
data-sources.html
5668
57-
- name: Commit nightly canonical refresh
69+
- name: Commit repaired website data
5870
uses: stefanzweifel/git-auto-commit-action@v5
5971
with:
60-
commit_message: "Refresh ParleyMap canonical data"
61-
file_pattern: "index.html ads.txt data/demo.json data/diagnostics/*.json data/diagnostics/*.md"
72+
commit_message: "Run ParleyMap nightly canonical repair"
73+
file_pattern: "index.html data/demo.json ads.txt privacy.html impressum.html about.html contact.html methodology.html data-sources.html data/diagnostics/*.json data/diagnostics/*.md data/crawler/*.json .github/workflows/nightly-refresh.yml scripts/repair-live-dataset.mjs scripts/validate-demo-data.mjs nightly-refresh.mjs"

0 commit comments

Comments
 (0)