diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml new file mode 100644 index 0000000..a4334d6 --- /dev/null +++ b/.github/workflows/archive.yml @@ -0,0 +1,53 @@ +name: Archive MAX seats + +# The SNCF feed only shows today's seat state. This keeps one snapshot a day on the public +# `data` branch (archive/max/YYYY/YYYY-MM-DD.json.gz), so seat odds can be computed later. +# A day already archived is never overwritten. + +on: + # After the feed's midday refresh and the 13:30 publish. + schedule: + - cron: "0 14 * * *" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: archive + cancel-in-progress: false + +jobs: + archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run fetch + - name: Check out the data branch + run: | + if git ls-remote --exit-code --heads origin data >/dev/null; then + git fetch --depth=1 origin data + git worktree add data-branch FETCH_HEAD + else + git worktree add --detach data-branch + git -C data-branch checkout --orphan data + git -C data-branch rm -rfq --ignore-unmatch . + fi + - run: npx tsx scripts/archive-max.ts data/raw/sncf-tgvmax.json data-branch/archive/max + - name: Commit and push + working-directory: data-branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add archive + if git diff --cached --quiet; then + echo "Nothing new to archive." + exit 0 + fi + git commit -q -m "Archive MAX seats for $(date -u +%F)" + git push origin HEAD:data diff --git a/docs/archive.md b/docs/archive.md new file mode 100644 index 0000000..e5c507c --- /dev/null +++ b/docs/archive.md @@ -0,0 +1,24 @@ +# MAX seat archive + +The SNCF feed only shows today's MAX seats. The **Archive MAX seats** workflow saves one +snapshot a day on the public `data` branch, so seat odds (by weekday, departure time and +days before travel) can be computed from the history. + +- Path: `archive/max/YYYY/YYYY-MM-DD.json.gz`, named by the day the feed was read (UTC). + A day already archived is never overwritten. +- Content (gzipped JSON): + +```json +{ + "snapshot": "2026-10-05", + "start": "2026-10-05", + "days": 31, + "services": [["PARIS (intramuros)", "LYON (intramuros)", "6601", "08:00", "11.0..."]] +} +``` + +Each service (origin, destination, train number, departure) is stored once. Its last field +has one character per day from `start`: `1` a free MAX seat, `0` no MAX seat, `.` not +running that day. + +Data: SNCF Open Data, Licence Ouverte. diff --git a/scripts/archive-max.ts b/scripts/archive-max.ts new file mode 100644 index 0000000..f8d78fa --- /dev/null +++ b/scripts/archive-max.ts @@ -0,0 +1,99 @@ +/** + * Archive today's MAX seat flags from the SNCF feed, so seat odds (by weekday, departure + * time and days before travel) can be computed later. The feed only shows the current + * state; anything not archived on the day is lost. + * + * One file per day, never overwritten: /YYYY/YYYY-MM-DD.json.gz. Each service + * (origin, destination, train, departure) is stored once with one character per day of + * the feed's window: "1" a free MAX seat, "0" full, "." not running that day. + * + * Run: npx tsx scripts/archive-max.ts [feed.json] [outDir] + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { gzipSync } from "node:zlib"; +import type { SncfRecord } from "../src/operators/sncf"; + +export interface MaxArchive { + /** Day the feed was read (UTC), "YYYY-MM-DD". */ + snapshot: string; + /** First travel day of the window; character i of each service is start + i days. */ + start: string; + days: number; + /** [origin, destination, trainNo, depart "HH:MM", per-day flags]. */ + services: [string, string, string, string, string][]; +} + +const DAY_MS = 86_400_000; + +function str(v: unknown): string { + return typeof v === "string" ? v.trim() : v == null ? "" : String(v).trim(); +} + +function hhmm(v: unknown): string { + const m = /^(\d{1,2}):(\d{2})/.exec(str(v)); + return m ? `${m[1]!.padStart(2, "0")}:${m[2]}` : ""; +} + +function dayIndex(date: string): number { + return Math.round(Date.parse(`${date}T00:00:00Z`) / DAY_MS); +} + +export function buildArchive(records: SncfRecord[], snapshot: string): MaxArchive { + const rows: { key: string; day: number; free: boolean }[] = []; + for (const r of records) { + const date = str(r.date); + const depart = hhmm(r.heure_depart); + const origin = str(r.origine); + const destination = str(r.destination); + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !depart || !origin || !destination) continue; + const key = [origin, destination, str(r.train_no), depart].join("\u0000"); + rows.push({ key, day: dayIndex(date), free: str(r.od_happy_card).toUpperCase() === "OUI" }); + } + if (rows.length === 0) return { snapshot, start: snapshot, days: 0, services: [] }; + + // A loop, not Math.min(...rows): the feed has ~360k rows, too many for spread arguments. + let first = Infinity; + let last = -Infinity; + for (const r of rows) { + first = Math.min(first, r.day); + last = Math.max(last, r.day); + } + const days = last - first + 1; + const flags = new Map(); + for (const r of rows) { + let f = flags.get(r.key); + if (!f) flags.set(r.key, (f = Array(days).fill("."))); + // A free seat on any duplicate row wins: the feed never lists a seat it doesn't have. + if (f[r.day - first] !== "1") f[r.day - first] = r.free ? "1" : "0"; + } + + const services = [...flags] + .map(([key, f]) => [...(key.split("\u0000") as [string, string, string, string]), f.join("")] as MaxArchive["services"][number]) + .sort((a, b) => a.join("|").localeCompare(b.join("|"))); + return { snapshot, start: new Date(first * DAY_MS).toISOString().slice(0, 10), days, services }; +} + +/** Write the day's archive; returns the path, or null when that day is already archived. */ +export function writeArchive(archive: MaxArchive, outDir: string): string | null { + const path = join(outDir, archive.snapshot.slice(0, 4), `${archive.snapshot}.json.gz`); + if (existsSync(path)) return null; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, gzipSync(JSON.stringify(archive))); + return path; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const feed = resolve(process.argv[2] ?? "data/raw/sncf-tgvmax.json"); + const outDir = resolve(process.argv[3] ?? "data/archive"); + const records = JSON.parse(readFileSync(feed, "utf-8")) as SncfRecord[]; + if (!Array.isArray(records) || records.length === 0) throw new Error(`no records in ${feed}`); + const archive = buildArchive(records, new Date().toISOString().slice(0, 10)); + const path = writeArchive(archive, outDir); + console.log( + path + ? `[archive] ${archive.services.length} services over ${archive.days} days → ${path}` + : `[archive] ${archive.snapshot} is already archived, left as is`, + ); +} diff --git a/tests/archive.test.ts b/tests/archive.test.ts new file mode 100644 index 0000000..1174f47 --- /dev/null +++ b/tests/archive.test.ts @@ -0,0 +1,47 @@ +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gunzipSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { buildArchive, writeArchive } from "../scripts/archive-max"; +import fixture from "./fixtures/sncf-tgvmax.sample.json"; + +const row = (date: string, train: string, seat: string) => ({ + date, + origine: "PARIS (intramuros)", + destination: "LYON (intramuros)", + heure_depart: "8:00", + heure_arrivee: "10:00", + train_no: train, + od_happy_card: seat, +}); + +describe("MAX seat archive", () => { + it("stores each service once with a flag per day", () => { + const a = buildArchive( + [row("2026-10-05", "6601", "OUI"), row("2026-10-07", "6601", "NON"), row("2026-10-06", "6605", "OUI")], + "2026-10-05", + ); + expect(a.start).toBe("2026-10-05"); + expect(a.days).toBe(3); + expect(a.services).toEqual([ + ["PARIS (intramuros)", "LYON (intramuros)", "6601", "08:00", "1.0"], + ["PARIS (intramuros)", "LYON (intramuros)", "6605", "08:00", ".1."], + ]); + }); + + it("keeps a free seat when the feed lists a service twice on one day", () => { + const a = buildArchive([row("2026-10-05", "6601", "NON"), row("2026-10-05", "6601", "OUI")], "2026-10-05"); + expect(a.services[0]?.[4]).toBe("1"); + }); + + it("archives the whole fixture and never overwrites a day", () => { + const a = buildArchive(fixture, "2026-06-25"); + expect(a.services.length).toBeGreaterThan(0); + const dir = mkdtempSync(join(tmpdir(), "archive-")); + const path = writeArchive(a, dir); + expect(path).toBe(join(dir, "2026", "2026-06-25.json.gz")); + expect(JSON.parse(gunzipSync(readFileSync(path!)).toString())).toEqual(a); + expect(writeArchive(a, dir)).toBeNull(); + }); +});