Add Renfe and Dutch timetables (Interrail)

.github/workflows/ci.yml

@@ -17,5 +17,23 @@ jobs:
       - run: npm ci
       - run: npm run typecheck
       - run: npm test
-      - name: Build the API from the fixture
-        run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json
+      - name: Build the API from the fixtures
+        run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05
+
+  live-feeds:
+    # Operator code changes are checked against the real feeds before they publish.
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npm run fetch
+      - run: npm run build:api
+      - name: Sizes
+        run: |
+          du -sh public/v1/*
+          for f in public/v1/*/*/all.json; do echo "$(du -h "$f" | cut -f1) $(gzip -c "$f" | wc -c | numfmt --to=si)B gzipped  $f"; done

AGENTS.md

@@ -3,7 +3,7 @@
 - Never mention AI tools or assistants anywhere that is pushed: no assistant co-author or
   session trailers, no "generated with" lines, in commits, PRs, comments, code or docs.
 - Merge pull requests with squash and merge.
-- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json`.
+- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05`.
 - `docs/format.md` is the contract with clients: change it in the same change as
   `src/format.ts`, `src/compact.ts` or `src/api.ts`.
 - Never claim a seat the data doesn't show: a pass whose seat no open data publishes

README.md

@@ -2,18 +2,27 @@
 
 > **Work in progress.** "train-api" is a placeholder name; the final name isn't chosen yet.
 
-Open train data for rail passes on SNCF: **MAX JEUNE**, **MAX SENIOR** and **Interrail**. It is built on top
-of [MAX-Finder](https://github.com/offware-apps/MAX-Finder): the same SNCF open-data feed,
-without MAX-Finder's "free MAX seat only" filter, so every train an Interrail holder can
-book shows up.
+Open train data for rail passes: **MAX JEUNE**, **MAX SENIOR** and **Interrail**, one simple
+API per operator, built from each operator's own open timetable:
+
+| Operator | Source | Licence | Passes |
+|----------|--------|---------|--------|
+| `sncf` | SNCF Open Data `tgvmax` (TGV INOUI, Intercités) | Licence Ouverte | MAX JEUNE, MAX SENIOR, Interrail |
+| `renfe` | Renfe Data, high-speed, long and medium distance (GTFS) | CC BY 4.0 | Interrail |
+| `nl` | Dutch national timetable via OVapi (GTFS): international and Intercity trains | CC0 | Interrail |
+
+The SNCF part is built on top of [MAX-Finder](https://github.com/offware-apps/MAX-Finder):
+the same feed, without MAX-Finder's "free MAX seat only" filter, so every train an
+Interrail holder can book shows up. Each operator is published on its own; trips that
+change between operators aren't joined here.
 
 No open data says whether an Interrail pass-holder seat is left, so Interrail trains are
 published as running, with the seat `unknown`.
 
 ## How it works
 
-1. `npm run fetch` downloads the SNCF `tgvmax` feed (every TGV INOUI and Intercités train
-   for ~30 days) to `data/raw/`.
+1. `npm run fetch` downloads every source to `data/raw/`: the SNCF `tgvmax` feed (every
+   TGV INOUI and Intercités train for ~30 days) and each GTFS operator's zip, unzipped.
 2. `npm run build:api` maps it to the [open train format](docs/format.md) and writes the
    static API to `public/v1/` in the compact snapshot format: each service is stored once
    with the dates it runs, about 20x smaller than the raw rows.
@@ -28,12 +37,14 @@ Live at <https://offware-apps.github.io/train-api/v1/index.json> once Pages is o
 ```sh
 npm ci
 npm run check                                              # typecheck + tests
-npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json  # build from the fixture
+npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json \
+  --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05    # build from the fixtures
 ```
 
 ## Next
 
 - The front-end on top of MAX-Finder's search core, with Interrail as the pass.
-- Other operators through the same format (Deutsche Bahn, Renfe).
+- More operators through the same GTFS reader (Deutsche Bahn, SNCB once its licence is checked).
 
-Data: SNCF Open Data, Licence Ouverte. Code: AGPL-3.0.
+Data: SNCF Open Data (Licence Ouverte), Renfe Data (CC BY 4.0), OVapi / NDOV (CC0).
+Code: AGPL-3.0.

docs/format.md

@@ -7,12 +7,12 @@ publishes it per pass in the compact snapshot format (`src/compact.ts`).
 
 | Field | Type | Meaning |
 |-------|------|---------|
-| `operator` | string | `sncf` for now |
+| `operator` | string | `sncf`, `renfe`, `nl` |
 | `date` | `YYYY-MM-DD` | departure date |
 | `origin`, `destination` | string | station names exactly as the operator publishes them |
 | `depart`, `arrive` | `HH:MM` | local time; an arrival before the departure is the next day |
 | `trainNo` | string | train number |
-| `category` | string, optional | line or train family (SNCF: the `axe`) |
+| `category` | string, optional | line or train family (SNCF: the `axe`; GTFS operators: `AVE`, `Eurostar`, `Intercity`…) |
 | `passes` | object | per pass: `{ bookable, seat }` |
 
 `seat` is `free` or `full` when the data says so, and `unknown` when no open data does.
@@ -31,6 +31,13 @@ its residence rule, which no timetable can check.
 | `max-senior` | as `max-jeune`, and the date is a weekday | known |
 | `interrail` | the train runs: every TGV INOUI, Intercités, night and international train takes a pass-holder reservation | unknown |
 
+GTFS operators (`renfe`, `nl`) publish only `interrail`. Each kept trip becomes one train
+per pair of stops it calls at, per date it runs, like an SNCF row; a departure after
+midnight is dated the next day. Renfe leaves out AVLO, which doesn't take Interrail. The
+Dutch feed keeps international and long-distance trains (Eurostar, ICE, EuroCity,
+Intercity direct, Nightjet) at every stop, and domestic Intercity trains only between the
+stations those serve.
+
 ## Compact snapshot
 
 Most of a 30-day timetable is the same service repeated every day, so each distinct

scripts/build-api.ts

@@ -1,24 +1,66 @@
 /**
- * Build the static API from a feed file already on disk (default: data/raw/sncf-tgvmax.json,
- * which `npm run fetch` downloads) into public/. Run: npm run build:api [-- <feed.json>]
+ * Build the static API into public/ from the sources on disk (see `npm run fetch`):
+ *   data/raw/sncf-tgvmax.json, or the feed file given as the first argument
+ *   data/raw/<operator>/       each GTFS operator, or `--gtfs <operator>=<dir>`
+ * `--from YYYY-MM-DD` sets the first day of the GTFS window (default: today, UTC).
+ * Sources that are missing are left out; the build fails only when none is there.
+ * Run: npm run build:api [-- <feed.json>] [--gtfs renfe=<dir>] [--from <date>]
  */
-import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { buildIndex, buildOperatorApi } from "../src/api";
+import { buildIndex, buildOperatorApi, type ApiFile, type ApiIndex } from "../src/api";
+import { readGtfs } from "../src/gtfs";
+import { GTFS_OPERATORS } from "../src/operators";
 import { mapSncfFeed } from "../src/operators/sncf";
 
-const feedPath = resolve(process.argv[2] ?? "data/raw/sncf-tgvmax.json");
+const args = process.argv.slice(2);
+const gtfsDirs = new Map<string, string>();
+let feedArg: string | undefined;
+let from = new Date().toISOString().slice(0, 10);
+for (let i = 0; i < args.length; i++) {
+  const a = args[i] as string;
+  if (a === "--gtfs") {
+    const [id, dir] = (args[++i] ?? "").split("=");
+    if (id && dir) gtfsDirs.set(id, resolve(dir));
+  } else if (a === "--from") from = args[++i] ?? from;
+  else feedArg = a;
+}
 const outDir = resolve("public");
+const files: ApiFile[] = [];
+const entries: ApiIndex["operators"] = [];
+const counts: string[] = [];
+
+const feedPath = resolve(feedArg ?? "data/raw/sncf-tgvmax.json");
+if (existsSync(feedPath)) {
+  const raw: unknown = JSON.parse(readFileSync(feedPath, "utf-8"));
+  if (Array.isArray(raw) && raw.length > 0) {
+    const sncf = buildOperatorApi("sncf", "SNCF Open Data — tgvmax (Licence Ouverte)", mapSncfFeed(raw));
+    files.push(...sncf.files);
+    entries.push(sncf.entry);
+  } else console.error(`[build-api] ${feedPath} is not a non-empty array; SNCF left out.`);
+} else console.error(`[build-api] no SNCF feed at ${feedPath}; SNCF left out.`);
 
-const raw: unknown = JSON.parse(readFileSync(feedPath, "utf-8"));
-if (!Array.isArray(raw) || raw.length === 0) {
-  console.error(`[build-api] ${feedPath} is not a non-empty array.`);
+for (const { op } of GTFS_OPERATORS) {
+  const dir = gtfsDirs.get(op.id) ?? resolve("data/raw", op.id);
+  if (!existsSync(join(dir, "stop_times.txt"))) {
+    if (gtfsDirs.size === 0 || gtfsDirs.has(op.id)) console.error(`[build-api] no ${op.id} feed at ${dir}; left out.`);
+    continue;
+  }
+  const trains = await readGtfs(dir, op, from);
+  if (trains.length === 0) {
+    console.error(`[build-api] ${op.id}: no trains from ${from}; left out.`);
+    continue;
+  }
+  const built = buildOperatorApi(op.id, op.source, trains);
+  files.push(...built.files);
+  entries.push(built.entry);
+}
+
+if (entries.length === 0) {
+  console.error("[build-api] no source to publish.");
   process.exit(1);
 }
-const trains = mapSncfFeed(raw);
-const updatedAt = new Date().toISOString();
-const sncf = buildOperatorApi("sncf", "SNCF Open Data — tgvmax (Licence Ouverte)", trains);
-const files = [...sncf.files, buildIndex(updatedAt, [sncf.entry])];
+files.push(buildIndex(new Date().toISOString(), entries));
 
 rmSync(join(outDir, "v1"), { recursive: true, force: true });
 let bytes = 0;
@@ -29,7 +71,8 @@ for (const f of files) {
   bytes += text.length;
   writeFileSync(p, text, "utf-8");
 }
-console.log(
-  `[build-api] ${raw.length} records → ${trains.length} trains → ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`,
-);
-for (const p of sncf.entry.passes) console.log(`[build-api]   ${p.id}: ${p.trainCount} bookable trains`);
+console.log(`[build-api] ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`);
+for (const e of entries) {
+  for (const p of e.passes) counts.push(`${e.id}/${p.id}: ${p.trainCount} trains, ${e.stationCount} stations`);
+}
+for (const c of counts) console.log(`[build-api]   ${c}`);

scripts/fetch.ts

@@ -1,34 +1,83 @@
 /**
- * Download the SNCF feed to data/raw/sncf-tgvmax.json (git-ignored). Retries with backoff
- * and never overwrites a good file with an empty or malformed response.
+ * Download every source to data/raw/ (git-ignored):
+ *   data/raw/sncf-tgvmax.json    the SNCF feed
+ *   data/raw/<operator>/*.txt    each GTFS operator's feed, unzipped
+ * Retries with backoff and never overwrites a good file with an empty or malformed
+ * response. A GTFS operator that can't be fetched is skipped (the build leaves it out);
+ * the run fails only when nothing at all could be fetched.
  * Run: npm run fetch
  */
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { GTFS_OPERATORS } from "../src/operators";
 import { SNCF_FEED_URL } from "../src/operators/sncf";
 
-const OUT = resolve("data/raw/sncf-tgvmax.json");
+const RAW = resolve("data/raw");
 const TRIES = 3;
 
-async function download(): Promise<unknown[]> {
+async function withRetries<T>(what: string, run: () => Promise<T>): Promise<T> {
   let last: unknown;
   for (let i = 1; i <= TRIES; i++) {
     try {
-      const res = await fetch(SNCF_FEED_URL, { signal: AbortSignal.timeout(120_000) });
-      if (!res.ok) throw new Error(`HTTP ${res.status}`);
-      const body: unknown = await res.json();
-      if (!Array.isArray(body) || body.length === 0) throw new Error("expected a non-empty JSON array");
-      return body;
+      return await run();
     } catch (err) {
       last = err;
-      console.error(`[fetch] attempt ${i}/${TRIES} failed: ${String(err)}`);
+      console.error(`[fetch] ${what}: attempt ${i}/${TRIES} failed: ${String(err)}`);
       if (i < TRIES) await new Promise((r) => setTimeout(r, 2_000 * i));
     }
   }
   throw last;
 }
 
-const rows = await download();
-mkdirSync(dirname(OUT), { recursive: true });
-writeFileSync(OUT, JSON.stringify(rows), "utf-8");
-console.log(`[fetch] ${rows.length} records → ${OUT}`);
+async function fetchSncf(): Promise<void> {
+  const rows = await withRetries("sncf", async () => {
+    const res = await fetch(SNCF_FEED_URL, { signal: AbortSignal.timeout(120_000) });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const body: unknown = await res.json();
+    if (!Array.isArray(body) || body.length === 0) throw new Error("expected a non-empty JSON array");
+    return body;
+  });
+  const out = join(RAW, "sncf-tgvmax.json");
+  mkdirSync(dirname(out), { recursive: true });
+  writeFileSync(out, JSON.stringify(rows), "utf-8");
+  console.log(`[fetch] sncf: ${rows.length} records → ${out}`);
+}
+
+async function fetchGtfs(id: string, url: string): Promise<void> {
+  const zip = await withRetries(id, async () => {
+    const res = await fetch(url, { signal: AbortSignal.timeout(600_000) });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const buf = Buffer.from(await res.arrayBuffer());
+    // "PK": a zip archive, not an error page.
+    if (buf.length < 100 || buf[0] !== 0x50 || buf[1] !== 0x4b) throw new Error("not a zip archive");
+    return buf;
+  });
+  const tmp = join(RAW, `${id}.zip`);
+  const dir = join(RAW, id);
+  const next = `${dir}.new`;
+  mkdirSync(RAW, { recursive: true });
+  writeFileSync(tmp, zip);
+  rmSync(next, { recursive: true, force: true });
+  execFileSync("unzip", ["-q", "-o", tmp, "-d", next]);
+  if (!existsSync(join(next, "stop_times.txt"))) throw new Error(`${id}: the archive has no stop_times.txt`);
+  rmSync(dir, { recursive: true, force: true });
+  execFileSync("mv", [next, dir]);
+  rmSync(tmp, { force: true });
+  console.log(`[fetch] ${id}: ${(zip.length / 1e6).toFixed(1)} MB → ${dir}`);
+}
+
+let ok = 0;
+const jobs: [string, () => Promise<void>][] = [
+  ["sncf", fetchSncf],
+  ...GTFS_OPERATORS.map(({ op, url }): [string, () => Promise<void>] => [op.id, () => fetchGtfs(op.id, url)]),
+];
+for (const [id, job] of jobs) {
+  try {
+    await job();
+    ok++;
+  } catch (err) {
+    console.error(`[fetch] ${id}: skipped (${String(err)})`);
+  }
+}
+if (ok === 0) process.exit(1);