From 6b10139f9ecde7c2513a2622f6f45f050e42a16d Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:28:22 +0000 Subject: [PATCH] Add Renfe and Dutch timetables through a GTFS reader Each operator stays its own API (v1/renfe, v1/nl), Interrail only. Renfe keeps every long and medium-distance family but AVLO; the Dutch feed keeps international and long-distance trains, and Intercity trains between the stations those serve. --- .github/workflows/ci.yml | 22 ++- AGENTS.md | 2 +- README.md | 29 ++- docs/format.md | 11 +- scripts/build-api.ts | 75 +++++-- scripts/fetch.ts | 81 ++++++-- src/gtfs.ts | 209 ++++++++++++++++++++ src/operators/index.ts | 9 + src/operators/nl.ts | 31 +++ src/operators/renfe.ts | 18 ++ tests/fixtures/gtfs-mini/agency.txt | 2 + tests/fixtures/gtfs-mini/calendar.txt | 3 + tests/fixtures/gtfs-mini/calendar_dates.txt | 3 + tests/fixtures/gtfs-mini/routes.txt | 4 + tests/fixtures/gtfs-mini/stop_times.txt | 9 + tests/fixtures/gtfs-mini/stops.txt | 5 + tests/fixtures/gtfs-mini/trips.txt | 4 + tests/gtfs.test.ts | 69 +++++++ 18 files changed, 540 insertions(+), 46 deletions(-) create mode 100644 src/gtfs.ts create mode 100644 src/operators/index.ts create mode 100644 src/operators/nl.ts create mode 100644 src/operators/renfe.ts create mode 100644 tests/fixtures/gtfs-mini/agency.txt create mode 100644 tests/fixtures/gtfs-mini/calendar.txt create mode 100644 tests/fixtures/gtfs-mini/calendar_dates.txt create mode 100644 tests/fixtures/gtfs-mini/routes.txt create mode 100644 tests/fixtures/gtfs-mini/stop_times.txt create mode 100644 tests/fixtures/gtfs-mini/stops.txt create mode 100644 tests/fixtures/gtfs-mini/trips.txt create mode 100644 tests/gtfs.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e4d3a4..5f8d24e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,5 +17,23 @@ jobs: - run: npm ci - run: npm run typecheck - run: npm test - - name: Build the API from the fixture - run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json + - name: Build the API from the fixtures + run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05 + + live-feeds: + # Operator code changes are checked against the real feeds before they publish. + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run fetch + - run: npm run build:api + - name: Sizes + run: | + du -sh public/v1/* + for f in public/v1/*/*/all.json; do echo "$(du -h "$f" | cut -f1) $(gzip -c "$f" | wc -c | numfmt --to=si)B gzipped $f"; done diff --git a/AGENTS.md b/AGENTS.md index 098bdd0..8679f79 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ - Never mention AI tools or assistants anywhere that is pushed: no assistant co-author or session trailers, no "generated with" lines, in commits, PRs, comments, code or docs. - Merge pull requests with squash and merge. -- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json`. +- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05`. - `docs/format.md` is the contract with clients: change it in the same change as `src/format.ts`, `src/compact.ts` or `src/api.ts`. - Never claim a seat the data doesn't show: a pass whose seat no open data publishes diff --git a/README.md b/README.md index 3ae3556..48834e0 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,27 @@ > **Work in progress.** "train-api" is a placeholder name; the final name isn't chosen yet. -Open train data for rail passes on SNCF: **MAX JEUNE**, **MAX SENIOR** and **Interrail**. It is built on top -of [MAX-Finder](https://github.com/offware-apps/MAX-Finder): the same SNCF open-data feed, -without MAX-Finder's "free MAX seat only" filter, so every train an Interrail holder can -book shows up. +Open train data for rail passes: **MAX JEUNE**, **MAX SENIOR** and **Interrail**, one simple +API per operator, built from each operator's own open timetable: + +| Operator | Source | Licence | Passes | +|----------|--------|---------|--------| +| `sncf` | SNCF Open Data `tgvmax` (TGV INOUI, Intercités) | Licence Ouverte | MAX JEUNE, MAX SENIOR, Interrail | +| `renfe` | Renfe Data, high-speed, long and medium distance (GTFS) | CC BY 4.0 | Interrail | +| `nl` | Dutch national timetable via OVapi (GTFS): international and Intercity trains | CC0 | Interrail | + +The SNCF part is built on top of [MAX-Finder](https://github.com/offware-apps/MAX-Finder): +the same feed, without MAX-Finder's "free MAX seat only" filter, so every train an +Interrail holder can book shows up. Each operator is published on its own; trips that +change between operators aren't joined here. No open data says whether an Interrail pass-holder seat is left, so Interrail trains are published as running, with the seat `unknown`. ## How it works -1. `npm run fetch` downloads the SNCF `tgvmax` feed (every TGV INOUI and Intercités train - for ~30 days) to `data/raw/`. +1. `npm run fetch` downloads every source to `data/raw/`: the SNCF `tgvmax` feed (every + TGV INOUI and Intercités train for ~30 days) and each GTFS operator's zip, unzipped. 2. `npm run build:api` maps it to the [open train format](docs/format.md) and writes the static API to `public/v1/` in the compact snapshot format: each service is stored once with the dates it runs, about 20x smaller than the raw rows. @@ -28,12 +37,14 @@ Live at once Pages is o ```sh npm ci npm run check # typecheck + tests -npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json # build from the fixture +npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json \ + --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05 # build from the fixtures ``` ## Next - The front-end on top of MAX-Finder's search core, with Interrail as the pass. -- Other operators through the same format (Deutsche Bahn, Renfe). +- More operators through the same GTFS reader (Deutsche Bahn, SNCB once its licence is checked). -Data: SNCF Open Data, Licence Ouverte. Code: AGPL-3.0. +Data: SNCF Open Data (Licence Ouverte), Renfe Data (CC BY 4.0), OVapi / NDOV (CC0). +Code: AGPL-3.0. diff --git a/docs/format.md b/docs/format.md index f14f163..59e66bf 100644 --- a/docs/format.md +++ b/docs/format.md @@ -7,12 +7,12 @@ publishes it per pass in the compact snapshot format (`src/compact.ts`). | Field | Type | Meaning | |-------|------|---------| -| `operator` | string | `sncf` for now | +| `operator` | string | `sncf`, `renfe`, `nl` | | `date` | `YYYY-MM-DD` | departure date | | `origin`, `destination` | string | station names exactly as the operator publishes them | | `depart`, `arrive` | `HH:MM` | local time; an arrival before the departure is the next day | | `trainNo` | string | train number | -| `category` | string, optional | line or train family (SNCF: the `axe`) | +| `category` | string, optional | line or train family (SNCF: the `axe`; GTFS operators: `AVE`, `Eurostar`, `Intercity`…) | | `passes` | object | per pass: `{ bookable, seat }` | `seat` is `free` or `full` when the data says so, and `unknown` when no open data does. @@ -31,6 +31,13 @@ its residence rule, which no timetable can check. | `max-senior` | as `max-jeune`, and the date is a weekday | known | | `interrail` | the train runs: every TGV INOUI, Intercités, night and international train takes a pass-holder reservation | unknown | +GTFS operators (`renfe`, `nl`) publish only `interrail`. Each kept trip becomes one train +per pair of stops it calls at, per date it runs, like an SNCF row; a departure after +midnight is dated the next day. Renfe leaves out AVLO, which doesn't take Interrail. The +Dutch feed keeps international and long-distance trains (Eurostar, ICE, EuroCity, +Intercity direct, Nightjet) at every stop, and domestic Intercity trains only between the +stations those serve. + ## Compact snapshot Most of a 30-day timetable is the same service repeated every day, so each distinct diff --git a/scripts/build-api.ts b/scripts/build-api.ts index f8c0d4f..07b0659 100644 --- a/scripts/build-api.ts +++ b/scripts/build-api.ts @@ -1,24 +1,66 @@ /** - * Build the static API from a feed file already on disk (default: data/raw/sncf-tgvmax.json, - * which `npm run fetch` downloads) into public/. Run: npm run build:api [-- ] + * Build the static API into public/ from the sources on disk (see `npm run fetch`): + * data/raw/sncf-tgvmax.json, or the feed file given as the first argument + * data/raw// each GTFS operator, or `--gtfs =` + * `--from YYYY-MM-DD` sets the first day of the GTFS window (default: today, UTC). + * Sources that are missing are left out; the build fails only when none is there. + * Run: npm run build:api [-- ] [--gtfs renfe=] [--from ] */ -import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; -import { buildIndex, buildOperatorApi } from "../src/api"; +import { buildIndex, buildOperatorApi, type ApiFile, type ApiIndex } from "../src/api"; +import { readGtfs } from "../src/gtfs"; +import { GTFS_OPERATORS } from "../src/operators"; import { mapSncfFeed } from "../src/operators/sncf"; -const feedPath = resolve(process.argv[2] ?? "data/raw/sncf-tgvmax.json"); +const args = process.argv.slice(2); +const gtfsDirs = new Map(); +let feedArg: string | undefined; +let from = new Date().toISOString().slice(0, 10); +for (let i = 0; i < args.length; i++) { + const a = args[i] as string; + if (a === "--gtfs") { + const [id, dir] = (args[++i] ?? "").split("="); + if (id && dir) gtfsDirs.set(id, resolve(dir)); + } else if (a === "--from") from = args[++i] ?? from; + else feedArg = a; +} const outDir = resolve("public"); +const files: ApiFile[] = []; +const entries: ApiIndex["operators"] = []; +const counts: string[] = []; + +const feedPath = resolve(feedArg ?? "data/raw/sncf-tgvmax.json"); +if (existsSync(feedPath)) { + const raw: unknown = JSON.parse(readFileSync(feedPath, "utf-8")); + if (Array.isArray(raw) && raw.length > 0) { + const sncf = buildOperatorApi("sncf", "SNCF Open Data — tgvmax (Licence Ouverte)", mapSncfFeed(raw)); + files.push(...sncf.files); + entries.push(sncf.entry); + } else console.error(`[build-api] ${feedPath} is not a non-empty array; SNCF left out.`); +} else console.error(`[build-api] no SNCF feed at ${feedPath}; SNCF left out.`); -const raw: unknown = JSON.parse(readFileSync(feedPath, "utf-8")); -if (!Array.isArray(raw) || raw.length === 0) { - console.error(`[build-api] ${feedPath} is not a non-empty array.`); +for (const { op } of GTFS_OPERATORS) { + const dir = gtfsDirs.get(op.id) ?? resolve("data/raw", op.id); + if (!existsSync(join(dir, "stop_times.txt"))) { + if (gtfsDirs.size === 0 || gtfsDirs.has(op.id)) console.error(`[build-api] no ${op.id} feed at ${dir}; left out.`); + continue; + } + const trains = await readGtfs(dir, op, from); + if (trains.length === 0) { + console.error(`[build-api] ${op.id}: no trains from ${from}; left out.`); + continue; + } + const built = buildOperatorApi(op.id, op.source, trains); + files.push(...built.files); + entries.push(built.entry); +} + +if (entries.length === 0) { + console.error("[build-api] no source to publish."); process.exit(1); } -const trains = mapSncfFeed(raw); -const updatedAt = new Date().toISOString(); -const sncf = buildOperatorApi("sncf", "SNCF Open Data — tgvmax (Licence Ouverte)", trains); -const files = [...sncf.files, buildIndex(updatedAt, [sncf.entry])]; +files.push(buildIndex(new Date().toISOString(), entries)); rmSync(join(outDir, "v1"), { recursive: true, force: true }); let bytes = 0; @@ -29,7 +71,8 @@ for (const f of files) { bytes += text.length; writeFileSync(p, text, "utf-8"); } -console.log( - `[build-api] ${raw.length} records → ${trains.length} trains → ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`, -); -for (const p of sncf.entry.passes) console.log(`[build-api] ${p.id}: ${p.trainCount} bookable trains`); +console.log(`[build-api] ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`); +for (const e of entries) { + for (const p of e.passes) counts.push(`${e.id}/${p.id}: ${p.trainCount} trains, ${e.stationCount} stations`); +} +for (const c of counts) console.log(`[build-api] ${c}`); diff --git a/scripts/fetch.ts b/scripts/fetch.ts index 3c6a01d..61cd01d 100644 --- a/scripts/fetch.ts +++ b/scripts/fetch.ts @@ -1,34 +1,83 @@ /** - * Download the SNCF feed to data/raw/sncf-tgvmax.json (git-ignored). Retries with backoff - * and never overwrites a good file with an empty or malformed response. + * Download every source to data/raw/ (git-ignored): + * data/raw/sncf-tgvmax.json the SNCF feed + * data/raw//*.txt each GTFS operator's feed, unzipped + * Retries with backoff and never overwrites a good file with an empty or malformed + * response. A GTFS operator that can't be fetched is skipped (the build leaves it out); + * the run fails only when nothing at all could be fetched. * Run: npm run fetch */ -import { mkdirSync, writeFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { GTFS_OPERATORS } from "../src/operators"; import { SNCF_FEED_URL } from "../src/operators/sncf"; -const OUT = resolve("data/raw/sncf-tgvmax.json"); +const RAW = resolve("data/raw"); const TRIES = 3; -async function download(): Promise { +async function withRetries(what: string, run: () => Promise): Promise { let last: unknown; for (let i = 1; i <= TRIES; i++) { try { - const res = await fetch(SNCF_FEED_URL, { signal: AbortSignal.timeout(120_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body: unknown = await res.json(); - if (!Array.isArray(body) || body.length === 0) throw new Error("expected a non-empty JSON array"); - return body; + return await run(); } catch (err) { last = err; - console.error(`[fetch] attempt ${i}/${TRIES} failed: ${String(err)}`); + console.error(`[fetch] ${what}: attempt ${i}/${TRIES} failed: ${String(err)}`); if (i < TRIES) await new Promise((r) => setTimeout(r, 2_000 * i)); } } throw last; } -const rows = await download(); -mkdirSync(dirname(OUT), { recursive: true }); -writeFileSync(OUT, JSON.stringify(rows), "utf-8"); -console.log(`[fetch] ${rows.length} records → ${OUT}`); +async function fetchSncf(): Promise { + const rows = await withRetries("sncf", async () => { + const res = await fetch(SNCF_FEED_URL, { signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const body: unknown = await res.json(); + if (!Array.isArray(body) || body.length === 0) throw new Error("expected a non-empty JSON array"); + return body; + }); + const out = join(RAW, "sncf-tgvmax.json"); + mkdirSync(dirname(out), { recursive: true }); + writeFileSync(out, JSON.stringify(rows), "utf-8"); + console.log(`[fetch] sncf: ${rows.length} records → ${out}`); +} + +async function fetchGtfs(id: string, url: string): Promise { + const zip = await withRetries(id, async () => { + const res = await fetch(url, { signal: AbortSignal.timeout(600_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const buf = Buffer.from(await res.arrayBuffer()); + // "PK": a zip archive, not an error page. + if (buf.length < 100 || buf[0] !== 0x50 || buf[1] !== 0x4b) throw new Error("not a zip archive"); + return buf; + }); + const tmp = join(RAW, `${id}.zip`); + const dir = join(RAW, id); + const next = `${dir}.new`; + mkdirSync(RAW, { recursive: true }); + writeFileSync(tmp, zip); + rmSync(next, { recursive: true, force: true }); + execFileSync("unzip", ["-q", "-o", tmp, "-d", next]); + if (!existsSync(join(next, "stop_times.txt"))) throw new Error(`${id}: the archive has no stop_times.txt`); + rmSync(dir, { recursive: true, force: true }); + execFileSync("mv", [next, dir]); + rmSync(tmp, { force: true }); + console.log(`[fetch] ${id}: ${(zip.length / 1e6).toFixed(1)} MB → ${dir}`); +} + +let ok = 0; +const jobs: [string, () => Promise][] = [ + ["sncf", fetchSncf], + ...GTFS_OPERATORS.map(({ op, url }): [string, () => Promise] => [op.id, () => fetchGtfs(op.id, url)]), +]; +for (const [id, job] of jobs) { + try { + await job(); + ok++; + } catch (err) { + console.error(`[fetch] ${id}: skipped (${String(err)})`); + } +} +if (ok === 0) process.exit(1); diff --git a/src/gtfs.ts b/src/gtfs.ts new file mode 100644 index 0000000..2cb6242 --- /dev/null +++ b/src/gtfs.ts @@ -0,0 +1,209 @@ +/** + * Read a GTFS timetable (the open format most European operators publish) into the open + * train format. Each kept trip becomes one `Train` per pair of stops it calls at, per date + * it runs, so a row reads like the SNCF feed's: "this train goes from A to B that day". + * + * Files are streamed line by line: a national feed's stop_times.txt can run to gigabytes. + */ +import { createReadStream, existsSync } from "node:fs"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import type { PassAvailability, PassId, Train } from "./format"; + +/** One CSV line → fields. Handles quoted fields and doubled quotes; trims padding. */ +export function parseCsvLine(line: string): string[] { + const out: string[] = []; + let field = ""; + let quoted = false; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (quoted) { + if (c === '"') { + if (line[i + 1] === '"') { + field += '"'; + i++; + } else quoted = false; + } else field += c; + } else if (c === '"') quoted = true; + else if (c === ",") { + out.push(field.trim()); + field = ""; + } else field += c; + } + out.push(field.trim()); + return out; +} + +/** Stream a GTFS table, calling `onRow` with each row keyed by column name. Missing file → no rows. */ +export async function readTable(dir: string, name: string, onRow: (row: Record) => void): Promise { + const path = join(dir, `${name}.txt`); + if (!existsSync(path)) return; + const lines = createInterface({ input: createReadStream(path, "utf-8"), crlfDelay: Infinity }); + let header: string[] | null = null; + for await (const raw of lines) { + if (!raw.trim()) continue; + const fields = parseCsvLine(raw); + if (!header) { + header = fields.map((h) => h.replace(/^/, "")); + continue; + } + const row: Record = {}; + for (let i = 0; i < header.length; i++) row[header[i] as string] = fields[i] ?? ""; + onRow(row); + } +} + +/** What to keep from one operator's feed, and how its trains map to passes. */ +export interface GtfsOperator { + id: string; + /** Attribution line for the API index: who publishes the data, under which licence. */ + source: string; + /** Keep this route? `agency` is the route's agency name. Returns the train family kept, or null. */ + route(route: Record, agency: string): string | null; + /** + * Families whose trains only count between stations the other kept families also serve. + * A national feed keeps its domestic Intercity this way between the long-distance + * stations, so the snapshot stays small. Default: none. + */ + narrow?: (family: string) => boolean; + passes(family: string): Partial>; +} + +/** "YYYYMMDD" → "YYYY-MM-DD". */ +function isoDate(d: string): string { + return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; +} + +function addDays(date: string, n: number): string { + return new Date(Date.parse(`${date}T12:00:00Z`) + n * 86_400_000).toISOString().slice(0, 10); +} + +/** GTFS "25:10:00" → minutes after midnight of the service day (1510), or NaN. */ +export function gtfsMinutes(t: string): number { + const m = /^(\d{1,2}):(\d{2})/.exec(t); + return m ? Number(m[1]) * 60 + Number(m[2]) : NaN; +} + +function hhmm(min: number): string { + const m = ((min % 1440) + 1440) % 1440; + return `${String(Math.floor(m / 60)).padStart(2, "0")}:${String(m % 60).padStart(2, "0")}`; +} + +/** The dates each service runs within [from, from + days). */ +async function serviceDates(dir: string, from: string, days: number): Promise>> { + const window = Array.from({ length: days }, (_, i) => addDays(from, i)); + const inWindow = new Set(window); + const out = new Map>(); + const get = (id: string): Set => { + let s = out.get(id); + if (!s) out.set(id, (s = new Set())); + return s; + }; + const weekdays = ["sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"]; + await readTable(dir, "calendar", (r) => { + const start = isoDate(r.start_date ?? ""); + const end = isoDate(r.end_date ?? ""); + for (const d of window) { + if (d < start || d > end) continue; + const day = weekdays[new Date(`${d}T12:00:00Z`).getUTCDay()] as string; + if (r[day] === "1") get(r.service_id ?? "").add(d); + } + }); + await readTable(dir, "calendar_dates", (r) => { + const d = isoDate(r.date ?? ""); + if (!inWindow.has(d)) return; + if (r.exception_type === "1") get(r.service_id ?? "").add(d); + else if (r.exception_type === "2") out.get(r.service_id ?? "")?.delete(d); + }); + return out; +} + +interface Call { + seq: number; + station: string; + arr: number; + dep: number; +} + +/** Read one operator's GTFS directory into trains running in [from, from + days). */ +export async function readGtfs(dir: string, op: GtfsOperator, from: string, days = 31): Promise { + const agencies = new Map(); + await readTable(dir, "agency", (r) => agencies.set(r.agency_id ?? "", r.agency_name ?? "")); + const onlyAgency = agencies.size === 1 ? [...agencies.values()][0] ?? "" : ""; + + const families = new Map(); + await readTable(dir, "routes", (r) => { + const family = op.route(r, agencies.get(r.agency_id ?? "") ?? onlyAgency); + if (family) families.set(r.route_id ?? "", family); + }); + + const trips = new Map(); + await readTable(dir, "trips", (r) => { + const family = families.get(r.route_id ?? ""); + if (family) trips.set(r.trip_id ?? "", { service: r.service_id ?? "", family, trainNo: r.trip_short_name ?? "" }); + }); + + // Platforms roll up to their station (parent_station), named after the station. + const names = new Map(); + const parents = new Map(); + await readTable(dir, "stops", (r) => { + names.set(r.stop_id ?? "", r.stop_name ?? ""); + if (r.parent_station) parents.set(r.stop_id ?? "", r.parent_station); + }); + const stationOf = (stop: string): string => names.get(parents.get(stop) ?? stop) ?? names.get(stop) ?? ""; + + const calls = new Map(); + await readTable(dir, "stop_times", (r) => { + const trip = r.trip_id ?? ""; + if (!trips.has(trip)) return; + const arr = gtfsMinutes(r.arrival_time || r.departure_time || ""); + const dep = gtfsMinutes(r.departure_time || r.arrival_time || ""); + if (Number.isNaN(arr) || Number.isNaN(dep)) return; + const list = calls.get(trip) ?? []; + list.push({ seq: Number(r.stop_sequence), station: stationOf(r.stop_id ?? ""), arr, dep }); + calls.set(trip, list); + }); + + const dates = await serviceDates(dir, from, days); + const wide = new Set(); + if (op.narrow) { + for (const [tripId, trip] of trips) { + if (op.narrow(trip.family)) continue; + for (const c of calls.get(tripId) ?? []) wide.add(c.station); + } + } + const out: Train[] = []; + for (const [tripId, trip] of trips) { + const runs = dates.get(trip.service); + const list = calls.get(tripId); + if (!runs?.size || !list) continue; + list.sort((a, b) => a.seq - b.seq); + const narrow = op.narrow?.(trip.family) ?? false; + const keep = list.filter((c, i) => c.station && c.station !== list[i - 1]?.station && (!narrow || wide.has(c.station))); + const passes = op.passes(trip.family); + for (let i = 0; i < keep.length; i++) { + for (let j = i + 1; j < keep.length; j++) { + const a = keep[i] as Call; + const b = keep[j] as Call; + if (a.station === b.station || b.arr < a.dep) continue; + // A departure past 24:00 belongs to the next calendar day. + const shift = Math.floor(a.dep / 1440); + for (const d of runs) { + const t: Train = { + operator: op.id, + date: addDays(d, shift), + origin: a.station, + destination: b.station, + depart: hhmm(a.dep), + arrive: hhmm(b.arr), + trainNo: trip.trainNo, + category: trip.family, + passes, + }; + out.push(t); + } + } + } + } + return out; +} diff --git a/src/operators/index.ts b/src/operators/index.ts new file mode 100644 index 0000000..285ac2b --- /dev/null +++ b/src/operators/index.ts @@ -0,0 +1,9 @@ +import type { GtfsOperator } from "../gtfs"; +import { NL, NL_GTFS_URL } from "./nl"; +import { RENFE, RENFE_GTFS_URL } from "./renfe"; + +/** Operators read from a GTFS feed: where to download it, and how to read it. */ +export const GTFS_OPERATORS: { op: GtfsOperator; url: string }[] = [ + { op: RENFE, url: RENFE_GTFS_URL }, + { op: NL, url: NL_GTFS_URL }, +]; diff --git a/src/operators/nl.ts b/src/operators/nl.ts new file mode 100644 index 0000000..3b3bc82 --- /dev/null +++ b/src/operators/nl.ts @@ -0,0 +1,31 @@ +import type { GtfsOperator } from "../gtfs"; + +/** The Netherlands' national timetable, every operator and mode (GTFS, CC0, via OVapi). */ +export const NL_GTFS_URL = "http://gtfs.openov.nl/gtfs-rt/gtfs-openov-nl.zip"; + +/** International and long-distance families: every stop they call at is kept. */ +const LONG_DISTANCE = ["eurostar", "ice", "eurocity", "eurocity direct", "intercity direct", "nightjet"]; + +/** Train family from the route's short name: "Intercity IC23" → "Intercity", "Eurostar" → "Eurostar". */ +function family(shortName: string): string { + const s = shortName.trim(); + const lower = s.toLowerCase(); + const long = [...LONG_DISTANCE].sort((a, b) => b.length - a.length).find((f) => lower === f || lower.startsWith(`${f} `)); + if (long) return s.slice(0, long.length); + if (lower.startsWith("intercity")) return "Intercity"; + return ""; +} + +export const NL: GtfsOperator = { + id: "nl", + source: "OVapi — Dutch national timetable, NDOV (CC0)", + route: (r) => { + if (r.route_type !== "2") return null; + return family(r.route_short_name ?? "") || null; + }, + // Domestic Intercity trains only count between the stations the long-distance trains + // serve (Amsterdam, Schiphol, Rotterdam, Utrecht, Arnhem…): every IC stop pair would + // make the snapshot tens of MB. + narrow: (f) => f === "Intercity", + passes: () => ({ interrail: { bookable: true, seat: "unknown" } }), +}; diff --git a/src/operators/renfe.ts b/src/operators/renfe.ts new file mode 100644 index 0000000..d7f4e04 --- /dev/null +++ b/src/operators/renfe.ts @@ -0,0 +1,18 @@ +import type { GtfsOperator } from "../gtfs"; + +/** Renfe's high-speed, long and medium-distance timetable (GTFS, CC BY 4.0). */ +export const RENFE_GTFS_URL = "https://ssl.renfe.com/gtransit/Fichero_AV_LD/google_transit.zip"; + +/** Renfe's low-cost brand, which doesn't take Interrail passes. */ +const NOT_INTERRAIL = new Set(["AVLO"]); + +export const RENFE: GtfsOperator = { + id: "renfe", + source: "Renfe Data — high-speed, long and medium distance (CC BY 4.0)", + route: (r) => { + const family = (r.route_short_name ?? "").trim(); + return r.route_type === "2" && family && !NOT_INTERRAIL.has(family.toUpperCase()) ? family : null; + }, + // Most of these trains need a paid pass-holder reservation; no open data says whether one is left. + passes: () => ({ interrail: { bookable: true, seat: "unknown" } }), +}; diff --git a/tests/fixtures/gtfs-mini/agency.txt b/tests/fixtures/gtfs-mini/agency.txt new file mode 100644 index 0000000..2f464f3 --- /dev/null +++ b/tests/fixtures/gtfs-mini/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_timezone +1071,RENFE OPERADORA,http://www.renfe.com,Europe/Madrid diff --git a/tests/fixtures/gtfs-mini/calendar.txt b/tests/fixtures/gtfs-mini/calendar.txt new file mode 100644 index 0000000..4be8b87 --- /dev/null +++ b/tests/fixtures/gtfs-mini/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +S1,1,1,1,1,1,0,0,20261005,20261011 +S2,0,0,0,0,0,0,0,20261005,20261011 diff --git a/tests/fixtures/gtfs-mini/calendar_dates.txt b/tests/fixtures/gtfs-mini/calendar_dates.txt new file mode 100644 index 0000000..430257f --- /dev/null +++ b/tests/fixtures/gtfs-mini/calendar_dates.txt @@ -0,0 +1,3 @@ +service_id,date,exception_type +S1,20261007,2 +S2,20261010,1 diff --git a/tests/fixtures/gtfs-mini/routes.txt b/tests/fixtures/gtfs-mini/routes.txt new file mode 100644 index 0000000..310762e --- /dev/null +++ b/tests/fixtures/gtfs-mini/routes.txt @@ -0,0 +1,4 @@ +route_id,agency_id,route_short_name,route_long_name,route_desc,route_type +R1,1071,AVE ,,,2 +R2,1071,AVLO,,,2 +R3,1071,BUS,,,3 diff --git a/tests/fixtures/gtfs-mini/stop_times.txt b/tests/fixtures/gtfs-mini/stop_times.txt new file mode 100644 index 0000000..e185de6 --- /dev/null +++ b/tests/fixtures/gtfs-mini/stop_times.txt @@ -0,0 +1,9 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence +T1,7:00:00,7:00:00,60000,01 +T1,8:20:00,8:22:00,P1,02 +T1,9:30:00,9:30:00,71801,03 +T2,10:00:00,10:00:00,60000,1 +T2,12:30:00,12:30:00,71801,2 +T3,23:30:00,23:30:00,60000,1 +T3,24:40:00,24:45:00,ZGZ,2 +T3,25:50:00,25:50:00,71801,3 diff --git a/tests/fixtures/gtfs-mini/stops.txt b/tests/fixtures/gtfs-mini/stops.txt new file mode 100644 index 0000000..6021148 --- /dev/null +++ b/tests/fixtures/gtfs-mini/stops.txt @@ -0,0 +1,5 @@ +stop_id,stop_code,stop_name,stop_lat,stop_lon,location_type,parent_station +60000,,Madrid-Puerta de Atocha-Almudena Grandes ,40.40,-3.69,, +P1,,Zaragoza platform 1,41.65,-0.91,0,ZGZ +ZGZ,,Zaragoza-Delicias,41.65,-0.91,1, +71801,,"Barcelona-Sants",41.38,2.14,, diff --git a/tests/fixtures/gtfs-mini/trips.txt b/tests/fixtures/gtfs-mini/trips.txt new file mode 100644 index 0000000..1e18fc7 --- /dev/null +++ b/tests/fixtures/gtfs-mini/trips.txt @@ -0,0 +1,4 @@ +route_id,service_id,trip_id,trip_headsign,trip_short_name +R1,S1,T1,,03063 +R2,S1,T2,,06101 +R1,S2,T3,,03171 diff --git a/tests/gtfs.test.ts b/tests/gtfs.test.ts new file mode 100644 index 0000000..4a1cadc --- /dev/null +++ b/tests/gtfs.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from "vitest"; +import { fileURLToPath } from "node:url"; +import { gtfsMinutes, parseCsvLine, readGtfs } from "../src/gtfs"; +import { RENFE } from "../src/operators/renfe"; +import { NL } from "../src/operators/nl"; + +const DIR = fileURLToPath(new URL("./fixtures/gtfs-mini", import.meta.url)); + +describe("GTFS reading", () => { + it("parses quoted, padded CSV and times past midnight", () => { + expect(parseCsvLine('a, "b, c" ,"d ""e"""')).toEqual(["a", "b, c", 'd "e"']); + expect(gtfsMinutes("8:30:00")).toBe(510); + expect(gtfsMinutes("25:50:00")).toBe(1550); + expect(gtfsMinutes("")).toBeNaN(); + }); + + it("turns each kept trip into one train per stop pair per running date", async () => { + const trains = await readGtfs(DIR, RENFE, "2026-10-05", 7); + const t1 = trains.filter((t) => t.trainNo === "03063"); + // Mon–Fri minus Wednesday's removal = 4 dates, 3 stop pairs each. + expect(new Set(t1.map((t) => t.date))).toEqual(new Set(["2026-10-05", "2026-10-06", "2026-10-08", "2026-10-09"])); + expect(t1).toHaveLength(12); + const leg = t1.find((t) => t.date === "2026-10-05" && t.destination === "Barcelona-Sants" && t.origin.startsWith("Madrid")); + expect(leg).toMatchObject({ + operator: "renfe", + origin: "Madrid-Puerta de Atocha-Almudena Grandes", + depart: "07:00", + arrive: "09:30", + category: "AVE", + passes: { interrail: { bookable: true, seat: "unknown" } }, + }); + // A platform reads as its station. + expect(t1.some((t) => t.origin === "Zaragoza-Delicias")).toBe(true); + }); + + it("skips the routes an operator leaves out (Avlo, buses)", async () => { + const trains = await readGtfs(DIR, RENFE, "2026-10-05", 7); + expect(trains.some((t) => t.trainNo === "06101")).toBe(false); + }); + + it("dates a stop pair by its departure, past midnight included", async () => { + const t3 = (await readGtfs(DIR, RENFE, "2026-10-05", 7)).filter((t) => t.trainNo === "03171"); + expect(t3.map((t) => [t.date, t.origin.slice(0, 6), t.depart, t.arrive]).sort()).toEqual([ + ["2026-10-10", "Madrid", "23:30", "00:40"], + ["2026-10-10", "Madrid", "23:30", "01:50"], + ["2026-10-11", "Zarago", "00:45", "01:50"], + ]); + }); + + it("keeps only the window asked for", async () => { + expect(await readGtfs(DIR, RENFE, "2026-11-01", 7)).toEqual([]); + }); +}); + +describe("Dutch operator", () => { + const route = (short: string, type = "2"): Record => ({ route_short_name: short, route_type: type }); + + it("keeps long-distance and Intercity trains, by family", () => { + expect(NL.route(route("Eurostar"), "NS International")).toBe("Eurostar"); + expect(NL.route(route("ICE"), "NS International")).toBe("ICE"); + expect(NL.route(route("Eurocity Direct"), "NS International")).toBe("Eurocity Direct"); + expect(NL.route(route("Intercity direct"), "NS")).toBe("Intercity direct"); + expect(NL.route(route("Intercity IC23"), "Blauwnet Keolis")).toBe("Intercity"); + expect(NL.route(route("Sprinter"), "NS")).toBeNull(); + expect(NL.route(route("Intercity", "3"), "Bus")).toBeNull(); + expect(NL.narrow?.("Intercity")).toBe(true); + expect(NL.narrow?.("Eurostar")).toBe(false); + }); +});