From e8eeb8b3d0f1f989dfb8e22768a209faceca8b51 Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:19:29 +0000 Subject: [PATCH] Add Great Britain and Sweden, read with keys from repository secrets Sweden comes from Trafiklab's GTFS Sverige 2 (CC0), which needs a free API key; Great Britain from Network Rail's open data SCHEDULE feed, which needs a free account. fetch reads TRAFIKLAB_KEY, NETWORK_RAIL_EMAIL and NETWORK_RAIL_PASSWORD and leaves an operator out while its key is unset. The Network Rail reader applies short-term overlays and cancellations over each train's permanent schedule date by date, keeps public calls of express trains and sleepers, and names stations from the feed's TIPLOC records. The stop-pair expansion is shared with the GTFS reader. --- .github/workflows/ci.yml | 7 +- .github/workflows/publish.yml | 5 + AGENTS.md | 2 +- README.md | 14 ++- docs/format.md | 8 +- scripts/build-api.ts | 16 ++- scripts/fetch.ts | 83 ++++++++++++--- src/gtfs.ts | 75 ++++++++----- src/operators/gb.ts | 181 ++++++++++++++++++++++++++++++++ src/operators/index.ts | 16 ++- src/operators/se.ts | 34 ++++++ tests/fixtures/gb-schedule.json | 14 +++ tests/gb.test.ts | 47 +++++++++ 13 files changed, 445 insertions(+), 57 deletions(-) create mode 100644 src/operators/gb.ts create mode 100644 src/operators/se.ts create mode 100644 tests/fixtures/gb-schedule.json create mode 100644 tests/gb.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb1dc11..e1db0fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - run: npm run typecheck - run: npm test - name: Build the API from the fixtures - run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05 + run: npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --gb tests/fixtures/gb-schedule.json --from 2026-10-05 live-feeds: # Operator code changes are checked against the real feeds before they publish. @@ -32,6 +32,11 @@ jobs: cache: npm - run: npm ci - run: npm run fetch + env: + # Feeds behind a free sign-up; each is left out while its secret isn't set. + TRAFIKLAB_KEY: ${{ secrets.TRAFIKLAB_KEY }} + NETWORK_RAIL_EMAIL: ${{ secrets.NETWORK_RAIL_EMAIL }} + NETWORK_RAIL_PASSWORD: ${{ secrets.NETWORK_RAIL_PASSWORD }} - name: Build the API from the real feeds run: /usr/bin/time -f "%e s, %M KB peak memory" npm run build:api env: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3c605a3..e20c67e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,6 +29,11 @@ jobs: cache: npm - run: npm ci - run: npm run fetch + env: + # Feeds behind a free sign-up; each is left out while its secret isn't set. + TRAFIKLAB_KEY: ${{ secrets.TRAFIKLAB_KEY }} + NETWORK_RAIL_EMAIL: ${{ secrets.NETWORK_RAIL_EMAIL }} + NETWORK_RAIL_PASSWORD: ${{ secrets.NETWORK_RAIL_PASSWORD }} - run: npm run build:api env: # The national feeds (Switzerland, Germany) expand to millions of rows. diff --git a/AGENTS.md b/AGENTS.md index 8679f79..7375cb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ - Never mention AI tools or assistants anywhere that is pushed: no assistant co-author or session trailers, no "generated with" lines, in commits, PRs, comments, code or docs. - Merge pull requests with squash and merge. -- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05`. +- Before a commit: `npm run check` and `npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json --gtfs renfe=tests/fixtures/gtfs-mini --gb tests/fixtures/gb-schedule.json --from 2026-10-05`. - `docs/format.md` is the contract with clients: change it in the same change as `src/format.ts`, `src/compact.ts` or `src/api.ts`. - Never claim a seat the data doesn't show: a pass whose seat no open data publishes diff --git a/README.md b/README.md index 92253c4..0b798ea 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ API per operator, built from each operator's own open timetable: | `no` | Entur (GTFS): Vy, Go-Ahead and SJ trains in Norway | NLOD | Interrail | | `fi` | Fintraffic / digitraffic.fi (GTFS): VR long-distance trains | CC BY 4.0 | Interrail | | `ie` | National Transport Authority (GTFS): Irish Rail, without the DART | CC BY 4.0 | Interrail | +| `se` | Samtrafiken via Trafiklab, GTFS Sverige 2: long-distance trains, regional between their stations. Needs `TRAFIKLAB_KEY` | CC0 | Interrail | +| `gb` | Network Rail open data SCHEDULE feed: express trains and sleepers. Needs `NETWORK_RAIL_EMAIL` and `NETWORK_RAIL_PASSWORD` | Network Rail licence (attribution) | Interrail | + +`se` and `gb` come from feeds behind a free sign-up: the publish workflow reads the keys +from repository secrets and leaves an operator out while its secret isn't set. The SNCF part is built on top of [MAX-Finder](https://github.com/offware-apps/MAX-Finder): the same feed, without MAX-Finder's "free MAX seat only" filter, so every train an @@ -44,18 +49,19 @@ Live at once Pages is o npm ci npm run check # typecheck + tests npm run build:api -- tests/fixtures/sncf-tgvmax.sample.json \ - --gtfs renfe=tests/fixtures/gtfs-mini --from 2026-10-05 # build from the fixtures + --gtfs renfe=tests/fixtures/gtfs-mini --gb tests/fixtures/gb-schedule.json \ + --from 2026-10-05 # build from the fixtures ``` ## Next - The front-end on top of MAX-Finder's search core, with Interrail as the pass. - More operators through the same GTFS reader. Waiting on a free sign-up or a licence - check: Great Britain (Network Rail), Sweden (Trafiklab key), Belgium (SNCB), Austria - (ÖBB), Denmark (Rejseplanen licence). + check: Belgium (SNCB), Austria (ÖBB), Denmark (Rejseplanen licence). Data: SNCF Open Data (Licence Ouverte), Renfe Data (CC BY 4.0), OVapi / NDOV (CC0), DELFI e.V. / gtfs.de (CC BY 4.0), Trenitalia via the Italian NAP and Clément Desouche (CC BY 4.0), opentransportdata.swiss, Entur (NLOD), Fintraffic / digitraffic.fi (CC BY 4.0), -National Transport Authority (CC BY 4.0). +National Transport Authority (CC BY 4.0), Samtrafiken (CC0), contains information of +Network Rail Infrastructure Limited. Code: AGPL-3.0. diff --git a/docs/format.md b/docs/format.md index 14044f9..7f23e03 100644 --- a/docs/format.md +++ b/docs/format.md @@ -7,7 +7,7 @@ publishes it per pass in the compact snapshot format (`src/compact.ts`). | Field | Type | Meaning | |-------|------|---------| -| `operator` | string | `sncf`, `renfe`, `nl`, `de`, `it`, `ch`, `no`, `fi`, `ie` | +| `operator` | string | `sncf`, `renfe`, `nl`, `de`, `it`, `ch`, `no`, `fi`, `ie`, `se`, `gb` | | `date` | `YYYY-MM-DD` | departure date | | `origin`, `destination` | string | station names exactly as the operator publishes them | | `depart`, `arrive` | `HH:MM` | local time; an arrival before the departure is the next day | @@ -43,6 +43,12 @@ suburban lines, charters and trains that don't take Interrail (Flytoget, the DAR panoramic trains) left out. `trainNo` is the trip's number, or the route's short name when the feed has none (Germany and Norway give only the line, e.g. `ICE 1`, `F6`). +Great Britain (`gb`) comes from Network Rail's SCHEDULE feed instead of GTFS. Each train's +permanent schedule is replaced, date by date, by any short-term overlay or cancellation +that covers that date. Only public calls count, express trains and sleepers are kept +(stopping trains, Heathrow Express, Lumo and Eurostar are left out), `trainNo` is the +headcode (e.g. `1S07`) and `category` the train operator (LNER, Avanti West Coast…). + ## Compact snapshot Most of a 30-day timetable is the same service repeated every day, so each distinct diff --git a/scripts/build-api.ts b/scripts/build-api.ts index 5e36dc5..42b442b 100644 --- a/scripts/build-api.ts +++ b/scripts/build-api.ts @@ -2,9 +2,10 @@ * Build the static API into public/ from the sources on disk (see `npm run fetch`): * data/raw/sncf-tgvmax.json, or the feed file given as the first argument * data/raw// each GTFS operator, or `--gtfs =` + * data/raw/gb-schedule.json.gz Network Rail's SCHEDULE extract, or `--gb ` * `--from YYYY-MM-DD` sets the first day of the GTFS window (default: today, UTC). * Sources that are missing are left out; the build fails only when none is there. - * Run: npm run build:api [-- ] [--gtfs renfe=] [--from ] + * Run: npm run build:api [-- ] [--gtfs renfe=] [--gb ] [--from ] */ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -12,18 +13,21 @@ import { buildIndex, buildOperatorApi, type ApiFile, type ApiIndex } from "../sr import type { Train } from "../src/format"; import { readGtfs } from "../src/gtfs"; import { GTFS_OPERATORS } from "../src/operators"; +import { GB_SOURCE, readNetworkRail } from "../src/operators/gb"; import { mapSncfFeed } from "../src/operators/sncf"; const args = process.argv.slice(2); const gtfsDirs = new Map(); let feedArg: string | undefined; +let gbArg: string | undefined; let from = new Date().toISOString().slice(0, 10); for (let i = 0; i < args.length; i++) { const a = args[i] as string; if (a === "--gtfs") { const [id, dir] = (args[++i] ?? "").split("="); if (id && dir) gtfsDirs.set(id, resolve(dir)); - } else if (a === "--from") from = args[++i] ?? from; + } else if (a === "--gb") gbArg = resolve(args[++i] ?? ""); + else if (a === "--from") from = args[++i] ?? from; else feedArg = a; } const outDir = resolve("public"); @@ -76,6 +80,14 @@ for (const { op } of GTFS_OPERATORS) { publish(op.id, op.source, trains); } +// Great Britain: Network Rail's SCHEDULE extract, when `npm run fetch` had the account to get it. +const gbPath = gbArg ?? ["gb-schedule.json.gz", "gb-schedule.json"].map((f) => resolve("data/raw", f)).find((p) => existsSync(p)); +if (gbPath) { + const trains = await readNetworkRail(gbPath, from); + if (trains.length > 0) publish("gb", GB_SOURCE, trains); + else console.error(`[build-api] gb: no trains from ${from}; left out.`); +} + if (entries.length === 0) { console.error("[build-api] no source to publish."); process.exit(1); diff --git a/scripts/fetch.ts b/scripts/fetch.ts index 6b0d261..bab5143 100644 --- a/scripts/fetch.ts +++ b/scripts/fetch.ts @@ -14,6 +14,7 @@ import { pipeline } from "node:stream/promises"; import type { ReadableStream as WebReadableStream } from "node:stream/web"; import { dirname, join, resolve } from "node:path"; import { GTFS_OPERATORS } from "../src/operators"; +import { NETWORK_RAIL_ENV, NETWORK_RAIL_URL } from "../src/operators/gb"; import { SNCF_FEED_URL } from "../src/operators/sncf"; const RAW = resolve("data/raw"); @@ -50,25 +51,43 @@ async function fetchSncf(): Promise { /** Files no reader uses; shapes.txt alone runs to gigabytes in some national feeds. */ const UNUSED = ["shapes.txt", "transfers.txt", "frequencies.txt", "pathways.txt", "levels.txt", "translations.txt"]; -async function fetchGtfs(id: string, url: string, headers?: Record): Promise { - const tmp = join(RAW, `${id}.zip`); - const dir = join(RAW, id); - const next = `${dir}.new`; - mkdirSync(RAW, { recursive: true }); - // Streamed to disk: national feeds run to hundreds of MB. - const bytes = await withRetries(id, async () => { +/** Stream `url` to `out`, retrying; `ok` checks the file's first bytes. Returns its size. */ +async function download( + id: string, + url: string, + out: string, + ok: (head: Buffer) => boolean, + headers?: Record, +): Promise { + mkdirSync(dirname(out), { recursive: true }); + return withRetries(id, async () => { const res = await fetch(url, { headers, signal: AbortSignal.timeout(900_000) }); if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`); - await pipeline(Readable.fromWeb(res.body as WebReadableStream), createWriteStream(tmp)); - const head = Buffer.alloc(2); - const fd = openSync(tmp, "r"); - readSync(fd, head, 0, 2, 0); - closeSync(fd); - const size = statSync(tmp).size; - // "PK": a zip archive, not an error page. - if (size < 100 || head[0] !== 0x50 || head[1] !== 0x4b) throw new Error("not a zip archive"); + await pipeline(Readable.fromWeb(res.body as WebReadableStream), createWriteStream(out)); + const size = statSync(out).size; + // Not an error page. + if (size < 100 || !ok(firstBytes(out))) throw new Error("unexpected file type"); return size; }); +} + +function firstBytes(path: string): Buffer { + const head = Buffer.alloc(2); + const fd = openSync(path, "r"); + readSync(fd, head, 0, 2, 0); + closeSync(fd); + return head; +} + +const isZip = (h: Buffer): boolean => h[0] === 0x50 && h[1] === 0x4b; +const isGzip = (h: Buffer): boolean => h[0] === 0x1f && h[1] === 0x8b; + +async function fetchGtfs(id: string, url: string, headers?: Record): Promise { + const tmp = join(RAW, `${id}.zip`); + const dir = join(RAW, id); + const next = `${dir}.new`; + // Streamed to disk: national feeds run to hundreds of MB. "PK": a zip archive. + const bytes = await download(id, url, tmp, isZip, headers); rmSync(next, { recursive: true, force: true }); execFileSync("unzip", ["-q", "-o", tmp, "-d", next, "-x", ...UNUSED], { stdio: ["ignore", "inherit", "ignore"] }); rmSync(tmp, { force: true }); @@ -78,17 +97,47 @@ async function fetchGtfs(id: string, url: string, headers?: Record { + const user = process.env[NETWORK_RAIL_ENV.user]; + const password = process.env[NETWORK_RAIL_ENV.password]; + if (!user || !password) throw new Missing(`${NETWORK_RAIL_ENV.user} and ${NETWORK_RAIL_ENV.password}`); + const auth = `Basic ${Buffer.from(`${user}:${password}`).toString("base64")}`; + // The answer redirects to a signed download URL; fetch drops the credentials on the way. + const tmp = join(RAW, "gb-schedule.new"); + const bytes = await download("gb", NETWORK_RAIL_URL, tmp, (h) => isGzip(h) || h[0] === 0x7b, { Authorization: auth }); + // Gzipped as published, or already inflated if the host sent it with Content-Encoding. + const gzipped = isGzip(firstBytes(tmp)); + const out = join(RAW, gzipped ? "gb-schedule.json.gz" : "gb-schedule.json"); + rmSync(join(RAW, "gb-schedule.json.gz"), { force: true }); + rmSync(join(RAW, "gb-schedule.json"), { force: true }); + execFileSync("mv", [tmp, out]); + console.log(`[fetch] gb: ${(bytes / 1e6).toFixed(1)} MB → ${out}`); +} + let ok = 0; const jobs: [string, () => Promise][] = [ ["sncf", fetchSncf], - ...GTFS_OPERATORS.map(({ op, url, headers }): [string, () => Promise] => [op.id, () => fetchGtfs(op.id, url, headers)]), + ...GTFS_OPERATORS.map(({ op, url, headers, keyEnv }): [string, () => Promise] => [ + op.id, + () => { + const key = keyEnv ? process.env[keyEnv] : undefined; + if (keyEnv && !key) throw new Missing(keyEnv); + return fetchGtfs(op.id, key ? url.replace("{key}", encodeURIComponent(key)) : url, headers); + }, + ]), + ["gb", fetchGb], ]; for (const [id, job] of jobs) { try { await job(); ok++; } catch (err) { - console.error(`[fetch] ${id}: skipped (${String(err)})`); + if (err instanceof Missing) console.log(`[fetch] ${id}: no ${err.message} set; left out.`); + else console.error(`[fetch] ${id}: skipped (${String(err)})`); } } if (ok === 0) process.exit(1); diff --git a/src/gtfs.ts b/src/gtfs.ts index ef7da8b..42e8f6f 100644 --- a/src/gtfs.ts +++ b/src/gtfs.ts @@ -105,12 +105,12 @@ function isoDate(d: string): string { return `${d.slice(0, 4)}-${d.slice(4, 6)}-${d.slice(6, 8)}`; } -function addDays(date: string, n: number): string { +export function addDays(date: string, n: number): string { return new Date(Date.parse(`${date}T12:00:00Z`) + n * 86_400_000).toISOString().slice(0, 10); } /** Shared strings for the values every row repeats, so millions of rows don't each hold a copy. */ -function interner(): (s: string) => string { +export function interner(): (s: string) => string { const seen = new Map(); return (s) => { const hit = seen.get(s); @@ -160,13 +160,55 @@ async function serviceDates(dir: string, from: string, days: number): Promise string, + operator: string, + calls: Call[], + runs: Iterable, + trainNo: string, + category: string, + passes: Partial>, +): void { + const no = str(trainNo); + const dates = [...runs]; + for (let i = 0; i < calls.length; i++) { + for (let j = i + 1; j < calls.length; j++) { + const a = calls[i] as Call; + const b = calls[j] as Call; + if (a.station === b.station || b.arr < a.dep) continue; + const shift = Math.floor(a.dep / 1440); + const depart = str(hhmm(a.dep)); + const arrive = str(hhmm(b.arr)); + for (const d of dates) { + out.push({ + operator, + date: str(shift ? addDays(d, shift) : d), + origin: a.station, + destination: b.station, + depart, + arrive, + trainNo: no, + category, + passes, + }); + } + } + } +} + /** * Read one operator's GTFS directory into trains running in [from, from + days). * The train number is the trip's short name, or the route's when the feed gives none @@ -234,32 +276,7 @@ export async function readGtfs(dir: string, op: GtfsOperator, from: string, days list.sort((a, b) => a.seq - b.seq); const narrow = op.narrow?.(trip.family) ?? false; const keep = list.filter((c, i) => c.station && c.station !== list[i - 1]?.station && (!narrow || wide.has(c.station))); - const passes = op.passes(trip.family); - const trainNo = str(trip.trainNo); - for (let i = 0; i < keep.length; i++) { - for (let j = i + 1; j < keep.length; j++) { - const a = keep[i] as Call; - const b = keep[j] as Call; - if (a.station === b.station || b.arr < a.dep) continue; - // A departure past 24:00 belongs to the next calendar day. - const shift = Math.floor(a.dep / 1440); - const depart = str(hhmm(a.dep)); - const arrive = str(hhmm(b.arr)); - for (const d of trip.runs) { - out.push({ - operator: op.id, - date: str(shift ? addDays(d, shift) : d), - origin: a.station, - destination: b.station, - depart, - arrive, - trainNo, - category: trip.family, - passes, - }); - } - } - } + pushStopPairs(out, str, op.id, keep, trip.runs, trip.trainNo, trip.family, op.passes(trip.family)); } return out; } diff --git a/src/operators/gb.ts b/src/operators/gb.ts new file mode 100644 index 0000000..d097cad --- /dev/null +++ b/src/operators/gb.ts @@ -0,0 +1,181 @@ +/** + * Great Britain's timetable from Network Rail's open data SCHEDULE feed (the daily full + * extract, newline-delimited JSON, gzipped). Needs a free Network Rail open data account: + * NETWORK_RAIL_EMAIL and NETWORK_RAIL_PASSWORD, repository secrets in the workflows. + * + * The feed lists each train's schedules with the days and date range they apply to, plus + * short-term overlays and cancellations (the STP indicator) that override the permanent + * schedule on their dates. Locations are TIPLOCs, named by the feed's own TIPLOC records. + */ +import { createReadStream, existsSync } from "node:fs"; +import { createInterface } from "node:readline"; +import { createGunzip } from "node:zlib"; +import type { PassAvailability, PassId, Train } from "../format"; +import { addDays, interner, pushStopPairs, type Call } from "../gtfs"; + +export const NETWORK_RAIL_URL = + "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"; +export const NETWORK_RAIL_ENV = { user: "NETWORK_RAIL_EMAIL", password: "NETWORK_RAIL_PASSWORD" }; +export const GB_SOURCE = "Network Rail open data — GB timetable (contains information of Network Rail Infrastructure Limited)"; + +/** Train operators by ATOC code, for the train's category. */ +const OPERATORS: Record = { + AW: "Transport for Wales", + CC: "c2c", + CH: "Chiltern Railways", + CS: "Caledonian Sleeper", + EM: "East Midlands Railway", + GC: "Grand Central", + GN: "Great Northern", + GR: "LNER", + GW: "GWR", + GX: "Gatwick Express", + HT: "Hull Trains", + LE: "Greater Anglia", + LM: "West Midlands Trains", + NT: "Northern", + SE: "Southeastern", + SN: "Southern", + SR: "ScotRail", + SW: "South Western Railway", + TL: "Thameslink", + TP: "TransPennine Express", + VT: "Avanti West Coast", + XC: "CrossCountry", +}; +/** Heathrow Express, Lumo and Eurostar don't take Interrail. */ +const NOT_INTERRAIL = new Set(["HX", "LD", "ES"]); +/** Express passenger trains and sleepers. Stopping trains (OO) would add every local stop pair. */ +const CATEGORIES = new Set(["XX", "XZ"]); +const PASSES: Partial> = { interrail: { bookable: true, seat: "unknown" } }; + +interface Location { + location_type?: string; + tiploc_code?: string; + public_arrival?: string | null; + public_departure?: string | null; +} + +interface Schedule { + uid: string; + stp: string; + start: string; + end: string; + days: string; + trainNo: string; + operator: string; + /** False for a train left out (a stopping train, a non-Interrail operator): it still wins its dates. */ + kept: boolean; + calls: { tiploc: string; arr: number; dep: number }[]; +} + +/** "0745" or "0745H" → minutes after midnight; null, "" or "0000" on a calling point → no public time. */ +function publicMinutes(t: string | null | undefined, allowMidnight: boolean): number | null { + if (!t || !/^\d{4}/.test(t) || (!allowMidnight && t.startsWith("0000"))) return null; + return Number(t.slice(0, 2)) * 60 + Number(t.slice(2, 4)); +} + +/** The public calls of a schedule, with times running past midnight kept increasing. */ +function publicCalls(locations: Location[]): Schedule["calls"] { + const out: Schedule["calls"] = []; + let offset = 0; + let last = -1; + const roll = (m: number): number => { + let t = m + offset; + if (t < last) { + offset += 1440; + t += 1440; + } + last = t; + return t; + }; + for (const l of locations) { + const type = l.location_type; + const first = type === "LO"; + const end = type === "LT"; + const arr = first ? null : publicMinutes(l.public_arrival, end); + const dep = end ? null : publicMinutes(l.public_departure, first); + if (arr === null && dep === null) continue; + const a = roll(arr ?? (dep as number)); + const d = dep === null ? a : roll(dep); + out.push({ tiploc: l.tiploc_code ?? "", arr: a, dep: d }); + } + return out; +} + +/** Does a schedule apply on this date ("YYYY-MM-DD")? `days` is Monday first. */ +function appliesOn(s: Schedule, date: string): boolean { + if (date < s.start || date > s.end) return false; + const weekday = (new Date(`${date}T12:00:00Z`).getUTCDay() + 6) % 7; + return s.days[weekday] === "1"; +} + +/** Short-term cancellations, overlays and new schedules win over the permanent one. */ +const STP_RANK: Record = { C: 0, N: 1, O: 1, P: 2 }; + +/** Read the gzipped (or plain) SCHEDULE extract into trains running in [from, from + days). */ +export async function readNetworkRail(path: string, from: string, days = 31): Promise { + if (!existsSync(path)) return []; + const raw = createReadStream(path); + const input = path.endsWith(".gz") ? raw.pipe(createGunzip()) : raw; + const names = new Map(); + const byUid = new Map(); + const until = addDays(from, days - 1); + for await (const line of createInterface({ input, crlfDelay: Infinity })) { + if (line.startsWith('{"TiplocV1"')) { + const t = (JSON.parse(line) as { TiplocV1: { tiploc_code?: string; tps_description?: string; description?: string } }).TiplocV1; + const name = (t.description || t.tps_description || "").trim(); + if (t.tiploc_code && name) names.set(t.tiploc_code, name); + continue; + } + if (!line.startsWith('{"JsonScheduleV1"')) continue; + const j = (JSON.parse(line) as { JsonScheduleV1: Record }).JsonScheduleV1; + const stp = String(j.CIF_stp_indicator ?? ""); + const start = String(j.schedule_start_date ?? ""); + const end = String(j.schedule_end_date ?? ""); + if (!(stp in STP_RANK) || end < from || start > until) continue; + const seg = (j.schedule_segment ?? {}) as { CIF_train_category?: string; signalling_id?: string; schedule_location?: Location[] }; + const operator = String(j.atoc_code ?? ""); + // Every schedule is kept for its dates, so an overlay onto a left-out train still hides the + // permanent one. Cancellations carry no train details. + const kept = stp !== "C" && CATEGORIES.has(seg.CIF_train_category ?? "") && !NOT_INTERRAIL.has(operator); + const s: Schedule = { + uid: String(j.CIF_train_uid ?? ""), + stp, + start, + end, + days: String(j.schedule_days_runs ?? ""), + trainNo: seg.signalling_id ?? "", + operator, + kept, + calls: kept ? publicCalls(seg.schedule_location ?? []) : [], + }; + const list = byUid.get(s.uid); + if (list) list.push(s); + else byUid.set(s.uid, [s]); + } + + const window = Array.from({ length: days }, (_, i) => addDays(from, i)); + const str = interner(); + const out: Train[] = []; + for (const schedules of byUid.values()) { + // The dates each schedule is the one in force on. + const runs = new Map(); + for (const d of window) { + let best: Schedule | undefined; + for (const s of schedules) { + if (appliesOn(s, d) && (!best || (STP_RANK[s.stp] ?? 9) < (STP_RANK[best.stp] ?? 9))) best = s; + } + if (!best?.kept) continue; + const list = runs.get(best); + if (list) list.push(d); + else runs.set(best, [d]); + } + for (const [s, dates] of runs) { + const calls: Call[] = s.calls.map((c, i) => ({ seq: i, station: names.get(c.tiploc) ?? c.tiploc, arr: c.arr, dep: c.dep })); + const kept = calls.filter((c, i) => c.station !== calls[i - 1]?.station); + pushStopPairs(out, str, "gb", kept, dates, s.trainNo, OPERATORS[s.operator] ?? s.operator, PASSES); + } + } + return out; +} diff --git a/src/operators/index.ts b/src/operators/index.ts index 8ae7b63..bffcccf 100644 --- a/src/operators/index.ts +++ b/src/operators/index.ts @@ -7,9 +7,20 @@ import { IT, IT_GTFS_URL } from "./it"; import { NL, NL_GTFS_URL } from "./nl"; import { NO, NO_GTFS_URL } from "./no"; import { RENFE, RENFE_GTFS_URL } from "./renfe"; +import { SE, SE_GTFS_URL, SE_HEADERS, SE_KEY_ENV } from "./se"; -/** Operators read from a GTFS feed: where to download it, and how to read it. */ -export const GTFS_OPERATORS: { op: GtfsOperator; url: string; headers?: Record }[] = [ +/** + * Operators read from a GTFS feed: where to download it, and how to read it. A feed behind + * a free sign-up names the environment variable holding its key (`keyEnv`, a repository + * secret in the workflows); `{key}` in the URL is replaced with it, and without it the + * operator is skipped. + */ +export const GTFS_OPERATORS: { + op: GtfsOperator; + url: string; + headers?: Record; + keyEnv?: string; +}[] = [ { op: RENFE, url: RENFE_GTFS_URL }, { op: NL, url: NL_GTFS_URL }, { op: DE, url: DE_GTFS_URL }, @@ -18,4 +29,5 @@ export const GTFS_OPERATORS: { op: GtfsOperator; url: string; headers?: Record = { + "101": "Snabbtåg", + "102": "Fjärrtåg", + "105": "Nattåg", + "103": "Regionaltåg", + "106": "Regionaltåg", +}; + +export const SE: GtfsOperator = { + id: "se", + source: "Samtrafiken via Trafiklab — GTFS Sverige 2 (CC0)", + route: (r, agency) => { + if (NOT_INTERRAIL.test(agency.trim())) return null; + // Commuter trains (pendeltåg, 109) and every non-rail mode are left out. + return FAMILIES[r.route_type ?? ""] ?? null; + }, + // Regional trains only count between the stations the long-distance trains serve. + narrow: (f) => f === "Regionaltåg", + passes: () => ({ interrail: { bookable: true, seat: "unknown" } }), +}; diff --git a/tests/fixtures/gb-schedule.json b/tests/fixtures/gb-schedule.json new file mode 100644 index 0000000..4780706 --- /dev/null +++ b/tests/fixtures/gb-schedule.json @@ -0,0 +1,14 @@ +{"JsonTimetableV1":{"classification":"public","timestamp":1,"owner":"Network Rail","Sender":{},"Metadata":{"type":"full","sequence":1}}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"KNGX","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"LONDON KINGS CROSS"}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"PBRO","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"PETERBOROUGH"}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"DONC","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"DONCASTER"}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"YORK","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"YORK"}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"HTRWJN","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"HEATHROW JN"}} +{"TiplocV1":{"transaction_type":"Create","tiploc_code":"LEEDS","nalco":"1","stanox":"1","crs_code":null,"description":null,"tps_description":"LEEDS"}} +{"JsonAssociationV1":{"transaction_type":"Create"}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"P","CIF_train_uid":"C10001","applicable_timetable":"Y","atoc_code":"GR","schedule_days_runs":"1111100","schedule_end_date":"2026-12-12","schedule_start_date":"2026-09-01","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"1S07","CIF_train_category":"XX","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"KNGX","departure":"2330","public_departure":"2330"},{"location_type":"LI","record_identity":"LI","tiploc_code":"PBRO","arrival":"2350","departure":"2352","public_arrival":"2350","public_departure":"2352","pass":null},{"location_type":"LI","record_identity":"LI","tiploc_code":"HTRWJN","arrival":"0005H","departure":"0005H","public_arrival":null,"public_departure":null,"pass":"0005H"},{"location_type":"LI","record_identity":"LI","tiploc_code":"DONC","arrival":"0040","departure":"0041","public_arrival":"0040","public_departure":"0041","pass":null},{"location_type":"LT","record_identity":"LT","tiploc_code":"YORK","arrival":"0130","public_arrival":"0130"}]}}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"O","CIF_train_uid":"C10001","applicable_timetable":"Y","atoc_code":"GR","schedule_days_runs":"0010000","schedule_end_date":"2026-10-07","schedule_start_date":"2026-10-07","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"1S07","CIF_train_category":"XX","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"KNGX","departure":"2345","public_departure":"2345"},{"location_type":"LT","record_identity":"LT","tiploc_code":"YORK","arrival":"0200","public_arrival":"0200"}]}}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"C","CIF_train_uid":"C10001","applicable_timetable":"Y","atoc_code":"GR","schedule_days_runs":"0001000","schedule_end_date":"2026-10-08","schedule_start_date":"2026-10-08","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"","CIF_train_category":"","schedule_location":[]}}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"O","CIF_train_uid":"C10001","applicable_timetable":"Y","atoc_code":"GR","schedule_days_runs":"0000100","schedule_end_date":"2026-10-09","schedule_start_date":"2026-10-09","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"1S07","CIF_train_category":"OO","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"KNGX","departure":"2330","public_departure":"2330"},{"location_type":"LT","record_identity":"LT","tiploc_code":"YORK","arrival":"0130","public_arrival":"0130"}]}}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"P","CIF_train_uid":"C20002","applicable_timetable":"Y","atoc_code":"NT","schedule_days_runs":"1111111","schedule_end_date":"2026-12-12","schedule_start_date":"2026-09-01","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"2D01","CIF_train_category":"OO","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"YORK","departure":"0800","public_departure":"0800"},{"location_type":"LT","record_identity":"LT","tiploc_code":"LEEDS","arrival":"0830","public_arrival":"0830"}]}}} +{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"P","CIF_train_uid":"C30003","applicable_timetable":"Y","atoc_code":"HX","schedule_days_runs":"1111111","schedule_end_date":"2026-12-12","schedule_start_date":"2026-09-01","train_status":"P","transaction_type":"Create","schedule_segment":{"signalling_id":"1X01","CIF_train_category":"XX","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"YORK","departure":"0900","public_departure":"0900"},{"location_type":"LT","record_identity":"LT","tiploc_code":"LEEDS","arrival":"0930","public_arrival":"0930"}]}}} diff --git a/tests/gb.test.ts b/tests/gb.test.ts new file mode 100644 index 0000000..1fd5547 --- /dev/null +++ b/tests/gb.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from "vitest"; +import { fileURLToPath } from "node:url"; +import { readNetworkRail } from "../src/operators/gb"; +import { SE } from "../src/operators/se"; + +const FEED = fileURLToPath(new URL("./fixtures/gb-schedule.json", import.meta.url)); + +describe("Network Rail schedule (Great Britain)", () => { + it("applies overlays and cancellations over the permanent schedule, date by date", async () => { + const trains = await readNetworkRail(FEED, "2026-10-05", 5); + const kgxYork = trains.filter((t) => t.origin === "LONDON KINGS CROSS" && t.destination === "YORK"); + expect(kgxYork.map((t) => [t.date, t.depart, t.arrive])).toEqual([ + ["2026-10-05", "23:30", "01:30"], + ["2026-10-06", "23:30", "01:30"], + // Wednesday's overlay; Thursday is cancelled; Friday's overlay is a stopping train, left out. + ["2026-10-07", "23:45", "02:00"], + ]); + expect(kgxYork[0]).toMatchObject({ operator: "gb", trainNo: "1S07", category: "LNER", passes: { interrail: { bookable: true, seat: "unknown" } } }); + }); + + it("keeps public calls only, and dates a departure after midnight the next day", async () => { + const trains = await readNetworkRail(FEED, "2026-10-05", 1); + expect(trains.some((t) => t.origin === "HEATHROW JN" || t.destination === "HEATHROW JN")).toBe(false); + expect(trains.find((t) => t.origin === "DONCASTER")).toMatchObject({ date: "2026-10-06", depart: "00:41", destination: "YORK" }); + expect(trains.find((t) => t.origin === "PETERBOROUGH" && t.destination === "DONCASTER")).toMatchObject({ date: "2026-10-05", depart: "23:52", arrive: "00:40" }); + }); + + it("leaves out stopping trains and operators that don't take Interrail", async () => { + const trains = await readNetworkRail(FEED, "2026-10-05", 5); + expect(trains.some((t) => t.trainNo === "2D01" || t.trainNo === "1X01")).toBe(false); + }); + + it("reads nothing when the extract is missing", async () => { + expect(await readNetworkRail("/nonexistent/gb.json.gz", "2026-10-05")).toEqual([]); + }); +}); + +describe("Swedish operator", () => { + it("keeps long-distance and regional trains, not commuter trains or non-Interrail operators", () => { + expect(SE.route({ route_type: "101" }, "SJ")).toBe("Snabbtåg"); + expect(SE.route({ route_type: "106" }, "Västtrafik")).toBe("Regionaltåg"); + expect(SE.narrow?.("Regionaltåg")).toBe(true); + expect(SE.route({ route_type: "109" }, "SL")).toBeNull(); + expect(SE.route({ route_type: "102" }, "Snälltåget")).toBeNull(); + expect(SE.route({ route_type: "700" }, "SJ")).toBeNull(); + }); +});