diff --git a/AGENTS.md b/AGENTS.md index 58dd3af..23dc38c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,8 @@ code that writes the files. - One folder under `v1/` per dataset. A new dataset also gets an entry in `src/catalog.ts` and its sources in `SOURCES`. -- Respect every licence: values come from Wikidata (CC0) only. Wikivoyage and Wikipedia - are linked, never copied. A new source needs its licence in `SOURCES` and the README. -- Never correct data here: errors are fixed on Wikidata, at each record's `source`. +- Respect every licence: a dataset carries the licence of its sources, and a share-alike + source (ODbL, CC BY-SA) makes the dataset share-alike too. Wikivoyage and Wikipedia are + linked, never copied. A new source needs its licence and credit in `src/sources.ts` and + the README. +- Never correct data here: errors are fixed at the source (Wikidata, GeoNames, Trainline…). diff --git a/README.md b/README.md index 89bcdf5..2b74da5 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # data The open data every Offware app reads, in one place: towns and cities worldwide with their -sights and links to free travel guides, and railway stations worldwide. Built from open -sources by GitHub Actions and served as static JSON on GitHub Pages, with no server, no key -and no cost. +sights and links to free travel guides, railway stations, cities, regions, airports, UNESCO +World Heritage sites, countries and Postcards' moments. Built from open sources by GitHub +Actions and served as static JSON on GitHub Pages, with no server, no key and no cost. [train-api](https://github.com/offware-apps/train-api) links its stations to these places, -and [Postcards](https://github.com/offware-apps/Postcards) reads the sights and stations. +and [Postcards](https://github.com/offware-apps/Postcards) reads every other dataset. Live at once Pages is on (Settings → Pages → Source: GitHub Actions). @@ -23,13 +23,39 @@ built, how many records it has and which apps read it. | | `v1/places/landmarks.json` | Every sight except UNESCO World Heritage sites, in Postcards' landmark shape | | | `v1/places/pack.json` | The sights as a Postcards community data pack | | stations | `v1/stations/railways.json` | Open railway stations worldwide, in Postcards' station shape | -| all | `v1/sources.json` | Every source and its licence | +| stations-europe | `v1/stations-europe/railways.json` | About 70,000 European stations from Trainline, same shape | +| cities | `v1/cities/top.json`, `v1/cities/all.json` | The 10,000 most populous cities, and all 135,000 | +| regions | `v1/regions/regions.json` | First-level regions (states, provinces, régions) with their names | +| airports | `v1/airports/airports.json` | Every airport with an IATA code | +| heritage | `v1/heritage/unesco.json` | Every UNESCO World Heritage site, once per country | +| countries | `v1/countries/*.json` | Common names, continents, languages and UN membership | +| moments | `v1/moments/moments.json` | Postcards' travel experiences, each citing its sources | +| all | `v1/sources.json` | Every source, its licence and the credit it asks for | Towns are kept from 5,000 people in Europe and from 15,000 elsewhere. The format is in [docs/format.md](docs/format.md). ## Sources and licences +Each dataset keeps the licence of what it is built from, given as `license` in +`catalog.json`. A share-alike licence (ODbL, CC BY-SA) carries over to the dataset, and +an app showing the data shows the source's `attribution` from `sources.json`. + +| Dataset | Licence | Source | +| --- | --- | --- | +| places, stations | CC0 1.0 | Wikidata | +| stations-europe | ODbL 1.0 | [Trainline EU stations](https://github.com/trainline-eu/stations) | +| cities | CC BY 4.0 | [GeoNames](https://www.geonames.org), by way of all-the-cities | +| regions | ODbL 1.0 | GeoNames, and [dr5hn countries-states-cities](https://github.com/dr5hn/countries-states-cities-database) for a few names | +| airports | ODbL 1.0 | [OpenFlights](https://openflights.org/data.html) (from OurAirports, public domain) | +| heritage | CC BY-SA 3.0 IGO | [UNESCO World Heritage Centre](https://whc.unesco.org/en/list/) | +| countries | ODbL 1.0 | [mledoze/countries](https://github.com/mledoze/countries), ISO 3166-1 codes | +| moments | Each item cites its sources | Written for Postcards (CC0), drawn from Wikidata, GeoNames, UNESCO and Wikivoyage | + +The vendored source files are in `data/reference/`: GeoNames' admin-1 names and city codes +(export of 2026-09-30, refreshed with `npm run city-admin1`, see `scripts/city-admin1.ts`), +UNESCO's list (retrieved 2026-07-11) and the moments. + - **Wikidata (CC0).** Every name, population, coordinate, category and UNESCO flag comes from Wikidata. CC0 asks for nothing, but every record still names its source: its `source` field is its Wikidata item. @@ -45,9 +71,9 @@ build. Corrections are never made in this repository. ```sh npm ci npm run check # typecheck + tests -npm run build -- tests/fixtures/raw.json # build from the fixture +npm run build -- tests/fixtures/raw.json # build from the fixture (add --offline to skip Trainline) npm run fetch -- LU MC # fetch two countries (needs query.wikidata.org) npm run build ``` -Code: AGPL-3.0. Data: CC0 1.0, from Wikidata. +Code: AGPL-3.0. Data: each dataset under its source's licence, as above. diff --git a/data/reference/admin1CodesASCII.txt.gz b/data/reference/admin1CodesASCII.txt.gz new file mode 100644 index 0000000..4e3d3ef Binary files /dev/null and b/data/reference/admin1CodesASCII.txt.gz differ diff --git a/data/reference/city-admin1.tsv.gz b/data/reference/city-admin1.tsv.gz new file mode 100644 index 0000000..38f2b29 Binary files /dev/null and b/data/reference/city-admin1.tsv.gz differ diff --git a/data/reference/moments.json b/data/reference/moments.json new file mode 100644 index 0000000..3465a83 --- /dev/null +++ b/data/reference/moments.json @@ -0,0 +1,113 @@ +{ + "name": "Postcards experiences (Moments) list", + "version": 1, + "license": "Mixed — only each moment's name/hint/emoji are app-authored (CC0-1.0); anchor coordinates and experience concepts are AGGREGATED from external openly-licensed sources listed in each item's `sources` (Wikidata CC0, GeoNames CC BY 4.0, UNESCO World Heritage & Intangible Cultural Heritage CC BY-SA 3.0 IGO, Wikivoyage travel topics CC BY-SA 3.0).", + "note": "V1. Moments are aggregated, not authored: anchor coordinates come from Wikidata (CC0) and GeoNames (CC BY 4.0); the experience concepts are attributed to UNESCO World Heritage / UNESCO Intangible Cultural Heritage (CC BY-SA 3.0 IGO) and Wikivoyage travel topics (CC BY-SA 3.0). Only each moment's short name, hint and emoji are app-authored (CC0-1.0). Added items carry a per-item `sources` list citing the specific record and its license. Each moment lists real places where it can be lived (name, coordinates, country) so the map can fly you there; the first spot is the primary anchor used to file the moment under its home continent and country. A community-maintained dataset can replace this file later — same shape, same ids.", + "experiences": [ + { "id": "xp-aurora-borealis", "emoji": "🌌", "name": "See the northern lights", "hint": "Aurora borealis; far northern latitudes in winter", "where": [ { "name": "Tromsø", "lat": 69.65, "lon": 18.96, "cc": "NO" }, { "name": "Fairbanks", "lat": 64.84, "lon": -147.72, "cc": "US" }, { "name": "Reykjavík", "lat": 64.13, "lon": -21.90, "cc": "IS" } ] }, + { "id": "xp-aurora-australis", "emoji": "🌠", "name": "See the southern lights", "hint": "Aurora australis; far southern latitudes", "where": [ { "name": "Ushuaia", "lat": -54.80, "lon": -68.30, "cc": "AR" }, { "name": "Queenstown", "lat": -45.03, "lon": 168.66, "cc": "NZ" }, { "name": "Hobart", "lat": -42.88, "lon": 147.33, "cc": "AU" } ] }, + { "id": "xp-midnight-sun", "emoji": "🌞", "name": "Experience the midnight sun", "hint": "Sun above the horizon at midnight; polar summer", "where": [ { "name": "Nordkapp", "lat": 71.17, "lon": 25.78, "cc": "NO" }, { "name": "Nuuk", "lat": 64.18, "lon": -51.69, "cc": "GL" } ] }, + { "id": "xp-polar-night", "emoji": "🌑", "name": "Live a polar night", "hint": "A day the sun never rises; polar winter", "where": [ { "name": "Longyearbyen", "lat": 78.22, "lon": 15.65, "cc": "SJ" } ] }, + { "id": "xp-santa-rovaniemi", "emoji": "🎅", "name": "Meet Santa Claus in Lapland", "hint": "Rovaniemi, Finland; the Santa Claus Village", "where": [ { "name": "Rovaniemi", "lat": 66.50, "lon": 25.73, "cc": "FI" } ] }, + { "id": "xp-total-solar-eclipse", "emoji": "🌘", "name": "Stand in a total solar eclipse", "hint": "Inside the path of totality; the path moves each time" }, + { "id": "xp-cross-equator", "emoji": "🌍", "name": "Cross the equator", "hint": "By land, sea or air", "where": [ { "name": "Mitad del Mundo", "lat": -0.00, "lon": -78.46, "cc": "EC" }, { "name": "Nanyuki", "lat": 0.01, "lon": 37.07, "cc": "KE" }, { "name": "Pontianak", "lat": -0.02, "lon": 109.34, "cc": "ID" } ] }, + { "id": "xp-cross-date-line", "emoji": "🕐", "name": "Cross the international date line", "hint": "Lose or gain a whole day", "where": [ { "name": "Taveuni", "lat": -16.85, "lon": 179.97, "cc": "FJ" } ] }, + { "id": "xp-arctic-circle", "emoji": "🧭", "name": "Cross the Arctic Circle", "hint": "66°33′ north", "where": [ { "name": "Rovaniemi", "lat": 66.55, "lon": 25.83, "cc": "FI" }, { "name": "Saltfjellet", "lat": 66.57, "lon": 15.36, "cc": "NO" } ] }, + { "id": "xp-antarctica", "emoji": "🐧", "name": "Set foot on Antarctica", "hint": "The seventh continent", "where": [ { "name": "Antarctic Peninsula", "lat": -63.50, "lon": -57.00, "cc": "AQ" } ] }, + { "id": "xp-safari", "emoji": "🦁", "name": "Go on a wildlife safari", "hint": "See big wildlife in its own habitat", "where": [ { "name": "Serengeti", "lat": -2.33, "lon": 34.83, "cc": "TZ" }, { "name": "Maasai Mara", "lat": -1.50, "lon": 35.10, "cc": "KE" }, { "name": "Kruger", "lat": -24.00, "lon": 31.50, "cc": "ZA" } ] }, + { "id": "xp-great-migration", "emoji": "🦓", "name": "Witness the great migration", "hint": "Wildebeest and zebra crossing the Mara River", "where": [ { "name": "Serengeti", "lat": -2.15, "lon": 34.69, "cc": "TZ" }, { "name": "Maasai Mara", "lat": -1.41, "lon": 35.00, "cc": "KE" } ] }, + { "id": "xp-mountain-gorillas", "emoji": "🦍", "name": "Trek to mountain gorillas", "hint": "Bwindi or the Virunga volcanoes", "where": [ { "name": "Bwindi", "lat": -1.06, "lon": 29.68, "cc": "UG" }, { "name": "Volcanoes NP", "lat": -1.48, "lon": 29.49, "cc": "RW" } ] }, + { "id": "xp-polar-bears", "emoji": "🐻‍❄️", "name": "See wild polar bears", "hint": "Churchill, on the shore of Hudson Bay", "where": [ { "name": "Churchill", "lat": 58.77, "lon": -94.17, "cc": "CA" } ] }, + { "id": "xp-whale-sharks", "emoji": "🦈", "name": "Swim with whale sharks", "hint": "The gentle giants of warm seas", "where": [ { "name": "Oslob", "lat": 9.46, "lon": 123.38, "cc": "PH" }, { "name": "Ningaloo", "lat": -22.70, "lon": 113.70, "cc": "AU" }, { "name": "La Paz", "lat": 24.16, "lon": -110.31, "cc": "MX" } ] }, + { "id": "xp-coral-reef", "emoji": "🐠", "name": "Snorkel or dive a coral reef", "hint": "A living reef, mask on", "where": [ { "name": "Great Barrier Reef", "lat": -18.28, "lon": 147.70, "cc": "AU" }, { "name": "Raja Ampat", "lat": -0.23, "lon": 130.52, "cc": "ID" }, { "name": "Red Sea (Sharm)", "lat": 27.91, "lon": 34.33, "cc": "EG" } ] }, + { "id": "xp-whales", "emoji": "🐋", "name": "Watch whales at sea", "hint": "From a boat or the shore", "where": [ { "name": "Húsavík", "lat": 66.04, "lon": -17.34, "cc": "IS" }, { "name": "Kaikōura", "lat": -42.40, "lon": 173.68, "cc": "NZ" }, { "name": "Monterey", "lat": 36.60, "lon": -121.90, "cc": "US" } ] }, + { "id": "xp-sea-turtles", "emoji": "🐢", "name": "See sea turtles nest", "hint": "A beach hatching under moonlight", "where": [ { "name": "Tortuguero", "lat": 10.54, "lon": -83.50, "cc": "CR" }, { "name": "Ras Al Jinz", "lat": 22.42, "lon": 59.83, "cc": "OM" } ] }, + { "id": "xp-wild-elephants", "emoji": "🐘", "name": "See wild elephants", "hint": "A herd in the open", "where": [ { "name": "Udawalawe", "lat": 6.44, "lon": 80.89, "cc": "LK" }, { "name": "Chobe", "lat": -18.62, "lon": 24.72, "cc": "BW" } ] }, + { "id": "xp-hot-air-balloon", "emoji": "🎈", "name": "Fly in a hot air balloon", "hint": "Cappadocia, Bagan, or anywhere aloft", "where": [ { "name": "Cappadocia", "lat": 38.64, "lon": 34.83, "cc": "TR" }, { "name": "Bagan", "lat": 21.17, "lon": 94.86, "cc": "MM" } ] }, + { "id": "xp-active-volcano", "emoji": "🌋", "name": "Look into an active volcano", "hint": "A glowing crater or fresh lava field", "where": [ { "name": "Kīlauea", "lat": 19.42, "lon": -155.29, "cc": "US" }, { "name": "Stromboli", "lat": 38.79, "lon": 15.21, "cc": "IT" }, { "name": "Mount Bromo", "lat": -7.94, "lon": 112.95, "cc": "ID" } ] }, + { "id": "xp-geyser", "emoji": "⛲", "name": "Watch a geyser erupt", "hint": "Iceland, Yellowstone, El Tatio", "where": [ { "name": "Geysir", "lat": 64.31, "lon": -20.30, "cc": "IS" }, { "name": "Yellowstone", "lat": 44.46, "lon": -110.83, "cc": "US" }, { "name": "El Tatio", "lat": -22.33, "lon": -68.01, "cc": "CL" } ] }, + { "id": "xp-hot-spring", "emoji": "♨️", "name": "Bathe in a natural hot spring", "hint": "Onsen, blue lagoon, wild pool", "where": [ { "name": "Blue Lagoon", "lat": 63.88, "lon": -22.45, "cc": "IS" }, { "name": "Hakone", "lat": 35.23, "lon": 139.02, "cc": "JP" }, { "name": "Pamukkale", "lat": 37.92, "lon": 29.12, "cc": "TR" } ] }, + { "id": "xp-desert-night", "emoji": "🏜️", "name": "Sleep a night in the desert", "hint": "Dunes and a sky full of stars", "where": [ { "name": "Merzouga", "lat": 31.10, "lon": -4.01, "cc": "MA" }, { "name": "Wadi Rum", "lat": 29.53, "lon": 35.42, "cc": "JO" }, { "name": "Atacama", "lat": -24.50, "lon": -69.25, "cc": "CL" } ] }, + { "id": "xp-milky-way", "emoji": "✨", "name": "See the Milky Way with bare eyes", "hint": "A true dark-sky place", "where": [ { "name": "Atacama", "lat": -24.50, "lon": -69.25, "cc": "CL" }, { "name": "Aoraki Dark Sky", "lat": -43.73, "lon": 170.10, "cc": "NZ" }, { "name": "NamibRand", "lat": -25.00, "lon": 16.00, "cc": "NA" } ] }, + { "id": "xp-high-peak", "emoji": "🏔️", "name": "Stand above 4,000 m", "hint": "A high summit or pass, on your own feet or not", "where": [ { "name": "Everest Base Camp", "lat": 28.00, "lon": 86.85, "cc": "NP" }, { "name": "Kilimanjaro", "lat": -3.07, "lon": 37.35, "cc": "TZ" }, { "name": "Mont Blanc", "lat": 45.83, "lon": 6.86, "cc": "FR" } ] }, + { "id": "xp-glacier", "emoji": "🧊", "name": "Walk on a glacier", "hint": "Crampons on ancient ice", "where": [ { "name": "Vatnajökull", "lat": 64.41, "lon": -16.83, "cc": "IS" }, { "name": "Perito Moreno", "lat": -50.47, "lon": -73.05, "cc": "AR" }, { "name": "Franz Josef", "lat": -43.47, "lon": 170.18, "cc": "NZ" } ] }, + { "id": "xp-fjord", "emoji": "⛰️", "name": "Cruise a great fjord", "hint": "Cliffs falling straight into deep water", "where": [ { "name": "Geirangerfjord", "lat": 62.10, "lon": 7.01, "cc": "NO" }, { "name": "Milford Sound", "lat": -44.67, "lon": 167.93, "cc": "NZ" } ] }, + { "id": "xp-salt-flat", "emoji": "🪞", "name": "Walk the world's largest salt flat", "hint": "A mirror to the sky after rain", "where": [ { "name": "Salar de Uyuni", "lat": -20.13, "lon": -67.49, "cc": "BO" } ] }, + { "id": "xp-giant-dunes", "emoji": "🐪", "name": "Climb a giant sand dune", "hint": "Some of the tallest dunes on earth", "where": [ { "name": "Sossusvlei", "lat": -24.73, "lon": 15.29, "cc": "NA" }, { "name": "Merzouga", "lat": 31.10, "lon": -4.01, "cc": "MA" } ] }, + { "id": "xp-great-waterfall", "emoji": "💦", "name": "Stand before a giant waterfall", "hint": "Iguazú, Victoria Falls, Niagara", "where": [ { "name": "Iguazú Falls", "lat": -25.69, "lon": -54.44, "cc": "AR" }, { "name": "Victoria Falls", "lat": -17.92, "lon": 25.86, "cc": "ZM" }, { "name": "Niagara Falls", "lat": 43.08, "lon": -79.07, "cc": "CA" } ] }, + { "id": "xp-grand-canyon", "emoji": "🏞️", "name": "Peer over a vast canyon", "hint": "A mile deep, cut by a river", "where": [ { "name": "Grand Canyon", "lat": 36.06, "lon": -112.14, "cc": "US" }, { "name": "Fish River Canyon", "lat": -27.60, "lon": 17.62, "cc": "NA" } ] }, + { "id": "xp-glowworm-cave", "emoji": "🕳️", "name": "Explore a cave of wonders", "hint": "Glowworms overhead, or the world's biggest cave", "where": [ { "name": "Waitomo", "lat": -38.26, "lon": 175.10, "cc": "NZ" }, { "name": "Sơn Đoòng", "lat": 17.45, "lon": 106.29, "cc": "VN" } ] }, + { "id": "xp-rainforest", "emoji": "🌴", "name": "Walk deep into rainforest", "hint": "The canopy loud with life", "where": [ { "name": "Amazon (Manaus)", "lat": -3.12, "lon": -60.02, "cc": "BR" }, { "name": "Borneo (Danum)", "lat": 4.97, "lon": 117.80, "cc": "MY" } ] }, + { "id": "xp-bioluminescence", "emoji": "🌊", "name": "Swim in bioluminescent water", "hint": "A glowing night sea", "where": [ { "name": "Mosquito Bay", "lat": 18.10, "lon": -65.44, "cc": "PR" }, { "name": "Vaadhoo", "lat": 3.20, "lon": 72.94, "cc": "MV" } ] }, + { "id": "xp-sail-open-sea", "emoji": "⛵", "name": "Sail out of sight of land", "hint": "Open water in every direction", "where": [ { "name": "Cape Horn", "lat": -55.98, "lon": -67.27, "cc": "CL" } ] }, + { "id": "xp-legendary-train", "emoji": "🚂", "name": "Ride a legendary long-distance train", "hint": "Trans-Siberian, the Ghan, a night train across a continent", "where": [ { "name": "Trans-Siberian (Moscow)", "lat": 55.76, "lon": 37.62, "cc": "RU" }, { "name": "The Ghan (Darwin)", "lat": -12.46, "lon": 130.84, "cc": "AU" } ] }, + { "id": "xp-night-train", "emoji": "🛌", "name": "Cross a border on a night train", "hint": "Fall asleep in one country, wake in another", "where": [ { "name": "Vienna", "lat": 48.19, "lon": 16.36, "cc": "AT" } ] }, + { "id": "xp-iconic-road-trip", "emoji": "🚗", "name": "Drive an iconic road", "hint": "Route 66, the Great Ocean Road, a coast highway", "where": [ { "name": "Route 66 (Flagstaff)", "lat": 35.20, "lon": -111.65, "cc": "US" }, { "name": "Great Ocean Road", "lat": -38.68, "lon": 143.10, "cc": "AU" } ] }, + { "id": "xp-gondola-venice", "emoji": "🚣", "name": "Glide a canal city by boat", "hint": "Venice by gondola, or the backwaters of Kerala", "where": [ { "name": "Venice", "lat": 45.44, "lon": 12.34, "cc": "IT" }, { "name": "Kerala backwaters", "lat": 9.50, "lon": 76.38, "cc": "IN" } ] }, + { "id": "xp-ski-snowboard", "emoji": "🎿", "name": "Ski or snowboard great powder", "hint": "The Alps or deep Japanese snow", "where": [ { "name": "Chamonix", "lat": 45.92, "lon": 6.87, "cc": "FR" }, { "name": "Niseko", "lat": 42.80, "lon": 140.69, "cc": "JP" } ] }, + { "id": "xp-dog-sled", "emoji": "🛷", "name": "Ride a dog sled", "hint": "A husky team across the snow", "where": [ { "name": "Tromsø", "lat": 69.65, "lon": 18.96, "cc": "NO" }, { "name": "Rovaniemi", "lat": 66.50, "lon": 25.73, "cc": "FI" } ] }, + { "id": "xp-ice-hotel", "emoji": "🏨", "name": "Sleep in an ice hotel", "hint": "A room carved from snow and ice", "where": [ { "name": "Jukkasjärvi", "lat": 67.85, "lon": 20.60, "cc": "SE" } ] }, + { "id": "xp-bungee", "emoji": "🪢", "name": "Take a leap: bungee or zipline", "hint": "Where the sport was born, or anywhere brave", "where": [ { "name": "Queenstown", "lat": -45.03, "lon": 168.66, "cc": "NZ" }, { "name": "Victoria Falls Bridge", "lat": -17.92, "lon": 25.85, "cc": "ZW" } ] }, + { "id": "xp-via-ferrata", "emoji": "🧗", "name": "Climb a via ferrata", "hint": "Cabled routes up sheer rock", "where": [ { "name": "Dolomites", "lat": 46.41, "lon": 11.84, "cc": "IT" } ] }, + { "id": "xp-rocket-launch", "emoji": "🚀", "name": "Watch a rocket launch", "hint": "A spaceport, live", "where": [ { "name": "Kennedy Space Center", "lat": 28.57, "lon": -80.65, "cc": "US" }, { "name": "Guiana Space Centre", "lat": 5.24, "lon": -52.77, "cc": "GF" } ] }, + { "id": "xp-pyramids-giza", "emoji": "🔺", "name": "Stand before the Pyramids", "hint": "Giza, the last ancient wonder", "where": [ { "name": "Giza", "lat": 29.98, "lon": 31.13, "cc": "EG" } ] }, + { "id": "xp-great-wall", "emoji": "🧱", "name": "Walk the Great Wall of China", "hint": "Ramparts over the ridgelines", "where": [ { "name": "Mutianyu", "lat": 40.43, "lon": 116.57, "cc": "CN" }, { "name": "Badaling", "lat": 40.36, "lon": 116.02, "cc": "CN" } ] }, + { "id": "xp-machu-picchu", "emoji": "🏯", "name": "Reach Machu Picchu", "hint": "The Inca city in the clouds", "where": [ { "name": "Machu Picchu", "lat": -13.16, "lon": -72.55, "cc": "PE" } ] }, + { "id": "xp-taj-mahal", "emoji": "🕌", "name": "See the Taj Mahal at dawn", "hint": "Marble turning gold at sunrise", "where": [ { "name": "Agra", "lat": 27.17, "lon": 78.04, "cc": "IN" } ] }, + { "id": "xp-petra", "emoji": "🏜️", "name": "Walk into Petra", "hint": "The Treasury at the end of the Siq", "where": [ { "name": "Petra", "lat": 30.33, "lon": 35.44, "cc": "JO" } ] }, + { "id": "xp-colosseum", "emoji": "🏛️", "name": "Stand in the Colosseum", "hint": "Two thousand years of Rome", "where": [ { "name": "Rome", "lat": 41.89, "lon": 12.49, "cc": "IT" } ] }, + { "id": "xp-angkor-wat", "emoji": "🛕", "name": "See Angkor Wat at sunrise", "hint": "Temple towers over the jungle", "where": [ { "name": "Siem Reap", "lat": 13.41, "lon": 103.87, "cc": "KH" } ] }, + { "id": "xp-christ-redeemer", "emoji": "⛰️", "name": "Look out from Christ the Redeemer", "hint": "Rio spread out far below", "where": [ { "name": "Rio de Janeiro", "lat": -22.95, "lon": -43.21, "cc": "BR" } ] }, + { "id": "xp-chichen-itza", "emoji": "🐍", "name": "Climb the steps of a Maya city", "hint": "Chichén Itzá, Tikal, Palenque", "where": [ { "name": "Chichén Itzá", "lat": 20.68, "lon": -88.57, "cc": "MX" }, { "name": "Tikal", "lat": 17.22, "lon": -89.62, "cc": "GT" } ] }, + { "id": "xp-eiffel-tower", "emoji": "🗼", "name": "Go up the Eiffel Tower", "hint": "Paris from the top", "where": [ { "name": "Paris", "lat": 48.86, "lon": 2.29, "cc": "FR" } ] }, + { "id": "xp-statue-liberty", "emoji": "🗽", "name": "Meet the Statue of Liberty", "hint": "New York harbour", "where": [ { "name": "New York", "lat": 40.69, "lon": -74.04, "cc": "US" } ] }, + { "id": "xp-sagrada-familia", "emoji": "⛪", "name": "Step inside the Sagrada Família", "hint": "Gaudí's forest of light", "where": [ { "name": "Barcelona", "lat": 41.40, "lon": 2.17, "cc": "ES" } ] }, + { "id": "xp-northern-lights-igloo", "emoji": "🛖", "name": "Sleep in a glass igloo", "hint": "The sky as your ceiling", "where": [ { "name": "Saariselkä", "lat": 68.42, "lon": 27.41, "cc": "FI" } ] }, + { "id": "xp-cherry-blossom", "emoji": "🌸", "name": "Catch cherry blossom season", "hint": "Hanami in Japan, or any full bloom", "where": [ { "name": "Kyoto", "lat": 35.01, "lon": 135.77, "cc": "JP" }, { "name": "Washington, D.C.", "lat": 38.88, "lon": -77.04, "cc": "US" } ] }, + { "id": "xp-fall-foliage", "emoji": "🍁", "name": "Chase autumn colour", "hint": "Whole forests turning red and gold", "where": [ { "name": "New England", "lat": 44.00, "lon": -71.50, "cc": "US" }, { "name": "Kyoto", "lat": 35.01, "lon": 135.77, "cc": "JP" } ] }, + { "id": "xp-tulip-fields", "emoji": "🌷", "name": "Walk through blooming flower fields", "hint": "Tulips or endless lavender rows", "where": [ { "name": "Keukenhof", "lat": 52.27, "lon": 4.55, "cc": "NL" }, { "name": "Valensole", "lat": 43.84, "lon": 5.98, "cc": "FR" } ] }, + { "id": "xp-vineyard-harvest", "emoji": "🍇", "name": "Join a grape harvest", "hint": "Wine country in autumn", "where": [ { "name": "Bordeaux", "lat": 44.84, "lon": -0.58, "cc": "FR" }, { "name": "Tuscany", "lat": 43.47, "lon": 11.14, "cc": "IT" }, { "name": "Mendoza", "lat": -32.89, "lon": -68.85, "cc": "AR" } ] }, + { "id": "xp-street-food-market", "emoji": "🍜", "name": "Feast at a night market", "hint": "A whole street of food after dark", "where": [ { "name": "Bangkok", "lat": 13.75, "lon": 100.49, "cc": "TH" }, { "name": "Marrakech", "lat": 31.63, "lon": -7.99, "cc": "MA" }, { "name": "Taipei", "lat": 25.04, "lon": 121.50, "cc": "TW" } ] }, + { "id": "xp-carnival", "emoji": "🎭", "name": "Join a world-famous carnival", "hint": "Rio, Venice, Notting Hill", "where": [ { "name": "Rio de Janeiro", "lat": -22.91, "lon": -43.18, "cc": "BR" }, { "name": "Venice", "lat": 45.43, "lon": 12.34, "cc": "IT" }, { "name": "Notting Hill", "lat": 51.52, "lon": -0.20, "cc": "GB" } ] }, + { "id": "xp-oktoberfest", "emoji": "🍺", "name": "Raise a stein at Oktoberfest", "hint": "Munich in autumn", "where": [ { "name": "Munich", "lat": 48.13, "lon": 11.55, "cc": "DE" } ] }, + { "id": "xp-la-tomatina", "emoji": "🍅", "name": "Throw tomatoes at La Tomatina", "hint": "A town square turned red", "where": [ { "name": "Buñol", "lat": 39.42, "lon": -0.79, "cc": "ES" } ] }, + { "id": "xp-holi", "emoji": "🎨", "name": "Play Holi, the festival of colours", "hint": "Clouds of colour in the streets", "where": [ { "name": "Mathura", "lat": 27.49, "lon": 77.67, "cc": "IN" } ] }, + { "id": "xp-dia-de-muertos", "emoji": "💀", "name": "See Día de Muertos", "hint": "Marigolds, altars and remembrance", "where": [ { "name": "Oaxaca", "lat": 17.06, "lon": -96.72, "cc": "MX" } ] }, + { "id": "xp-songkran", "emoji": "💧", "name": "Join a water festival", "hint": "Songkran, the world's biggest water fight", "where": [ { "name": "Chiang Mai", "lat": 18.79, "lon": 98.98, "cc": "TH" } ] }, + { "id": "xp-lantern-festival", "emoji": "🏮", "name": "Release a sky lantern", "hint": "A night sky full of light", "where": [ { "name": "Chiang Mai (Yi Peng)", "lat": 18.79, "lon": 98.98, "cc": "TH" }, { "name": "Pingxi", "lat": 25.02, "lon": 121.74, "cc": "TW" } ] }, + { "id": "xp-running-of-bulls", "emoji": "🐂", "name": "See the running of the bulls", "hint": "San Fermín in Pamplona", "where": [ { "name": "Pamplona", "lat": 42.82, "lon": -1.64, "cc": "ES" } ] }, + { "id": "xp-new-year-abroad", "emoji": "🎆", "name": "Celebrate New Year in another country", "hint": "Fireworks far from home", "where": [ { "name": "Sydney", "lat": -33.86, "lon": 151.21, "cc": "AU" }, { "name": "Edinburgh", "lat": 55.95, "lon": -3.19, "cc": "GB" } ] }, + { "id": "xp-pilgrimage-walk", "emoji": "🥾", "name": "Walk a great pilgrimage route", "hint": "The Camino de Santiago, or the Kumano Kodō", "where": [ { "name": "Santiago de Compostela", "lat": 42.88, "lon": -8.54, "cc": "ES" }, { "name": "Kumano Kodō", "lat": 33.84, "lon": 135.77, "cc": "JP" } ] }, + { "id": "xp-grand-opera", "emoji": "🎼", "name": "See a show at a grand opera house", "hint": "Under a famous roof", "where": [ { "name": "Sydney Opera House", "lat": -33.857, "lon": 151.215, "cc": "AU" }, { "name": "La Scala", "lat": 45.47, "lon": 9.19, "cc": "IT" } ] }, + { "id": "xp-kilimanjaro", "emoji": "🏔️", "name": "Summit Kilimanjaro", "hint": "Uhuru Peak, the roof of Africa", "where": [ { "name": "Uhuru Peak", "lat": -3.0758, "lon": 37.3533, "cc": "TZ", "wikidata": "Q7296" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Kilimanjaro National Park", "license": "CC BY-SA 3.0 IGO", "version": "WHS 403", "url": "https://whc.unesco.org/en/list/403" }, { "dataset": "Wikidata — Mount Kilimanjaro", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q7296" } ] }, + { "id": "xp-baobab-avenue", "emoji": "🌳", "name": "Walk the Avenue of the Baobabs", "hint": "Giant baobabs at sunset, Menabe", "where": [ { "name": "Avenue of the Baobabs", "lat": -20.2506, "lon": 44.4183, "cc": "MG" } ], "sources": [ { "dataset": "Wikidata — Avenue of the Baobabs (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-cape-agulhas", "emoji": "🧭", "name": "Stand at Africa's southern tip", "hint": "Cape Agulhas, where two oceans meet", "where": [ { "name": "Cape Agulhas", "lat": -34.8333, "lon": 20.0, "cc": "ZA" } ], "sources": [ { "dataset": "Wikidata — Cape Agulhas (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-galapagos", "emoji": "🐢", "name": "Meet the Galápagos wildlife", "hint": "Fearless creatures of Darwin's islands", "where": [ { "name": "Puerto Ayora (Galápagos)", "lat": -0.7436, "lon": -90.3016, "cc": "EC" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Galápagos Islands", "license": "CC BY-SA 3.0 IGO", "version": "WHS 1", "url": "https://whc.unesco.org/en/list/1" }, { "dataset": "GeoNames — Puerto Ayora", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-inca-trail", "emoji": "🥾", "name": "Trek the Inca Trail", "hint": "Four days over the Andes to Machu Picchu", "where": [ { "name": "Machu Picchu", "lat": -13.1631, "lon": -72.545, "cc": "PE", "wikidata": "Q676203" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Historic Sanctuary of Machu Picchu", "license": "CC BY-SA 3.0 IGO", "version": "WHS 274", "url": "https://whc.unesco.org/en/list/274" }, { "dataset": "Wikivoyage — Inca Trail", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Inca_Trail" } ] }, + { "id": "xp-ushuaia", "emoji": "⚓", "name": "Reach the end of the world", "hint": "Ushuaia, the world's southernmost city", "where": [ { "name": "Ushuaia", "lat": -54.8019, "lon": -68.303, "cc": "AR" } ], "sources": [ { "dataset": "Wikivoyage — Ushuaia", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Ushuaia" }, { "dataset": "GeoNames — Ushuaia", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-rapa-nui-moai", "emoji": "🗿", "name": "Stand among the moai", "hint": "Rapa Nui's giant stone figures", "where": [ { "name": "Ahu Tongariki", "lat": -27.1256, "lon": -109.2769, "cc": "CL" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Rapa Nui National Park", "license": "CC BY-SA 3.0 IGO", "version": "WHS 715", "url": "https://whc.unesco.org/en/list/715" }, { "dataset": "Wikidata — Ahu Tongariki (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-monument-valley", "emoji": "🏜️", "name": "Drive through Monument Valley", "hint": "Red sandstone buttes on Navajo land", "where": [ { "name": "Monument Valley", "lat": 36.998, "lon": -110.0985, "cc": "US" } ], "sources": [ { "dataset": "Wikidata — Monument Valley (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-pantanal", "emoji": "🐆", "name": "Track wildlife in the Pantanal", "hint": "The world's largest tropical wetland", "where": [ { "name": "Corumbá (Pantanal)", "lat": -19.0086, "lon": -57.6528, "cc": "BR" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Pantanal Conservation Area", "license": "CC BY-SA 3.0 IGO", "version": "WHS 999", "url": "https://whc.unesco.org/en/list/999" }, { "dataset": "GeoNames — Corumbá", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-mount-fuji", "emoji": "🗻", "name": "Climb Mount Fuji", "hint": "Japan's sacred summit at sunrise", "where": [ { "name": "Mount Fuji", "lat": 35.3606, "lon": 138.7274, "cc": "JP", "wikidata": "Q39231" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Fujisan, sacred place and source of artistic inspiration", "license": "CC BY-SA 3.0 IGO", "version": "WHS 1418", "url": "https://whc.unesco.org/en/list/1418" }, { "dataset": "Wikidata — Mount Fuji", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q39231" } ] }, + { "id": "xp-ha-long-bay", "emoji": "🚤", "name": "Cruise Ha Long Bay", "hint": "Limestone karsts rising from the sea", "where": [ { "name": "Ha Long Bay", "lat": 20.9101, "lon": 107.1839, "cc": "VN", "wikidata": "Q190638" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Ha Long Bay", "license": "CC BY-SA 3.0 IGO", "version": "WHS 672", "url": "https://whc.unesco.org/en/list/672" }, { "dataset": "Wikidata — Hạ Long Bay", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q190638" } ] }, + { "id": "xp-borobudur", "emoji": "🛕", "name": "See Borobudur at dawn", "hint": "The world's largest Buddhist temple", "where": [ { "name": "Borobudur", "lat": -7.6079, "lon": 110.2038, "cc": "ID" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Borobudur Temple Compounds", "license": "CC BY-SA 3.0 IGO", "version": "WHS 592", "url": "https://whc.unesco.org/en/list/592" }, { "dataset": "Wikidata — Borobudur (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-terracotta-army", "emoji": "🗿", "name": "Face the Terracotta Army", "hint": "Thousands of clay warriors, Xi'an", "where": [ { "name": "Terracotta Army", "lat": 34.3841, "lon": 109.2785, "cc": "CN" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Mausoleum of the First Qin Emperor", "license": "CC BY-SA 3.0 IGO", "version": "WHS 441", "url": "https://whc.unesco.org/en/list/441" }, { "dataset": "Wikidata — Terracotta Army (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-dead-sea-float", "emoji": "🧂", "name": "Float in the Dead Sea", "hint": "The lowest point on land, too salty to sink", "where": [ { "name": "Dead Sea (Jordan shore)", "lat": 31.559, "lon": 35.4732, "cc": "JO" } ], "sources": [ { "dataset": "Wikidata — Dead Sea (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-registan", "emoji": "🕌", "name": "Stand on the Registan", "hint": "Samarkand's three great madrasas", "where": [ { "name": "Registan", "lat": 39.6542, "lon": 66.9758, "cc": "UZ" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Samarkand – Crossroad of Cultures", "license": "CC BY-SA 3.0 IGO", "version": "WHS 603", "url": "https://whc.unesco.org/en/list/603" }, { "dataset": "Wikidata — Registan (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-persepolis", "emoji": "🏛️", "name": "Walk through Persepolis", "hint": "Ceremonial capital of ancient Persia", "where": [ { "name": "Persepolis", "lat": 29.9354, "lon": 52.8916, "cc": "IR" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Persepolis", "license": "CC BY-SA 3.0 IGO", "version": "WHS 114", "url": "https://whc.unesco.org/en/list/114" }, { "dataset": "Wikidata — Persepolis (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-santorini-sunset", "emoji": "🌅", "name": "Watch the sunset over Santorini", "hint": "Whitewashed Oia above the caldera", "where": [ { "name": "Oia", "lat": 36.4618, "lon": 25.3753, "cc": "GR" } ], "sources": [ { "dataset": "Wikivoyage — Santorini", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Santorini" }, { "dataset": "GeoNames — Oia", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-acropolis", "emoji": "🏛️", "name": "Stand on the Acropolis", "hint": "The Parthenon above Athens", "where": [ { "name": "Acropolis of Athens", "lat": 37.9715, "lon": 23.7267, "cc": "GR", "wikidata": "Q131013" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Acropolis, Athens", "license": "CC BY-SA 3.0 IGO", "version": "WHS 404", "url": "https://whc.unesco.org/en/list/404" }, { "dataset": "Wikidata — Acropolis of Athens", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q131013" } ] }, + { "id": "xp-stonehenge-solstice", "emoji": "🌄", "name": "See the solstice at Stonehenge", "hint": "Sunrise through the ancient stones", "where": [ { "name": "Stonehenge", "lat": 51.1789, "lon": -1.8262, "cc": "GB", "wikidata": "Q39671" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Stonehenge, Avebury and Associated Sites", "license": "CC BY-SA 3.0 IGO", "version": "WHS 373", "url": "https://whc.unesco.org/en/list/373" }, { "dataset": "Wikidata — Stonehenge", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q39671" } ] }, + { "id": "xp-bernina-railway", "emoji": "🚞", "name": "Ride the Bernina Express", "hint": "Over the Alps by the Rhaetian Railway", "where": [ { "name": "Ospizio Bernina", "lat": 46.4106, "lon": 10.0219, "cc": "CH" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Rhaetian Railway in the Albula / Bernina Landscapes", "license": "CC BY-SA 3.0 IGO", "version": "WHS 1276", "url": "https://whc.unesco.org/en/list/1276" }, { "dataset": "Wikidata — Bernina railway (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-cabo-da-roca", "emoji": "🌊", "name": "Reach the western edge of Europe", "hint": "Cabo da Roca, the continent's westernmost point", "where": [ { "name": "Cabo da Roca", "lat": 38.7803, "lon": -9.4989, "cc": "PT" } ], "sources": [ { "dataset": "Wikidata — Cabo da Roca (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-flamenco", "emoji": "💃", "name": "Feel live flamenco", "hint": "Andalusian song, guitar and dance", "where": [ { "name": "Seville", "lat": 37.3891, "lon": -5.9845, "cc": "ES" } ], "sources": [ { "dataset": "UNESCO Representative List of the Intangible Cultural Heritage — Flamenco (inscribed 2010)", "license": "CC BY-SA 3.0 IGO", "url": "https://ich.unesco.org/en/lists" }, { "dataset": "GeoNames — Seville", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-uluru", "emoji": "🔴", "name": "See Uluru change colour", "hint": "The great monolith at sunset", "where": [ { "name": "Uluru", "lat": -25.3444, "lon": 131.0369, "cc": "AU", "wikidata": "Q140565" } ], "sources": [ { "dataset": "UNESCO World Heritage List — Uluru-Kata Tjuta National Park", "license": "CC BY-SA 3.0 IGO", "version": "WHS 447", "url": "https://whc.unesco.org/en/list/447" }, { "dataset": "Wikidata — Uluru", "license": "CC0-1.0", "url": "https://www.wikidata.org/wiki/Q140565" } ] }, + { "id": "xp-whitsundays", "emoji": "🏝️", "name": "Sail the Whitsundays", "hint": "Whitehaven's white silica sand", "where": [ { "name": "Whitehaven Beach", "lat": -20.2833, "lon": 149.0389, "cc": "AU" } ], "sources": [ { "dataset": "Wikivoyage — Whitsunday Islands", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Whitsunday_Islands" }, { "dataset": "Wikidata — Whitehaven Beach (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-rotorua-powhiri", "emoji": "🪶", "name": "Receive a Māori welcome", "hint": "A pōwhiri and hāngī in Rotorua", "where": [ { "name": "Rotorua", "lat": -38.1368, "lon": 176.2497, "cc": "NZ" } ], "sources": [ { "dataset": "Wikivoyage — Rotorua", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Rotorua" }, { "dataset": "GeoNames — Rotorua", "license": "CC BY 4.0", "url": "https://www.geonames.org" } ] }, + { "id": "xp-emperor-penguins", "emoji": "🐧", "name": "Visit an emperor penguin colony", "hint": "Chicks huddled on the Antarctic ice", "where": [ { "name": "Snow Hill Island", "lat": -64.4667, "lon": -57.3667, "cc": "AQ" } ], "sources": [ { "dataset": "Wikivoyage — Antarctica", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Antarctica" }, { "dataset": "Wikidata — Snow Hill Island (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" } ] }, + { "id": "xp-drake-passage", "emoji": "🌊", "name": "Cross the Drake Passage", "hint": "The wild sea to Antarctica", "scope": "worldwide", "where": [ { "name": "Drake Passage", "lat": -58.0, "lon": -65.0, "cc": "AQ" } ], "sources": [ { "dataset": "Wikidata — Drake Passage (coordinate)", "license": "CC0-1.0", "url": "https://www.wikidata.org" }, { "dataset": "Wikivoyage — Antarctica", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Antarctica" } ] }, + { "id": "xp-circumnavigate", "emoji": "🌐", "name": "Circle the whole globe", "hint": "Set out one way and come home the other", "scope": "worldwide", "sources": [ { "dataset": "Wikivoyage — Round the world flights", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Round_the_world_flights" } ] }, + { "id": "xp-cross-ocean", "emoji": "🛳️", "name": "Cross an ocean by sea", "hint": "Days of open water, no land in sight", "scope": "worldwide", "sources": [ { "dataset": "Wikivoyage — Cruise ships", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Cruise_ships" } ] }, + { "id": "xp-meteor-shower", "emoji": "☄️", "name": "Catch a meteor shower at its peak", "hint": "Perseids or Geminids under a dark sky", "scope": "worldwide", "sources": [ { "dataset": "Wikivoyage — Stargazing", "license": "CC BY-SA 3.0", "url": "https://en.wikivoyage.org/wiki/Stargazing" } ] } + ] +} diff --git a/data/reference/whc-sites-2025.json.gz b/data/reference/whc-sites-2025.json.gz new file mode 100644 index 0000000..6d08097 Binary files /dev/null and b/data/reference/whc-sites-2025.json.gz differ diff --git a/docs/format.md b/docs/format.md index ce696d6..ffef13e 100644 --- a/docs/format.md +++ b/docs/format.md @@ -1,9 +1,10 @@ # Format (v1) -All files are JSON under `/v1/`. Coordinates are WGS 84 decimal degrees, rounded to 5 -places (about 1 m). Countries are ISO 3166-1 alpha-2 codes. Every record carries its -Wikidata id; `source` is the URL of that Wikidata item, where its values come from and -where an error can be corrected. +All files are JSON under `/v1/`. Coordinates are WGS 84 decimal degrees. Countries are +ISO 3166-1 alpha-2 codes. In places and stations, coordinates are rounded to 5 places +(about 1 m) and every record carries its Wikidata id; `source` is the URL of that Wikidata +item, where its values come from and where an error can be corrected. The other datasets +keep the ids and rounding of their sources. ## `catalog.json` @@ -101,6 +102,58 @@ Open mainline railway stations with coordinates and a UIC code or a Wikipedia (o The landmarks as a Postcards community data pack (`"format": "postcards-pack"`), at most 50,000 of them, the best known kept. +## `stations-europe/railways.json` + +The same shape as `stations/railways.json`, with `"license": "ODbL-1.0"` and +`"coverage": "europe"`. Ids are `tl-` and Trainline's own id. + +## `cities/top.json`, `cities/all.json` + +The 10,000 most populous cities, and every city, most populous first. `id` is the GeoNames +id; `population` is null when GeoNames doesn't know it; `subdivisionId` is a region in +`regions/regions.json`, or null. + +```json +[{ "id": "2988507", "name": "Paris", "countryIso2": "FR", "subdivisionId": "FR-11", + "lat": 48.8534, "lon": 2.3488, "population": 2138551 }] +``` + +## `regions/regions.json` + +```json +[{ "id": "FR-11", "countryIso2": "FR", "name": "Île-de-France" }] +``` + +## `airports/airports.json` + +```json +[{ "id": "CDG", "name": "Charles de Gaulle International Airport", "city": "Paris", + "countryIso2": "FR", "lat": 49.0128, "lon": 2.55 }] +``` + +## `heritage/unesco.json` + +One entry per country a site is in; a site in several countries has the country code at +the end of its id. `lat` and `lon` are 0 when UNESCO gives no coordinates. + +```json +[{ "id": "whs-paris-banks-of-the-seine", "name": "Paris, Banks of the Seine", "countryIso2": "FR", + "lat": 48.8655, "lon": 2.3211, "category": "cultural" }] +``` + +## `countries/*.json` + +Objects keyed by ISO 3166-1 alpha-2 code: `names.json` and `article-names.json` +(`"RU": "Russia"`), `continents.json` (`"FR": "Europe"`), `languages.json` +(`"FR": [{ "code": "fra", "name": "French" }]`) and `sovereignty.json` (`"un"` or +`"territory"`). + +## `moments/moments.json` + +Postcards' moments: `{ name, version, license, note, experiences: [...] }`, each +experience citing its own `sources`. + ## `sources.json` -Every source, its licence and what is taken from it. +Every source, its licence, what is taken from it, and the `attribution` its licence asks +an app to show. diff --git a/package-lock.json b/package-lock.json index 85366aa..cbb6b04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,13 @@ "name": "data", "version": "0.1.0", "license": "AGPL-3.0-only", + "dependencies": { + "airport-data": "^1.0.1", + "all-the-cities": "^3.1.0", + "country-state-city": "^3.2.1", + "i18n-iso-countries": "^7.14.0", + "world-countries": "^5.1.0" + }, "devDependencies": { "@types/node": "^22.10.0", "tsx": "^4.19.2", @@ -961,6 +968,21 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/airport-data": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/airport-data/-/airport-data-1.0.1.tgz", + "integrity": "sha512-Y3k2q5I/CwLKW1mul7ECRQQSEQpBm5jebmgutjO/DLbnBp6CG7jaKD58uu8O8HS0oip23TurO6czXBh5Cet9zg==", + "license": "Unlicense" + }, + "node_modules/all-the-cities": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/all-the-cities/-/all-the-cities-3.1.0.tgz", + "integrity": "sha512-Jx+mZR7lUVSX+qk785T3MRkAJYMvUpOmeNQSCbUl1WcCNpBUKJsQJ6uZzOl3/bWXVIUx1UhDFs8ASDHSI3rvzg==", + "license": "MIT", + "dependencies": { + "pbf": "^3.2.1" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -1008,6 +1030,12 @@ "node": ">= 16" } }, + "node_modules/country-state-city": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/country-state-city/-/country-state-city-3.2.1.tgz", + "integrity": "sha512-kxbanqMc6izjhc/EHkGPCTabSPZ2G6eG4/97akAYHJUN4stzzFEvQPZoF8oXDQ+10gM/O/yUmISCR1ZVxyb6EA==", + "license": "GPL-3.0" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1036,6 +1064,12 @@ "node": ">=6" } }, + "node_modules/diacritics": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/diacritics/-/diacritics-1.3.0.tgz", + "integrity": "sha512-wlwEkqcsaxvPJML+rDh/2iS824jbREk6DUMUKkEaSlxdYHeS43cClJtsWglvw2RfeXGm6ohKDqsXteJ5sP5enA==", + "license": "MIT" + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1120,6 +1154,38 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/i18n-iso-countries": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/i18n-iso-countries/-/i18n-iso-countries-7.14.0.tgz", + "integrity": "sha512-nXHJZYtNrfsi1UQbyRqm3Gou431elgLjKl//CYlnBGt5aTWdRPH1PiS2T/p/n8Q8LnqYqzQJik3Q7mkwvLokeg==", + "license": "MIT", + "dependencies": { + "diacritics": "1.3.0" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -1180,6 +1246,19 @@ "node": ">= 14.16" } }, + "node_modules/pbf": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.3.0.tgz", + "integrity": "sha512-XDF38WCH3z5OV/OVa8GKUNtLAyneuzbCisx7QUCF8Q6Nutx0WnJrQe5O+kOtBlLfRNUws98Y58Lblp+NJG5T4Q==", + "license": "BSD-3-Clause", + "dependencies": { + "ieee754": "^1.1.12", + "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -1216,6 +1295,21 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/protocol-buffers-schema": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", + "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", + "license": "MIT" + }, + "node_modules/resolve-protobuf-schema": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", + "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "license": "MIT", + "dependencies": { + "protocol-buffers-schema": "^3.3.1" + } + }, "node_modules/rollup": { "version": "4.64.5", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.64.5.tgz", @@ -1972,6 +2066,11 @@ "engines": { "node": ">=8" } + }, + "node_modules/world-countries": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/world-countries/-/world-countries-5.1.0.tgz", + "integrity": "sha512-CXR6EBvTbArDlDDIWU3gfKb7Qk0ck2WNZ234b/A0vuecPzIfzzxH+O6Ejnvg1sT8XuiZjVlzOH0h08ZtaO7g0w==" } } } diff --git a/package.json b/package.json index 6d67f15..a4145ab 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "data", "version": "0.1.0", "private": true, - "description": "The open data every Offware app reads: towns, sights, guide links and railway stations worldwide, served as static JSON.", + "description": "The open data every Offware app reads: towns, sights, guide links, railway stations, cities, regions, airports, UNESCO sites, countries and moments, served as static JSON.", "license": "AGPL-3.0-only", "type": "module", "scripts": { @@ -10,6 +10,7 @@ "test": "vitest run", "fetch": "tsx scripts/fetch.ts", "build": "tsx scripts/build.ts", + "city-admin1": "tsx scripts/city-admin1.ts", "check": "npm run typecheck && npm test" }, "devDependencies": { @@ -17,5 +18,12 @@ "tsx": "^4.19.2", "typescript": "^5.6.3", "vitest": "^2.1.8" + }, + "dependencies": { + "airport-data": "^1.0.1", + "all-the-cities": "^3.1.0", + "country-state-city": "^3.2.1", + "i18n-iso-countries": "^7.14.0", + "world-countries": "^5.1.0" } } diff --git a/scripts/build.ts b/scripts/build.ts index b670404..4d6fb8e 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -1,20 +1,25 @@ /** - * Build every dataset into public/v1/ from data/raw/wikidata.json (see `npm run fetch`), - * or from the file given as the first argument. - * Run: npm run build [-- ] + * Build every dataset into public/v1/: places and stations from data/raw/wikidata.json + * (see `npm run fetch`) or the file given as the first argument, and the rest from + * src/reference/. `--offline` leaves out the one that is read from the network. + * Run: npm run build [-- ] [--offline] */ import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { buildFiles } from "../src/places"; +import { buildReference } from "../src/reference"; import type { Raw } from "../src/wikidata"; -const input = resolve(process.argv[2] ?? "data/raw/wikidata.json"); +const args = process.argv.slice(2); +const input = resolve(args.find((a) => !a.startsWith("--")) ?? "data/raw/wikidata.json"); const raw = JSON.parse(readFileSync(input, "utf-8")) as Raw; if (!Array.isArray(raw.cities) || raw.cities.length === 0) throw new Error(`no towns or cities in ${input}`); const outDir = resolve("public"); rmSync(join(outDir, "v1"), { recursive: true, force: true }); -const files = buildFiles(raw, raw.fetchedAt ?? new Date().toISOString()); +const updatedAt = raw.fetchedAt ?? new Date().toISOString(); +const reference = await buildReference(updatedAt, { network: !args.includes("--offline") }); +const files = [...buildFiles(raw, updatedAt, reference.records), ...reference.files]; let bytes = 0; for (const f of files) { const p = join(outDir, f.path); @@ -24,4 +29,5 @@ for (const f of files) { writeFileSync(p, text, "utf-8"); } const sights = Object.values(raw.sights).reduce((n, s) => n + s.length, 0); -console.log(`[build] ${raw.cities.length} places, ${sights} sights, ${raw.stations?.length ?? 0} stations, ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`); +const others = Object.entries(reference.records).map(([k, n]) => `${n} ${k}`).join(", "); +console.log(`[build] ${raw.cities.length} places, ${sights} sights, ${raw.stations?.length ?? 0} stations, ${others}; ${files.length} files, ${(bytes / 1e6).toFixed(1)} MB in ${outDir}`); diff --git a/scripts/city-admin1.ts b/scripts/city-admin1.ts new file mode 100644 index 0000000..e52edc0 --- /dev/null +++ b/scripts/city-admin1.ts @@ -0,0 +1,37 @@ +/** + * Vendor GeoNames' current admin-1 code for every city all-the-cities lists, so the cities + * and regions datasets take codes and names from one GeoNames day (src/reference/cities.ts). + * Run it with admin1CodesASCII.txt from the same day, gzipped into data/reference/. + * + * curl -LO https://download.geonames.org/export/dump/cities500.zip && unzip cities500.zip + * curl -L https://download.geonames.org/export/dump/admin1CodesASCII.txt | gzip -9 > data/reference/admin1CodesASCII.txt.gz + * npm run city-admin1 -- cities500.txt + * + * Writes data/reference/city-admin1.tsv.gz: one "\t." line per city + * cities500 still lists (CC BY 4.0, GeoNames). + */ +import { createRequire } from "node:module"; +import { readFileSync, writeFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +import { join } from "node:path"; +import { REFERENCE_DIR } from "../src/reference/common"; + +const require = createRequire(import.meta.url); +const allCities = require("all-the-cities") as { cityId: number }[]; + +const source = process.argv[2]; +if (!source) { + console.error("usage: npm run city-admin1 -- "); + process.exit(1); +} + +const wanted = new Set(allCities.map((c) => String(c.cityId))); +const rows: [number, string][] = []; +for (const line of readFileSync(source, "utf8").split("\n")) { + const f = line.split("\t"); + if (f.length < 11 || !wanted.has(f[0]!)) continue; + rows.push([Number(f[0]), `${f[0]}\t${f[8]}.${f[10]}`]); +} +rows.sort((a, b) => a[0] - b[0]); +writeFileSync(join(REFERENCE_DIR, "city-admin1.tsv.gz"), gzipSync(rows.map((r) => r[1]).join("\n") + "\n", { level: 9 })); +console.log(`cities with a current admin-1 code: ${rows.length} of ${wanted.size}`); diff --git a/src/catalog.ts b/src/catalog.ts index c738356..e134e45 100644 --- a/src/catalog.ts +++ b/src/catalog.ts @@ -48,6 +48,85 @@ export const DATASETS: Dataset[] = [ usedBy: ["postcards"], files: [{ path: "stations/railways.json", content: "every station, in Postcards' station shape" }], }, + { + id: "stations-europe", + name: "Railway stations in Europe", + description: "About 70,000 European stations a traveller can pick, from Trainline's open database. City groupings and airports are left out.", + license: "ODbL-1.0", + sources: ["trainline"], + schedule: "weekly", + usedBy: ["postcards"], + files: [{ path: "stations-europe/railways.json", content: "every station, in Postcards' station shape" }], + }, + { + id: "cities", + name: "Cities worldwide", + description: "Every city GeoNames lists in the all-the-cities package (about 135,000), with population, coordinates and first-level region.", + license: "CC-BY-4.0", + sources: ["geonames"], + schedule: "weekly", + usedBy: ["postcards"], + files: [ + { path: "cities/top.json", content: "the 10,000 most populous cities" }, + { path: "cities/all.json", content: "every city, most populous first" }, + ], + }, + { + id: "regions", + name: "First-level regions", + description: "The regions cities are in (states, provinces, French régions), by GeoNames admin-1 code, with their names.", + license: "ODbL-1.0", + sources: ["geonames", "dr5hn"], + schedule: "weekly", + usedBy: ["postcards"], + files: [{ path: "regions/regions.json", content: "every region: id, country, name" }], + }, + { + id: "airports", + name: "Airports", + description: "Every airport with an IATA code, with its city, country and coordinates.", + license: "ODbL-1.0", + sources: ["openflights"], + schedule: "weekly", + usedBy: ["postcards"], + files: [{ path: "airports/airports.json", content: "every airport, by IATA code" }], + }, + { + id: "heritage", + name: "UNESCO World Heritage sites", + description: "Every World Heritage site through the 2025 session, once per country it is in.", + license: "CC-BY-SA-3.0-IGO", + sources: ["unesco-whc"], + schedule: "weekly", + usedBy: ["postcards"], + files: [{ path: "heritage/unesco.json", content: "every site: id, name, country, coordinates, category" }], + }, + { + id: "countries", + name: "Countries", + description: "Each country's common name, continent, spoken languages and UN membership, by ISO 3166-1 code.", + license: "ODbL-1.0", + sources: ["world-countries", "iso-3166"], + schedule: "weekly", + usedBy: ["postcards"], + files: [ + { path: "countries/names.json", content: "common English names" }, + { path: "countries/article-names.json", content: "Wikivoyage and Wikipedia article titles" }, + { path: "countries/continents.json", content: "continent of each country" }, + { path: "countries/languages.json", content: "spoken languages of each country" }, + { path: "countries/sovereignty.json", content: "UN member state or territory" }, + ], + }, + { + id: "moments", + name: "Moments", + description: "Travel experiences to collect, each tied to a place and citing where it comes from.", + license: "CC0-1.0 AND CC-BY-4.0 AND CC-BY-SA-3.0-IGO AND CC-BY-SA-4.0", + sources: ["wikidata", "geonames", "unesco-whc", "unesco-ich", "wikivoyage"], + schedule: "weekly", + usedBy: ["postcards"], + files: [{ path: "moments/moments.json", content: "every moment, with the sources of each" }], + }, ]; export interface Catalog { diff --git a/src/places.ts b/src/places.ts index b32e063..f00f7cf 100644 --- a/src/places.ts +++ b/src/places.ts @@ -23,33 +23,9 @@ export const ATTRIBUTION = "Data from Wikidata (CC0). Guide links point to Wikiv /** The Wikidata page of an item: where its values come from, and where anyone can correct them. */ export const wikidataUrl = (id: string): string => `https://www.wikidata.org/wiki/${id}`; -export const SOURCES = [ - { - id: "wikidata", - name: "Wikidata", - url: "https://www.wikidata.org", - license: "CC0-1.0", - licenseUrl: "https://creativecommons.org/publicdomain/zero/1.0/", - use: "Every name, population, coordinate, category and UNESCO flag, and the links below.", - fix: "Each record's `source` is its Wikidata item: correct an error there and it reaches this dataset on the next weekly build.", - }, - { - id: "wikivoyage", - name: "Wikivoyage", - url: "https://en.wikivoyage.org", - license: "CC-BY-SA-4.0", - licenseUrl: "https://creativecommons.org/licenses/by-sa/4.0/", - use: "Links to guide articles only. No article text is copied.", - }, - { - id: "wikipedia", - name: "Wikipedia", - url: "https://en.wikipedia.org", - license: "CC-BY-SA-4.0", - licenseUrl: "https://creativecommons.org/licenses/by-sa/4.0/", - use: "Links to articles only. No article text is copied.", - }, -]; +import { SOURCES } from "./sources"; + +export { SOURCES }; export interface PlaceSight { wikidata: string; @@ -156,7 +132,8 @@ export function stationsFile(stations: Station[], retrieved: string): unknown { }; } -export function buildFiles(raw: Raw, updatedAt: string): { path: string; body: unknown }[] { +/** `records` adds the record counts of datasets built elsewhere (src/reference/) to the catalog. */ +export function buildFiles(raw: Raw, updatedAt: string, records: Record = {}): { path: string; body: unknown }[] { const ids = placeIds(raw.cities); const cities = [...raw.cities].sort((a, b) => b.population - a.population || a.wikidata.localeCompare(b.wikidata)); const files: { path: string; body: unknown }[] = []; @@ -208,7 +185,7 @@ export function buildFiles(raw: Raw, updatedAt: string): { path: string; body: u files.push({ path: "v1/places/index.json", body: index }); files.push({ path: "v1/sources.json", body: { version: 1, updatedAt, sources: SOURCES } }); - files.push({ path: "v1/catalog.json", body: buildCatalog(updatedAt, { places: cities.length, stations: raw.stations?.length ?? 0 }) }); + files.push({ path: "v1/catalog.json", body: buildCatalog(updatedAt, { ...records, places: cities.length, stations: raw.stations?.length ?? 0 }) }); files.push({ path: "v1/stations/railways.json", body: stationsFile(raw.stations ?? [], updatedAt) }); files.push({ path: "v1/places/landmarks.json", body: landmarks }); files.push({ diff --git a/src/reference/airports.ts b/src/reference/airports.ts new file mode 100644 index 0000000..5980ec0 --- /dev/null +++ b/src/reference/airports.ts @@ -0,0 +1,73 @@ +/** + * The airports dataset: IATA-coded airports from OpenFlights (ODbL, aggregated from the + * public-domain OurAirports), by way of the airport-data package. + * + * airports/airports.json + * + * Moved from Postcards' scripts/build-reference.mjs; the output is the same. + */ +import { createRequire } from "node:module"; +import type { Built } from "./common"; + +const require = createRequire(import.meta.url); + +interface SourceAirport { + iata?: string; + name: string; + city?: string; + country: string; + latitude: number; + longitude: number; +} + +export interface Airport { + id: string; + name: string; + city: string; + countryIso2: string; + lat: number; + lon: number; +} + +// A few OpenFlights country names differ from ISO 3166 English names. "Netherlands +// Antilles" is left out on purpose: it split in 2010 into CW, SX and BQ and the source +// can't say which, so those airports are dropped rather than guessed. +const COUNTRY_OVERRIDES: Record = { + "Congo (Kinshasa)": "CD", "Congo (Brazzaville)": "CG", Burma: "MM", Laos: "LA", + Syria: "SY", Micronesia: "FM", "Virgin Islands": "VI", "British Virgin Islands": "VG", + "Falkland Islands": "FK", Moldova: "MD", Macedonia: "MK", Swaziland: "SZ", Macau: "MO", + Brunei: "BN", "East Timor": "TL", "Midway Islands": "UM", "Johnston Atoll": "UM", + "Wake Island": "UM", +}; + +const round4 = (n: number): number => Math.round(n * 1e4) / 1e4; + +export function buildAirports(): Built { + const source = require("airport-data") as SourceAirport[]; + const countries = require("i18n-iso-countries") as { + registerLocale(locale: unknown): void; + getAlpha2Code(name: string, lang: string): string | undefined; + }; + countries.registerLocale(require("i18n-iso-countries/langs/en.json")); + + const airports: Airport[] = []; + const seen = new Set(); + for (const a of source) { + const iata = a.iata; + if (!iata || !/^[A-Z]{3}$/.test(iata) || seen.has(iata)) continue; + if (!Number.isFinite(a.latitude) || !Number.isFinite(a.longitude)) continue; + const cc = COUNTRY_OVERRIDES[a.country] ?? countries.getAlpha2Code(a.country, "en"); + if (!cc) continue; + seen.add(iata); + airports.push({ + id: iata, + name: a.name, + city: a.city && a.city !== a.name ? a.city : "", + countryIso2: cc, + lat: round4(a.latitude), + lon: round4(a.longitude), + }); + } + airports.sort((a, b) => a.id.localeCompare(b.id)); + return { files: [{ path: "v1/airports/airports.json", body: airports }], records: { airports: airports.length } }; +} diff --git a/src/reference/cities.ts b/src/reference/cities.ts new file mode 100644 index 0000000..4412f49 --- /dev/null +++ b/src/reference/cities.ts @@ -0,0 +1,187 @@ +/** + * The cities and regions datasets, from GeoNames (CC BY 4.0) by way of the all-the-cities + * package, with each city's admin-1 code and each code's name from one GeoNames day, + * vendored in data/reference/. A code GeoNames doesn't name takes a name from the dr5hn + * countries-states-cities database (ODbL), matched by nearest centroid. + * + * cities/top.json the 10,000 most populous cities + * cities/all.json every city all-the-cities lists (about 135,000) + * regions/regions.json first-level regions, named + * + * Moved from Postcards' scripts/build-reference.mjs; the output is the same. + */ +import { createRequire } from "node:module"; +import { readFileSync } from "node:fs"; +import { gunzipSync } from "node:zlib"; +import { join } from "node:path"; +import { REFERENCE_DIR, type Built } from "./common"; + +const require = createRequire(import.meta.url); + +interface SourceCity { + cityId: number; + name: string; + country: string; + adminCode?: string; + population: number; + loc: { coordinates: [number, number] }; +} + +export interface CityRecord { + id: string; + name: string; + countryIso2: string; + subdivisionId: string | null; + lat: number; + lon: number; + population: number | null; +} + +export interface Subdivision { + id: string; + countryIso2: string; + name: string; +} + +/** The cities apps bundle and load at start-up; the rest is a download on demand. */ +export const TOP_COUNT = 10_000; + +// France: GeoNames names four of the 13 metropolitan regions in English ("Brittany"), +// so all 13 take their own French names, by INSEE region code. +const FR_REGION_NAMES: Record = { + 11: "Île-de-France", 24: "Centre-Val de Loire", 27: "Bourgogne-Franche-Comté", + 28: "Normandie", 32: "Hauts-de-France", 44: "Grand Est", 52: "Pays de la Loire", + 53: "Bretagne", 75: "Nouvelle-Aquitaine", 76: "Occitanie", 84: "Auvergne-Rhône-Alpes", + 93: "Provence-Alpes-Côte d'Azur", 94: "Corse", +}; + +const round4 = (n: number): number => Math.round(n * 1e4) / 1e4; + +function tsv(file: string): Map { + return new Map( + gunzipSync(readFileSync(join(REFERENCE_DIR, file))) + .toString("utf8") + .split("\n") + .filter(Boolean) + .map((line) => { + const [key = "", value = ""] = line.split("\t"); + return [key, value]; + }), + ); +} + +export function buildCities(): Built { + const allCities = require("all-the-cities") as SourceCity[]; + const { State } = require("country-state-city") as { + State: { getStatesOfCountry(cc: string): { name: string; latitude: string | null; longitude: string | null }[] }; + }; + + // GeoNames admin-1 names by "CC-code", from admin1CodesASCII.txt. + const admin1Names = new Map([...tsv("admin1CodesASCII.txt.gz")].map(([code, name]) => [code.replace(".", "-"), name])); + // Each city's admin-1 code on the same GeoNames day, by geonameid. all-the-cities carries + // codes from an older snapshot, and GeoNames has since given some to other regions. + const cityAdmin1 = tsv("city-admin1.tsv.gz"); + + const statesByCountry = new Map(); + const statesOf = (cc: string) => { + let list = statesByCountry.get(cc); + if (!list) { + list = (State.getStatesOfCountry(cc) || []) + .map((s) => ({ name: s.name, lat: parseFloat(s.latitude ?? ""), lon: parseFloat(s.longitude ?? "") })) + .filter((s) => Number.isFinite(s.lat) && Number.isFinite(s.lon) && !(s.lat === 0 && s.lon === 0)); + statesByCountry.set(cc, list); + } + return list; + }; + + // Cities, each with its current region code where GeoNames still lists it. + const cities: (CityRecord & { unlisted?: boolean })[] = []; + const seen = new Set(); + for (const c of allCities) { + if (!c.cityId || seen.has(c.cityId)) continue; + const [lon, lat] = c.loc.coordinates; + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + seen.add(c.cityId); + const current = cityAdmin1.get(String(c.cityId)); + const [cc, admin1] = current ? current.split(".") : []; + cities.push({ + id: String(c.cityId), + name: c.name, + countryIso2: c.country, + subdivisionId: cc === c.country && admin1 ? `${cc}-${admin1}` : null, + lat: round4(lat), + lon: round4(lon), + // GeoNames uses 0 for "unknown population": null, never a fake 0. + population: c.population > 0 ? c.population : null, + // Only a city GeoNames no longer lists falls back to its nearest neighbour. + unlisted: !current && !!c.adminCode, + }); + } + const coded = new Map(); + for (const c of cities) { + if (!c.subdivisionId) continue; + const list = coded.get(c.countryIso2) ?? []; + list.push(c); + coded.set(c.countryIso2, list); + } + for (const c of cities) { + if (c.unlisted) { + let best: CityRecord | null = null; + let bestD = Infinity; + for (const o of coded.get(c.countryIso2) ?? []) { + const d = (o.lat - c.lat) ** 2 + (o.lon - c.lon) ** 2; + if (d < bestD) { + bestD = d; + best = o; + } + } + if (best) c.subdivisionId = best.subdivisionId; + } + delete c.unlisted; + } + + const regions = new Map(); + for (const c of cities) { + if (!c.subdivisionId) continue; + let r = regions.get(c.subdivisionId); + if (!r) { + r = { cc: c.countryIso2, adminCode: c.subdivisionId.slice(c.countryIso2.length + 1), sumLat: 0, sumLon: 0, n: 0 }; + regions.set(c.subdivisionId, r); + } + r.sumLat += c.lat; + r.sumLon += c.lon; + r.n++; + } + cities.sort((a, b) => (b.population ?? 0) - (a.population ?? 0)); + + const nearestName = (cc: string, lat: number, lon: number): string | null => { + let best: string | null = null; + let bestD = Infinity; + for (const s of statesOf(cc)) { + const d = (s.lat - lat) ** 2 + (s.lon - lon) ** 2; + if (d < bestD) { + bestD = d; + best = s.name; + } + } + return best; + }; + const subdivisions: Subdivision[] = []; + for (const [id, r] of regions) { + const name = + (r.cc === "FR" ? FR_REGION_NAMES[r.adminCode] : undefined) ?? + admin1Names.get(id) ?? + nearestName(r.cc, r.sumLat / r.n, r.sumLon / r.n); + subdivisions.push({ id, countryIso2: r.cc, name: name ?? `${r.cc} region ${r.adminCode}` }); + } + subdivisions.sort((a, b) => a.id.localeCompare(b.id)); + + return { + files: [ + { path: "v1/cities/top.json", body: cities.slice(0, TOP_COUNT) }, + { path: "v1/cities/all.json", body: cities }, + { path: "v1/regions/regions.json", body: subdivisions }, + ], + records: { cities: cities.length, regions: subdivisions.length }, + }; +} diff --git a/src/reference/common.ts b/src/reference/common.ts new file mode 100644 index 0000000..354c79c --- /dev/null +++ b/src/reference/common.ts @@ -0,0 +1,11 @@ +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +/** Vendored source files: data/reference/. */ +export const REFERENCE_DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "data", "reference"); + +/** Files under public/, and the record count of each dataset they belong to. */ +export interface Built { + files: { path: string; body: unknown }[]; + records: Record; +} diff --git a/src/reference/countries.ts b/src/reference/countries.ts new file mode 100644 index 0000000..0a7bd2d --- /dev/null +++ b/src/reference/countries.ts @@ -0,0 +1,68 @@ +/** + * The countries dataset, from world-countries (ODbL) and ISO 3166-1 codes (public facts, + * by way of i18n-iso-countries, MIT): + * + * countries/names.json { "RU": "Russia", … } common English name + * countries/article-names.json the same names: the Wikivoyage and Wikipedia article titles + * countries/continents.json { "FR": "Europe", … } + * countries/languages.json { "FR": [{ "code": "fra", "name": "French" }], … } + * countries/sovereignty.json { "US": "un", "HK": "territory", … } + * + * Moved from Postcards' scripts/build-languages.mjs and build-sovereignty.mjs; the output is + * the same. + */ +import { createRequire } from "node:module"; +import type { Built } from "./common"; + +const require = createRequire(import.meta.url); + +interface SourceCountry { + cca2: string; + region: string; + unMember: boolean; + name: { common: string }; + languages?: Record; +} + +// Against the UN's own member list (193 members): world-countries 5.1.0 marks the Holy See +// a member, but it is a permanent observer (https://www.un.org/en/about-us/member-states). +const UN_MEMBER_OVERRIDES: Record = { VA: "territory" }; + +export function buildCountries(): Built { + const world = require("world-countries") as SourceCountry[]; + const iso = require("i18n-iso-countries") as { + registerLocale(locale: unknown): void; + getNames(lang: string): Record; + alpha2ToNumeric(code: string): string | undefined; + }; + iso.registerLocale(require("i18n-iso-countries/langs/en.json")); + + const names: Record = {}; + const languages: Record = {}; + const continents: Record = {}; + for (const c of world) { + if (!c.cca2) continue; + if (c.name?.common) names[c.cca2] = c.name.common; + const langs = Object.entries(c.languages ?? {}).map(([code, name]) => ({ code, name })); + if (langs.length) languages[c.cca2] = langs; + } + for (const c of [...world].sort((a, b) => a.cca2.localeCompare(b.cca2))) continents[c.cca2] = c.region; + + const byCca2 = new Map(world.map((c) => [c.cca2, c])); + const sovereignty: Record = {}; + for (const code of Object.keys(iso.getNames("en"))) { + if (!iso.alpha2ToNumeric(code)) continue; + sovereignty[code] = UN_MEMBER_OVERRIDES[code] ?? (byCca2.get(code)?.unMember ? "un" : "territory"); + } + + return { + files: [ + { path: "v1/countries/names.json", body: names }, + { path: "v1/countries/article-names.json", body: names }, + { path: "v1/countries/continents.json", body: continents }, + { path: "v1/countries/languages.json", body: languages }, + { path: "v1/countries/sovereignty.json", body: sovereignty }, + ], + records: { countries: Object.keys(names).length }, + }; +} diff --git a/src/reference/heritage.ts b/src/reference/heritage.ts new file mode 100644 index 0000000..84bad01 --- /dev/null +++ b/src/reference/heritage.ts @@ -0,0 +1,127 @@ +/** + * The heritage dataset: the UNESCO World Heritage List (CC BY-SA 3.0 IGO, © UNESCO World + * Heritage Centre), complete through the 47th session (July 2025). The source payload is + * vendored at data/reference/whc-sites-2025.json.gz, retrieved 2026-07-11 from a public + * mirror of whc.unesco.org's syndication data, so the build needs no network. + * + * heritage/unesco.json { id, name, countryIso2, lat, lon, category } + * + * A site in several countries is listed once per country, its id suffixed with the country + * code. A site the source has no coordinates for keeps lat 0 / lon 0, read as "no + * coordinates", never guessed. The one site without a country code (Old City of Jerusalem) + * is left out rather than given one. Names are made plain text. + * + * Moved from Postcards' scripts/build-heritage-full.mjs; the output is the same. + */ +import { readFileSync } from "node:fs"; +import { gunzipSync } from "node:zlib"; +import { join } from "node:path"; +import { REFERENCE_DIR, type Built } from "./common"; + +interface SourceSite { + id_no: number | string; + name_en: string; + category: string; + iso_codes?: string; + coordinates?: { lat?: number; lon?: number }; +} + +export interface HeritageSite { + id: string; + name: string; + countryIso2: string; + lat: number; + lon: number; + category: "cultural" | "natural" | "mixed"; +} + +const ENTITIES: Record = { + quot: '"', apos: "'", nbsp: " ", ndash: "–", mdash: "—", rsquo: "’", lsquo: "‘", eacute: "é", egrave: "è", +}; + +/** Strips the HTML tags and entities the UNESCO feed sometimes embeds (). */ +export function plainName(raw: string): string { + return String(raw) + .replace(/<[^>]*>/g, "") + .replace(/&/g, "&") + .replace(/&#(\d+);/g, (_, n: string) => String.fromCodePoint(Number(n))) + .replace(/&(quot|apos|nbsp|ndash|mdash|rsquo|lsquo|eacute|egrave);/g, (_, e: string) => ENTITIES[e] ?? "") + .replace(/\s+/g, " ") + .trim(); +} + +function slugify(name: string): string { + return name + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .replace(/[øØ]/g, "o") + .replace(/[æÆ]/g, "ae") + .replace(/[œŒ]/g, "oe") + .replace(/[ßẞ]/g, "ss") + .replace(/[đĐðÐ]/g, "d") + .replace(/[þÞ]/g, "th") + .replace(/[łŁ]/g, "l") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} + +const CATEGORY: Record = { Cultural: "cultural", Natural: "natural", Mixed: "mixed" }; + +export function buildHeritage(): Built { + const sites = JSON.parse(gunzipSync(readFileSync(join(REFERENCE_DIR, "whc-sites-2025.json.gz"))).toString("utf8")) as SourceSite[]; + if (!Array.isArray(sites) || sites.length < 1100) throw new Error("heritage: unexpected source"); + + const out: (HeritageSite & { idNo?: string })[] = []; + const sitesKept = new Set(); + for (const s of sites) { + const name = plainName(s.name_en); + const category = CATEGORY[s.category]; + if (!name || !category) throw new Error(`heritage: site ${s.id_no} has a bad name or category`); + const isos = [ + ...new Set( + String(s.iso_codes ?? "") + .split(",") + .map((c) => c.trim().toUpperCase()) + .filter((c) => /^[A-Z]{2}$/.test(c)), + ), + ]; + if (isos.length === 0) continue; + sitesKept.add(String(s.id_no)); + const c = s.coordinates; + const has = typeof c?.lat === "number" && typeof c?.lon === "number" && Number.isFinite(c.lat) && Number.isFinite(c.lon); + const lat = has ? Number(c!.lat!.toFixed(4)) : 0; + const lon = has ? Number(c!.lon!.toFixed(4)) : 0; + const slug = slugify(name); + for (const iso of isos) { + out.push({ + id: isos.length > 1 ? `whs-${slug}-${iso.toLowerCase()}` : `whs-${slug}`, + name, + countryIso2: iso, + lat, + lon, + category, + idNo: String(s.id_no), + }); + } + } + + // Two sites can share a slug: the source's own id tells them apart. + const counts = new Map(); + for (const e of out) counts.set(e.id, (counts.get(e.id) ?? 0) + 1); + for (const e of out) { + if ((counts.get(e.id) ?? 0) > 1) e.id = `${e.id}-${e.idNo}`; + delete e.idNo; + } + out.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + const ids = new Set(); + for (const e of out) { + if (ids.has(e.id)) throw new Error(`heritage: duplicate id ${e.id}`); + ids.add(e.id); + if (!/^whs-[a-z0-9-]+$/.test(e.id) || /[<>]/.test(e.name) || Math.abs(e.lat) > 90 || Math.abs(e.lon) > 180) { + throw new Error(`heritage: bad entry ${e.id}`); + } + } + return { files: [{ path: "v1/heritage/unesco.json", body: out }], records: { heritage: sitesKept.size } }; +} diff --git a/src/reference/index.ts b/src/reference/index.ts new file mode 100644 index 0000000..fcf3b28 --- /dev/null +++ b/src/reference/index.ts @@ -0,0 +1,28 @@ +/** + * Every dataset built from a source other than Wikidata's query service: cities, regions, + * airports, UNESCO sites, European stations, countries and moments. They need no Wikidata + * fetch; European stations are read from GitHub, and when that fails they're left out of + * this build rather than failing it. + */ +import { buildAirports } from "./airports"; +import { buildCities } from "./cities"; +import type { Built } from "./common"; +import { buildCountries } from "./countries"; +import { buildHeritage } from "./heritage"; +import { buildMoments } from "./moments"; +import { buildTrainline } from "./trainline"; + +export async function buildReference(retrieved: string, options: { network?: boolean } = {}): Promise { + const parts = [buildCities(), buildAirports(), buildHeritage(), buildCountries(), buildMoments()]; + if (options.network !== false) { + try { + parts.push(await buildTrainline(retrieved)); + } catch (err) { + console.error(`[reference] European stations left out of this build (${String(err)})`); + } + } + return { + files: parts.flatMap((p) => p.files), + records: Object.assign({}, ...parts.map((p) => p.records)) as Record, + }; +} diff --git a/src/reference/moments.ts b/src/reference/moments.ts new file mode 100644 index 0000000..28b8842 --- /dev/null +++ b/src/reference/moments.ts @@ -0,0 +1,18 @@ +/** + * The moments dataset: Postcards' list of travel experiences ("Moments"). Each moment's + * name, hint and emoji were written for it (CC0); its anchor coordinates and the experience + * itself come from the sources each item cites (Wikidata, GeoNames, UNESCO's World + * Heritage and Intangible Cultural Heritage lists, Wikivoyage). Kept as written in + * data/reference/moments.json. + * + * moments/moments.json + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { REFERENCE_DIR, type Built } from "./common"; + +export function buildMoments(): Built { + const body = JSON.parse(readFileSync(join(REFERENCE_DIR, "moments.json"), "utf-8")) as { experiences?: unknown[] }; + if (!Array.isArray(body.experiences)) throw new Error("moments: no experiences"); + return { files: [{ path: "v1/moments/moments.json", body }], records: { moments: body.experiences.length } }; +} diff --git a/src/reference/trainline.ts b/src/reference/trainline.ts new file mode 100644 index 0000000..02a5d50 --- /dev/null +++ b/src/reference/trainline.ts @@ -0,0 +1,72 @@ +/** + * The European stations dataset: the Trainline EU open stations database (ODbL), about + * 70,000 stations, read from GitHub at build time. + * + * stations-europe/railways.json in the same { _source, stations } shape as stations/railways.json + * + * Only stations a traveller can pick: city groupings, which sit on the city centre, and + * airports are left out. Ids are "tl-" and Trainline's own id. + * + * Moved from Postcards' scripts/build-railways.mjs; the output is the same. + */ +import type { Built } from "./common"; + +export const TRAINLINE_CSV = "https://raw.githubusercontent.com/trainline-eu/stations/master/stations.csv"; +const UA = "offware-data/0.1 (https://github.com/offware-apps/data)"; + +export interface TrainlineStation { + id: string; + name: string; + countryIso2: string; + subdivisionId: null; + lat: number; + lon: number; +} + +/** Trainline's CSV is `;`-separated and unquoted in the columns read here. */ +export function parseTrainline(text: string): TrainlineStation[] { + const lines = text.split("\n"); + const header = (lines[0] ?? "").split(";"); + const col = (name: string): number => header.indexOf(name); + const [iId, iName, iLat, iLon, iCc, iSugg, iCity, iAir] = ["id", "name", "latitude", "longitude", "country", "is_suggestable", "is_city", "is_airport"].map(col) as number[]; + const byId = new Map(); + for (const line of lines.slice(1)) { + if (!line) continue; + const f = line.split(";"); + if (f[iSugg!] !== "t" || f[iCity!] === "t" || f[iAir!] === "t") continue; + // An empty field is no coordinate, not 0. + const lat = f[iLat!] ? Number(f[iLat!]) : NaN; + const lon = f[iLon!] ? Number(f[iLon!]) : NaN; + const cc = (f[iCc!] || "").trim().toUpperCase(); + const name = (f[iName!] || "").trim(); + if (!name || !/^[A-Z]{2}$/.test(cc) || !Number.isFinite(lat) || !Number.isFinite(lon)) continue; + const id = `tl-${f[iId!]}`; + if (!byId.has(id)) byId.set(id, { id, name, countryIso2: cc, subdivisionId: null, lat, lon }); + } + return [...byId.values()].sort((a, b) => a.name.localeCompare(b.name)); +} + +export function trainlineFile(stations: TrainlineStation[], retrieved: string): unknown { + return { + _source: { + dataset: "Trainline EU open stations database (railway stations)", + url: "https://github.com/trainline-eu/stations", + license: "ODbL-1.0", + retrieved: retrieved.slice(0, 10), + note: "Suggestable stations with coordinates; airports and city-grouping hubs excluded. Coverage is Europe-focused.", + coverage: "europe", + }, + stations, + }; +} + +export async function buildTrainline(retrieved: string): Promise { + const res = await fetch(TRAINLINE_CSV, { headers: { "User-Agent": UA }, signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`Trainline CSV: HTTP ${res.status}`); + const stations = parseTrainline(await res.text()); + if (stations.length < 10_000) throw new Error(`Trainline CSV: only ${stations.length} stations`); + return { + files: [{ path: "v1/stations-europe/railways.json", body: trainlineFile(stations, retrieved) }], + records: { "stations-europe": stations.length }, + }; +} diff --git a/src/sources.ts b/src/sources.ts new file mode 100644 index 0000000..72844c4 --- /dev/null +++ b/src/sources.ts @@ -0,0 +1,121 @@ +/** + * v1/sources.json: every source a dataset takes values from, its licence, what is taken, + * and the credit its licence asks for. A share-alike source (ODbL, CC BY-SA) keeps its + * licence on every dataset built from it: see `license` in catalog.json. + */ +export interface Source { + id: string; + name: string; + url: string; + license: string; + licenseUrl: string; + use: string; + /** The credit to show wherever the data is used. */ + attribution?: string; + /** Where an error is corrected. */ + fix?: string; +} + +const CC0 = "https://creativecommons.org/publicdomain/zero/1.0/"; +const ODBL = "https://opendatacommons.org/licenses/odbl/1-0/"; + +export const SOURCES: Source[] = [ + { + id: "wikidata", + name: "Wikidata", + url: "https://www.wikidata.org", + license: "CC0-1.0", + licenseUrl: CC0, + use: "Every name, population, coordinate, category and UNESCO flag in places and stations, and the guide links.", + fix: "Each record's `source` is its Wikidata item: correct an error there and it reaches this dataset on the next weekly build.", + }, + { + id: "wikivoyage", + name: "Wikivoyage", + url: "https://en.wikivoyage.org", + license: "CC-BY-SA-4.0", + licenseUrl: "https://creativecommons.org/licenses/by-sa/4.0/", + use: "Links to guide articles, and the travel topics some moments are drawn from. No article text is copied.", + }, + { + id: "wikipedia", + name: "Wikipedia", + url: "https://en.wikipedia.org", + license: "CC-BY-SA-4.0", + licenseUrl: "https://creativecommons.org/licenses/by-sa/4.0/", + use: "Links to articles only. No article text is copied.", + }, + { + id: "geonames", + name: "GeoNames", + url: "https://www.geonames.org", + license: "CC-BY-4.0", + licenseUrl: "https://creativecommons.org/licenses/by/4.0/", + use: "Every city (by way of the all-the-cities package), each city's first-level region code and each code's name (export of 2026-09-30).", + attribution: "Contains data from GeoNames (https://www.geonames.org), CC BY 4.0.", + fix: "Errors are corrected on geonames.org.", + }, + { + id: "dr5hn", + name: "countries-states-cities database (dr5hn)", + url: "https://github.com/dr5hn/countries-states-cities-database", + license: "ODbL-1.0", + licenseUrl: ODBL, + use: "The name of a region GeoNames gives a code but no name, matched by nearest centroid.", + attribution: "Contains information from the countries-states-cities database, made available under the ODbL.", + }, + { + id: "openflights", + name: "OpenFlights airports", + url: "https://openflights.org/data.html", + license: "ODbL-1.0", + licenseUrl: ODBL, + use: "Every airport with an IATA code (by way of the airport-data package). OpenFlights aggregates OurAirports, which is public domain.", + attribution: "Contains information from OpenFlights (https://openflights.org/data.html), made available under the ODbL.", + }, + { + id: "unesco-whc", + name: "UNESCO World Heritage List", + url: "https://whc.unesco.org/en/list/", + license: "CC-BY-SA-3.0-IGO", + licenseUrl: "https://creativecommons.org/licenses/by-sa/3.0/igo/", + use: "Every World Heritage site through the 47th session (2025): name, countries, coordinates and category. Some moments are drawn from it.", + attribution: "© UNESCO World Heritage Centre (https://whc.unesco.org), CC BY-SA 3.0 IGO.", + }, + { + id: "unesco-ich", + name: "UNESCO Intangible Cultural Heritage lists", + url: "https://ich.unesco.org/en/lists", + license: "CC-BY-SA-3.0-IGO", + licenseUrl: "https://creativecommons.org/licenses/by-sa/3.0/igo/", + use: "The traditions some moments are drawn from; each such moment cites its element.", + attribution: "© UNESCO (https://ich.unesco.org), CC BY-SA 3.0 IGO.", + }, + { + id: "trainline", + name: "Trainline EU open stations database", + url: "https://github.com/trainline-eu/stations", + license: "ODbL-1.0", + licenseUrl: ODBL, + use: "European railway stations: name, country and coordinates.", + attribution: "Contains information from the Trainline EU stations database (https://github.com/trainline-eu/stations), made available under the ODbL.", + fix: "Errors are corrected in trainline-eu/stations.", + }, + { + id: "world-countries", + name: "world-countries (mledoze/countries)", + url: "https://github.com/mledoze/countries", + license: "ODbL-1.0", + licenseUrl: ODBL, + use: "Each country's common name, region, languages and UN membership.", + attribution: "Contains information from mledoze/countries (https://github.com/mledoze/countries), made available under the ODbL.", + }, + { + id: "iso-3166", + name: "ISO 3166-1 country codes (via i18n-iso-countries)", + url: "https://github.com/michaelwittig/node-i18n-iso-countries", + license: "MIT", + licenseUrl: "https://github.com/michaelwittig/node-i18n-iso-countries/blob/master/LICENSE", + use: "The list of country codes. The codes themselves are public facts.", + }, +]; diff --git a/tests/places.test.ts b/tests/places.test.ts index 27c42e7..9a71cbf 100644 --- a/tests/places.test.ts +++ b/tests/places.test.ts @@ -103,14 +103,27 @@ describe("dataset files", () => { expect(stations.stations.map((s) => s.id)).toEqual(["Q745942", "Q801124"]); const sources = byPath.get("v1/sources.json") as { sources: { id: string; license: string }[] }; - expect(sources.sources.map((s) => `${s.id} ${s.license}`)).toEqual(["wikidata CC0-1.0", "wikivoyage CC-BY-SA-4.0", "wikipedia CC-BY-SA-4.0"]); + expect(sources.sources.slice(0, 3).map((s) => `${s.id} ${s.license}`)).toEqual(["wikidata CC0-1.0", "wikivoyage CC-BY-SA-4.0", "wikipedia CC-BY-SA-4.0"]); const catalog = byPath.get("v1/catalog.json") as Catalog; expect(catalog.baseUrl).toBe("https://offware-apps.github.io/data/v1"); - expect(catalog.datasets.map((d) => `${d.id} ${d.records}`)).toEqual(["places 4", "stations 2"]); + expect(catalog.datasets.map((d) => `${d.id} ${d.records}`)).toEqual([ + "places 4", "stations 2", "stations-europe 0", "cities 0", "regions 0", "airports 0", "heritage 0", "countries 0", "moments 0", + ]); + // A dataset built elsewhere gets its count from the caller. + const withCounts = buildFiles(raw(), "2026-10-12T00:00:00.000Z", { cities: 7 }).find((f) => f.path === "v1/catalog.json")!.body as Catalog; + expect(withCounts.datasets.find((d) => d.id === "cities")?.records).toBe(7); // Every source a dataset names is listed with its licence. const listed = new Set(sources.sources.map((s) => s.id)); for (const d of catalog.datasets) for (const s of d.sources) expect(listed.has(s)).toBe(true); + // A share-alike source's licence is the licence of every dataset built from it. + const licenseOf = new Map(sources.sources.map((s) => [s.id, s.license])); + for (const d of catalog.datasets) { + for (const s of d.sources) { + const l = licenseOf.get(s)!; + if (l.startsWith("ODbL") || (l.startsWith("CC-BY-SA") && d.id !== "places")) expect(d.license).toContain(l); + } + } }); it("never gives a town the name of one of the dataset's files", () => { diff --git a/tests/reference.test.ts b/tests/reference.test.ts new file mode 100644 index 0000000..14bbdba --- /dev/null +++ b/tests/reference.test.ts @@ -0,0 +1,88 @@ +import { readFileSync } from "node:fs"; +import { gunzipSync } from "node:zlib"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { buildCities, type CityRecord, type Subdivision } from "../src/reference/cities"; +import { REFERENCE_DIR } from "../src/reference/common"; +import { buildCountries } from "../src/reference/countries"; +import { plainName } from "../src/reference/heritage"; +import { buildMoments } from "../src/reference/moments"; +import { parseTrainline, trainlineFile } from "../src/reference/trainline"; + +describe("reference datasets", () => { + it("keeps only Trainline stations a traveller can pick", () => { + const csv = [ + "id;name;slug;uic;latitude;longitude;parent_station_id;country;time_zone;is_city;is_main_station;is_airport;is_suggestable", + "1;Paris Gare de Lyon;paris-gare-de-lyon;8768600;48.844;2.373;4916;fr;Europe/Paris;f;t;f;t", + "4916;Paris;paris;;48.85;2.35;;FR;Europe/Paris;t;f;f;t", + "2;Paris CDG Airport;cdg;;49.0;2.55;;FR;Europe/Paris;f;f;t;t", + "3;Hidden;hidden;;48.0;2.0;;FR;Europe/Paris;f;f;f;f", + "5;No coordinates;none;;;;;FR;Europe/Paris;f;f;f;t", + "1;Paris Gare de Lyon;paris-gare-de-lyon;8768600;48.844;2.373;4916;FR;Europe/Paris;f;t;f;t", + ].join("\n"); + const stations = parseTrainline(csv); + expect(stations).toEqual([{ id: "tl-1", name: "Paris Gare de Lyon", countryIso2: "FR", subdivisionId: null, lat: 48.844, lon: 2.373 }]); + expect(trainlineFile(stations, "2026-10-23T03:00:00Z")).toMatchObject({ _source: { license: "ODbL-1.0", retrieved: "2026-10-23", coverage: "europe" } }); + }); + + it("makes UNESCO site names plain text", () => { + expect(plainName("Historic Centre of Rome & the Holy See’s sites")).toBe("Historic Centre of Rome & the Holy See’s sites"); + }); + + it("counts the Holy See as a territory, as the UN does", () => { + const sovereignty = buildCountries().files.find((f) => f.path === "v1/countries/sovereignty.json")!.body as Record; + expect(sovereignty.VA).toBe("territory"); + expect(sovereignty.FR).toBe("un"); + expect(Object.values(sovereignty).filter((v) => v === "un")).toHaveLength(193); + }); + + it("publishes the moments as written", () => { + const built = buildMoments(); + expect(built.records.moments).toBeGreaterThan(100); + expect(built.files[0]!.path).toBe("v1/moments/moments.json"); + }); + + describe("cities and regions", () => { + const built = buildCities(); + const file = (path: string): T => built.files.find((f) => f.path === path)!.body as T; + const cities = file("v1/cities/all.json"); + const regions = new Map(file("v1/regions/regions.json").map((r) => [r.id, r.name])); + // Both vendored files are one GeoNames day: each city's region code and each code's + // name. Mixing days put Hanoi under "Quang Tri" when Vietnam renumbered in 2025. + const vendored = (name: string) => + new Map( + gunzipSync(readFileSync(join(REFERENCE_DIR, name))) + .toString("utf8") + .split("\n") + .filter(Boolean) + .map((line) => line.split("\t") as [string, string]), + ); + + it("gives every city GeoNames lists the region code it lists today", () => { + const current = vendored("city-admin1.tsv.gz"); + const wrong = cities + .filter((c) => { + const code = current.get(c.id); + const listed = code?.startsWith(`${c.countryIso2}.`) && !code.endsWith("."); + return listed && c.subdivisionId !== code!.replace(".", "-"); + }) + .map((c) => `${c.name} ${c.id}: ${c.subdivisionId}`); + expect(wrong).toEqual([]); + }); + + it("names every region by GeoNames' name for its code, France in French", () => { + const wrong = [...vendored("admin1CodesASCII.txt.gz")] + .map(([code, name]) => ({ id: code.replace(".", "-"), name, ours: regions.get(code.replace(".", "-")) })) + .filter((s) => s.ours !== undefined && !s.id.startsWith("FR-") && s.ours !== s.name) + .map((s) => `${s.id}: ${s.ours}`); + expect(wrong).toEqual([]); + expect(regions.get("FR-53")).toBe("Bretagne"); + }); + + it("keeps the most populous first, and the top 10,000 apart", () => { + expect(file("v1/cities/top.json")).toHaveLength(10_000); + expect(cities[0]!.population).toBeGreaterThanOrEqual(cities[1]!.population ?? 0); + expect(cities.find((c) => c.id === "2988507")).toMatchObject({ name: "Paris", subdivisionId: "FR-11" }); + }); + }); +});