From 07ad7dc076ce69d384d9e6d0483ad18f4458aa09 Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:19:17 +0000 Subject: [PATCH 1/4] Probe why some countries have few or no towns (temporary) --- .github/workflows/ci.yml | 2 ++ scripts/probe.ts | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 scripts/probe.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33ae143..0ccb332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: node-version: 22 cache: npm - run: npm ci + - name: Probe town gaps + run: npx tsx scripts/probe.ts - name: Fetch three countries from Wikidata # Great Britain and the United States are the heaviest countries for the time limit. run: npm run fetch -- MC GB US diff --git a/scripts/probe.ts b/scripts/probe.ts new file mode 100644 index 0000000..40a6bb0 --- /dev/null +++ b/scripts/probe.ts @@ -0,0 +1,32 @@ +// Temporary: checks why some countries come back with few or no towns. +const ENDPOINT = "https://query.wikidata.org/sparql"; +const USER_AGENT = "offware-data/0.1 (https://github.com/offware-apps/data)"; +const Q: [string, string][] = [ + ["NL holders of P297", `SELECT ?c ?cLabel WHERE { ?c wdt:P297 "NL" . SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } }`], + ["Amsterdam P17/P31", `SELECT ?p17 ?p31 WHERE { wd:Q727 wdt:P17 ?p17 . OPTIONAL { wd:Q727 wdt:P31 ?p31 } }`], + ["Utrecht station P17", `SELECT ?p17 WHERE { wd:Q1022934 wdt:P17 ?p17 }`], + ["NL pop>=5000 by P17", `SELECT ?country (COUNT(DISTINCT ?city) AS ?n) WHERE { VALUES ?country { wd:Q55 wd:Q29999 } ?city wdt:P17 ?country ; wdt:P1082 ?p . FILTER(?p >= 5000) } GROUP BY ?country`], + ["Kingdom P297", `SELECT ?v WHERE { wd:Q29999 wdt:P297 ?v }`], + ["class paths", `SELECT ?cls ?settlement ?municipality WHERE { VALUES ?cls { wd:Q484170 wd:Q747074 wd:Q2039348 wd:Q15284 wd:Q2074737 wd:Q1529096 wd:Q262166 } BIND(EXISTS { ?cls wdt:P279* wd:Q486972 } AS ?settlement) BIND(EXISTS { ?cls wdt:P279* wd:Q15284 } AS ?municipality) }`], + ["FR communes >=5000", `SELECT (COUNT(DISTINCT ?c) AS ?n) (COUNT(DISTINCT ?s) AS ?settled) WHERE { ?c wdt:P31 wd:Q484170 ; wdt:P1082 ?p . FILTER(?p >= 5000) OPTIONAL { ?c wdt:P31 ?cls . ?cls wdt:P279* wd:Q486972 . BIND(?c AS ?s) } }`], + ["IT comuni >=5000", `SELECT (COUNT(DISTINCT ?c) AS ?n) (COUNT(DISTINCT ?s) AS ?settled) WHERE { ?c wdt:P31 wd:Q747074 ; wdt:P1082 ?p . FILTER(?p >= 5000) OPTIONAL { ?c wdt:P31 ?cls . ?cls wdt:P279* wd:Q486972 . BIND(?c AS ?s) } }`], + ["TR pop>=5000 classes", `SELECT ?cls ?clsLabel (COUNT(DISTINCT ?c) AS ?n) WHERE { ?country wdt:P297 "TR" . ?c wdt:P17 ?country ; wdt:P1082 ?p ; wdt:P31 ?cls . FILTER(?p >= 5000) SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } } GROUP BY ?cls ?clsLabel ORDER BY DESC(?n) LIMIT 25`], + ["FR pop>=5000 classes", `SELECT ?cls ?clsLabel (COUNT(DISTINCT ?c) AS ?n) WHERE { ?c wdt:P17 wd:Q142 ; wdt:P1082 ?p ; wdt:P31 ?cls . FILTER(?p >= 5000) SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } } GROUP BY ?cls ?clsLabel ORDER BY DESC(?n) LIMIT 15`], +]; +for (const [name, query] of Q) { + const t = Date.now(); + try { + const res = await fetch(ENDPOINT, { + method: "POST", + headers: { "User-Agent": USER_AGENT, Accept: "application/sparql-results+json", "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ query }), + signal: AbortSignal.timeout(90_000), + }); + const text = await res.text(); + const rows = res.ok ? (JSON.parse(text) as { results: { bindings: Record[] } }).results.bindings.map((b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value.replace("http://www.wikidata.org/entity/", "")]))) : text.slice(0, 200); + console.log(`### ${name} (${res.status}, ${((Date.now() - t) / 1000).toFixed(1)} s)\n${JSON.stringify(rows)}`); + } catch (err) { + console.log(`### ${name}: ${String(err)}`); + } + await new Promise((r) => setTimeout(r, 2000)); +} From 97242ac30d6763ff5f7901fdbec28e8378b89758 Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:48:17 +0000 Subject: [PATCH 2/4] Find Dutch towns and stations, communes and comuni, and leave out neighbourhoods NL's ISO code sits on the Kingdom of the Netherlands, while Dutch towns and stations name the Netherlands as their country, so none were found. French communes, Italian comuni and Dutch gemeenten are municipalities on Wikidata, not settlements, so most were missed. Turkey's thousands of mahalle are neighbourhoods, not towns. --- .github/workflows/ci.yml | 4 ++-- scripts/probe.ts | 32 -------------------------------- src/wikidata.ts | 31 ++++++++++++++++++++++++++----- tests/places.test.ts | 11 ++++++++++- 4 files changed, 38 insertions(+), 40 deletions(-) delete mode 100644 scripts/probe.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ccb332..f0a4960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,8 @@ jobs: node-version: 22 cache: npm - run: npm ci - - name: Probe town gaps - run: npx tsx scripts/probe.ts + - name: Probe the countries that came back short (temporary) + run: npm run fetch -- NL TR FR IT - name: Fetch three countries from Wikidata # Great Britain and the United States are the heaviest countries for the time limit. run: npm run fetch -- MC GB US diff --git a/scripts/probe.ts b/scripts/probe.ts deleted file mode 100644 index 40a6bb0..0000000 --- a/scripts/probe.ts +++ /dev/null @@ -1,32 +0,0 @@ -// Temporary: checks why some countries come back with few or no towns. -const ENDPOINT = "https://query.wikidata.org/sparql"; -const USER_AGENT = "offware-data/0.1 (https://github.com/offware-apps/data)"; -const Q: [string, string][] = [ - ["NL holders of P297", `SELECT ?c ?cLabel WHERE { ?c wdt:P297 "NL" . SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } }`], - ["Amsterdam P17/P31", `SELECT ?p17 ?p31 WHERE { wd:Q727 wdt:P17 ?p17 . OPTIONAL { wd:Q727 wdt:P31 ?p31 } }`], - ["Utrecht station P17", `SELECT ?p17 WHERE { wd:Q1022934 wdt:P17 ?p17 }`], - ["NL pop>=5000 by P17", `SELECT ?country (COUNT(DISTINCT ?city) AS ?n) WHERE { VALUES ?country { wd:Q55 wd:Q29999 } ?city wdt:P17 ?country ; wdt:P1082 ?p . FILTER(?p >= 5000) } GROUP BY ?country`], - ["Kingdom P297", `SELECT ?v WHERE { wd:Q29999 wdt:P297 ?v }`], - ["class paths", `SELECT ?cls ?settlement ?municipality WHERE { VALUES ?cls { wd:Q484170 wd:Q747074 wd:Q2039348 wd:Q15284 wd:Q2074737 wd:Q1529096 wd:Q262166 } BIND(EXISTS { ?cls wdt:P279* wd:Q486972 } AS ?settlement) BIND(EXISTS { ?cls wdt:P279* wd:Q15284 } AS ?municipality) }`], - ["FR communes >=5000", `SELECT (COUNT(DISTINCT ?c) AS ?n) (COUNT(DISTINCT ?s) AS ?settled) WHERE { ?c wdt:P31 wd:Q484170 ; wdt:P1082 ?p . FILTER(?p >= 5000) OPTIONAL { ?c wdt:P31 ?cls . ?cls wdt:P279* wd:Q486972 . BIND(?c AS ?s) } }`], - ["IT comuni >=5000", `SELECT (COUNT(DISTINCT ?c) AS ?n) (COUNT(DISTINCT ?s) AS ?settled) WHERE { ?c wdt:P31 wd:Q747074 ; wdt:P1082 ?p . FILTER(?p >= 5000) OPTIONAL { ?c wdt:P31 ?cls . ?cls wdt:P279* wd:Q486972 . BIND(?c AS ?s) } }`], - ["TR pop>=5000 classes", `SELECT ?cls ?clsLabel (COUNT(DISTINCT ?c) AS ?n) WHERE { ?country wdt:P297 "TR" . ?c wdt:P17 ?country ; wdt:P1082 ?p ; wdt:P31 ?cls . FILTER(?p >= 5000) SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } } GROUP BY ?cls ?clsLabel ORDER BY DESC(?n) LIMIT 25`], - ["FR pop>=5000 classes", `SELECT ?cls ?clsLabel (COUNT(DISTINCT ?c) AS ?n) WHERE { ?c wdt:P17 wd:Q142 ; wdt:P1082 ?p ; wdt:P31 ?cls . FILTER(?p >= 5000) SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } } GROUP BY ?cls ?clsLabel ORDER BY DESC(?n) LIMIT 15`], -]; -for (const [name, query] of Q) { - const t = Date.now(); - try { - const res = await fetch(ENDPOINT, { - method: "POST", - headers: { "User-Agent": USER_AGENT, Accept: "application/sparql-results+json", "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ query }), - signal: AbortSignal.timeout(90_000), - }); - const text = await res.text(); - const rows = res.ok ? (JSON.parse(text) as { results: { bindings: Record[] } }).results.bindings.map((b) => Object.fromEntries(Object.entries(b).map(([k, v]) => [k, v.value.replace("http://www.wikidata.org/entity/", "")]))) : text.slice(0, 200); - console.log(`### ${name} (${res.status}, ${((Date.now() - t) / 1000).toFixed(1)} s)\n${JSON.stringify(rows)}`); - } catch (err) { - console.log(`### ${name}: ${String(err)}`); - } - await new Promise((r) => setTimeout(r, 2000)); -} diff --git a/src/wikidata.ts b/src/wikidata.ts index e5f8287..ea086bd 100644 --- a/src/wikidata.ts +++ b/src/wikidata.ts @@ -77,6 +77,27 @@ export const COUNTRIES_QUERY = `SELECT DISTINCT ?iso2 WHERE { FILTER NOT EXISTS { ?country wdt:P576 ?dissolved . } }`; +/** + * Where the ISO code sits on an item other than the one towns and stations name as their + * country: "NL" is on the Kingdom of the Netherlands, while Dutch towns and stations are in + * the Netherlands, its European part. + */ +const COUNTRY_ITEMS: Record = { NL: "Q55" }; + +/** Binds ?country to the country with this ISO 3166-1 code. */ +function country(countryIso2: string): string { + const item = COUNTRY_ITEMS[countryIso2]; + return item ? `BIND(wd:${item} AS ?country)` : `?country wdt:P297 "${countryIso2}" .`; +} + +/** + * What a town is: a human settlement, or a municipality (French communes, Italian comuni and + * Dutch gemeenten are municipalities, not settlements, on Wikidata). Neighbourhoods are left + * out: Turkey alone has thousands of mahalle of 5,000 people or more. + */ +const TOWN_CLASSES = `FILTER(?cls NOT IN (wd:Q17051044, wd:Q123705, wd:Q2983893)) + { ?cls wdt:P279* wd:Q486972 . } UNION { ?cls wdt:P279* wd:Q15284 . }`; + export function parseCountries(result: unknown): string[] { const codes = bindings(result) .map((b) => b.iso2?.value?.toUpperCase() ?? "") @@ -96,15 +117,15 @@ export function citiesQuery(countryIso2: string, byPopulation = false): string { // read every item of the country. Slower for small countries, but never times out. const towns = byPopulation ? `hint:SubQuery hint:optimizer "None" . - ?country wdt:P297 "${countryIso2}" . + ${country(countryIso2)} ?city wdt:P1082 ?p0 . FILTER(?p0 >= ${minPopulation(countryIso2)}) ?city wdt:P17 ?country ; wdt:P31 ?cls . - ?cls wdt:P279* wd:Q486972 .` - : `?country wdt:P297 "${countryIso2}" . + ${TOWN_CLASSES}` + : `${country(countryIso2)} ?city wdt:P17 ?country ; wdt:P1082 ?p0 ; wdt:P31 ?cls . FILTER(?p0 >= ${minPopulation(countryIso2)}) - ?cls wdt:P279* wd:Q486972 .`; + ${TOWN_CLASSES}`; return `SELECT ?city ?cityLabel (MAX(?p) AS ?pop) (SAMPLE(?c) AS ?coord) (SAMPLE(?v) AS ?voy) (SAMPLE(?w) AS ?wp) (SAMPLE(?s) AS ?site) (GROUP_CONCAT(DISTINCT ?alt; separator="|") AS ?names) WITH { @@ -286,7 +307,7 @@ export function stationsQuery(countryIso2: string): string { WITH { SELECT ?station (SAMPLE(?c) AS ?coord) WHERE { hint:SubQuery hint:optimizer "None" . - ?country wdt:P297 "${countryIso2}" . + ${country(countryIso2)} ?station wdt:P31 wd:Q55488 . ?station wdt:P17 ?country ; wdt:P625 ?c ; wikibase:sitelinks ?links . FILTER NOT EXISTS { ?station wdt:P576 ?closed . } diff --git a/tests/places.test.ts b/tests/places.test.ts index 9a71cbf..8023b31 100644 --- a/tests/places.test.ts +++ b/tests/places.test.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { PACK_MAX, buildFiles, placeIds, slug, type Place, type PlacesIndex } from "../src/places"; import type { Catalog } from "../src/catalog"; -import { citiesQuery, minPopulation, parseCities, parseCountries, parseSights, parseStations, point, sightsQuery, type Raw } from "../src/wikidata"; +import { citiesQuery, minPopulation, parseCities, parseCountries, parseSights, parseStations, point, sightsQuery, stationsQuery, type Raw } from "../src/wikidata"; const fixture = (name: string): unknown => JSON.parse(readFileSync(new URL(`./fixtures/${name}`, import.meta.url), "utf-8")); @@ -55,8 +55,17 @@ describe("Wikidata results", () => { it("builds queries for a country and a batch of cities", () => { expect(citiesQuery("FR")).toContain('wdt:P297 "FR"'); + // Communes are municipalities on Wikidata, not settlements. + expect(citiesQuery("FR")).toContain("wd:Q15284"); expect(sightsQuery(["Q90", "Q456"])).toContain("VALUES ?city { wd:Q90 wd:Q456 }"); }); + + it("finds Dutch towns and stations by the country they name", () => { + expect(citiesQuery("NL")).toContain("BIND(wd:Q55 AS ?country)"); + expect(citiesQuery("NL", true)).toContain("BIND(wd:Q55 AS ?country)"); + expect(stationsQuery("NL")).toContain("BIND(wd:Q55 AS ?country)"); + expect(stationsQuery("NL")).not.toContain("P297"); + }); }); describe("dataset files", () => { From 7141656c9d8f7da63cbcd15ccfcb30e1fb52151c Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:48:35 +0000 Subject: [PATCH 3/4] Check Pages before the fetch, not after it --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 18c19f2..1a401ea 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,10 +29,11 @@ jobs: with: node-version: 22 cache: npm + # First, so a repository without Pages fails in seconds, not after the whole fetch. + - uses: actions/configure-pages@v5 - run: npm ci - run: npm run fetch - run: npm run build - - uses: actions/configure-pages@v5 - uses: actions/upload-pages-artifact@v3 with: path: public From 2d728dfa6ce07a0422a638b696fa7a3035ea036b Mon Sep 17 00:00:00 2001 From: David <60177543+davd-gzl@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:26:50 +0000 Subject: [PATCH 4/4] Drop the temporary probe --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0a4960..33ae143 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,6 @@ jobs: node-version: 22 cache: npm - run: npm ci - - name: Probe the countries that came back short (temporary) - run: npm run fetch -- NL TR FR IT - name: Fetch three countries from Wikidata # Great Britain and the United States are the heaviest countries for the time limit. run: npm run fetch -- MC GB US