xcc.es
[READ-ONLY] Mirror of https://github.com/mrgnw/ananas. learn multiple languages at once
ananas.xcc.es
1/**
2 * Fetches country data from Wikidata, including:
3 * - ISO 3166-1 alpha-2 codes
4 * - Official/primary languages
5 * - Language speaker counts
6 * - Flag emoji
7 *
8 * Run with: bun run fetch:countries
9 */
10
11const WIKIDATA_ENDPOINT = 'https://query.wikidata.org/sparql';
12
13const QUERY = `
14SELECT DISTINCT ?country ?countryLabel ?isoCode ?iso3Code ?flagImage
15(GROUP_CONCAT(DISTINCT ?nativeName; separator="|") as ?nativeNames)
16(GROUP_CONCAT(DISTINCT ?lang; separator="|") as ?languages)
17(GROUP_CONCAT(DISTINCT ?langLabel; separator="|") as ?languageLabels)
18(GROUP_CONCAT(DISTINCT ?langIso; separator="|") as ?languageIsos)
19(GROUP_CONCAT(DISTINCT ?langIso1; separator="|") as ?languageIso1s)
20(GROUP_CONCAT(DISTINCT ?speakers; separator="|") as ?speakerCounts)
21WHERE {
22 ?country wdt:P31 wd:Q3624078 . # Instance of: sovereign state
23 ?country wdt:P297 ?isoCode . # ISO 3166-1 alpha-2 code
24 ?country wdt:P298 ?iso3Code . # ISO 3166-1 alpha-3 code
25
26 # Get official native name (P1448)
27 OPTIONAL {
28 ?country wdt:P1448 ?nativeName .
29 }
30
31 # Get flag image
32 OPTIONAL { ?country wdt:P41 ?flagImage . }
33
34 # Get official languages
35 OPTIONAL {
36 ?country wdt:P37 ?lang . # Official language
37 ?lang rdfs:label ?langLabel .
38 FILTER(LANG(?langLabel) = "en")
39
40 # Get ISO codes for languages
41 OPTIONAL { ?lang wdt:P220 ?langIso } # ISO 639-3 code
42 OPTIONAL { ?lang wdt:P218 ?langIso1 } # ISO 639-1 code
43
44 # Get speaker counts where available
45 OPTIONAL {
46 ?lang wdt:P1098 ?speakers . # Number of speakers
47 }
48 }
49
50 # Get English labels
51 SERVICE wikibase:label {
52 bd:serviceParam wikibase:language "en" .
53 }
54}
55GROUP BY ?country ?countryLabel ?isoCode ?iso3Code ?flagImage
56ORDER BY ?isoCode
57`;
58
59async function fetchWikidataCountries() {
60 const response = await fetch(WIKIDATA_ENDPOINT, {
61 method: 'POST',
62 headers: {
63 'Content-Type': 'application/x-www-form-urlencoded',
64 'Accept': 'application/json',
65 'User-Agent': 'Ananas Language Learning App/1.0'
66 },
67 body: 'query=' + encodeURIComponent(QUERY)
68 });
69
70 if (!response.ok) {
71 throw new Error(`HTTP error! status: ${response.status}`);
72 }
73
74 const data = await response.json();
75
76 // Process the results into a more usable format
77 let countries = data.results.bindings.map(country => {
78 const languages = country.languages?.value.split('|').filter(Boolean) || [];
79 const languageLabels = country.languageLabels?.value.split('|').filter(Boolean) || [];
80 const languageIsos = country.languageIsos?.value.split('|').filter(Boolean) || [];
81 const languageIso1s = country.languageIso1s?.value.split('|').filter(Boolean) || [];
82 const speakerCounts = country.speakerCounts?.value.split('|').map(Number).filter(Boolean) || [];
83
84 // Create language objects with labels, ISO codes and speaker counts
85 let languagesWithData = languages.map((lang, index) => ({
86 id: lang.split('/').pop(),
87 name: languageLabels[index],
88 iso: languageIsos[index],
89 iso1: languageIso1s[index] || null,
90 speakers: speakerCounts[index] || null,
91 speakers_m: speakerCounts[index] ? +(speakerCounts[index] / 1000000).toFixed(1) : null
92 }));
93
94 // Sort languages by id for idempotency
95 languagesWithData.sort((a, b) => {
96 if (!a.id && !b.id) return 0;
97 if (!a.id) return 1;
98 if (!b.id) return -1;
99 return a.id.localeCompare(b.id);
100 });
101
102 return {
103 wikidata_id: country.country.value.split('/').pop(),
104 name: country.countryLabel.value,
105 native_name: country.nativeNames?.value.split('|')[0] || null,
106 iso: country.isoCode.value.toLowerCase(),
107 iso3: country.iso3Code.value.toLowerCase(),
108 flag: String.fromCodePoint(
109 ...country.isoCode.value
110 .toUpperCase()
111 .split('')
112 .map(char => char.charCodeAt(0) + 127397)
113 ),
114 languages: languagesWithData
115 };
116 });
117
118 // Remove duplicate countries by wikidata_id
119 const seen = new Set();
120 countries = countries.filter(country => {
121 if (seen.has(country.wikidata_id)) return false;
122 seen.add(country.wikidata_id);
123 return true;
124 });
125
126 // Ensure languages are sorted by id, then name for stability
127 countries.forEach(country => {
128 country.languages.sort((a, b) => {
129 if (a.id && b.id) {
130 const cmp = a.id.localeCompare(b.id);
131 if (cmp !== 0) return cmp;
132 }
133 if (a.name && b.name) return a.name.localeCompare(b.name);
134 if (a.name) return -1;
135 if (b.name) return 1;
136 return 0;
137 });
138 });
139
140 // Sort countries by name for idempotency and readability
141 countries.sort((a, b) => a.name.localeCompare(b.name));
142
143 // Write the results to a JSON file
144 await Bun.write(
145 './src/lib/data/wikidata-countries.json',
146 JSON.stringify(countries, null, 2)
147 );
148
149 console.log(`Wrote ${countries.length} countries to wikidata-countries.json`);
150}
151
152fetchWikidataCountries().catch(console.error);