xcc.es
[READ-ONLY] Mirror of https://github.com/mrgnw/ananas. learn multiple languages at once
ananas.xcc.es
1#!/usr/bin/env bun
2import fs from 'fs'
3import path from 'path'
4import { fileURLToPath } from 'url'
5import { createPatch } from 'diff'
6import chalk from 'chalk';
7import { diffLines } from 'diff';
8
9const __dirname = path.dirname(fileURLToPath(import.meta.url))
10const OUTPUT_DIR = path.join(__dirname, '../src/lib/data')
11
12const SPARQL_QUERY = fs.readFileSync(path.join(__dirname, 'languages.sql'), 'utf-8')
13
14// UNESCO status mapping
15const UNESCO_STATUS = {
16 "20672082": "extinct",
17 "20672083": "critically endangered",
18 "20672084": "severely endangered",
19 "20672085": "definitely endangered",
20 "20672086": "vulnerable",
21 "20672087": "safe",
22 "20672088": "definitely endangered"
23}
24
25// Ethnologue status mapping
26const ETHNOLOGUE_STATUS = {
27 "29051540": "0 International",
28 "29051541": "1 National",
29 "29051542": "2 Provincial",
30 "29051543": "3 Wider Communication",
31 "29051544": "4 Educational",
32 "29051545": "5 Developing",
33 "29051546": "6a Vigorous",
34 "29051547": "6b Threatened",
35 "29051548": "7 Shifting",
36 "29051549": "8a Moribund",
37 "29051550": "8b Nearly Extinct",
38 "29051551": "9 Dormant",
39 "29051552": "10 Extinct"
40}
41
42async function fetchWikidataSpeakers() {
43 console.log('Fetching comprehensive language data from Wikidata...')
44
45 const url = 'https://query.wikidata.org/sparql'
46 const response = await fetch(url, {
47 method: 'POST',
48 headers: {
49 'Content-Type': 'application/x-www-form-urlencoded',
50 'Accept': 'application/sparql-results+json',
51 'User-Agent': 'AnanasDictionary/1.0 (https://github.com/mrgnw/ananas)'
52 },
53 body: 'query=' + encodeURIComponent(SPARQL_QUERY)
54 })
55
56 if (!response.ok) {
57 throw new Error(`HTTP error! status: ${response.status}`)
58 }
59
60 const data = await response.json()
61 const languages = data.results.bindings.map(lang => {
62 // First create object with iso codes first
63 const obj = {
64 iso: lang.iso.value,
65 iso1: lang.iso1?.value || null
66 }
67
68 // Case-insensitive deduplication for native names
69 const dedupeNames = (names) => {
70 const seen = new Map()
71 return names
72 .filter(Boolean)
73 .map(name => {
74 // Normalize by removing all case differences
75 const normalized = name.toLowerCase()
76
77 // If we haven't seen this name before or the new version looks better
78 if (!seen.has(normalized) || shouldPreferName(name, seen.get(normalized))) {
79 seen.set(normalized, name)
80 }
81 return seen.get(normalized)
82 })
83 .filter((name, index, arr) => arr.indexOf(name) === index)
84 .sort()
85 }
86
87 // Helper to decide which version of a name to keep
88 const shouldPreferName = (newName, existingName) => {
89 // If it's a CamelCase vs lowercase difference (e.g., ChiShona vs chiShona),
90 // prefer the one that follows proper capitalization (first letter capital)
91 if (newName.charAt(0).toUpperCase() === newName.charAt(0) &&
92 existingName.charAt(0).toLowerCase() === existingName.charAt(0)) {
93 return true
94 }
95
96 // For other cases, prefer the one with more uppercase letters
97 const newUpperCount = newName.replace(/[^A-Z]/g, '').length
98 const existingUpperCount = existingName.replace(/[^A-Z]/g, '').length
99 return newUpperCount > existingUpperCount
100 }
101
102 // Then add remaining properties in alphabetical order
103 const sortedProps = Object.entries({
104 countries: [...new Set(lang.countries?.value.split('|').filter(Boolean))]?.sort() || [],
105 ethnologueStatus: lang.ethnologueStatus?.value ? ETHNOLOGUE_STATUS[lang.ethnologueStatus.value] : null,
106 langLabel: lang.langLabel.value,
107 nativeNames: dedupeNames(lang.nativeNames?.value.split(', ') || []),
108 nativeSpeakers_k: parseInt(lang.nativeSpeakers_k.value),
109 ...(lang.rtl?.value === "true" ? { rtl: true } : {}),
110 unescoStatus: lang.unescoStatus?.value ? UNESCO_STATUS[lang.unescoStatus.value] : null,
111 writingSystems: [...new Set(lang.writingSystems?.value.split(', ').filter(Boolean))]?.sort() || []
112 }).sort(([a], [b]) => a.localeCompare(b))
113
114 return Object.assign(obj, Object.fromEntries(sortedProps))
115 })
116
117 // Deduplicate by iso code (keep first occurrence, log duplicates)
118 const uniqueLanguages = [];
119 const seenIso = new Set();
120 for (const lang of languages) {
121 const existingIdx = uniqueLanguages.findIndex(l => l.iso === lang.iso);
122 if (existingIdx === -1) {
123 uniqueLanguages.push(lang);
124 seenIso.add(lang.iso);
125 } else {
126 const original = uniqueLanguages[existingIdx];
127 // Merge logic
128 const merged = { ...original };
129 // Native speakers: pick largest
130 merged.nativeSpeakers_k = Math.max(original.nativeSpeakers_k || 0, lang.nativeSpeakers_k || 0);
131 // langLabel: pick from record with largest nativeSpeakers_k
132 merged.langLabel = (lang.nativeSpeakers_k || 0) > (original.nativeSpeakers_k || 0) ? lang.langLabel : original.langLabel;
133 // Arrays: merge and dedupe
134 const mergeSet = (a, b) => Array.from(new Set([...(a || []), ...(b || [])])).sort();
135 merged.nativeNames = mergeSet(original.nativeNames, lang.nativeNames);
136 merged.countries = mergeSet(original.countries, lang.countries);
137 merged.writingSystems = mergeSet(original.writingSystems, lang.writingSystems);
138 // All other fields: keep from record with largest nativeSpeakers_k
139 for (const key of Object.keys(lang)) {
140 if (!(key in merged)) merged[key] = lang[key];
141 if (!['iso','iso1','nativeSpeakers_k','langLabel','nativeNames','countries','writingSystems'].includes(key)) {
142 if ((lang.nativeSpeakers_k || 0) > (original.nativeSpeakers_k || 0)) {
143 merged[key] = lang[key];
144 }
145 }
146 }
147 // Print debug diff
148 const mergedStr = JSON.stringify(merged, null, 2);
149 const originalStr = JSON.stringify(original, null, 2);
150 const duplicateStr = JSON.stringify(lang, null, 2);
151 const diffResult = diffLines(originalStr, mergedStr);
152 console.debug(chalk.yellow(`Merged duplicate iso code: ${lang.iso}`));
153 diffResult.forEach(part => {
154 if (part.added) {
155 process.stdout.write(chalk.green(part.value));
156 } else if (part.removed) {
157 process.stdout.write(chalk.red(part.value));
158 } else {
159 process.stdout.write(chalk.gray(part.value));
160 }
161 });
162 process.stdout.write('\n');
163 uniqueLanguages[existingIdx] = merged;
164 }
165 }
166
167 // Write to file
168 const outputPath = path.join(OUTPUT_DIR, 'wikidata-languages.json')
169 fs.writeFileSync(outputPath, JSON.stringify(uniqueLanguages, null, 2))
170 console.log(`Wrote ${uniqueLanguages.length} languages to ${outputPath}`)
171}
172
173fetchWikidataSpeakers().catch(console.error)