nandi.uk
This repository has no description
11 kB
314 lines
1/**
2 * island:cleanup
3 *
4 * Detect and remove low-quality edges from the island-detect SQLite DB.
5 * Runs in dry-run mode by default — use --apply to actually clean.
6 *
7 * Categories:
8 * at-uri — source or target is an at:// URI (not HTTPS)
9 * null-relation — relation is NULL or empty
10 * case-variant — relation is lowercase variant of a known type
11 * bsky-app-noise — bsky.app/profile/HANDLE/post/RKEY URLs (post permalinks, not content)
12 * same-domain-trivial — source and target share a registered domain, one is a subpage of the other
13 *
14 * Usage:
15 * bun island:cleanup # dry-run: report what would be cleaned
16 * bun island:cleanup --apply # actually apply the cleanup
17 * bun island:cleanup --apply --json # apply + JSON output
18 */
19
20import { resolve, dirname } from 'node:path';
21import { fileURLToPath } from 'node:url';
22import { Database } from 'bun:sqlite';
23import { loadDotEnv } from './cli-utils.js';
24import { domainFromUrl } from './island-shared.js';
25
26const __dirname = dirname(fileURLToPath(import.meta.url));
27await loadDotEnv(resolve(__dirname, '..', '.env'));
28
29const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
30const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
31
32// --- Edge classification ---
33
34interface Edge {
35 source: string;
36 target: string;
37 relation: string | null;
38 did: string;
39}
40
41interface CleanupReport {
42 atUri: { count: number; samples: Edge[] };
43 nullRelation: { count: number; samples: Edge[] };
44 caseVariant: { count: number; samples: Edge[]; normalized: Map<string, string> };
45 bskyAppNoise: { count: number; samples: Edge[] };
46 sameDomainTrivial: { count: number; samples: Edge[] };
47 totalEdges: number;
48}
49
50const BSky_POST_RE = /^https:\/\/bsky\.app\/profile\/[^/]+\/post\//;
51const CANONICAL_RELATIONS = new Set([
52 'RELATED', 'LEADS_TO', 'SUPPORTS', 'SUPPLEMENT', 'SUPPLEMENTS',
53 'EXPLAINER', 'HELPFUL', 'ADDRESSES', 'ANNOTATES', 'OPPOSES',
54 'CONTAINS', 'RELATES_TO',
55]);
56
57/** Extract registered domain: strip www., take hostname. */
58function registeredDomain(url: string): string {
59 try {
60 return new URL(url).hostname.replace(/^www\./, '');
61 } catch {
62 return url;
63 }
64}
65
66/** Check if one URL is a trivial same-domain link (subpage → root, www → non-www). */
67function isTrivialSameDomain(source: string, target: string): boolean {
68 const sDomain = registeredDomain(source);
69 const tDomain = registeredDomain(target);
70 if (sDomain !== tDomain) return false;
71
72 // Normalize scheme and www only (keep query params — different ?v= means different content)
73 const normalize = (u: string) => {
74 try {
75 const parsed = new URL(u);
76 parsed.hostname = parsed.hostname.replace(/^www\./, '');
77 // Force https
78 if (parsed.protocol === 'http:') parsed.protocol = 'https:';
79 parsed.hash = '';
80 return parsed.toString().replace(/\/+$/, '');
81 } catch {
82 return u.replace(/\/+$/, '');
83 }
84 };
85
86 const s = normalize(source);
87 const t = normalize(target);
88
89 // Exact duplicate after scheme/www normalization
90 if (s === t) return true;
91
92 // One is the root (no path beyond /) and the other is a subpage
93 // This catches: attie.ai/login → attera.org/ but NOT: youtube.com/watch?v=X → youtube.com/watch?v=Y
94 const sUrl = new URL(s);
95 const tUrl = new URL(t);
96 const sPath = sUrl.pathname.replace(/\/+$/, '');
97 const tPath = tUrl.pathname.replace(/\/+$/, '');
98
99 // Only trivial if one has an empty path (root) and the other doesn't
100 if ((sPath === '' || sPath === '/') && tPath !== '' && tPath !== '/') return true;
101 if ((tPath === '' || tPath === '/') && sPath !== '' && sPath !== '/') return true;
102
103 return false;
104}
105
106function classifyEdges(edges: Edge[]): CleanupReport {
107 const report: CleanupReport = {
108 atUri: { count: 0, samples: [] },
109 nullRelation: { count: 0, samples: [] },
110 caseVariant: { count: 0, samples: [], normalized: new Map() },
111 bskyAppNoise: { count: 0, samples: [] },
112 sameDomainTrivial: { count: 0, samples: [] },
113 totalEdges: edges.length,
114 };
115
116 const addSample = (arr: Edge[], e: Edge) => {
117 if (arr.length < 5) arr.push(e);
118 };
119
120 for (const e of edges) {
121 // at-uri: source or target starts with at://
122 if (e.source.startsWith('at://') || e.target.startsWith('at://')) {
123 report.atUri.count++;
124 addSample(report.atUri.samples, e);
125 continue; // don't double-count
126 }
127
128 // null-relation
129 if (!e.relation || e.relation.trim() === '') {
130 report.nullRelation.count++;
131 addSample(report.nullRelation.samples, e);
132 continue;
133 }
134
135 // case-variant: relation != UPPER(relation) and UPPER is a known type
136 const upper = e.relation.toUpperCase();
137 if (e.relation !== upper && CANONICAL_RELATIONS.has(upper)) {
138 report.caseVariant.count++;
139 report.caseVariant.normalized.set(e.relation, upper);
140 addSample(report.caseVariant.samples, e);
141 continue;
142 }
143
144 // bsky-app-noise: post permalinks
145 if (BSky_POST_RE.test(e.source) || BSky_POST_RE.test(e.target)) {
146 report.bskyAppNoise.count++;
147 addSample(report.bskyAppNoise.samples, e);
148 continue;
149 }
150
151 // same-domain-trivial
152 if (isTrivialSameDomain(e.source, e.target)) {
153 report.sameDomainTrivial.count++;
154 addSample(report.sameDomainTrivial.samples, e);
155 }
156 }
157
158 return report;
159}
160
161// --- Formatting ---
162
163function formatEdge(e: Edge): string {
164 const rel = e.relation ?? '(null)';
165 return ` ${e.source.slice(0, 55).padEnd(58)} → ${e.target.slice(0, 55)} [${rel}]`;
166}
167
168function printReport(report: CleanupReport): void {
169 console.log(`\nEdge cleanup report (${report.totalEdges} total edges)\n`);
170 console.log('─'.repeat(72));
171
172 const categories: Array<{ label: string; count: number; action: string; samples: Edge[] }> = [
173 { label: 'at-uri', count: report.atUri.count, action: 'delete', samples: report.atUri.samples },
174 { label: 'null-relation', count: report.nullRelation.count, action: 'delete', samples: report.nullRelation.samples },
175 { label: 'case-variant', count: report.caseVariant.count, action: 'normalize', samples: report.caseVariant.samples },
176 { label: 'bsky-app-noise', count: report.bskyAppNoise.count, action: 'delete', samples: report.bskyAppNoise.samples },
177 { label: 'same-domain-trivial', count: report.sameDomainTrivial.count, action: 'delete', samples: report.sameDomainTrivial.samples },
178 ];
179
180 for (const cat of categories) {
181 if (cat.count === 0) continue;
182 console.log(`\n${cat.label}: ${cat.count} edges (${cat.action})`);
183 for (const e of cat.samples) console.log(formatEdge(e));
184 if (cat.count > cat.samples.length) console.log(` ... and ${cat.count - cat.samples.length} more`);
185 }
186
187 // Show case-variant normalization map
188 if (report.caseVariant.count > 0) {
189 console.log('\nNormalization map:');
190 for (const [from, to] of report.caseVariant.normalized) {
191 console.log(` ${from} → ${to}`);
192 }
193 }
194
195 const totalDelete = report.atUri.count + report.nullRelation.count + report.bskyAppNoise.count + report.sameDomainTrivial.count;
196 const totalNormalize = report.caseVariant.count;
197 console.log('\n' + '─'.repeat(72));
198 console.log(`Summary: ${totalDelete} edges to delete, ${totalNormalize} relations to normalize`);
199 console.log(`Remaining: ${report.totalEdges - totalDelete} edges`);
200 console.log('\nRun with --apply to execute.');
201}
202
203// --- Apply cleanup ---
204
205function applyCleanup(db: Database, report: CleanupReport): { deleted: number; normalized: number } {
206 let deleted = 0;
207 let normalized = 0;
208
209 // Delete at-uri edges
210 if (report.atUri.count > 0) {
211 const result = db.run("DELETE FROM edges WHERE source LIKE 'at://%' OR target LIKE 'at://%'");
212 deleted += result.changes;
213 }
214
215 // Delete null-relation edges
216 if (report.nullRelation.count > 0) {
217 const result = db.run("DELETE FROM edges WHERE relation IS NULL OR relation = ''");
218 deleted += result.changes;
219 }
220
221 // Delete bsky-app-noise edges (post permalinks)
222 if (report.bskyAppNoise.count > 0) {
223 const rows = db.query("SELECT source, target, did FROM edges WHERE source LIKE 'https://bsky.app/profile/%/post/%' OR target LIKE 'https://bsky.app/profile/%/post/%'").all() as Edge[];
224 const stmt = db.query('DELETE FROM edges WHERE source = ? AND target = ? AND did = ?');
225 for (const r of rows) {
226 stmt.run(r.source, r.target, r.did);
227 }
228 deleted += rows.length;
229 }
230
231 // Delete same-domain-trivial edges
232 if (report.sameDomainTrivial.count > 0) {
233 // Re-scan to find exact rows to delete
234 const allEdges = db.query('SELECT source, target, relation, did FROM edges').all() as Edge[];
235 const stmt = db.query('DELETE FROM edges WHERE source = ? AND target = ? AND did = ?');
236 for (const e of allEdges) {
237 if (isTrivialSameDomain(e.source, e.target)) {
238 stmt.run(e.source, e.target, e.did);
239 deleted++;
240 }
241 }
242 }
243
244 // Normalize case-variant relations
245 if (report.caseVariant.count > 0) {
246 for (const [from, to] of report.caseVariant.normalized) {
247 const result = db.query('UPDATE edges SET relation = ? WHERE relation = ?').run(to, from);
248 normalized += result.changes;
249 }
250 }
251
252 return { deleted, normalized };
253}
254
255// --- Main ---
256
257async function main(): Promise<void> {
258 const args = process.argv.slice(2);
259 const apply = args.includes('--apply');
260 const wantJson = args.includes('--json');
261
262 let db: Database;
263 try {
264 db = new Database(DB_PATH);
265 db.exec('PRAGMA journal_mode = WAL');
266 db.exec('PRAGMA synchronous = NORMAL');
267 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
268 if (count === 0) {
269 console.error('No edges found. Run `bun island:discover` first.');
270 db.close();
271 process.exit(1);
272 }
273 } catch {
274 console.error('Database not found. Run `bun island:discover` first.');
275 process.exit(1);
276 }
277
278 const edges = db.query('SELECT source, target, relation, did FROM edges').all() as Edge[];
279 const report = classifyEdges(edges);
280
281 if (wantJson) {
282 const jsonReport = {
283 totalEdges: report.totalEdges,
284 atUri: report.atUri.count,
285 nullRelation: report.nullRelation.count,
286 caseVariant: report.caseVariant.count,
287 caseVariantMap: Object.fromEntries(report.caseVariant.normalized),
288 bskyAppNoise: report.bskyAppNoise.count,
289 sameDomainTrivial: report.sameDomainTrivial.count,
290 };
291 if (apply) {
292 const result = applyCleanup(db, report);
293 console.log(JSON.stringify({ ...jsonReport, applied: result }, null, 2));
294 } else {
295 console.log(JSON.stringify(jsonReport, null, 2));
296 }
297 } else {
298 printReport(report);
299
300 if (apply) {
301 const result = applyCleanup(db, report);
302 console.log(`\nApplied: ${result.deleted} edges deleted, ${result.normalized} relations normalized`);
303 const remaining = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c;
304 console.log(`Remaining edges: ${remaining}`);
305 }
306 }
307
308 db.close();
309}
310
311main().catch((err) => {
312 console.error('Fatal:', err instanceof Error ? err.message : String(err));
313 process.exit(1);
314});