nandi.uk
This repository has no description
13 kB
346 lines
1/**
2 * island:discover
3 *
4 * Find connected components in the global network.cosmik.connection graph.
5 * Uses SQLite to cache network data so we don't rebuild each time.
6 *
7 * Usage:
8 * bun island:discover # find all connected components
9 * bun island:discover --refresh # force re-fetch all DIDs
10 * bun island:discover <island-id> # show full details for an island
11 * bun island:discover --min-size 5 # only show components with >= 5 nodes
12 * bun island:discover --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { createHash } from 'node:crypto';
18import { Database } from 'bun:sqlite';
19import { loadDotEnv } from './cli-utils.js';
20import { islandId, domainFromUrl, computeCentroid, type Component } from './island-shared.js';
21
22const __dirname = dirname(fileURLToPath(import.meta.url));
23await loadDotEnv(resolve(__dirname, '..', '.env'));
24
25const RELAY = process.env.ATPROTO_RELAY ?? 'https://bsky.network';
26const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
27const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
28const COLLECTION = 'network.cosmik.connection';
29const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
30
31// --- SQLite ---
32
33function openDb(): Database {
34 const db = new Database(DB_PATH, { create: true });
35 db.exec('PRAGMA journal_mode = WAL');
36 db.exec('PRAGMA synchronous = NORMAL');
37 db.exec(`
38 CREATE TABLE IF NOT EXISTS edges (
39 source TEXT NOT NULL,
40 target TEXT NOT NULL,
41 relation TEXT,
42 did TEXT NOT NULL,
43 fetched_at TEXT NOT NULL,
44 PRIMARY KEY (source, target, did)
45 );
46 CREATE TABLE IF NOT EXISTS dids (
47 did TEXT PRIMARY KEY,
48 handle TEXT,
49 pds TEXT,
50 edge_count INTEGER DEFAULT 0,
51 fetched_at TEXT
52 );
53 CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source);
54 CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target);
55 `);
56 return db;
57}
58
59// --- Network discovery ---
60
61async function discoverDIDs(): Promise<string[]> {
62 const dids: string[] = [];
63 let cursor = '';
64 while (true) {
65 const url = new URL(`${RELAY}/xrpc/com.atproto.sync.listReposByCollection`);
66 url.searchParams.set('collection', COLLECTION);
67 url.searchParams.set('limit', '1000');
68 if (cursor) url.searchParams.set('cursor', cursor);
69 const res = await fetch(url.toString());
70 if (!res.ok) break;
71 const data = await res.json() as { repos: Array<{ did: string }>; cursor?: string };
72 for (const r of data.repos) dids.push(r.did);
73 cursor = data.cursor ?? '';
74 if (!cursor) break;
75 await new Promise((r) => setTimeout(r, 200));
76 }
77 return dids;
78}
79
80async function resolveDidDoc(did: string): Promise<{ pds: string | null; handle: string | null } | null> {
81 try {
82 const res = await fetch(`https://plc.directory/${did}`);
83 if (!res.ok) return null;
84 const doc = await res.json() as { service?: Array<{ id: string; serviceEndpoint: string }>; alsoKnownAs?: string[] };
85 const pds = doc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint ?? null;
86 const handle = doc.alsoKnownAs?.find((h) => h.startsWith('at://'))?.replace('at://', '') ?? null;
87 return { pds, handle };
88 } catch { return null; }
89}
90
91async function fetchConnectionRecords(pds: string, did: string): Promise<Array<{ source: string; target: string; relation?: string; atUri: string }>> {
92 const records: Array<{ source: string; target: string; relation?: string; atUri: string }> = [];
93 let cursor = '';
94 while (true) {
95 const url = new URL(`${pds}/xrpc/com.atproto.repo.listRecords`);
96 url.searchParams.set('repo', did);
97 url.searchParams.set('collection', COLLECTION);
98 url.searchParams.set('limit', '100');
99 if (cursor) url.searchParams.set('cursor', cursor);
100 const res = await fetch(url.toString());
101 if (!res.ok) break;
102 const data = await res.json() as {
103 records: Array<{ uri: string; value: { source?: string; target?: string; connectionType?: string } }>;
104 cursor?: string;
105 };
106 for (const r of data.records) {
107 if (r.value.source && r.value.target) {
108 records.push({ source: r.value.source, target: r.value.target, relation: r.value.connectionType, atUri: r.uri });
109 }
110 }
111 cursor = data.cursor ?? '';
112 if (!cursor) break;
113 await new Promise((r) => setTimeout(r, 100));
114 }
115 return records;
116}
117
118// --- Cache operations ---
119
120function isCached(db: Database, did: string): boolean {
121 const row = db.query('SELECT fetched_at FROM dids WHERE did = ?').get(did) as { fetched_at: string } | null;
122 if (!row) return false;
123 return Date.now() - new Date(row.fetched_at).getTime() < CACHE_TTL_MS;
124}
125
126function upsertDid(db: Database, did: string, handle: string | null, pds: string | null, edgeCount: number): void {
127 const now = new Date().toISOString();
128 db.query(`
129 INSERT INTO dids (did, handle, pds, edge_count, fetched_at)
130 VALUES (?, ?, ?, ?, ?)
131 ON CONFLICT(did) DO UPDATE SET handle=?, pds=?, edge_count=?, fetched_at=?
132 `).run(did, handle, pds, edgeCount, now, handle, pds, edgeCount, now);
133}
134
135function upsertEdges(db: Database, did: string, records: Array<{ source: string; target: string; relation?: string; atUri: string }>): void {
136 const now = new Date().toISOString();
137 // Add at_uri column if missing
138 try { db.exec('ALTER TABLE edges ADD COLUMN at_uri TEXT'); } catch {}
139 const insert = db.query(`
140 INSERT INTO edges (source, target, relation, did, at_uri, fetched_at)
141 VALUES (?, ?, ?, ?, ?, ?)
142 ON CONFLICT(source, target, did) DO UPDATE SET relation=?, at_uri=?, fetched_at=?
143 `);
144 // Delete stale edges for this DID before inserting
145 db.query('DELETE FROM edges WHERE did = ?').run(did);
146 for (const r of records) {
147 insert.run(r.source, r.target, r.relation ?? null, did, r.atUri, now, r.relation ?? null, r.atUri, now);
148 }
149}
150
151// --- Connected components ---
152
153function findComponents(db: Database): Component[] {
154 // Build adjacency list from all edges
155 const adj = new Map<string, Set<string>>();
156 const edgeDids = new Map<string, Set<string>>(); // node -> which DIDs have edges touching it
157
158 const rows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
159 for (const row of rows) {
160 const s = row.source;
161 const t = row.target;
162 if (!adj.has(s)) adj.set(s, new Set());
163 if (!adj.has(t)) adj.set(t, new Set());
164 adj.get(s)!.add(t);
165 adj.get(t)!.add(s);
166 if (!edgeDids.has(s)) edgeDids.set(s, new Set());
167 if (!edgeDids.has(t)) edgeDids.set(t, new Set());
168 edgeDids.get(s)!.add(row.did);
169 edgeDids.get(t)!.add(row.did);
170 }
171
172 // BFS to find components
173 const visited = new Set<string>();
174 const components: Component[] = [];
175
176 for (const node of adj.keys()) {
177 if (visited.has(node)) continue;
178 const compNodes = new Set<string>();
179 const compDids = new Set<string>();
180 const queue = [node];
181 while (queue.length > 0) {
182 const current = queue.pop()!;
183 if (visited.has(current)) continue;
184 visited.add(current);
185 compNodes.add(current);
186 for (const did of edgeDids.get(current) ?? []) compDids.add(did);
187 for (const neighbor of adj.get(current) ?? []) {
188 if (!visited.has(neighbor)) queue.push(neighbor);
189 }
190 }
191 const lexmin = [...compNodes].sort()[0];
192 const centroid = computeCentroid(adj, compNodes);
193 components.push({ nodes: compNodes, dids: compDids, lexmin, centroid, islandId: islandId(centroid) });
194 }
195
196 return components.sort((a, b) => b.nodes.size - a.nodes.size);
197}
198
199// --- Main ---
200
201async function main(): Promise<void> {
202 const args = process.argv.slice(2);
203 const wantJson = args.includes('--json');
204 const refresh = args.includes('--refresh');
205 const targetId = args.find((a) => !a.startsWith('--'));
206 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
207 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
208
209 const db = openDb();
210
211 console.error('Discovering network DIDs...');
212 const networkDids = await discoverDIDs();
213 console.error(`Found ${networkDids.length} DIDs`);
214
215 // Fetch edges for each DID (use cache unless --refresh), parallelized
216 let fetched = 0;
217 let cached = 0;
218 const toFetch: string[] = [];
219 for (const did of networkDids) {
220 if (!refresh && isCached(db, did)) {
221 cached++;
222 } else {
223 toFetch.push(did);
224 }
225 }
226 console.error(`Cached: ${cached}, Fetching: ${toFetch.length}`);
227
228 const CONCURRENCY = 10;
229 for (let i = 0; i < toFetch.length; i += CONCURRENCY) {
230 const batch = toFetch.slice(i, i + CONCURRENCY);
231 const results = await Promise.allSettled(batch.map(async (did) => {
232 const didDoc = await resolveDidDoc(did);
233 if (!didDoc || !didDoc.pds) return null;
234 const records = await fetchConnectionRecords(didDoc.pds, did);
235 return { did, handle: didDoc.handle, pds: didDoc.pds, records };
236 }));
237 for (const r of results) {
238 if (r.status === 'fulfilled' && r.value) {
239 const { did, handle, pds, records } = r.value;
240 upsertDid(db, did, handle, pds, records.length);
241 upsertEdges(db, did, records);
242 fetched++;
243 }
244 }
245 console.error(`[${Math.min(i + CONCURRENCY, toFetch.length)}/${toFetch.length}] fetched=${fetched}`);
246 }
247 console.error(`Fetched: ${fetched}, Cached: ${cached}`);
248
249 // Stats
250 const totalEdges = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c;
251 const totalDids = (db.query('SELECT COUNT(*) as c FROM dids').get() as { c: number }).c;
252 const uniqueUrls = new Set<string>();
253 const urlRows = db.query('SELECT DISTINCT source FROM edges UNION SELECT DISTINCT target FROM edges').all() as Array<{ source: string }>;
254 for (const r of urlRows) uniqueUrls.add(r.source);
255
256 console.error(`Database: ${totalEdges} edges, ${totalDids} DIDs, ${uniqueUrls.size} unique URLs`);
257
258 // Find connected components
259 const components = findComponents(db);
260 const filtered = components.filter((c) => c.nodes.size >= minSize);
261
262 // Show a specific island by ID
263 if (targetId) {
264 const comp = components.find((c) => c.islandId === targetId);
265 if (!comp) {
266 console.error(`Island ${targetId} not found.`);
267 // Suggest similar IDs
268 const similar = filtered.filter((c) => c.islandId.startsWith(targetId.slice(0, 4)));
269 if (similar.length > 0) {
270 console.error(`Similar IDs:`);
271 for (const s of similar.slice(0, 5)) console.error(` ${s.islandId} ${s.nodes.size} nodes ${domainFromUrl(s.centroid ?? s.lexmin)}`);
272 } else {
273 console.error(`Run without args to list available islands.`);
274 }
275 process.exit(1);
276 }
277 const didPlaceholders = [...comp.dids].map(() => '?').join(',');
278 const didRows = db.query(`SELECT did, handle, edge_count FROM dids WHERE did IN (${didPlaceholders})`).all(...comp.dids) as Array<{ did: string; handle: string | null; edge_count: number }>;
279 if (wantJson) {
280 console.log(JSON.stringify({
281 id: comp.islandId,
282 centroid: comp.centroid,
283 lexmin: comp.lexmin,
284 nodeCount: comp.nodes.size,
285 didCount: comp.dids.size,
286 nodes: [...comp.nodes].sort(),
287 dids: didRows,
288 }, null, 2));
289 } else {
290 console.log(`Island ${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
291 console.log(`Centroid: ${comp.centroid}`);
292 console.log('\nNodes:');
293 for (const url of [...comp.nodes].sort()) console.log(` ${url}`);
294 console.log('\nDIDs:');
295 for (const d of didRows) console.log(` ${d.handle ?? d.did} (${d.edge_count} edges)`);
296 }
297 db.close();
298 return;
299 }
300
301 if (wantJson) {
302 console.log(JSON.stringify({
303 totalEdges,
304 totalDids,
305 uniqueUrls: uniqueUrls.size,
306 componentCount: filtered.length,
307 components: filtered.map((c) => ({
308 id: c.islandId,
309 centroid: c.centroid,
310 lexmin: c.lexmin,
311 nodeCount: c.nodes.size,
312 didCount: c.dids.size,
313 sampleNodes: [...c.nodes].sort().slice(0, 10),
314 dids: [...c.dids],
315 })),
316 }, null, 2));
317 db.close();
318 return;
319 }
320
321 console.log(`\nIsland discover: ${totalDids} DIDs, ${totalEdges} edges, ${uniqueUrls.size} unique URLs`);
322 console.log(`Connected components (>= ${minSize} nodes): ${filtered.length}\n`);
323
324 for (const comp of filtered.slice(0, 30)) {
325 const domain = domainFromUrl(comp.centroid);
326 console.log(`${comp.islandId} ${comp.nodes.size} nodes, ${comp.dids.size} DIDs ${domain}`);
327 }
328
329 // Show which component we're in (if we can determine our DID)
330 const ourDid = process.env.BLUESKY_IDENTIFIER
331 ? (db.query('SELECT did FROM dids WHERE handle = ? OR did LIKE ?').get(process.env.BLUESKY_IDENTIFIER, `%${process.env.BLUESKY_IDENTIFIER}%`) as { did: string } | null)
332 : null;
333 if (ourDid) {
334 const ourComp = components.find((c) => c.dids.has(ourDid.did));
335 if (ourComp) {
336 console.log(`\nOur island: ${ourComp.islandId} (${ourComp.nodes.size} nodes, shared with ${ourComp.dids.size - 1} other DIDs)`);
337 }
338 }
339
340 db.close();
341}
342
343main().catch((err) => {
344 console.error('Fatal:', err instanceof Error ? err.message : String(err));
345 process.exit(1);
346});