nandi.uk
This repository has no description
12 kB
326 lines
1/**
2 * island:discover
3 *
4 * Find connected components in the global network.cosmik.connection graph.
5 * Uses SQLite to cache network data so we don't rebuild each time.
6 *
7 * Usage:
8 * bun island:discover # find all connected components
9 * bun island:discover --refresh # force re-fetch all DIDs
10 * bun island:discover --component 1 # show full details for component 1
11 * bun island:discover --min-size 5 # only show components with >= 5 nodes
12 * bun island:discover --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { execFileSync } from 'node:fs';
18import { Database } from 'bun:sqlite';
19import { loadDotEnv } from './cli-utils.js';
20
21const __dirname = dirname(fileURLToPath(import.meta.url));
22await loadDotEnv(resolve(__dirname, '..', '.env'));
23
24const RELAY = process.env.ATPROTO_RELAY ?? 'https://bsky.network';
25const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
26const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
27const COLLECTION = 'network.cosmik.connection';
28const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
29
30interface Component {
31 id: number;
32 nodes: Set<string>;
33 dids: Set<string>;
34}
35
36// --- SQLite ---
37
38function openDb(): Database {
39 const db = new Database(DB_PATH, { create: true });
40 db.exec('PRAGMA journal_mode = WAL');
41 db.exec('PRAGMA synchronous = NORMAL');
42 db.exec(`
43 CREATE TABLE IF NOT EXISTS edges (
44 source TEXT NOT NULL,
45 target TEXT NOT NULL,
46 relation TEXT,
47 did TEXT NOT NULL,
48 fetched_at TEXT NOT NULL,
49 PRIMARY KEY (source, target, did)
50 );
51 CREATE TABLE IF NOT EXISTS dids (
52 did TEXT PRIMARY KEY,
53 handle TEXT,
54 pds TEXT,
55 edge_count INTEGER DEFAULT 0,
56 fetched_at TEXT
57 );
58 CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source);
59 CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target);
60 `);
61 return db;
62}
63
64// --- Network discovery ---
65
66async function discoverDIDs(): Promise<string[]> {
67 const dids: string[] = [];
68 let cursor = '';
69 while (true) {
70 const url = new URL(`${RELAY}/xrpc/com.atproto.sync.listReposByCollection`);
71 url.searchParams.set('collection', COLLECTION);
72 url.searchParams.set('limit', '1000');
73 if (cursor) url.searchParams.set('cursor', cursor);
74 const res = await fetch(url.toString());
75 if (!res.ok) break;
76 const data = await res.json() as { repos: Array<{ did: string }>; cursor?: string };
77 for (const r of data.repos) dids.push(r.did);
78 cursor = data.cursor ?? '';
79 if (!cursor) break;
80 await new Promise((r) => setTimeout(r, 200));
81 }
82 return dids;
83}
84
85async function resolvePDS(did: string): Promise<string | null> {
86 try {
87 const res = await fetch(`https://plc.directory/${did}`);
88 if (!res.ok) return null;
89 const doc = await res.json() as { service?: Array<{ id: string; serviceEndpoint: string }> };
90 return doc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint ?? null;
91 } catch { return null; }
92}
93
94async function resolveHandle(did: string): Promise<string | null> {
95 try {
96 const res = await fetch(`https://plc.directory/${did}`);
97 if (!res.ok) return null;
98 const doc = await res.json() as { alsoKnownAs?: string[] };
99 return doc.alsoKnownAs?.find((h) => h.startsWith('at://'))?.replace('at://', '') ?? null;
100 } catch { return null; }
101}
102
103async function fetchConnectionRecords(pds: string, did: string): Promise<Array<{ source: string; target: string; relation?: string }>> {
104 const records: Array<{ source: string; target: string; relation?: string }> = [];
105 let cursor = '';
106 while (true) {
107 const url = new URL(`${pds}/xrpc/com.atproto.repo.listRecords`);
108 url.searchParams.set('repo', did);
109 url.searchParams.set('collection', COLLECTION);
110 url.searchParams.set('limit', '100');
111 if (cursor) url.searchParams.set('cursor', cursor);
112 const res = await fetch(url.toString());
113 if (!res.ok) break;
114 const data = await res.json() as {
115 records: Array<{ value: { source?: string; target?: string; connectionType?: string } }>;
116 cursor?: string;
117 };
118 for (const r of data.records) {
119 if (r.value.source && r.value.target) {
120 records.push({ source: r.value.source, target: r.value.target, relation: r.value.connectionType });
121 }
122 }
123 cursor = data.cursor ?? '';
124 if (!cursor) break;
125 await new Promise((r) => setTimeout(r, 100));
126 }
127 return records;
128}
129
130// --- Cache operations ---
131
132function isCached(db: Database, did: string): boolean {
133 const row = db.query('SELECT fetched_at FROM dids WHERE did = ?').get(did) as { fetched_at: string } | null;
134 if (!row) return false;
135 return Date.now() - new Date(row.fetched_at).getTime() < CACHE_TTL_MS;
136}
137
138function upsertDid(db: Database, did: string, handle: string | null, pds: string | null, edgeCount: number): void {
139 const now = new Date().toISOString();
140 db.query(`
141 INSERT INTO dids (did, handle, pds, edge_count, fetched_at)
142 VALUES (?, ?, ?, ?, ?)
143 ON CONFLICT(did) DO UPDATE SET handle=?, pds=?, edge_count=?, fetched_at=?
144 `).run(did, handle, pds, edgeCount, now, handle, pds, edgeCount, now);
145}
146
147function upsertEdges(db: Database, did: string, records: Array<{ source: string; target: string; relation?: string }>): void {
148 const now = new Date().toISOString();
149 const insert = db.query(`
150 INSERT INTO edges (source, target, relation, did, fetched_at)
151 VALUES (?, ?, ?, ?, ?)
152 ON CONFLICT(source, target, did) DO UPDATE SET relation=?, fetched_at=?
153 `);
154 // Delete stale edges for this DID before inserting
155 db.query('DELETE FROM edges WHERE did = ?').run(did);
156 for (const r of records) {
157 insert.run(r.source, r.target, r.relation ?? null, did, now, r.relation ?? null, now);
158 }
159}
160
161// --- Connected components ---
162
163function findComponents(db: Database): Component[] {
164 // Build adjacency list from all edges
165 const adj = new Map<string, Set<string>>();
166 const edgeDids = new Map<string, Set<string>>(); // node -> which DIDs have edges touching it
167
168 const rows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
169 for (const row of rows) {
170 const s = row.source;
171 const t = row.target;
172 if (!adj.has(s)) adj.set(s, new Set());
173 if (!adj.has(t)) adj.set(t, new Set());
174 adj.get(s)!.add(t);
175 adj.get(t)!.add(s);
176 if (!edgeDids.has(s)) edgeDids.set(s, new Set());
177 if (!edgeDids.has(t)) edgeDids.set(t, new Set());
178 edgeDids.get(s)!.add(row.did);
179 edgeDids.get(t)!.add(row.did);
180 }
181
182 // BFS to find components
183 const visited = new Set<string>();
184 const components: Component[] = [];
185 let compId = 0;
186
187 for (const node of adj.keys()) {
188 if (visited.has(node)) continue;
189 compId++;
190 const compNodes = new Set<string>();
191 const compDids = new Set<string>();
192 const queue = [node];
193 while (queue.length > 0) {
194 const current = queue.pop()!;
195 if (visited.has(current)) continue;
196 visited.add(current);
197 compNodes.add(current);
198 for (const did of edgeDids.get(current) ?? []) compDids.add(did);
199 for (const neighbor of adj.get(current) ?? []) {
200 if (!visited.has(neighbor)) queue.push(neighbor);
201 }
202 }
203 components.push({ id: compId, nodes: compNodes, dids: compDids });
204 }
205
206 return components.sort((a, b) => b.nodes.size - a.nodes.size);
207}
208
209// --- Main ---
210
211async function main(): Promise<void> {
212 const args = process.argv.slice(2);
213 const wantJson = args.includes('--json');
214 const refresh = args.includes('--refresh');
215 const componentArg = args.find((a) => a.startsWith('--component='));
216 const showComponent = componentArg ? Number(componentArg.split('=')[1]) : null;
217 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
218 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
219
220 const db = openDb();
221
222 console.error('Discovering network DIDs...');
223 const networkDids = await discoverDIDs();
224 console.error(`Found ${networkDids.length} DIDs`);
225
226 // Fetch edges for each DID (use cache unless --refresh)
227 let fetched = 0;
228 let cached = 0;
229 for (let i = 0; i < networkDids.length; i++) {
230 const did = networkDids[i];
231 if (!refresh && isCached(db, did)) {
232 cached++;
233 continue;
234 }
235 console.error(`[${i + 1}/${networkDids.length}] Fetching ${did}...`);
236 const pds = await resolvePDS(did);
237 if (!pds) { console.error(' Could not resolve PDS'); continue; }
238 const handle = await resolveHandle(did);
239 const records = await fetchConnectionRecords(pds, did);
240 upsertDid(db, did, handle, pds, records.length);
241 upsertEdges(db, did, records);
242 fetched++;
243 }
244 console.error(`Fetched: ${fetched}, Cached: ${cached}`);
245
246 // Stats
247 const totalEdges = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c;
248 const totalDids = (db.query('SELECT COUNT(*) as c FROM dids').get() as { c: number }).c;
249 const uniqueUrls = new Set<string>();
250 const urlRows = db.query('SELECT DISTINCT source FROM edges UNION SELECT DISTINCT target FROM edges').all() as Array<{ source: string }>;
251 for (const r of urlRows) uniqueUrls.add(r.source);
252
253 console.error(`Database: ${totalEdges} edges, ${totalDids} DIDs, ${uniqueUrls.size} unique URLs`);
254
255 // Find connected components
256 const components = findComponents(db);
257 const filtered = components.filter((c) => c.nodes.size >= minSize);
258
259 if (showComponent !== null) {
260 const comp = components[showComponent - 1];
261 if (!comp) { console.error(`Component ${showComponent} not found`); process.exit(1); }
262 const didRows = db.query('SELECT did, handle, edge_count FROM dids WHERE did IN (?)').all(...comp.dids) as Array<{ did: string; handle: string | null; edge_count: number }>;
263 if (wantJson) {
264 console.log(JSON.stringify({
265 id: comp.id,
266 nodeCount: comp.nodes.size,
267 didCount: comp.dids.size,
268 nodes: [...comp.nodes].sort(),
269 dids: didRows,
270 }, null, 2));
271 } else {
272 console.log(`Component ${showComponent}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
273 console.log('\nNodes:');
274 for (const url of [...comp.nodes].sort()) console.log(` ${url}`);
275 console.log('\nDIDs:');
276 for (const d of didRows) console.log(` ${d.handle ?? d.did} (${d.edge_count} edges)`);
277 }
278 db.close();
279 return;
280 }
281
282 if (wantJson) {
283 console.log(JSON.stringify({
284 totalEdges,
285 totalDids,
286 uniqueUrls: uniqueUrls.size,
287 componentCount: filtered.length,
288 components: filtered.map((c) => ({
289 id: c.id,
290 nodeCount: c.nodes.size,
291 didCount: c.dids.size,
292 sampleNodes: [...c.nodes].sort().slice(0, 10),
293 dids: [...c.dids],
294 })),
295 }, null, 2));
296 db.close();
297 return;
298 }
299
300 console.log(`\nIsland detect: ${totalDids} DIDs, ${totalEdges} edges, ${uniqueUrls.size} unique URLs`);
301 console.log(`Connected components (>= ${minSize} nodes): ${filtered.length}\n`);
302
303 for (const comp of filtered.slice(0, 30)) {
304 const sampleNodes = [...comp.nodes].sort().slice(0, 5);
305 console.log(`Component ${comp.id}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
306 console.log(` URLs: ${sampleNodes.join(', ')}${comp.nodes.size > 5 ? ` ... (+${comp.nodes.size - 5})` : ''}`);
307 }
308
309 // Show which component we're in (if we can determine our DID)
310 const ourDid = process.env.BLUESKY_IDENTIFIER
311 ? (db.query('SELECT did FROM dids WHERE handle = ? OR did LIKE ?').get(process.env.BLUESKY_IDENTIFIER, `%${process.env.BLUESKY_IDENTIFIER}%`) as { did: string } | null)
312 : null;
313 if (ourDid) {
314 const ourComp = components.find((c) => c.dids.has(ourDid.did));
315 if (ourComp) {
316 console.log(`\nOur island: Component ${ourComp.id} (${ourComp.nodes.size} nodes, shared with ${ourComp.dids.size - 1} other DIDs)`);
317 }
318 }
319
320 db.close();
321}
322
323main().catch((err) => {
324 console.error('Fatal:', err instanceof Error ? err.message : String(err));
325 process.exit(1);
326});