nandi.uk
This repository has no description
9.4 kB
298 lines
1/**
2 * island:detect
3 *
4 * Match our carry graph against discovered components to find closest island.
5 * Uses the SQLite cache from island:discover.
6 *
7 * Usage:
8 * bun island:detect # find which component we're closest to
9 * bun island:detect --top 5 # show top 5 closest components
10 * bun island:detect --json # machine-readable output
11 */
12
13import { resolve, dirname } from 'node:path';
14import { fileURLToPath } from 'node:url';
15import { execFileSync } from 'node:child_process';
16import { Database } from 'bun:sqlite';
17import { AtpAgent } from '@atproto/api';
18import { loadDotEnv } from './cli-utils.js';
19
20const __dirname = dirname(fileURLToPath(import.meta.url));
21await loadDotEnv(resolve(__dirname, '..', '.env'));
22
23const RELAY = process.env.ATPROTO_RELAY ?? 'https://bsky.network';
24const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
25const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
26const ATPROTO_SERVICE = process.env.ATPROTO_SERVICE ?? 'https://bsky.social';
27const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
28const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
29const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
30const COLLECTION = 'network.cosmik.connection';
31
32interface Component {
33 id: number;
34 nodes: Set<string>;
35 dids: Set<string>;
36}
37
38interface MatchResult {
39 componentId: number;
40 nodeCount: number;
41 didCount: number;
42 sharedCount: number;
43 similarity: number;
44 sharedUrls: string[];
45}
46
47// --- SQLite ---
48
49function openDb(): Database {
50 const db = new Database(DB_PATH, { readonly: true });
51 return db;
52}
53
54// --- Our graph ---
55
56function normalizeUrl(url: string): string {
57 try {
58 const u = new URL(url);
59 u.hash = '';
60 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
61 return u.toString();
62 } catch {
63 return url;
64 }
65}
66
67function carryQueryJson<T>(domain: string, fields: string[]): T[] {
68 try {
69 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
70 cwd: CARRY_DIR,
71 encoding: 'utf-8',
72 timeout: 15_000,
73 });
74 return JSON.parse(out) as T[];
75 } catch {
76 return [];
77 }
78}
79
80function buildOurUrlSet(): Set<string> {
81 const urls = new Set<string>();
82
83 // From carry connections
84 const connections = carryQueryJson<{ source?: string; target?: string }>('org.latha.connection', ['source', 'target']);
85 const entities = carryQueryJson<{ name?: string; url?: string }>('org.latha.entity', ['name', 'url']);
86 const entityUrls = new Map<string, string>();
87 for (const e of entities) {
88 if (e.name && e.url?.startsWith('https://')) {
89 entityUrls.set(e.name, e.url);
90 }
91 }
92
93 for (const c of connections) {
94 const sUrl = entityUrls.get(c.source ?? '') ?? (c.source?.startsWith('https://') ? c.source : undefined);
95 const tUrl = entityUrls.get(c.target ?? '') ?? (c.target?.startsWith('https://') ? c.target : undefined);
96 if (sUrl) urls.add(normalizeUrl(sUrl));
97 if (tUrl) urls.add(normalizeUrl(tUrl));
98 }
99
100 return urls;
101}
102
103async function fetchOurPdsUrls(): Promise<Set<string>> {
104 const urls = new Set<string>();
105 if (!BLUESKY_HANDLE) return urls;
106
107 try {
108 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
109 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
110 const did = agent.session!.did;
111
112 // Resolve PDS
113 const doc = await fetch(`https://plc.directory/${did}`).then((r) => r.json()) as { service?: Array<{ id: string; serviceEndpoint: string }> };
114 const pds = doc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint ?? PDS_SERVICE;
115
116 // Fetch our connection records
117 let cursor = '';
118 while (true) {
119 const url = new URL(`${pds}/xrpc/com.atproto.repo.listRecords`);
120 url.searchParams.set('repo', did);
121 url.searchParams.set('collection', COLLECTION);
122 url.searchParams.set('limit', '100');
123 if (cursor) url.searchParams.set('cursor', cursor);
124
125 const res = await fetch(url.toString());
126 if (!res.ok) break;
127 const data = await res.json() as {
128 records: Array<{ value: { source?: string; target?: string } }>;
129 cursor?: string;
130 };
131
132 for (const r of data.records) {
133 if (r.value.source?.startsWith('https://')) urls.add(normalizeUrl(r.value.source));
134 if (r.value.target?.startsWith('https://')) urls.add(normalizeUrl(r.value.target));
135 }
136
137 cursor = data.cursor ?? '';
138 if (!cursor) break;
139 }
140 } catch {}
141
142 return urls;
143}
144
145// --- Components ---
146
147function findComponents(db: Database): Component[] {
148 const adj = new Map<string, Set<string>>();
149 const edgeDids = new Map<string, Set<string>>();
150
151 const rows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
152 for (const row of rows) {
153 const s = row.source;
154 const t = row.target;
155 if (!adj.has(s)) adj.set(s, new Set());
156 if (!adj.has(t)) adj.set(t, new Set());
157 adj.get(s)!.add(t);
158 adj.get(t)!.add(s);
159 if (!edgeDids.has(s)) edgeDids.set(s, new Set());
160 if (!edgeDids.has(t)) edgeDids.set(t, new Set());
161 edgeDids.get(s)!.add(row.did);
162 edgeDids.get(t)!.add(row.did);
163 }
164
165 const visited = new Set<string>();
166 const components: Component[] = [];
167 let compId = 0;
168
169 for (const node of adj.keys()) {
170 if (visited.has(node)) continue;
171 compId++;
172 const compNodes = new Set<string>();
173 const compDids = new Set<string>();
174 const queue = [node];
175 while (queue.length > 0) {
176 const current = queue.pop()!;
177 if (visited.has(current)) continue;
178 visited.add(current);
179 compNodes.add(current);
180 for (const did of edgeDids.get(current) ?? []) compDids.add(did);
181 for (const neighbor of adj.get(current) ?? []) {
182 if (!visited.has(neighbor)) queue.push(neighbor);
183 }
184 }
185 components.push({ id: compId, nodes: compNodes, dids: compDids });
186 }
187
188 return components.sort((a, b) => b.nodes.size - a.nodes.size);
189}
190
191function jaccard(a: Set<string>, b: Set<string>): number {
192 if (a.size === 0 && b.size === 0) return 0;
193 let intersection = 0;
194 for (const item of a) {
195 if (b.has(item)) intersection++;
196 }
197 const union = a.size + b.size - intersection;
198 return union === 0 ? 0 : intersection / union;
199}
200
201function sharedUrls(a: Set<string>, b: Set<string>): string[] {
202 const shared: string[] = [];
203 for (const url of a) {
204 if (b.has(url)) shared.push(url);
205 }
206 return shared.sort();
207}
208
209// --- Main ---
210
211async function main(): Promise<void> {
212 const args = process.argv.slice(2);
213 const wantJson = args.includes('--json');
214 const topArg = args.find((a) => a.startsWith('--top='));
215 const topN = topArg ? Number(topArg.split('=')[1]) : 5;
216
217 // Check if discover has been run
218 let db: Database;
219 try {
220 db = openDb();
221 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
222 if (count === 0) {
223 console.error('No components found. Run `bun island:discover` first.');
224 process.exit(1);
225 }
226 } catch {
227 console.error('Database not found. Run `bun island:discover` first.');
228 process.exit(1);
229 }
230
231 console.error('Building our carry graph URL set...');
232 const ourUrls = buildOurUrlSet();
233 const pdsUrls = await fetchOurPdsUrls();
234 for (const url of pdsUrls) ourUrls.add(url);
235 console.error(`Our graph: ${ourUrls.size} unique HTTPS URLs`);
236
237 console.error('Finding components...');
238 const components = findComponents(db);
239 console.error(`Found ${components.length} components`);
240
241 // Match against each component
242 const matches: MatchResult[] = [];
243 for (const comp of components) {
244 const sim = jaccard(ourUrls, comp.nodes);
245 const shared = sharedUrls(ourUrls, comp.nodes);
246 if (shared.length > 0) {
247 matches.push({
248 componentId: comp.id,
249 nodeCount: comp.nodes.size,
250 didCount: comp.dids.size,
251 sharedCount: shared.length,
252 similarity: sim,
253 sharedUrls: shared,
254 });
255 }
256 }
257
258 matches.sort((a, b) => b.similarity - a.similarity);
259 const topMatches = matches.slice(0, topN);
260
261 if (wantJson) {
262 console.log(JSON.stringify({
263 ourUrlCount: ourUrls.size,
264 componentCount: components.length,
265 matchCount: matches.length,
266 topMatches: topMatches.map((m) => ({
267 ...m,
268 sharedUrls: m.sharedUrls.slice(0, 20),
269 })),
270 }, null, 2));
271 db.close();
272 return;
273 }
274
275 console.log(`\nIsland detect: our graph (${ourUrls.size} URLs) matched against ${components.length} components`);
276 console.log(`Components with overlap: ${matches.length}\n`);
277
278 for (const m of topMatches) {
279 console.log(`Component ${m.componentId}: ${m.nodeCount} nodes, ${m.didCount} DIDs`);
280 console.log(` Similarity: ${(m.similarity * 100).toFixed(2)}% (${m.sharedCount} shared URLs)`);
281 for (const url of m.sharedUrls) {
282 console.log(` ${url}`);
283 }
284 console.log();
285 }
286
287 if (topMatches.length > 0) {
288 const best = topMatches[0];
289 console.log(`\n→ We belong to Component ${best.componentId} (${(best.similarity * 100).toFixed(1)}% match)`);
290 }
291
292 db.close();
293}
294
295main().catch((err) => {
296 console.error('Fatal:', err instanceof Error ? err.message : String(err));
297 process.exit(1);
298});