nandi.uk
This repository has no description
7.7 kB
242 lines
1/**
2 * island:detect
3 *
4 * Match our carry graph against discovered components to find closest island.
5 * Uses the SQLite cache from island:discover.
6 *
7 * Usage:
8 * bun island:detect # find which island we're closest to
9 * bun island:detect --top 5 # show top 5 closest islands
10 * bun island:detect --json # machine-readable output
11 */
12
13import { resolve, dirname } from 'node:path';
14import { fileURLToPath } from 'node:url';
15import { execFileSync } from 'node:child_process';
16import { AtpAgent } from '@atproto/api';
17import { loadDotEnv } from './cli-utils.js';
18import { openDb, findComponents, islandId, domainFromUrl, type Component } from './island-shared.js';
19
20const __dirname = dirname(fileURLToPath(import.meta.url));
21await loadDotEnv(resolve(__dirname, '..', '.env'));
22
23const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
24const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
25const ATPROTO_SERVICE = process.env.ATPROTO_SERVICE ?? 'https://bsky.social';
26const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
27const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
28const COLLECTION = 'network.cosmik.connection';
29
30interface MatchResult {
31 islandId: string;
32 lexmin: string;
33 nodeCount: number;
34 didCount: number;
35 sharedCount: number;
36 similarity: number;
37 sharedUrls: string[];
38}
39
40// --- Our graph ---
41
42function normalizeUrl(url: string): string {
43 try {
44 const u = new URL(url);
45 u.hash = '';
46 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
47 return u.toString();
48 } catch {
49 return url;
50 }
51}
52
53function carryQueryJson<T>(domain: string, fields: string[]): T[] {
54 try {
55 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
56 cwd: CARRY_DIR,
57 encoding: 'utf-8',
58 timeout: 15_000,
59 });
60 return JSON.parse(out) as T[];
61 } catch {
62 return [];
63 }
64}
65
66function buildOurUrlSet(): Set<string> {
67 const urls = new Set<string>();
68
69 // From carry connections
70 const connections = carryQueryJson<{ source?: string; target?: string }>('org.latha.connection', ['source', 'target']);
71 const entities = carryQueryJson<{ name?: string; url?: string }>('org.latha.entity', ['name', 'url']);
72 const entityUrls = new Map<string, string>();
73 for (const e of entities) {
74 if (e.name && e.url?.startsWith('https://')) {
75 entityUrls.set(e.name, e.url);
76 }
77 }
78
79 for (const c of connections) {
80 const sUrl = entityUrls.get(c.source ?? '') ?? (c.source?.startsWith('https://') ? c.source : undefined);
81 const tUrl = entityUrls.get(c.target ?? '') ?? (c.target?.startsWith('https://') ? c.target : undefined);
82 if (sUrl) urls.add(normalizeUrl(sUrl));
83 if (tUrl) urls.add(normalizeUrl(tUrl));
84 }
85
86 return urls;
87}
88
89async function fetchOurPdsUrls(): Promise<Set<string>> {
90 const urls = new Set<string>();
91 if (!BLUESKY_HANDLE) return urls;
92
93 try {
94 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
95 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
96 const did = agent.session!.did;
97
98 // Resolve PDS
99 const doc = await fetch(`https://plc.directory/${did}`).then((r) => r.json()) as { service?: Array<{ id: string; serviceEndpoint: string }> };
100 const pds = doc.service?.find((s) => s.id === '#atproto_pds')?.serviceEndpoint ?? PDS_SERVICE;
101
102 // Fetch our connection records
103 let cursor = '';
104 while (true) {
105 const url = new URL(`${pds}/xrpc/com.atproto.repo.listRecords`);
106 url.searchParams.set('repo', did);
107 url.searchParams.set('collection', COLLECTION);
108 url.searchParams.set('limit', '100');
109 if (cursor) url.searchParams.set('cursor', cursor);
110
111 const res = await fetch(url.toString());
112 if (!res.ok) break;
113 const data = await res.json() as {
114 records: Array<{ value: { source?: string; target?: string } }>;
115 cursor?: string;
116 };
117
118 for (const r of data.records) {
119 if (r.value.source?.startsWith('https://')) urls.add(normalizeUrl(r.value.source));
120 if (r.value.target?.startsWith('https://')) urls.add(normalizeUrl(r.value.target));
121 }
122
123 cursor = data.cursor ?? '';
124 if (!cursor) break;
125 }
126 } catch {}
127
128 return urls;
129}
130
131// --- Similarity ---
132
133function jaccard(a: Set<string>, b: Set<string>): number {
134 if (a.size === 0 && b.size === 0) return 0;
135 let intersection = 0;
136 for (const item of a) {
137 if (b.has(item)) intersection++;
138 }
139 const union = a.size + b.size - intersection;
140 return union === 0 ? 0 : intersection / union;
141}
142
143function sharedUrls(a: Set<string>, b: Set<string>): string[] {
144 const shared: string[] = [];
145 for (const url of a) {
146 if (b.has(url)) shared.push(url);
147 }
148 return shared.sort();
149}
150
151// --- Main ---
152
153async function main(): Promise<void> {
154 const args = process.argv.slice(2);
155 const wantJson = args.includes('--json');
156 const topArg = args.find((a) => a.startsWith('--top='));
157 const topN = topArg ? Number(topArg.split('=')[1]) : 5;
158
159 // Check if discover has been run
160 let db;
161 try {
162 db = openDb();
163 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
164 if (count === 0) {
165 console.error('No components found. Run `bun island:discover` first.');
166 process.exit(1);
167 }
168 } catch {
169 console.error('Database not found. Run `bun island:discover` first.');
170 process.exit(1);
171 }
172
173 console.error('Building our carry graph URL set...');
174 const ourUrls = buildOurUrlSet();
175 const pdsUrls = await fetchOurPdsUrls();
176 for (const url of pdsUrls) ourUrls.add(url);
177 console.error(`Our graph: ${ourUrls.size} unique HTTPS URLs`);
178
179 console.error('Finding components...');
180 const components = findComponents(db);
181 console.error(`Found ${components.length} components`);
182
183 // Match against each component
184 const matches: MatchResult[] = [];
185 for (const comp of components) {
186 const sim = jaccard(ourUrls, comp.nodes);
187 const shared = sharedUrls(ourUrls, comp.nodes);
188 if (shared.length > 0) {
189 matches.push({
190 islandId: comp.islandId,
191 lexmin: comp.lexmin,
192 nodeCount: comp.nodes.size,
193 didCount: comp.dids.size,
194 sharedCount: shared.length,
195 similarity: sim,
196 sharedUrls: shared,
197 });
198 }
199 }
200
201 matches.sort((a, b) => b.similarity - a.similarity);
202 const topMatches = matches.slice(0, topN);
203
204 if (wantJson) {
205 console.log(JSON.stringify({
206 ourUrlCount: ourUrls.size,
207 componentCount: components.length,
208 matchCount: matches.length,
209 topMatches: topMatches.map((m) => ({
210 ...m,
211 sharedUrls: m.sharedUrls.slice(0, 20),
212 })),
213 }, null, 2));
214 db.close();
215 return;
216 }
217
218 console.log(`\nIsland detect: our graph (${ourUrls.size} URLs) matched against ${components.length} components`);
219 console.log(`Islands with overlap: ${matches.length}\n`);
220
221 for (const m of topMatches) {
222 const domain = domainFromUrl(m.lexmin);
223 console.log(`${m.islandId} ${m.nodeCount} nodes, ${m.didCount} DIDs ${domain}`);
224 console.log(` Similarity: ${(m.similarity * 100).toFixed(2)}% (${m.sharedCount} shared URLs)`);
225 for (const url of m.sharedUrls) {
226 console.log(` ${url}`);
227 }
228 console.log();
229 }
230
231 if (topMatches.length > 0) {
232 const best = topMatches[0];
233 console.log(`\n→ We belong to island ${best.islandId} (${(best.similarity * 100).toFixed(1)}% match)`);
234 }
235
236 db.close();
237}
238
239main().catch((err) => {
240 console.error('Fatal:', err instanceof Error ? err.message : String(err));
241 process.exit(1);
242});