nandi.uk
This repository has no description
11 kB
329 lines
1/**
2 * connection:detect
3 *
4 * Compare our carry graph against other DIDs' connection records.
5 * Discovers DIDs publishing network.cosmik.connection records,
6 * fetches their edge sets, and computes Jaccard similarity against our graph.
7 *
8 * Usage:
9 * bun connection:detect # compare against all network DIDs
10 * bun connection:detect --limit 10 # only check 10 DIDs
11 * bun connection:detect --threshold 0.05 # only show similarity >= 0.05
12 * bun connection:detect --json # machine-readable output
13 * bun connection:detect --write # create connection records for top matches
14 */
15
16import { resolve, dirname } from 'node:path';
17import { fileURLToPath } from 'node:url';
18import { execFileSync } from 'node:child_process';
19import { AtpAgent } from '@atproto/api';
20import { loadDotEnv } from './cli-utils.js';
21
22const __dirname = dirname(fileURLToPath(import.meta.url));
23await loadDotEnv(resolve(__dirname, '..', '.env'));
24
25const RELAY = process.env.ATPROTO_RELAY ?? 'https://bsky.network';
26const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
27const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
28const ATPROTO_SERVICE = process.env.ATPROTO_SERVICE ?? 'https://bsky.social';
29const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
30const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
31const COLLECTION = 'network.cosmik.connection';
32
33interface NetworkIsland {
34 did: string;
35 handle?: string;
36 pds?: string;
37 urls: Set<string>;
38 edgeCount: number;
39}
40
41interface SimilarityResult {
42 did: string;
43 handle?: string;
44 edgeCount: number;
45 sharedCount: number;
46 similarity: number;
47 sharedUrls: string[];
48}
49
50// --- Network discovery ---
51
52async function discoverDIDs(limit?: number): Promise<string[]> {
53 const dids: string[] = [];
54 let cursor = '';
55 while (true) {
56 const url = new URL(`${RELAY}/xrpc/com.atproto.sync.listReposByCollection`);
57 url.searchParams.set('collection', COLLECTION);
58 url.searchParams.set('limit', '1000');
59 if (cursor) url.searchParams.set('cursor', cursor);
60
61 const res = await fetch(url.toString());
62 if (!res.ok) {
63 console.error(`Relay query failed (${res.status}): ${await res.text().catch(() => '')}`);
64 break;
65 }
66 const data = await res.json() as { repos: Array<{ did: string }>; cursor?: string };
67 for (const r of data.repos) {
68 dids.push(r.did);
69 if (limit && dids.length >= limit) return dids;
70 }
71 cursor = data.cursor ?? '';
72 if (!cursor) break;
73 await new Promise((r) => setTimeout(r, 200)); // rate limit
74 }
75 return dids;
76}
77
78async function resolvePDS(did: string): Promise<string | null> {
79 try {
80 const res = await fetch(`https://plc.directory/${did}`);
81 if (!res.ok) return null;
82 const doc = await res.json() as { service?: Array<{ id: string; serviceEndpoint: string }> };
83 const pds = doc.service?.find((s) => s.id === '#atproto_pds');
84 return pds?.serviceEndpoint ?? null;
85 } catch {
86 return null;
87 }
88}
89
90async function resolveHandle(did: string): Promise<string | null> {
91 try {
92 const res = await fetch(`https://plc.directory/${did}`);
93 if (!res.ok) return null;
94 const doc = await res.json() as { alsoKnownAs?: string[] };
95 const handle = doc.alsoKnownAs?.find((h) => h.startsWith('at://'));
96 return handle?.replace('at://', '') ?? null;
97 } catch {
98 return null;
99 }
100}
101
102async function fetchConnectionRecords(pds: string, did: string): Promise<Array<{ source?: string; target?: string; connectionType?: string }>> {
103 const records: Array<{ source?: string; target?: string; connectionType?: string }> = [];
104 let cursor = '';
105 while (true) {
106 const url = new URL(`${pds}/xrpc/com.atproto.repo.listRecords`);
107 url.searchParams.set('repo', did);
108 url.searchParams.set('collection', COLLECTION);
109 url.searchParams.set('limit', '100');
110 if (cursor) url.searchParams.set('cursor', cursor);
111
112 const res = await fetch(url.toString());
113 if (!res.ok) break;
114 const data = await res.json() as { records: Array<{ value: { source?: string; target?: string; connectionType?: string } }>; cursor?: string };
115 for (const r of data.records) {
116 records.push(r.value);
117 }
118 cursor = data.cursor ?? '';
119 if (!cursor) break;
120 await new Promise((r) => setTimeout(r, 100));
121 }
122 return records;
123}
124
125// --- Our graph ---
126
127function normalizeUrl(url: string): string {
128 try {
129 const u = new URL(url);
130 u.hash = '';
131 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
132 return u.toString();
133 } catch {
134 return url;
135 }
136}
137
138function buildOurUrlSet(): Set<string> {
139 const urls = new Set<string>();
140
141 // From carry connections
142 const connections = carryQueryJson<{ source?: string; target?: string }>('org.latha.connection', ['source', 'target']);
143 const entities = carryQueryJson<{ name?: string; url?: string }>('org.latha.entity', ['name', 'url']);
144 const entityUrls = new Map<string, string>();
145 for (const e of entities) {
146 if (e.name && e.url?.startsWith('https://')) {
147 entityUrls.set(e.name, e.url);
148 }
149 }
150
151 for (const c of connections) {
152 const sUrl = entityUrls.get(c.source ?? '') ?? (c.source?.startsWith('https://') ? c.source : undefined);
153 const tUrl = entityUrls.get(c.target ?? '') ?? (c.target?.startsWith('https://') ? c.target : undefined);
154 if (sUrl) urls.add(normalizeUrl(sUrl));
155 if (tUrl) urls.add(normalizeUrl(tUrl));
156 }
157
158 return urls;
159}
160
161function carryQueryJson<T>(domain: string, fields: string[]): T[] {
162 try {
163 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
164 cwd: CARRY_DIR,
165 encoding: 'utf-8',
166 timeout: 15_000,
167 });
168 return JSON.parse(out) as T[];
169 } catch {
170 return [];
171 }
172}
173
174// --- Similarity ---
175
176function jaccard(a: Set<string>, b: Set<string>): number {
177 if (a.size === 0 && b.size === 0) return 0;
178 let intersection = 0;
179 for (const item of a) {
180 if (b.has(item)) intersection++;
181 }
182 const union = a.size + b.size - intersection;
183 return union === 0 ? 0 : intersection / union;
184}
185
186function sharedUrls(a: Set<string>, b: Set<string>): string[] {
187 const shared: string[] = [];
188 for (const url of a) {
189 if (b.has(url)) shared.push(url);
190 }
191 return shared.sort();
192}
193
194// --- Main ---
195
196async function main(): Promise<void> {
197 const args = process.argv.slice(2);
198 const wantJson = args.includes('--json');
199 const wantWrite = args.includes('--write');
200 const limitArg = args.find((a) => a.startsWith('--limit='));
201 const limit = limitArg ? Number(limitArg.split('=')[1]) : undefined;
202 const thresholdArg = args.find((a) => a.startsWith('--threshold='));
203 const threshold = thresholdArg ? Number(thresholdArg.split('=')[1]) : 0.01;
204
205 console.error('Building our carry graph URL set...');
206 const ourUrls = buildOurUrlSet();
207
208 // Also fetch our own PDS connections
209 const ourDid = await resolveOurDid();
210 if (ourDid) {
211 const pds = await resolvePDS(ourDid);
212 if (pds) {
213 const ourRecords = await fetchConnectionRecords(pds, ourDid);
214 for (const r of ourRecords) {
215 if (r.source?.startsWith('https://')) ourUrls.add(normalizeUrl(r.source));
216 if (r.target?.startsWith('https://')) ourUrls.add(normalizeUrl(r.target));
217 }
218 }
219 }
220
221 console.error(`Our graph: ${ourUrls.size} unique HTTPS URLs`);
222 console.error('Discovering network DIDs...');
223
224 const dids = await discoverDIDs(limit);
225 console.error(`Found ${dids.length} DIDs with ${COLLECTION} records`);
226
227 // Skip our own DID
228 const otherDids = ourDid ? dids.filter((d) => d !== ourDid) : dids;
229
230 const results: SimilarityResult[] = [];
231
232 for (let i = 0; i < otherDids.length; i++) {
233 const did = otherDids[i];
234 console.error(`[${i + 1}/${otherDids.length}] Fetching ${did}...`);
235
236 const pds = await resolvePDS(did);
237 if (!pds) {
238 console.error(` Could not resolve PDS`);
239 continue;
240 }
241
242 const records = await fetchConnectionRecords(pds, did);
243 const theirUrls = new Set<string>();
244 for (const r of records) {
245 if (r.source?.startsWith('https://')) theirUrls.add(normalizeUrl(r.source));
246 if (r.target?.startsWith('https://')) theirUrls.add(normalizeUrl(r.target));
247 }
248
249 const sim = jaccard(ourUrls, theirUrls);
250 const shared = sharedUrls(ourUrls, theirUrls);
251
252 if (sim >= threshold) {
253 const handle = await resolveHandle(did);
254 results.push({
255 did,
256 handle: handle ?? undefined,
257 edgeCount: records.length,
258 sharedCount: shared.length,
259 similarity: sim,
260 sharedUrls: shared,
261 });
262 }
263 }
264
265 results.sort((a, b) => b.similarity - a.similarity);
266
267 if (wantWrite && results.length > 0) {
268 const agent = new AtpAgent({ service: PDS_SERVICE });
269 if (BLUESKY_HANDLE && BLUESKY_APP_PASSWORD) {
270 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
271 }
272 let wrote = 0;
273 for (const r of results.slice(0, 10)) {
274 if (r.similarity < 0.05) continue;
275 const profileUrl = r.handle ? `https://bsky.app/profile/${r.handle}` : `https://bsky.app/profile/${r.did}`;
276 try {
277 await agent.api.com.atproto.repo.createRecord({
278 repo: agent.session!.did,
279 collection: COLLECTION,
280 record: {
281 $type: COLLECTION,
282 source: profileUrl,
283 target: `https://bsky.app/profile/${BLUESKY_HANDLE}`,
284 connectionType: 'SIMILAR_ISLAND',
285 note: `Island similarity: ${(r.similarity * 100).toFixed(1)}% (${r.sharedCount} shared URLs)`,
286 createdAt: new Date().toISOString(),
287 },
288 });
289 wrote++;
290 } catch {
291 // skip
292 }
293 }
294 console.error(`Wrote ${wrote} similarity connection records`);
295 }
296
297 if (wantJson) {
298 console.log(JSON.stringify({ ourUrlCount: ourUrls.size, didCount: otherDids.length, results }, null, 2));
299 return;
300 }
301
302 console.log(`\nConnection detect: compared against ${otherDids.length} network DIDs`);
303 console.log(`Our graph: ${ourUrls.size} unique HTTPS URLs`);
304 console.log(`Matches (similarity >= ${threshold}): ${results.length}\n`);
305
306 for (const r of results.slice(0, 20)) {
307 const label = r.handle ?? r.did;
308 console.log(`${r.similarity.toFixed(4)} | ${label} (${r.edgeCount} edges, ${r.sharedCount} shared)`);
309 for (const url of r.sharedUrls) {
310 console.log(` ${url}`);
311 }
312 }
313}
314
315async function resolveOurDid(): Promise<string | null> {
316 if (!BLUESKY_HANDLE) return null;
317 try {
318 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
319 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
320 return agent.session!.did;
321 } catch {
322 return null;
323 }
324}
325
326main().catch((err) => {
327 console.error('Fatal:', err instanceof Error ? err.message : String(err));
328 process.exit(1);
329});