nandi.uk
This repository has no description
20 kB
443 lines
1/**
2 * island:explore <island-id>
3 *
4 * Local-first island explorer that ingests an org.latha.island record, identifies
5 * fringe nodes, and emits endofunctor suggestions for re-attaching them to the island.
6 *
7 * Accepts either an island hash (12 hex chars, e.g. a1b2c3d4e5f6) or an AT URI.
8 * When given an island hash, resolves it to the component's lexmin and looks up
9 * the published record on the PDS.
10 *
11 * Scoring is carry-backed: crawled page text is matched against the carry entity
12 * and citation index, and overlap with island edges determines confidence.
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
18import { execFileSync } from 'node:child_process';
19import { AtpAgent } from '@atproto/api';
20import { loadDotEnv } from './cli-utils.js';
21import { findComponentById, islandId } from './island-shared.js';
22import { Database } from 'bun:sqlite';
23import { JSDOM, VirtualConsole } from 'jsdom';
24
25const __dirname = dirname(fileURLToPath(import.meta.url));
26await loadDotEnv(resolve(__dirname, '..', '.env'));
27
28const SEMBLE_API = 'https://api.semble.so';
29const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
30const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
31const ATPROTO_SERVICE = process.env.ATPROTO_SERVICE ?? 'https://bsky.social';
32const PDS_IDENTIFIER = process.env.PDS_IDENTIFIER ?? BLUESKY_HANDLE;
33const PDS_APP_PASSWORD = process.env.PDS_APP_PASSWORD ?? BLUESKY_APP_PASSWORD;
34const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
35const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
36const FEED_LIMIT = Number(process.env.ISLAND_EXPLORE_LIMIT ?? '50');
37const PROVENANCE_PATH = process.env.ISLAND_EXPLORE_PROVENANCE ?? resolve(CARRY_DIR, '.island-explore.json');
38
39interface IslandRecord {
40 $type?: string;
41 source?: { uri?: string; collection?: string };
42 analysis?: { title?: string; themes?: string[]; highlights?: string[]; tensions?: string[]; openQuestions?: string[]; synthesis?: string };
43 edges?: Array<{ source?: string; target?: string; relation?: string; title?: string; description?: string }>;
44 connections?: Array<{ source?: string; target?: string; connectionType?: string; note?: string; uri?: string }>;
45}
46
47interface CardContent { url?: string; title?: string; description?: string; author?: string; siteName?: string; type?: string; }
48interface FeedItem {
49 id: string;
50 user: { handle?: string; name?: string };
51 createdAt: string;
52 activityType: 'CARD_COLLECTED' | 'CONNECTION_CREATED';
53 card?: { url?: string; cardContent?: CardContent };
54 connection?: { sourceCard?: { url?: string; title?: string; cardContent?: CardContent }; targetCard?: { url?: string; title?: string; cardContent?: CardContent }; relation?: string };
55}
56interface FeedResponse { activities: FeedItem[]; }
57
58type Suggestion = { source: string; target: string; connectionType: string; confidence: number; evidence: string; fringe: boolean; };
59
60interface CarryEntity { name: string; url: string; description: string }
61interface CarryCitation { url: string; title: string; takeaway: string; domain: string }
62interface CarryIndex {
63 entitiesByName: Map<string, CarryEntity>;
64 entitiesByUrl: Map<string, CarryEntity>;
65 citationsByUrl: Map<string, CarryCitation>;
66 islandEntityUrls: Set<string>;
67}
68
69function normalizeUrl(url: string | undefined): string | null {
70 if (!url) return null;
71 try {
72 const u = new URL(url);
73 u.hash = '';
74 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
75 if (u.protocol !== 'https:') u.protocol = 'https:';
76 return u.toString();
77 } catch {
78 return url.trim().startsWith('https://') ? url.trim() : null;
79 }
80}
81
82function isHttps(url: string | null): url is string {
83 return !!url && url.startsWith('https://');
84}
85
86function carryQueryJson<T>(domain: string, fields: string[]): T[] {
87 try {
88 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
89 cwd: CARRY_DIR,
90 encoding: 'utf-8',
91 timeout: 15_000,
92 });
93 return JSON.parse(out) as T[];
94 } catch {
95 return [];
96 }
97}
98
99function buildCarryIndex(islandTargets: Set<string>): CarryIndex {
100 const entitiesByName = new Map<string, CarryEntity>();
101 const entitiesByUrl = new Map<string, CarryEntity>();
102
103 const rawEntities = carryQueryJson<{ name?: string; url?: string; description?: string }>('org.latha.entity', ['name', 'url', 'description']);
104 for (const e of rawEntities) {
105 if (!e.name) continue;
106 const ent: CarryEntity = { name: e.name, url: e.url ?? '', description: e.description ?? '' };
107 entitiesByName.set(e.name.toLowerCase(), ent);
108 if (e.url) {
109 const norm = normalizeUrl(e.url);
110 if (norm) entitiesByUrl.set(norm, ent);
111 }
112 }
113
114 const citationsByUrl = new Map<string, CarryCitation>();
115 const rawCitations = carryQueryJson<{ url?: string; title?: string; takeaway?: string; domain?: string }>('org.latha.citation', ['url', 'title', 'takeaway', 'domain']);
116 for (const c of rawCitations) {
117 if (!c.url) continue;
118 const norm = normalizeUrl(c.url);
119 if (norm) citationsByUrl.set(norm, { url: norm, title: c.title ?? '', takeaway: c.takeaway ?? '', domain: c.domain ?? '' });
120 }
121
122 const islandEntityUrls = new Set<string>();
123 for (const target of islandTargets) {
124 if (entitiesByUrl.has(target)) islandEntityUrls.add(target);
125 if (citationsByUrl.has(target)) islandEntityUrls.add(target);
126 }
127
128 return { entitiesByName, entitiesByUrl, citationsByUrl, islandEntityUrls };
129}
130
131function extractCarryRefs(text: string, index: CarryIndex): { entities: CarryEntity[]; citations: CarryCitation[]; islandHits: number } {
132 const lower = text.toLowerCase();
133 const foundEntities: CarryEntity[] = [];
134 const foundCitations: CarryCitation[] = [];
135 const seen = new Set<string>();
136 let islandHits = 0;
137
138 for (const [name, entity] of index.entitiesByName) {
139 if (name.length < 4 || seen.has(name)) continue;
140 if (lower.includes(name)) {
141 seen.add(name);
142 foundEntities.push(entity);
143 const norm = normalizeUrl(entity.url);
144 if (norm && index.islandEntityUrls.has(norm)) islandHits++;
145 }
146 }
147
148 const urlPattern = /https?:\/\/[^\s)"'<>]+/gi;
149 const urls = lower.match(urlPattern) ?? [];
150 for (const raw of urls) {
151 const norm = normalizeUrl(raw);
152 if (!norm || seen.has(norm)) continue;
153 seen.add(norm);
154 const citation = index.citationsByUrl.get(norm);
155 if (citation) {
156 foundCitations.push(citation);
157 if (index.islandEntityUrls.has(norm)) islandHits++;
158 }
159 const entity = index.entitiesByUrl.get(norm);
160 if (entity && !foundEntities.includes(entity)) {
161 foundEntities.push(entity);
162 if (index.islandEntityUrls.has(norm)) islandHits++;
163 }
164 }
165
166 return { entities: foundEntities, citations: foundCitations, islandHits };
167}
168
169function scoreCarryOverlap(refs: ReturnType<typeof extractCarryRefs>, islandTargets: Set<string>): { score: number; evidence: string; type: string } {
170 const totalRefs = refs.entities.length + refs.citations.length;
171 if (totalRefs === 0) return { score: 0, evidence: 'no carry refs found', type: 'RELATED' };
172
173 const islandRatio = refs.islandHits / totalRefs;
174 const score = Math.min(0.97, 0.3 + islandRatio * 0.5 + Math.min(totalRefs, 10) * 0.02);
175
176 const entityNames = refs.entities.map((e) => e.name).slice(0, 8);
177 const citationTitles = refs.citations.map((c) => c.title).slice(0, 4);
178 const evidence = [
179 entityNames.length > 0 ? `entities: ${entityNames.join(', ')}` : '',
180 citationTitles.length > 0 ? `citations: ${citationTitles.join(', ')}` : '',
181 `island overlap: ${refs.islandHits}/${totalRefs}`,
182 ].filter(Boolean).join('; ');
183
184 const type = islandRatio > 0.5 ? 'LEADS_TO'
185 : islandRatio > 0.2 ? 'SUPPORTS'
186 : refs.entities.some((e) => indexIsInIsland(e, islandTargets)) ? 'ANNOTATES'
187 : 'RELATED';
188
189 return { score, evidence, type };
190}
191
192function indexIsInIsland(entity: CarryEntity, islandTargets: Set<string>): boolean {
193 const norm = normalizeUrl(entity.url);
194 return !!norm && islandTargets.has(norm);
195}
196
197async function loginToSemble(): Promise<string> {
198 if (!BLUESKY_HANDLE || !BLUESKY_APP_PASSWORD) throw new Error('BLUESKY_IDENTIFIER and BLUESKY_APP_PASSWORD must be set');
199 const res = await fetch(`${SEMBLE_API}/api/users/login/app-password`, {
200 method: 'POST', headers: { 'Content-Type': 'application/json' },
201 body: JSON.stringify({ identifier: BLUESKY_HANDLE, appPassword: BLUESKY_APP_PASSWORD }), redirect: 'manual',
202 });
203 if (res.status !== 200 && res.status !== 302) {
204 const body = await res.text().catch(() => '');
205 throw new Error(`Semble login failed (${res.status}): ${body.slice(0, 200)}`);
206 }
207 const setCookieHeaders = res.headers.getSetCookie?.() ?? [];
208 if (setCookieHeaders.length === 0) {
209 const rawCookie = res.headers.get('set-cookie');
210 if (rawCookie) setCookieHeaders.push(rawCookie);
211 }
212 if (setCookieHeaders.length === 0) throw new Error('Semble login succeeded but no cookies returned');
213 return setCookieHeaders.map((c: string) => c.split(';')[0].trim()).join('; ');
214}
215
216async function fetchFollowingFeed(cookies: string): Promise<FeedItem[]> {
217 const url = new URL(`${SEMBLE_API}/api/feeds/following`);
218 url.searchParams.set('limit', String(FEED_LIMIT));
219 const res = await fetch(url.toString(), { headers: { Cookie: cookies, Accept: 'application/json' } });
220 if (!res.ok) {
221 const body = await res.text().catch(() => '');
222 throw new Error(`Semble feed fetch failed (${res.status}): ${body.slice(0, 200)}`);
223 }
224 const data = (await res.json()) as FeedResponse;
225 return data.activities ?? [];
226}
227
228async function scrapeUrlText(url: string): Promise<string | null> {
229 try {
230 const res = await fetch(url, { signal: AbortSignal.timeout(10_000), headers: { 'User-Agent': 'Mozilla/5.0 (compatible; island-explore/1.0)', Accept: 'text/html' }, redirect: 'follow' });
231 if (!res.ok) return null;
232 const dom = new JSDOM(await res.text(), { url, virtualConsole: new VirtualConsole({ silent: true }) });
233 const doc = dom.window.document;
234 for (const el of doc.querySelectorAll('nav, script, style, footer, header, aside')) el.remove();
235 const root = doc.querySelector('article') ?? doc.querySelector('main') ?? doc.body;
236 const text = root?.textContent?.replace(/\s+/g, ' ').trim() ?? '';
237 return text.length >= 50 ? text.slice(0, 5000) : null;
238 } catch {
239 return null;
240 }
241}
242
243async function fetchIslandRecord(aturi: string): Promise<IslandRecord> {
244 const localPath = resolve(CARRY_DIR, '.islands', `${aturi.replace(/[^a-zA-Z0-9._-]+/g, '_')}.json`);
245 if (existsSync(localPath)) return JSON.parse(readFileSync(localPath, 'utf-8')) as IslandRecord;
246 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
247 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
248 const parts = aturi.split('/');
249 if (parts.length < 5) throw new Error(`Invalid AT URI: ${aturi}`);
250 const collection = parts[3];
251 const rkey = parts[4];
252 const res = await agent.api.com.atproto.repo.getRecord({ repo: parts[2], collection, rkey });
253 return res.data.value as IslandRecord;
254}
255
256function islandEdgeTargets(island: IslandRecord): string[] {
257 // Support both `edges` (legacy) and `connections` (lexicon) field names
258 const allEdges = [
259 ...(island.edges ?? []),
260 ...(island.connections ?? []).map((c) => ({ source: c.source, target: c.target, relation: c.connectionType })),
261 ];
262 return [...new Set(allEdges.map((e) => normalizeUrl(e.target)).filter(isHttps))];
263}
264
265function carryAssertEdge(source: string, target: string, relation: string, context: string): void {
266 const payload = [`edge:`, ` org.latha.connection:`, ` source: ${JSON.stringify(source)}`, ` target: ${JSON.stringify(target)}`, ` relation: ${JSON.stringify(relation)}`, ` context: ${JSON.stringify(context.slice(0, 1000))}`, ''].join('\n');
267 execFileSync('carry', ['assert', '-'], { cwd: CARRY_DIR, input: payload, encoding: 'utf-8' });
268}
269
270async function createPdsEdge(agent: AtpAgent, source: string, target: string, connectionType: string, note: string): Promise<string | null> {
271 const r = await agent.api.com.atproto.repo.createRecord({ repo: agent.session!.did, collection: 'network.cosmik.connection', record: { $type: 'network.cosmik.connection', source, target, connectionType, note, createdAt: new Date().toISOString() } });
272 return r.data.uri;
273}
274
275async function main(): Promise<void> {
276 const input = process.argv[2];
277 if (!input) throw new Error('Usage: bun island:explore <island-id|aturi> [--json] [--write]');
278 const wantJson = process.argv.includes('--json');
279 const wantWrite = process.argv.includes('--write');
280 const limitArg = process.argv.find((arg) => arg.startsWith('--limit='));
281 const limit = limitArg ? Number(limitArg.split('=')[1]) : FEED_LIMIT;
282
283 // Resolve island hash to AT URI
284 let aturi: string;
285 const ISLAND_HASH_RE = /^[0-9a-f]{12}$/;
286 if (ISLAND_HASH_RE.test(input)) {
287 // Island hash — resolve to component lexmin, then find published record
288 const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
289 let db: Database;
290 try {
291 db = new Database(DB_PATH, { readonly: true });
292 } catch {
293 throw new Error('Database not found. Run `bun island:discover` first.');
294 }
295 const component = findComponentById(db, input);
296 db.close();
297 if (!component) throw new Error(`Island ${input} not found in discovery DB. Run \`bun island:discover\` first.`);
298 console.error(`Resolved island ${input} → lexmin ${component.lexmin}`);
299
300 // Look for a published org.latha.island record on our PDS
301 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
302 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
303 const did = agent.session!.did;
304 let cursor = '';
305 let foundUri: string | null = null;
306 while (true) {
307 const url = new URL(`${PDS_SERVICE}/xrpc/com.atproto.repo.listRecords`);
308 url.searchParams.set('repo', did);
309 url.searchParams.set('collection', 'org.latha.island');
310 url.searchParams.set('limit', '100');
311 if (cursor) url.searchParams.set('cursor', cursor);
312 const res = await fetch(url.toString());
313 if (!res.ok) break;
314 const data = await res.json() as { records: Array<{ uri: string; value: { source?: { uri?: string } } }>; cursor?: string };
315 for (const r of data.records) {
316 if (r.value.source?.uri === component.lexmin) {
317 foundUri = r.uri;
318 break;
319 }
320 }
321 if (foundUri) break;
322 cursor = data.cursor ?? '';
323 if (!cursor) break;
324 }
325 if (!foundUri) throw new Error(`No published org.latha.island record found for lexmin ${component.lexmin}. Run \`bun island:publish ${input}\` first.`);
326 aturi = foundUri;
327 console.error(`Found record: ${aturi}`);
328 } else {
329 aturi = input;
330 }
331
332 const island = await fetchIslandRecord(aturi);
333 const targets = islandEdgeTargets(island);
334 if (targets.length === 0) {
335 throw new Error(`Island record has no canonical https edge URLs in edges (found ${island.edges?.length ?? 0} edges)`);
336 }
337 const islandTargetSet = new Set(targets);
338
339 console.error(`Building carry index...`);
340 const carryIndex = buildCarryIndex(islandTargetSet);
341 console.error(`Carry index: ${carryIndex.entitiesByName.size} entities, ${carryIndex.citationsByUrl.size} citations, ${carryIndex.islandEntityUrls.size} island overlaps`);
342
343 const cookies = await loginToSemble();
344 const feed = (await fetchFollowingFeed(cookies)).slice(0, limit);
345
346 const entityIndex = new Map<string, { url?: string }>();
347 for (const e of carryQueryJson<{ name?: string; url?: string }>('org.latha.entity', ['name', 'url'])) {
348 if (e.url) entityIndex.set(normalizeUrl(e.url) ?? e.url, e);
349 }
350 const carryConnections = new Set<string>();
351
352 const fringeUrls = new Map<string, CardContent>();
353 for (const item of feed) {
354 const url = item.activityType === 'CARD_COLLECTED'
355 ? normalizeUrl(item.card?.cardContent?.url ?? item.card?.url)
356 : normalizeUrl(item.connection?.sourceCard?.url ?? item.connection?.targetCard?.url);
357 const card = item.activityType === 'CARD_COLLECTED' ? item.card?.cardContent : item.connection?.targetCard?.cardContent ?? item.connection?.sourceCard?.cardContent;
358 if (url && card && !entityIndex.has(url)) fringeUrls.set(url, card);
359 }
360
361 const suggestions: Suggestion[] = [];
362 for (const [url, card] of fringeUrls) {
363 const crawled = await scrapeUrlText(url);
364 if (!crawled) continue;
365 const text = `${card.title ?? ''} ${card.description ?? ''} ${crawled}`;
366 const refs = extractCarryRefs(text, carryIndex);
367 if (refs.entities.length + refs.citations.length === 0) continue;
368 const scored = scoreCarryOverlap(refs, islandTargetSet);
369 if (scored.score < 0.3) continue;
370 // Suggest edges to the specific island targets that overlap
371 const overlapTargets = refs.entities
372 .map((e) => normalizeUrl(e.url))
373 .filter((u): u is string => !!u && islandTargetSet.has(u));
374 const citationOverlapTargets = refs.citations
375 .map((c) => c.url)
376 .filter((u) => islandTargetSet.has(u));
377 const allTargets = [...new Set([...overlapTargets, ...citationOverlapTargets])];
378 if (allTargets.length === 0) {
379 // No direct overlap — suggest to the first island target as a weaker connection
380 suggestions.push({ source: url, target: targets[0], connectionType: scored.type, confidence: scored.score * 0.6, evidence: `${scored.evidence} (no direct island overlap)`, fringe: true });
381 } else {
382 for (const target of allTargets) {
383 suggestions.push({ source: url, target, connectionType: scored.type, confidence: scored.score, evidence: scored.evidence, fringe: true });
384 }
385 }
386 }
387
388 const output = {
389 island: { aturi, title: island.analysis?.title ?? island.source?.uri ?? 'Island', targets, carryEntities: carryIndex.entitiesByName.size, carryCitations: carryIndex.citationsByUrl.size, islandOverlaps: carryIndex.islandEntityUrls.size },
390 fringeCount: fringeUrls.size,
391 suggestionCount: suggestions.length,
392 suggestions: suggestions.sort((a, b) => b.confidence - a.confidence),
393 writeEnabled: wantWrite,
394 };
395
396 if (wantWrite) {
397 const pds = new AtpAgent({ service: PDS_SERVICE });
398 if (PDS_IDENTIFIER && PDS_APP_PASSWORD) await pds.login({ identifier: PDS_IDENTIFIER, password: PDS_APP_PASSWORD });
399 let wrote = 0;
400 for (const suggestion of output.suggestions) {
401 if (suggestion.confidence < 0.6) continue;
402 const source = normalizeUrl(suggestion.source);
403 const target = normalizeUrl(suggestion.target);
404 if (!isHttps(source) || !isHttps(target)) continue;
405 const key = `${source}::${target}`;
406 if (carryConnections.has(key)) continue;
407 carryAssertEdge(source, target, suggestion.connectionType, suggestion.evidence);
408 carryConnections.add(key);
409 const uri = await createPdsEdge(pds, source, target, suggestion.connectionType, suggestion.evidence);
410 if (uri) {
411 wrote += 1;
412 console.log(`Wrote edge: ${uri}`);
413 }
414 }
415 saveJson(PROVENANCE_PATH, { aturi, wrote, wroteAt: new Date().toISOString(), suggestions: output.suggestions });
416 output.writeEnabled = true;
417 }
418
419 if (wantJson) {
420 console.log(JSON.stringify(output, null, 2));
421 return;
422 }
423
424 console.log(`Island: ${output.island.title} (${aturi})`);
425 console.log(`Targets: ${output.island.targets.length} edge URLs`);
426 console.log(`Carry: ${output.island.carryEntities} entities, ${output.island.carryCitations} citations, ${output.island.islandOverlaps} island overlaps`);
427 console.log(`Fringe nodes: ${output.fringeCount}`);
428 console.log(`Suggestions: ${output.suggestionCount}`);
429 for (const s of output.suggestions.slice(0, 20)) {
430 console.log(`- ${s.connectionType} ${s.source} -> ${s.target} (${s.confidence.toFixed(2)})`);
431 console.log(` ${s.evidence}`);
432 }
433}
434
435function saveJson(path: string, value: unknown): void {
436 mkdirSync(dirname(path), { recursive: true });
437 writeFileSync(path, JSON.stringify(value, null, 2) + '\n', 'utf-8');
438}
439
440main().catch((err) => {
441 console.error('Fatal:', err instanceof Error ? err.message : String(err));
442 process.exit(1);
443});