nandi.uk
This repository has no description
18 kB
382 lines
1/**
2 * island:explore <aturi>
3 *
4 * Local-first island explorer that ingests an org.latha.strata record, identifies
5 * fringe nodes, and emits endofunctor suggestions for re-attaching them to the island.
6 *
7 * Scoring is carry-backed: crawled page text is matched against the carry entity
8 * and citation index, and overlap with island edges determines confidence.
9 */
10
11import { resolve, dirname } from 'node:path';
12import { fileURLToPath } from 'node:url';
13import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
14import { execFileSync } from 'node:child_process';
15import { AtpAgent } from '@atproto/api';
16import { loadDotEnv } from './cli-utils.js';
17import { JSDOM, VirtualConsole } from 'jsdom';
18
19const __dirname = dirname(fileURLToPath(import.meta.url));
20await loadDotEnv(resolve(__dirname, '..', '.env'));
21
22const SEMBLE_API = 'https://api.semble.so';
23const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
24const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
25const ATPROTO_SERVICE = process.env.ATPROTO_SERVICE ?? 'https://bsky.social';
26const PDS_IDENTIFIER = process.env.PDS_IDENTIFIER ?? BLUESKY_HANDLE;
27const PDS_APP_PASSWORD = process.env.PDS_APP_PASSWORD ?? BLUESKY_APP_PASSWORD;
28const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
29const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
30const FEED_LIMIT = Number(process.env.ISLAND_EXPLORE_LIMIT ?? '50');
31const PROVENANCE_PATH = process.env.ISLAND_EXPLORE_PROVENANCE ?? resolve(CARRY_DIR, '.island-explore.json');
32
33interface IslandRecord {
34 $type?: string;
35 source?: { uri?: string; collection?: string };
36 analysis?: { title?: string; themes?: string[]; highlights?: string[]; tensions?: string[]; openQuestions?: string[]; synthesis?: string };
37 edges?: Array<{ source?: string; target?: string; relation?: string; title?: string; description?: string }>;
38}
39
40interface CardContent { url?: string; title?: string; description?: string; author?: string; siteName?: string; type?: string; }
41interface FeedItem {
42 id: string;
43 user: { handle?: string; name?: string };
44 createdAt: string;
45 activityType: 'CARD_COLLECTED' | 'CONNECTION_CREATED';
46 card?: { url?: string; cardContent?: CardContent };
47 connection?: { sourceCard?: { url?: string; title?: string; cardContent?: CardContent }; targetCard?: { url?: string; title?: string; cardContent?: CardContent }; relation?: string };
48}
49interface FeedResponse { activities: FeedItem[]; }
50
51type Suggestion = { source: string; target: string; connectionType: string; confidence: number; evidence: string; fringe: boolean; };
52
53interface CarryEntity { name: string; url: string; description: string }
54interface CarryCitation { url: string; title: string; takeaway: string; domain: string }
55interface CarryIndex {
56 entitiesByName: Map<string, CarryEntity>;
57 entitiesByUrl: Map<string, CarryEntity>;
58 citationsByUrl: Map<string, CarryCitation>;
59 islandEntityUrls: Set<string>;
60}
61
62function normalizeUrl(url: string | undefined): string | null {
63 if (!url) return null;
64 try {
65 const u = new URL(url);
66 u.hash = '';
67 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
68 if (u.protocol !== 'https:') u.protocol = 'https:';
69 return u.toString();
70 } catch {
71 return url.trim().startsWith('https://') ? url.trim() : null;
72 }
73}
74
75function isHttps(url: string | null): url is string {
76 return !!url && url.startsWith('https://');
77}
78
79function carryQueryJson<T>(domain: string, fields: string[]): T[] {
80 try {
81 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
82 cwd: CARRY_DIR,
83 encoding: 'utf-8',
84 timeout: 15_000,
85 });
86 return JSON.parse(out) as T[];
87 } catch {
88 return [];
89 }
90}
91
92function buildCarryIndex(islandTargets: Set<string>): CarryIndex {
93 const entitiesByName = new Map<string, CarryEntity>();
94 const entitiesByUrl = new Map<string, CarryEntity>();
95
96 const rawEntities = carryQueryJson<{ name?: string; url?: string; description?: string }>('org.latha.entity', ['name', 'url', 'description']);
97 for (const e of rawEntities) {
98 if (!e.name) continue;
99 const ent: CarryEntity = { name: e.name, url: e.url ?? '', description: e.description ?? '' };
100 entitiesByName.set(e.name.toLowerCase(), ent);
101 if (e.url) {
102 const norm = normalizeUrl(e.url);
103 if (norm) entitiesByUrl.set(norm, ent);
104 }
105 }
106
107 const citationsByUrl = new Map<string, CarryCitation>();
108 const rawCitations = carryQueryJson<{ url?: string; title?: string; takeaway?: string; domain?: string }>('org.latha.citation', ['url', 'title', 'takeaway', 'domain']);
109 for (const c of rawCitations) {
110 if (!c.url) continue;
111 const norm = normalizeUrl(c.url);
112 if (norm) citationsByUrl.set(norm, { url: norm, title: c.title ?? '', takeaway: c.takeaway ?? '', domain: c.domain ?? '' });
113 }
114
115 const islandEntityUrls = new Set<string>();
116 for (const target of islandTargets) {
117 if (entitiesByUrl.has(target)) islandEntityUrls.add(target);
118 if (citationsByUrl.has(target)) islandEntityUrls.add(target);
119 }
120
121 return { entitiesByName, entitiesByUrl, citationsByUrl, islandEntityUrls };
122}
123
124function extractCarryRefs(text: string, index: CarryIndex): { entities: CarryEntity[]; citations: CarryCitation[]; islandHits: number } {
125 const lower = text.toLowerCase();
126 const foundEntities: CarryEntity[] = [];
127 const foundCitations: CarryCitation[] = [];
128 const seen = new Set<string>();
129 let islandHits = 0;
130
131 for (const [name, entity] of index.entitiesByName) {
132 if (name.length < 4 || seen.has(name)) continue;
133 if (lower.includes(name)) {
134 seen.add(name);
135 foundEntities.push(entity);
136 const norm = normalizeUrl(entity.url);
137 if (norm && index.islandEntityUrls.has(norm)) islandHits++;
138 }
139 }
140
141 const urlPattern = /https?:\/\/[^\s)"'<>]+/gi;
142 const urls = lower.match(urlPattern) ?? [];
143 for (const raw of urls) {
144 const norm = normalizeUrl(raw);
145 if (!norm || seen.has(norm)) continue;
146 seen.add(norm);
147 const citation = index.citationsByUrl.get(norm);
148 if (citation) {
149 foundCitations.push(citation);
150 if (index.islandEntityUrls.has(norm)) islandHits++;
151 }
152 const entity = index.entitiesByUrl.get(norm);
153 if (entity && !foundEntities.includes(entity)) {
154 foundEntities.push(entity);
155 if (index.islandEntityUrls.has(norm)) islandHits++;
156 }
157 }
158
159 return { entities: foundEntities, citations: foundCitations, islandHits };
160}
161
162function scoreCarryOverlap(refs: ReturnType<typeof extractCarryRefs>, islandTargets: Set<string>): { score: number; evidence: string; type: string } {
163 const totalRefs = refs.entities.length + refs.citations.length;
164 if (totalRefs === 0) return { score: 0, evidence: 'no carry refs found', type: 'RELATED' };
165
166 const islandRatio = refs.islandHits / totalRefs;
167 const score = Math.min(0.97, 0.3 + islandRatio * 0.5 + Math.min(totalRefs, 10) * 0.02);
168
169 const entityNames = refs.entities.map((e) => e.name).slice(0, 8);
170 const citationTitles = refs.citations.map((c) => c.title).slice(0, 4);
171 const evidence = [
172 entityNames.length > 0 ? `entities: ${entityNames.join(', ')}` : '',
173 citationTitles.length > 0 ? `citations: ${citationTitles.join(', ')}` : '',
174 `island overlap: ${refs.islandHits}/${totalRefs}`,
175 ].filter(Boolean).join('; ');
176
177 const type = islandRatio > 0.5 ? 'LEADS_TO'
178 : islandRatio > 0.2 ? 'SUPPORTS'
179 : refs.entities.some((e) => indexIsInIsland(e, islandTargets)) ? 'ANNOTATES'
180 : 'RELATED';
181
182 return { score, evidence, type };
183}
184
185function indexIsInIsland(entity: CarryEntity, islandTargets: Set<string>): boolean {
186 const norm = normalizeUrl(entity.url);
187 return !!norm && islandTargets.has(norm);
188}
189
190async function loginToSemble(): Promise<string> {
191 if (!BLUESKY_HANDLE || !BLUESKY_APP_PASSWORD) throw new Error('BLUESKY_IDENTIFIER and BLUESKY_APP_PASSWORD must be set');
192 const res = await fetch(`${SEMBLE_API}/api/users/login/app-password`, {
193 method: 'POST', headers: { 'Content-Type': 'application/json' },
194 body: JSON.stringify({ identifier: BLUESKY_HANDLE, appPassword: BLUESKY_APP_PASSWORD }), redirect: 'manual',
195 });
196 if (res.status !== 200 && res.status !== 302) {
197 const body = await res.text().catch(() => '');
198 throw new Error(`Semble login failed (${res.status}): ${body.slice(0, 200)}`);
199 }
200 const setCookieHeaders = res.headers.getSetCookie?.() ?? [];
201 if (setCookieHeaders.length === 0) {
202 const rawCookie = res.headers.get('set-cookie');
203 if (rawCookie) setCookieHeaders.push(rawCookie);
204 }
205 if (setCookieHeaders.length === 0) throw new Error('Semble login succeeded but no cookies returned');
206 return setCookieHeaders.map((c: string) => c.split(';')[0].trim()).join('; ');
207}
208
209async function fetchFollowingFeed(cookies: string): Promise<FeedItem[]> {
210 const url = new URL(`${SEMBLE_API}/api/feeds/following`);
211 url.searchParams.set('limit', String(FEED_LIMIT));
212 const res = await fetch(url.toString(), { headers: { Cookie: cookies, Accept: 'application/json' } });
213 if (!res.ok) {
214 const body = await res.text().catch(() => '');
215 throw new Error(`Semble feed fetch failed (${res.status}): ${body.slice(0, 200)}`);
216 }
217 const data = (await res.json()) as FeedResponse;
218 return data.activities ?? [];
219}
220
221async function scrapeUrlText(url: string): Promise<string | null> {
222 try {
223 const res = await fetch(url, { signal: AbortSignal.timeout(10_000), headers: { 'User-Agent': 'Mozilla/5.0 (compatible; island-explore/1.0)', Accept: 'text/html' }, redirect: 'follow' });
224 if (!res.ok) return null;
225 const dom = new JSDOM(await res.text(), { url, virtualConsole: new VirtualConsole({ silent: true }) });
226 const doc = dom.window.document;
227 for (const el of doc.querySelectorAll('nav, script, style, footer, header, aside')) el.remove();
228 const root = doc.querySelector('article') ?? doc.querySelector('main') ?? doc.body;
229 const text = root?.textContent?.replace(/\s+/g, ' ').trim() ?? '';
230 return text.length >= 50 ? text.slice(0, 5000) : null;
231 } catch {
232 return null;
233 }
234}
235
236async function fetchIslandRecord(aturi: string): Promise<IslandRecord> {
237 const localPath = resolve(CARRY_DIR, '.islands', `${aturi.replace(/[^a-zA-Z0-9._-]+/g, '_')}.json`);
238 if (existsSync(localPath)) return JSON.parse(readFileSync(localPath, 'utf-8')) as IslandRecord;
239 const agent = new AtpAgent({ service: ATPROTO_SERVICE });
240 await agent.login({ identifier: BLUESKY_HANDLE, password: BLUESKY_APP_PASSWORD });
241 const parts = aturi.split('/');
242 if (parts.length < 5) throw new Error(`Invalid AT URI: ${aturi}`);
243 const collection = parts[3];
244 const rkey = parts[4];
245 const res = await agent.api.com.atproto.repo.getRecord({ repo: parts[2], collection, rkey });
246 return res.data.value as IslandRecord;
247}
248
249function islandEdgeTargets(island: IslandRecord): string[] {
250 return [...new Set((island.edges ?? []).map((e) => normalizeUrl(e.target)).filter(isHttps))];
251}
252
253function carryAssertEdge(source: string, target: string, relation: string, context: string): void {
254 const payload = [`edge:`, ` org.latha.connection:`, ` source: ${JSON.stringify(source)}`, ` target: ${JSON.stringify(target)}`, ` relation: ${JSON.stringify(relation)}`, ` context: ${JSON.stringify(context.slice(0, 1000))}`, ''].join('\n');
255 execFileSync('carry', ['assert', '-'], { cwd: CARRY_DIR, input: payload, encoding: 'utf-8' });
256}
257
258async function createPdsEdge(agent: AtpAgent, source: string, target: string, connectionType: string, note: string): Promise<string | null> {
259 const r = await agent.api.com.atproto.repo.createRecord({ repo: agent.session!.did, collection: 'network.cosmik.connection', record: { $type: 'network.cosmik.connection', source, target, connectionType, note, createdAt: new Date().toISOString() } });
260 return r.data.uri;
261}
262
263async function main(): Promise<void> {
264 const aturi = process.argv[2];
265 if (!aturi) throw new Error('Usage: bun island:explore <aturi> [--json] [--write]');
266 const wantJson = process.argv.includes('--json');
267 const wantWrite = process.argv.includes('--write');
268 const limitArg = process.argv.find((arg) => arg.startsWith('--limit='));
269 const limit = limitArg ? Number(limitArg.split('=')[1]) : FEED_LIMIT;
270
271 const island = await fetchIslandRecord(aturi);
272 const targets = islandEdgeTargets(island);
273 if (targets.length === 0) {
274 throw new Error(`Island record has no canonical https edge URLs in edges (found ${island.edges?.length ?? 0} edges)`);
275 }
276 const islandTargetSet = new Set(targets);
277
278 console.error(`Building carry index...`);
279 const carryIndex = buildCarryIndex(islandTargetSet);
280 console.error(`Carry index: ${carryIndex.entitiesByName.size} entities, ${carryIndex.citationsByUrl.size} citations, ${carryIndex.islandEntityUrls.size} island overlaps`);
281
282 const cookies = await loginToSemble();
283 const feed = (await fetchFollowingFeed(cookies)).slice(0, limit);
284
285 const entityIndex = new Map<string, { url?: string }>();
286 for (const e of carryQueryJson<{ name?: string; url?: string }>('org.latha.entity', ['name', 'url'])) {
287 if (e.url) entityIndex.set(normalizeUrl(e.url) ?? e.url, e);
288 }
289 const carryConnections = new Set<string>();
290
291 const fringeUrls = new Map<string, CardContent>();
292 for (const item of feed) {
293 const url = item.activityType === 'CARD_COLLECTED'
294 ? normalizeUrl(item.card?.cardContent?.url ?? item.card?.url)
295 : normalizeUrl(item.connection?.sourceCard?.url ?? item.connection?.targetCard?.url);
296 const card = item.activityType === 'CARD_COLLECTED' ? item.card?.cardContent : item.connection?.targetCard?.cardContent ?? item.connection?.sourceCard?.cardContent;
297 if (url && card && !entityIndex.has(url)) fringeUrls.set(url, card);
298 }
299
300 const suggestions: Suggestion[] = [];
301 for (const [url, card] of fringeUrls) {
302 const crawled = await scrapeUrlText(url);
303 if (!crawled) continue;
304 const text = `${card.title ?? ''} ${card.description ?? ''} ${crawled}`;
305 const refs = extractCarryRefs(text, carryIndex);
306 if (refs.entities.length + refs.citations.length === 0) continue;
307 const scored = scoreCarryOverlap(refs, islandTargetSet);
308 if (scored.score < 0.3) continue;
309 // Suggest edges to the specific island targets that overlap
310 const overlapTargets = refs.entities
311 .map((e) => normalizeUrl(e.url))
312 .filter((u): u is string => !!u && islandTargetSet.has(u));
313 const citationOverlapTargets = refs.citations
314 .map((c) => c.url)
315 .filter((u) => islandTargetSet.has(u));
316 const allTargets = [...new Set([...overlapTargets, ...citationOverlapTargets])];
317 if (allTargets.length === 0) {
318 // No direct overlap — suggest to the first island target as a weaker connection
319 suggestions.push({ source: url, target: targets[0], connectionType: scored.type, confidence: scored.score * 0.6, evidence: `${scored.evidence} (no direct island overlap)`, fringe: true });
320 } else {
321 for (const target of allTargets) {
322 suggestions.push({ source: url, target, connectionType: scored.type, confidence: scored.score, evidence: scored.evidence, fringe: true });
323 }
324 }
325 }
326
327 const output = {
328 island: { aturi, title: island.analysis?.title ?? island.source?.uri ?? 'Island', targets, carryEntities: carryIndex.entitiesByName.size, carryCitations: carryIndex.citationsByUrl.size, islandOverlaps: carryIndex.islandEntityUrls.size },
329 fringeCount: fringeUrls.size,
330 suggestionCount: suggestions.length,
331 suggestions: suggestions.sort((a, b) => b.confidence - a.confidence),
332 writeEnabled: wantWrite,
333 };
334
335 if (wantWrite) {
336 const pds = new AtpAgent({ service: PDS_SERVICE });
337 if (PDS_IDENTIFIER && PDS_APP_PASSWORD) await pds.login({ identifier: PDS_IDENTIFIER, password: PDS_APP_PASSWORD });
338 let wrote = 0;
339 for (const suggestion of output.suggestions) {
340 if (suggestion.confidence < 0.6) continue;
341 const source = normalizeUrl(suggestion.source);
342 const target = normalizeUrl(suggestion.target);
343 if (!isHttps(source) || !isHttps(target)) continue;
344 const key = `${source}::${target}`;
345 if (carryConnections.has(key)) continue;
346 carryAssertEdge(source, target, suggestion.connectionType, suggestion.evidence);
347 carryConnections.add(key);
348 const uri = await createPdsEdge(pds, source, target, suggestion.connectionType, suggestion.evidence);
349 if (uri) {
350 wrote += 1;
351 console.log(`Wrote edge: ${uri}`);
352 }
353 }
354 saveJson(PROVENANCE_PATH, { aturi, wrote, wroteAt: new Date().toISOString(), suggestions: output.suggestions });
355 output.writeEnabled = true;
356 }
357
358 if (wantJson) {
359 console.log(JSON.stringify(output, null, 2));
360 return;
361 }
362
363 console.log(`Island: ${output.island.title} (${aturi})`);
364 console.log(`Targets: ${output.island.targets.length} edge URLs`);
365 console.log(`Carry: ${output.island.carryEntities} entities, ${output.island.carryCitations} citations, ${output.island.islandOverlaps} island overlaps`);
366 console.log(`Fringe nodes: ${output.fringeCount}`);
367 console.log(`Suggestions: ${output.suggestionCount}`);
368 for (const s of output.suggestions.slice(0, 20)) {
369 console.log(`- ${s.connectionType} ${s.source} -> ${s.target} (${s.confidence.toFixed(2)})`);
370 console.log(` ${s.evidence}`);
371 }
372}
373
374function saveJson(path: string, value: unknown): void {
375 mkdirSync(dirname(path), { recursive: true });
376 writeFileSync(path, JSON.stringify(value, null, 2) + '\n', 'utf-8');
377}
378
379main().catch((err) => {
380 console.error('Fatal:', err instanceof Error ? err.message : String(err));
381 process.exit(1);
382});