nandi.uk
This repository has no description
21 kB
522 lines
1/**
2 * island:analyze <island-id>
3 *
4 * Analyze a discovered component using an LLM agent (letta-code-sdk).
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze a1b2c3d4e5f6 # analyze by island hash
9 * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { Database } from 'bun:sqlite';
18import { AtpAgent } from '@atproto/api';
19import {
20 createAgent,
21 createSession,
22 type AnyAgentTool,
23 type AgentToolResult,
24 type SDKResultMessage,
25} from '@letta-ai/letta-code-sdk';
26import { loadDotEnv } from './cli-utils.js';
27import { findComponents, domainFromUrl, type Component, type EdgeRow, fetchComponentEdges, resolveDidHandles } from './island-shared.js';
28
29const __dirname = dirname(fileURLToPath(import.meta.url));
30await loadDotEnv(resolve(__dirname, '..', '.env'));
31await loadDotEnv('/home/nandi/code/swarm/.env'); // fallback for shared credentials
32
33const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
34const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
35const BLUESKY_HANDLE = process.env.BLUESKY_IDENTIFIER ?? '';
36const BLUESKY_APP_PASSWORD = process.env.BLUESKY_APP_PASSWORD ?? '';
37const PDS_IDENTIFIER = process.env.PDS_IDENTIFIER ?? BLUESKY_HANDLE;
38const PDS_APP_PASSWORD = process.env.PDS_APP_PASSWORD ?? BLUESKY_APP_PASSWORD;
39const PDS_SERVICE = process.env.PDS_SERVICE ?? 'https://amanita.us-east.host.bsky.network';
40
41// --- Types ---
42
43interface AnalysisResult {
44 islandId: string;
45 centroid: string;
46 title: string;
47 themes: string[];
48 highlights: string[];
49 tensions: string[];
50 openQuestions: string[];
51 synthesis: string;
52 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
53 analyzedAt: string;
54}
55
56// --- SQLite ---
57
58function openDb(): Database {
59 const db = new Database(DB_PATH, { create: true });
60 db.exec('PRAGMA journal_mode = WAL');
61 db.exec('PRAGMA synchronous = NORMAL');
62 db.exec(`
63 CREATE TABLE IF NOT EXISTS analyses (
64 island_id TEXT PRIMARY KEY,
65 title TEXT,
66 themes TEXT,
67 highlights TEXT,
68 tensions TEXT,
69 open_questions TEXT,
70 synthesis TEXT,
71 new_connections TEXT,
72 analyzed_at TEXT NOT NULL
73 );
74 CREATE TABLE IF NOT EXISTS config (
75 key TEXT PRIMARY KEY,
76 value TEXT NOT NULL
77 );
78 `);
79 return db;
80}
81
82function saveAnalysis(db: Database, result: AnalysisResult): void {
83 db.query(`
84 INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
85 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
86 ON CONFLICT(island_id) DO UPDATE SET
87 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
88 tensions=excluded.tensions, open_questions=excluded.open_questions,
89 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
90 `).run(
91 result.islandId,
92 result.title,
93 JSON.stringify(result.themes),
94 JSON.stringify(result.highlights),
95 JSON.stringify(result.tensions),
96 JSON.stringify(result.openQuestions),
97 result.synthesis,
98 JSON.stringify(result.newConnections),
99 result.analyzedAt,
100 );
101}
102
103function loadAnalysis(db: Database, islandId: string): AnalysisResult | null {
104 const row = db.query('SELECT * FROM analyses WHERE island_id = ?').get(islandId) as Record<string, string> | null;
105 if (!row) return null;
106 return {
107 islandId: row.island_id,
108 centroid: '',
109 title: row.title ?? '',
110 themes: JSON.parse(row.themes ?? '[]'),
111 highlights: JSON.parse(row.highlights ?? '[]'),
112 tensions: JSON.parse(row.tensions ?? '[]'),
113 openQuestions: JSON.parse(row.open_questions ?? '[]'),
114 synthesis: row.synthesis ?? '',
115 newConnections: JSON.parse(row.new_connections ?? '[]'),
116 analyzedAt: row.analyzed_at,
117 };
118}
119
120function buildIslandRecord(db: Database, component: Component, result: AnalysisResult): Record<string, unknown> {
121 const edges = fetchComponentEdges(db, component);
122 const connections = edges
123 .filter((e) => e.atUri)
124 .map((e) => ({ uri: e.atUri }));
125
126 return {
127 $type: 'org.latha.island',
128 source: {
129 uri: component.centroid,
130 collection: 'network.cosmik.connection',
131 },
132 connections,
133 analysis: {
134 title: result.title,
135 themes: result.themes,
136 highlights: result.highlights,
137 tensions: result.tensions,
138 openQuestions: result.openQuestions,
139 synthesis: result.synthesis,
140 },
141 createdAt: result.analyzedAt,
142 updatedAt: new Date().toISOString(),
143 };
144}
145
146async function publishIslandAnalysis(db: Database, component: Component, result: AnalysisResult): Promise<string> {
147 if (!PDS_IDENTIFIER || !PDS_APP_PASSWORD) {
148 throw new Error('PDS_IDENTIFIER and PDS_APP_PASSWORD (or BLUESKY_IDENTIFIER/BLUESKY_APP_PASSWORD) must be set');
149 }
150
151 const record = buildIslandRecord(db, component, result);
152 const agent = new AtpAgent({ service: PDS_SERVICE });
153 await agent.login({ identifier: PDS_IDENTIFIER, password: PDS_APP_PASSWORD });
154
155 const published = await agent.api.com.atproto.repo.putRecord({
156 repo: agent.session!.did,
157 collection: 'org.latha.island',
158 rkey: component.islandId,
159 record,
160 });
161
162 const appview = process.env.STIGMERGIC_APPVIEW ?? 'https://stigmergic.latha.org';
163 try {
164 const resp = await fetch(`${appview}/xrpc/org.latha.island.notifyOfUpdate`, {
165 method: 'POST',
166 headers: { 'Content-Type': 'application/json' },
167 body: JSON.stringify({ did: agent.session!.did }),
168 });
169 if (!resp.ok) {
170 console.error(` Appview notify failed: ${resp.status} (non-fatal, cron will catch up)`);
171 }
172 } catch (e: any) {
173 console.error(` Appview notify error: ${e.message} (non-fatal, cron will catch up)`);
174 }
175
176 return published.data.uri;
177}
178
179// --- Build island context from SQLite data ---
180
181function buildIslandContext(db: Database, comp: Component): string {
182 const edges = fetchComponentEdges(db, comp);
183 const didHandles = resolveDidHandles(db, comp.dids);
184
185 // Domain distribution
186 const domainCounts = new Map<string, number>();
187 for (const url of comp.nodes) {
188 const d = domainFromUrl(url);
189 domainCounts.set(d, (domainCounts.get(d) ?? 0) + 1);
190 }
191 const topDomains = [...domainCounts.entries()]
192 .sort((a, b) => b[1] - a[1])
193 .slice(0, 15)
194 .map(([d, c]) => `${d} (${c})`)
195 .join(', ');
196
197 // Edge type distribution
198 const edgeTypes = new Map<string, number>();
199 for (const e of edges) {
200 const t = e.relation || 'relates';
201 edgeTypes.set(t, (edgeTypes.get(t) ?? 0) + 1);
202 }
203 const edgeTypeStr = [...edgeTypes.entries()]
204 .sort((a, b) => b[1] - a[1])
205 .map(([t, c]) => `${t} (${c})`)
206 .join(', ');
207
208 // Contributors
209 const contributorStr = [...comp.dids]
210 .map((d) => didHandles.get(d) ?? d)
211 .join(', ');
212
213 // Vertices (grouped by domain for readability)
214 const verticesByDomain = new Map<string, string[]>();
215 for (const url of comp.nodes) {
216 const d = domainFromUrl(url);
217 if (!verticesByDomain.has(d)) verticesByDomain.set(d, []);
218 verticesByDomain.get(d)!.push(url);
219 }
220 const vertexSections = [...verticesByDomain.entries()]
221 .sort((a, b) => b[1].length - a[1].length)
222 .slice(0, 10)
223 .map(([domain, urls]) => `${domain}:\n${urls.slice(0, 5).map((u) => ` ${u}`).join('\n')}${urls.length > 5 ? `\n ... and ${urls.length - 5} more` : ''}`)
224 .join('\n\n');
225
226 // Edges (sample)
227 const edgeSample = edges.slice(0, 40)
228 .map((e) => {
229 const handle = didHandles.get(e.did) ?? e.did;
230 return `${e.atUri ?? ''}: ${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}] (by ${handle})`;
231 })
232 .join('\n');
233
234 return [
235 `# Island ${comp.islandId}`,
236 '',
237 `## Stats`,
238 `- ${comp.nodes.size} vertices, ${edges.length} edges, ${comp.dids.size} contributors`,
239 `- Top domains: ${topDomains}`,
240 `- Edge types: ${edgeTypeStr}`,
241 `- Contributors: ${contributorStr}`,
242 '',
243 `## Vertices`,
244 vertexSections,
245 '',
246 `## Edges (sample)`,
247 edgeSample,
248 edges.length > 40 ? `\n... and ${edges.length - 40} more edges` : '',
249 ].join('\n');
250}
251
252// --- island_save tool ---
253
254let pendingResult: AnalysisResult | null = null;
255
256const islandSaveTool: AnyAgentTool = {
257 label: 'island_save',
258 name: 'island_save',
259 description: `Save your analysis of this island. Call this tool once when you have completed your analysis. This is the ONLY way to save your work — you MUST call this tool.`,
260 parameters: {
261 type: 'object',
262 properties: {
263 title: { type: 'string', description: 'Concise sentence capturing the core narrative' },
264 themes: { type: 'string', description: 'Comma-separated key themes' },
265 highlights: { type: 'string', description: 'Comma-separated AT-URIs of semantically significant EDGE records only. Use exact at://.../network.cosmik.connection/... URIs from the Edges section. Do not output domains, vertex URLs, project names, or descriptions.' },
266 tensions: { type: 'string', description: 'Comma-separated structural tensions. Max 3. Before writing a tension, ask: does this already fit under one I wrote? If so, fold it in as a parenthetical example, don\'t add a new one. The pattern "AI does X vs. people resist X" is ONE tension, not three. BAD: "AI industry builds infrastructure vs communities resist", "AI companies promise benefit vs communities bear costs", "AI enables content at scale vs quality unravels". GOOD: "AI centralizes benefits while externalizing costs — datacenters displace communities, content pollution degrades shared information, and promised benefits accrue to companies while communities absorb the damage"' },
267 openQuestions: { type: 'string', description: 'Semicolon-separated questions raised by this island. IMPORTANT: each item must be one complete question ending in ?; do not use commas as separators because questions often contain commas.' },
268 synthesis: { type: 'string', description: `Structured synthesis in markdown. FORMAT RULES:
2691. Start with a 1-2 sentence thesis, then use ### subheadings for each major thread
2702. Under each subheading: 2-3 short paragraphs (max 3 sentences each)
2713. EVERY URL from the island must appear as a [short label](url) link — never bare URLs
2724. **Bold** every proper noun, tool name, and key term on first mention
2735. Use bullet lists for enumerations (3+ items), not prose
2746. Close with a "## What binds this island" section: 1-2 sentences on the core tension
275BAD: "This island maps the sprawling backlash against AI's material and informational consequences. The dominant cluster 404media.co's 68 URLs documents communities fighting..."
276GOOD: "This island traces two parallel crises: **AI's physical footprint** and its **information pollution**.\n\n### Datacenter resistance\n[404 Media](https://404media.co) dominates with 68 URLs documenting communities from [Amarillo](https://www.404media.co/project-matador-datacenter-amarillo-texas/) to [Maine](https://www.404media.co/maine-datacenter-construction-bill-ld-307/) fighting datacenter construction..."` },
277 newConnections: { type: 'string', description: 'Semicolon-separated new edges as: source|target|type|note' },
278 },
279 required: ['title', 'themes', 'synthesis'],
280 },
281 execute: async (_id, args): Promise<AgentToolResult<unknown>> => {
282 const a = args as Record<string, string>;
283 const parseList = (s: string | undefined) => (s ? s.split(',').map(x => x.trim()).filter(Boolean) : []);
284 const parseQuestions = (s: string | undefined) => {
285 if (!s) return [];
286 const parts = s.includes(';') ? s.split(';') : s.match(/[^?]+\?/g) ?? [s];
287 return parts.map(x => x.trim()).filter(Boolean);
288 };
289 const parseConnections = (s: string | undefined) => {
290 if (!s) return [];
291 return s.split(';').map(x => x.trim()).filter(Boolean).map(part => {
292 const [source, target, connectionType, ...noteParts] = part.split('|').map(x => x.trim());
293 return { source: source ?? '', target: target ?? '', connectionType: connectionType ?? 'related-to', note: noteParts.join('|') || '' };
294 });
295 };
296 pendingResult = {
297 islandId: '',
298 centroid: '',
299 title: a.title ?? '',
300 themes: parseList(a.themes),
301 highlights: parseList(a.highlights).filter(h => h.startsWith('at://')),
302 tensions: parseList(a.tensions),
303 openQuestions: parseQuestions(a.openQuestions),
304 synthesis: a.synthesis ?? '',
305 newConnections: parseConnections(a.newConnections),
306 analyzedAt: new Date().toISOString(),
307 };
308 return {
309 content: [{ type: 'text', text: 'Analysis saved.' }],
310 };
311 },
312};
313
314// --- Main ---
315
316async function main(): Promise<void> {
317 const args = process.argv.slice(2);
318 const wantJson = args.includes('--json');
319 const force = args.includes('--force');
320 const publish = args.includes('--publish');
321 const analyzeAll = args.includes('--all');
322 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
323 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
324 const modelArg = args.find((a) => a.startsWith('--model='));
325 const model = modelArg ? modelArg.split('=')[1] : undefined;
326
327 const targetId = args.find((a) => !a.startsWith('--'));
328
329 if (!targetId && !analyzeAll) {
330 console.error('Usage: bun island:analyze <island-id> [--force] [--publish] [--json] [--model=auto-fast]');
331 console.error(' bun island:analyze --all [--min-size 5] [--force] [--publish] [--json]');
332 process.exit(1);
333 }
334
335 let db: Database;
336 try {
337 db = openDb();
338 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
339 if (count === 0) {
340 console.error('No edges found. Run `bun island:discover` first.');
341 process.exit(1);
342 }
343 } catch {
344 console.error('Database not found. Run `bun island:discover` first.');
345 process.exit(1);
346 }
347
348 console.error('Finding components...');
349 const components = findComponents(db);
350
351 let targets: Component[];
352 if (analyzeAll) {
353 targets = components.filter((c) => c.nodes.size >= minSize);
354 } else {
355 const comp = components.find((c) => c.islandId === targetId);
356 if (!comp) {
357 console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`);
358 process.exit(1);
359 }
360 targets = [comp];
361 }
362
363 console.error(`Analyzing ${targets.length} island(s)...`);
364
365 // Reuse existing agent or create one
366 const sessionOpts: Record<string, unknown> = {
367 tools: [islandSaveTool],
368 permissionMode: 'bypassPermissions' as const,
369 };
370 if (model) sessionOpts.model = model;
371
372 let agentId: string;
373 const savedAgentId = (db.query('SELECT value FROM config WHERE key = ?').get('island_analyzer_agent_id') as { value: string } | null)?.value;
374
375 if (savedAgentId) {
376 agentId = savedAgentId;
377 console.error(`Reusing agent: ${agentId}`);
378 } else {
379 agentId = await createAgent({
380 persona: `You are a research analyst that synthesizes connected components (islands) from a knowledge graph into structured analysis records.
381
382Given an island — a cluster of URLs connected by semantic edges — you MUST call the island_save tool with your analysis. Do NOT just write text — you MUST use the island_save tool call.
383
384Your output must be a single island_save tool call with these fields:
385- title: concise sentence capturing the island's core narrative
386- themes: list of key themes
387- highlights: comma-separated AT-URIs of semantically significant EDGE records only. Use exact at://.../network.cosmik.connection/... URIs from the Edges section. Do not output domains, vertex URLs, project names, or descriptions.
388- tensions: max 3 structural tensions. "AI does X vs people resist X" is ONE tension, not three — fold variants in as parenthetical examples.
389- openQuestions: semicolon-separated questions raised by this island. Each item must be exactly one complete question ending in ?. Do not separate questions with commas; use semicolons.
390- synthesis: structured markdown with ### subheadings per thread, short paragraphs (max 3 sentences), [label](url) for every URL, **bold** for key terms, bullet lists for 3+ items. Close with "## What binds this island". See the synthesis field description for full format rules.
391- newConnections: suggested new edges between vertices
392
393CRITICAL: You MUST call island_save. Do not just describe your analysis in text. Call the tool.`,
394 human: 'island-analyzer',
395 tools: [islandSaveTool],
396 permissionMode: 'bypassPermissions',
397 memfs: false,
398 systemInfoReminder: false,
399 });
400 db.query('INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run('island_analyzer_agent_id', agentId);
401 console.error(`Created agent: ${agentId}`);
402 }
403
404 const session = createSession(agentId, {
405 memfsStartup: 'skip',
406 ...sessionOpts,
407 });
408
409 await session.initialize();
410
411 const results: AnalysisResult[] = [];
412 const published = new Map<string, string>();
413
414 for (const comp of targets) {
415 // Check if already analyzed
416 if (!force) {
417 const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null;
418 if (existing) {
419 console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
420 const cached = loadAnalysis(db, comp.islandId);
421 if (cached) {
422 cached.centroid = comp.centroid;
423 results.push(cached);
424 if (publish) {
425 const atUri = await publishIslandAnalysis(db, comp, cached);
426 published.set(comp.islandId, atUri);
427 console.error(` Published: ${atUri}`);
428 }
429 }
430 continue;
431 }
432 }
433
434 console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
435
436 const context = buildIslandContext(db, comp);
437 pendingResult = null;
438
439 const prompt = `Analyze this island and call island_save with your analysis.
440
441${context}`;
442
443 const result: SDKResultMessage = await session.runTurn(prompt);
444
445 console.error(` Result: success=${result.success}, stopReason=${result.stopReason ?? 'none'}, duration=${result.durationMs}ms`);
446
447 if (!result.success) {
448 console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`);
449 continue;
450 }
451
452 if (!pendingResult) {
453 console.error(` Agent did not call island_save`);
454 // Log what the agent produced
455 if (result.result) {
456 console.error(` Agent output: ${result.result.slice(0, 500)}`);
457 }
458 continue;
459 }
460
461 const analysisResult = pendingResult;
462
463 // Fill in island metadata
464 analysisResult.islandId = comp.islandId;
465 analysisResult.centroid = comp.centroid;
466
467 saveAnalysis(db, analysisResult);
468 results.push(analysisResult);
469
470 if (publish) {
471 const atUri = await publishIslandAnalysis(db, comp, analysisResult);
472 published.set(comp.islandId, atUri);
473 console.error(` Published: ${atUri}`);
474 }
475
476 console.error(` Title: ${analysisResult.title.slice(0, 80)}`);
477 console.error(` Themes: ${analysisResult.themes.join(', ') || 'none'}`);
478 console.error(` New connections: ${analysisResult.newConnections.length}`);
479 console.error(` Tensions: ${analysisResult.tensions.length}`);
480 }
481
482 await session[Symbol.asyncDispose]();
483
484 if (wantJson) {
485 console.log(JSON.stringify(results.map((r) => ({
486 ...r,
487 publishedAtUri: published.get(r.islandId),
488 })), null, 2));
489 db.close();
490 return;
491 }
492
493 for (const r of results) {
494 console.log(`\n${r.islandId}: ${r.title}`);
495 const atUri = published.get(r.islandId);
496 if (atUri) console.log(` Published: ${atUri}`);
497 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
498 console.log(` New connections: ${r.newConnections.length}`);
499 if (r.newConnections.length > 0) {
500 for (const nc of r.newConnections.slice(0, 10)) {
501 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
502 }
503 if (r.newConnections.length > 10) {
504 console.log(` ... and ${r.newConnections.length - 10} more`);
505 }
506 }
507 console.log(` Tensions: ${r.tensions.length}`);
508 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
509 console.log(` Highlights: ${r.highlights.length}`);
510 for (const h of r.highlights.slice(0, 5)) console.log(` ${h.slice(0, 100)}`);
511 console.log(` Open questions: ${r.openQuestions.length}`);
512 for (const q of r.openQuestions) console.log(` ${q.slice(0, 100)}`);
513 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}${r.synthesis.length > 300 ? '...' : ''}`);
514 }
515
516 db.close();
517}
518
519main().catch((err) => {
520 console.error('Fatal:', err instanceof Error ? err.message : String(err));
521 process.exit(1);
522});