nandi.uk
This repository has no description
15 kB
412 lines
1/**
2 * island:analyze <island-id>
3 *
4 * Analyze a discovered component using an LLM agent (letta-code-sdk).
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze a1b2c3d4e5f6 # analyze by island hash
9 * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { Database } from 'bun:sqlite';
18import {
19 createAgent,
20 createSession,
21 type AnyAgentTool,
22 type AgentToolResult,
23 type SDKResultMessage,
24} from '@letta-ai/letta-code-sdk';
25import { loadDotEnv } from './cli-utils.js';
26import { findComponents, domainFromUrl, type Component, type EdgeRow, fetchComponentEdges, resolveDidHandles } from './island-shared.js';
27
28const __dirname = dirname(fileURLToPath(import.meta.url));
29await loadDotEnv(resolve(__dirname, '..', '.env'));
30
31const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
32const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
33
34// --- Types ---
35
36interface AnalysisResult {
37 islandId: string;
38 lexmin: string;
39 title: string;
40 themes: string[];
41 highlights: string[];
42 tensions: string[];
43 openQuestions: string[];
44 synthesis: string;
45 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
46 analyzedAt: string;
47}
48
49// --- SQLite ---
50
51function openDb(): Database {
52 const db = new Database(DB_PATH, { create: true });
53 db.exec('PRAGMA journal_mode = WAL');
54 db.exec('PRAGMA synchronous = NORMAL');
55 db.exec(`
56 CREATE TABLE IF NOT EXISTS analyses (
57 island_id TEXT PRIMARY KEY,
58 title TEXT,
59 themes TEXT,
60 highlights TEXT,
61 tensions TEXT,
62 open_questions TEXT,
63 synthesis TEXT,
64 new_connections TEXT,
65 analyzed_at TEXT NOT NULL
66 );
67 `);
68 return db;
69}
70
71function saveAnalysis(db: Database, result: AnalysisResult): void {
72 db.query(`
73 INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
74 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
75 ON CONFLICT(island_id) DO UPDATE SET
76 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
77 tensions=excluded.tensions, open_questions=excluded.open_questions,
78 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
79 `).run(
80 result.islandId,
81 result.title,
82 JSON.stringify(result.themes),
83 JSON.stringify(result.highlights),
84 JSON.stringify(result.tensions),
85 JSON.stringify(result.openQuestions),
86 result.synthesis,
87 JSON.stringify(result.newConnections),
88 result.analyzedAt,
89 );
90}
91
92function loadAnalysis(db: Database, islandId: string): AnalysisResult | null {
93 const row = db.query('SELECT * FROM analyses WHERE island_id = ?').get(islandId) as Record<string, string> | null;
94 if (!row) return null;
95 return {
96 islandId: row.island_id,
97 lexmin: '',
98 title: row.title ?? '',
99 themes: JSON.parse(row.themes ?? '[]'),
100 highlights: JSON.parse(row.highlights ?? '[]'),
101 tensions: JSON.parse(row.tensions ?? '[]'),
102 openQuestions: JSON.parse(row.open_questions ?? '[]'),
103 synthesis: row.synthesis ?? '',
104 newConnections: JSON.parse(row.new_connections ?? '[]'),
105 analyzedAt: row.analyzed_at,
106 };
107}
108
109// --- Build island context from SQLite data ---
110
111function buildIslandContext(db: Database, comp: Component): string {
112 const edges = fetchComponentEdges(db, comp);
113 const didHandles = resolveDidHandles(db, comp.dids);
114
115 // Domain distribution
116 const domainCounts = new Map<string, number>();
117 for (const url of comp.nodes) {
118 const d = domainFromUrl(url);
119 domainCounts.set(d, (domainCounts.get(d) ?? 0) + 1);
120 }
121 const topDomains = [...domainCounts.entries()]
122 .sort((a, b) => b[1] - a[1])
123 .slice(0, 15)
124 .map(([d, c]) => `${d} (${c})`)
125 .join(', ');
126
127 // Edge type distribution
128 const edgeTypes = new Map<string, number>();
129 for (const e of edges) {
130 const t = e.relation || 'relates';
131 edgeTypes.set(t, (edgeTypes.get(t) ?? 0) + 1);
132 }
133 const edgeTypeStr = [...edgeTypes.entries()]
134 .sort((a, b) => b[1] - a[1])
135 .map(([t, c]) => `${t} (${c})`)
136 .join(', ');
137
138 // Contributors
139 const contributorStr = [...comp.dids]
140 .map((d) => didHandles.get(d) ?? d)
141 .join(', ');
142
143 // Vertices (grouped by domain for readability)
144 const verticesByDomain = new Map<string, string[]>();
145 for (const url of comp.nodes) {
146 const d = domainFromUrl(url);
147 if (!verticesByDomain.has(d)) verticesByDomain.set(d, []);
148 verticesByDomain.get(d)!.push(url);
149 }
150 const vertexSections = [...verticesByDomain.entries()]
151 .sort((a, b) => b[1].length - a[1].length)
152 .slice(0, 10)
153 .map(([domain, urls]) => `${domain}:\n${urls.slice(0, 5).map((u) => ` ${u}`).join('\n')}${urls.length > 5 ? `\n ... and ${urls.length - 5} more` : ''}`)
154 .join('\n\n');
155
156 // Edges (sample)
157 const edgeSample = edges.slice(0, 40)
158 .map((e) => {
159 const handle = didHandles.get(e.did) ?? e.did;
160 return `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}] (by ${handle})`;
161 })
162 .join('\n');
163
164 return [
165 `# Island ${comp.islandId}`,
166 '',
167 `## Stats`,
168 `- ${comp.nodes.size} vertices, ${edges.length} edges, ${comp.dids.size} contributors`,
169 `- Top domains: ${topDomains}`,
170 `- Edge types: ${edgeTypeStr}`,
171 `- Contributors: ${contributorStr}`,
172 '',
173 `## Vertices`,
174 vertexSections,
175 '',
176 `## Edges (sample)`,
177 edgeSample,
178 edges.length > 40 ? `\n... and ${edges.length - 40} more edges` : '',
179 ].join('\n');
180}
181
182// --- island_save tool ---
183
184let pendingResult: AnalysisResult | null = null;
185
186const islandSaveTool: AnyAgentTool = {
187 label: 'island_save',
188 name: 'island_save',
189 description: `Save your analysis of this island. Call once when you have completed your analysis.
190
191Fields:
192- title: concise sentence capturing the island's core narrative (max 300 chars)
193- themes: list of key themes (max 20, each max 100 chars)
194- highlights: list of vertex URLs or edge descriptions that are semantically significant (max 20)
195- tensions: contradictions or unresolved tensions within the island (max 10, each max 500 chars)
196- openQuestions: questions raised by this island (max 10, each max 500 chars)
197- synthesis: prose synthesis connecting the island's vertices and edges into a coherent narrative (max 20000 chars)
198- newConnections: suggested new edges between vertices, with source URL, target URL, connectionType, and a note explaining why (max 50)`,
199 parameters: {
200 type: 'object',
201 properties: {
202 title: { type: 'string', description: 'Concise sentence capturing the core narrative' },
203 themes: { type: 'array', items: { type: 'string' }, description: 'Key themes' },
204 highlights: { type: 'array', items: { type: 'string' }, description: 'Semantically significant vertices or edges' },
205 tensions: { type: 'array', items: { type: 'string' }, description: 'Contradictions or unresolved tensions' },
206 openQuestions: { type: 'array', items: { type: 'string' }, description: 'Questions raised by this island' },
207 synthesis: { type: 'string', description: 'Prose synthesis of the island' },
208 newConnections: {
209 type: 'array',
210 items: {
211 type: 'object',
212 properties: {
213 source: { type: 'string', description: 'Source vertex URL' },
214 target: { type: 'string', description: 'Target vertex URL' },
215 connectionType: { type: 'string', description: 'Relationship type (e.g. related-to, supports, contradicts)' },
216 note: { type: 'string', description: 'Why this connection exists' },
217 },
218 required: ['source', 'target', 'connectionType', 'note'],
219 },
220 description: 'Suggested new edges between vertices',
221 },
222 },
223 required: ['title', 'themes', 'synthesis'],
224 },
225 execute: async (_id, args): Promise<AgentToolResult<unknown>> => {
226 const a = args as Record<string, unknown>;
227 pendingResult = {
228 islandId: '',
229 lexmin: '',
230 title: String(a.title ?? ''),
231 themes: (a.themes as string[]) ?? [],
232 highlights: (a.highlights as string[]) ?? [],
233 tensions: (a.tensions as string[]) ?? [],
234 openQuestions: (a.openQuestions as string[]) ?? [],
235 synthesis: String(a.synthesis ?? ''),
236 newConnections: (a.newConnections as Array<{ source: string; target: string; connectionType: string; note: string }>) ?? [],
237 analyzedAt: new Date().toISOString(),
238 };
239 return {
240 content: [{ type: 'text', text: 'Analysis saved.' }],
241 };
242 },
243};
244
245// --- Main ---
246
247async function main(): Promise<void> {
248 const args = process.argv.slice(2);
249 const wantJson = args.includes('--json');
250 const force = args.includes('--force');
251 const analyzeAll = args.includes('--all');
252 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
253 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
254 const modelArg = args.find((a) => a.startsWith('--model='));
255 const model = modelArg ? modelArg.split('=')[1] : 'auto-fast';
256
257 const targetId = args.find((a) => !a.startsWith('--'));
258
259 if (!targetId && !analyzeAll) {
260 console.error('Usage: bun island:analyze <island-id> [--force] [--json] [--model=auto-fast]');
261 console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
262 process.exit(1);
263 }
264
265 let db: Database;
266 try {
267 db = openDb();
268 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
269 if (count === 0) {
270 console.error('No edges found. Run `bun island:discover` first.');
271 process.exit(1);
272 }
273 } catch {
274 console.error('Database not found. Run `bun island:discover` first.');
275 process.exit(1);
276 }
277
278 console.error('Finding components...');
279 const components = findComponents(db);
280
281 let targets: Component[];
282 if (analyzeAll) {
283 targets = components.filter((c) => c.nodes.size >= minSize);
284 } else {
285 const comp = components.find((c) => c.islandId === targetId);
286 if (!comp) {
287 console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`);
288 process.exit(1);
289 }
290 targets = [comp];
291 }
292
293 console.error(`Analyzing ${targets.length} island(s)...`);
294
295 // Create agent
296 const agentId = await createAgent({
297 model,
298 persona: `You are a research analyst that synthesizes connected components (islands) from a knowledge graph into structured analysis records.
299
300Given an island — a cluster of URLs connected by semantic edges — you produce:
3011. A title that captures the island's core narrative in one sentence
3022. Themes that describe what this cluster is about
3033. Highlights — vertices or edges that are semantically significant
3044. Tensions — contradictions or unresolved questions within the cluster
3055. Open questions raised by this island
3066. A synthesis — a prose narrative connecting the vertices and edges into a coherent story
3077. New connections — suggested edges between vertices that would strengthen the island
308
309You always call island_save once when done. You do not ask for permission.`,
310 human: 'island-analyzer',
311 tools: [islandSaveTool],
312 allowedTools: ['island_save'],
313 permissionMode: 'bypassPermissions',
314 memfs: false,
315 systemInfoReminder: false,
316 });
317
318 console.error(`Created agent: ${agentId}`);
319
320 const session = createSession(agentId, {
321 memfsStartup: 'skip',
322 });
323
324 await session.initialize();
325
326 const results: AnalysisResult[] = [];
327
328 for (const comp of targets) {
329 // Check if already analyzed
330 if (!force) {
331 const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null;
332 if (existing) {
333 console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
334 const cached = loadAnalysis(db, comp.islandId);
335 if (cached) {
336 cached.lexmin = comp.lexmin;
337 results.push(cached);
338 }
339 continue;
340 }
341 }
342
343 console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
344
345 const context = buildIslandContext(db, comp);
346 pendingResult = null;
347
348 const prompt = `Analyze this island and call island_save with your analysis.
349
350${context}`;
351
352 const result: SDKResultMessage = await session.runTurn(prompt);
353
354 if (!result.success) {
355 console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`);
356 continue;
357 }
358
359 if (!pendingResult) {
360 console.error(` Agent did not call island_save`);
361 continue;
362 }
363
364 // Fill in island metadata
365 pendingResult.islandId = comp.islandId;
366 pendingResult.lexmin = comp.lexmin;
367
368 saveAnalysis(db, pendingResult);
369 results.push(pendingResult);
370
371 console.error(` Title: ${pendingResult.title.slice(0, 80)}`);
372 console.error(` Themes: ${pendingResult.themes.join(', ') || 'none'}`);
373 console.error(` New connections: ${pendingResult.newConnections.length}`);
374 console.error(` Tensions: ${pendingResult.tensions.length}`);
375 }
376
377 await session[Symbol.asyncDispose]();
378
379 if (wantJson) {
380 console.log(JSON.stringify(results, null, 2));
381 db.close();
382 return;
383 }
384
385 for (const r of results) {
386 console.log(`\n${r.islandId}: ${r.title}`);
387 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
388 console.log(` New connections: ${r.newConnections.length}`);
389 if (r.newConnections.length > 0) {
390 for (const nc of r.newConnections.slice(0, 10)) {
391 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
392 }
393 if (r.newConnections.length > 10) {
394 console.log(` ... and ${r.newConnections.length - 10} more`);
395 }
396 }
397 console.log(` Tensions: ${r.tensions.length}`);
398 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
399 console.log(` Highlights: ${r.highlights.length}`);
400 for (const h of r.highlights.slice(0, 5)) console.log(` ${h.slice(0, 100)}`);
401 console.log(` Open questions: ${r.openQuestions.length}`);
402 for (const q of r.openQuestions) console.log(` ${q.slice(0, 100)}`);
403 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}${r.synthesis.length > 300 ? '...' : ''}`);
404 }
405
406 db.close();
407}
408
409main().catch((err) => {
410 console.error('Fatal:', err instanceof Error ? err.message : String(err));
411 process.exit(1);
412});