nandi.uk
This repository has no description
15 kB
423 lines
1/**
2 * island:analyze <island-id>
3 *
4 * Analyze a discovered component using an LLM agent (letta-code-sdk).
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze a1b2c3d4e5f6 # analyze by island hash
9 * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { Database } from 'bun:sqlite';
18import {
19 createAgent,
20 createSession,
21 type AnyAgentTool,
22 type AgentToolResult,
23 type SDKResultMessage,
24} from '@letta-ai/letta-code-sdk';
25import { loadDotEnv } from './cli-utils.js';
26import { findComponents, domainFromUrl, type Component, type EdgeRow, fetchComponentEdges, resolveDidHandles } from './island-shared.js';
27
28const __dirname = dirname(fileURLToPath(import.meta.url));
29await loadDotEnv(resolve(__dirname, '..', '.env'));
30
31const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
32const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
33
34// --- Types ---
35
36interface AnalysisResult {
37 islandId: string;
38 lexmin: string;
39 title: string;
40 themes: string[];
41 highlights: string[];
42 tensions: string[];
43 openQuestions: string[];
44 synthesis: string;
45 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
46 analyzedAt: string;
47}
48
49// --- SQLite ---
50
51function openDb(): Database {
52 const db = new Database(DB_PATH, { create: true });
53 db.exec('PRAGMA journal_mode = WAL');
54 db.exec('PRAGMA synchronous = NORMAL');
55 db.exec(`
56 CREATE TABLE IF NOT EXISTS analyses (
57 island_id TEXT PRIMARY KEY,
58 title TEXT,
59 themes TEXT,
60 highlights TEXT,
61 tensions TEXT,
62 open_questions TEXT,
63 synthesis TEXT,
64 new_connections TEXT,
65 analyzed_at TEXT NOT NULL
66 );
67 CREATE TABLE IF NOT EXISTS config (
68 key TEXT PRIMARY KEY,
69 value TEXT NOT NULL
70 );
71 `);
72 return db;
73}
74
75function saveAnalysis(db: Database, result: AnalysisResult): void {
76 db.query(`
77 INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
78 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
79 ON CONFLICT(island_id) DO UPDATE SET
80 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
81 tensions=excluded.tensions, open_questions=excluded.open_questions,
82 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
83 `).run(
84 result.islandId,
85 result.title,
86 JSON.stringify(result.themes),
87 JSON.stringify(result.highlights),
88 JSON.stringify(result.tensions),
89 JSON.stringify(result.openQuestions),
90 result.synthesis,
91 JSON.stringify(result.newConnections),
92 result.analyzedAt,
93 );
94}
95
96function loadAnalysis(db: Database, islandId: string): AnalysisResult | null {
97 const row = db.query('SELECT * FROM analyses WHERE island_id = ?').get(islandId) as Record<string, string> | null;
98 if (!row) return null;
99 return {
100 islandId: row.island_id,
101 lexmin: '',
102 title: row.title ?? '',
103 themes: JSON.parse(row.themes ?? '[]'),
104 highlights: JSON.parse(row.highlights ?? '[]'),
105 tensions: JSON.parse(row.tensions ?? '[]'),
106 openQuestions: JSON.parse(row.open_questions ?? '[]'),
107 synthesis: row.synthesis ?? '',
108 newConnections: JSON.parse(row.new_connections ?? '[]'),
109 analyzedAt: row.analyzed_at,
110 };
111}
112
113// --- Build island context from SQLite data ---
114
115function buildIslandContext(db: Database, comp: Component): string {
116 const edges = fetchComponentEdges(db, comp);
117 const didHandles = resolveDidHandles(db, comp.dids);
118
119 // Domain distribution
120 const domainCounts = new Map<string, number>();
121 for (const url of comp.nodes) {
122 const d = domainFromUrl(url);
123 domainCounts.set(d, (domainCounts.get(d) ?? 0) + 1);
124 }
125 const topDomains = [...domainCounts.entries()]
126 .sort((a, b) => b[1] - a[1])
127 .slice(0, 15)
128 .map(([d, c]) => `${d} (${c})`)
129 .join(', ');
130
131 // Edge type distribution
132 const edgeTypes = new Map<string, number>();
133 for (const e of edges) {
134 const t = e.relation || 'relates';
135 edgeTypes.set(t, (edgeTypes.get(t) ?? 0) + 1);
136 }
137 const edgeTypeStr = [...edgeTypes.entries()]
138 .sort((a, b) => b[1] - a[1])
139 .map(([t, c]) => `${t} (${c})`)
140 .join(', ');
141
142 // Contributors
143 const contributorStr = [...comp.dids]
144 .map((d) => didHandles.get(d) ?? d)
145 .join(', ');
146
147 // Vertices (grouped by domain for readability)
148 const verticesByDomain = new Map<string, string[]>();
149 for (const url of comp.nodes) {
150 const d = domainFromUrl(url);
151 if (!verticesByDomain.has(d)) verticesByDomain.set(d, []);
152 verticesByDomain.get(d)!.push(url);
153 }
154 const vertexSections = [...verticesByDomain.entries()]
155 .sort((a, b) => b[1].length - a[1].length)
156 .slice(0, 10)
157 .map(([domain, urls]) => `${domain}:\n${urls.slice(0, 5).map((u) => ` ${u}`).join('\n')}${urls.length > 5 ? `\n ... and ${urls.length - 5} more` : ''}`)
158 .join('\n\n');
159
160 // Edges (sample)
161 const edgeSample = edges.slice(0, 40)
162 .map((e) => {
163 const handle = didHandles.get(e.did) ?? e.did;
164 return `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}] (by ${handle})`;
165 })
166 .join('\n');
167
168 return [
169 `# Island ${comp.islandId}`,
170 '',
171 `## Stats`,
172 `- ${comp.nodes.size} vertices, ${edges.length} edges, ${comp.dids.size} contributors`,
173 `- Top domains: ${topDomains}`,
174 `- Edge types: ${edgeTypeStr}`,
175 `- Contributors: ${contributorStr}`,
176 '',
177 `## Vertices`,
178 vertexSections,
179 '',
180 `## Edges (sample)`,
181 edgeSample,
182 edges.length > 40 ? `\n... and ${edges.length - 40} more edges` : '',
183 ].join('\n');
184}
185
186// --- island_save tool ---
187
188let pendingResult: AnalysisResult | null = null;
189
190const islandSaveTool: AnyAgentTool = {
191 label: 'island_save',
192 name: 'island_save',
193 description: `Save your analysis of this island. Call this tool once when you have completed your analysis. This is the ONLY way to save your work — you MUST call this tool.`,
194 parameters: {
195 type: 'object',
196 properties: {
197 title: { type: 'string', description: 'Concise sentence capturing the core narrative' },
198 themes: { type: 'string', description: 'Comma-separated key themes' },
199 highlights: { type: 'string', description: 'Comma-separated semantically significant vertices or edges' },
200 tensions: { type: 'string', description: 'Comma-separated contradictions or unresolved tensions' },
201 openQuestions: { type: 'string', description: 'Comma-separated questions raised by this island' },
202 synthesis: { type: 'string', description: 'Prose synthesis of the island' },
203 newConnections: { type: 'string', description: 'Semicolon-separated new edges as: source|target|type|note' },
204 },
205 required: ['title', 'themes', 'synthesis'],
206 },
207 execute: async (_id, args): Promise<AgentToolResult<unknown>> => {
208 const a = args as Record<string, string>;
209 const parseList = (s: string | undefined) => (s ? s.split(',').map(x => x.trim()).filter(Boolean) : []);
210 const parseConnections = (s: string | undefined) => {
211 if (!s) return [];
212 return s.split(';').map(x => x.trim()).filter(Boolean).map(part => {
213 const [source, target, connectionType, ...noteParts] = part.split('|').map(x => x.trim());
214 return { source: source ?? '', target: target ?? '', connectionType: connectionType ?? 'related-to', note: noteParts.join('|') || '' };
215 });
216 };
217 pendingResult = {
218 islandId: '',
219 lexmin: '',
220 title: a.title ?? '',
221 themes: parseList(a.themes),
222 highlights: parseList(a.highlights),
223 tensions: parseList(a.tensions),
224 openQuestions: parseList(a.openQuestions),
225 synthesis: a.synthesis ?? '',
226 newConnections: parseConnections(a.newConnections),
227 analyzedAt: new Date().toISOString(),
228 };
229 return {
230 content: [{ type: 'text', text: 'Analysis saved.' }],
231 };
232 },
233};
234
235// --- Main ---
236
237async function main(): Promise<void> {
238 const args = process.argv.slice(2);
239 const wantJson = args.includes('--json');
240 const force = args.includes('--force');
241 const analyzeAll = args.includes('--all');
242 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
243 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
244 const modelArg = args.find((a) => a.startsWith('--model='));
245 const model = modelArg ? modelArg.split('=')[1] : undefined;
246
247 const targetId = args.find((a) => !a.startsWith('--'));
248
249 if (!targetId && !analyzeAll) {
250 console.error('Usage: bun island:analyze <island-id> [--force] [--json] [--model=auto-fast]');
251 console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
252 process.exit(1);
253 }
254
255 let db: Database;
256 try {
257 db = openDb();
258 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
259 if (count === 0) {
260 console.error('No edges found. Run `bun island:discover` first.');
261 process.exit(1);
262 }
263 } catch {
264 console.error('Database not found. Run `bun island:discover` first.');
265 process.exit(1);
266 }
267
268 console.error('Finding components...');
269 const components = findComponents(db);
270
271 let targets: Component[];
272 if (analyzeAll) {
273 targets = components.filter((c) => c.nodes.size >= minSize);
274 } else {
275 const comp = components.find((c) => c.islandId === targetId);
276 if (!comp) {
277 console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`);
278 process.exit(1);
279 }
280 targets = [comp];
281 }
282
283 console.error(`Analyzing ${targets.length} island(s)...`);
284
285 // Reuse existing agent or create one
286 const sessionOpts: Record<string, unknown> = {
287 tools: [islandSaveTool],
288 permissionMode: 'bypassPermissions' as const,
289 };
290 if (model) sessionOpts.model = model;
291
292 let agentId: string;
293 const savedAgentId = (db.query('SELECT value FROM config WHERE key = ?').get('island_analyzer_agent_id') as { value: string } | null)?.value;
294
295 if (savedAgentId) {
296 agentId = savedAgentId;
297 console.error(`Reusing agent: ${agentId}`);
298 } else {
299 agentId = await createAgent({
300 persona: `You are a research analyst that synthesizes connected components (islands) from a knowledge graph into structured analysis records.
301
302Given an island — a cluster of URLs connected by semantic edges — you MUST call the island_save tool with your analysis. Do NOT just write text — you MUST use the island_save tool call.
303
304Your output must be a single island_save tool call with these fields:
305- title: concise sentence capturing the island's core narrative
306- themes: list of key themes
307- highlights: semantically significant vertices or edges
308- tensions: contradictions or unresolved tensions
309- openQuestions: questions raised by this island
310- synthesis: prose narrative connecting the vertices and edges
311- newConnections: suggested new edges between vertices
312
313CRITICAL: You MUST call island_save. Do not just describe your analysis in text. Call the tool.`,
314 human: 'island-analyzer',
315 tools: [islandSaveTool],
316 permissionMode: 'bypassPermissions',
317 memfs: false,
318 systemInfoReminder: false,
319 });
320 db.query('INSERT INTO config (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run('island_analyzer_agent_id', agentId);
321 console.error(`Created agent: ${agentId}`);
322 }
323
324 const session = createSession(agentId, {
325 memfsStartup: 'skip',
326 ...sessionOpts,
327 });
328
329 await session.initialize();
330
331 const results: AnalysisResult[] = [];
332
333 for (const comp of targets) {
334 // Check if already analyzed
335 if (!force) {
336 const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null;
337 if (existing) {
338 console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
339 const cached = loadAnalysis(db, comp.islandId);
340 if (cached) {
341 cached.lexmin = comp.lexmin;
342 results.push(cached);
343 }
344 continue;
345 }
346 }
347
348 console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
349
350 const context = buildIslandContext(db, comp);
351 pendingResult = null;
352
353 const prompt = `Analyze this island and call island_save with your analysis.
354
355${context}`;
356
357 const result: SDKResultMessage = await session.runTurn(prompt);
358
359 console.error(` Result: success=${result.success}, stopReason=${result.stopReason ?? 'none'}, duration=${result.durationMs}ms`);
360
361 if (!result.success) {
362 console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`);
363 continue;
364 }
365
366 if (!pendingResult) {
367 console.error(` Agent did not call island_save`);
368 // Log what the agent produced
369 if (result.result) {
370 console.error(` Agent output: ${result.result.slice(0, 500)}`);
371 }
372 continue;
373 }
374
375 // Fill in island metadata
376 pendingResult.islandId = comp.islandId;
377 pendingResult.lexmin = comp.lexmin;
378
379 saveAnalysis(db, pendingResult);
380 results.push(pendingResult);
381
382 console.error(` Title: ${pendingResult.title.slice(0, 80)}`);
383 console.error(` Themes: ${pendingResult.themes.join(', ') || 'none'}`);
384 console.error(` New connections: ${pendingResult.newConnections.length}`);
385 console.error(` Tensions: ${pendingResult.tensions.length}`);
386 }
387
388 await session[Symbol.asyncDispose]();
389
390 if (wantJson) {
391 console.log(JSON.stringify(results, null, 2));
392 db.close();
393 return;
394 }
395
396 for (const r of results) {
397 console.log(`\n${r.islandId}: ${r.title}`);
398 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
399 console.log(` New connections: ${r.newConnections.length}`);
400 if (r.newConnections.length > 0) {
401 for (const nc of r.newConnections.slice(0, 10)) {
402 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
403 }
404 if (r.newConnections.length > 10) {
405 console.log(` ... and ${r.newConnections.length - 10} more`);
406 }
407 }
408 console.log(` Tensions: ${r.tensions.length}`);
409 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
410 console.log(` Highlights: ${r.highlights.length}`);
411 for (const h of r.highlights.slice(0, 5)) console.log(` ${h.slice(0, 100)}`);
412 console.log(` Open questions: ${r.openQuestions.length}`);
413 for (const q of r.openQuestions) console.log(` ${q.slice(0, 100)}`);
414 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}${r.synthesis.length > 300 ? '...' : ''}`);
415 }
416
417 db.close();
418}
419
420main().catch((err) => {
421 console.error('Fatal:', err instanceof Error ? err.message : String(err));
422 process.exit(1);
423});