nandi.uk
This repository has no description
15 kB
408 lines
1/**
2 * island:analyze <island-id>
3 *
4 * Analyze a discovered component using an LLM agent (letta-code-sdk).
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze a1b2c3d4e5f6 # analyze by island hash
9 * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { Database } from 'bun:sqlite';
18import {
19 createAgent,
20 createSession,
21 type AnyAgentTool,
22 type AgentToolResult,
23 type SDKResultMessage,
24} from '@letta-ai/letta-code-sdk';
25import { loadDotEnv } from './cli-utils.js';
26import { findComponents, domainFromUrl, type Component, type EdgeRow, fetchComponentEdges, resolveDidHandles } from './island-shared.js';
27
28const __dirname = dirname(fileURLToPath(import.meta.url));
29await loadDotEnv(resolve(__dirname, '..', '.env'));
30
31const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
32const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
33
34// --- Types ---
35
36interface AnalysisResult {
37 islandId: string;
38 lexmin: string;
39 title: string;
40 themes: string[];
41 highlights: string[];
42 tensions: string[];
43 openQuestions: string[];
44 synthesis: string;
45 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
46 analyzedAt: string;
47}
48
49// --- SQLite ---
50
51function openDb(): Database {
52 const db = new Database(DB_PATH, { create: true });
53 db.exec('PRAGMA journal_mode = WAL');
54 db.exec('PRAGMA synchronous = NORMAL');
55 db.exec(`
56 CREATE TABLE IF NOT EXISTS analyses (
57 island_id TEXT PRIMARY KEY,
58 title TEXT,
59 themes TEXT,
60 highlights TEXT,
61 tensions TEXT,
62 open_questions TEXT,
63 synthesis TEXT,
64 new_connections TEXT,
65 analyzed_at TEXT NOT NULL
66 );
67 `);
68 return db;
69}
70
71function saveAnalysis(db: Database, result: AnalysisResult): void {
72 db.query(`
73 INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
74 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
75 ON CONFLICT(island_id) DO UPDATE SET
76 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
77 tensions=excluded.tensions, open_questions=excluded.open_questions,
78 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
79 `).run(
80 result.islandId,
81 result.title,
82 JSON.stringify(result.themes),
83 JSON.stringify(result.highlights),
84 JSON.stringify(result.tensions),
85 JSON.stringify(result.openQuestions),
86 result.synthesis,
87 JSON.stringify(result.newConnections),
88 result.analyzedAt,
89 );
90}
91
92function loadAnalysis(db: Database, islandId: string): AnalysisResult | null {
93 const row = db.query('SELECT * FROM analyses WHERE island_id = ?').get(islandId) as Record<string, string> | null;
94 if (!row) return null;
95 return {
96 islandId: row.island_id,
97 lexmin: '',
98 title: row.title ?? '',
99 themes: JSON.parse(row.themes ?? '[]'),
100 highlights: JSON.parse(row.highlights ?? '[]'),
101 tensions: JSON.parse(row.tensions ?? '[]'),
102 openQuestions: JSON.parse(row.open_questions ?? '[]'),
103 synthesis: row.synthesis ?? '',
104 newConnections: JSON.parse(row.new_connections ?? '[]'),
105 analyzedAt: row.analyzed_at,
106 };
107}
108
109// --- Build island context from SQLite data ---
110
111function buildIslandContext(db: Database, comp: Component): string {
112 const edges = fetchComponentEdges(db, comp);
113 const didHandles = resolveDidHandles(db, comp.dids);
114
115 // Domain distribution
116 const domainCounts = new Map<string, number>();
117 for (const url of comp.nodes) {
118 const d = domainFromUrl(url);
119 domainCounts.set(d, (domainCounts.get(d) ?? 0) + 1);
120 }
121 const topDomains = [...domainCounts.entries()]
122 .sort((a, b) => b[1] - a[1])
123 .slice(0, 15)
124 .map(([d, c]) => `${d} (${c})`)
125 .join(', ');
126
127 // Edge type distribution
128 const edgeTypes = new Map<string, number>();
129 for (const e of edges) {
130 const t = e.relation || 'relates';
131 edgeTypes.set(t, (edgeTypes.get(t) ?? 0) + 1);
132 }
133 const edgeTypeStr = [...edgeTypes.entries()]
134 .sort((a, b) => b[1] - a[1])
135 .map(([t, c]) => `${t} (${c})`)
136 .join(', ');
137
138 // Contributors
139 const contributorStr = [...comp.dids]
140 .map((d) => didHandles.get(d) ?? d)
141 .join(', ');
142
143 // Vertices (grouped by domain for readability)
144 const verticesByDomain = new Map<string, string[]>();
145 for (const url of comp.nodes) {
146 const d = domainFromUrl(url);
147 if (!verticesByDomain.has(d)) verticesByDomain.set(d, []);
148 verticesByDomain.get(d)!.push(url);
149 }
150 const vertexSections = [...verticesByDomain.entries()]
151 .sort((a, b) => b[1].length - a[1].length)
152 .slice(0, 10)
153 .map(([domain, urls]) => `${domain}:\n${urls.slice(0, 5).map((u) => ` ${u}`).join('\n')}${urls.length > 5 ? `\n ... and ${urls.length - 5} more` : ''}`)
154 .join('\n\n');
155
156 // Edges (sample)
157 const edgeSample = edges.slice(0, 40)
158 .map((e) => {
159 const handle = didHandles.get(e.did) ?? e.did;
160 return `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}] (by ${handle})`;
161 })
162 .join('\n');
163
164 return [
165 `# Island ${comp.islandId}`,
166 '',
167 `## Stats`,
168 `- ${comp.nodes.size} vertices, ${edges.length} edges, ${comp.dids.size} contributors`,
169 `- Top domains: ${topDomains}`,
170 `- Edge types: ${edgeTypeStr}`,
171 `- Contributors: ${contributorStr}`,
172 '',
173 `## Vertices`,
174 vertexSections,
175 '',
176 `## Edges (sample)`,
177 edgeSample,
178 edges.length > 40 ? `\n... and ${edges.length - 40} more edges` : '',
179 ].join('\n');
180}
181
182// --- island_save tool ---
183
184let pendingResult: AnalysisResult | null = null;
185
186const islandSaveTool: AnyAgentTool = {
187 label: 'island_save',
188 name: 'island_save',
189 description: `Save your analysis of this island. Call once when you have completed your analysis.
190
191Fields:
192- title: concise sentence capturing the island's core narrative (max 300 chars)
193- themes: list of key themes (max 20, each max 100 chars)
194- highlights: list of vertex URLs or edge descriptions that are semantically significant (max 20)
195- tensions: contradictions or unresolved tensions within the island (max 10, each max 500 chars)
196- openQuestions: questions raised by this island (max 10, each max 500 chars)
197- synthesis: prose synthesis connecting the island's vertices and edges into a coherent narrative (max 20000 chars)
198- newConnections: suggested new edges between vertices, with source URL, target URL, connectionType, and a note explaining why (max 50)`,
199 parameters: {
200 type: 'object',
201 properties: {
202 title: { type: 'string', description: 'Concise sentence capturing the core narrative' },
203 themes: { type: 'array', items: { type: 'string' }, description: 'Key themes' },
204 highlights: { type: 'array', items: { type: 'string' }, description: 'Semantically significant vertices or edges' },
205 tensions: { type: 'array', items: { type: 'string' }, description: 'Contradictions or unresolved tensions' },
206 openQuestions: { type: 'array', items: { type: 'string' }, description: 'Questions raised by this island' },
207 synthesis: { type: 'string', description: 'Prose synthesis of the island' },
208 newConnections: {
209 type: 'array',
210 items: {
211 type: 'object',
212 properties: {
213 source: { type: 'string', description: 'Source vertex URL' },
214 target: { type: 'string', description: 'Target vertex URL' },
215 connectionType: { type: 'string', description: 'Relationship type (e.g. related-to, supports, contradicts)' },
216 note: { type: 'string', description: 'Why this connection exists' },
217 },
218 required: ['source', 'target', 'connectionType', 'note'],
219 },
220 description: 'Suggested new edges between vertices',
221 },
222 },
223 required: ['title', 'themes', 'synthesis'],
224 },
225 execute: async (_id, args): Promise<AgentToolResult<unknown>> => {
226 const a = args as Record<string, unknown>;
227 pendingResult = {
228 islandId: '',
229 lexmin: '',
230 title: String(a.title ?? ''),
231 themes: (a.themes as string[]) ?? [],
232 highlights: (a.highlights as string[]) ?? [],
233 tensions: (a.tensions as string[]) ?? [],
234 openQuestions: (a.openQuestions as string[]) ?? [],
235 synthesis: String(a.synthesis ?? ''),
236 newConnections: (a.newConnections as Array<{ source: string; target: string; connectionType: string; note: string }>) ?? [],
237 analyzedAt: new Date().toISOString(),
238 };
239 return {
240 content: [{ type: 'text', text: 'Analysis saved.' }],
241 };
242 },
243};
244
245// --- Main ---
246
247async function main(): Promise<void> {
248 const args = process.argv.slice(2);
249 const wantJson = args.includes('--json');
250 const force = args.includes('--force');
251 const analyzeAll = args.includes('--all');
252 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
253 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
254
255 const targetId = args.find((a) => !a.startsWith('--'));
256
257 if (!targetId && !analyzeAll) {
258 console.error('Usage: bun island:analyze <island-id> [--force] [--json]');
259 console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
260 process.exit(1);
261 }
262
263 let db: Database;
264 try {
265 db = openDb();
266 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
267 if (count === 0) {
268 console.error('No edges found. Run `bun island:discover` first.');
269 process.exit(1);
270 }
271 } catch {
272 console.error('Database not found. Run `bun island:discover` first.');
273 process.exit(1);
274 }
275
276 console.error('Finding components...');
277 const components = findComponents(db);
278
279 let targets: Component[];
280 if (analyzeAll) {
281 targets = components.filter((c) => c.nodes.size >= minSize);
282 } else {
283 const comp = components.find((c) => c.islandId === targetId);
284 if (!comp) {
285 console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`);
286 process.exit(1);
287 }
288 targets = [comp];
289 }
290
291 console.error(`Analyzing ${targets.length} island(s)...`);
292
293 // Create agent
294 const agentId = await createAgent({
295 persona: `You are a research analyst that synthesizes connected components (islands) from a knowledge graph into structured analysis records.
296
297Given an island — a cluster of URLs connected by semantic edges — you produce:
2981. A title that captures the island's core narrative in one sentence
2992. Themes that describe what this cluster is about
3003. Highlights — vertices or edges that are semantically significant
3014. Tensions — contradictions or unresolved questions within the cluster
3025. Open questions raised by this island
3036. A synthesis — a prose narrative connecting the vertices and edges into a coherent story
3047. New connections — suggested edges between vertices that would strengthen the island
305
306You always call island_save once when done. You do not ask for permission.`,
307 human: 'island-analyzer',
308 tools: [islandSaveTool],
309 permissionMode: 'bypassPermissions',
310 memfs: false,
311 systemInfoReminder: false,
312 });
313
314 console.error(`Created agent: ${agentId}`);
315
316 const session = createSession(agentId, {
317 memfsStartup: 'skip',
318 });
319
320 await session.initialize();
321
322 const results: AnalysisResult[] = [];
323
324 for (const comp of targets) {
325 // Check if already analyzed
326 if (!force) {
327 const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null;
328 if (existing) {
329 console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
330 const cached = loadAnalysis(db, comp.islandId);
331 if (cached) {
332 cached.lexmin = comp.lexmin;
333 results.push(cached);
334 }
335 continue;
336 }
337 }
338
339 console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
340
341 const context = buildIslandContext(db, comp);
342 pendingResult = null;
343
344 const prompt = `Analyze this island and call island_save with your analysis.
345
346${context}`;
347
348 const result: SDKResultMessage = await session.runTurn(prompt);
349
350 if (!result.success) {
351 console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`);
352 continue;
353 }
354
355 if (!pendingResult) {
356 console.error(` Agent did not call island_save`);
357 continue;
358 }
359
360 // Fill in island metadata
361 pendingResult.islandId = comp.islandId;
362 pendingResult.lexmin = comp.lexmin;
363
364 saveAnalysis(db, pendingResult);
365 results.push(pendingResult);
366
367 console.error(` Title: ${pendingResult.title.slice(0, 80)}`);
368 console.error(` Themes: ${pendingResult.themes.join(', ') || 'none'}`);
369 console.error(` New connections: ${pendingResult.newConnections.length}`);
370 console.error(` Tensions: ${pendingResult.tensions.length}`);
371 }
372
373 await session[Symbol.asyncDispose]();
374
375 if (wantJson) {
376 console.log(JSON.stringify(results, null, 2));
377 db.close();
378 return;
379 }
380
381 for (const r of results) {
382 console.log(`\n${r.islandId}: ${r.title}`);
383 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
384 console.log(` New connections: ${r.newConnections.length}`);
385 if (r.newConnections.length > 0) {
386 for (const nc of r.newConnections.slice(0, 10)) {
387 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
388 }
389 if (r.newConnections.length > 10) {
390 console.log(` ... and ${r.newConnections.length - 10} more`);
391 }
392 }
393 console.log(` Tensions: ${r.tensions.length}`);
394 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
395 console.log(` Highlights: ${r.highlights.length}`);
396 for (const h of r.highlights.slice(0, 5)) console.log(` ${h.slice(0, 100)}`);
397 console.log(` Open questions: ${r.openQuestions.length}`);
398 for (const q of r.openQuestions) console.log(` ${q.slice(0, 100)}`);
399 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}${r.synthesis.length > 300 ? '...' : ''}`);
400 }
401
402 db.close();
403}
404
405main().catch((err) => {
406 console.error('Fatal:', err instanceof Error ? err.message : String(err));
407 process.exit(1);
408});