nandi.uk
This repository has no description
17 kB
474 lines
1/**
2 * island:analyze <island-id>
3 *
4 * Analyze a discovered component against carry data.
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze a1b2c3d4e5f6 # analyze by island hash
9 * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { execFileSync } from 'node:child_process';
18import { Database } from 'bun:sqlite';
19import { loadDotEnv } from './cli-utils.js';
20import { findComponents, islandId, domainFromUrl, type Component } from './island-shared.js';
21
22const __dirname = dirname(fileURLToPath(import.meta.url));
23await loadDotEnv(resolve(__dirname, '..', '.env'));
24
25const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
26const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
27
28// --- Types ---
29
30interface CarryEntity { name: string; url: string; description: string }
31interface CarryCitation { url: string; title: string; takeaway: string; domain: string }
32interface CarryConnection { source: string; target: string; relation: string; context: string }
33
34interface AnalysisResult {
35 islandId: string;
36 lexmin: string;
37 title: string;
38 themes: string[];
39 highlights: string[];
40 tensions: string[];
41 openQuestions: string[];
42 synthesis: string;
43 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
44 analyzedAt: string;
45}
46
47// --- SQLite ---
48
49function openDb(): Database {
50 const db = new Database(DB_PATH, { create: true });
51 db.exec('PRAGMA journal_mode = WAL');
52 db.exec('PRAGMA synchronous = NORMAL');
53 db.exec(`
54 CREATE TABLE IF NOT EXISTS analyses (
55 island_id TEXT PRIMARY KEY,
56 title TEXT,
57 themes TEXT,
58 highlights TEXT,
59 tensions TEXT,
60 open_questions TEXT,
61 synthesis TEXT,
62 new_connections TEXT,
63 analyzed_at TEXT NOT NULL
64 );
65 `);
66 return db;
67}
68
69// --- Carry data ---
70
71function carryQueryJson<T>(domain: string, fields: string[]): T[] {
72 try {
73 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
74 cwd: CARRY_DIR,
75 encoding: 'utf-8',
76 timeout: 15_000,
77 });
78 return JSON.parse(out) as T[];
79 } catch {
80 return [];
81 }
82}
83
84function buildEntityUrlIndex(): Map<string, string> {
85 const entities = carryQueryJson<CarryEntity>('org.latha.entity', ['name', 'url']);
86 const index = new Map<string, string>();
87 for (const e of entities) {
88 if (e.name && e.url?.startsWith('https://')) {
89 index.set(e.name, e.url);
90 }
91 }
92 return index;
93}
94
95// --- Analysis ---
96
97const THEME_KEYWORDS: Record<string, string[]> = {
98 'open science': ['open science', 'open access', 'oa', 'preprint'],
99 'data sharing': ['data sharing', 'shared data', 'open data', 'dataset'],
100 'reproducibility': ['reproducib', 'replicab', 'replication'],
101 'decentralization': ['decentraliz', 'federat', 'self-host'],
102 'knowledge graphs': ['knowledge graph', 'ontolog', 'linked data', 'semantic'],
103 'ai/ml': ['machine learning', 'deep learning', 'neural', 'llm', 'gpt', 'claude'],
104 'community': ['community', 'collective', 'cooperative', 'commons'],
105 'governance': ['governance', 'policy', 'regulation', 'standards'],
106 'infrastructure': ['infrastructure', 'platform', 'tooling', 'pipeline'],
107 'stigmergy': ['stigmerg', 'pheromone', 'emergent', 'self-organiz'],
108 'provenance': ['provenance', 'lineage', 'traceability', 'attribution'],
109 'doi': ['doi', 'zenodo', 'citeable', 'citation'],
110 'privacy': ['privacy', 'encryption', 'e2ee', 'consent'],
111 'sovereignty': ['sovereignty', 'self-determin', 'autonomy', 'agency'],
112 'at protocol': ['atproto', 'at protocol', 'bluesky', 'pds', 'appview'],
113 'semantic web': ['semantic web', 'rdf', 'sparql', 'linked data'],
114};
115
116function extractThemes(text: string): string[] {
117 const lower = text.toLowerCase();
118 return Object.entries(THEME_KEYWORDS)
119 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
120 .map(([theme]) => theme);
121}
122
123function matchCarryConnections(
124 islandUrls: Set<string>,
125 carryConnections: CarryConnection[],
126 entityUrlIndex: Map<string, string>,
127): Array<{ source: string; target: string; connectionType: string; note: string }> {
128 const connections: Array<{ source: string; target: string; connectionType: string; note: string }> = [];
129
130 for (const conn of carryConnections) {
131 const srcUrl = entityUrlIndex.get(conn.source) || (conn.source.startsWith('https://') ? conn.source : '');
132 const tgtUrl = entityUrlIndex.get(conn.target) || (conn.target.startsWith('https://') ? conn.target : '');
133
134 if (!srcUrl && !tgtUrl) continue;
135
136 const srcInIsland = srcUrl && islandUrls.has(srcUrl);
137 const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
138
139 // Both in island → already connected
140 if (srcInIsland && tgtInIsland) continue;
141
142 // One side in island → bridge connection
143 if (srcInIsland && tgtUrl) {
144 connections.push({
145 source: srcUrl, target: tgtUrl,
146 connectionType: conn.relation,
147 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
148 });
149 } else if (tgtInIsland && srcUrl) {
150 connections.push({
151 source: tgtUrl, target: srcUrl,
152 connectionType: conn.relation,
153 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
154 });
155 } else if (srcUrl && tgtUrl) {
156 // Neither in island — nearby carry connection
157 connections.push({
158 source: srcUrl, target: tgtUrl,
159 connectionType: conn.relation,
160 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
161 });
162 }
163 }
164
165 // Deduplicate
166 const seen = new Set<string>();
167 return connections.filter(c => {
168 const key = `${c.source}|${c.target}`;
169 if (seen.has(key)) return false;
170 seen.add(key);
171 return true;
172 }).slice(0, 50);
173}
174
175function findHotEdges(
176 db: Database,
177 islandUrls: Set<string>,
178 newConnections: Array<{ source: string; target: string }>,
179): string[] {
180 const highlights: string[] = [];
181
182 const newEndpoints = new Set<string>();
183 for (const nc of newConnections) {
184 newEndpoints.add(nc.source);
185 newEndpoints.add(nc.target);
186 }
187
188 // Get all edges touching this component
189 const edgeRows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
190 const scored: Array<{ source: string; target: string; score: number }> = [];
191
192 for (const row of edgeRows) {
193 if (!islandUrls.has(row.source) && !islandUrls.has(row.target)) continue;
194 let score = 0;
195 if (newEndpoints.has(row.source) || newEndpoints.has(row.target)) score += 3;
196 if (row.source.startsWith('https://') && row.target.startsWith('https://')) score += 1;
197 if (score > 0) scored.push({ source: row.source, target: row.target, score });
198 }
199
200 scored.sort((a, b) => b.score - a.score);
201 for (const s of scored.slice(0, 10)) {
202 highlights.push(`${s.source} → ${s.target}`);
203 }
204
205 return highlights;
206}
207
208function findTensions(islandText: string, carryCitations: CarryCitation[]): string[] {
209 const tensions: string[] = [];
210 const lower = islandText.toLowerCase();
211
212 for (const c of carryCitations) {
213 if (!c.takeaway) continue;
214 const t = c.takeaway.toLowerCase();
215 if (t.includes('contradict') || t.includes('however') || t.includes('tension') || t.includes('but')) {
216 // Check if citation URL is in the island
217 if ((c.url && lower.includes(c.url.toLowerCase())) || (c.domain && lower.includes(c.domain.toLowerCase()))) {
218 tensions.push(c.takeaway.slice(0, 500));
219 }
220 }
221 }
222
223 return tensions.slice(0, 5);
224}
225
226function generateOpenQuestions(
227 nodeCount: number,
228 edgeCount: number,
229 themes: string[],
230 tensions: string[],
231 newConnections: Array<{ source: string; target: string }>,
232): string[] {
233 const questions: string[] = [];
234
235 if (newConnections.length > 0) {
236 questions.push(`${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`);
237 }
238
239 if (tensions.length > 0) {
240 questions.push(`What evidence would resolve the ${tensions.length} tension(s) identified?`);
241 }
242
243 const edgeTypes = new Set(newConnections.map(c => c.connectionType));
244 if (edgeTypes.size > 1) {
245 questions.push(`This island has ${edgeTypes.size} connection types. What's the dominant relationship?`);
246 }
247
248 if (nodeCount > 5) {
249 questions.push(`This is a ${nodeCount}-vertex cluster. Are there sub-clusters within it?`);
250 }
251
252 if (themes.includes('doi') || themes.includes('open science')) {
253 questions.push('What data artifacts does this cluster produce, and are they in Zenodo?');
254 }
255
256 return questions.slice(0, 5);
257}
258
259function synthesize(
260 nodeCount: number,
261 edgeCount: number,
262 didCount: number,
263 themes: string[],
264 highlights: string[],
265 tensions: string[],
266 openQuestions: string[],
267 newConnections: Array<{ source: string; target: string }>,
268): string {
269 const clusterDesc = `${nodeCount} vertices, ${edgeCount} edges, ${didCount} DIDs`;
270 const themeStr = themes.length > 0 ? `Themes: ${themes.join(', ')}.` : '';
271 const touchpointStr = newConnections.length > 0
272 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
273 : 'No carry touchpoints found.';
274 const highlightStr = highlights.length > 0
275 ? `${highlights.length} existing edges flagged as semantically significant.`
276 : '';
277 const tensionStr = tensions.length > 0
278 ? `There ${tensions.length === 1 ? 'is 1 tension' : `are ${tensions.length} tensions`} with your existing claims.`
279 : '';
280 const questionStr = openQuestions.length > 0
281 ? `Open questions: ${openQuestions.join(' ')}`
282 : '';
283
284 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(' ');
285}
286
287// --- Analyze a single component ---
288
289function analyzeComponent(
290 db: Database,
291 comp: Component,
292 carryConnections: CarryConnection[],
293 entityUrlIndex: Map<string, string>,
294 carryCitations: CarryCitation[],
295): AnalysisResult {
296 const islandUrls = comp.nodes;
297
298 // Build text corpus from node URLs for theme extraction
299 const islandText = [...islandUrls].join(' ');
300
301 // Extract themes
302 const themes = extractThemes(islandText);
303
304 // Match carry connections
305 const newConnections = matchCarryConnections(islandUrls, carryConnections, entityUrlIndex);
306
307 // Find hot edges
308 const highlights = findHotEdges(db, islandUrls, newConnections);
309
310 // Find tensions
311 const tensions = findTensions(islandText, carryCitations);
312
313 // Count edges for this component
314 const allEdges = db.query('SELECT source, target FROM edges').all() as Array<{ source: string; target: string }>;
315 const edgeCount = allEdges.filter(e => islandUrls.has(e.source) || islandUrls.has(e.target)).length;
316
317 // Open questions
318 const openQuestions = generateOpenQuestions(islandUrls.size, edgeCount, themes, tensions, newConnections);
319
320 // Synthesis
321 const synthesis = synthesize(
322 islandUrls.size, edgeCount, comp.dids.size,
323 themes, highlights, tensions, openQuestions, newConnections,
324 );
325
326 // Title: first theme or first few words of synthesis
327 const title = themes.length > 0
328 ? themes.slice(0, 3).map(t => t.charAt(0).toUpperCase() + t.slice(1)).join(', ')
329 : synthesis.slice(0, 200);
330
331 return {
332 islandId: comp.islandId,
333 lexmin: comp.lexmin,
334 title,
335 themes,
336 highlights,
337 tensions,
338 openQuestions,
339 synthesis,
340 newConnections,
341 analyzedAt: new Date().toISOString(),
342 };
343}
344
345// --- Save analysis to SQLite ---
346
347function saveAnalysis(db: Database, result: AnalysisResult): void {
348 db.query(`
349 INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
350 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
351 ON CONFLICT(island_id) DO UPDATE SET
352 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
353 tensions=excluded.tensions, open_questions=excluded.open_questions,
354 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
355 `).run(
356 result.islandId,
357 result.title,
358 JSON.stringify(result.themes),
359 JSON.stringify(result.highlights),
360 JSON.stringify(result.tensions),
361 JSON.stringify(result.openQuestions),
362 result.synthesis,
363 JSON.stringify(result.newConnections),
364 result.analyzedAt,
365 );
366}
367
368// --- Main ---
369
370async function main(): Promise<void> {
371 const args = process.argv.slice(2);
372 const wantJson = args.includes('--json');
373 const force = args.includes('--force');
374 const analyzeAll = args.includes('--all');
375 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
376 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
377
378 // Get island hash from args
379 const targetId = args.find((a) => !a.startsWith('--'));
380
381 if (!targetId && !analyzeAll) {
382 console.error('Usage: bun island:analyze <island-id> [--force] [--json]');
383 console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
384 process.exit(1);
385 }
386
387 let db: Database;
388 try {
389 db = openDb();
390 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
391 if (count === 0) {
392 console.error('No edges found. Run `bun island:discover` first.');
393 process.exit(1);
394 }
395 } catch {
396 console.error('Database not found. Run `bun island:discover` first.');
397 process.exit(1);
398 }
399
400 console.error('Loading carry data...');
401 const carryConnections = carryQueryJson<CarryConnection>('org.latha.connection', ['source', 'target', 'relation', 'context']);
402 const carryCitations = carryQueryJson<CarryCitation>('org.latha.citation', ['url', 'title', 'takeaway', 'domain']);
403 const entityUrlIndex = buildEntityUrlIndex();
404 console.error(`Carry: ${carryConnections.length} connections, ${carryCitations.length} citations, ${entityUrlIndex.size} entity URLs`);
405
406 console.error('Finding components...');
407 const components = findComponents(db);
408
409 let targets: Component[];
410 if (analyzeAll) {
411 targets = components.filter((c) => c.nodes.size >= minSize);
412 } else {
413 const comp = components.find((c) => c.islandId === targetId);
414 if (!comp) {
415 console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`);
416 process.exit(1);
417 }
418 targets = [comp];
419 }
420
421 console.error(`Analyzing ${targets.length} island(s)...`);
422
423 const results: AnalysisResult[] = [];
424
425 for (const comp of targets) {
426 // Check if already analyzed
427 if (!force) {
428 const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null;
429 if (existing) {
430 console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
431 continue;
432 }
433 }
434
435 console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
436 const result = analyzeComponent(db, comp, carryConnections, entityUrlIndex, carryCitations);
437 saveAnalysis(db, result);
438 results.push(result);
439 console.error(` Themes: ${result.themes.join(', ') || 'none'}`);
440 console.error(` New connections: ${result.newConnections.length}`);
441 console.error(` Tensions: ${result.tensions.length}`);
442 }
443
444 if (wantJson) {
445 console.log(JSON.stringify(results, null, 2));
446 db.close();
447 return;
448 }
449
450 for (const r of results) {
451 console.log(`\n${r.islandId}: ${r.title}`);
452 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
453 console.log(` New connections: ${r.newConnections.length}`);
454 if (r.newConnections.length > 0) {
455 for (const nc of r.newConnections.slice(0, 10)) {
456 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
457 }
458 if (r.newConnections.length > 10) {
459 console.log(` ... and ${r.newConnections.length - 10} more`);
460 }
461 }
462 console.log(` Tensions: ${r.tensions.length}`);
463 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
464 console.log(` Highlights: ${r.highlights.length}`);
465 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}...`);
466 }
467
468 db.close();
469}
470
471main().catch((err) => {
472 console.error('Fatal:', err instanceof Error ? err.message : String(err));
473 process.exit(1);
474});