nandi.uk
This repository has no description
18 kB
524 lines
1/**
2 * island:analyze <component-id>
3 *
4 * Analyze a discovered component against carry data.
5 * Writes analysis results to SQLite (not PDS — use publish for that).
6 *
7 * Usage:
8 * bun island:analyze 1 # analyze component 1
9 * bun island:analyze 1 --force # re-analyze even if cached
10 * bun island:analyze --all # analyze all components
11 * bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12 * bun island:analyze 1 --json # machine-readable output
13 */
14
15import { resolve, dirname } from 'node:path';
16import { fileURLToPath } from 'node:url';
17import { execFileSync } from 'node:fs';
18import { Database } from 'bun:sqlite';
19import { loadDotEnv } from './cli-utils.js';
20
21const __dirname = dirname(fileURLToPath(import.meta.url));
22await loadDotEnv(resolve(__dirname, '..', '.env'));
23
24const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
25const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
26const COLLECTION = 'network.cosmik.connection';
27
28// --- Types ---
29
30interface Component {
31 id: number;
32 nodes: Set<string>;
33 dids: Set<string>;
34}
35
36interface CarryEntity { name: string; url: string; description: string }
37interface CarryCitation { url: string; title: string; takeaway: string; domain: string }
38interface CarryConnection { source: string; target: string; relation: string; context: string }
39
40interface AnalysisResult {
41 componentId: number;
42 title: string;
43 themes: string[];
44 highlights: string[];
45 tensions: string[];
46 openQuestions: string[];
47 synthesis: string;
48 newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
49 analyzedAt: string;
50}
51
52// --- SQLite ---
53
54function openDb(): Database {
55 const db = new Database(DB_PATH, { create: true });
56 db.exec('PRAGMA journal_mode = WAL');
57 db.exec('PRAGMA synchronous = NORMAL');
58 db.exec(`
59 CREATE TABLE IF NOT EXISTS analyses (
60 component_id INTEGER PRIMARY KEY,
61 title TEXT,
62 themes TEXT,
63 highlights TEXT,
64 tensions TEXT,
65 open_questions TEXT,
66 synthesis TEXT,
67 new_connections TEXT,
68 analyzed_at TEXT NOT NULL
69 );
70 `);
71 return db;
72}
73
74// --- Carry data ---
75
76function carryQueryJson<T>(domain: string, fields: string[]): T[] {
77 try {
78 const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
79 cwd: CARRY_DIR,
80 encoding: 'utf-8',
81 timeout: 15_000,
82 });
83 return JSON.parse(out) as T[];
84 } catch {
85 return [];
86 }
87}
88
89function buildEntityUrlIndex(): Map<string, string> {
90 const entities = carryQueryJson<CarryEntity>('org.latha.entity', ['name', 'url']);
91 const index = new Map<string, string>();
92 for (const e of entities) {
93 if (e.name && e.url?.startsWith('https://')) {
94 index.set(e.name, e.url);
95 }
96 }
97 return index;
98}
99
100// --- Connected components (from SQLite edges) ---
101
102function findComponents(db: Database): Component[] {
103 const adj = new Map<string, Set<string>>();
104 const edgeDids = new Map<string, Set<string>>();
105
106 const rows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
107 for (const row of rows) {
108 const s = row.source;
109 const t = row.target;
110 if (!adj.has(s)) adj.set(s, new Set());
111 if (!adj.has(t)) adj.set(t, new Set());
112 adj.get(s)!.add(t);
113 adj.get(t)!.add(s);
114 if (!edgeDids.has(s)) edgeDids.set(s, new Set());
115 if (!edgeDids.has(t)) edgeDids.set(t, new Set());
116 edgeDids.get(s)!.add(row.did);
117 edgeDids.get(t)!.add(row.did);
118 }
119
120 const visited = new Set<string>();
121 const components: Component[] = [];
122 let compId = 0;
123
124 for (const node of adj.keys()) {
125 if (visited.has(node)) continue;
126 compId++;
127 const compNodes = new Set<string>();
128 const compDids = new Set<string>();
129 const queue = [node];
130 while (queue.length > 0) {
131 const current = queue.pop()!;
132 if (visited.has(current)) continue;
133 visited.add(current);
134 compNodes.add(current);
135 for (const did of edgeDids.get(current) ?? []) compDids.add(did);
136 for (const neighbor of adj.get(current) ?? []) {
137 if (!visited.has(neighbor)) queue.push(neighbor);
138 }
139 }
140 components.push({ id: compId, nodes: compNodes, dids: compDids });
141 }
142
143 return components.sort((a, b) => b.nodes.size - a.nodes.size);
144}
145
146// --- Analysis ---
147
148const THEME_KEYWORDS: Record<string, string[]> = {
149 'open science': ['open science', 'open access', 'oa', 'preprint'],
150 'data sharing': ['data sharing', 'shared data', 'open data', 'dataset'],
151 'reproducibility': ['reproducib', 'replicab', 'replication'],
152 'decentralization': ['decentraliz', 'federat', 'self-host'],
153 'knowledge graphs': ['knowledge graph', 'ontolog', 'linked data', 'semantic'],
154 'ai/ml': ['machine learning', 'deep learning', 'neural', 'llm', 'gpt', 'claude'],
155 'community': ['community', 'collective', 'cooperative', 'commons'],
156 'governance': ['governance', 'policy', 'regulation', 'standards'],
157 'infrastructure': ['infrastructure', 'platform', 'tooling', 'pipeline'],
158 'stigmergy': ['stigmerg', 'pheromone', 'emergent', 'self-organiz'],
159 'provenance': ['provenance', 'lineage', 'traceability', 'attribution'],
160 'doi': ['doi', 'zenodo', 'citeable', 'citation'],
161 'privacy': ['privacy', 'encryption', 'e2ee', 'consent'],
162 'sovereignty': ['sovereignty', 'self-determin', 'autonomy', 'agency'],
163 'at protocol': ['atproto', 'at protocol', 'bluesky', 'pds', 'appview'],
164 'semantic web': ['semantic web', 'rdf', 'sparql', 'linked data'],
165};
166
167function extractThemes(text: string): string[] {
168 const lower = text.toLowerCase();
169 return Object.entries(THEME_KEYWORDS)
170 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
171 .map(([theme]) => theme);
172}
173
174function matchCarryConnections(
175 islandUrls: Set<string>,
176 carryConnections: CarryConnection[],
177 entityUrlIndex: Map<string, string>,
178): Array<{ source: string; target: string; connectionType: string; note: string }> {
179 const connections: Array<{ source: string; target: string; connectionType: string; note: string }> = [];
180
181 for (const conn of carryConnections) {
182 const srcUrl = entityUrlIndex.get(conn.source) || (conn.source.startsWith('https://') ? conn.source : '');
183 const tgtUrl = entityUrlIndex.get(conn.target) || (conn.target.startsWith('https://') ? conn.target : '');
184
185 if (!srcUrl && !tgtUrl) continue;
186
187 const srcInIsland = srcUrl && islandUrls.has(srcUrl);
188 const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
189
190 // Both in island → already connected
191 if (srcInIsland && tgtInIsland) continue;
192
193 // One side in island → bridge connection
194 if (srcInIsland && tgtUrl) {
195 connections.push({
196 source: srcUrl, target: tgtUrl,
197 connectionType: conn.relation,
198 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
199 });
200 } else if (tgtInIsland && srcUrl) {
201 connections.push({
202 source: tgtUrl, target: srcUrl,
203 connectionType: conn.relation,
204 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
205 });
206 } else if (srcUrl && tgtUrl) {
207 // Neither in island — nearby carry connection
208 connections.push({
209 source: srcUrl, target: tgtUrl,
210 connectionType: conn.relation,
211 note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
212 });
213 }
214 }
215
216 // Deduplicate
217 const seen = new Set<string>();
218 return connections.filter(c => {
219 const key = `${c.source}|${c.target}`;
220 if (seen.has(key)) return false;
221 seen.add(key);
222 return true;
223 }).slice(0, 50);
224}
225
226function findHotEdges(
227 db: Database,
228 islandUrls: Set<string>,
229 newConnections: Array<{ source: string; target: string }>,
230): string[] {
231 const highlights: string[] = [];
232
233 const newEndpoints = new Set<string>();
234 for (const nc of newConnections) {
235 newEndpoints.add(nc.source);
236 newEndpoints.add(nc.target);
237 }
238
239 // Get all edges touching this component
240 const edgeRows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
241 const scored: Array<{ source: string; target: string; score: number }> = [];
242
243 for (const row of edgeRows) {
244 if (!islandUrls.has(row.source) && !islandUrls.has(row.target)) continue;
245 let score = 0;
246 if (newEndpoints.has(row.source) || newEndpoints.has(row.target)) score += 3;
247 if (row.source.startsWith('https://') && row.target.startsWith('https://')) score += 1;
248 if (score > 0) scored.push({ source: row.source, target: row.target, score });
249 }
250
251 scored.sort((a, b) => b.score - a.score);
252 for (const s of scored.slice(0, 10)) {
253 highlights.push(`${s.source} → ${s.target}`);
254 }
255
256 return highlights;
257}
258
259function findTensions(islandText: string, carryCitations: CarryCitation[]): string[] {
260 const tensions: string[] = [];
261 const lower = islandText.toLowerCase();
262
263 for (const c of carryCitations) {
264 const t = c.takeaway.toLowerCase();
265 if (t.includes('contradict') || t.includes('however') || t.includes('tension') || t.includes('but')) {
266 // Check if citation URL is in the island
267 if (lower.includes(c.url.toLowerCase()) || lower.includes(c.domain.toLowerCase())) {
268 tensions.push(c.takeaway.slice(0, 500));
269 }
270 }
271 }
272
273 return tensions.slice(0, 5);
274}
275
276function generateOpenQuestions(
277 nodeCount: number,
278 edgeCount: number,
279 themes: string[],
280 tensions: string[],
281 newConnections: Array<{ source: string; target: string }>,
282): string[] {
283 const questions: string[] = [];
284
285 if (newConnections.length > 0) {
286 questions.push(`${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`);
287 }
288
289 if (tensions.length > 0) {
290 questions.push(`What evidence would resolve the ${tensions.length} tension(s) identified?`);
291 }
292
293 const edgeTypes = new Set(newConnections.map(c => c.connectionType));
294 if (edgeTypes.size > 1) {
295 questions.push(`This island has ${edgeTypes.size} connection types. What's the dominant relationship?`);
296 }
297
298 if (nodeCount > 5) {
299 questions.push(`This is a ${nodeCount}-vertex cluster. Are there sub-clusters within it?`);
300 }
301
302 if (themes.includes('doi') || themes.includes('open science')) {
303 questions.push('What data artifacts does this cluster produce, and are they in Zenodo?');
304 }
305
306 return questions.slice(0, 5);
307}
308
309function synthesize(
310 nodeCount: number,
311 edgeCount: number,
312 didCount: number,
313 themes: string[],
314 highlights: string[],
315 tensions: string[],
316 openQuestions: string[],
317 newConnections: Array<{ source: string; target: string }>,
318): string {
319 const clusterDesc = `${nodeCount} vertices, ${edgeCount} edges, ${didCount} DIDs`;
320 const themeStr = themes.length > 0 ? `Themes: ${themes.join(', ')}.` : '';
321 const touchpointStr = newConnections.length > 0
322 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
323 : 'No carry touchpoints found.';
324 const highlightStr = highlights.length > 0
325 ? `${highlights.length} existing edges flagged as semantically significant.`
326 : '';
327 const tensionStr = tensions.length > 0
328 ? `There ${tensions.length === 1 ? 'is 1 tension' : `are ${tensions.length} tensions`} with your existing claims.`
329 : '';
330 const questionStr = openQuestions.length > 0
331 ? `Open questions: ${openQuestions.join(' ')}`
332 : '';
333
334 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(' ');
335}
336
337// --- Analyze a single component ---
338
339function analyzeComponent(
340 db: Database,
341 comp: Component,
342 carryConnections: CarryConnection[],
343 entityUrlIndex: Map<string, string>,
344 carryCitations: CarryCitation[],
345): AnalysisResult {
346 const islandUrls = comp.nodes;
347
348 // Build text corpus from node URLs for theme extraction
349 const islandText = [...islandUrls].join(' ');
350
351 // Extract themes
352 const themes = extractThemes(islandText);
353
354 // Match carry connections
355 const newConnections = matchCarryConnections(islandUrls, carryConnections, entityUrlIndex);
356
357 // Find hot edges
358 const highlights = findHotEdges(db, islandUrls, newConnections);
359
360 // Find tensions
361 const tensions = findTensions(islandText, carryCitations);
362
363 // Count edges for this component
364 const allEdges = db.query('SELECT source, target FROM edges').all() as Array<{ source: string; target: string }>;
365 const edgeCount = allEdges.filter(e => islandUrls.has(e.source) || islandUrls.has(e.target)).length;
366
367 // Open questions
368 const openQuestions = generateOpenQuestions(islandUrls.size, edgeCount, themes, tensions, newConnections);
369
370 // Synthesis
371 const synthesis = synthesize(
372 islandUrls.size, edgeCount, comp.dids.size,
373 themes, highlights, tensions, openQuestions, newConnections,
374 );
375
376 // Title: first theme or first few words of synthesis
377 const title = themes.length > 0
378 ? themes.slice(0, 3).map(t => t.charAt(0).toUpperCase() + t.slice(1)).join(', ')
379 : synthesis.slice(0, 200);
380
381 return {
382 componentId: comp.id,
383 title,
384 themes,
385 highlights,
386 tensions,
387 openQuestions,
388 synthesis,
389 newConnections,
390 analyzedAt: new Date().toISOString(),
391 };
392}
393
394// --- Save analysis to SQLite ---
395
396function saveAnalysis(db: Database, result: AnalysisResult): void {
397 db.query(`
398 INSERT INTO analyses (component_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
399 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
400 ON CONFLICT(component_id) DO UPDATE SET
401 title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
402 tensions=excluded.tensions, open_questions=excluded.open_questions,
403 synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
404 `).run(
405 result.componentId,
406 result.title,
407 JSON.stringify(result.themes),
408 JSON.stringify(result.highlights),
409 JSON.stringify(result.tensions),
410 JSON.stringify(result.openQuestions),
411 result.synthesis,
412 JSON.stringify(result.newConnections),
413 result.analyzedAt,
414 );
415}
416
417// --- Main ---
418
419async function main(): Promise<void> {
420 const args = process.argv.slice(2);
421 const wantJson = args.includes('--json');
422 const force = args.includes('--force');
423 const analyzeAll = args.includes('--all');
424 const minSizeArg = args.find((a) => a.startsWith('--min-size='));
425 const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
426
427 // Get component ID from args
428 const compArg = args.find((a) => !a.startsWith('--') && /^\d+$/.test(a));
429 const compId = compArg ? Number(compArg) : null;
430
431 if (!compId && !analyzeAll) {
432 console.error('Usage: bun island:analyze <component-id> [--force] [--json]');
433 console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
434 process.exit(1);
435 }
436
437 let db: Database;
438 try {
439 db = openDb();
440 const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
441 if (count === 0) {
442 console.error('No edges found. Run `bun island:discover` first.');
443 process.exit(1);
444 }
445 } catch {
446 console.error('Database not found. Run `bun island:discover` first.');
447 process.exit(1);
448 }
449
450 console.error('Loading carry data...');
451 const carryConnections = carryQueryJson<CarryConnection>('org.latha.connection', ['source', 'target', 'relation', 'context']);
452 const carryCitations = carryQueryJson<CarryCitation>('org.latha.citation', ['url', 'title', 'takeaway', 'domain']);
453 const entityUrlIndex = buildEntityUrlIndex();
454 console.error(`Carry: ${carryConnections.length} connections, ${carryCitations.length} citations, ${entityUrlIndex.size} entity URLs`);
455
456 console.error('Finding components...');
457 const components = findComponents(db);
458
459 let targets: Component[];
460 if (analyzeAll) {
461 targets = components.filter((c) => c.nodes.size >= minSize);
462 } else {
463 const comp = components.find((c) => c.id === compId);
464 if (!comp) {
465 console.error(`Component ${compId} not found. Available: ${components.slice(0, 10).map((c) => c.id).join(', ')}...`);
466 process.exit(1);
467 }
468 targets = [comp];
469 }
470
471 console.error(`Analyzing ${targets.length} component(s)...`);
472
473 const results: AnalysisResult[] = [];
474
475 for (const comp of targets) {
476 // Check if already analyzed
477 if (!force) {
478 const existing = db.query('SELECT analyzed_at FROM analyses WHERE component_id = ?').get(comp.id) as { analyzed_at: string } | null;
479 if (existing) {
480 console.error(`Component ${comp.id}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
481 continue;
482 }
483 }
484
485 console.error(`Component ${comp.id}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
486 const result = analyzeComponent(db, comp, carryConnections, entityUrlIndex, carryCitations);
487 saveAnalysis(db, result);
488 results.push(result);
489 console.error(` Themes: ${result.themes.join(', ') || 'none'}`);
490 console.error(` New connections: ${result.newConnections.length}`);
491 console.error(` Tensions: ${result.tensions.length}`);
492 }
493
494 if (wantJson) {
495 console.log(JSON.stringify(results, null, 2));
496 db.close();
497 return;
498 }
499
500 for (const r of results) {
501 console.log(`\nComponent ${r.componentId}: ${r.title}`);
502 console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
503 console.log(` New connections: ${r.newConnections.length}`);
504 if (r.newConnections.length > 0) {
505 for (const nc of r.newConnections.slice(0, 10)) {
506 console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
507 }
508 if (r.newConnections.length > 10) {
509 console.log(` ... and ${r.newConnections.length - 10} more`);
510 }
511 }
512 console.log(` Tensions: ${r.tensions.length}`);
513 for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
514 console.log(` Highlights: ${r.highlights.length}`);
515 console.log(` Synthesis: ${r.synthesis.slice(0, 300)}...`);
516 }
517
518 db.close();
519}
520
521main().catch((err) => {
522 console.error('Fatal:', err instanceof Error ? err.message : String(err));
523 process.exit(1);
524});