/** * island:analyze * * Analyze a discovered component against carry data. * Writes analysis results to SQLite (not PDS — use publish for that). * * Usage: * bun island:analyze a1b2c3d4e5f6 # analyze by island hash * bun island:analyze a1b2c3d4e5f6 --force # re-analyze even if cached * bun island:analyze --all # analyze all components * bun island:analyze --all --min-size 5 # only components with >= 5 nodes * bun island:analyze a1b2c3d4e5f6 --json # machine-readable output */ import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; import { Database } from 'bun:sqlite'; import { loadDotEnv } from './cli-utils.js'; import { findComponents, islandId, domainFromUrl, type Component } from './island-shared.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); await loadDotEnv(resolve(__dirname, '..', '.env')); const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault'; const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db'); // --- Types --- interface CarryEntity { name: string; url: string; description: string } interface CarryCitation { url: string; title: string; takeaway: string; domain: string } interface CarryConnection { source: string; target: string; relation: string; context: string } interface AnalysisResult { islandId: string; lexmin: string; title: string; themes: string[]; highlights: string[]; tensions: string[]; openQuestions: string[]; synthesis: string; newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>; analyzedAt: string; } // --- SQLite --- function openDb(): Database { const db = new Database(DB_PATH, { create: true }); db.exec('PRAGMA journal_mode = WAL'); db.exec('PRAGMA synchronous = NORMAL'); db.exec(` CREATE TABLE IF NOT EXISTS analyses ( island_id TEXT PRIMARY KEY, title TEXT, themes TEXT, highlights TEXT, tensions TEXT, open_questions TEXT, synthesis TEXT, new_connections TEXT, analyzed_at TEXT NOT NULL ); `); return db; } // --- Carry data --- function carryQueryJson(domain: string, fields: string[]): T[] { try { const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], { cwd: CARRY_DIR, encoding: 'utf-8', timeout: 15_000, }); return JSON.parse(out) as T[]; } catch { return []; } } function buildEntityUrlIndex(): Map { const entities = carryQueryJson('org.latha.entity', ['name', 'url']); const index = new Map(); for (const e of entities) { if (e.name && e.url?.startsWith('https://')) { index.set(e.name, e.url); } } return index; } // --- Analysis --- const THEME_KEYWORDS: Record = { 'open science': ['open science', 'open access', 'oa', 'preprint'], 'data sharing': ['data sharing', 'shared data', 'open data', 'dataset'], 'reproducibility': ['reproducib', 'replicab', 'replication'], 'decentralization': ['decentraliz', 'federat', 'self-host'], 'knowledge graphs': ['knowledge graph', 'ontolog', 'linked data', 'semantic'], 'ai/ml': ['machine learning', 'deep learning', 'neural', 'llm', 'gpt', 'claude'], 'community': ['community', 'collective', 'cooperative', 'commons'], 'governance': ['governance', 'policy', 'regulation', 'standards'], 'infrastructure': ['infrastructure', 'platform', 'tooling', 'pipeline'], 'stigmergy': ['stigmerg', 'pheromone', 'emergent', 'self-organiz'], 'provenance': ['provenance', 'lineage', 'traceability', 'attribution'], 'doi': ['doi', 'zenodo', 'citeable', 'citation'], 'privacy': ['privacy', 'encryption', 'e2ee', 'consent'], 'sovereignty': ['sovereignty', 'self-determin', 'autonomy', 'agency'], 'at protocol': ['atproto', 'at protocol', 'bluesky', 'pds', 'appview'], 'semantic web': ['semantic web', 'rdf', 'sparql', 'linked data'], }; function extractThemes(text: string): string[] { const lower = text.toLowerCase(); return Object.entries(THEME_KEYWORDS) .filter(([_, terms]) => terms.some(t => lower.includes(t))) .map(([theme]) => theme); } function matchCarryConnections( islandUrls: Set, carryConnections: CarryConnection[], entityUrlIndex: Map, ): Array<{ source: string; target: string; connectionType: string; note: string }> { const connections: Array<{ source: string; target: string; connectionType: string; note: string }> = []; for (const conn of carryConnections) { const srcUrl = entityUrlIndex.get(conn.source) || (conn.source.startsWith('https://') ? conn.source : ''); const tgtUrl = entityUrlIndex.get(conn.target) || (conn.target.startsWith('https://') ? conn.target : ''); if (!srcUrl && !tgtUrl) continue; const srcInIsland = srcUrl && islandUrls.has(srcUrl); const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl); // Both in island → already connected if (srcInIsland && tgtInIsland) continue; // One side in island → bridge connection if (srcInIsland && tgtUrl) { connections.push({ source: srcUrl, target: tgtUrl, connectionType: conn.relation, note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`, }); } else if (tgtInIsland && srcUrl) { connections.push({ source: tgtUrl, target: srcUrl, connectionType: conn.relation, note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`, }); } else if (srcUrl && tgtUrl) { // Neither in island — nearby carry connection connections.push({ source: srcUrl, target: tgtUrl, connectionType: conn.relation, note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`, }); } } // Deduplicate const seen = new Set(); return connections.filter(c => { const key = `${c.source}|${c.target}`; if (seen.has(key)) return false; seen.add(key); return true; }).slice(0, 50); } function findHotEdges( db: Database, islandUrls: Set, newConnections: Array<{ source: string; target: string }>, ): string[] { const highlights: string[] = []; const newEndpoints = new Set(); for (const nc of newConnections) { newEndpoints.add(nc.source); newEndpoints.add(nc.target); } // Get all edges touching this component const edgeRows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>; const scored: Array<{ source: string; target: string; score: number }> = []; for (const row of edgeRows) { if (!islandUrls.has(row.source) && !islandUrls.has(row.target)) continue; let score = 0; if (newEndpoints.has(row.source) || newEndpoints.has(row.target)) score += 3; if (row.source.startsWith('https://') && row.target.startsWith('https://')) score += 1; if (score > 0) scored.push({ source: row.source, target: row.target, score }); } scored.sort((a, b) => b.score - a.score); for (const s of scored.slice(0, 10)) { highlights.push(`${s.source} → ${s.target}`); } return highlights; } function findTensions(islandText: string, carryCitations: CarryCitation[]): string[] { const tensions: string[] = []; const lower = islandText.toLowerCase(); for (const c of carryCitations) { if (!c.takeaway) continue; const t = c.takeaway.toLowerCase(); if (t.includes('contradict') || t.includes('however') || t.includes('tension') || t.includes('but')) { // Check if citation URL is in the island if ((c.url && lower.includes(c.url.toLowerCase())) || (c.domain && lower.includes(c.domain.toLowerCase()))) { tensions.push(c.takeaway.slice(0, 500)); } } } return tensions.slice(0, 5); } function generateOpenQuestions( nodeCount: number, edgeCount: number, themes: string[], tensions: string[], newConnections: Array<{ source: string; target: string }>, ): string[] { const questions: string[] = []; if (newConnections.length > 0) { questions.push(`${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`); } if (tensions.length > 0) { questions.push(`What evidence would resolve the ${tensions.length} tension(s) identified?`); } const edgeTypes = new Set(newConnections.map(c => c.connectionType)); if (edgeTypes.size > 1) { questions.push(`This island has ${edgeTypes.size} connection types. What's the dominant relationship?`); } if (nodeCount > 5) { questions.push(`This is a ${nodeCount}-vertex cluster. Are there sub-clusters within it?`); } if (themes.includes('doi') || themes.includes('open science')) { questions.push('What data artifacts does this cluster produce, and are they in Zenodo?'); } return questions.slice(0, 5); } function synthesize( nodeCount: number, edgeCount: number, didCount: number, themes: string[], highlights: string[], tensions: string[], openQuestions: string[], newConnections: Array<{ source: string; target: string }>, ): string { const clusterDesc = `${nodeCount} vertices, ${edgeCount} edges, ${didCount} DIDs`; const themeStr = themes.length > 0 ? `Themes: ${themes.join(', ')}.` : ''; const touchpointStr = newConnections.length > 0 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.` : 'No carry touchpoints found.'; const highlightStr = highlights.length > 0 ? `${highlights.length} existing edges flagged as semantically significant.` : ''; const tensionStr = tensions.length > 0 ? `There ${tensions.length === 1 ? 'is 1 tension' : `are ${tensions.length} tensions`} with your existing claims.` : ''; const questionStr = openQuestions.length > 0 ? `Open questions: ${openQuestions.join(' ')}` : ''; return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(' '); } // --- Analyze a single component --- function analyzeComponent( db: Database, comp: Component, carryConnections: CarryConnection[], entityUrlIndex: Map, carryCitations: CarryCitation[], ): AnalysisResult { const islandUrls = comp.nodes; // Build text corpus from node URLs for theme extraction const islandText = [...islandUrls].join(' '); // Extract themes const themes = extractThemes(islandText); // Match carry connections const newConnections = matchCarryConnections(islandUrls, carryConnections, entityUrlIndex); // Find hot edges const highlights = findHotEdges(db, islandUrls, newConnections); // Find tensions const tensions = findTensions(islandText, carryCitations); // Count edges for this component const allEdges = db.query('SELECT source, target FROM edges').all() as Array<{ source: string; target: string }>; const edgeCount = allEdges.filter(e => islandUrls.has(e.source) || islandUrls.has(e.target)).length; // Open questions const openQuestions = generateOpenQuestions(islandUrls.size, edgeCount, themes, tensions, newConnections); // Synthesis const synthesis = synthesize( islandUrls.size, edgeCount, comp.dids.size, themes, highlights, tensions, openQuestions, newConnections, ); // Title: first theme or first few words of synthesis const title = themes.length > 0 ? themes.slice(0, 3).map(t => t.charAt(0).toUpperCase() + t.slice(1)).join(', ') : synthesis.slice(0, 200); return { islandId: comp.islandId, lexmin: comp.lexmin, title, themes, highlights, tensions, openQuestions, synthesis, newConnections, analyzedAt: new Date().toISOString(), }; } // --- Save analysis to SQLite --- function saveAnalysis(db: Database, result: AnalysisResult): void { db.query(` INSERT INTO analyses (island_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(island_id) DO UPDATE SET title=excluded.title, themes=excluded.themes, highlights=excluded.highlights, tensions=excluded.tensions, open_questions=excluded.open_questions, synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at `).run( result.islandId, result.title, JSON.stringify(result.themes), JSON.stringify(result.highlights), JSON.stringify(result.tensions), JSON.stringify(result.openQuestions), result.synthesis, JSON.stringify(result.newConnections), result.analyzedAt, ); } // --- Main --- async function main(): Promise { const args = process.argv.slice(2); const wantJson = args.includes('--json'); const force = args.includes('--force'); const analyzeAll = args.includes('--all'); const minSizeArg = args.find((a) => a.startsWith('--min-size=')); const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3; // Get island hash from args const targetId = args.find((a) => !a.startsWith('--')); if (!targetId && !analyzeAll) { console.error('Usage: bun island:analyze [--force] [--json]'); console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]'); process.exit(1); } let db: Database; try { db = openDb(); const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0; if (count === 0) { console.error('No edges found. Run `bun island:discover` first.'); process.exit(1); } } catch { console.error('Database not found. Run `bun island:discover` first.'); process.exit(1); } console.error('Loading carry data...'); const carryConnections = carryQueryJson('org.latha.connection', ['source', 'target', 'relation', 'context']); const carryCitations = carryQueryJson('org.latha.citation', ['url', 'title', 'takeaway', 'domain']); const entityUrlIndex = buildEntityUrlIndex(); console.error(`Carry: ${carryConnections.length} connections, ${carryCitations.length} citations, ${entityUrlIndex.size} entity URLs`); console.error('Finding components...'); const components = findComponents(db); let targets: Component[]; if (analyzeAll) { targets = components.filter((c) => c.nodes.size >= minSize); } else { const comp = components.find((c) => c.islandId === targetId); if (!comp) { console.error(`Island ${targetId} not found. Run \`bun island:discover\` to list available islands.`); process.exit(1); } targets = [comp]; } console.error(`Analyzing ${targets.length} island(s)...`); const results: AnalysisResult[] = []; for (const comp of targets) { // Check if already analyzed if (!force) { const existing = db.query('SELECT analyzed_at FROM analyses WHERE island_id = ?').get(comp.islandId) as { analyzed_at: string } | null; if (existing) { console.error(`${comp.islandId}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`); continue; } } console.error(`${comp.islandId}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`); const result = analyzeComponent(db, comp, carryConnections, entityUrlIndex, carryCitations); saveAnalysis(db, result); results.push(result); console.error(` Themes: ${result.themes.join(', ') || 'none'}`); console.error(` New connections: ${result.newConnections.length}`); console.error(` Tensions: ${result.tensions.length}`); } if (wantJson) { console.log(JSON.stringify(results, null, 2)); db.close(); return; } for (const r of results) { console.log(`\n${r.islandId}: ${r.title}`); console.log(` Themes: ${r.themes.join(', ') || 'none'}`); console.log(` New connections: ${r.newConnections.length}`); if (r.newConnections.length > 0) { for (const nc of r.newConnections.slice(0, 10)) { console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`); } if (r.newConnections.length > 10) { console.log(` ... and ${r.newConnections.length - 10} more`); } } console.log(` Tensions: ${r.tensions.length}`); for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`); console.log(` Highlights: ${r.highlights.length}`); console.log(` Synthesis: ${r.synthesis.slice(0, 300)}...`); } db.close(); } main().catch((err) => { console.error('Fatal:', err instanceof Error ? err.message : String(err)); process.exit(1); });