nandi.uk
This repository has no description
Add island:analyze — analyze discovered components against carry data
Runs carry-backed analysis on connected components found by island:discover:
- Matches carry connections against island URLs to find bridge edges
- Extracts themes from island text corpus
- Flags hot edges connected to carry touchpoints
- Finds tensions from carry citations
- Generates synthesis and open questions
- Writes results to SQLite (not PDS — use island:publish for that)
Usage: bun island:analyze 1 | bun island:analyze --all
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
nandi
co-author
Letta Code
date
(May 15, 2026, 4:29 AM UTC)
commit
a497e137
a497e1377e8a36c71bbe4a85a533d572207fb19a
parent
2013d82e
2013d82e8271dee3b592b21465bec7d29a57e78a
+526
+2
package.json
+2
package.json
···
11
11
"island:explore": "bun run src/island-explore.ts",
12
12
"island:detect": "bun run src/island-detect.ts",
13
13
"island:discover": "bun run src/island-discover.ts",
14
+
"island:analyze": "bun run src/island-analyze.ts",
14
15
"connection:detect": "bun run src/connection-detect.ts",
16
+
"island:publish": "bun run src/island-publish.ts",
15
17
"vault:analyze": "bun run src/vault-analyze.ts"
16
18
},
17
19
"dependencies": {
+524
src/island-analyze.ts
+524
src/island-analyze.ts
···
1
+
/**
2
+
* island:analyze <component-id>
3
+
*
4
+
* Analyze a discovered component against carry data.
5
+
* Writes analysis results to SQLite (not PDS — use publish for that).
6
+
*
7
+
* Usage:
8
+
* bun island:analyze 1 # analyze component 1
9
+
* bun island:analyze 1 --force # re-analyze even if cached
10
+
* bun island:analyze --all # analyze all components
11
+
* bun island:analyze --all --min-size 5 # only components with >= 5 nodes
12
+
* bun island:analyze 1 --json # machine-readable output
13
+
*/
14
+
15
+
import { resolve, dirname } from 'node:path';
16
+
import { fileURLToPath } from 'node:url';
17
+
import { execFileSync } from 'node:fs';
18
+
import { Database } from 'bun:sqlite';
19
+
import { loadDotEnv } from './cli-utils.js';
20
+
21
+
const __dirname = dirname(fileURLToPath(import.meta.url));
22
+
await loadDotEnv(resolve(__dirname, '..', '.env'));
23
+
24
+
const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
25
+
const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db');
26
+
const COLLECTION = 'network.cosmik.connection';
27
+
28
+
// --- Types ---
29
+
30
+
interface Component {
31
+
id: number;
32
+
nodes: Set<string>;
33
+
dids: Set<string>;
34
+
}
35
+
36
+
interface CarryEntity { name: string; url: string; description: string }
37
+
interface CarryCitation { url: string; title: string; takeaway: string; domain: string }
38
+
interface CarryConnection { source: string; target: string; relation: string; context: string }
39
+
40
+
interface AnalysisResult {
41
+
componentId: number;
42
+
title: string;
43
+
themes: string[];
44
+
highlights: string[];
45
+
tensions: string[];
46
+
openQuestions: string[];
47
+
synthesis: string;
48
+
newConnections: Array<{ source: string; target: string; connectionType: string; note: string }>;
49
+
analyzedAt: string;
50
+
}
51
+
52
+
// --- SQLite ---
53
+
54
+
function openDb(): Database {
55
+
const db = new Database(DB_PATH, { create: true });
56
+
db.exec('PRAGMA journal_mode = WAL');
57
+
db.exec('PRAGMA synchronous = NORMAL');
58
+
db.exec(`
59
+
CREATE TABLE IF NOT EXISTS analyses (
60
+
component_id INTEGER PRIMARY KEY,
61
+
title TEXT,
62
+
themes TEXT,
63
+
highlights TEXT,
64
+
tensions TEXT,
65
+
open_questions TEXT,
66
+
synthesis TEXT,
67
+
new_connections TEXT,
68
+
analyzed_at TEXT NOT NULL
69
+
);
70
+
`);
71
+
return db;
72
+
}
73
+
74
+
// --- Carry data ---
75
+
76
+
function carryQueryJson<T>(domain: string, fields: string[]): T[] {
77
+
try {
78
+
const out = execFileSync('carry', ['query', domain, ...fields, '--format', 'json'], {
79
+
cwd: CARRY_DIR,
80
+
encoding: 'utf-8',
81
+
timeout: 15_000,
82
+
});
83
+
return JSON.parse(out) as T[];
84
+
} catch {
85
+
return [];
86
+
}
87
+
}
88
+
89
+
function buildEntityUrlIndex(): Map<string, string> {
90
+
const entities = carryQueryJson<CarryEntity>('org.latha.entity', ['name', 'url']);
91
+
const index = new Map<string, string>();
92
+
for (const e of entities) {
93
+
if (e.name && e.url?.startsWith('https://')) {
94
+
index.set(e.name, e.url);
95
+
}
96
+
}
97
+
return index;
98
+
}
99
+
100
+
// --- Connected components (from SQLite edges) ---
101
+
102
+
function findComponents(db: Database): Component[] {
103
+
const adj = new Map<string, Set<string>>();
104
+
const edgeDids = new Map<string, Set<string>>();
105
+
106
+
const rows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
107
+
for (const row of rows) {
108
+
const s = row.source;
109
+
const t = row.target;
110
+
if (!adj.has(s)) adj.set(s, new Set());
111
+
if (!adj.has(t)) adj.set(t, new Set());
112
+
adj.get(s)!.add(t);
113
+
adj.get(t)!.add(s);
114
+
if (!edgeDids.has(s)) edgeDids.set(s, new Set());
115
+
if (!edgeDids.has(t)) edgeDids.set(t, new Set());
116
+
edgeDids.get(s)!.add(row.did);
117
+
edgeDids.get(t)!.add(row.did);
118
+
}
119
+
120
+
const visited = new Set<string>();
121
+
const components: Component[] = [];
122
+
let compId = 0;
123
+
124
+
for (const node of adj.keys()) {
125
+
if (visited.has(node)) continue;
126
+
compId++;
127
+
const compNodes = new Set<string>();
128
+
const compDids = new Set<string>();
129
+
const queue = [node];
130
+
while (queue.length > 0) {
131
+
const current = queue.pop()!;
132
+
if (visited.has(current)) continue;
133
+
visited.add(current);
134
+
compNodes.add(current);
135
+
for (const did of edgeDids.get(current) ?? []) compDids.add(did);
136
+
for (const neighbor of adj.get(current) ?? []) {
137
+
if (!visited.has(neighbor)) queue.push(neighbor);
138
+
}
139
+
}
140
+
components.push({ id: compId, nodes: compNodes, dids: compDids });
141
+
}
142
+
143
+
return components.sort((a, b) => b.nodes.size - a.nodes.size);
144
+
}
145
+
146
+
// --- Analysis ---
147
+
148
+
const THEME_KEYWORDS: Record<string, string[]> = {
149
+
'open science': ['open science', 'open access', 'oa', 'preprint'],
150
+
'data sharing': ['data sharing', 'shared data', 'open data', 'dataset'],
151
+
'reproducibility': ['reproducib', 'replicab', 'replication'],
152
+
'decentralization': ['decentraliz', 'federat', 'self-host'],
153
+
'knowledge graphs': ['knowledge graph', 'ontolog', 'linked data', 'semantic'],
154
+
'ai/ml': ['machine learning', 'deep learning', 'neural', 'llm', 'gpt', 'claude'],
155
+
'community': ['community', 'collective', 'cooperative', 'commons'],
156
+
'governance': ['governance', 'policy', 'regulation', 'standards'],
157
+
'infrastructure': ['infrastructure', 'platform', 'tooling', 'pipeline'],
158
+
'stigmergy': ['stigmerg', 'pheromone', 'emergent', 'self-organiz'],
159
+
'provenance': ['provenance', 'lineage', 'traceability', 'attribution'],
160
+
'doi': ['doi', 'zenodo', 'citeable', 'citation'],
161
+
'privacy': ['privacy', 'encryption', 'e2ee', 'consent'],
162
+
'sovereignty': ['sovereignty', 'self-determin', 'autonomy', 'agency'],
163
+
'at protocol': ['atproto', 'at protocol', 'bluesky', 'pds', 'appview'],
164
+
'semantic web': ['semantic web', 'rdf', 'sparql', 'linked data'],
165
+
};
166
+
167
+
function extractThemes(text: string): string[] {
168
+
const lower = text.toLowerCase();
169
+
return Object.entries(THEME_KEYWORDS)
170
+
.filter(([_, terms]) => terms.some(t => lower.includes(t)))
171
+
.map(([theme]) => theme);
172
+
}
173
+
174
+
function matchCarryConnections(
175
+
islandUrls: Set<string>,
176
+
carryConnections: CarryConnection[],
177
+
entityUrlIndex: Map<string, string>,
178
+
): Array<{ source: string; target: string; connectionType: string; note: string }> {
179
+
const connections: Array<{ source: string; target: string; connectionType: string; note: string }> = [];
180
+
181
+
for (const conn of carryConnections) {
182
+
const srcUrl = entityUrlIndex.get(conn.source) || (conn.source.startsWith('https://') ? conn.source : '');
183
+
const tgtUrl = entityUrlIndex.get(conn.target) || (conn.target.startsWith('https://') ? conn.target : '');
184
+
185
+
if (!srcUrl && !tgtUrl) continue;
186
+
187
+
const srcInIsland = srcUrl && islandUrls.has(srcUrl);
188
+
const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
189
+
190
+
// Both in island → already connected
191
+
if (srcInIsland && tgtInIsland) continue;
192
+
193
+
// One side in island → bridge connection
194
+
if (srcInIsland && tgtUrl) {
195
+
connections.push({
196
+
source: srcUrl, target: tgtUrl,
197
+
connectionType: conn.relation,
198
+
note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
199
+
});
200
+
} else if (tgtInIsland && srcUrl) {
201
+
connections.push({
202
+
source: tgtUrl, target: srcUrl,
203
+
connectionType: conn.relation,
204
+
note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
205
+
});
206
+
} else if (srcUrl && tgtUrl) {
207
+
// Neither in island — nearby carry connection
208
+
connections.push({
209
+
source: srcUrl, target: tgtUrl,
210
+
connectionType: conn.relation,
211
+
note: conn.context || `${conn.source} → ${conn.relation} → ${conn.target}`,
212
+
});
213
+
}
214
+
}
215
+
216
+
// Deduplicate
217
+
const seen = new Set<string>();
218
+
return connections.filter(c => {
219
+
const key = `${c.source}|${c.target}`;
220
+
if (seen.has(key)) return false;
221
+
seen.add(key);
222
+
return true;
223
+
}).slice(0, 50);
224
+
}
225
+
226
+
function findHotEdges(
227
+
db: Database,
228
+
islandUrls: Set<string>,
229
+
newConnections: Array<{ source: string; target: string }>,
230
+
): string[] {
231
+
const highlights: string[] = [];
232
+
233
+
const newEndpoints = new Set<string>();
234
+
for (const nc of newConnections) {
235
+
newEndpoints.add(nc.source);
236
+
newEndpoints.add(nc.target);
237
+
}
238
+
239
+
// Get all edges touching this component
240
+
const edgeRows = db.query('SELECT source, target, did FROM edges').all() as Array<{ source: string; target: string; did: string }>;
241
+
const scored: Array<{ source: string; target: string; score: number }> = [];
242
+
243
+
for (const row of edgeRows) {
244
+
if (!islandUrls.has(row.source) && !islandUrls.has(row.target)) continue;
245
+
let score = 0;
246
+
if (newEndpoints.has(row.source) || newEndpoints.has(row.target)) score += 3;
247
+
if (row.source.startsWith('https://') && row.target.startsWith('https://')) score += 1;
248
+
if (score > 0) scored.push({ source: row.source, target: row.target, score });
249
+
}
250
+
251
+
scored.sort((a, b) => b.score - a.score);
252
+
for (const s of scored.slice(0, 10)) {
253
+
highlights.push(`${s.source} → ${s.target}`);
254
+
}
255
+
256
+
return highlights;
257
+
}
258
+
259
+
function findTensions(islandText: string, carryCitations: CarryCitation[]): string[] {
260
+
const tensions: string[] = [];
261
+
const lower = islandText.toLowerCase();
262
+
263
+
for (const c of carryCitations) {
264
+
const t = c.takeaway.toLowerCase();
265
+
if (t.includes('contradict') || t.includes('however') || t.includes('tension') || t.includes('but')) {
266
+
// Check if citation URL is in the island
267
+
if (lower.includes(c.url.toLowerCase()) || lower.includes(c.domain.toLowerCase())) {
268
+
tensions.push(c.takeaway.slice(0, 500));
269
+
}
270
+
}
271
+
}
272
+
273
+
return tensions.slice(0, 5);
274
+
}
275
+
276
+
function generateOpenQuestions(
277
+
nodeCount: number,
278
+
edgeCount: number,
279
+
themes: string[],
280
+
tensions: string[],
281
+
newConnections: Array<{ source: string; target: string }>,
282
+
): string[] {
283
+
const questions: string[] = [];
284
+
285
+
if (newConnections.length > 0) {
286
+
questions.push(`${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`);
287
+
}
288
+
289
+
if (tensions.length > 0) {
290
+
questions.push(`What evidence would resolve the ${tensions.length} tension(s) identified?`);
291
+
}
292
+
293
+
const edgeTypes = new Set(newConnections.map(c => c.connectionType));
294
+
if (edgeTypes.size > 1) {
295
+
questions.push(`This island has ${edgeTypes.size} connection types. What's the dominant relationship?`);
296
+
}
297
+
298
+
if (nodeCount > 5) {
299
+
questions.push(`This is a ${nodeCount}-vertex cluster. Are there sub-clusters within it?`);
300
+
}
301
+
302
+
if (themes.includes('doi') || themes.includes('open science')) {
303
+
questions.push('What data artifacts does this cluster produce, and are they in Zenodo?');
304
+
}
305
+
306
+
return questions.slice(0, 5);
307
+
}
308
+
309
+
function synthesize(
310
+
nodeCount: number,
311
+
edgeCount: number,
312
+
didCount: number,
313
+
themes: string[],
314
+
highlights: string[],
315
+
tensions: string[],
316
+
openQuestions: string[],
317
+
newConnections: Array<{ source: string; target: string }>,
318
+
): string {
319
+
const clusterDesc = `${nodeCount} vertices, ${edgeCount} edges, ${didCount} DIDs`;
320
+
const themeStr = themes.length > 0 ? `Themes: ${themes.join(', ')}.` : '';
321
+
const touchpointStr = newConnections.length > 0
322
+
? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
323
+
: 'No carry touchpoints found.';
324
+
const highlightStr = highlights.length > 0
325
+
? `${highlights.length} existing edges flagged as semantically significant.`
326
+
: '';
327
+
const tensionStr = tensions.length > 0
328
+
? `There ${tensions.length === 1 ? 'is 1 tension' : `are ${tensions.length} tensions`} with your existing claims.`
329
+
: '';
330
+
const questionStr = openQuestions.length > 0
331
+
? `Open questions: ${openQuestions.join(' ')}`
332
+
: '';
333
+
334
+
return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(' ');
335
+
}
336
+
337
+
// --- Analyze a single component ---
338
+
339
+
function analyzeComponent(
340
+
db: Database,
341
+
comp: Component,
342
+
carryConnections: CarryConnection[],
343
+
entityUrlIndex: Map<string, string>,
344
+
carryCitations: CarryCitation[],
345
+
): AnalysisResult {
346
+
const islandUrls = comp.nodes;
347
+
348
+
// Build text corpus from node URLs for theme extraction
349
+
const islandText = [...islandUrls].join(' ');
350
+
351
+
// Extract themes
352
+
const themes = extractThemes(islandText);
353
+
354
+
// Match carry connections
355
+
const newConnections = matchCarryConnections(islandUrls, carryConnections, entityUrlIndex);
356
+
357
+
// Find hot edges
358
+
const highlights = findHotEdges(db, islandUrls, newConnections);
359
+
360
+
// Find tensions
361
+
const tensions = findTensions(islandText, carryCitations);
362
+
363
+
// Count edges for this component
364
+
const allEdges = db.query('SELECT source, target FROM edges').all() as Array<{ source: string; target: string }>;
365
+
const edgeCount = allEdges.filter(e => islandUrls.has(e.source) || islandUrls.has(e.target)).length;
366
+
367
+
// Open questions
368
+
const openQuestions = generateOpenQuestions(islandUrls.size, edgeCount, themes, tensions, newConnections);
369
+
370
+
// Synthesis
371
+
const synthesis = synthesize(
372
+
islandUrls.size, edgeCount, comp.dids.size,
373
+
themes, highlights, tensions, openQuestions, newConnections,
374
+
);
375
+
376
+
// Title: first theme or first few words of synthesis
377
+
const title = themes.length > 0
378
+
? themes.slice(0, 3).map(t => t.charAt(0).toUpperCase() + t.slice(1)).join(', ')
379
+
: synthesis.slice(0, 200);
380
+
381
+
return {
382
+
componentId: comp.id,
383
+
title,
384
+
themes,
385
+
highlights,
386
+
tensions,
387
+
openQuestions,
388
+
synthesis,
389
+
newConnections,
390
+
analyzedAt: new Date().toISOString(),
391
+
};
392
+
}
393
+
394
+
// --- Save analysis to SQLite ---
395
+
396
+
function saveAnalysis(db: Database, result: AnalysisResult): void {
397
+
db.query(`
398
+
INSERT INTO analyses (component_id, title, themes, highlights, tensions, open_questions, synthesis, new_connections, analyzed_at)
399
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
400
+
ON CONFLICT(component_id) DO UPDATE SET
401
+
title=excluded.title, themes=excluded.themes, highlights=excluded.highlights,
402
+
tensions=excluded.tensions, open_questions=excluded.open_questions,
403
+
synthesis=excluded.synthesis, new_connections=excluded.new_connections, analyzed_at=excluded.analyzed_at
404
+
`).run(
405
+
result.componentId,
406
+
result.title,
407
+
JSON.stringify(result.themes),
408
+
JSON.stringify(result.highlights),
409
+
JSON.stringify(result.tensions),
410
+
JSON.stringify(result.openQuestions),
411
+
result.synthesis,
412
+
JSON.stringify(result.newConnections),
413
+
result.analyzedAt,
414
+
);
415
+
}
416
+
417
+
// --- Main ---
418
+
419
+
async function main(): Promise<void> {
420
+
const args = process.argv.slice(2);
421
+
const wantJson = args.includes('--json');
422
+
const force = args.includes('--force');
423
+
const analyzeAll = args.includes('--all');
424
+
const minSizeArg = args.find((a) => a.startsWith('--min-size='));
425
+
const minSize = minSizeArg ? Number(minSizeArg.split('=')[1]) : 3;
426
+
427
+
// Get component ID from args
428
+
const compArg = args.find((a) => !a.startsWith('--') && /^\d+$/.test(a));
429
+
const compId = compArg ? Number(compArg) : null;
430
+
431
+
if (!compId && !analyzeAll) {
432
+
console.error('Usage: bun island:analyze <component-id> [--force] [--json]');
433
+
console.error(' bun island:analyze --all [--min-size 5] [--force] [--json]');
434
+
process.exit(1);
435
+
}
436
+
437
+
let db: Database;
438
+
try {
439
+
db = openDb();
440
+
const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
441
+
if (count === 0) {
442
+
console.error('No edges found. Run `bun island:discover` first.');
443
+
process.exit(1);
444
+
}
445
+
} catch {
446
+
console.error('Database not found. Run `bun island:discover` first.');
447
+
process.exit(1);
448
+
}
449
+
450
+
console.error('Loading carry data...');
451
+
const carryConnections = carryQueryJson<CarryConnection>('org.latha.connection', ['source', 'target', 'relation', 'context']);
452
+
const carryCitations = carryQueryJson<CarryCitation>('org.latha.citation', ['url', 'title', 'takeaway', 'domain']);
453
+
const entityUrlIndex = buildEntityUrlIndex();
454
+
console.error(`Carry: ${carryConnections.length} connections, ${carryCitations.length} citations, ${entityUrlIndex.size} entity URLs`);
455
+
456
+
console.error('Finding components...');
457
+
const components = findComponents(db);
458
+
459
+
let targets: Component[];
460
+
if (analyzeAll) {
461
+
targets = components.filter((c) => c.nodes.size >= minSize);
462
+
} else {
463
+
const comp = components.find((c) => c.id === compId);
464
+
if (!comp) {
465
+
console.error(`Component ${compId} not found. Available: ${components.slice(0, 10).map((c) => c.id).join(', ')}...`);
466
+
process.exit(1);
467
+
}
468
+
targets = [comp];
469
+
}
470
+
471
+
console.error(`Analyzing ${targets.length} component(s)...`);
472
+
473
+
const results: AnalysisResult[] = [];
474
+
475
+
for (const comp of targets) {
476
+
// Check if already analyzed
477
+
if (!force) {
478
+
const existing = db.query('SELECT analyzed_at FROM analyses WHERE component_id = ?').get(comp.id) as { analyzed_at: string } | null;
479
+
if (existing) {
480
+
console.error(`Component ${comp.id}: already analyzed at ${existing.analyzed_at} (use --force to re-analyze)`);
481
+
continue;
482
+
}
483
+
}
484
+
485
+
console.error(`Component ${comp.id}: ${comp.nodes.size} nodes, ${comp.dids.size} DIDs`);
486
+
const result = analyzeComponent(db, comp, carryConnections, entityUrlIndex, carryCitations);
487
+
saveAnalysis(db, result);
488
+
results.push(result);
489
+
console.error(` Themes: ${result.themes.join(', ') || 'none'}`);
490
+
console.error(` New connections: ${result.newConnections.length}`);
491
+
console.error(` Tensions: ${result.tensions.length}`);
492
+
}
493
+
494
+
if (wantJson) {
495
+
console.log(JSON.stringify(results, null, 2));
496
+
db.close();
497
+
return;
498
+
}
499
+
500
+
for (const r of results) {
501
+
console.log(`\nComponent ${r.componentId}: ${r.title}`);
502
+
console.log(` Themes: ${r.themes.join(', ') || 'none'}`);
503
+
console.log(` New connections: ${r.newConnections.length}`);
504
+
if (r.newConnections.length > 0) {
505
+
for (const nc of r.newConnections.slice(0, 10)) {
506
+
console.log(` ${nc.source} → ${nc.target} (${nc.connectionType})`);
507
+
}
508
+
if (r.newConnections.length > 10) {
509
+
console.log(` ... and ${r.newConnections.length - 10} more`);
510
+
}
511
+
}
512
+
console.log(` Tensions: ${r.tensions.length}`);
513
+
for (const t of r.tensions) console.log(` ${t.slice(0, 100)}`);
514
+
console.log(` Highlights: ${r.highlights.length}`);
515
+
console.log(` Synthesis: ${r.synthesis.slice(0, 300)}...`);
516
+
}
517
+
518
+
db.close();
519
+
}
520
+
521
+
main().catch((err) => {
522
+
console.error('Fatal:', err instanceof Error ? err.message : String(err));
523
+
process.exit(1);
524
+
});