nandi.uk
This repository has no description
23 kB
672 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Vault + carry markdown from local filesystem (synced via ob/git)
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// For carry/vault notes without URLs, uses the Letta API to discover
6// canonical URLs via web search.
7// Returns structured analysis + new connection records to be created on PDS.
8
9import { readdirSync, readFileSync, statSync } from "fs";
10import { join, extname } from "path";
11
12const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
13const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
14const LETTA_API_KEY = process.env.LETTA_API_KEY || "";
15const LETTA_BASE_URL = process.env.LETTA_BASE_URL || "https://api.letta.com";
16const LETTA_AGENT_ID = process.env.LETTA_AGENT_ID || "";
17
18// ─── Types ──────────────────────────────────────────────────────────
19
20interface VaultNote {
21 path: string;
22 content: string;
23}
24
25interface CarryNote {
26 path: string;
27 domain: string;
28 content: string;
29}
30
31interface VertexInfo {
32 uri: string;
33 title: string;
34 description: string;
35 type: string;
36}
37
38interface EdgeInfo {
39 uri: string;
40 did: string;
41 source: string;
42 target: string;
43 connectionType: string;
44 note: string;
45}
46
47interface CarryEntity {
48 atUri: string;
49 name: string;
50 entityType: string;
51 description?: string;
52 url?: string; // The entity's canonical https:// URL
53}
54
55interface CarryCitation {
56 atUri: string;
57 url: string;
58 title: string;
59 takeaway: string;
60 confidence?: string;
61}
62
63interface CarryReasoning {
64 atUri: string;
65 claim: string;
66 evidence: string;
67 reasoningType?: string;
68 url?: string; // The reasoning's source https:// URL
69}
70
71interface CarryData {
72 entities: CarryEntity[];
73 citations: CarryCitation[];
74 reasonings: CarryReasoning[];
75}
76
77interface NewConnection {
78 source: string; // island vertex URI
79 target: string; // carry record AT URI
80 connectionType: string; // e.g. "network.cosmik.connection#annotates"
81 note: string; // why this connection exists
82}
83
84interface AnalysisInput {
85 did: string;
86 island: {
87 vertices: VertexInfo[];
88 edges: EdgeInfo[];
89 };
90 carryData?: CarryData;
91}
92
93interface AnalysisResult {
94 themes: string[];
95 highlights: string[];
96 tensions: string[];
97 openQuestions: string[];
98 synthesis: string;
99 newConnections: NewConnection[];
100}
101
102// ─── Main analysis function ────────────────────────────────────────
103
104export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
105 const { island, carryData } = input;
106
107 // 1. Read vault + carry from filesystem
108 const vaultNotes = readVault();
109 const carryNotes = readCarry();
110
111 // 2. Build corpus from the entire island
112 const islandText = buildIslandText(island);
113
114 // 3. Extract themes from the island
115 const themes = extractThemes(islandText);
116
117 // 4. Find carry touchpoints — semantic matches from D1 records
118 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] });
119
120 // 5. Find vault/carry touchpoints — semantic matches from filesystem notes
121 // (async — may call Letta API for URL discovery)
122 const fsConnections = await findFilesystemTouchpoints(island, vaultNotes, carryNotes);
123 newConnections.push(...fsConnections);
124
125 // 6. Flag hot edges — existing edges that are semantically significant
126 const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections);
127
128 // 7. Find tensions from carry notes + reasonings
129 const tensions = [
130 ...findTensionsFromNotes(islandText, carryNotes),
131 ...findTensionsFromReasonings(islandText, carryData?.reasonings || []),
132 ];
133
134 // 8. Generate open questions
135 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
136
137 // 9. Synthesize prose
138 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
139
140 // Deduplicate newConnections
141 const seen = new Set<string>();
142 const deduped = newConnections.filter(c => {
143 const key = `${c.source}|${c.target}`;
144 if (seen.has(key)) return false;
145 seen.add(key);
146 return true;
147 });
148
149 return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
150}
151
152// ─── Filesystem readers ────────────────────────────────────────────
153
154function readVault(): VaultNote[] {
155 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
156 path: relPath,
157 content,
158 }));
159}
160
161function readCarry(): CarryNote[] {
162 const notes: CarryNote[] = [];
163 try {
164 const domains = readdirSync(CARRY_PATH);
165 for (const domain of domains) {
166 const domainPath = join(CARRY_PATH, domain);
167 if (!statSync(domainPath).isDirectory()) continue;
168 if (domain.startsWith(".")) continue;
169 const files = readMarkdownTree(domainPath);
170 for (const { relPath, content } of files) {
171 notes.push({ path: relPath, domain, content });
172 }
173 }
174 } catch (e: any) {
175 console.error("Failed to read carry:", e.message);
176 }
177 return notes;
178}
179
180function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
181 const results: Array<{ relPath: string; content: string }> = [];
182 try {
183 const entries = readdirSync(root, { withFileTypes: true });
184 for (const entry of entries) {
185 const fullPath = join(root, entry.name);
186 if (entry.name.startsWith(".")) continue;
187 if (entry.isDirectory()) {
188 results.push(...readMarkdownTree(fullPath));
189 } else if (extname(entry.name) === ".md") {
190 try {
191 const content = readFileSync(fullPath, "utf-8");
192 results.push({ relPath: fullPath.replace(root + "/", ""), content });
193 } catch {}
194 }
195 }
196 } catch {}
197 return results;
198}
199
200// ─── Build island text corpus ──────────────────────────────────────
201
202function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
203 const parts: string[] = [];
204
205 for (const v of island.vertices) {
206 parts.push(v.title);
207 if (v.description) parts.push(v.description);
208 }
209
210 for (const e of island.edges) {
211 if (e.note) parts.push(e.note);
212 }
213
214 return parts.join(" ");
215}
216
217// ─── Carry touchpoint matching (D1 records) ────────────────────────
218
219function findCarryTouchpoints(
220 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
221 carryData: CarryData,
222): NewConnection[] {
223 const connections: NewConnection[] = [];
224
225 // --- Match citations by URL ---
226 for (const cite of carryData.citations) {
227 if (!cite.url) continue; // Need an https:// URL as target
228 // Direct URL match: citation URL is a vertex URI
229 const vertexMatch = island.vertices.find(v => v.uri === cite.url);
230 if (vertexMatch) {
231 connections.push({
232 source: cite.url,
233 target: cite.url, // Self-referential — the citation IS about this vertex
234 connectionType: "network.cosmik.connection#annotates",
235 note: cite.takeaway.slice(0, 500),
236 });
237 continue;
238 }
239
240 // Semantic match: citation title/takeaway overlaps with island vertices
241 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
242 const bestVertex = findBestVertexMatch(citeText, island.vertices);
243 if (bestVertex) {
244 const overlap = keywordOverlap(citeText, `${bestVertex.title} ${bestVertex.description || ""}`);
245 if (overlap.length >= 2) {
246 connections.push({
247 source: bestVertex.uri,
248 target: cite.url,
249 connectionType: "network.cosmik.connection#annotates",
250 note: cite.takeaway.slice(0, 500),
251 });
252 }
253 }
254 }
255
256 // --- Match entities by name/description ---
257 for (const entity of carryData.entities) {
258 if (!entity.url) continue; // Need an https:// URL as target
259 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
260 const bestVertex = findBestVertexMatch(entityText, island.vertices);
261 if (bestVertex) {
262 const overlap = keywordOverlap(entityText, `${bestVertex.title} ${bestVertex.description || ""}`);
263 if (overlap.length >= 2) {
264 connections.push({
265 source: bestVertex.uri,
266 target: entity.url,
267 connectionType: "network.cosmik.connection#relates",
268 note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
269 });
270 }
271 }
272 }
273
274 // --- Match reasonings by claim/evidence ---
275 for (const reasoning of carryData.reasonings) {
276 if (!reasoning.url) continue; // Need an https:// URL as target
277 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
278 const bestVertex = findBestVertexMatch(reasoningText, island.vertices);
279 if (bestVertex) {
280 const overlap = keywordOverlap(reasoningText, `${bestVertex.title} ${bestVertex.description || ""}`);
281 if (overlap.length >= 2) {
282 const connType = reasoning.reasoningType?.includes("contradict")
283 ? "network.cosmik.connection#contradicts"
284 : "network.cosmik.connection#extends";
285 connections.push({
286 source: bestVertex.uri,
287 target: reasoning.url,
288 connectionType: connType,
289 note: reasoning.claim.slice(0, 500),
290 });
291 }
292 }
293 }
294
295 return connections;
296}
297
298// ─── Letta URL discovery ────────────────────────────────────────────
299// For carry/vault notes that lack an https:// URL, ask the Letta agent
300// to find the canonical URL via web search.
301
302async function discoverUrl(semanticDescription: string): Promise<string | null> {
303 if (!LETTA_API_KEY || !LETTA_AGENT_ID) return null;
304
305 try {
306 const resp = await fetch(`${LETTA_BASE_URL}/v1/agents/${LETTA_AGENT_ID}/messages`, {
307 method: "POST",
308 headers: {
309 "Authorization": `Bearer ${LETTA_API_KEY}`,
310 "Content-Type": "application/json",
311 },
312 body: JSON.stringify({
313 messages: [{
314 role: "user",
315 content: `Find the canonical URL for: ${semanticDescription}`,
316 }],
317 }),
318 });
319
320 if (!resp.ok) {
321 console.error(`[letta] API error: ${resp.status}`);
322 return null;
323 }
324
325 const data = await resp.json() as { messages: Array<{ message_type: string; content?: string }> };
326 // Find the assistant message — it should contain just the URL
327 for (const msg of data.messages || []) {
328 if (msg.message_type === "assistant_message" && msg.content) {
329 // Extract URL from the response
330 const urlMatch = msg.content.match(/https?:\/\/[^\s)\]]+/);
331 if (urlMatch) return urlMatch[0];
332 }
333 }
334 return null;
335 } catch (e: any) {
336 console.error("[letta] discoverUrl error:", e.message);
337 return null;
338 }
339}
340
341// ─── Filesystem touchpoint matching (vault + carry notes) ───────────
342
343function extractUrl(content: string): string | null {
344 // Pull the first https:// URL from the note content
345 const match = content.match(/https?:\/\/[^\s)\]]+/);
346 return match ? match[0] : null;
347}
348
349function extractTitle(content: string): string {
350 return content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || "";
351}
352
353async function findFilesystemTouchpoints(
354 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
355 vaultNotes: VaultNote[],
356 carryNotes: CarryNote[],
357): Promise<NewConnection[]> {
358 const connections: NewConnection[] = [];
359 const islandText = buildIslandText(island).toLowerCase();
360
361 // Match vault notes to island vertices
362 for (const note of vaultNotes) {
363 const noteLower = note.content.toLowerCase();
364 const overlap = keywordOverlap(islandText, noteLower);
365 if (overlap.length < 3) continue;
366
367 let noteUrl = extractUrl(note.content);
368 const title = extractTitle(note.content) || note.path;
369
370 // If no URL in the note, ask Letta to discover one
371 if (!noteUrl && LETTA_API_KEY && LETTA_AGENT_ID) {
372 noteUrl = await discoverUrl(title || note.path);
373 }
374 if (!noteUrl) continue;
375
376 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
377 if (bestVertex) {
378 connections.push({
379 source: bestVertex.uri,
380 target: noteUrl,
381 connectionType: "network.cosmik.connection#annotates",
382 note: `Vault note: ${title.slice(0, 200)}`,
383 });
384 }
385 }
386
387 // Match carry notes to island vertices
388 for (const note of carryNotes) {
389 const noteLower = note.content.toLowerCase();
390 const overlap = keywordOverlap(islandText, noteLower);
391 if (overlap.length < 2) continue;
392
393 let noteUrl = extractUrl(note.content);
394 const title = extractTitle(note.content) || note.path;
395
396 // If no URL in the note, ask Letta to discover one
397 if (!noteUrl && LETTA_API_KEY && LETTA_AGENT_ID) {
398 noteUrl = await discoverUrl(`Carry [${note.domain}]: ${title}`);
399 }
400 if (!noteUrl) continue;
401
402 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
403 if (bestVertex) {
404 connections.push({
405 source: bestVertex.uri,
406 target: noteUrl,
407 connectionType: "network.cosmik.connection#relates",
408 note: `Carry [${note.domain}]: ${title.slice(0, 200)}`,
409 });
410 }
411 }
412
413 return connections;
414}
415
416function findBestVertexMatch(
417 text: string,
418 vertices: VertexInfo[],
419): VertexInfo | null {
420 let bestScore = 0;
421 let bestVertex: VertexInfo | null = null;
422
423 for (const v of vertices) {
424 const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
425 const overlap = keywordOverlap(text, vertexText);
426 if (overlap.length > bestScore) {
427 bestScore = overlap.length;
428 bestVertex = v;
429 }
430 }
431
432 return bestScore >= 2 ? bestVertex : null;
433}
434
435// ─── Analysis functions ─────────────────────────────────────────────
436
437function extractThemes(text: string): string[] {
438 const keywords: Record<string, string[]> = {
439 "open science": ["open science", "open access", "oa", "preprint"],
440 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
441 "reproducibility": ["reproducib", "replicab", "replication"],
442 "decentralization": ["decentraliz", "federat", "self-host"],
443 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
444 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
445 "community": ["community", "collective", "cooperative", "commons"],
446 "governance": ["governance", "policy", "regulation", "standards"],
447 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
448 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
449 "provenance": ["provenance", "lineage", "traceability", "attribution"],
450 "doi": ["doi", "zenodo", "citeable", "citation"],
451 "privacy": ["privacy", "encryption", "e2ee", "consent"],
452 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
453 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
454 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
455 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
456 };
457
458 const lower = text.toLowerCase();
459 return Object.entries(keywords)
460 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
461 .map(([theme]) => theme);
462}
463
464function findHotEdges(
465 islandText: string,
466 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
467 vaultNotes: VaultNote[],
468 carryNotes: CarryNote[],
469 newConnections: NewConnection[],
470): string[] {
471 const highlights: string[] = [];
472 const lower = islandText.toLowerCase();
473
474 // Score each edge by semantic significance
475 const edgeScores = new Map<string, number>();
476
477 for (const edge of island.edges) {
478 let score = 0;
479
480 // Check if source or target appears in vault notes
481 for (const note of vaultNotes) {
482 const noteLower = note.content.toLowerCase();
483 const overlap = keywordOverlap(lower, noteLower);
484 if (overlap.length >= 2) {
485 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
486 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
487 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
488 if (targetTitle && noteLower.includes(targetTitle)) score += 2;
489 }
490 }
491
492 // Check if source or target appears in carry notes
493 for (const note of carryNotes) {
494 const noteLower = note.content.toLowerCase();
495 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
496 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
497 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
498 if (targetTitle && noteLower.includes(targetTitle)) score += 1;
499 }
500
501 // Edges with notes are more interesting
502 if (edge.note && edge.note.length > 10) score += 1;
503
504 // Edges with semantic connection types are more interesting
505 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
506 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
507
508 // Boost edges whose source or target is also connected to carry data
509 for (const nc of newConnections) {
510 if (edge.source === nc.source || edge.target === nc.source) score += 3;
511 if (edge.source === nc.target || edge.target === nc.target) score += 3;
512 }
513
514 if (score > 0) {
515 edgeScores.set(edge.uri, score);
516 }
517 }
518
519 // Return top edges by score
520 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
521 for (const [uri] of sorted.slice(0, 10)) {
522 highlights.push(uri);
523 }
524
525 return highlights;
526}
527
528function findTensionsFromNotes(
529 islandText: string,
530 carryNotes: CarryNote[],
531): string[] {
532 const tensions: string[] = [];
533 const lower = islandText.toLowerCase();
534
535 for (const note of carryNotes) {
536 const noteLower = note.content.toLowerCase();
537 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
538 const overlap = keywordOverlap(lower, noteLower);
539 if (overlap.length >= 1) {
540 tensions.push(
541 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
542 );
543 }
544 }
545 }
546
547 return tensions.slice(0, 5);
548}
549
550function findTensionsFromReasonings(
551 islandText: string,
552 reasonings: CarryReasoning[],
553): string[] {
554 const tensions: string[] = [];
555 const lower = islandText.toLowerCase();
556
557 for (const r of reasonings) {
558 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
559 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
560 const overlap = keywordOverlap(lower, rLower);
561 if (overlap.length >= 1) {
562 tensions.push(r.claim.slice(0, 500));
563 }
564 }
565 }
566
567 return tensions.slice(0, 5);
568}
569
570function generateOpenQuestions(
571 islandText: string,
572 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
573 highlights: string[],
574 tensions: string[],
575 newConnections: NewConnection[],
576): string[] {
577 const questions: string[] = [];
578 const lower = islandText.toLowerCase();
579
580 if (newConnections.length > 0) {
581 questions.push(
582 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
583 );
584 }
585
586 if (tensions.length > 0) {
587 questions.push(
588 `What evidence would resolve the ${tensions.length} tension(s) identified?`
589 );
590 }
591
592 const vertexCount = island.vertices.length;
593 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
594
595 if (edgeTypes.size > 1) {
596 questions.push(
597 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
598 );
599 }
600
601 if (vertexCount > 5) {
602 questions.push(
603 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
604 );
605 }
606
607 if (lower.includes("data") || lower.includes("dataset")) {
608 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
609 }
610 if (lower.includes("standard") || lower.includes("protocol")) {
611 questions.push("Does this cluster propose a new standard, and who are the adopters?");
612 }
613
614 return questions.slice(0, 5);
615}
616
617function synthesize(
618 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
619 themes: string[],
620 highlights: string[],
621 tensions: string[],
622 openQuestions: string[],
623 newConnections: NewConnection[],
624): string {
625 const vertexCount = island.vertices.length;
626 const edgeCount = island.edges.length;
627 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
628
629 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
630
631 const themeStr = themes.length > 0
632 ? `Themes: ${themes.join(", ")}.`
633 : "";
634
635 const touchpointStr = newConnections.length > 0
636 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
637 : "No carry touchpoints found.";
638
639 const highlightStr = highlights.length > 0
640 ? `${highlights.length} existing edges flagged as semantically significant.`
641 : "";
642
643 const tensionStr = tensions.length > 0
644 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
645 : "";
646
647 const questionStr = openQuestions.length > 0
648 ? `Open questions: ${openQuestions.join(" ")}`
649 : "";
650
651 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
652}
653
654// ─── Utilities ──────────────────────────────────────────────────────
655
656const STOP_WORDS = new Set([
657 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
658 "have", "has", "had", "do", "does", "did", "will", "would", "could",
659 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
660 "on", "with", "at", "by", "from", "as", "into", "through", "during",
661 "before", "after", "above", "below", "between", "and", "but", "or",
662 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
663 "http", "https", "www", "com", "org",
664]);
665
666function keywordOverlap(textA: string, textB: string): string[] {
667 const wordsA = new Set(
668 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
669 );
670 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
671 return wordsB.filter(w => wordsA.has(w));
672}