nandi.uk
This repository has no description
23 kB
670 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Vault + carry markdown from local filesystem (synced via ob/git)
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// For carry/vault notes without URLs, uses the Letta API to discover
6// canonical URLs via web search.
7// Returns structured analysis + new connection records to be created on PDS.
8
9import { readFileSync, existsSync } from "fs";
10import { join } from "path";
11
12const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
13
14// ─── Types ──────────────────────────────────────────────────────────
15
16interface VertexInfo {
17 uri: string;
18 title: string;
19 description: string;
20 type: string;
21}
22
23interface EdgeInfo {
24 uri: string;
25 did: string;
26 source: string;
27 target: string;
28 connectionType: string;
29 note: string;
30}
31
32interface CarryEntity {
33 atUri: string;
34 name: string;
35 entityType: string;
36 description?: string;
37 url?: string; // The entity's canonical https:// URL
38}
39
40interface CarryCitation {
41 atUri: string;
42 url: string;
43 title: string;
44 takeaway: string;
45 confidence?: string;
46}
47
48interface CarryReasoning {
49 atUri: string;
50 claim: string;
51 evidence: string;
52 reasoningType?: string;
53 url?: string; // The reasoning's source https:// URL
54}
55
56interface CarryData {
57 entities: CarryEntity[];
58 citations: CarryCitation[];
59 reasonings: CarryReasoning[];
60 connections: CarryConnection[];
61}
62
63// Structured carry connection — a semantic triple
64interface CarryConnection {
65 sourceName: string; // e.g. "Khayame"
66 targetName: string; // e.g. "Vickers Appreciative Systems"
67 sourceUrl: string; // e.g. "https://link.springer.com/..."
68 targetUrl: string; // e.g. "https://pubadmin.institute/..."
69 relation: string; // e.g. "intellectual_influence"
70 context: string; // e.g. "Khayame draws on Vickers' appreciative systems..."
71}
72
73interface NewConnection {
74 source: string; // island vertex URI
75 target: string; // carry record AT URI
76 connectionType: string; // e.g. "network.cosmik.connection#annotates"
77 note: string; // why this connection exists
78}
79
80interface AnalysisInput {
81 did: string;
82 island: {
83 vertices: VertexInfo[];
84 edges: EdgeInfo[];
85 };
86 carryData?: CarryData;
87}
88
89interface AnalysisResult {
90 themes: string[];
91 highlights: string[];
92 tensions: string[];
93 openQuestions: string[];
94 synthesis: string;
95 newConnections: NewConnection[];
96}
97
98// ─── Main analysis function ────────────────────────────────────────
99
100export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
101 const { island, carryData } = input;
102
103 // 1. Read carry JSON exports (from `carry query --format json` during sync)
104 const carryEntities = readCarryEntities();
105 const carryCitations = readCarryCitations();
106 const carryConnections = readCarryConnections();
107 const entityUrlIndex = buildEntityUrlIndex();
108
109 // 2. Build island URL set for fast lookup
110 const islandText = buildIslandText(island);
111 const islandUrls = new Set(island.vertices.map(v => v.uri));
112 const islandTokenSet = tokenize(islandText);
113 const vertexTokenSets = new Map<string, Set<string>>();
114 for (const v of island.vertices) {
115 vertexTokenSets.set(v.uri, tokenize(`${v.title} ${v.description || ""}`));
116 }
117
118 // 3. Compute IDF for TF-IDF scoring
119 const allDocuments = [
120 ...carryEntities.map(e => `${e.name} ${e.description || ""}`),
121 ...carryCitations.map(c => `${c.title} ${c.takeaway || ""}`),
122 ...island.vertices.map(v => `${v.title} ${v.description || ""}`),
123 ];
124 const idf = computeIdf(allDocuments);
125
126 // 4. Extract themes
127 const themes = extractThemes(islandText);
128
129 // 5. Match carry data against island — structured semantic triples only
130 const newConnections = matchCarryData(
131 island, islandUrls, islandTokenSet, vertexTokenSets, idf,
132 carryEntities, carryCitations, carryConnections, entityUrlIndex,
133 carryData || { entities: [], citations: [], reasonings: [], connections: [] },
134 );
135
136 // 6. Flag hot edges
137 const highlights = findHotEdges(islandText, island, newConnections);
138
139 // 7. Find tensions from carry reasonings
140 const tensions = findTensionsFromReasonings(islandText, carryData?.reasonings || []);
141
142 // 8. Generate open questions
143 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
144
145 // 9. Synthesize prose
146 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
147
148 // Deduplicate newConnections
149 const seen = new Set<string>();
150 const deduped = newConnections.filter(c => {
151 const key = `${c.source}|${c.target}`;
152 if (seen.has(key)) return false;
153 seen.add(key);
154 return true;
155 });
156
157 return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
158}
159
160// ─── Carry JSON readers ────────────────────────────────────────────
161// Reads carry data exported by `carry query --format json` during sync.
162
163interface CarryEntityRecord {
164 id: string;
165 name: string;
166 url?: string;
167 type?: string;
168 description?: string;
169}
170
171interface CarryCitationRecord {
172 id: string;
173 title: string;
174 url?: string;
175 takeaway?: string;
176}
177
178interface CarryConnectionRecord {
179 id: string;
180 relation: string;
181 source?: string;
182 target?: string;
183 context?: string;
184}
185
186function readCarryJson<T>(filename: string): T[] {
187 const path = join(CARRY_PATH, filename);
188 try {
189 if (!existsSync(path)) return [];
190 const data = readFileSync(path, "utf-8");
191 return JSON.parse(data) as T[];
192 } catch (e: any) {
193 console.error(`[carry] Failed to read ${filename}:`, e.message);
194 return [];
195 }
196}
197
198function readCarryEntities(): CarryEntityRecord[] {
199 return readCarryJson<CarryEntityRecord>("org.latha.entity.json");
200}
201
202function readCarryCitations(): CarryCitationRecord[] {
203 return readCarryJson<CarryCitationRecord>("org.latha.citation.json");
204}
205
206function readCarryConnections(): CarryConnectionRecord[] {
207 return readCarryJson<CarryConnectionRecord>("org.latha.connection.json");
208}
209
210// Build entity name → URL lookup from carry export
211function buildEntityUrlIndex(): Map<string, string> {
212 const index = new Map<string, string>();
213 for (const e of readCarryEntities()) {
214 const url = e.url?.trim().replace(/^"|"$/g, "");
215 if (url && url.startsWith("http")) {
216 const name = e.name?.trim().replace(/^"|"$/g, "");
217 if (name) index.set(name, url);
218 }
219 }
220 return index;
221}
222
223// ─── Build island text corpus ──────────────────────────────────────
224
225function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
226 const parts: string[] = [];
227
228 for (const v of island.vertices) {
229 parts.push(v.title);
230 if (v.description) parts.push(v.description);
231 }
232
233 for (const e of island.edges) {
234 if (e.note) parts.push(e.note);
235 }
236
237 return parts.join(" ");
238}
239
240// ─── Core mapper: match carry semantic triples against island ──────
241
242function matchCarryData(
243 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
244 islandUrls: Set<string>,
245 islandTokenSet: Set<string>,
246 vertexTokenSets: Map<string, Set<string>>,
247 idf: Map<string, number>,
248 carryEntities: CarryEntityRecord[],
249 carryCitations: CarryCitationRecord[],
250 carryConnections: CarryConnectionRecord[],
251 entityUrlIndex: Map<string, string>,
252 d1CarryData: CarryData,
253): NewConnection[] {
254 const connections: NewConnection[] = [];
255
256 // ── 1. Carry connections (semantic triples) ──────────────────────
257 for (const conn of carryConnections) {
258 const srcName = conn.source || "";
259 const tgtName = conn.target || "";
260 const srcUrl = entityUrlIndex.get(srcName) || "";
261 const tgtUrl = entityUrlIndex.get(tgtName) || "";
262
263 const srcInIsland = srcUrl && islandUrls.has(srcUrl);
264 const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
265
266 if (srcInIsland && tgtUrl) {
267 connections.push({
268 source: srcUrl,
269 target: tgtUrl,
270 connectionType: `network.cosmik.connection#${conn.relation}`,
271 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
272 });
273 }
274
275 if (tgtInIsland && srcUrl) {
276 connections.push({
277 source: tgtUrl,
278 target: srcUrl,
279 connectionType: `network.cosmik.connection#${conn.relation}`,
280 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
281 });
282 }
283
284 // Neither in island — try TF-IDF name match
285 if (!srcInIsland && !tgtInIsland && (srcUrl || tgtUrl)) {
286 const nameToMatch = srcName || tgtName;
287 const urlToConnect = srcUrl || tgtUrl || "";
288 if (!urlToConnect) continue;
289
290 const nameTerms = tokenize(nameToMatch);
291 const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 4.0);
292 if (bestVertex) {
293 connections.push({
294 source: bestVertex.uri,
295 target: urlToConnect,
296 connectionType: `network.cosmik.connection#${conn.relation}`,
297 note: `${nameToMatch} (${conn.relation}) — carry connection`,
298 });
299 }
300 }
301 }
302
303 // ── 2. Carry entities — match by URL or name ──────────────────────
304 for (const entity of carryEntities) {
305 const url = entity.url?.trim().replace(/^"|"$/g, "");
306 if (!url || !url.startsWith("http")) continue;
307 if (islandUrls.has(url)) continue;
308
309 const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`);
310 const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0);
311 if (bestVertex) {
312 connections.push({
313 source: bestVertex.uri,
314 target: url,
315 connectionType: "network.cosmik.connection#relates",
316 note: `Related: ${entity.name} (${entity.type || "entity"})`,
317 });
318 }
319 }
320
321 // ── 3. Carry citations — match by URL or title ───────────────────
322 for (const cite of carryCitations) {
323 const url = cite.url?.trim().replace(/^"|"$/g, "");
324 if (!url || !url.startsWith("http")) continue;
325 if (islandUrls.has(url)) continue;
326
327 const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`);
328 const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0);
329 if (bestVertex) {
330 connections.push({
331 source: bestVertex.uri,
332 target: url,
333 connectionType: "network.cosmik.connection#annotates",
334 note: cite.title.slice(0, 200),
335 });
336 }
337 }
338
339 // ── 4. D1 carry data (entities, citations, reasonings from PDS) ──
340 for (const entity of d1CarryData.entities) {
341 if (!entity.url || !entity.url.startsWith("http")) continue;
342 if (islandUrls.has(entity.url)) continue;
343
344 const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`);
345 const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0);
346 if (bestVertex) {
347 connections.push({
348 source: bestVertex.uri,
349 target: entity.url,
350 connectionType: "network.cosmik.connection#relates",
351 note: `Related: ${entity.name} (${entity.entityType.replace("org.latha.strata.defs#", "")})`,
352 });
353 }
354 }
355
356 for (const cite of d1CarryData.citations) {
357 if (!cite.url || !cite.url.startsWith("http")) continue;
358 if (islandUrls.has(cite.url)) continue;
359
360 const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`);
361 const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0);
362 if (bestVertex) {
363 connections.push({
364 source: bestVertex.uri,
365 target: cite.url,
366 connectionType: "network.cosmik.connection#annotates",
367 note: cite.takeaway?.slice(0, 200) || cite.title.slice(0, 200),
368 });
369 }
370 }
371
372 for (const reasoning of d1CarryData.reasonings) {
373 if (!reasoning.url || !reasoning.url.startsWith("http")) continue;
374 if (islandUrls.has(reasoning.url)) continue;
375
376 const reasoningTerms = tokenize(`${reasoning.claim} ${reasoning.evidence}`);
377 const bestVertex = findBestVertexMatch(reasoningTerms, island.vertices, vertexTokenSets, idf, 4.0);
378 if (bestVertex) {
379 const connType = reasoning.reasoningType?.includes("contradict")
380 ? "network.cosmik.connection#contradicts"
381 : "network.cosmik.connection#extends";
382 connections.push({
383 source: bestVertex.uri,
384 target: reasoning.url,
385 connectionType: connType,
386 note: reasoning.claim.slice(0, 500),
387 });
388 }
389 }
390
391 return connections;
392}
393
394function findBestVertexMatch(
395 noteTerms: Set<string>,
396 vertices: VertexInfo[],
397 vertexTokenSets: Map<string, Set<string>>,
398 idf: Map<string, number>,
399 minScore: number = 3.0,
400): VertexInfo | null {
401 let bestScore = 0;
402 let bestVertex: VertexInfo | null = null;
403
404 for (const v of vertices) {
405 const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`);
406 const score = tfidfScore(noteTerms, vSet, idf);
407 if (score > bestScore) {
408 bestScore = score;
409 bestVertex = v;
410 }
411 }
412
413 return bestScore >= minScore ? bestVertex : null;
414}
415
416// ─── Analysis functions ─────────────────────────────────────────────
417
418function extractThemes(text: string): string[] {
419 const keywords: Record<string, string[]> = {
420 "open science": ["open science", "open access", "oa", "preprint"],
421 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
422 "reproducibility": ["reproducib", "replicab", "replication"],
423 "decentralization": ["decentraliz", "federat", "self-host"],
424 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
425 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
426 "community": ["community", "collective", "cooperative", "commons"],
427 "governance": ["governance", "policy", "regulation", "standards"],
428 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
429 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
430 "provenance": ["provenance", "lineage", "traceability", "attribution"],
431 "doi": ["doi", "zenodo", "citeable", "citation"],
432 "privacy": ["privacy", "encryption", "e2ee", "consent"],
433 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
434 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
435 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
436 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
437 };
438
439 const lower = text.toLowerCase();
440 return Object.entries(keywords)
441 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
442 .map(([theme]) => theme);
443}
444
445function findHotEdges(
446 islandText: string,
447 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
448 newConnections: NewConnection[],
449): string[] {
450 const highlights: string[] = [];
451
452 // Score each edge by semantic significance
453 const edgeScores = new Map<string, number>();
454
455 for (const edge of island.edges) {
456 let score = 0;
457
458 // Edges with notes are more interesting
459 if (edge.note && edge.note.length > 10) score += 1;
460
461 // Edges with semantic connection types are more interesting
462 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
463 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
464
465 // Boost edges whose source or target is also connected to carry data
466 for (const nc of newConnections) {
467 if (edge.source === nc.source || edge.target === nc.source) score += 3;
468 if (edge.source === nc.target || edge.target === nc.target) score += 3;
469 }
470
471 if (score > 0) {
472 edgeScores.set(edge.uri, score);
473 }
474 }
475
476 // Return top edges by score
477 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
478 for (const [uri] of sorted.slice(0, 10)) {
479 highlights.push(uri);
480 }
481
482 return highlights;
483}
484
485function findTensionsFromReasonings(
486 islandText: string,
487 reasonings: CarryReasoning[],
488): string[] {
489 const tensions: string[] = [];
490 const lowerSet = tokenize(islandText);
491
492 for (const r of reasonings) {
493 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
494 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
495 const overlap = keywordOverlap(lowerSet, rLower);
496 if (overlap.length >= 1) {
497 tensions.push(r.claim.slice(0, 500));
498 }
499 }
500 }
501
502 return tensions.slice(0, 5);
503}
504
505function generateOpenQuestions(
506 islandText: string,
507 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
508 highlights: string[],
509 tensions: string[],
510 newConnections: NewConnection[],
511): string[] {
512 const questions: string[] = [];
513 const lower = islandText.toLowerCase();
514
515 if (newConnections.length > 0) {
516 questions.push(
517 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
518 );
519 }
520
521 if (tensions.length > 0) {
522 questions.push(
523 `What evidence would resolve the ${tensions.length} tension(s) identified?`
524 );
525 }
526
527 const vertexCount = island.vertices.length;
528 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
529
530 if (edgeTypes.size > 1) {
531 questions.push(
532 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
533 );
534 }
535
536 if (vertexCount > 5) {
537 questions.push(
538 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
539 );
540 }
541
542 if (lower.includes("data") || lower.includes("dataset")) {
543 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
544 }
545 if (lower.includes("standard") || lower.includes("protocol")) {
546 questions.push("Does this cluster propose a new standard, and who are the adopters?");
547 }
548
549 return questions.slice(0, 5);
550}
551
552function synthesize(
553 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
554 themes: string[],
555 highlights: string[],
556 tensions: string[],
557 openQuestions: string[],
558 newConnections: NewConnection[],
559): string {
560 const vertexCount = island.vertices.length;
561 const edgeCount = island.edges.length;
562 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
563
564 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
565
566 const themeStr = themes.length > 0
567 ? `Themes: ${themes.join(", ")}.`
568 : "";
569
570 const touchpointStr = newConnections.length > 0
571 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
572 : "No carry touchpoints found.";
573
574 const highlightStr = highlights.length > 0
575 ? `${highlights.length} existing edges flagged as semantically significant.`
576 : "";
577
578 const tensionStr = tensions.length > 0
579 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
580 : "";
581
582 const questionStr = openQuestions.length > 0
583 ? `Open questions: ${openQuestions.join(" ")}`
584 : "";
585
586 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
587}
588
589// ─── Utilities ──────────────────────────────────────────────────────
590
591const STOP_WORDS = new Set([
592 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
593 "have", "has", "had", "do", "does", "did", "will", "would", "could",
594 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
595 "on", "with", "at", "by", "from", "as", "into", "through", "during",
596 "before", "after", "above", "below", "between", "and", "but", "or",
597 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
598 "http", "https", "www", "com", "org",
599 // Common vault noise terms
600 "checkin", "check", "daily", "today", "yesterday", "tomorrow",
601 "morning", "evening", "week", "month", "year",
602 "also", "just", "like", "really", "very", "much", "some",
603 "more", "most", "other", "about", "which", "when", "what",
604 "there", "their", "these", "those", "each", "every", "all",
605 "make", "made", "thing", "things", "something", "anything",
606 "need", "want", "know", "think", "going", "come", "get",
607 "good", "great", "nice", "well", "still", "even", "back",
608 "first", "last", "new", "old", "own", "same", "another",
609 "many", "such", "only", "over", "than", "too", "very",
610 "because", "since", "while", "where", "how", "why", "who",
611 "work", "working", "time", "people", "using", "used", "use",
612]);
613
614function tokenize(text: string): Set<string> {
615 return new Set(
616 text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
617 );
618}
619
620function tokenizeArray(text: string): string[] {
621 return text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
622}
623
624// ─── TF-IDF scoring ────────────────────────────────────────────────
625// Compute IDF across all notes, then score note-island matches by
626// the sum of IDF weights of shared terms. This naturally downweights
627// common terms like "checkin" and upweights distinctive terms like
628// "stigmergy" or "nanopublication".
629
630function computeIdf(documents: string[]): Map<string, number> {
631 const docCount = documents.length;
632 const df = new Map<string, number>(); // document frequency
633
634 for (const doc of documents) {
635 const terms = new Set(tokenizeArray(doc));
636 for (const t of terms) {
637 df.set(t, (df.get(t) || 0) + 1);
638 }
639 }
640
641 const idf = new Map<string, number>();
642 for (const [term, freq] of df) {
643 // Smooth IDF: log((N+1)/(df+1)) + 1
644 idf.set(term, Math.log((docCount + 1) / (freq + 1)) + 1);
645 }
646 return idf;
647}
648
649function tfidfScore(
650 noteTerms: Set<string>,
651 targetTerms: Set<string>,
652 idf: Map<string, number>,
653): number {
654 let score = 0;
655 for (const t of noteTerms) {
656 if (targetTerms.has(t)) {
657 score += idf.get(t) || 0;
658 }
659 }
660 return score;
661}
662
663function keywordOverlap(setA: Set<string>, textB: string): string[] {
664 const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
665 return wordsB.filter(w => setA.has(w));
666}
667
668function keywordOverlapSets(setA: Set<string>, setB: Set<string>): string[] {
669 return [...setA].filter(w => setB.has(w));
670}