nandi.uk
This repository has no description
21 kB
609 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Vault + carry markdown from local filesystem (synced via ob/git)
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// Returns structured analysis + new connection records to be created on PDS.
6
7import { readdirSync, readFileSync, statSync } from "fs";
8import { join, extname } from "path";
9
10const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
11const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
12
13// ─── Types ──────────────────────────────────────────────────────────
14
15interface VaultNote {
16 path: string;
17 content: string;
18}
19
20interface CarryNote {
21 path: string;
22 domain: string;
23 content: string;
24}
25
26interface VertexInfo {
27 uri: string;
28 title: string;
29 description: string;
30 type: string;
31}
32
33interface EdgeInfo {
34 uri: string;
35 did: string;
36 source: string;
37 target: string;
38 connectionType: string;
39 note: string;
40}
41
42interface CarryEntity {
43 atUri: string;
44 name: string;
45 entityType: string;
46 description?: string;
47 url?: string; // The entity's canonical https:// URL
48}
49
50interface CarryCitation {
51 atUri: string;
52 url: string;
53 title: string;
54 takeaway: string;
55 confidence?: string;
56}
57
58interface CarryReasoning {
59 atUri: string;
60 claim: string;
61 evidence: string;
62 reasoningType?: string;
63 url?: string; // The reasoning's source https:// URL
64}
65
66interface CarryData {
67 entities: CarryEntity[];
68 citations: CarryCitation[];
69 reasonings: CarryReasoning[];
70}
71
72interface NewConnection {
73 source: string; // island vertex URI
74 target: string; // carry record AT URI
75 connectionType: string; // e.g. "network.cosmik.connection#annotates"
76 note: string; // why this connection exists
77}
78
79interface AnalysisInput {
80 did: string;
81 island: {
82 vertices: VertexInfo[];
83 edges: EdgeInfo[];
84 };
85 carryData?: CarryData;
86}
87
88interface AnalysisResult {
89 themes: string[];
90 highlights: string[];
91 tensions: string[];
92 openQuestions: string[];
93 synthesis: string;
94 newConnections: NewConnection[];
95}
96
97// ─── Main analysis function ────────────────────────────────────────
98
99export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
100 const { island, carryData } = input;
101
102 // 1. Read vault + carry from filesystem
103 const vaultNotes = readVault();
104 const carryNotes = readCarry();
105
106 // 2. Build corpus from the entire island
107 const islandText = buildIslandText(island);
108
109 // 3. Extract themes from the island
110 const themes = extractThemes(islandText);
111
112 // 4. Find carry touchpoints — semantic matches from D1 records
113 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] });
114
115 // 5. Find vault/carry touchpoints — semantic matches from filesystem notes
116 const fsConnections = findFilesystemTouchpoints(island, vaultNotes, carryNotes);
117 newConnections.push(...fsConnections);
118
119 // 6. Flag hot edges — existing edges that are semantically significant
120 const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections);
121
122 // 7. Find tensions from carry notes + reasonings
123 const tensions = [
124 ...findTensionsFromNotes(islandText, carryNotes),
125 ...findTensionsFromReasonings(islandText, carryData?.reasonings || []),
126 ];
127
128 // 8. Generate open questions
129 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
130
131 // 9. Synthesize prose
132 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
133
134 // Deduplicate newConnections
135 const seen = new Set<string>();
136 const deduped = newConnections.filter(c => {
137 const key = `${c.source}|${c.target}`;
138 if (seen.has(key)) return false;
139 seen.add(key);
140 return true;
141 });
142
143 return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
144}
145
146// ─── Filesystem readers ────────────────────────────────────────────
147
148function readVault(): VaultNote[] {
149 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
150 path: relPath,
151 content,
152 }));
153}
154
155function readCarry(): CarryNote[] {
156 const notes: CarryNote[] = [];
157 try {
158 const domains = readdirSync(CARRY_PATH);
159 for (const domain of domains) {
160 const domainPath = join(CARRY_PATH, domain);
161 if (!statSync(domainPath).isDirectory()) continue;
162 if (domain.startsWith(".")) continue;
163 const files = readMarkdownTree(domainPath);
164 for (const { relPath, content } of files) {
165 notes.push({ path: relPath, domain, content });
166 }
167 }
168 } catch (e: any) {
169 console.error("Failed to read carry:", e.message);
170 }
171 return notes;
172}
173
174function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
175 const results: Array<{ relPath: string; content: string }> = [];
176 try {
177 const entries = readdirSync(root, { withFileTypes: true });
178 for (const entry of entries) {
179 const fullPath = join(root, entry.name);
180 if (entry.name.startsWith(".")) continue;
181 if (entry.isDirectory()) {
182 results.push(...readMarkdownTree(fullPath));
183 } else if (extname(entry.name) === ".md") {
184 try {
185 const content = readFileSync(fullPath, "utf-8");
186 results.push({ relPath: fullPath.replace(root + "/", ""), content });
187 } catch {}
188 }
189 }
190 } catch {}
191 return results;
192}
193
194// ─── Build island text corpus ──────────────────────────────────────
195
196function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
197 const parts: string[] = [];
198
199 for (const v of island.vertices) {
200 parts.push(v.title);
201 if (v.description) parts.push(v.description);
202 }
203
204 for (const e of island.edges) {
205 if (e.note) parts.push(e.note);
206 }
207
208 return parts.join(" ");
209}
210
211// ─── Carry touchpoint matching (D1 records) ────────────────────────
212
213function findCarryTouchpoints(
214 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
215 carryData: CarryData,
216): NewConnection[] {
217 const connections: NewConnection[] = [];
218
219 // --- Match citations by URL ---
220 for (const cite of carryData.citations) {
221 if (!cite.url) continue; // Need an https:// URL as target
222 // Direct URL match: citation URL is a vertex URI
223 const vertexMatch = island.vertices.find(v => v.uri === cite.url);
224 if (vertexMatch) {
225 connections.push({
226 source: cite.url,
227 target: cite.url, // Self-referential — the citation IS about this vertex
228 connectionType: "network.cosmik.connection#annotates",
229 note: cite.takeaway.slice(0, 500),
230 });
231 continue;
232 }
233
234 // Semantic match: citation title/takeaway overlaps with island vertices
235 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
236 const bestVertex = findBestVertexMatch(citeText, island.vertices);
237 if (bestVertex) {
238 const overlap = keywordOverlap(citeText, `${bestVertex.title} ${bestVertex.description || ""}`);
239 if (overlap.length >= 2) {
240 connections.push({
241 source: bestVertex.uri,
242 target: cite.url,
243 connectionType: "network.cosmik.connection#annotates",
244 note: cite.takeaway.slice(0, 500),
245 });
246 }
247 }
248 }
249
250 // --- Match entities by name/description ---
251 for (const entity of carryData.entities) {
252 if (!entity.url) continue; // Need an https:// URL as target
253 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
254 const bestVertex = findBestVertexMatch(entityText, island.vertices);
255 if (bestVertex) {
256 const overlap = keywordOverlap(entityText, `${bestVertex.title} ${bestVertex.description || ""}`);
257 if (overlap.length >= 2) {
258 connections.push({
259 source: bestVertex.uri,
260 target: entity.url,
261 connectionType: "network.cosmik.connection#relates",
262 note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
263 });
264 }
265 }
266 }
267
268 // --- Match reasonings by claim/evidence ---
269 for (const reasoning of carryData.reasonings) {
270 if (!reasoning.url) continue; // Need an https:// URL as target
271 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
272 const bestVertex = findBestVertexMatch(reasoningText, island.vertices);
273 if (bestVertex) {
274 const overlap = keywordOverlap(reasoningText, `${bestVertex.title} ${bestVertex.description || ""}`);
275 if (overlap.length >= 2) {
276 const connType = reasoning.reasoningType?.includes("contradict")
277 ? "network.cosmik.connection#contradicts"
278 : "network.cosmik.connection#extends";
279 connections.push({
280 source: bestVertex.uri,
281 target: reasoning.url,
282 connectionType: connType,
283 note: reasoning.claim.slice(0, 500),
284 });
285 }
286 }
287 }
288
289 return connections;
290}
291
292// ─── Filesystem touchpoint matching (vault + carry notes) ───────────
293
294function extractUrl(content: string): string | null {
295 // Pull the first https:// URL from the note content
296 const match = content.match(/https?:\/\/[^\s)\]]+/);
297 return match ? match[0] : null;
298}
299
300function findFilesystemTouchpoints(
301 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
302 vaultNotes: VaultNote[],
303 carryNotes: CarryNote[],
304): NewConnection[] {
305 const connections: NewConnection[] = [];
306 const islandText = buildIslandText(island).toLowerCase();
307
308 // Match vault notes to island vertices
309 for (const note of vaultNotes) {
310 const noteLower = note.content.toLowerCase();
311 const overlap = keywordOverlap(islandText, noteLower);
312 if (overlap.length < 3) continue;
313
314 const noteUrl = extractUrl(note.content);
315 if (!noteUrl) continue; // No URL = can't form a connection
316
317 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
318 if (bestVertex) {
319 const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path;
320 connections.push({
321 source: bestVertex.uri,
322 target: noteUrl,
323 connectionType: "network.cosmik.connection#annotates",
324 note: `Vault note: ${title.slice(0, 200)}`,
325 });
326 }
327 }
328
329 // Match carry notes to island vertices
330 for (const note of carryNotes) {
331 const noteLower = note.content.toLowerCase();
332 const overlap = keywordOverlap(islandText, noteLower);
333 if (overlap.length < 2) continue;
334
335 const noteUrl = extractUrl(note.content);
336 if (!noteUrl) continue; // No URL = can't form a connection
337
338 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
339 if (bestVertex) {
340 const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path;
341 connections.push({
342 source: bestVertex.uri,
343 target: noteUrl,
344 connectionType: "network.cosmik.connection#relates",
345 note: `Carry [${note.domain}]: ${title.slice(0, 200)}`,
346 });
347 }
348 }
349
350 return connections;
351}
352
353function findBestVertexMatch(
354 text: string,
355 vertices: VertexInfo[],
356): VertexInfo | null {
357 let bestScore = 0;
358 let bestVertex: VertexInfo | null = null;
359
360 for (const v of vertices) {
361 const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
362 const overlap = keywordOverlap(text, vertexText);
363 if (overlap.length > bestScore) {
364 bestScore = overlap.length;
365 bestVertex = v;
366 }
367 }
368
369 return bestScore >= 2 ? bestVertex : null;
370}
371
372// ─── Analysis functions ─────────────────────────────────────────────
373
374function extractThemes(text: string): string[] {
375 const keywords: Record<string, string[]> = {
376 "open science": ["open science", "open access", "oa", "preprint"],
377 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
378 "reproducibility": ["reproducib", "replicab", "replication"],
379 "decentralization": ["decentraliz", "federat", "self-host"],
380 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
381 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
382 "community": ["community", "collective", "cooperative", "commons"],
383 "governance": ["governance", "policy", "regulation", "standards"],
384 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
385 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
386 "provenance": ["provenance", "lineage", "traceability", "attribution"],
387 "doi": ["doi", "zenodo", "citeable", "citation"],
388 "privacy": ["privacy", "encryption", "e2ee", "consent"],
389 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
390 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
391 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
392 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
393 };
394
395 const lower = text.toLowerCase();
396 return Object.entries(keywords)
397 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
398 .map(([theme]) => theme);
399}
400
401function findHotEdges(
402 islandText: string,
403 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
404 vaultNotes: VaultNote[],
405 carryNotes: CarryNote[],
406 newConnections: NewConnection[],
407): string[] {
408 const highlights: string[] = [];
409 const lower = islandText.toLowerCase();
410
411 // Score each edge by semantic significance
412 const edgeScores = new Map<string, number>();
413
414 for (const edge of island.edges) {
415 let score = 0;
416
417 // Check if source or target appears in vault notes
418 for (const note of vaultNotes) {
419 const noteLower = note.content.toLowerCase();
420 const overlap = keywordOverlap(lower, noteLower);
421 if (overlap.length >= 2) {
422 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
423 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
424 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
425 if (targetTitle && noteLower.includes(targetTitle)) score += 2;
426 }
427 }
428
429 // Check if source or target appears in carry notes
430 for (const note of carryNotes) {
431 const noteLower = note.content.toLowerCase();
432 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
433 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
434 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
435 if (targetTitle && noteLower.includes(targetTitle)) score += 1;
436 }
437
438 // Edges with notes are more interesting
439 if (edge.note && edge.note.length > 10) score += 1;
440
441 // Edges with semantic connection types are more interesting
442 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
443 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
444
445 // Boost edges whose source or target is also connected to carry data
446 for (const nc of newConnections) {
447 if (edge.source === nc.source || edge.target === nc.source) score += 3;
448 if (edge.source === nc.target || edge.target === nc.target) score += 3;
449 }
450
451 if (score > 0) {
452 edgeScores.set(edge.uri, score);
453 }
454 }
455
456 // Return top edges by score
457 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
458 for (const [uri] of sorted.slice(0, 10)) {
459 highlights.push(uri);
460 }
461
462 return highlights;
463}
464
465function findTensionsFromNotes(
466 islandText: string,
467 carryNotes: CarryNote[],
468): string[] {
469 const tensions: string[] = [];
470 const lower = islandText.toLowerCase();
471
472 for (const note of carryNotes) {
473 const noteLower = note.content.toLowerCase();
474 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
475 const overlap = keywordOverlap(lower, noteLower);
476 if (overlap.length >= 1) {
477 tensions.push(
478 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
479 );
480 }
481 }
482 }
483
484 return tensions.slice(0, 5);
485}
486
487function findTensionsFromReasonings(
488 islandText: string,
489 reasonings: CarryReasoning[],
490): string[] {
491 const tensions: string[] = [];
492 const lower = islandText.toLowerCase();
493
494 for (const r of reasonings) {
495 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
496 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
497 const overlap = keywordOverlap(lower, rLower);
498 if (overlap.length >= 1) {
499 tensions.push(r.claim.slice(0, 500));
500 }
501 }
502 }
503
504 return tensions.slice(0, 5);
505}
506
507function generateOpenQuestions(
508 islandText: string,
509 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
510 highlights: string[],
511 tensions: string[],
512 newConnections: NewConnection[],
513): string[] {
514 const questions: string[] = [];
515 const lower = islandText.toLowerCase();
516
517 if (newConnections.length > 0) {
518 questions.push(
519 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
520 );
521 }
522
523 if (tensions.length > 0) {
524 questions.push(
525 `What evidence would resolve the ${tensions.length} tension(s) identified?`
526 );
527 }
528
529 const vertexCount = island.vertices.length;
530 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
531
532 if (edgeTypes.size > 1) {
533 questions.push(
534 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
535 );
536 }
537
538 if (vertexCount > 5) {
539 questions.push(
540 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
541 );
542 }
543
544 if (lower.includes("data") || lower.includes("dataset")) {
545 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
546 }
547 if (lower.includes("standard") || lower.includes("protocol")) {
548 questions.push("Does this cluster propose a new standard, and who are the adopters?");
549 }
550
551 return questions.slice(0, 5);
552}
553
554function synthesize(
555 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
556 themes: string[],
557 highlights: string[],
558 tensions: string[],
559 openQuestions: string[],
560 newConnections: NewConnection[],
561): string {
562 const vertexCount = island.vertices.length;
563 const edgeCount = island.edges.length;
564 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
565
566 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
567
568 const themeStr = themes.length > 0
569 ? `Themes: ${themes.join(", ")}.`
570 : "";
571
572 const touchpointStr = newConnections.length > 0
573 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
574 : "No carry touchpoints found.";
575
576 const highlightStr = highlights.length > 0
577 ? `${highlights.length} existing edges flagged as semantically significant.`
578 : "";
579
580 const tensionStr = tensions.length > 0
581 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
582 : "";
583
584 const questionStr = openQuestions.length > 0
585 ? `Open questions: ${openQuestions.join(" ")}`
586 : "";
587
588 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
589}
590
591// ─── Utilities ──────────────────────────────────────────────────────
592
593const STOP_WORDS = new Set([
594 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
595 "have", "has", "had", "do", "does", "did", "will", "would", "could",
596 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
597 "on", "with", "at", "by", "from", "as", "into", "through", "during",
598 "before", "after", "above", "below", "between", "and", "but", "or",
599 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
600 "http", "https", "www", "com", "org",
601]);
602
603function keywordOverlap(textA: string, textB: string): string[] {
604 const wordsA = new Set(
605 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
606 );
607 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
608 return wordsB.filter(w => wordsA.has(w));
609}