nandi.uk
This repository has no description
21 kB
594 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Vault + carry markdown from local filesystem (synced via ob/git)
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// Returns structured analysis + new connection records to be created on PDS.
6
7import { readdirSync, readFileSync, statSync } from "fs";
8import { join, extname } from "path";
9
10const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
11const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
12
13// ─── Types ──────────────────────────────────────────────────────────
14
15interface VaultNote {
16 path: string;
17 content: string;
18}
19
20interface CarryNote {
21 path: string;
22 domain: string;
23 content: string;
24}
25
26interface VertexInfo {
27 uri: string;
28 title: string;
29 description: string;
30 type: string;
31}
32
33interface EdgeInfo {
34 uri: string;
35 did: string;
36 source: string;
37 target: string;
38 connectionType: string;
39 note: string;
40}
41
42interface CarryEntity {
43 atUri: string;
44 name: string;
45 entityType: string;
46 description?: string;
47}
48
49interface CarryCitation {
50 atUri: string;
51 url: string;
52 title: string;
53 takeaway: string;
54 confidence?: string;
55}
56
57interface CarryReasoning {
58 atUri: string;
59 claim: string;
60 evidence: string;
61 reasoningType?: string;
62}
63
64interface CarryData {
65 entities: CarryEntity[];
66 citations: CarryCitation[];
67 reasonings: CarryReasoning[];
68}
69
70interface NewConnection {
71 source: string; // island vertex URI
72 target: string; // carry record AT URI
73 connectionType: string; // e.g. "network.cosmik.connection#annotates"
74 note: string; // why this connection exists
75}
76
77interface AnalysisInput {
78 did: string;
79 island: {
80 vertices: VertexInfo[];
81 edges: EdgeInfo[];
82 };
83 carryData?: CarryData;
84}
85
86interface AnalysisResult {
87 themes: string[];
88 highlights: string[];
89 tensions: string[];
90 openQuestions: string[];
91 synthesis: string;
92 newConnections: NewConnection[];
93}
94
95// ─── Main analysis function ────────────────────────────────────────
96
97export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
98 const { island, carryData } = input;
99
100 // 1. Read vault + carry from filesystem
101 const vaultNotes = readVault();
102 const carryNotes = readCarry();
103
104 // 2. Build corpus from the entire island
105 const islandText = buildIslandText(island);
106
107 // 3. Extract themes from the island
108 const themes = extractThemes(islandText);
109
110 // 4. Find carry touchpoints — semantic matches from D1 records
111 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] });
112
113 // 5. Find vault/carry touchpoints — semantic matches from filesystem notes
114 const fsConnections = findFilesystemTouchpoints(island, vaultNotes, carryNotes);
115 newConnections.push(...fsConnections);
116
117 // 6. Flag hot edges — existing edges that are semantically significant
118 const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections);
119
120 // 7. Find tensions from carry notes + reasonings
121 const tensions = [
122 ...findTensionsFromNotes(islandText, carryNotes),
123 ...findTensionsFromReasonings(islandText, carryData?.reasonings || []),
124 ];
125
126 // 8. Generate open questions
127 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
128
129 // 9. Synthesize prose
130 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
131
132 // Deduplicate newConnections
133 const seen = new Set<string>();
134 const deduped = newConnections.filter(c => {
135 const key = `${c.source}|${c.target}`;
136 if (seen.has(key)) return false;
137 seen.add(key);
138 return true;
139 });
140
141 return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
142}
143
144// ─── Filesystem readers ────────────────────────────────────────────
145
146function readVault(): VaultNote[] {
147 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
148 path: relPath,
149 content,
150 }));
151}
152
153function readCarry(): CarryNote[] {
154 const notes: CarryNote[] = [];
155 try {
156 const domains = readdirSync(CARRY_PATH);
157 for (const domain of domains) {
158 const domainPath = join(CARRY_PATH, domain);
159 if (!statSync(domainPath).isDirectory()) continue;
160 if (domain.startsWith(".")) continue;
161 const files = readMarkdownTree(domainPath);
162 for (const { relPath, content } of files) {
163 notes.push({ path: relPath, domain, content });
164 }
165 }
166 } catch (e: any) {
167 console.error("Failed to read carry:", e.message);
168 }
169 return notes;
170}
171
172function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
173 const results: Array<{ relPath: string; content: string }> = [];
174 try {
175 const entries = readdirSync(root, { withFileTypes: true });
176 for (const entry of entries) {
177 const fullPath = join(root, entry.name);
178 if (entry.name.startsWith(".")) continue;
179 if (entry.isDirectory()) {
180 results.push(...readMarkdownTree(fullPath));
181 } else if (extname(entry.name) === ".md") {
182 try {
183 const content = readFileSync(fullPath, "utf-8");
184 results.push({ relPath: fullPath.replace(root + "/", ""), content });
185 } catch {}
186 }
187 }
188 } catch {}
189 return results;
190}
191
192// ─── Build island text corpus ──────────────────────────────────────
193
194function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
195 const parts: string[] = [];
196
197 for (const v of island.vertices) {
198 parts.push(v.title);
199 if (v.description) parts.push(v.description);
200 }
201
202 for (const e of island.edges) {
203 if (e.note) parts.push(e.note);
204 }
205
206 return parts.join(" ");
207}
208
209// ─── Carry touchpoint matching (D1 records) ────────────────────────
210
211function findCarryTouchpoints(
212 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
213 carryData: CarryData,
214): NewConnection[] {
215 const connections: NewConnection[] = [];
216
217 // --- Match citations by URL ---
218 for (const cite of carryData.citations) {
219 // Direct URL match: citation URL is a vertex URI
220 const vertexMatch = island.vertices.find(v => v.uri === cite.url);
221 if (vertexMatch) {
222 connections.push({
223 source: cite.url,
224 target: cite.atUri,
225 connectionType: "network.cosmik.connection#annotates",
226 note: cite.takeaway.slice(0, 500),
227 });
228 continue;
229 }
230
231 // Semantic match: citation title/takeaway overlaps with island vertices
232 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
233 const bestVertex = findBestVertexMatch(citeText, island.vertices);
234 if (bestVertex) {
235 const overlap = keywordOverlap(citeText, `${bestVertex.title} ${bestVertex.description || ""}`);
236 if (overlap.length >= 2) {
237 connections.push({
238 source: bestVertex.uri,
239 target: cite.atUri,
240 connectionType: "network.cosmik.connection#annotates",
241 note: cite.takeaway.slice(0, 500),
242 });
243 }
244 }
245 }
246
247 // --- Match entities by name/description ---
248 for (const entity of carryData.entities) {
249 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
250 const bestVertex = findBestVertexMatch(entityText, island.vertices);
251 if (bestVertex) {
252 const overlap = keywordOverlap(entityText, `${bestVertex.title} ${bestVertex.description || ""}`);
253 if (overlap.length >= 2) {
254 connections.push({
255 source: bestVertex.uri,
256 target: entity.atUri,
257 connectionType: "network.cosmik.connection#relates",
258 note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
259 });
260 }
261 }
262 }
263
264 // --- Match reasonings by claim/evidence ---
265 for (const reasoning of carryData.reasonings) {
266 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
267 const bestVertex = findBestVertexMatch(reasoningText, island.vertices);
268 if (bestVertex) {
269 const overlap = keywordOverlap(reasoningText, `${bestVertex.title} ${bestVertex.description || ""}`);
270 if (overlap.length >= 2) {
271 const connType = reasoning.reasoningType?.includes("contradict")
272 ? "network.cosmik.connection#contradicts"
273 : "network.cosmik.connection#extends";
274 connections.push({
275 source: bestVertex.uri,
276 target: reasoning.atUri,
277 connectionType: connType,
278 note: reasoning.claim.slice(0, 500),
279 });
280 }
281 }
282 }
283
284 return connections;
285}
286
287// ─── Filesystem touchpoint matching (vault + carry notes) ───────────
288
289function findFilesystemTouchpoints(
290 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
291 vaultNotes: VaultNote[],
292 carryNotes: CarryNote[],
293): NewConnection[] {
294 const connections: NewConnection[] = [];
295 const islandText = buildIslandText(island).toLowerCase();
296
297 // Match vault notes to island vertices
298 for (const note of vaultNotes) {
299 const noteLower = note.content.toLowerCase();
300 const overlap = keywordOverlap(islandText, noteLower);
301 if (overlap.length < 3) continue;
302
303 // Find the best vertex for this vault note
304 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
305 if (bestVertex) {
306 // Extract title from first heading or first line
307 const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path;
308 connections.push({
309 source: bestVertex.uri,
310 target: `vault://${note.path}`,
311 connectionType: "network.cosmik.connection#annotates",
312 note: `Vault note: ${title.slice(0, 200)}`,
313 });
314 }
315 }
316
317 // Match carry notes to island vertices
318 for (const note of carryNotes) {
319 const noteLower = note.content.toLowerCase();
320 const overlap = keywordOverlap(islandText, noteLower);
321 if (overlap.length < 2) continue;
322
323 const bestVertex = findBestVertexMatch(noteLower, island.vertices);
324 if (bestVertex) {
325 const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path;
326 connections.push({
327 source: bestVertex.uri,
328 target: `carry://${note.domain}/${note.path}`,
329 connectionType: "network.cosmik.connection#relates",
330 note: `Carry [${note.domain}]: ${title.slice(0, 200)}`,
331 });
332 }
333 }
334
335 return connections;
336}
337
338function findBestVertexMatch(
339 text: string,
340 vertices: VertexInfo[],
341): VertexInfo | null {
342 let bestScore = 0;
343 let bestVertex: VertexInfo | null = null;
344
345 for (const v of vertices) {
346 const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
347 const overlap = keywordOverlap(text, vertexText);
348 if (overlap.length > bestScore) {
349 bestScore = overlap.length;
350 bestVertex = v;
351 }
352 }
353
354 return bestScore >= 2 ? bestVertex : null;
355}
356
357// ─── Analysis functions ─────────────────────────────────────────────
358
359function extractThemes(text: string): string[] {
360 const keywords: Record<string, string[]> = {
361 "open science": ["open science", "open access", "oa", "preprint"],
362 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
363 "reproducibility": ["reproducib", "replicab", "replication"],
364 "decentralization": ["decentraliz", "federat", "self-host"],
365 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
366 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
367 "community": ["community", "collective", "cooperative", "commons"],
368 "governance": ["governance", "policy", "regulation", "standards"],
369 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
370 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
371 "provenance": ["provenance", "lineage", "traceability", "attribution"],
372 "doi": ["doi", "zenodo", "citeable", "citation"],
373 "privacy": ["privacy", "encryption", "e2ee", "consent"],
374 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
375 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
376 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
377 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
378 };
379
380 const lower = text.toLowerCase();
381 return Object.entries(keywords)
382 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
383 .map(([theme]) => theme);
384}
385
386function findHotEdges(
387 islandText: string,
388 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
389 vaultNotes: VaultNote[],
390 carryNotes: CarryNote[],
391 newConnections: NewConnection[],
392): string[] {
393 const highlights: string[] = [];
394 const lower = islandText.toLowerCase();
395
396 // Score each edge by semantic significance
397 const edgeScores = new Map<string, number>();
398
399 for (const edge of island.edges) {
400 let score = 0;
401
402 // Check if source or target appears in vault notes
403 for (const note of vaultNotes) {
404 const noteLower = note.content.toLowerCase();
405 const overlap = keywordOverlap(lower, noteLower);
406 if (overlap.length >= 2) {
407 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
408 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
409 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
410 if (targetTitle && noteLower.includes(targetTitle)) score += 2;
411 }
412 }
413
414 // Check if source or target appears in carry notes
415 for (const note of carryNotes) {
416 const noteLower = note.content.toLowerCase();
417 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
418 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
419 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
420 if (targetTitle && noteLower.includes(targetTitle)) score += 1;
421 }
422
423 // Edges with notes are more interesting
424 if (edge.note && edge.note.length > 10) score += 1;
425
426 // Edges with semantic connection types are more interesting
427 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
428 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
429
430 // Boost edges whose source or target is also connected to carry data
431 for (const nc of newConnections) {
432 if (edge.source === nc.source || edge.target === nc.source) score += 3;
433 if (edge.source === nc.target || edge.target === nc.target) score += 3;
434 }
435
436 if (score > 0) {
437 edgeScores.set(edge.uri, score);
438 }
439 }
440
441 // Return top edges by score
442 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
443 for (const [uri] of sorted.slice(0, 10)) {
444 highlights.push(uri);
445 }
446
447 return highlights;
448}
449
450function findTensionsFromNotes(
451 islandText: string,
452 carryNotes: CarryNote[],
453): string[] {
454 const tensions: string[] = [];
455 const lower = islandText.toLowerCase();
456
457 for (const note of carryNotes) {
458 const noteLower = note.content.toLowerCase();
459 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
460 const overlap = keywordOverlap(lower, noteLower);
461 if (overlap.length >= 1) {
462 tensions.push(
463 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
464 );
465 }
466 }
467 }
468
469 return tensions.slice(0, 5);
470}
471
472function findTensionsFromReasonings(
473 islandText: string,
474 reasonings: CarryReasoning[],
475): string[] {
476 const tensions: string[] = [];
477 const lower = islandText.toLowerCase();
478
479 for (const r of reasonings) {
480 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
481 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
482 const overlap = keywordOverlap(lower, rLower);
483 if (overlap.length >= 1) {
484 tensions.push(r.claim.slice(0, 500));
485 }
486 }
487 }
488
489 return tensions.slice(0, 5);
490}
491
492function generateOpenQuestions(
493 islandText: string,
494 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
495 highlights: string[],
496 tensions: string[],
497 newConnections: NewConnection[],
498): string[] {
499 const questions: string[] = [];
500 const lower = islandText.toLowerCase();
501
502 if (newConnections.length > 0) {
503 questions.push(
504 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
505 );
506 }
507
508 if (tensions.length > 0) {
509 questions.push(
510 `What evidence would resolve the ${tensions.length} tension(s) identified?`
511 );
512 }
513
514 const vertexCount = island.vertices.length;
515 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
516
517 if (edgeTypes.size > 1) {
518 questions.push(
519 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
520 );
521 }
522
523 if (vertexCount > 5) {
524 questions.push(
525 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
526 );
527 }
528
529 if (lower.includes("data") || lower.includes("dataset")) {
530 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
531 }
532 if (lower.includes("standard") || lower.includes("protocol")) {
533 questions.push("Does this cluster propose a new standard, and who are the adopters?");
534 }
535
536 return questions.slice(0, 5);
537}
538
539function synthesize(
540 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
541 themes: string[],
542 highlights: string[],
543 tensions: string[],
544 openQuestions: string[],
545 newConnections: NewConnection[],
546): string {
547 const vertexCount = island.vertices.length;
548 const edgeCount = island.edges.length;
549 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
550
551 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
552
553 const themeStr = themes.length > 0
554 ? `Themes: ${themes.join(", ")}.`
555 : "";
556
557 const touchpointStr = newConnections.length > 0
558 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
559 : "No carry touchpoints found.";
560
561 const highlightStr = highlights.length > 0
562 ? `${highlights.length} existing edges flagged as semantically significant.`
563 : "";
564
565 const tensionStr = tensions.length > 0
566 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
567 : "";
568
569 const questionStr = openQuestions.length > 0
570 ? `Open questions: ${openQuestions.join(" ")}`
571 : "";
572
573 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
574}
575
576// ─── Utilities ──────────────────────────────────────────────────────
577
578const STOP_WORDS = new Set([
579 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
580 "have", "has", "had", "do", "does", "did", "will", "would", "could",
581 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
582 "on", "with", "at", "by", "from", "as", "into", "through", "during",
583 "before", "after", "above", "below", "between", "and", "but", "or",
584 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
585 "http", "https", "www", "com", "org",
586]);
587
588function keywordOverlap(textA: string, textB: string): string[] {
589 const wordsA = new Set(
590 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
591 );
592 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
593 return wordsB.filter(w => wordsA.has(w));
594}