nandi.uk
This repository has no description
13 kB
378 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Reads vault (markdown) + carry (markdown) from local filesystem
3// Cross-references an island (cluster of linked URLs) against local knowledge
4// Returns structured analysis with carry cross-refs
5//
6// This runs inside a Cloudflare Container with synced vault + carry data.
7
8import { readdirSync, readFileSync, statSync } from "fs";
9import { join, extname } from "path";
10
11const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
12const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
13
14// ─── Types ──────────────────────────────────────────────────────────
15
16interface VaultNote {
17 path: string;
18 content: string;
19}
20
21interface CarryNote {
22 path: string;
23 domain: string;
24 content: string;
25}
26
27interface VertexInfo {
28 uri: string;
29 title: string;
30 description: string;
31 type: string;
32}
33
34interface EdgeInfo {
35 uri: string;
36 did: string;
37 source: string;
38 target: string;
39 connectionType: string;
40 note: string;
41}
42
43interface CarryRef {
44 carryId: string;
45 atUri: string;
46 summary: string;
47}
48
49interface AnalysisInput {
50 did: string;
51 island: {
52 vertices: VertexInfo[];
53 edges: EdgeInfo[];
54 };
55}
56
57interface AnalysisResult {
58 themes: string[];
59 highlights: string[];
60 tensions: string[];
61 openQuestions: string[];
62 synthesis: string;
63 carryRefs: CarryRef[];
64}
65
66// ─── Main analysis function ────────────────────────────────────────
67
68export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
69 const { island } = input;
70
71 // 1. Read vault notes from filesystem
72 const vaultNotes = readVault();
73
74 // 2. Read carry notes from filesystem
75 const carryNotes = readCarry();
76
77 // 3. Build corpus from the entire island
78 const islandText = buildIslandText(island);
79
80 // 4. Extract themes from the island
81 const themes = extractThemes(islandText);
82
83 // 5. Flag hot edges — existing edges that are semantically significant
84 const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes);
85
86 // 6. Find tensions
87 const tensions = findTensions(islandText, carryNotes);
88
89 // 7. Generate open questions
90 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions);
91
92 // 8. Synthesize prose
93 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions);
94
95 // 9. Build carry cross-refs
96 const carryRefs = buildCarryRefs(connections);
97
98 return { themes, highlights, tensions, openQuestions, synthesis, carryRefs };
99}
100
101// ─── Filesystem readers ────────────────────────────────────────────
102
103function readVault(): VaultNote[] {
104 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
105 path: relPath,
106 content,
107 }));
108}
109
110function readCarry(): CarryNote[] {
111 const notes: CarryNote[] = [];
112 try {
113 const domains = readdirSync(CARRY_PATH);
114 for (const domain of domains) {
115 const domainPath = join(CARRY_PATH, domain);
116 if (!statSync(domainPath).isDirectory()) continue;
117 if (domain.startsWith(".")) continue;
118 const files = readMarkdownTree(domainPath);
119 for (const { relPath, content } of files) {
120 notes.push({ path: relPath, domain, content });
121 }
122 }
123 } catch (e: any) {
124 console.error("Failed to read carry:", e.message);
125 }
126 return notes;
127}
128
129function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
130 const results: Array<{ relPath: string; content: string }> = [];
131 try {
132 const entries = readdirSync(root, { withFileTypes: true });
133 for (const entry of entries) {
134 const fullPath = join(root, entry.name);
135 if (entry.name.startsWith(".")) continue;
136 if (entry.isDirectory()) {
137 results.push(...readMarkdownTree(fullPath));
138 } else if (extname(entry.name) === ".md") {
139 try {
140 const content = readFileSync(fullPath, "utf-8");
141 results.push({ relPath: fullPath.replace(root + "/", ""), content });
142 } catch {}
143 }
144 }
145 } catch {}
146 return results;
147}
148
149// ─── Build island text corpus ──────────────────────────────────────
150
151function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
152 const parts: string[] = [];
153
154 for (const v of island.vertices) {
155 parts.push(v.title);
156 if (v.description) parts.push(v.description);
157 }
158
159 for (const e of island.edges) {
160 if (e.note) parts.push(e.note);
161 }
162
163 return parts.join(" ");
164}
165
166// ─── Analysis functions ─────────────────────────────────────────────
167
168function extractThemes(text: string): string[] {
169 const keywords: Record<string, string[]> = {
170 "open science": ["open science", "open access", "oa", "preprint"],
171 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
172 "reproducibility": ["reproducib", "replicab", "replication"],
173 "decentralization": ["decentraliz", "federat", "self-host"],
174 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
175 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
176 "community": ["community", "collective", "cooperative", "commons"],
177 "governance": ["governance", "policy", "regulation", "standards"],
178 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
179 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
180 "provenance": ["provenance", "lineage", "traceability", "attribution"],
181 "doi": ["doi", "zenodo", "citeable", "citation"],
182 "privacy": ["privacy", "encryption", "e2ee", "consent"],
183 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
184 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
185 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
186 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
187 };
188
189 const lower = text.toLowerCase();
190 return Object.entries(keywords)
191 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
192 .map(([theme]) => theme);
193}
194
195function findHotEdges(
196 islandText: string,
197 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
198 vaultNotes: VaultNote[],
199 carryNotes: CarryNote[],
200): string[] {
201 const highlights: string[] = [];
202 const lower = islandText.toLowerCase();
203
204 // Score each edge by how much it connects to vault/carry knowledge
205 const edgeScores = new Map<string, number>();
206
207 for (const edge of island.edges) {
208 let score = 0;
209
210 // Check if source or target appears in vault notes
211 for (const note of vaultNotes) {
212 const noteLower = note.content.toLowerCase();
213 const overlap = keywordOverlap(lower, noteLower);
214 if (overlap.length >= 2) {
215 // Does this edge's source or target title appear in the vault note?
216 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
217 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
218 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
219 if (targetTitle && noteLower.includes(targetTitle)) score += 2;
220 }
221 }
222
223 // Check if source or target appears in carry notes
224 for (const note of carryNotes) {
225 const noteLower = note.content.toLowerCase();
226 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
227 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
228 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
229 if (targetTitle && noteLower.includes(targetTitle)) score += 1;
230 }
231
232 // Edges with notes are more interesting
233 if (edge.note && edge.note.length > 10) score += 1;
234
235 // Edges with semantic connection types are more interesting
236 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
237 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
238
239 if (score > 0) {
240 edgeScores.set(edge.uri, score);
241 }
242 }
243
244 // Return top edges by score
245 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
246 for (const [uri] of sorted.slice(0, 10)) {
247 highlights.push(uri);
248 }
249
250 return highlights;
251}
252
253function findTensions(
254 islandText: string,
255 carryNotes: CarryNote[],
256): string[] {
257 const tensions: string[] = [];
258 const lower = islandText.toLowerCase();
259
260 for (const note of carryNotes) {
261 const noteLower = note.content.toLowerCase();
262 // Look for tension markers in carry notes
263 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
264 const overlap = keywordOverlap(lower, noteLower);
265 if (overlap.length >= 1) {
266 tensions.push(
267 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
268 );
269 }
270 }
271 }
272
273 return tensions.slice(0, 5);
274}
275
276function generateOpenQuestions(
277 islandText: string,
278 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
279 highlights: string[],
280 tensions: string[],
281): string[] {
282 const questions: string[] = [];
283 const lower = islandText.toLowerCase();
284
285 if (highlights.length > 0) {
286 questions.push(
287 `How does this island extend or challenge your existing knowledge? ${highlights.length} edges flagged as significant.`
288 );
289 }
290
291 if (tensions.length > 0) {
292 questions.push(
293 `What evidence would resolve the ${tensions.length} tension(s) identified?`
294 );
295 }
296
297 const vertexCount = island.vertices.length;
298 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
299
300 if (edgeTypes.size > 1) {
301 questions.push(
302 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
303 );
304 }
305
306 if (vertexCount > 5) {
307 questions.push(
308 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
309 );
310 }
311
312 if (lower.includes("data") || lower.includes("dataset")) {
313 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
314 }
315 if (lower.includes("standard") || lower.includes("protocol")) {
316 questions.push("Does this cluster propose a new standard, and who are the adopters?");
317 }
318
319 return questions.slice(0, 5);
320}
321
322function synthesize(
323 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
324 themes: string[],
325 highlights: string[],
326 tensions: string[],
327 openQuestions: string[],
328): string {
329 const vertexCount = island.vertices.length;
330 const edgeCount = island.edges.length;
331 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
332
333 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
334
335 const themeStr = themes.length > 0
336 ? `Themes: ${themes.join(", ")}.`
337 : "";
338
339 const highlightStr = highlights.length > 0
340 ? `${highlights.length} edges flagged as semantically significant — connecting to your existing knowledge.`
341 : "No edges flagged as significant against your existing knowledge.";
342
343 const tensionStr = tensions.length > 0
344 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
345 : "";
346
347 const questionStr = openQuestions.length > 0
348 ? `Open questions: ${openQuestions.join(" ")}`
349 : "";
350
351 return [`Island (${clusterDesc}).`, themeStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
352}
353
354function buildCarryRefs(highlights: string[]): CarryRef[] {
355 // No carry-specific refs without the old connection objects
356 // Carry cross-refs are now implicit via the hot edge flags
357 return [];
358}
359
360// ─── Utilities ──────────────────────────────────────────────────────
361
362const STOP_WORDS = new Set([
363 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
364 "have", "has", "had", "do", "does", "did", "will", "would", "could",
365 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
366 "on", "with", "at", "by", "from", "as", "into", "through", "during",
367 "before", "after", "above", "below", "between", "and", "but", "or",
368 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
369 "http", "https", "www", "com", "org",
370]);
371
372function keywordOverlap(textA: string, textB: string): string[] {
373 const wordsA = new Set(
374 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
375 );
376 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
377 return wordsB.filter(w => wordsA.has(w));
378}