nandi.uk
This repository has no description
13 kB
380 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Reads vault (markdown) + carry (markdown) from local filesystem
3// Cross-references an island (cluster of linked URLs) against local knowledge
4// Returns structured analysis with carry cross-refs
5//
6// This runs inside a Cloudflare Container with synced vault + carry data.
7
8import { readdirSync, readFileSync, statSync } from "fs";
9import { join, extname } from "path";
10
11const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
12const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
13
14// ─── Types ──────────────────────────────────────────────────────────
15
16interface VaultNote {
17 path: string;
18 content: string;
19}
20
21interface CarryNote {
22 path: string;
23 domain: string;
24 content: string;
25}
26
27interface VertexInfo {
28 uri: string;
29 title: string;
30 description: string;
31 type: string;
32}
33
34interface EdgeInfo {
35 uri: string;
36 did: string;
37 source: string;
38 target: string;
39 connectionType: string;
40 note: string;
41}
42
43interface Connection {
44 description: string;
45 targetUri: string;
46 relation: string;
47 source: "vault" | "carry";
48 sourcePath?: string;
49}
50
51interface CarryRef {
52 carryId: string;
53 atUri: string;
54 summary: string;
55}
56
57interface AnalysisInput {
58 did: string;
59 island: {
60 vertices: VertexInfo[];
61 edges: EdgeInfo[];
62 };
63}
64
65interface AnalysisResult {
66 themes: string[];
67 connections: Connection[];
68 tensions: string[];
69 openQuestions: string[];
70 synthesis: string;
71 carryRefs: CarryRef[];
72}
73
74// ─── Main analysis function ────────────────────────────────────────
75
76export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
77 const { island } = input;
78
79 // 1. Read vault notes from filesystem
80 const vaultNotes = readVault();
81
82 // 2. Read carry notes from filesystem
83 const carryNotes = readCarry();
84
85 // 3. Build corpus from the entire island
86 const islandText = buildIslandText(island);
87
88 // 4. Extract themes from the island
89 const themes = extractThemes(islandText);
90
91 // 5. Find connections to vault + carry
92 const connections = findConnections(islandText, island, vaultNotes, carryNotes);
93
94 // 6. Find tensions
95 const tensions = findTensions(islandText, carryNotes);
96
97 // 7. Generate open questions
98 const openQuestions = generateOpenQuestions(islandText, island, connections, tensions);
99
100 // 8. Synthesize prose
101 const synthesis = synthesize(island, themes, connections, tensions, openQuestions);
102
103 // 9. Build carry cross-refs
104 const carryRefs = buildCarryRefs(connections);
105
106 return { themes, connections, tensions, openQuestions, synthesis, carryRefs };
107}
108
109// ─── Filesystem readers ────────────────────────────────────────────
110
111function readVault(): VaultNote[] {
112 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
113 path: relPath,
114 content,
115 }));
116}
117
118function readCarry(): CarryNote[] {
119 const notes: CarryNote[] = [];
120 try {
121 const domains = readdirSync(CARRY_PATH);
122 for (const domain of domains) {
123 const domainPath = join(CARRY_PATH, domain);
124 if (!statSync(domainPath).isDirectory()) continue;
125 if (domain.startsWith(".")) continue;
126 const files = readMarkdownTree(domainPath);
127 for (const { relPath, content } of files) {
128 notes.push({ path: relPath, domain, content });
129 }
130 }
131 } catch (e: any) {
132 console.error("Failed to read carry:", e.message);
133 }
134 return notes;
135}
136
137function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
138 const results: Array<{ relPath: string; content: string }> = [];
139 try {
140 const entries = readdirSync(root, { withFileTypes: true });
141 for (const entry of entries) {
142 const fullPath = join(root, entry.name);
143 if (entry.name.startsWith(".")) continue;
144 if (entry.isDirectory()) {
145 results.push(...readMarkdownTree(fullPath));
146 } else if (extname(entry.name) === ".md") {
147 try {
148 const content = readFileSync(fullPath, "utf-8");
149 results.push({ relPath: fullPath.replace(root + "/", ""), content });
150 } catch {}
151 }
152 }
153 } catch {}
154 return results;
155}
156
157// ─── Build island text corpus ──────────────────────────────────────
158
159function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
160 const parts: string[] = [];
161
162 for (const v of island.vertices) {
163 parts.push(v.title);
164 if (v.description) parts.push(v.description);
165 }
166
167 for (const e of island.edges) {
168 if (e.note) parts.push(e.note);
169 }
170
171 return parts.join(" ");
172}
173
174// ─── Analysis functions ─────────────────────────────────────────────
175
176function extractThemes(text: string): string[] {
177 const keywords: Record<string, string[]> = {
178 "open science": ["open science", "open access", "oa", "preprint"],
179 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
180 "reproducibility": ["reproducib", "replicab", "replication"],
181 "decentralization": ["decentraliz", "federat", "self-host"],
182 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
183 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
184 "community": ["community", "collective", "cooperative", "commons"],
185 "governance": ["governance", "policy", "regulation", "standards"],
186 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
187 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
188 "provenance": ["provenance", "lineage", "traceability", "attribution"],
189 "doi": ["doi", "zenodo", "citeable", "citation"],
190 "privacy": ["privacy", "encryption", "e2ee", "consent"],
191 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
192 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
193 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
194 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
195 };
196
197 const lower = text.toLowerCase();
198 return Object.entries(keywords)
199 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
200 .map(([theme]) => theme);
201}
202
203function findConnections(
204 islandText: string,
205 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
206 vaultNotes: VaultNote[],
207 carryNotes: CarryNote[],
208): Connection[] {
209 const connections: Connection[] = [];
210 const lower = islandText.toLowerCase();
211
212 // Check vault notes for overlapping themes
213 for (const note of vaultNotes) {
214 const noteLower = note.content.toLowerCase();
215 const overlap = keywordOverlap(lower, noteLower);
216 if (overlap.length >= 2) {
217 connections.push({
218 description: `Vault note "${note.path}" shares themes: ${overlap.join(", ")}`,
219 targetUri: "",
220 relation: "contextualizes",
221 source: "vault",
222 sourcePath: note.path,
223 });
224 }
225 }
226
227 // Check carry notes for related claims
228 for (const note of carryNotes) {
229 const noteLower = note.content.toLowerCase();
230 const overlap = keywordOverlap(lower, noteLower);
231 if (overlap.length >= 1) {
232 // Extract wikilinks as potential AT URIs
233 const wikilinks = note.content.match(/\[\[([^\]]+)\]\]/g) || [];
234 const atUri = wikilinks[0]?.replace("[[", "").replace("]]", "") || "";
235
236 connections.push({
237 description: `Carry (${note.domain}): "${note.path}" — overlapping: ${overlap.join(", ")}`,
238 targetUri: atUri,
239 relation: "supports",
240 source: "carry",
241 sourcePath: note.path,
242 });
243 }
244 }
245
246 return connections.slice(0, 10);
247}
248
249function findTensions(
250 islandText: string,
251 carryNotes: CarryNote[],
252): string[] {
253 const tensions: string[] = [];
254 const lower = islandText.toLowerCase();
255
256 for (const note of carryNotes) {
257 const noteLower = note.content.toLowerCase();
258 // Look for tension markers in carry notes
259 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
260 const overlap = keywordOverlap(lower, noteLower);
261 if (overlap.length >= 1) {
262 tensions.push(
263 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
264 );
265 }
266 }
267 }
268
269 return tensions.slice(0, 5);
270}
271
272function generateOpenQuestions(
273 islandText: string,
274 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
275 connections: Connection[],
276 tensions: string[],
277): string[] {
278 const questions: string[] = [];
279 const lower = islandText.toLowerCase();
280
281 if (connections.length > 0) {
282 const carryCount = connections.filter(c => c.source === "carry").length;
283 const vaultCount = connections.filter(c => c.source === "vault").length;
284 questions.push(
285 `How does this island extend or challenge your existing ${carryCount} carry and ${vaultCount} vault connections?`
286 );
287 }
288
289 if (tensions.length > 0) {
290 questions.push(
291 `What evidence would resolve the ${tensions.length} tension(s) identified?`
292 );
293 }
294
295 const vertexCount = island.vertices.length;
296 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
297
298 if (edgeTypes.size > 1) {
299 questions.push(
300 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
301 );
302 }
303
304 if (vertexCount > 5) {
305 questions.push(
306 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
307 );
308 }
309
310 if (lower.includes("data") || lower.includes("dataset")) {
311 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
312 }
313 if (lower.includes("standard") || lower.includes("protocol")) {
314 questions.push("Does this cluster propose a new standard, and who are the adopters?");
315 }
316
317 return questions.slice(0, 5);
318}
319
320function synthesize(
321 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
322 themes: string[],
323 connections: Connection[],
324 tensions: string[],
325 openQuestions: string[],
326): string {
327 const vertexCount = island.vertices.length;
328 const edgeCount = island.edges.length;
329 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
330
331 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
332
333 const themeStr = themes.length > 0
334 ? `Themes: ${themes.join(", ")}.`
335 : "";
336
337 const connStr = connections.length > 0
338 ? `Connects to ${connections.length} items in your knowledge base: ${connections.slice(0, 3).map(c => c.description).join("; ")}.`
339 : "No direct connections found in your existing knowledge.";
340
341 const tensionStr = tensions.length > 0
342 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
343 : "";
344
345 const questionStr = openQuestions.length > 0
346 ? `Open questions: ${openQuestions.join(" ")}`
347 : "";
348
349 return [`Island (${clusterDesc}).`, themeStr, connStr, tensionStr, questionStr].filter(Boolean).join(" ");
350}
351
352function buildCarryRefs(connections: Connection[]): CarryRef[] {
353 return connections
354 .filter(c => c.source === "carry")
355 .map(c => ({
356 carryId: c.sourcePath || "",
357 atUri: c.targetUri || "",
358 summary: c.description,
359 }));
360}
361
362// ─── Utilities ──────────────────────────────────────────────────────
363
364const STOP_WORDS = new Set([
365 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
366 "have", "has", "had", "do", "does", "did", "will", "would", "could",
367 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
368 "on", "with", "at", "by", "from", "as", "into", "through", "during",
369 "before", "after", "above", "below", "between", "and", "but", "or",
370 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
371 "http", "https", "www", "com", "org",
372]);
373
374function keywordOverlap(textA: string, textB: string): string[] {
375 const wordsA = new Set(
376 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
377 );
378 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
379 return wordsB.filter(w => wordsA.has(w));
380}