nandi.uk
This repository has no description
14 kB
412 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Reads vault (markdown) + carry (JSON) from R2
3// Cross-references source content against local knowledge
4// Returns structured analysis with carry cross-refs
5//
6// This runs inside a Cloudflare Container with R2 access.
7// The container is per-user (each DID gets its own instance).
8
9import {
10 S3Client,
11 GetObjectCommand,
12 ListObjectsV2Command,
13} from "@aws-sdk/client-s3";
14
15// R2 config — injected via Worker envVars when the container starts
16const R2_ACCOUNT_ID = process.env.R2_ACCOUNT_ID || "";
17const R2_ACCESS_KEY = process.env.R2_ACCESS_KEY || "";
18const R2_SECRET_KEY = process.env.R2_SECRET_KEY || "";
19
20const s3 = new S3Client({
21 region: "auto",
22 endpoint: `https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
23 credentials: {
24 accessKeyId: R2_ACCESS_KEY,
25 secretAccessKey: R2_SECRET_KEY,
26 },
27});
28
29const VAULT_BUCKET = "stigmergic-vault";
30const CARRY_BUCKET = "stigmergic-carry";
31
32// ─── Types ──────────────────────────────────────────────────────────
33
34interface VaultNote {
35 path: string;
36 content: string;
37}
38
39interface CarryClaim {
40 id: string;
41 body: string;
42 summary: string;
43 sentiment: string;
44 qualifier: string;
45 atUri: string;
46 entityType: string;
47}
48
49interface Connection {
50 description: string;
51 targetUri: string;
52 relation: string;
53 source: "vault" | "carry";
54 sourcePath?: string;
55 carryId?: string;
56}
57
58interface CarryRef {
59 entityType: "claim" | "citation" | "entity" | "reasoning";
60 carryId: string;
61 atUri: string;
62 summary: string;
63}
64
65interface AnalysisInput {
66 sourceUri: string;
67 sourceCid?: string;
68 sourceRecord: any;
69 did: string;
70}
71
72interface AnalysisResult {
73 themes: string[];
74 connections: Connection[];
75 tensions: string[];
76 openQuestions: string[];
77 synthesis: string;
78 carryRefs: CarryRef[];
79}
80
81// ─── Main analysis function ────────────────────────────────────────
82
83export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
84 const { sourceRecord, did } = input;
85
86 // 1. Read vault notes from R2
87 const vaultNotes = await readVault(did);
88
89 // 2. Read carry claims from R2
90 const carryClaims = await readCarry(did);
91
92 // 3. Extract themes from source record
93 const sourceText = extractText(sourceRecord);
94 const themes = extractThemes(sourceText);
95
96 // 4. Find connections in vault + carry
97 const connections = findConnections(sourceText, vaultNotes, carryClaims);
98
99 // 5. Find tensions (contradictions, gaps)
100 const tensions = findTensions(sourceText, carryClaims);
101
102 // 6. Generate open questions
103 const openQuestions = generateOpenQuestions(sourceText, connections, tensions);
104
105 // 7. Synthesize prose
106 const synthesis = synthesize(
107 sourceText,
108 themes,
109 connections,
110 tensions,
111 openQuestions
112 );
113
114 // 8. Build carry cross-refs
115 const carryRefs = buildCarryRefs(connections, carryClaims);
116
117 return {
118 themes,
119 connections,
120 tensions,
121 openQuestions,
122 synthesis,
123 carryRefs,
124 };
125}
126
127// ─── R2 readers ─────────────────────────────────────────────────────
128
129async function readVault(did: string): Promise<VaultNote[]> {
130 const prefix = `${did}/`;
131 try {
132 const list = await s3.send(
133 new ListObjectsV2Command({ Bucket: VAULT_BUCKET, Prefix: prefix })
134 );
135
136 const notes: VaultNote[] = [];
137 for (const obj of list.Contents || []) {
138 if (!obj.Key?.endsWith(".md")) continue;
139 try {
140 const get = await s3.send(
141 new GetObjectCommand({ Bucket: VAULT_BUCKET, Key: obj.Key })
142 );
143 const body = await get.Body?.transformToString();
144 if (body) {
145 notes.push({
146 path: obj.Key.replace(prefix, ""),
147 content: body,
148 });
149 }
150 } catch (e) {
151 console.error(`Failed to read vault file ${obj.Key}:`, e);
152 }
153 }
154 return notes;
155 } catch (e) {
156 console.error("Failed to list vault objects:", e);
157 return [];
158 }
159}
160
161async function readCarry(did: string): Promise<CarryClaim[]> {
162 const prefix = `${did}/`;
163 try {
164 const list = await s3.send(
165 new ListObjectsV2Command({ Bucket: CARRY_BUCKET, Prefix: prefix })
166 );
167
168 const claims: CarryClaim[] = [];
169 for (const obj of list.Contents || []) {
170 if (!obj.Key?.endsWith(".json")) continue;
171 try {
172 const get = await s3.send(
173 new GetObjectCommand({ Bucket: CARRY_BUCKET, Key: obj.Key })
174 );
175 const body = await get.Body?.transformToString();
176 if (body) {
177 try {
178 const data = JSON.parse(body);
179 if (Array.isArray(data)) {
180 claims.push(...data.map(normalizeClaim));
181 } else {
182 claims.push(normalizeClaim(data));
183 }
184 } catch {
185 // Skip malformed JSON
186 }
187 }
188 } catch (e) {
189 console.error(`Failed to read carry file ${obj.Key}:`, e);
190 }
191 }
192 return claims;
193 } catch (e) {
194 console.error("Failed to list carry objects:", e);
195 return [];
196 }
197}
198
199// ─── Analysis functions ─────────────────────────────────────────────
200
201function extractText(record: any): string {
202 // Extract text from various Semble record shapes
203 const parts: string[] = [];
204 if (record.title) parts.push(record.title);
205 if (record.text) parts.push(record.text);
206 if (record.body) parts.push(record.body);
207 if (record.content) parts.push(record.content);
208 if (record.description) parts.push(record.description);
209 if (record.embed?.title) parts.push(record.embed.title);
210 if (record.embed?.description) parts.push(record.embed.description);
211 // Semble card shape
212 if (record.type === "URL" && record.content?.url) parts.push(record.content.url);
213 if (record.content?.metadata?.title) parts.push(record.content.metadata.title);
214 if (record.content?.metadata?.description) parts.push(record.content.metadata.description);
215 if (record.type === "NOTE" && record.content?.text) parts.push(record.content.text);
216 return parts.join(" ");
217}
218
219function extractThemes(text: string): string[] {
220 // Keyword-based theme extraction. In production, call an LLM.
221 const keywords: Record<string, string[]> = {
222 "open science": ["open science", "open access", "oa", "preprint"],
223 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
224 "reproducibility": ["reproducib", "replicab", "replication"],
225 "atproto": ["atproto", "at protocol", "bluesky", "bsky", "semble"],
226 "decentralization": ["decentraliz", "federat", "self-host"],
227 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
228 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
229 "community": ["community", "collective", "cooperative", "commons"],
230 "governance": ["governance", "policy", "regulation", "standards"],
231 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
232 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
233 "provenance": ["provenance", "lineage", "traceability", "attribution"],
234 "doi": ["doi", "zenodo", "citeable", "citation"],
235 "privacy": ["privacy", "encryption", "e2ee", "consent"],
236 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
237 };
238
239 const lower = text.toLowerCase();
240 return Object.entries(keywords)
241 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
242 .map(([theme]) => theme);
243}
244
245function findConnections(
246 sourceText: string,
247 vaultNotes: VaultNote[],
248 carryClaims: CarryClaim[]
249): Connection[] {
250 const connections: Connection[] = [];
251 const lower = sourceText.toLowerCase();
252
253 // Check vault notes for overlapping themes
254 for (const note of vaultNotes) {
255 const noteLower = note.content.toLowerCase();
256 const overlap = keywordOverlap(lower, noteLower);
257 if (overlap.length >= 2) {
258 connections.push({
259 description: `Vault note "${note.path}" shares themes: ${overlap.join(", ")}`,
260 targetUri: "",
261 relation: "contextualizes",
262 source: "vault",
263 sourcePath: note.path,
264 });
265 }
266 }
267
268 // Check carry claims for related claims
269 for (const claim of carryClaims) {
270 const claimText = (claim.body || claim.summary || "").toLowerCase();
271 const overlap = keywordOverlap(lower, claimText);
272 if (overlap.length >= 1) {
273 connections.push({
274 description: `Carry claim: "${claim.summary || claim.body?.slice(0, 100)}"`,
275 targetUri: claim.atUri || "",
276 relation: claim.sentiment === "negative" ? "contradicts" : "supports",
277 source: "carry",
278 carryId: claim.id,
279 });
280 }
281 }
282
283 return connections.slice(0, 10);
284}
285
286function findTensions(
287 sourceText: string,
288 carryClaims: CarryClaim[]
289): string[] {
290 const tensions: string[] = [];
291 const lower = sourceText.toLowerCase();
292
293 for (const claim of carryClaims) {
294 if (claim.sentiment === "negative" || claim.qualifier === "but" || claim.qualifier === "however") {
295 const claimText = (claim.body || claim.summary || "").toLowerCase();
296 const overlap = keywordOverlap(lower, claimText);
297 if (overlap.length >= 1) {
298 tensions.push(
299 `Your carry data notes a tension: "${claim.summary || claim.body?.slice(0, 150)}"`
300 );
301 }
302 }
303 }
304
305 return tensions.slice(0, 5);
306}
307
308function generateOpenQuestions(
309 sourceText: string,
310 connections: Connection[],
311 tensions: string[]
312): string[] {
313 const questions: string[] = [];
314
315 if (connections.length > 0) {
316 const carryCount = connections.filter(c => c.source === "carry").length;
317 questions.push(
318 `How does this source extend or challenge your existing ${carryCount} carry claims?`
319 );
320 }
321
322 if (tensions.length > 0) {
323 questions.push(
324 `What evidence would resolve the ${tensions.length} tension(s) identified?`
325 );
326 }
327
328 const lower = sourceText.toLowerCase();
329 if (lower.includes("data") || lower.includes("dataset")) {
330 questions.push("What data artifacts does this source produce, and are they in Zenodo?");
331 }
332 if (lower.includes("standard") || lower.includes("protocol")) {
333 questions.push("Does this propose a new standard, and who are the adopters?");
334 }
335 if (lower.includes("community") || lower.includes("collective")) {
336 questions.push("What governance model does this community use?");
337 }
338
339 return questions.slice(0, 5);
340}
341
342function synthesize(
343 sourceText: string,
344 themes: string[],
345 connections: Connection[],
346 tensions: string[],
347 openQuestions: string[]
348): string {
349 // Template-based synthesis. In production, call an LLM.
350 const themeStr = themes.length > 0
351 ? `This source touches on ${themes.join(", ")}.`
352 : "This source covers a specific topic.";
353
354 const connStr = connections.length > 0
355 ? `It connects to ${connections.length} items in your knowledge base: ${connections.slice(0, 3).map(c => c.description).join("; ")}.`
356 : "No direct connections found in your existing knowledge.";
357
358 const tensionStr = tensions.length > 0
359 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
360 : "";
361
362 const questionStr = openQuestions.length > 0
363 ? `Open questions: ${openQuestions.join(" ")}`
364 : "";
365
366 return [themeStr, connStr, tensionStr, questionStr].filter(Boolean).join(" ");
367}
368
369function buildCarryRefs(
370 connections: Connection[],
371 _carryClaims: CarryClaim[]
372): CarryRef[] {
373 return connections
374 .filter(c => c.source === "carry")
375 .map(c => ({
376 entityType: "claim" as const,
377 carryId: c.carryId || "",
378 atUri: c.targetUri || "",
379 summary: c.description,
380 }));
381}
382
383// ─── Utilities ──────────────────────────────────────────────────────
384
385const STOP_WORDS = new Set([
386 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
387 "have", "has", "had", "do", "does", "did", "will", "would", "could",
388 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
389 "on", "with", "at", "by", "from", "as", "into", "through", "during",
390 "before", "after", "above", "below", "between", "and", "but", "or",
391 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
392]);
393
394function keywordOverlap(textA: string, textB: string): string[] {
395 const wordsA = new Set(
396 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
397 );
398 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
399 return wordsB.filter(w => wordsA.has(w));
400}
401
402function normalizeClaim(data: any): CarryClaim {
403 return {
404 id: data.id || data.claimId || crypto.randomUUID(),
405 body: data.body || data.text || data.claim || "",
406 summary: data.summary || data.title || "",
407 sentiment: data.sentiment || "neutral",
408 qualifier: data.qualifier || "",
409 atUri: data.atUri || "",
410 entityType: data.entityType || "claim",
411 };
412}