nandi.uk
This repository has no description
15 kB
422 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against carry data
3// (entities, citations, reasonings from D1) passed in by the worker.
4// Returns structured analysis + new connection records to be created on PDS.
5
6// ─── Types ──────────────────────────────────────────────────────────
7
8interface VertexInfo {
9 uri: string;
10 title: string;
11 description: string;
12 type: string;
13}
14
15interface EdgeInfo {
16 uri: string;
17 did: string;
18 source: string;
19 target: string;
20 connectionType: string;
21 note: string;
22}
23
24interface CarryEntity {
25 atUri: string;
26 name: string;
27 entityType: string;
28 description?: string;
29}
30
31interface CarryCitation {
32 atUri: string;
33 url: string;
34 title: string;
35 takeaway: string;
36 confidence?: string;
37}
38
39interface CarryReasoning {
40 atUri: string;
41 claim: string;
42 evidence: string;
43 reasoningType?: string;
44}
45
46interface CarryData {
47 entities: CarryEntity[];
48 citations: CarryCitation[];
49 reasonings: CarryReasoning[];
50}
51
52interface NewConnection {
53 source: string; // island vertex URI
54 target: string; // carry record AT URI
55 connectionType: string; // e.g. "network.cosmik.connection#annotates"
56 note: string; // why this connection exists
57}
58
59interface AnalysisInput {
60 did: string;
61 island: {
62 vertices: VertexInfo[];
63 edges: EdgeInfo[];
64 };
65 carryData?: CarryData;
66}
67
68interface AnalysisResult {
69 themes: string[];
70 highlights: string[];
71 tensions: string[];
72 openQuestions: string[];
73 synthesis: string;
74 newConnections: NewConnection[];
75}
76
77// ─── Main analysis function ────────────────────────────────────────
78
79export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
80 const { island, carryData } = input;
81
82 // 1. Build corpus from the entire island
83 const islandText = buildIslandText(island);
84
85 // 2. Extract themes from the island
86 const themes = extractThemes(islandText);
87
88 // 3. Find carry touchpoints — semantic matches between carry records and island vertices
89 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] });
90
91 // 4. Flag hot edges — existing edges that are semantically significant
92 const highlights = findHotEdges(islandText, island, newConnections);
93
94 // 5. Find tensions from carry reasonings
95 const tensions = findTensions(islandText, carryData?.reasonings || []);
96
97 // 6. Generate open questions
98 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
99
100 // 7. Synthesize prose
101 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
102
103 return { themes, highlights, tensions, openQuestions, synthesis, newConnections };
104}
105
106// ─── Build island text corpus ──────────────────────────────────────
107
108function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
109 const parts: string[] = [];
110
111 for (const v of island.vertices) {
112 parts.push(v.title);
113 if (v.description) parts.push(v.description);
114 }
115
116 for (const e of island.edges) {
117 if (e.note) parts.push(e.note);
118 }
119
120 return parts.join(" ");
121}
122
123// ─── Carry touchpoint matching ──────────────────────────────────────
124
125function findCarryTouchpoints(
126 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
127 carryData: CarryData,
128): NewConnection[] {
129 const connections: NewConnection[] = [];
130 const islandText = buildIslandText(island).toLowerCase();
131
132 // Build a vertex index for fast lookup
133 const vertexByUri = new Map(island.vertices.map(v => [v.uri, v]));
134
135 // --- Match citations by URL ---
136 for (const cite of carryData.citations) {
137 // Direct URL match: citation URL is a vertex URI
138 if (vertexByUri.has(cite.url)) {
139 connections.push({
140 source: cite.url,
141 target: cite.atUri,
142 connectionType: "network.cosmik.connection#annotates",
143 note: cite.takeaway.slice(0, 500),
144 });
145 continue;
146 }
147
148 // Semantic match: citation title/takeaway overlaps with island vertices
149 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
150 const bestVertex = findBestVertexMatch(citeText, island.vertices);
151 if (bestVertex) {
152 const overlap = keywordOverlap(citeText, bestVertex.title + " " + (bestVertex.description || ""));
153 if (overlap.length >= 2) {
154 connections.push({
155 source: bestVertex.uri,
156 target: cite.atUri,
157 connectionType: "network.cosmik.connection#annotates",
158 note: cite.takeaway.slice(0, 500),
159 });
160 }
161 }
162 }
163
164 // --- Match entities by name/description ---
165 for (const entity of carryData.entities) {
166 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
167 const bestVertex = findBestVertexMatch(entityText, island.vertices);
168 if (bestVertex) {
169 const overlap = keywordOverlap(entityText, bestVertex.title + " " + (bestVertex.description || ""));
170 if (overlap.length >= 2) {
171 connections.push({
172 source: bestVertex.uri,
173 target: entity.atUri,
174 connectionType: "network.cosmik.connection#relates",
175 note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
176 });
177 }
178 }
179 }
180
181 // --- Match reasonings by claim/evidence ---
182 for (const reasoning of carryData.reasonings) {
183 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
184 const bestVertex = findBestVertexMatch(reasoningText, island.vertices);
185 if (bestVertex) {
186 const overlap = keywordOverlap(reasoningText, bestVertex.title + " " + (bestVertex.description || ""));
187 if (overlap.length >= 2) {
188 const connType = reasoning.reasoningType?.includes("contradict")
189 ? "network.cosmik.connection#contradicts"
190 : "network.cosmik.connection#extends";
191 connections.push({
192 source: bestVertex.uri,
193 target: reasoning.atUri,
194 connectionType: connType,
195 note: reasoning.claim.slice(0, 500),
196 });
197 }
198 }
199 }
200
201 // Deduplicate by (source, target) pair
202 const seen = new Set<string>();
203 return connections.filter(c => {
204 const key = `${c.source}|${c.target}`;
205 if (seen.has(key)) return false;
206 seen.add(key);
207 return true;
208 }).slice(0, 20);
209}
210
211function findBestVertexMatch(
212 text: string,
213 vertices: VertexInfo[],
214): VertexInfo | null {
215 let bestScore = 0;
216 let bestVertex: VertexInfo | null = null;
217
218 for (const v of vertices) {
219 const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
220 const overlap = keywordOverlap(text, vertexText);
221 if (overlap.length > bestScore) {
222 bestScore = overlap.length;
223 bestVertex = v;
224 }
225 }
226
227 return bestScore >= 2 ? bestVertex : null;
228}
229
230// ─── Analysis functions ─────────────────────────────────────────────
231
232function extractThemes(text: string): string[] {
233 const keywords: Record<string, string[]> = {
234 "open science": ["open science", "open access", "oa", "preprint"],
235 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
236 "reproducibility": ["reproducib", "replicab", "replication"],
237 "decentralization": ["decentraliz", "federat", "self-host"],
238 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
239 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
240 "community": ["community", "collective", "cooperative", "commons"],
241 "governance": ["governance", "policy", "regulation", "standards"],
242 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
243 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
244 "provenance": ["provenance", "lineage", "traceability", "attribution"],
245 "doi": ["doi", "zenodo", "citeable", "citation"],
246 "privacy": ["privacy", "encryption", "e2ee", "consent"],
247 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
248 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
249 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
250 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
251 };
252
253 const lower = text.toLowerCase();
254 return Object.entries(keywords)
255 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
256 .map(([theme]) => theme);
257}
258
259function findHotEdges(
260 islandText: string,
261 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
262 newConnections: NewConnection[],
263): string[] {
264 const highlights: string[] = [];
265
266 // Score each edge by semantic significance
267 const edgeScores = new Map<string, number>();
268
269 for (const edge of island.edges) {
270 let score = 0;
271
272 // Edges with notes are more interesting
273 if (edge.note && edge.note.length > 10) score += 1;
274
275 // Edges with semantic connection types are more interesting
276 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
277 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
278
279 // Boost edges whose source or target is also connected to carry data
280 for (const nc of newConnections) {
281 if (edge.source === nc.source || edge.target === nc.source) score += 3;
282 if (edge.source === nc.target || edge.target === nc.target) score += 3;
283 }
284
285 if (score > 0) {
286 edgeScores.set(edge.uri, score);
287 }
288 }
289
290 // Return top edges by score
291 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
292 for (const [uri] of sorted.slice(0, 10)) {
293 highlights.push(uri);
294 }
295
296 return highlights;
297}
298
299function findTensions(
300 islandText: string,
301 reasonings: CarryReasoning[],
302): string[] {
303 const tensions: string[] = [];
304 const lower = islandText.toLowerCase();
305
306 for (const r of reasonings) {
307 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
308 // Look for contradiction markers
309 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
310 const overlap = keywordOverlap(lower, rLower);
311 if (overlap.length >= 1) {
312 tensions.push(r.claim.slice(0, 500));
313 }
314 }
315 }
316
317 return tensions.slice(0, 5);
318}
319
320function generateOpenQuestions(
321 islandText: string,
322 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
323 highlights: string[],
324 tensions: string[],
325 newConnections: NewConnection[],
326): string[] {
327 const questions: string[] = [];
328 const lower = islandText.toLowerCase();
329
330 if (newConnections.length > 0) {
331 questions.push(
332 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
333 );
334 }
335
336 if (tensions.length > 0) {
337 questions.push(
338 `What evidence would resolve the ${tensions.length} tension(s) identified?`
339 );
340 }
341
342 const vertexCount = island.vertices.length;
343 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
344
345 if (edgeTypes.size > 1) {
346 questions.push(
347 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
348 );
349 }
350
351 if (vertexCount > 5) {
352 questions.push(
353 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
354 );
355 }
356
357 if (lower.includes("data") || lower.includes("dataset")) {
358 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
359 }
360 if (lower.includes("standard") || lower.includes("protocol")) {
361 questions.push("Does this cluster propose a new standard, and who are the adopters?");
362 }
363
364 return questions.slice(0, 5);
365}
366
367function synthesize(
368 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
369 themes: string[],
370 highlights: string[],
371 tensions: string[],
372 openQuestions: string[],
373 newConnections: NewConnection[],
374): string {
375 const vertexCount = island.vertices.length;
376 const edgeCount = island.edges.length;
377 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
378
379 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
380
381 const themeStr = themes.length > 0
382 ? `Themes: ${themes.join(", ")}.`
383 : "";
384
385 const touchpointStr = newConnections.length > 0
386 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
387 : "No carry touchpoints found.";
388
389 const highlightStr = highlights.length > 0
390 ? `${highlights.length} existing edges flagged as semantically significant.`
391 : "";
392
393 const tensionStr = tensions.length > 0
394 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
395 : "";
396
397 const questionStr = openQuestions.length > 0
398 ? `Open questions: ${openQuestions.join(" ")}`
399 : "";
400
401 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
402}
403
404// ─── Utilities ──────────────────────────────────────────────────────
405
406const STOP_WORDS = new Set([
407 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
408 "have", "has", "had", "do", "does", "did", "will", "would", "could",
409 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
410 "on", "with", "at", "by", "from", "as", "into", "through", "during",
411 "before", "after", "above", "below", "between", "and", "but", "or",
412 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
413 "http", "https", "www", "com", "org",
414]);
415
416function keywordOverlap(textA: string, textB: string): string[] {
417 const wordsA = new Set(
418 textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
419 );
420 const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
421 return wordsB.filter(w => wordsA.has(w));
422}