nandi.uk
This repository has no description
18 kB
504 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Carry semantic triples from `carry query --format json`
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// Follows carry semantics: connections are source → relation → target.
6// If one side is in the island, the other side becomes a new connection.
7// Returns structured analysis + new connection records to be created on PDS.
8
9import { readFileSync, existsSync } from "fs";
10import { join } from "path";
11
12const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
13
14// ─── Types ──────────────────────────────────────────────────────────
15
16interface VertexInfo {
17 uri: string;
18 title: string;
19 description: string;
20 type: string;
21}
22
23interface EdgeInfo {
24 uri: string;
25 did: string;
26 source: string;
27 target: string;
28 connectionType: string;
29 note: string;
30}
31
32interface CarryEntity {
33 atUri: string;
34 name: string;
35 entityType: string;
36 description?: string;
37 url?: string; // The entity's canonical https:// URL
38}
39
40interface CarryCitation {
41 atUri: string;
42 url: string;
43 title: string;
44 takeaway: string;
45 confidence?: string;
46}
47
48interface CarryReasoning {
49 atUri: string;
50 claim: string;
51 evidence: string;
52 reasoningType?: string;
53 url?: string; // The reasoning's source https:// URL
54}
55
56interface CarryData {
57 entities: CarryEntity[];
58 citations: CarryCitation[];
59 reasonings: CarryReasoning[];
60 connections: CarryConnection[];
61}
62
63// Structured carry connection — a semantic triple
64interface CarryConnection {
65 sourceName: string; // e.g. "Khayame"
66 targetName: string; // e.g. "Vickers Appreciative Systems"
67 sourceUrl: string; // e.g. "https://link.springer.com/..."
68 targetUrl: string; // e.g. "https://pubadmin.institute/..."
69 relation: string; // e.g. "intellectual_influence"
70 context: string; // e.g. "Khayame draws on Vickers' appreciative systems..."
71}
72
73interface NewConnection {
74 source: string; // island vertex URI
75 target: string; // carry record AT URI
76 connectionType: string; // e.g. "network.cosmik.connection#annotates"
77 note: string; // why this connection exists
78}
79
80interface AnalysisInput {
81 did: string;
82 island: {
83 vertices: VertexInfo[];
84 edges: EdgeInfo[];
85 };
86 carryData?: CarryData;
87}
88
89interface AnalysisResult {
90 themes: string[];
91 highlights: string[];
92 tensions: string[];
93 openQuestions: string[];
94 synthesis: string;
95 newConnections: NewConnection[];
96}
97
98// ─── Main analysis function ────────────────────────────────────────
99
100export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
101 const { island, carryData } = input;
102
103 // 1. Read carry JSON exports (from `carry query --format json` during sync)
104 const carryEntities = readCarryEntities();
105 const carryCitations = readCarryCitations();
106 const carryConnections = readCarryConnections();
107 const entityUrlIndex = buildEntityUrlIndex();
108
109 // 2. Build island URL set for fast lookup
110 const islandText = buildIslandText(island);
111 const islandUrls = new Set(island.vertices.map(v => v.uri));
112
113 // 3. Extract themes
114 const themes = extractThemes(islandText);
115
116 // 4. Match carry data against island — follow carry semantics
117 const newConnections = matchCarryData(
118 island, islandUrls,
119 carryEntities, carryCitations, carryConnections, entityUrlIndex,
120 carryData || { entities: [], citations: [], reasonings: [], connections: [] },
121 );
122
123 // 6. Flag hot edges
124 const highlights = findHotEdges(islandText, island, newConnections);
125
126 // 7. Find tensions from carry reasonings
127 const tensions = findTensionsFromReasonings(islandText, carryData?.reasonings || []);
128
129 // Deduplicate newConnections
130 const seen = new Set<string>();
131 const deduped = newConnections.filter(c => {
132 const key = `${c.source}|${c.target}`;
133 if (seen.has(key)) return false;
134 seen.add(key);
135 return true;
136 }).slice(0, 20);
137
138 // Regenerate synthesis and openQuestions with deduped count
139 const finalQuestions = generateOpenQuestions(islandText, island, highlights, tensions, deduped);
140 const finalSynthesis = synthesize(island, themes, highlights, tensions, finalQuestions, deduped);
141
142 return { themes, highlights, tensions, openQuestions: finalQuestions, synthesis: finalSynthesis, newConnections: deduped };
143}
144
145// ─── Carry JSON readers ────────────────────────────────────────────
146// Reads carry data exported by `carry query --format json` during sync.
147
148interface CarryEntityRecord {
149 id: string;
150 name: string;
151 url?: string;
152 type?: string;
153 description?: string;
154}
155
156interface CarryCitationRecord {
157 id: string;
158 title: string;
159 url?: string;
160 takeaway?: string;
161}
162
163interface CarryConnectionRecord {
164 id: string;
165 relation: string;
166 source?: string;
167 target?: string;
168 context?: string;
169}
170
171function readCarryJson<T>(filename: string): T[] {
172 const path = join(CARRY_PATH, filename);
173 try {
174 if (!existsSync(path)) return [];
175 const data = readFileSync(path, "utf-8");
176 return JSON.parse(data) as T[];
177 } catch (e: any) {
178 console.error(`[carry] Failed to read ${filename}:`, e.message);
179 return [];
180 }
181}
182
183function readCarryEntities(): CarryEntityRecord[] {
184 return readCarryJson<CarryEntityRecord>("org.latha.entity.json");
185}
186
187function readCarryCitations(): CarryCitationRecord[] {
188 return readCarryJson<CarryCitationRecord>("org.latha.citation.json");
189}
190
191function readCarryConnections(): CarryConnectionRecord[] {
192 return readCarryJson<CarryConnectionRecord>("org.latha.connection.json");
193}
194
195// Build entity name → URL lookup from carry export
196function buildEntityUrlIndex(): Map<string, string> {
197 const index = new Map<string, string>();
198 for (const e of readCarryEntities()) {
199 const url = e.url?.trim().replace(/^"|"$/g, "");
200 if (url && url.startsWith("http")) {
201 const name = e.name?.trim().replace(/^"|"$/g, "");
202 if (name) index.set(name, url);
203 }
204 }
205 return index;
206}
207
208// ─── Build island text corpus ──────────────────────────────────────
209
210function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
211 const parts: string[] = [];
212
213 for (const v of island.vertices) {
214 parts.push(v.title);
215 if (v.description) parts.push(v.description);
216 }
217
218 for (const e of island.edges) {
219 if (e.note) parts.push(e.note);
220 }
221
222 return parts.join(" ");
223}
224
225// ─── Core mapper: match carry semantic triples against island ──────
226
227function matchCarryData(
228 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
229 islandUrls: Set<string>,
230 carryEntities: CarryEntityRecord[],
231 _carryCitations: CarryCitationRecord[],
232 carryConnections: CarryConnectionRecord[],
233 entityUrlIndex: Map<string, string>,
234 _d1CarryData: CarryData,
235): NewConnection[] {
236 const connections: NewConnection[] = [];
237
238 // ── 1. Carry connections (semantic triples) ──────────────────────
239 // A carry connection is source → relation → target.
240 // If one side's URL is in the island, the other side becomes a new connection.
241 // If both sides are in the island, it's an existing edge (skip).
242 // If neither side is in the island, it's a nearby carry connection
243 // that could expand the island if linked.
244 for (const conn of carryConnections) {
245 const srcName = conn.source || "";
246 const tgtName = conn.target || "";
247 const srcUrl = entityUrlIndex.get(srcName) || "";
248 const tgtUrl = entityUrlIndex.get(tgtName) || "";
249
250 if (!srcUrl && !tgtUrl) continue; // Neither side has a URL — skip
251
252 const srcInIsland = srcUrl && islandUrls.has(srcUrl);
253 const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
254
255 // Both in island → already connected, skip
256 if (srcInIsland && tgtInIsland) continue;
257
258 // One side in island → bridge connection
259 if (srcInIsland && tgtUrl) {
260 connections.push({
261 source: srcUrl,
262 target: tgtUrl,
263 connectionType: `network.cosmik.connection#${conn.relation}`,
264 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
265 });
266 } else if (tgtInIsland && srcUrl) {
267 connections.push({
268 source: tgtUrl,
269 target: srcUrl,
270 connectionType: `network.cosmik.connection#${conn.relation}`,
271 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
272 });
273 } else if (srcUrl && tgtUrl) {
274 // Neither side in island — emit as a nearby carry connection
275 // (both sides have URLs, so it's a real edge that could expand the island)
276 connections.push({
277 source: srcUrl,
278 target: tgtUrl,
279 connectionType: `network.cosmik.connection#${conn.relation}`,
280 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
281 });
282 }
283 // One side has URL but the other doesn't — can't create a full connection
284 }
285
286 // ── 2. Carry entities with URLs not in island ───────────────────
287 // These are entities that the user tracks but aren't yet in the island.
288 // We don't try to match them by keyword — we just list them as
289 // "nearby entities" for the synthesis to reference.
290 // (No NewConnection created — entities without a connection triple
291 // don't have a semantic relation to the island.)
292
293 return connections;
294}
295
296// ─── Analysis functions ─────────────────────────────────────────────
297
298function extractThemes(text: string): string[] {
299 const keywords: Record<string, string[]> = {
300 "open science": ["open science", "open access", "oa", "preprint"],
301 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
302 "reproducibility": ["reproducib", "replicab", "replication"],
303 "decentralization": ["decentraliz", "federat", "self-host"],
304 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
305 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
306 "community": ["community", "collective", "cooperative", "commons"],
307 "governance": ["governance", "policy", "regulation", "standards"],
308 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
309 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
310 "provenance": ["provenance", "lineage", "traceability", "attribution"],
311 "doi": ["doi", "zenodo", "citeable", "citation"],
312 "privacy": ["privacy", "encryption", "e2ee", "consent"],
313 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
314 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
315 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
316 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
317 };
318
319 const lower = text.toLowerCase();
320 return Object.entries(keywords)
321 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
322 .map(([theme]) => theme);
323}
324
325function findHotEdges(
326 islandText: string,
327 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
328 newConnections: NewConnection[],
329): string[] {
330 const highlights: string[] = [];
331
332 // Score each edge by semantic significance
333 const edgeScores = new Map<string, number>();
334
335 for (const edge of island.edges) {
336 let score = 0;
337
338 // Edges with notes are more interesting
339 if (edge.note && edge.note.length > 10) score += 1;
340
341 // Edges with semantic connection types are more interesting
342 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
343 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
344
345 // Boost edges whose source or target is also connected to carry data
346 for (const nc of newConnections) {
347 if (edge.source === nc.source || edge.target === nc.source) score += 3;
348 if (edge.source === nc.target || edge.target === nc.target) score += 3;
349 }
350
351 if (score > 0) {
352 edgeScores.set(edge.uri, score);
353 }
354 }
355
356 // Return top edges by score
357 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
358 for (const [uri] of sorted.slice(0, 10)) {
359 highlights.push(uri);
360 }
361
362 return highlights;
363}
364
365function findTensionsFromReasonings(
366 islandText: string,
367 reasonings: CarryReasoning[],
368): string[] {
369 const tensions: string[] = [];
370 const lowerSet = tokenize(islandText);
371
372 for (const r of reasonings) {
373 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
374 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
375 const overlap = keywordOverlap(lowerSet, rLower);
376 if (overlap.length >= 1) {
377 tensions.push(r.claim.slice(0, 500));
378 }
379 }
380 }
381
382 return tensions.slice(0, 5);
383}
384
385function generateOpenQuestions(
386 islandText: string,
387 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
388 highlights: string[],
389 tensions: string[],
390 newConnections: NewConnection[],
391): string[] {
392 const questions: string[] = [];
393 const lower = islandText.toLowerCase();
394
395 if (newConnections.length > 0) {
396 questions.push(
397 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
398 );
399 }
400
401 if (tensions.length > 0) {
402 questions.push(
403 `What evidence would resolve the ${tensions.length} tension(s) identified?`
404 );
405 }
406
407 const vertexCount = island.vertices.length;
408 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
409
410 if (edgeTypes.size > 1) {
411 questions.push(
412 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
413 );
414 }
415
416 if (vertexCount > 5) {
417 questions.push(
418 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
419 );
420 }
421
422 if (lower.includes("data") || lower.includes("dataset")) {
423 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
424 }
425 if (lower.includes("standard") || lower.includes("protocol")) {
426 questions.push("Does this cluster propose a new standard, and who are the adopters?");
427 }
428
429 return questions.slice(0, 5);
430}
431
432function synthesize(
433 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
434 themes: string[],
435 highlights: string[],
436 tensions: string[],
437 openQuestions: string[],
438 newConnections: NewConnection[],
439): string {
440 const vertexCount = island.vertices.length;
441 const edgeCount = island.edges.length;
442 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
443
444 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
445
446 const themeStr = themes.length > 0
447 ? `Themes: ${themes.join(", ")}.`
448 : "";
449
450 const touchpointStr = newConnections.length > 0
451 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
452 : "No carry touchpoints found.";
453
454 const highlightStr = highlights.length > 0
455 ? `${highlights.length} existing edges flagged as semantically significant.`
456 : "";
457
458 const tensionStr = tensions.length > 0
459 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
460 : "";
461
462 const questionStr = openQuestions.length > 0
463 ? `Open questions: ${openQuestions.join(" ")}`
464 : "";
465
466 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
467}
468
469// ─── Utilities ──────────────────────────────────────────────────────
470
471const STOP_WORDS = new Set([
472 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
473 "have", "has", "had", "do", "does", "did", "will", "would", "could",
474 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
475 "on", "with", "at", "by", "from", "as", "into", "through", "during",
476 "before", "after", "above", "below", "between", "and", "but", "or",
477 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
478 "http", "https", "www", "com", "org",
479 // Common vault noise terms
480 "checkin", "check", "daily", "today", "yesterday", "tomorrow",
481 "morning", "evening", "week", "month", "year",
482 "also", "just", "like", "really", "very", "much", "some",
483 "more", "most", "other", "about", "which", "when", "what",
484 "there", "their", "these", "those", "each", "every", "all",
485 "make", "made", "thing", "things", "something", "anything",
486 "need", "want", "know", "think", "going", "come", "get",
487 "good", "great", "nice", "well", "still", "even", "back",
488 "first", "last", "new", "old", "own", "same", "another",
489 "many", "such", "only", "over", "than", "too", "very",
490 "because", "since", "while", "where", "how", "why", "who",
491 "work", "working", "time", "people", "using", "used", "use",
492]);
493
494function tokenize(text: string): Set<string> {
495 return new Set(
496 text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
497 );
498}
499
500
501function keywordOverlap(setA: Set<string>, textB: string): string[] {
502 const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
503 return wordsB.filter(w => setA.has(w));
504}