refactor: follow carry semantics, remove keyword matching entirely · nandi.uk/stigmergic@9d554df

+40 -206

1 changed file

Expand all

container

analysis.ts

+40 -206

container/analysis.ts

··· 1 1 // ─── Strata Analysis Engine ───────────────────────────────────────── 2 2 // Cross-references an island (cluster of linked URLs) against: 3 - // 1. Vault + carry markdown from local filesystem (synced via ob/git) 3 + // 1. Carry semantic triples from `carry query --format json` 4 4 // 2. Structured carry records from D1 (entities, citations, reasonings) 5 - // For carry/vault notes without URLs, uses the Letta API to discover 6 - // canonical URLs via web search. 5 + // Follows carry semantics: connections are source → relation → target. 6 + // If one side is in the island, the other side becomes a new connection. 7 7 // Returns structured analysis + new connection records to be created on PDS. 8 8 9 9 import { readFileSync, existsSync } from "fs"; ··· 109 109 // 2. Build island URL set for fast lookup 110 110 const islandText = buildIslandText(island); 111 111 const islandUrls = new Set(island.vertices.map(v => v.uri)); 112 - const islandTokenSet = tokenize(islandText); 113 - const vertexTokenSets = new Map<string, Set<string>>(); 114 - for (const v of island.vertices) { 115 - vertexTokenSets.set(v.uri, tokenize(`${v.title} ${v.description || ""}`)); 116 - } 117 112 118 - // 3. Compute IDF for TF-IDF scoring 119 - const allDocuments = [ 120 - ...carryEntities.map(e => `${e.name} ${e.description || ""}`), 121 - ...carryCitations.map(c => `${c.title} ${c.takeaway || ""}`), 122 - ...island.vertices.map(v => `${v.title} ${v.description || ""}`), 123 - ]; 124 - const idf = computeIdf(allDocuments); 125 - 126 - // 4. Extract themes 113 + // 3. Extract themes 127 114 const themes = extractThemes(islandText); 128 115 129 - // 5. Match carry data against island — structured semantic triples only 116 + // 4. Match carry data against island — follow carry semantics 130 117 const newConnections = matchCarryData( 131 - island, islandUrls, islandTokenSet, vertexTokenSets, idf, 118 + island, islandUrls, 132 119 carryEntities, carryCitations, carryConnections, entityUrlIndex, 133 120 carryData || { entities: [], citations: [], reasonings: [], connections: [] }, 134 121 ); ··· 138 125 139 126 // 7. Find tensions from carry reasonings 140 127 const tensions = findTensionsFromReasonings(islandText, carryData?.reasonings || []); 141 - 142 - // 8. Generate open questions 143 - const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections); 144 - 145 - // 9. Synthesize prose 146 - const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections); 147 128 148 129 // Deduplicate newConnections 149 130 const seen = new Set<string>(); ··· 152 133 if (seen.has(key)) return false; 153 134 seen.add(key); 154 135 return true; 155 - }); 136 + }).slice(0, 20); 156 137 157 - return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) }; 138 + // Regenerate synthesis and openQuestions with deduped count 139 + const finalQuestions = generateOpenQuestions(islandText, island, highlights, tensions, deduped); 140 + const finalSynthesis = synthesize(island, themes, highlights, tensions, finalQuestions, deduped); 141 + 142 + return { themes, highlights, tensions, openQuestions: finalQuestions, synthesis: finalSynthesis, newConnections: deduped }; 158 143 } 159 144 160 145 // ─── Carry JSON readers ──────────────────────────────────────────── ··· 242 227 function matchCarryData( 243 228 island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 244 229 islandUrls: Set<string>, 245 - islandTokenSet: Set<string>, 246 - vertexTokenSets: Map<string, Set<string>>, 247 - idf: Map<string, number>, 248 230 carryEntities: CarryEntityRecord[], 249 - carryCitations: CarryCitationRecord[], 231 + _carryCitations: CarryCitationRecord[], 250 232 carryConnections: CarryConnectionRecord[], 251 233 entityUrlIndex: Map<string, string>, 252 - d1CarryData: CarryData, 234 + _d1CarryData: CarryData, 253 235 ): NewConnection[] { 254 236 const connections: NewConnection[] = []; 255 237 256 238 // ── 1. Carry connections (semantic triples) ────────────────────── 239 + // A carry connection is source → relation → target. 240 + // If one side's URL is in the island, the other side becomes a new connection. 241 + // If both sides are in the island, it's an existing edge (skip). 242 + // If neither side is in the island, it's a nearby carry connection 243 + // that could expand the island if linked. 257 244 for (const conn of carryConnections) { 258 245 const srcName = conn.source || ""; 259 246 const tgtName = conn.target || ""; 260 247 const srcUrl = entityUrlIndex.get(srcName) || ""; 261 248 const tgtUrl = entityUrlIndex.get(tgtName) || ""; 262 249 250 + if (!srcUrl && !tgtUrl) continue; // Neither side has a URL — skip 251 + 263 252 const srcInIsland = srcUrl && islandUrls.has(srcUrl); 264 253 const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl); 265 254 255 + // Both in island → already connected, skip 256 + if (srcInIsland && tgtInIsland) continue; 257 + 258 + // One side in island → bridge connection 266 259 if (srcInIsland && tgtUrl) { 267 260 connections.push({ 268 261 source: srcUrl, ··· 270 263 connectionType: `network.cosmik.connection#${conn.relation}`, 271 264 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`, 272 265 }); 273 - } 274 - 275 - if (tgtInIsland && srcUrl) { 266 + } else if (tgtInIsland && srcUrl) { 276 267 connections.push({ 277 268 source: tgtUrl, 278 269 target: srcUrl, 279 270 connectionType: `network.cosmik.connection#${conn.relation}`, 280 271 note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`, 281 272 }); 282 - } 283 - 284 - // Neither in island — try TF-IDF name match 285 - if (!srcInIsland && !tgtInIsland && (srcUrl || tgtUrl)) { 286 - const nameToMatch = srcName || tgtName; 287 - const urlToConnect = srcUrl || tgtUrl || ""; 288 - if (!urlToConnect) continue; 289 - 290 - const nameTerms = tokenize(nameToMatch); 291 - const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 4.0); 292 - if (bestVertex) { 293 - connections.push({ 294 - source: bestVertex.uri, 295 - target: urlToConnect, 296 - connectionType: `network.cosmik.connection#${conn.relation}`, 297 - note: `${nameToMatch} (${conn.relation}) — carry connection`, 298 - }); 299 - } 300 - } 301 - } 302 - 303 - // ── 2. Carry entities — match by URL or name ────────────────────── 304 - for (const entity of carryEntities) { 305 - const url = entity.url?.trim().replace(/^"|"$/g, ""); 306 - if (!url || !url.startsWith("http")) continue; 307 - if (islandUrls.has(url)) continue; 308 - 309 - const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`); 310 - const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0); 311 - if (bestVertex) { 312 - connections.push({ 313 - source: bestVertex.uri, 314 - target: url, 315 - connectionType: "network.cosmik.connection#relates", 316 - note: `Related: ${entity.name} (${entity.type || "entity"})`, 317 - }); 318 - } 319 - } 320 - 321 - // ── 3. Carry citations — match by URL or title ─────────────────── 322 - for (const cite of carryCitations) { 323 - const url = cite.url?.trim().replace(/^"|"$/g, ""); 324 - if (!url || !url.startsWith("http")) continue; 325 - if (islandUrls.has(url)) continue; 326 - 327 - const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`); 328 - const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0); 329 - if (bestVertex) { 330 - connections.push({ 331 - source: bestVertex.uri, 332 - target: url, 333 - connectionType: "network.cosmik.connection#annotates", 334 - note: cite.title.slice(0, 200), 335 - }); 336 - } 337 - } 338 - 339 - // ── 4. D1 carry data (entities, citations, reasonings from PDS) ── 340 - for (const entity of d1CarryData.entities) { 341 - if (!entity.url || !entity.url.startsWith("http")) continue; 342 - if (islandUrls.has(entity.url)) continue; 343 - 344 - const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`); 345 - const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0); 346 - if (bestVertex) { 273 + } else if (srcUrl && tgtUrl) { 274 + // Neither side in island — emit as a nearby carry connection 275 + // (both sides have URLs, so it's a real edge that could expand the island) 347 276 connections.push({ 348 - source: bestVertex.uri, 349 - target: entity.url, 350 - connectionType: "network.cosmik.connection#relates", 351 - note: `Related: ${entity.name} (${entity.entityType.replace("org.latha.strata.defs#", "")})`, 277 + source: srcUrl, 278 + target: tgtUrl, 279 + connectionType: `network.cosmik.connection#${conn.relation}`, 280 + note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`, 352 281 }); 353 282 } 283 + // One side has URL but the other doesn't — can't create a full connection 354 284 } 355 285 356 - for (const cite of d1CarryData.citations) { 357 - if (!cite.url || !cite.url.startsWith("http")) continue; 358 - if (islandUrls.has(cite.url)) continue; 359 - 360 - const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`); 361 - const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0); 362 - if (bestVertex) { 363 - connections.push({ 364 - source: bestVertex.uri, 365 - target: cite.url, 366 - connectionType: "network.cosmik.connection#annotates", 367 - note: cite.takeaway?.slice(0, 200) || cite.title.slice(0, 200), 368 - }); 369 - } 370 - } 371 - 372 - for (const reasoning of d1CarryData.reasonings) { 373 - if (!reasoning.url || !reasoning.url.startsWith("http")) continue; 374 - if (islandUrls.has(reasoning.url)) continue; 375 - 376 - const reasoningTerms = tokenize(`${reasoning.claim} ${reasoning.evidence}`); 377 - const bestVertex = findBestVertexMatch(reasoningTerms, island.vertices, vertexTokenSets, idf, 4.0); 378 - if (bestVertex) { 379 - const connType = reasoning.reasoningType?.includes("contradict") 380 - ? "network.cosmik.connection#contradicts" 381 - : "network.cosmik.connection#extends"; 382 - connections.push({ 383 - source: bestVertex.uri, 384 - target: reasoning.url, 385 - connectionType: connType, 386 - note: reasoning.claim.slice(0, 500), 387 - }); 388 - } 389 - } 286 + // ── 2. Carry entities with URLs not in island ─────────────────── 287 + // These are entities that the user tracks but aren't yet in the island. 288 + // We don't try to match them by keyword — we just list them as 289 + // "nearby entities" for the synthesis to reference. 290 + // (No NewConnection created — entities without a connection triple 291 + // don't have a semantic relation to the island.) 390 292 391 293 return connections; 392 - } 393 - 394 - function findBestVertexMatch( 395 - noteTerms: Set<string>, 396 - vertices: VertexInfo[], 397 - vertexTokenSets: Map<string, Set<string>>, 398 - idf: Map<string, number>, 399 - minScore: number = 3.0, 400 - ): VertexInfo | null { 401 - let bestScore = 0; 402 - let bestVertex: VertexInfo | null = null; 403 - 404 - for (const v of vertices) { 405 - const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`); 406 - const score = tfidfScore(noteTerms, vSet, idf); 407 - if (score > bestScore) { 408 - bestScore = score; 409 - bestVertex = v; 410 - } 411 - } 412 - 413 - return bestScore >= minScore ? bestVertex : null; 414 294 } 415 295 416 296 // ─── Analysis functions ───────────────────────────────────────────── ··· 617 497 ); 618 498 } 619 499 620 - function tokenizeArray(text: string): string[] { 621 - return text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)); 622 - } 623 - 624 - // ─── TF-IDF scoring ──────────────────────────────────────────────── 625 - // Compute IDF across all notes, then score note-island matches by 626 - // the sum of IDF weights of shared terms. This naturally downweights 627 - // common terms like "checkin" and upweights distinctive terms like 628 - // "stigmergy" or "nanopublication". 629 - 630 - function computeIdf(documents: string[]): Map<string, number> { 631 - const docCount = documents.length; 632 - const df = new Map<string, number>(); // document frequency 633 - 634 - for (const doc of documents) { 635 - const terms = new Set(tokenizeArray(doc)); 636 - for (const t of terms) { 637 - df.set(t, (df.get(t) || 0) + 1); 638 - } 639 - } 640 - 641 - const idf = new Map<string, number>(); 642 - for (const [term, freq] of df) { 643 - // Smooth IDF: log((N+1)/(df+1)) + 1 644 - idf.set(term, Math.log((docCount + 1) / (freq + 1)) + 1); 645 - } 646 - return idf; 647 - } 648 - 649 - function tfidfScore( 650 - noteTerms: Set<string>, 651 - targetTerms: Set<string>, 652 - idf: Map<string, number>, 653 - ): number { 654 - let score = 0; 655 - for (const t of noteTerms) { 656 - if (targetTerms.has(t)) { 657 - score += idf.get(t) || 0; 658 - } 659 - } 660 - return score; 661 - } 662 500 663 501 function keywordOverlap(setA: Set<string>, textB: string): string[] { 664 502 const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)); 665 503 return wordsB.filter(w => setA.has(w)); 666 504 } 667 - 668 - function keywordOverlapSets(setA: Set<string>, setB: Set<string>): string[] { 669 - return [...setA].filter(w => setB.has(w)); 670 - }

Configure Feed

Configure Feed