perf: precompute token sets for keyword overlap matching · nandi.uk/stigmergic@81ed8ca

+73 -39

1 changed file

Expand all

container

analysis.ts

+73 -39

container/analysis.ts

··· 108 108 const vaultNotes = readVault(); 109 109 const carryNotes = readCarry(); 110 110 111 - // 2. Build corpus from the entire island 111 + // 2. Build corpus from the entire island + precompute token sets 112 112 const islandText = buildIslandText(island); 113 + const islandTokenSet = tokenize(islandText); 114 + const vertexTokenSets = new Map<string, Set<string>>(); 115 + for (const v of island.vertices) { 116 + vertexTokenSets.set(v.uri, tokenize(`${v.title} ${v.description || ""}`)); 117 + } 113 118 114 119 // 3. Extract themes from the island 115 120 const themes = extractThemes(islandText); 116 121 117 122 // 4. Find carry touchpoints — semantic matches from D1 records 118 - const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] }); 123 + const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] }, vertexTokenSets); 119 124 120 125 // 5. Find vault/carry touchpoints — semantic matches from filesystem notes 121 126 // (async — may call Letta API for URL discovery) 122 - const fsConnections = await findFilesystemTouchpoints(island, vaultNotes, carryNotes); 127 + const fsConnections = await findFilesystemTouchpoints(island, vaultNotes, carryNotes, islandTokenSet, vertexTokenSets); 123 128 newConnections.push(...fsConnections); 124 129 125 130 // 6. Flag hot edges — existing edges that are semantically significant ··· 219 224 function findCarryTouchpoints( 220 225 island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 221 226 carryData: CarryData, 227 + vertexTokenSets: Map<string, Set<string>>, 222 228 ): NewConnection[] { 223 229 const connections: NewConnection[] = []; 224 230 ··· 239 245 240 246 // Semantic match: citation title/takeaway overlaps with island vertices 241 247 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase(); 242 - const bestVertex = findBestVertexMatch(citeText, island.vertices); 248 + const bestVertex = findBestVertexMatch(citeText, island.vertices, vertexTokenSets); 243 249 if (bestVertex) { 244 - const overlap = keywordOverlap(citeText, `${bestVertex.title} ${bestVertex.description || ""}`); 250 + const overlap = keywordOverlapSets(tokenize(citeText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`)); 245 251 if (overlap.length >= 2) { 246 252 connections.push({ 247 253 source: bestVertex.uri, ··· 257 263 for (const entity of carryData.entities) { 258 264 if (!entity.url) continue; // Need an https:// URL as target 259 265 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase(); 260 - const bestVertex = findBestVertexMatch(entityText, island.vertices); 266 + const bestVertex = findBestVertexMatch(entityText, island.vertices, vertexTokenSets); 261 267 if (bestVertex) { 262 - const overlap = keywordOverlap(entityText, `${bestVertex.title} ${bestVertex.description || ""}`); 268 + const overlap = keywordOverlapSets(tokenize(entityText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`)); 263 269 if (overlap.length >= 2) { 264 270 connections.push({ 265 271 source: bestVertex.uri, ··· 275 281 for (const reasoning of carryData.reasonings) { 276 282 if (!reasoning.url) continue; // Need an https:// URL as target 277 283 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase(); 278 - const bestVertex = findBestVertexMatch(reasoningText, island.vertices); 284 + const bestVertex = findBestVertexMatch(reasoningText, island.vertices, vertexTokenSets); 279 285 if (bestVertex) { 280 - const overlap = keywordOverlap(reasoningText, `${bestVertex.title} ${bestVertex.description || ""}`); 286 + const overlap = keywordOverlapSets(tokenize(reasoningText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`)); 281 287 if (overlap.length >= 2) { 282 288 const connType = reasoning.reasoningType?.includes("contradict") 283 289 ? "network.cosmik.connection#contradicts" ··· 297 303 298 304 // ─── Letta URL discovery ──────────────────────────────────────────── 299 305 // For carry/vault notes that lack an https:// URL, ask the Letta agent 300 - // to find the canonical URL via web search. 306 + // to find the canonical URL via web search. Only called for notes that 307 + // already matched the island (passed keyword overlap), so the volume 308 + // is small. Results are cached in-memory for the lifetime of the container. 309 + 310 + const discoveredUrls = new Map<string, string | null>(); 301 311 302 312 async function discoverUrl(semanticDescription: string): Promise<string | null> { 303 313 if (!LETTA_API_KEY || !LETTA_AGENT_ID) return null; 304 314 315 + // Check cache 316 + const cacheKey = semanticDescription.slice(0, 200); 317 + if (discoveredUrls.has(cacheKey)) return discoveredUrls.get(cacheKey)!; 318 + 305 319 try { 306 320 const resp = await fetch(`${LETTA_BASE_URL}/v1/agents/${LETTA_AGENT_ID}/messages`, { 307 321 method: "POST", ··· 309 323 "Authorization": `Bearer ${LETTA_API_KEY}`, 310 324 "Content-Type": "application/json", 311 325 }, 326 + signal: AbortSignal.timeout(15000), // 15s timeout per call 312 327 body: JSON.stringify({ 313 328 messages: [{ 314 329 role: "user", ··· 319 334 320 335 if (!resp.ok) { 321 336 console.error(`[letta] API error: ${resp.status}`); 337 + discoveredUrls.set(cacheKey, null); 322 338 return null; 323 339 } 324 340 325 341 const data = await resp.json() as { messages: Array<{ message_type: string; content?: string }> }; 326 - // Find the assistant message — it should contain just the URL 327 342 for (const msg of data.messages || []) { 328 343 if (msg.message_type === "assistant_message" && msg.content) { 329 - // Extract URL from the response 330 344 const urlMatch = msg.content.match(/https?:\/\/[^\s)\]]+/); 331 - if (urlMatch) return urlMatch[0]; 345 + if (urlMatch) { 346 + discoveredUrls.set(cacheKey, urlMatch[0]); 347 + return urlMatch[0]; 348 + } 332 349 } 333 350 } 351 + discoveredUrls.set(cacheKey, null); 334 352 return null; 335 353 } catch (e: any) { 336 354 console.error("[letta] discoverUrl error:", e.message); 355 + discoveredUrls.set(cacheKey, null); 337 356 return null; 338 357 } 339 358 } ··· 354 373 island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 355 374 vaultNotes: VaultNote[], 356 375 carryNotes: CarryNote[], 376 + islandTokenSet: Set<string>, 377 + vertexTokenSets: Map<string, Set<string>>, 357 378 ): Promise<NewConnection[]> { 358 379 const connections: NewConnection[] = []; 359 - const islandText = buildIslandText(island).toLowerCase(); 380 + const MAX_LETTA_CALLS = 10; 381 + let lettaCalls = 0; 360 382 361 383 // Match vault notes to island vertices 362 384 for (const note of vaultNotes) { 363 385 const noteLower = note.content.toLowerCase(); 364 - const overlap = keywordOverlap(islandText, noteLower); 386 + const overlap = keywordOverlap(islandTokenSet, noteLower); 365 387 if (overlap.length < 3) continue; 366 388 367 389 let noteUrl = extractUrl(note.content); 368 390 const title = extractTitle(note.content) || note.path; 369 391 370 - // If no URL in the note, ask Letta to discover one 371 - if (!noteUrl && LETTA_API_KEY && LETTA_AGENT_ID) { 392 + if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) { 372 393 noteUrl = await discoverUrl(title || note.path); 394 + lettaCalls++; 373 395 } 374 396 if (!noteUrl) continue; 375 397 376 - const bestVertex = findBestVertexMatch(noteLower, island.vertices); 398 + const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets); 377 399 if (bestVertex) { 378 400 connections.push({ 379 401 source: bestVertex.uri, ··· 387 409 // Match carry notes to island vertices 388 410 for (const note of carryNotes) { 389 411 const noteLower = note.content.toLowerCase(); 390 - const overlap = keywordOverlap(islandText, noteLower); 412 + const overlap = keywordOverlap(islandTokenSet, noteLower); 391 413 if (overlap.length < 2) continue; 392 414 393 415 let noteUrl = extractUrl(note.content); 394 416 const title = extractTitle(note.content) || note.path; 395 417 396 - // If no URL in the note, ask Letta to discover one 397 - if (!noteUrl && LETTA_API_KEY && LETTA_AGENT_ID) { 418 + if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) { 398 419 noteUrl = await discoverUrl(`Carry [${note.domain}]: ${title}`); 420 + lettaCalls++; 399 421 } 400 422 if (!noteUrl) continue; 401 423 402 - const bestVertex = findBestVertexMatch(noteLower, island.vertices); 424 + const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets); 403 425 if (bestVertex) { 404 426 connections.push({ 405 427 source: bestVertex.uri, ··· 416 438 function findBestVertexMatch( 417 439 text: string, 418 440 vertices: VertexInfo[], 441 + vertexTokenSets?: Map<string, Set<string>>, 419 442 ): VertexInfo | null { 420 443 let bestScore = 0; 421 444 let bestVertex: VertexInfo | null = null; 422 445 423 446 for (const v of vertices) { 424 - const vertexText = `${v.title} ${v.description || ""}`.toLowerCase(); 425 - const overlap = keywordOverlap(text, vertexText); 447 + let overlap: string[]; 448 + if (vertexTokenSets) { 449 + const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`); 450 + overlap = keywordOverlapSets(tokenize(text), vSet); 451 + } else { 452 + const vertexText = `${v.title} ${v.description || ""}`.toLowerCase(); 453 + overlap = keywordOverlap(tokenize(text), vertexText); 454 + } 426 455 if (overlap.length > bestScore) { 427 456 bestScore = overlap.length; 428 457 bestVertex = v; ··· 469 498 newConnections: NewConnection[], 470 499 ): string[] { 471 500 const highlights: string[] = []; 472 - const lower = islandText.toLowerCase(); 501 + const lowerSet = tokenize(islandText); 473 502 474 503 // Score each edge by semantic significance 475 504 const edgeScores = new Map<string, number>(); 476 505 477 506 for (const edge of island.edges) { 478 507 let score = 0; 508 + const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || ""; 509 + const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || ""; 479 510 480 511 // Check if source or target appears in vault notes 481 512 for (const note of vaultNotes) { 482 513 const noteLower = note.content.toLowerCase(); 483 - const overlap = keywordOverlap(lower, noteLower); 514 + const overlap = keywordOverlap(lowerSet, noteLower); 484 515 if (overlap.length >= 2) { 485 - const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || ""; 486 - const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || ""; 487 516 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2; 488 517 if (targetTitle && noteLower.includes(targetTitle)) score += 2; 489 518 } ··· 492 521 // Check if source or target appears in carry notes 493 522 for (const note of carryNotes) { 494 523 const noteLower = note.content.toLowerCase(); 495 - const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || ""; 496 - const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || ""; 497 524 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1; 498 525 if (targetTitle && noteLower.includes(targetTitle)) score += 1; 499 526 } ··· 530 557 carryNotes: CarryNote[], 531 558 ): string[] { 532 559 const tensions: string[] = []; 533 - const lower = islandText.toLowerCase(); 560 + const lowerSet = tokenize(islandText); 534 561 535 562 for (const note of carryNotes) { 536 563 const noteLower = note.content.toLowerCase(); 537 564 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) { 538 - const overlap = keywordOverlap(lower, noteLower); 565 + const overlap = keywordOverlap(lowerSet, noteLower); 539 566 if (overlap.length >= 1) { 540 567 tensions.push( 541 568 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes` ··· 552 579 reasonings: CarryReasoning[], 553 580 ): string[] { 554 581 const tensions: string[] = []; 555 - const lower = islandText.toLowerCase(); 582 + const lowerSet = tokenize(islandText); 556 583 557 584 for (const r of reasonings) { 558 585 const rLower = `${r.claim} ${r.evidence}`.toLowerCase(); 559 586 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) { 560 - const overlap = keywordOverlap(lower, rLower); 587 + const overlap = keywordOverlap(lowerSet, rLower); 561 588 if (overlap.length >= 1) { 562 589 tensions.push(r.claim.slice(0, 500)); 563 590 } ··· 663 690 "http", "https", "www", "com", "org", 664 691 ]); 665 692 666 - function keywordOverlap(textA: string, textB: string): string[] { 667 - const wordsA = new Set( 668 - textA.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)) 693 + function tokenize(text: string): Set<string> { 694 + return new Set( 695 + text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)) 669 696 ); 670 - const wordsB = textB.split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)); 671 - return wordsB.filter(w => wordsA.has(w)); 697 + } 698 + 699 + function keywordOverlap(setA: Set<string>, textB: string): string[] { 700 + const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w)); 701 + return wordsB.filter(w => setA.has(w)); 702 + } 703 + 704 + function keywordOverlapSets(setA: Set<string>, setB: Set<string>): string[] { 705 + return [...setA].filter(w => setB.has(w)); 672 706 }

Configure Feed

Configure Feed