nandi.uk
This repository has no description
25 kB
706 lines
1// ─── Strata Analysis Engine ─────────────────────────────────────────
2// Cross-references an island (cluster of linked URLs) against:
3// 1. Vault + carry markdown from local filesystem (synced via ob/git)
4// 2. Structured carry records from D1 (entities, citations, reasonings)
5// For carry/vault notes without URLs, uses the Letta API to discover
6// canonical URLs via web search.
7// Returns structured analysis + new connection records to be created on PDS.
8
9import { readdirSync, readFileSync, statSync } from "fs";
10import { join, extname } from "path";
11
12const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
13const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
14const LETTA_API_KEY = process.env.LETTA_API_KEY || "";
15const LETTA_BASE_URL = process.env.LETTA_BASE_URL || "https://api.letta.com";
16const LETTA_AGENT_ID = process.env.LETTA_AGENT_ID || "";
17
18// ─── Types ──────────────────────────────────────────────────────────
19
20interface VaultNote {
21 path: string;
22 content: string;
23}
24
25interface CarryNote {
26 path: string;
27 domain: string;
28 content: string;
29}
30
31interface VertexInfo {
32 uri: string;
33 title: string;
34 description: string;
35 type: string;
36}
37
38interface EdgeInfo {
39 uri: string;
40 did: string;
41 source: string;
42 target: string;
43 connectionType: string;
44 note: string;
45}
46
47interface CarryEntity {
48 atUri: string;
49 name: string;
50 entityType: string;
51 description?: string;
52 url?: string; // The entity's canonical https:// URL
53}
54
55interface CarryCitation {
56 atUri: string;
57 url: string;
58 title: string;
59 takeaway: string;
60 confidence?: string;
61}
62
63interface CarryReasoning {
64 atUri: string;
65 claim: string;
66 evidence: string;
67 reasoningType?: string;
68 url?: string; // The reasoning's source https:// URL
69}
70
71interface CarryData {
72 entities: CarryEntity[];
73 citations: CarryCitation[];
74 reasonings: CarryReasoning[];
75}
76
77interface NewConnection {
78 source: string; // island vertex URI
79 target: string; // carry record AT URI
80 connectionType: string; // e.g. "network.cosmik.connection#annotates"
81 note: string; // why this connection exists
82}
83
84interface AnalysisInput {
85 did: string;
86 island: {
87 vertices: VertexInfo[];
88 edges: EdgeInfo[];
89 };
90 carryData?: CarryData;
91}
92
93interface AnalysisResult {
94 themes: string[];
95 highlights: string[];
96 tensions: string[];
97 openQuestions: string[];
98 synthesis: string;
99 newConnections: NewConnection[];
100}
101
102// ─── Main analysis function ────────────────────────────────────────
103
104export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
105 const { island, carryData } = input;
106
107 // 1. Read vault + carry from filesystem
108 const vaultNotes = readVault();
109 const carryNotes = readCarry();
110
111 // 2. Build corpus from the entire island + precompute token sets
112 const islandText = buildIslandText(island);
113 const islandTokenSet = tokenize(islandText);
114 const vertexTokenSets = new Map<string, Set<string>>();
115 for (const v of island.vertices) {
116 vertexTokenSets.set(v.uri, tokenize(`${v.title} ${v.description || ""}`));
117 }
118
119 // 3. Extract themes from the island
120 const themes = extractThemes(islandText);
121
122 // 4. Find carry touchpoints — semantic matches from D1 records
123 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] }, vertexTokenSets);
124
125 // 5. Find vault/carry touchpoints — semantic matches from filesystem notes
126 // (async — may call Letta API for URL discovery)
127 const fsConnections = await findFilesystemTouchpoints(island, vaultNotes, carryNotes, islandTokenSet, vertexTokenSets);
128 newConnections.push(...fsConnections);
129
130 // 6. Flag hot edges — existing edges that are semantically significant
131 const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections);
132
133 // 7. Find tensions from carry notes + reasonings
134 const tensions = [
135 ...findTensionsFromNotes(islandText, carryNotes),
136 ...findTensionsFromReasonings(islandText, carryData?.reasonings || []),
137 ];
138
139 // 8. Generate open questions
140 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
141
142 // 9. Synthesize prose
143 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
144
145 // Deduplicate newConnections
146 const seen = new Set<string>();
147 const deduped = newConnections.filter(c => {
148 const key = `${c.source}|${c.target}`;
149 if (seen.has(key)) return false;
150 seen.add(key);
151 return true;
152 });
153
154 return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
155}
156
157// ─── Filesystem readers ────────────────────────────────────────────
158
159function readVault(): VaultNote[] {
160 return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
161 path: relPath,
162 content,
163 }));
164}
165
166function readCarry(): CarryNote[] {
167 const notes: CarryNote[] = [];
168 try {
169 const domains = readdirSync(CARRY_PATH);
170 for (const domain of domains) {
171 const domainPath = join(CARRY_PATH, domain);
172 if (!statSync(domainPath).isDirectory()) continue;
173 if (domain.startsWith(".")) continue;
174 const files = readMarkdownTree(domainPath);
175 for (const { relPath, content } of files) {
176 notes.push({ path: relPath, domain, content });
177 }
178 }
179 } catch (e: any) {
180 console.error("Failed to read carry:", e.message);
181 }
182 return notes;
183}
184
185function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
186 const results: Array<{ relPath: string; content: string }> = [];
187 try {
188 const entries = readdirSync(root, { withFileTypes: true });
189 for (const entry of entries) {
190 const fullPath = join(root, entry.name);
191 if (entry.name.startsWith(".")) continue;
192 if (entry.isDirectory()) {
193 results.push(...readMarkdownTree(fullPath));
194 } else if (extname(entry.name) === ".md") {
195 try {
196 const content = readFileSync(fullPath, "utf-8");
197 results.push({ relPath: fullPath.replace(root + "/", ""), content });
198 } catch {}
199 }
200 }
201 } catch {}
202 return results;
203}
204
205// ─── Build island text corpus ──────────────────────────────────────
206
207function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
208 const parts: string[] = [];
209
210 for (const v of island.vertices) {
211 parts.push(v.title);
212 if (v.description) parts.push(v.description);
213 }
214
215 for (const e of island.edges) {
216 if (e.note) parts.push(e.note);
217 }
218
219 return parts.join(" ");
220}
221
222// ─── Carry touchpoint matching (D1 records) ────────────────────────
223
224function findCarryTouchpoints(
225 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
226 carryData: CarryData,
227 vertexTokenSets: Map<string, Set<string>>,
228): NewConnection[] {
229 const connections: NewConnection[] = [];
230
231 // --- Match citations by URL ---
232 for (const cite of carryData.citations) {
233 if (!cite.url) continue; // Need an https:// URL as target
234 // Direct URL match: citation URL is a vertex URI
235 const vertexMatch = island.vertices.find(v => v.uri === cite.url);
236 if (vertexMatch) {
237 connections.push({
238 source: cite.url,
239 target: cite.url, // Self-referential — the citation IS about this vertex
240 connectionType: "network.cosmik.connection#annotates",
241 note: cite.takeaway.slice(0, 500),
242 });
243 continue;
244 }
245
246 // Semantic match: citation title/takeaway overlaps with island vertices
247 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
248 const bestVertex = findBestVertexMatch(citeText, island.vertices, vertexTokenSets);
249 if (bestVertex) {
250 const overlap = keywordOverlapSets(tokenize(citeText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
251 if (overlap.length >= 2) {
252 connections.push({
253 source: bestVertex.uri,
254 target: cite.url,
255 connectionType: "network.cosmik.connection#annotates",
256 note: cite.takeaway.slice(0, 500),
257 });
258 }
259 }
260 }
261
262 // --- Match entities by name/description ---
263 for (const entity of carryData.entities) {
264 if (!entity.url) continue; // Need an https:// URL as target
265 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
266 const bestVertex = findBestVertexMatch(entityText, island.vertices, vertexTokenSets);
267 if (bestVertex) {
268 const overlap = keywordOverlapSets(tokenize(entityText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
269 if (overlap.length >= 2) {
270 connections.push({
271 source: bestVertex.uri,
272 target: entity.url,
273 connectionType: "network.cosmik.connection#relates",
274 note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
275 });
276 }
277 }
278 }
279
280 // --- Match reasonings by claim/evidence ---
281 for (const reasoning of carryData.reasonings) {
282 if (!reasoning.url) continue; // Need an https:// URL as target
283 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
284 const bestVertex = findBestVertexMatch(reasoningText, island.vertices, vertexTokenSets);
285 if (bestVertex) {
286 const overlap = keywordOverlapSets(tokenize(reasoningText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
287 if (overlap.length >= 2) {
288 const connType = reasoning.reasoningType?.includes("contradict")
289 ? "network.cosmik.connection#contradicts"
290 : "network.cosmik.connection#extends";
291 connections.push({
292 source: bestVertex.uri,
293 target: reasoning.url,
294 connectionType: connType,
295 note: reasoning.claim.slice(0, 500),
296 });
297 }
298 }
299 }
300
301 return connections;
302}
303
304// ─── Letta URL discovery ────────────────────────────────────────────
305// For carry/vault notes that lack an https:// URL, ask the Letta agent
306// to find the canonical URL via web search. Only called for notes that
307// already matched the island (passed keyword overlap), so the volume
308// is small. Results are cached in-memory for the lifetime of the container.
309
310const discoveredUrls = new Map<string, string | null>();
311
312async function discoverUrl(semanticDescription: string): Promise<string | null> {
313 if (!LETTA_API_KEY || !LETTA_AGENT_ID) return null;
314
315 // Check cache
316 const cacheKey = semanticDescription.slice(0, 200);
317 if (discoveredUrls.has(cacheKey)) return discoveredUrls.get(cacheKey)!;
318
319 try {
320 const resp = await fetch(`${LETTA_BASE_URL}/v1/agents/${LETTA_AGENT_ID}/messages`, {
321 method: "POST",
322 headers: {
323 "Authorization": `Bearer ${LETTA_API_KEY}`,
324 "Content-Type": "application/json",
325 },
326 signal: AbortSignal.timeout(15000), // 15s timeout per call
327 body: JSON.stringify({
328 messages: [{
329 role: "user",
330 content: `Find the canonical URL for: ${semanticDescription}`,
331 }],
332 }),
333 });
334
335 if (!resp.ok) {
336 console.error(`[letta] API error: ${resp.status}`);
337 discoveredUrls.set(cacheKey, null);
338 return null;
339 }
340
341 const data = await resp.json() as { messages: Array<{ message_type: string; content?: string }> };
342 for (const msg of data.messages || []) {
343 if (msg.message_type === "assistant_message" && msg.content) {
344 const urlMatch = msg.content.match(/https?:\/\/[^\s)\]]+/);
345 if (urlMatch) {
346 discoveredUrls.set(cacheKey, urlMatch[0]);
347 return urlMatch[0];
348 }
349 }
350 }
351 discoveredUrls.set(cacheKey, null);
352 return null;
353 } catch (e: any) {
354 console.error("[letta] discoverUrl error:", e.message);
355 discoveredUrls.set(cacheKey, null);
356 return null;
357 }
358}
359
360// ─── Filesystem touchpoint matching (vault + carry notes) ───────────
361
362function extractUrl(content: string): string | null {
363 // Pull the first https:// URL from the note content
364 const match = content.match(/https?:\/\/[^\s)\]]+/);
365 return match ? match[0] : null;
366}
367
368function extractTitle(content: string): string {
369 return content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || "";
370}
371
372async function findFilesystemTouchpoints(
373 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
374 vaultNotes: VaultNote[],
375 carryNotes: CarryNote[],
376 islandTokenSet: Set<string>,
377 vertexTokenSets: Map<string, Set<string>>,
378): Promise<NewConnection[]> {
379 const connections: NewConnection[] = [];
380 const MAX_LETTA_CALLS = 10;
381 let lettaCalls = 0;
382
383 // Match vault notes to island vertices
384 for (const note of vaultNotes) {
385 const noteLower = note.content.toLowerCase();
386 const overlap = keywordOverlap(islandTokenSet, noteLower);
387 if (overlap.length < 3) continue;
388
389 let noteUrl = extractUrl(note.content);
390 const title = extractTitle(note.content) || note.path;
391
392 if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) {
393 noteUrl = await discoverUrl(title || note.path);
394 lettaCalls++;
395 }
396 if (!noteUrl) continue;
397
398 const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets);
399 if (bestVertex) {
400 connections.push({
401 source: bestVertex.uri,
402 target: noteUrl,
403 connectionType: "network.cosmik.connection#annotates",
404 note: `Vault note: ${title.slice(0, 200)}`,
405 });
406 }
407 }
408
409 // Match carry notes to island vertices
410 for (const note of carryNotes) {
411 const noteLower = note.content.toLowerCase();
412 const overlap = keywordOverlap(islandTokenSet, noteLower);
413 if (overlap.length < 2) continue;
414
415 let noteUrl = extractUrl(note.content);
416 const title = extractTitle(note.content) || note.path;
417
418 if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) {
419 noteUrl = await discoverUrl(`Carry [${note.domain}]: ${title}`);
420 lettaCalls++;
421 }
422 if (!noteUrl) continue;
423
424 const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets);
425 if (bestVertex) {
426 connections.push({
427 source: bestVertex.uri,
428 target: noteUrl,
429 connectionType: "network.cosmik.connection#relates",
430 note: `Carry [${note.domain}]: ${title.slice(0, 200)}`,
431 });
432 }
433 }
434
435 return connections;
436}
437
438function findBestVertexMatch(
439 text: string,
440 vertices: VertexInfo[],
441 vertexTokenSets?: Map<string, Set<string>>,
442): VertexInfo | null {
443 let bestScore = 0;
444 let bestVertex: VertexInfo | null = null;
445
446 for (const v of vertices) {
447 let overlap: string[];
448 if (vertexTokenSets) {
449 const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`);
450 overlap = keywordOverlapSets(tokenize(text), vSet);
451 } else {
452 const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
453 overlap = keywordOverlap(tokenize(text), vertexText);
454 }
455 if (overlap.length > bestScore) {
456 bestScore = overlap.length;
457 bestVertex = v;
458 }
459 }
460
461 return bestScore >= 2 ? bestVertex : null;
462}
463
464// ─── Analysis functions ─────────────────────────────────────────────
465
466function extractThemes(text: string): string[] {
467 const keywords: Record<string, string[]> = {
468 "open science": ["open science", "open access", "oa", "preprint"],
469 "data sharing": ["data sharing", "shared data", "open data", "dataset"],
470 "reproducibility": ["reproducib", "replicab", "replication"],
471 "decentralization": ["decentraliz", "federat", "self-host"],
472 "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
473 "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
474 "community": ["community", "collective", "cooperative", "commons"],
475 "governance": ["governance", "policy", "regulation", "standards"],
476 "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
477 "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
478 "provenance": ["provenance", "lineage", "traceability", "attribution"],
479 "doi": ["doi", "zenodo", "citeable", "citation"],
480 "privacy": ["privacy", "encryption", "e2ee", "consent"],
481 "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
482 "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
483 "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
484 "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
485 };
486
487 const lower = text.toLowerCase();
488 return Object.entries(keywords)
489 .filter(([_, terms]) => terms.some(t => lower.includes(t)))
490 .map(([theme]) => theme);
491}
492
493function findHotEdges(
494 islandText: string,
495 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
496 vaultNotes: VaultNote[],
497 carryNotes: CarryNote[],
498 newConnections: NewConnection[],
499): string[] {
500 const highlights: string[] = [];
501 const lowerSet = tokenize(islandText);
502
503 // Score each edge by semantic significance
504 const edgeScores = new Map<string, number>();
505
506 for (const edge of island.edges) {
507 let score = 0;
508 const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
509 const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
510
511 // Check if source or target appears in vault notes
512 for (const note of vaultNotes) {
513 const noteLower = note.content.toLowerCase();
514 const overlap = keywordOverlap(lowerSet, noteLower);
515 if (overlap.length >= 2) {
516 if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
517 if (targetTitle && noteLower.includes(targetTitle)) score += 2;
518 }
519 }
520
521 // Check if source or target appears in carry notes
522 for (const note of carryNotes) {
523 const noteLower = note.content.toLowerCase();
524 if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
525 if (targetTitle && noteLower.includes(targetTitle)) score += 1;
526 }
527
528 // Edges with notes are more interesting
529 if (edge.note && edge.note.length > 10) score += 1;
530
531 // Edges with semantic connection types are more interesting
532 const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
533 if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
534
535 // Boost edges whose source or target is also connected to carry data
536 for (const nc of newConnections) {
537 if (edge.source === nc.source || edge.target === nc.source) score += 3;
538 if (edge.source === nc.target || edge.target === nc.target) score += 3;
539 }
540
541 if (score > 0) {
542 edgeScores.set(edge.uri, score);
543 }
544 }
545
546 // Return top edges by score
547 const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
548 for (const [uri] of sorted.slice(0, 10)) {
549 highlights.push(uri);
550 }
551
552 return highlights;
553}
554
555function findTensionsFromNotes(
556 islandText: string,
557 carryNotes: CarryNote[],
558): string[] {
559 const tensions: string[] = [];
560 const lowerSet = tokenize(islandText);
561
562 for (const note of carryNotes) {
563 const noteLower = note.content.toLowerCase();
564 if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
565 const overlap = keywordOverlap(lowerSet, noteLower);
566 if (overlap.length >= 1) {
567 tensions.push(
568 `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
569 );
570 }
571 }
572 }
573
574 return tensions.slice(0, 5);
575}
576
577function findTensionsFromReasonings(
578 islandText: string,
579 reasonings: CarryReasoning[],
580): string[] {
581 const tensions: string[] = [];
582 const lowerSet = tokenize(islandText);
583
584 for (const r of reasonings) {
585 const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
586 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
587 const overlap = keywordOverlap(lowerSet, rLower);
588 if (overlap.length >= 1) {
589 tensions.push(r.claim.slice(0, 500));
590 }
591 }
592 }
593
594 return tensions.slice(0, 5);
595}
596
597function generateOpenQuestions(
598 islandText: string,
599 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
600 highlights: string[],
601 tensions: string[],
602 newConnections: NewConnection[],
603): string[] {
604 const questions: string[] = [];
605 const lower = islandText.toLowerCase();
606
607 if (newConnections.length > 0) {
608 questions.push(
609 `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
610 );
611 }
612
613 if (tensions.length > 0) {
614 questions.push(
615 `What evidence would resolve the ${tensions.length} tension(s) identified?`
616 );
617 }
618
619 const vertexCount = island.vertices.length;
620 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
621
622 if (edgeTypes.size > 1) {
623 questions.push(
624 `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
625 );
626 }
627
628 if (vertexCount > 5) {
629 questions.push(
630 `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
631 );
632 }
633
634 if (lower.includes("data") || lower.includes("dataset")) {
635 questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
636 }
637 if (lower.includes("standard") || lower.includes("protocol")) {
638 questions.push("Does this cluster propose a new standard, and who are the adopters?");
639 }
640
641 return questions.slice(0, 5);
642}
643
644function synthesize(
645 island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
646 themes: string[],
647 highlights: string[],
648 tensions: string[],
649 openQuestions: string[],
650 newConnections: NewConnection[],
651): string {
652 const vertexCount = island.vertices.length;
653 const edgeCount = island.edges.length;
654 const edgeTypes = new Set(island.edges.map(e => e.connectionType));
655
656 const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;
657
658 const themeStr = themes.length > 0
659 ? `Themes: ${themes.join(", ")}.`
660 : "";
661
662 const touchpointStr = newConnections.length > 0
663 ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
664 : "No carry touchpoints found.";
665
666 const highlightStr = highlights.length > 0
667 ? `${highlights.length} existing edges flagged as semantically significant.`
668 : "";
669
670 const tensionStr = tensions.length > 0
671 ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
672 : "";
673
674 const questionStr = openQuestions.length > 0
675 ? `Open questions: ${openQuestions.join(" ")}`
676 : "";
677
678 return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
679}
680
681// ─── Utilities ──────────────────────────────────────────────────────
682
683const STOP_WORDS = new Set([
684 "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
685 "have", "has", "had", "do", "does", "did", "will", "would", "could",
686 "should", "may", "might", "shall", "can", "to", "of", "in", "for",
687 "on", "with", "at", "by", "from", "as", "into", "through", "during",
688 "before", "after", "above", "below", "between", "and", "but", "or",
689 "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
690 "http", "https", "www", "com", "org",
691]);
692
693function tokenize(text: string): Set<string> {
694 return new Set(
695 text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
696 );
697}
698
699function keywordOverlap(setA: Set<string>, textB: string): string[] {
700 const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
701 return wordsB.filter(w => setA.has(w));
702}
703
704function keywordOverlapSets(setA: Set<string>, setB: Set<string>): string[] {
705 return [...setA].filter(w => setB.has(w));
706}