hotsocket.fyi
This repository has no description
1import fs from "fs/promises"
2import path from "path"
3import { createHash } from "crypto"
4import { fileURLToPath } from "url"
5import type { Message } from "./types.js"
6import { getDb, isVecAvailable, MEMORY_EMBEDDING_DIMENSIONS } from "./db.js"
7import { EMBEDDING_DIMENSIONS, EMBEDDING_MODEL, embeddingsConfigured, embedTexts } from "./embeddings.js"
8import { recordMetric } from "./metrics.js"
9
10const HOME_DIR = path.resolve(fileURLToPath(import.meta.url), "../../home")
11const MEMORIES_DIR = path.join(HOME_DIR, "memories")
12const JOURNAL_DIR = path.join(MEMORIES_DIR, "journal")
13const PEOPLE_DIR = path.join(MEMORIES_DIR, "people")
14const CORE_FILE = path.join(MEMORIES_DIR, "core.md")
15const ALIASES_FILE = path.join(MEMORIES_DIR, "aliases.json")
16
17const MEMORY_RECALL_HEADER = "[memory recall v1]"
18const MEMORY_RECALL_NOTE =
19 "Potentially relevant long-term notes. Use only if helpful; trust newer conversation details if anything conflicts."
20const MEMORY_RECALL_MAX_CHUNKS = 4
21const MEMORY_RECALL_MAX_CHUNKS_HARD_CAP = 8
22const MEMORY_RECALL_MAX_CHARS = 1_500
23const MEMORY_RECALL_PER_EXTRA_PERSON_CHARS = 400
24const MEMORY_QUERY_TOKEN_LIMIT = 12
25const MEMORY_RECALL_COOLDOWN_TURNS = 7
26const MEMORY_EMBEDDING_BATCH_SIZE = 24
27const MEMORY_SEMANTIC_MIN_SIMILARITY = 0.18
28const MEMORY_SEMANTIC_STRONG_SIMILARITY = 0.32
29const MEMORY_CHATTER_SIMILARITY_THRESHOLD = 0.74
30const MEMORY_RECALL_INTENT_SIMILARITY_THRESHOLD = 0.55
31const SCHEDULED_HEARTBEAT_CONTENT = "Scheduled heartbeat."
32const MEMORY_EMBEDDING_PROTOTYPES = [
33 { id: 1, name: "affection-love", category: "chatter", text: "i love you so much sweetie <33" },
34 { id: 2, name: "cat-greeting", category: "chatter", text: "boop mraow meow hi sweetie" },
35 { id: 3, name: "celebration", category: "chatter", text: "yay yayy lets gooooo <33" },
36 { id: 4, name: "goodnight", category: "chatter", text: "goodnight sweet dreams rest well" },
37 { id: 101, name: "who-person", category: "recall_intent", text: "who is this person what do i know about them" },
38 { id: 102, name: "past-event", category: "recall_intent", text: "what happened before remember when that event happened" },
39 { id: 103, name: "task-context", category: "recall_intent", text: "what context do i need for this task or project" },
40 { id: 104, name: "system-lesson", category: "recall_intent", text: "what lesson or instruction should i remember here" },
41] as const
42const MEMORY_STOP_WORDS = new Set([
43 "a",
44 "an",
45 "and",
46 "are",
47 "as",
48 "at",
49 "be",
50 "been",
51 "but",
52 "by",
53 "for",
54 "from",
55 "had",
56 "has",
57 "have",
58 "he",
59 "her",
60 "hers",
61 "him",
62 "his",
63 "i",
64 "if",
65 "in",
66 "into",
67 "is",
68 "it",
69 "its",
70 "me",
71 "my",
72 "of",
73 "on",
74 "or",
75 "our",
76 "she",
77 "that",
78 "the",
79 "their",
80 "them",
81 "there",
82 "they",
83 "this",
84 "to",
85 "up",
86 "us",
87 "was",
88 "we",
89 "were",
90 "with",
91 "you",
92 "your",
93])
94
95type MemoryKind = "core" | "journal" | "people"
96
97type MemoryDocumentRow = {
98 id: number
99 path: string
100 content_hash: string
101 mtime_ms: number
102 kind?: string
103 title?: string
104}
105
106type MemoryChunkInput = {
107 title: string
108 headingPath: string | null
109 text: string
110 tags: string
111}
112
113type MemoryHit = {
114 chunkId: number
115 path: string
116 kind: MemoryKind
117 documentTitle: string
118 title: string
119 headingPath: string | null
120 text: string
121 rank: number
122 semanticDistance?: number
123 semanticSimilarity?: number
124}
125
126export type MemorySearchResult = {
127 chunkId: number
128 kind: MemoryKind
129 path: string
130 source: string
131 title: string
132 headingPath: string | null
133 preview: string
134}
135
136type MemoryQueryParts = {
137 sender: string | null
138 source: string | null
139 body: string
140}
141
142type MemorySearchProfile = {
143 normalized: string
144 sender: string | null
145 senderAliases: string[]
146 bodyTokens: string[]
147 bodyPeople: string[]
148 tokens: string[]
149 personQuery: boolean
150 eventQuery: boolean
151 bodyInformative: boolean
152}
153
154type MemoryHitSignal = {
155 overlap: number
156 strongOverlap: boolean
157 bodyOverlap: number
158 senderMatch: boolean
159}
160
161type MemoryEmbeddingRow = {
162 chunkId: number
163 path: string
164 kind: MemoryKind
165 documentTitle: string
166 title: string
167 headingPath: string | null
168 text: string
169 tags: string | null
170 model: string | null
171 dimensions: number | null
172 contentHash: string | null
173}
174
175type SemanticQuerySignal = {
176 vector: number[]
177 chatterSimilarity: number | null
178 recallIntentSimilarity: number | null
179}
180
181export type AliasMap = Record<string, string[]>
182
183function normalizeText(value: string): string {
184 return value.replace(/\r\n/g, "\n").replace(/\s+/g, " ").trim()
185}
186
187function trimForPrompt(value: string, maxChars: number): string {
188 if (value.length <= maxChars) return value
189 if (maxChars <= 3) return ".".repeat(maxChars)
190 return `${value.slice(0, maxChars - 3).trimEnd()}...`
191}
192
193function basenameWithoutExt(filePath: string): string {
194 return path.basename(filePath, path.extname(filePath))
195}
196
197function titleFromPath(filePath: string, fallback: string): string {
198 const base = basenameWithoutExt(filePath).replace(/[-_]+/g, " ").trim()
199 return base ? base : fallback
200}
201
202function detectMemoryKind(filePath: string): MemoryKind | null {
203 if (filePath === CORE_FILE) return "core"
204 if (filePath.startsWith(`${JOURNAL_DIR}${path.sep}`)) return "journal"
205 if (filePath.startsWith(`${PEOPLE_DIR}${path.sep}`)) return "people"
206 return null
207}
208
209async function pathExists(target: string): Promise<boolean> {
210 try {
211 await fs.access(target)
212 return true
213 } catch {
214 return false
215 }
216}
217
218function normalizeHandle(handle: string): string {
219 return handle.trim().replace(/^@+/, "").toLowerCase()
220}
221
222let aliasCache: { mtimeMs: number; map: AliasMap } | null = null
223
224async function loadAliasMap(): Promise<AliasMap> {
225 try {
226 const stat = await fs.stat(ALIASES_FILE)
227 if (aliasCache && aliasCache.mtimeMs === stat.mtimeMs) return aliasCache.map
228 const raw = await fs.readFile(ALIASES_FILE, "utf-8")
229 const parsed = JSON.parse(raw) as unknown
230 const map: AliasMap = {}
231 if (parsed && typeof parsed === "object") {
232 for (const [key, value] of Object.entries(parsed as Record<string, unknown>)) {
233 const handle = normalizeHandle(key)
234 if (!handle) continue
235 const list = Array.isArray(value) ? value : [value]
236 const aliases = list
237 .map((v) => (typeof v === "string" ? normalizeHandle(v) : ""))
238 .filter((v) => v && v !== handle)
239 if (aliases.length > 0) map[handle] = Array.from(new Set(aliases))
240 }
241 }
242 aliasCache = { mtimeMs: stat.mtimeMs, map }
243 return map
244 } catch {
245 aliasCache = { mtimeMs: 0, map: {} }
246 return {}
247 }
248}
249
250async function writeAliasMap(map: AliasMap): Promise<void> {
251 await fs.mkdir(MEMORIES_DIR, { recursive: true })
252 const sorted: AliasMap = {}
253 for (const key of Object.keys(map).sort()) sorted[key] = [...map[key]!].sort()
254 await fs.writeFile(ALIASES_FILE, `${JSON.stringify(sorted, null, 2)}\n`, "utf-8")
255 aliasCache = null
256}
257
258function resolveAliases(handle: string | null, map: AliasMap): string[] {
259 if (!handle) return []
260 const seen = new Set<string>([handle])
261 const out: string[] = []
262 const queue = [handle]
263 while (queue.length > 0) {
264 const current = queue.shift()!
265 const next = map[current] ?? []
266 for (const alias of next) {
267 if (seen.has(alias)) continue
268 seen.add(alias)
269 out.push(alias)
270 queue.push(alias)
271 }
272 }
273 return out
274}
275
276export async function listAliases(): Promise<AliasMap> {
277 return loadAliasMap()
278}
279
280export async function setAlias(handle: string, canonical: string): Promise<AliasMap> {
281 const h = normalizeHandle(handle)
282 const c = normalizeHandle(canonical)
283 if (!h || !c) throw new Error("alias handle and canonical must be non-empty")
284 const map = await loadAliasMap()
285 if (h === c) return map
286 const existing = new Set(map[h] ?? [])
287 existing.add(c)
288 map[h] = Array.from(existing)
289 await writeAliasMap(map)
290 return map
291}
292
293export async function removeAlias(handle: string, canonical?: string): Promise<AliasMap> {
294 const h = normalizeHandle(handle)
295 if (!h) throw new Error("alias handle must be non-empty")
296 const map = await loadAliasMap()
297 if (!map[h]) return map
298 if (canonical) {
299 const c = normalizeHandle(canonical)
300 map[h] = map[h]!.filter((entry) => entry !== c)
301 if (map[h]!.length === 0) delete map[h]
302 } else {
303 delete map[h]
304 }
305 await writeAliasMap(map)
306 return map
307}
308
309async function walkMarkdownFiles(root: string): Promise<string[]> {
310 if (!(await pathExists(root))) return []
311
312 const found: string[] = []
313 const entries = await fs.readdir(root, { withFileTypes: true })
314 for (const entry of entries) {
315 const fullPath = path.join(root, entry.name)
316 if (entry.isDirectory()) {
317 found.push(...(await walkMarkdownFiles(fullPath)))
318 continue
319 }
320 if (entry.isFile() && entry.name.endsWith(".md")) found.push(fullPath)
321 }
322
323 return found.sort()
324}
325
326async function listMemoryFiles(): Promise<string[]> {
327 const files: string[] = []
328 if (await pathExists(CORE_FILE)) files.push(CORE_FILE)
329 files.push(...(await walkMarkdownFiles(JOURNAL_DIR)))
330 files.push(...(await walkMarkdownFiles(PEOPLE_DIR)))
331 return files
332}
333
334function contentHash(content: string): string {
335 return createHash("sha1").update(content).digest("hex")
336}
337
338function embeddingInputHash(content: string): string {
339 return createHash("sha256").update(content).digest("hex")
340}
341
342function vectorParam(vector: number[]): Float32Array {
343 return new Float32Array(vector)
344}
345
346function embeddingTextForChunk(row: {
347 path: string
348 kind: MemoryKind
349 documentTitle: string
350 title: string
351 headingPath: string | null
352 text: string
353 tags?: string | null
354}): string {
355 const relativePath = path.relative(HOME_DIR, row.path)
356 return [
357 `kind: ${row.kind}`,
358 `file: ${relativePath}`,
359 `document: ${row.documentTitle}`,
360 `title: ${row.title}`,
361 row.headingPath ? `section: ${row.headingPath}` : null,
362 row.tags ? `tags: ${row.tags}` : null,
363 "",
364 row.text,
365 ]
366 .filter((part): part is string => part !== null)
367 .join("\n")
368}
369
370function chunkLargeSection(text: string, maxChars = 900): string[] {
371 const paragraphs = text
372 .split(/\n\s*\n/g)
373 .map((part) => part.trim())
374 .filter(Boolean)
375
376 if (paragraphs.length === 0) return []
377
378 const chunks: string[] = []
379 let current = ""
380
381 for (const paragraph of paragraphs) {
382 const next = current ? `${current}\n\n${paragraph}` : paragraph
383 if (next.length <= maxChars || current.length === 0) {
384 current = next
385 continue
386 }
387 chunks.push(current)
388 current = paragraph
389 }
390
391 if (current) chunks.push(current)
392 return chunks
393}
394
395function parseMarkdownDocument(filePath: string, content: string): { title: string; chunks: MemoryChunkInput[] } {
396 const lines = content.replace(/\r\n/g, "\n").split("\n")
397 const h1 = lines.find((line) => /^#\s+/.test(line))
398 const title = h1 ? h1.replace(/^#\s+/, "").trim() : titleFromPath(filePath, "Memory")
399 const headingStack: string[] = []
400 let sectionLines: string[] = []
401 let sectionTitle = title
402 const chunks: MemoryChunkInput[] = []
403
404 const flushSection = () => {
405 const body = sectionLines.join("\n").trim()
406 if (!body) {
407 sectionLines = []
408 return
409 }
410
411 const headingPath = headingStack.length > 0 ? headingStack.join(" > ") : null
412 const tags = [basenameWithoutExt(filePath), ...headingStack].join(" ").trim()
413 for (const part of chunkLargeSection(body)) {
414 chunks.push({
415 title: sectionTitle || title,
416 headingPath,
417 text: part,
418 tags,
419 })
420 }
421 sectionLines = []
422 }
423
424 for (const line of lines) {
425 const headingMatch = line.match(/^(#{1,6})\s+(.*)$/)
426 if (!headingMatch) {
427 sectionLines.push(line)
428 continue
429 }
430
431 flushSection()
432
433 const level = headingMatch[1]!.length
434 const heading = headingMatch[2]!.trim()
435 if (level === 1) {
436 sectionTitle = heading || title
437 headingStack.length = 0
438 continue
439 }
440
441 while (headingStack.length >= level - 1) headingStack.pop()
442 headingStack.push(heading)
443 sectionTitle = heading || title
444 }
445
446 flushSection()
447
448 if (chunks.length === 0) {
449 const body = content.trim()
450 if (body) {
451 chunks.push({
452 title,
453 headingPath: null,
454 text: body,
455 tags: basenameWithoutExt(filePath),
456 })
457 }
458 }
459
460 return { title, chunks }
461}
462
463function memoryRecallCooldownTurns(kind: MemoryKind): number {
464 if (kind === "people" || kind === "journal") return MEMORY_RECALL_COOLDOWN_TURNS
465 return 0
466}
467
468function isCoolingDown(hit: MemoryHit, cooldowns: Record<number, number>, currentTurn: number): boolean {
469 const lastTurn = cooldowns[hit.chunkId]
470 if (typeof lastTurn !== "number") return false
471 return currentTurn - lastTurn < memoryRecallCooldownTurns(hit.kind)
472}
473
474function isMemoryRecallSkippedMessage(content: string): boolean {
475 if (content.startsWith(MEMORY_RECALL_HEADER)) return true
476 if (content.startsWith("[system]")) return true
477 if (content.includes("scan snapshot:") && !content.includes("[discord batch]")) return true
478 if (/\[discord batch\]/i.test(content) && conciseDiscordBatchMemoryQuery(content) === null) return true
479 return false
480}
481
482function latestMemoryRecallQuery(conversation: Message[]): string | null {
483 for (let i = conversation.length - 1; i >= 0; i -= 1) {
484 const message = conversation[i]
485 if (!message || message.role !== "user") continue
486 if (typeof message.content !== "string") continue
487
488 const content = message.content.trim()
489 if (!content || isMemoryRecallSkippedMessage(content)) continue
490 if (content === SCHEDULED_HEARTBEAT_CONTENT) continue
491 return content
492 }
493 return null
494}
495
496function discordChannelLabel(channelId: string | null, fallbackContext: string | null, isDm: boolean): string {
497 if (isDm) return "DM"
498
499 const fallback = fallbackContext
500 ?.replace(/^context:\s*/i, "")
501 .replace(/\s*\(\d+\)\s*$/, "")
502 .trim()
503
504 if (channelId) {
505 try {
506 const row = getDb()
507 .prepare("select guild_id, guild_name, channel_name, is_dm from discord_channels where channel_id = ?")
508 .get(channelId) as
509 | {
510 guild_id: string | null
511 guild_name: string | null
512 channel_name: string | null
513 is_dm: number
514 }
515 | undefined
516
517 if (row?.is_dm) return "DM"
518 if (row) {
519 const guild = row.guild_name ?? row.guild_id
520 const channel = row.channel_name ?? channelId
521 if (guild && channel) return `${guild}/#${channel}`
522 if (channel) return `#${channel}`
523 }
524 } catch {
525 // If the main db is unavailable in tests or scripts, keep the parsed context.
526 }
527 }
528
529 if (fallback) return fallback
530 return channelId ? `#${channelId}` : "channel"
531}
532
533function conciseDiscordMemoryQuery(raw: string): MemoryQueryParts | null {
534 const withoutWakeEnvelope = raw.replace(/^\[(wake|incoming|harness restarted)[^\n]*\]\s*/gi, "").trim()
535 if (!/\[discord\/(?:dm|channel)\]/i.test(withoutWakeEnvelope)) return null
536
537 const blocks = withoutWakeEnvelope
538 .split(/\n\s*\n/g)
539 .map((block) => block.trim())
540 .filter(Boolean)
541 const headerBlock = blocks[0] ?? withoutWakeEnvelope
542 const message = blocks.length > 1 ? blocks.slice(1).join("\n\n").trim() : ""
543
544 const lines = headerBlock.split("\n").map((line) => line.trim()).filter(Boolean)
545 const discordLine = lines.find((line) => /^\[discord\/(?:dm|channel)\]/i.test(line)) ?? ""
546 const contextLine = lines.find((line) => /^context:\s*/i.test(line)) ?? null
547 const isDm = /\[discord\/dm\]/i.test(discordLine)
548 const author = discordLine.match(/@(\S+)/)?.[1] ?? null
549 const context = contextLine?.replace(/^context:\s*/i, "").trim() ?? ""
550 const dmChannelId = context.match(/^DM\s+(\d+)/i)?.[1] ?? null
551 const namedChannelId = context.match(/\((\d+)\)\s*$/)?.[1] ?? null
552 const channelId = dmChannelId ?? namedChannelId
553 const location = discordChannelLabel(channelId, contextLine, isDm)
554
555 if (!author && !location && !message) return null
556 return {
557 sender: author ? normalizeHandle(author) : null,
558 source: location || null,
559 body: message,
560 }
561}
562
563function extractBulletSection(raw: string, label: string): string[] {
564 const escapedLabel = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
565 const match = raw.match(new RegExp(`(?:^|\\n)${escapedLabel}:\\n([\\s\\S]*?)(?:\\n\\n[^\\n:]+:|$)`, "i"))
566 if (!match) return []
567
568 return match[1]
569 .split("\n")
570 .map((line) => line.trim())
571 .filter((line) => line.startsWith("- "))
572 .map((line) => line.slice(2).trim())
573 .filter((line) => line && line !== "(none)")
574}
575
576function conciseDiscordBatchMemoryQuery(raw: string): MemoryQueryParts | null {
577 const withoutWakeEnvelope = raw.replace(/^\[(wake|incoming|harness restarted)[^\n]*\]\s*/gi, "").trim()
578 if (!/\[discord batch\]/i.test(withoutWakeEnvelope)) return null
579
580 const pending = extractBulletSection(withoutWakeEnvelope, "pending preview")
581 const selected = pending.filter((entry) => !/^\(none\)$/i.test(entry)).slice(-3)
582 if (selected.length === 0) return null
583
584 const senders: string[] = []
585 const sources: string[] = []
586 const bodies: string[] = []
587 const entryPattern = /(?:^|\s)\[([^\]]+)\]\s+\[[^\]]+\]\s+@([^:]+):\s*(.*)$/i
588 for (const entry of selected) {
589 const match = entry.match(entryPattern)
590 if (!match) {
591 bodies.push(entry)
592 continue
593 }
594 const [, location, author, body] = match
595 if (author) senders.push(normalizeHandle(author))
596 if (location) sources.push(location.trim())
597 if (body) bodies.push(body.trim())
598 }
599
600 const lastSender = senders.length > 0 ? senders[senders.length - 1]! : null
601 const lastSource = sources.length > 0 ? sources[sources.length - 1]! : null
602 const body = bodies.filter(Boolean).join("\n").trim()
603
604 if (!lastSender && !lastSource && !body) return null
605 return { sender: lastSender, source: lastSource, body }
606}
607
608function memoryQueryForUserMessage(raw: string): MemoryQueryParts {
609 return (
610 conciseDiscordMemoryQuery(raw) ??
611 conciseDiscordBatchMemoryQuery(raw) ??
612 { sender: null, source: null, body: raw }
613 )
614}
615
616function memoryQueryToString(parts: MemoryQueryParts): string {
617 const pieces = [
618 parts.sender ? `@${parts.sender}` : null,
619 parts.source,
620 parts.body || null,
621 ].filter((value): value is string => Boolean(value && value.trim()))
622 return pieces.join("\n")
623}
624
625function normalizeBodyText(raw: string): string {
626 return raw
627 .replace(/@([a-z0-9_.-]+)/gi, " $1 ")
628 .replace(/\b\d{6,}\b/g, " ")
629 .replace(/[^\p{L}\p{N}\s'-]+/gu, " ")
630 .toLowerCase()
631 .trim()
632}
633
634function tokensFromText(raw: string): string[] {
635 const clean = normalizeBodyText(raw)
636 const tokens = clean
637 .split(/\s+/)
638 .map((token) => token.replace(/^['-]+|['-]+$/g, ""))
639 .filter((token) => token.length >= 2 || /\d{2,}/.test(token))
640 .filter((token) => !MEMORY_STOP_WORDS.has(token))
641
642 const unique: string[] = []
643 const seen = new Set<string>()
644 for (const token of tokens) {
645 if (seen.has(token)) continue
646 seen.add(token)
647 unique.push(token)
648 if (unique.length >= MEMORY_QUERY_TOKEN_LIMIT) break
649 }
650 return unique
651}
652
653function searchTokens(raw: string): string[] {
654 return tokensFromText(raw)
655}
656
657const BODY_INFORMATIVE_BM25_THRESHOLD = -5
658
659function bestBodyBm25(bodyTokens: string[]): number | null {
660 if (bodyTokens.length === 0) return null
661 const query = bodyTokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(" OR ")
662 try {
663 const row = getDb()
664 .prepare(
665 "select bm25(memory_chunks_fts, 5.0, 2.0, 1.0, 0.5) as r from memory_chunks_fts where memory_chunks_fts match ? order by r limit 1",
666 )
667 .get(query) as { r: number } | undefined
668 return row?.r ?? null
669 } catch {
670 return null
671 }
672}
673
674function computeBodyInformativeness(bodyTokens: string[], bodyPeople: string[]): boolean {
675 if (bodyPeople.length > 0) return true
676 if (bodyTokens.length === 0) return false
677 const top = bestBodyBm25(bodyTokens)
678 if (top === null) return false
679 return top <= BODY_INFORMATIVE_BM25_THRESHOLD
680}
681
682async function knownPeopleHandles(aliasMap: AliasMap): Promise<Set<string>> {
683 const handles = new Set<string>()
684 if (await pathExists(PEOPLE_DIR)) {
685 const entries = await fs.readdir(PEOPLE_DIR, { withFileTypes: true })
686 for (const entry of entries) {
687 if (!entry.isFile() || !entry.name.endsWith(".md")) continue
688 const base = basenameWithoutExt(entry.name).toLowerCase()
689 if (base) handles.add(base)
690 }
691 }
692 for (const [key, values] of Object.entries(aliasMap)) {
693 handles.add(key)
694 for (const value of values) handles.add(value)
695 }
696 return handles
697}
698
699async function buildSearchProfile(parts: MemoryQueryParts): Promise<MemorySearchProfile> {
700 const aliasMap = await loadAliasMap()
701 const sender = parts.sender ? normalizeHandle(parts.sender) : null
702 const senderAliases = resolveAliases(sender, aliasMap)
703 const bodyTokens = parts.body ? tokensFromText(parts.body) : []
704
705 const known = await knownPeopleHandles(aliasMap)
706 const inlineMentions = parts.body
707 ? Array.from(parts.body.matchAll(/@([a-z0-9_.-]+)/gi)).map((match) => normalizeHandle(match[1]!))
708 : []
709 const bodyPeopleSet = new Set<string>()
710 const senderSet = new Set<string>([sender ?? "", ...senderAliases].filter(Boolean))
711 for (const token of [...bodyTokens, ...inlineMentions]) {
712 if (!token || senderSet.has(token)) continue
713 if (known.has(token)) bodyPeopleSet.add(token)
714 }
715 const bodyPeople = Array.from(bodyPeopleSet)
716 for (const person of [...bodyPeople]) {
717 for (const alias of resolveAliases(person, aliasMap)) {
718 if (!senderSet.has(alias)) bodyPeopleSet.add(alias)
719 }
720 }
721 const bodyPeopleResolved = Array.from(bodyPeopleSet)
722
723 const combined: string[] = []
724 const seen = new Set<string>()
725 const push = (value: string | null | undefined) => {
726 if (!value) return
727 const lower = value.toLowerCase()
728 if (seen.has(lower)) return
729 seen.add(lower)
730 combined.push(lower)
731 }
732 push(sender)
733 for (const alias of senderAliases) push(alias)
734 for (const person of bodyPeopleResolved) push(person)
735 for (const token of bodyTokens) push(token)
736
737 const normalized = [sender ? `@${sender}` : "", parts.source ?? "", parts.body ?? ""]
738 .filter(Boolean)
739 .join(" ")
740 .toLowerCase()
741
742 const bodyInformative = computeBodyInformativeness(bodyTokens, bodyPeopleResolved)
743
744 return {
745 normalized,
746 sender,
747 senderAliases,
748 bodyTokens,
749 bodyPeople: bodyPeopleResolved,
750 tokens: combined.slice(0, MEMORY_QUERY_TOKEN_LIMIT),
751 bodyInformative,
752 personQuery:
753 Boolean(sender) ||
754 bodyPeopleResolved.length > 0 ||
755 /\b(who is|who's|tell me about|about)\b/.test(normalized) ||
756 /\bname\b/.test(normalized) ||
757 /\bfriend\b/.test(normalized),
758 eventQuery:
759 /\b(what happened|when|yesterday|today|tonight|earlier|before|after|restart|restarted|session|wake)\b/.test(
760 normalized,
761 ) ||
762 /\b\d{4}-\d{2}-\d{2}\b/.test(normalized) ||
763 /\b\d{1,2}\/\d{1,2}(?:\/\d{2,4})?\b/.test(normalized) ||
764 /\b(january|february|march|april|may|june|july|august|september|october|november|december)\b/.test(
765 normalized,
766 ),
767 }
768}
769
770function buildSearchQuery(profile: MemorySearchProfile): string | null {
771 if (profile.tokens.length === 0) return null
772 return profile.tokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(" OR ")
773}
774
775async function readMemoryDocumentRows(): Promise<Map<string, MemoryDocumentRow>> {
776 const rows = getDb()
777 .prepare("select id, path, kind, title, content_hash, mtime_ms from memory_documents")
778 .all() as MemoryDocumentRow[]
779
780 return new Map(rows.map((row) => [row.path, row]))
781}
782
783export async function syncMemoryIndex(): Promise<void> {
784 const db = getDb()
785 const files = await listMemoryFiles()
786 const known = await readMemoryDocumentRows()
787 const present = new Set(files)
788
789 const deleteChunksByDocument = db.prepare("delete from memory_chunks where document_id = ?")
790 const insertDocument = db.prepare(`
791 insert into memory_documents (path, kind, title, mtime_ms, content_hash, updated_at)
792 values (@path, @kind, @title, @mtime_ms, @content_hash, datetime('now'))
793 on conflict(path) do update set
794 kind = excluded.kind,
795 title = excluded.title,
796 mtime_ms = excluded.mtime_ms,
797 content_hash = excluded.content_hash,
798 updated_at = datetime('now')
799 `)
800 const selectDocumentId = db.prepare("select id from memory_documents where path = ?")
801 const insertChunk = db.prepare(`
802 insert into memory_chunks (document_id, chunk_index, title, heading_path, chunk_text, tags)
803 values (?, ?, ?, ?, ?, ?)
804 `)
805 const deleteDocumentByPath = db.prepare("delete from memory_documents where path = ?")
806
807 const updates: Array<{
808 path: string
809 kind: MemoryKind
810 title: string
811 mtimeMs: number
812 hash: string
813 chunks: MemoryChunkInput[]
814 action: "inserted" | "updated"
815 }> = []
816
817 for (const filePath of files) {
818 const kind = detectMemoryKind(filePath)
819 if (!kind) continue
820
821 const [content, stat] = await Promise.all([fs.readFile(filePath, "utf-8"), fs.stat(filePath)])
822 const hash = contentHash(content)
823 const previous = known.get(filePath)
824 if (previous && previous.content_hash === hash && previous.mtime_ms === Math.floor(stat.mtimeMs)) continue
825
826 const parsed = parseMarkdownDocument(filePath, content)
827 updates.push({
828 path: filePath,
829 kind,
830 title: parsed.title,
831 mtimeMs: Math.floor(stat.mtimeMs),
832 hash,
833 chunks: parsed.chunks,
834 action: previous ? "updated" : "inserted",
835 })
836 }
837
838 const removedPaths: string[] = []
839
840 db.transaction(() => {
841 for (const item of updates) {
842 insertDocument.run({
843 path: item.path,
844 kind: item.kind,
845 title: item.title,
846 mtime_ms: item.mtimeMs,
847 content_hash: item.hash,
848 })
849 const row = selectDocumentId.get(item.path) as { id: number } | undefined
850 if (!row) continue
851
852 deleteChunksByDocument.run(row.id)
853 item.chunks.forEach((chunk, index) => {
854 insertChunk.run(row.id, index, chunk.title, chunk.headingPath, chunk.text, chunk.tags)
855 })
856
857 console.log(
858 `[memory] ${item.action} kind=${item.kind} chunks=${item.chunks.length} path=${path.relative(HOME_DIR, item.path)}`,
859 )
860 }
861
862 for (const filePath of known.keys()) {
863 if (present.has(filePath)) continue
864 deleteDocumentByPath.run(filePath)
865 removedPaths.push(filePath)
866 console.log(`[memory] removed path=${path.relative(HOME_DIR, filePath)}`)
867 }
868 })()
869
870 if (updates.length === 0 && removedPaths.length === 0) {
871 await syncMemoryEmbeddings()
872 return
873 }
874
875 await syncMemoryEmbeddings()
876}
877
878let embeddingSkipWarned = false
879
880async function syncMemoryEmbeddings(): Promise<void> {
881 if (!isVecAvailable()) return
882 if (!embeddingsConfigured()) {
883 if (!embeddingSkipWarned) {
884 console.warn("[memory] embeddings disabled: set EMBEDDING_API_KEY")
885 embeddingSkipWarned = true
886 }
887 return
888 }
889 if (EMBEDDING_DIMENSIONS !== MEMORY_EMBEDDING_DIMENSIONS) {
890 if (!embeddingSkipWarned) {
891 console.warn(
892 `[memory] embeddings disabled: EMBEDDING_DIMENSIONS=${EMBEDDING_DIMENSIONS} but sqlite-vec table is ${MEMORY_EMBEDDING_DIMENSIONS}`,
893 )
894 embeddingSkipWarned = true
895 }
896 return
897 }
898
899 const db = getDb()
900 try {
901 await syncMemoryEmbeddingPrototypes()
902 } catch (err: any) {
903 console.warn(`[memory] prototype embedding sync failed: ${err?.message ?? String(err)}`)
904 return
905 }
906 db.prepare("delete from memory_embedding_meta where chunk_id not in (select id from memory_chunks)").run()
907 db.prepare("delete from memory_chunk_vec where rowid not in (select id from memory_chunks)").run()
908
909 const rows = db
910 .prepare(`
911 select
912 c.id as chunkId,
913 d.path as path,
914 d.kind as kind,
915 d.title as documentTitle,
916 c.title as title,
917 c.heading_path as headingPath,
918 c.chunk_text as text,
919 c.tags as tags,
920 m.model as model,
921 m.dimensions as dimensions,
922 m.content_hash as contentHash
923 from memory_chunks c
924 join memory_documents d on d.id = c.document_id
925 left join memory_embedding_meta m on m.chunk_id = c.id
926 order by d.kind, d.path, c.chunk_index
927 `)
928 .all() as MemoryEmbeddingRow[]
929
930 const pending = rows
931 .map((row) => {
932 const text = embeddingTextForChunk(row)
933 return { ...row, embeddingText: text, embeddingHash: embeddingInputHash(text) }
934 })
935 .filter(
936 (row) =>
937 row.model !== EMBEDDING_MODEL ||
938 row.dimensions !== MEMORY_EMBEDDING_DIMENSIONS ||
939 row.contentHash !== row.embeddingHash,
940 )
941
942 if (pending.length === 0) return
943
944 const upsertMeta = db.prepare(`
945 insert into memory_embedding_meta (chunk_id, model, dimensions, content_hash, updated_at)
946 values (?, ?, ?, ?, datetime('now'))
947 on conflict(chunk_id) do update set
948 model = excluded.model,
949 dimensions = excluded.dimensions,
950 content_hash = excluded.content_hash,
951 updated_at = datetime('now')
952 `)
953 const upsertVector = db.prepare("insert or replace into memory_chunk_vec(rowid, embedding) values (?, ?)")
954
955 let embedded = 0
956 for (let i = 0; i < pending.length; i += MEMORY_EMBEDDING_BATCH_SIZE) {
957 const batch = pending.slice(i, i + MEMORY_EMBEDDING_BATCH_SIZE)
958 let vectors: number[][]
959 try {
960 vectors = await embedTexts(batch.map((row) => row.embeddingText))
961 } catch (err: any) {
962 console.warn(`[memory] embedding batch failed: ${err?.message ?? String(err)}`)
963 return
964 }
965
966 db.transaction(() => {
967 batch.forEach((row, index) => {
968 const vector = vectors[index]
969 if (!vector) return
970 if (vector.length !== MEMORY_EMBEDDING_DIMENSIONS) {
971 throw new Error(`embedding dimension mismatch: got ${vector.length}, expected ${MEMORY_EMBEDDING_DIMENSIONS}`)
972 }
973 upsertVector.run(BigInt(row.chunkId), vectorParam(vector))
974 upsertMeta.run(row.chunkId, EMBEDDING_MODEL, MEMORY_EMBEDDING_DIMENSIONS, row.embeddingHash)
975 embedded += 1
976 })
977 })()
978 }
979
980 console.log(`[memory] embedded chunks=${embedded} model=${EMBEDDING_MODEL} dimensions=${MEMORY_EMBEDDING_DIMENSIONS}`)
981}
982
983async function syncMemoryEmbeddingPrototypes(): Promise<void> {
984 const db = getDb()
985 const rows = db
986 .prepare("select id, name, category, model, dimensions, content_hash as contentHash from memory_embedding_prototypes")
987 .all() as Array<{
988 id: number
989 name: string
990 category: string
991 model: string
992 dimensions: number
993 contentHash: string
994 }>
995 const known = new Map(rows.map((row) => [row.id, row]))
996 const pending = MEMORY_EMBEDDING_PROTOTYPES.map((prototype) => ({
997 ...prototype,
998 hash: embeddingInputHash(`${prototype.category}\n${prototype.name}\n${prototype.text}`),
999 })).filter((prototype) => {
1000 const row = known.get(prototype.id)
1001 return (
1002 !row ||
1003 row.name !== prototype.name ||
1004 row.category !== prototype.category ||
1005 row.model !== EMBEDDING_MODEL ||
1006 row.dimensions !== MEMORY_EMBEDDING_DIMENSIONS ||
1007 row.contentHash !== prototype.hash
1008 )
1009 })
1010
1011 if (pending.length === 0) return
1012
1013 const vectors = await embedTexts(pending.map((prototype) => prototype.text))
1014 const upsertPrototype = db.prepare(`
1015 insert into memory_embedding_prototypes (id, name, category, model, dimensions, content_hash, updated_at)
1016 values (?, ?, ?, ?, ?, ?, datetime('now'))
1017 on conflict(id) do update set
1018 name = excluded.name,
1019 category = excluded.category,
1020 model = excluded.model,
1021 dimensions = excluded.dimensions,
1022 content_hash = excluded.content_hash,
1023 updated_at = datetime('now')
1024 `)
1025 const upsertVector = db.prepare("insert or replace into memory_prototype_vec(rowid, embedding) values (?, ?)")
1026
1027 db.transaction(() => {
1028 pending.forEach((prototype, index) => {
1029 const vector = vectors[index]
1030 if (!vector) return
1031 if (vector.length !== MEMORY_EMBEDDING_DIMENSIONS) {
1032 throw new Error(`prototype embedding dimension mismatch: got ${vector.length}, expected ${MEMORY_EMBEDDING_DIMENSIONS}`)
1033 }
1034 upsertVector.run(BigInt(prototype.id), vectorParam(vector))
1035 upsertPrototype.run(
1036 prototype.id,
1037 prototype.name,
1038 prototype.category,
1039 EMBEDDING_MODEL,
1040 MEMORY_EMBEDDING_DIMENSIONS,
1041 prototype.hash,
1042 )
1043 })
1044 })()
1045}
1046
1047function senderHandles(profile: MemorySearchProfile): string[] {
1048 const out: string[] = []
1049 if (profile.sender) out.push(profile.sender)
1050 for (const alias of profile.senderAliases) out.push(alias)
1051 return out
1052}
1053
1054function allPersonHandles(profile: MemorySearchProfile): string[] {
1055 const seen = new Set<string>()
1056 const out: string[] = []
1057 for (const handle of [...senderHandles(profile), ...profile.bodyPeople]) {
1058 if (!handle || seen.has(handle)) continue
1059 seen.add(handle)
1060 out.push(handle)
1061 }
1062 return out
1063}
1064
1065function targetPersonHandles(profile: MemorySearchProfile): string[] {
1066 return profile.bodyPeople.length > 0 ? profile.bodyPeople : allPersonHandles(profile)
1067}
1068
1069function hitMatchesHandle(hit: MemoryHit, handle: string): boolean {
1070 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
1071 const pathHaystack = hit.path.toLowerCase()
1072 return titleHaystack.includes(handle) || pathHaystack.includes(`/${handle}.md`)
1073}
1074
1075function hitMentionsHandle(hit: MemoryHit, handle: string): boolean {
1076 return hitMatchesHandle(hit, handle) || hitSearchHaystack(hit).includes(handle)
1077}
1078
1079function scoreMemoryHit(hit: MemoryHit, profile: MemorySearchProfile): number {
1080 let score = hit.rank
1081
1082 if (profile.personQuery && !profile.eventQuery) {
1083 if (hit.kind === "people") score -= 2
1084 if (hit.kind === "core") score -= 0.5
1085 if (hit.kind === "journal") score += 1.5
1086 } else if (profile.eventQuery && !profile.personQuery) {
1087 if (hit.kind === "journal") score -= 1.5
1088 if (hit.kind === "people") score += 0.75
1089 if (hit.kind === "core") score += 0.25
1090 } else {
1091 if (hit.kind === "people") score -= 0.5
1092 if (hit.kind === "core") score -= 0.25
1093 }
1094
1095 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
1096 if (profile.bodyTokens.some((token) => titleHaystack.includes(token))) score -= 0.35
1097
1098 const handles = allPersonHandles(profile)
1099 if (handles.length > 0) {
1100 const fullHaystack = `${titleHaystack} ${hit.text.toLowerCase()}`
1101 const pathHaystack = hit.path.toLowerCase()
1102 if (handles.some((h) => titleHaystack.includes(h) || pathHaystack.includes(`/${h}.md`))) {
1103 score -= 3
1104 } else if (handles.some((h) => fullHaystack.includes(h))) {
1105 score -= 1
1106 }
1107 }
1108
1109 if (profile.bodyPeople.length > 0) {
1110 const targetHandles = targetPersonHandles(profile)
1111 const matchesTarget = targetHandles.some((handle) => hitMatchesHandle(hit, handle))
1112 if (matchesTarget) {
1113 score -= 4
1114 } else if (hit.kind === "people") {
1115 score += 3
1116 } else if (hit.kind === "core" && !profileHasCoreIntent(profile)) {
1117 score += 3.5
1118 }
1119 }
1120
1121 return score
1122}
1123
1124function hitSearchHaystack(hit: MemoryHit): string {
1125 return `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)} ${hit.text}`.toLowerCase()
1126}
1127
1128function memoryHitSignal(hit: MemoryHit, profile: MemorySearchProfile): MemoryHitSignal {
1129 const haystack = hitSearchHaystack(hit)
1130 let overlap = 0
1131 let strongOverlap = false
1132 let bodyOverlap = 0
1133
1134 for (const token of profile.tokens) {
1135 if (!haystack.includes(token)) continue
1136 overlap += 1
1137 if (token.length >= 5 || /[0-9]/.test(token)) strongOverlap = true
1138 }
1139 for (const token of profile.bodyTokens) {
1140 if (haystack.includes(token)) bodyOverlap += 1
1141 }
1142
1143 const handles = allPersonHandles(profile)
1144 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
1145 const pathHaystack = hit.path.toLowerCase()
1146 const senderMatch =
1147 handles.length > 0 &&
1148 handles.some((h) => titleHaystack.includes(h) || pathHaystack.includes(`/${h}.md`) || haystack.includes(h))
1149
1150 return { overlap, strongOverlap, bodyOverlap, senderMatch }
1151}
1152
1153function shouldInjectHits(hits: MemoryHit[], profile: MemorySearchProfile): boolean {
1154 if (hits.length === 0) return false
1155
1156 const topSignal = memoryHitSignal(hits[0]!, profile)
1157
1158 if (profile.sender || profile.bodyPeople.length > 0) {
1159 if (!profile.bodyInformative) return false
1160 if (topSignal.senderMatch) return true
1161 if (topSignal.bodyOverlap >= 2) return true
1162 if (topSignal.bodyOverlap >= 1 && topSignal.strongOverlap) return true
1163 return false
1164 }
1165
1166 if (profile.eventQuery && !profile.personQuery) {
1167 return topSignal.overlap >= 1 && hits.some((hit) => hit.kind === "journal")
1168 }
1169
1170 if (profile.personQuery && !profile.eventQuery) {
1171 return topSignal.overlap >= 1 && hits.some((hit) => hit.kind === "people" || hit.kind === "core")
1172 }
1173
1174 if (topSignal.bodyOverlap >= 2) return true
1175 if (topSignal.bodyOverlap >= 1 && topSignal.strongOverlap) return true
1176 if (topSignal.overlap >= 2 && topSignal.strongOverlap) return true
1177 return false
1178}
1179
1180function profileHasExplicitRecallIntent(profile: MemorySearchProfile): boolean {
1181 if (profile.bodyPeople.length > 0) return true
1182 if (profile.eventQuery) return true
1183 if (/\b(who is|who's|tell me about|remember|recall|what happened|what do i know|context)\b/.test(profile.normalized)) {
1184 return true
1185 }
1186 return false
1187}
1188
1189function profileHasCoreIntent(profile: MemorySearchProfile): boolean {
1190 return /\b(core|identity|system|environment|lesson|rule|instruction|workflow|tool|config|memory)\b/.test(profile.normalized)
1191}
1192
1193async function semanticQuerySignal(memoryQuery: string): Promise<SemanticQuerySignal | null> {
1194 if (!isVecAvailable() || !embeddingsConfigured() || EMBEDDING_DIMENSIONS !== MEMORY_EMBEDDING_DIMENSIONS) return null
1195 const [vector] = await embedTexts([memoryQuery])
1196 if (!vector || vector.length !== MEMORY_EMBEDDING_DIMENSIONS) return null
1197
1198 const rows = getDb()
1199 .prepare(`
1200 select p.category as category, v.distance as distance
1201 from memory_prototype_vec v
1202 join memory_embedding_prototypes p on p.id = v.rowid
1203 where v.embedding match ?
1204 and k = ?
1205 order by v.distance
1206 `)
1207 .all(vectorParam(vector), 8) as Array<{ category: string; distance: number }>
1208
1209 let chatterSimilarity: number | null = null
1210 let recallIntentSimilarity: number | null = null
1211 for (const row of rows) {
1212 const similarity = 1 - row.distance
1213 if (row.category === "chatter") chatterSimilarity = Math.max(chatterSimilarity ?? -Infinity, similarity)
1214 if (row.category === "recall_intent") {
1215 recallIntentSimilarity = Math.max(recallIntentSimilarity ?? -Infinity, similarity)
1216 }
1217 }
1218
1219 return {
1220 vector,
1221 chatterSimilarity,
1222 recallIntentSimilarity,
1223 }
1224}
1225
1226function shouldSkipForSemanticChatter(profile: MemorySearchProfile, signal: SemanticQuerySignal | null): boolean {
1227 if (!signal) return false
1228 if (profileHasExplicitRecallIntent(profile)) return false
1229 if (profile.bodyTokens.length > 5) return false
1230 const chatter = signal.chatterSimilarity ?? 0
1231 const recallIntent = signal.recallIntentSimilarity ?? 0
1232 return chatter >= MEMORY_CHATTER_SIMILARITY_THRESHOLD && recallIntent < MEMORY_RECALL_INTENT_SIMILARITY_THRESHOLD
1233}
1234
1235function semanticDistancesForQuery(vector: number[], limit: number): Map<number, number> {
1236 if (!isVecAvailable()) return new Map()
1237 const rows = getDb()
1238 .prepare(`
1239 select rowid as chunkId, distance
1240 from memory_chunk_vec
1241 where embedding match ?
1242 and k = ?
1243 order by distance
1244 `)
1245 .all(vectorParam(vector), limit) as Array<{ chunkId: number; distance: number }>
1246 return new Map(rows.map((row) => [row.chunkId, row.distance]))
1247}
1248
1249function exactPersonHits(profile: MemorySearchProfile): MemoryHit[] {
1250 if (profile.bodyPeople.length === 0) return []
1251 const db = getDb()
1252 const hits: MemoryHit[] = []
1253 const seen = new Set<number>()
1254 const stmt = db.prepare(`
1255 select
1256 c.id as chunkId,
1257 d.path as path,
1258 d.kind as kind,
1259 d.title as documentTitle,
1260 c.title as title,
1261 c.heading_path as headingPath,
1262 c.chunk_text as text,
1263 -10.0 as rank
1264 from memory_chunks c
1265 join memory_documents d on d.id = c.document_id
1266 where d.kind = 'people'
1267 and lower(d.path) like ?
1268 order by c.chunk_index
1269 `)
1270
1271 for (const handle of profile.bodyPeople) {
1272 const rows = stmt.all(`%/people/${handle.toLowerCase()}.md`) as MemoryHit[]
1273 for (const row of rows) {
1274 if (seen.has(row.chunkId)) continue
1275 seen.add(row.chunkId)
1276 hits.push(row)
1277 }
1278 }
1279 return hits
1280}
1281
1282function scoreMemoryHitWithSemantic(hit: MemoryHit, profile: MemorySearchProfile): number {
1283 let score = scoreMemoryHit(hit, profile)
1284 const coreIntent = profileHasCoreIntent(profile)
1285
1286 if (hit.semanticSimilarity !== undefined) {
1287 score -= hit.semanticSimilarity * 2.5
1288 if (hit.kind === "core" && !coreIntent && hit.semanticSimilarity < MEMORY_SEMANTIC_STRONG_SIMILARITY) {
1289 score += 1.25
1290 }
1291 } else {
1292 score += 0.75
1293 if (hit.kind === "core" && !coreIntent) score += 1.25
1294 }
1295
1296 if (hit.kind === "core" && !coreIntent) score += 0.75
1297 return score
1298}
1299
1300async function searchMemory(
1301 profile: MemorySearchProfile,
1302 cooldowns: Record<number, number>,
1303 currentTurn: number,
1304 limit: number,
1305 semanticSignal: SemanticQuerySignal | null = null,
1306): Promise<MemoryHit[]> {
1307 const db = getDb()
1308 const query = buildSearchQuery(profile)
1309 if (!query) return []
1310 const rows = db
1311 .prepare(`
1312 select
1313 c.id as chunkId,
1314 d.path as path,
1315 d.kind as kind,
1316 d.title as documentTitle,
1317 c.title as title,
1318 c.heading_path as headingPath,
1319 c.chunk_text as text,
1320 bm25(memory_chunks_fts, 5.0, 2.0, 1.0, 0.5) as rank
1321 from memory_chunks_fts
1322 join memory_chunks c on c.id = memory_chunks_fts.rowid
1323 join memory_documents d on d.id = c.document_id
1324 where memory_chunks_fts match ?
1325 order by rank
1326 limit ?
1327 `)
1328 .all(query, Math.max(limit * 8, limit)) as MemoryHit[]
1329
1330 for (const hit of exactPersonHits(profile)) {
1331 if (rows.some((row) => row.chunkId === hit.chunkId)) continue
1332 rows.push(hit)
1333 }
1334
1335 if (semanticSignal) {
1336 const distances = semanticDistancesForQuery(semanticSignal.vector, Math.max(limit * 16, 64))
1337 for (const row of rows) {
1338 const distance = distances.get(row.chunkId)
1339 if (distance === undefined) continue
1340 row.semanticDistance = distance
1341 row.semanticSimilarity = 1 - distance
1342 }
1343 }
1344
1345 const ordered = rows.sort((a, b) => scoreMemoryHitWithSemantic(a, profile) - scoreMemoryHitWithSemantic(b, profile))
1346 const pool =
1347 profile.personQuery && !profile.eventQuery
1348 ? (() => {
1349 const structured = ordered.filter((row) => row.kind === "people" || row.kind === "core")
1350 return structured.length > 0 ? structured : ordered
1351 })()
1352 : profile.eventQuery && !profile.personQuery
1353 ? (() => {
1354 const journal = ordered.filter((row) => row.kind === "journal")
1355 return journal.length > 0 ? journal : ordered
1356 })()
1357 : ordered
1358
1359 const deduped: MemoryHit[] = []
1360 const seenChunkIds = new Set<number>()
1361 const seenPaths = new Set<string>()
1362
1363 const handles = targetPersonHandles(profile)
1364 const exactTargetAvailable =
1365 profile.bodyPeople.length > 0 && pool.some((row) => handles.some((handle) => hitMatchesHandle(row, handle)))
1366 if (handles.length >= 2) {
1367 for (const handle of handles) {
1368 if (deduped.length >= limit) break
1369 const candidate = pool.find(
1370 (row) =>
1371 !seenChunkIds.has(row.chunkId) &&
1372 !seenPaths.has(row.path) &&
1373 !isCoolingDown(row, cooldowns, currentTurn) &&
1374 hitMatchesHandle(row, handle),
1375 )
1376 if (!candidate) continue
1377 seenChunkIds.add(candidate.chunkId)
1378 seenPaths.add(candidate.path)
1379 deduped.push(candidate)
1380 }
1381 }
1382
1383 for (const row of pool) {
1384 if (deduped.length >= limit) break
1385 if (seenChunkIds.has(row.chunkId)) continue
1386 if (isCoolingDown(row, cooldowns, currentTurn)) continue
1387 const exactTargetMatch = profile.bodyPeople.length > 0 && handles.some((handle) => hitMatchesHandle(row, handle))
1388 if (exactTargetAvailable && !exactTargetMatch && row.kind !== "journal") {
1389 continue
1390 }
1391 if (profile.bodyPeople.length > 0 && !profile.bodyPeople.some((handle) => hitMentionsHandle(row, handle))) {
1392 continue
1393 }
1394 if (
1395 semanticSignal &&
1396 row.kind === "core" &&
1397 !profileHasCoreIntent(profile) &&
1398 (row.semanticSimilarity ?? 0) < MEMORY_SEMANTIC_MIN_SIMILARITY
1399 ) {
1400 continue
1401 }
1402 seenChunkIds.add(row.chunkId)
1403 deduped.push(row)
1404 }
1405
1406 return deduped
1407}
1408
1409function formatMemorySource(hit: MemoryHit): string {
1410 const relativePath = path.relative(HOME_DIR, hit.path)
1411 const pieces = [`[${hit.kind}]`, hit.documentTitle]
1412 if (hit.headingPath && hit.headingPath !== hit.documentTitle) pieces.push(hit.headingPath)
1413 pieces.push(relativePath)
1414 return pieces.join(" / ")
1415}
1416
1417function toMemorySearchResult(hit: MemoryHit): MemorySearchResult {
1418 return {
1419 chunkId: hit.chunkId,
1420 kind: hit.kind,
1421 path: hit.path,
1422 source: formatMemorySource(hit),
1423 title: hit.title,
1424 headingPath: hit.headingPath,
1425 preview: trimForPrompt(normalizeText(hit.text), 280),
1426 }
1427}
1428
1429function buildMemoryRecallMessage(hits: MemoryHit[], maxChars: number): string {
1430 const lines = [MEMORY_RECALL_HEADER, MEMORY_RECALL_NOTE, ""]
1431 let usedChars = lines.join("\n").length
1432
1433 for (const hit of hits) {
1434 const source = formatMemorySource(hit)
1435 const remaining = Math.max(120, maxChars - usedChars - source.length - 10)
1436 const body = trimForPrompt(normalizeText(hit.text), Math.min(280, remaining))
1437 const block = `- ${source}\n ${body}`
1438
1439 if (usedChars + block.length > maxChars && lines.length > 3) break
1440 lines.push(block)
1441 usedChars += block.length + 1
1442 }
1443
1444 return lines.join("\n").trim()
1445}
1446
1447export async function buildCompletionMessages(
1448 conversation: Message[],
1449 cooldowns: Record<number, number>,
1450 currentTurn: number,
1451): Promise<{ messages: Message[]; recalledChunkIds: number[] }> {
1452 const memoryQuerySource = latestMemoryRecallQuery(conversation)
1453 if (!memoryQuerySource) return { messages: conversation, recalledChunkIds: [] }
1454 const queryParts = memoryQueryForUserMessage(memoryQuerySource)
1455 const memoryQuery = memoryQueryToString(queryParts)
1456
1457 await syncMemoryIndex()
1458
1459 const profile = await buildSearchProfile(queryParts)
1460 if (profile.tokens.length === 0) return { messages: conversation, recalledChunkIds: [] }
1461
1462 if ((profile.sender || profile.bodyPeople.length > 0) && !profile.bodyInformative) {
1463 console.log(
1464 `[memory] skipped query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} sender=${profile.sender ?? "-"} reason=trivial-body`,
1465 )
1466 return { messages: conversation, recalledChunkIds: [] }
1467 }
1468
1469 let semanticSignal: SemanticQuerySignal | null = null
1470 try {
1471 semanticSignal = await semanticQuerySignal(memoryQuery)
1472 } catch (err: any) {
1473 console.warn(`[memory] semantic query failed: ${err?.message ?? String(err)}`)
1474 }
1475
1476 if (shouldSkipForSemanticChatter(profile, semanticSignal)) {
1477 console.log(
1478 `[memory] skipped query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} sender=${profile.sender ?? "-"} reason=semantic-chatter chatter=${semanticSignal?.chatterSimilarity?.toFixed(3) ?? "-"} recallIntent=${semanticSignal?.recallIntentSimilarity?.toFixed(3) ?? "-"}`,
1479 )
1480 return { messages: conversation, recalledChunkIds: [] }
1481 }
1482
1483 const personCount = allPersonHandles(profile).length
1484 const recallLimit = Math.min(
1485 MEMORY_RECALL_MAX_CHUNKS_HARD_CAP,
1486 personCount >= 2 ? MEMORY_RECALL_MAX_CHUNKS + (personCount - 1) : MEMORY_RECALL_MAX_CHUNKS,
1487 )
1488 const hits = await searchMemory(profile, cooldowns, currentTurn, recallLimit, semanticSignal)
1489 const aliasInfo = profile.senderAliases.length > 0 ? ` aliases=${profile.senderAliases.join(",")}` : ""
1490 const peopleInfo = profile.bodyPeople.length > 0 ? ` people=${profile.bodyPeople.join(",")}` : ""
1491 const debugTag = `sender=${profile.sender ?? "-"}${aliasInfo}${peopleInfo} personQuery=${profile.personQuery} eventQuery=${profile.eventQuery}`
1492 if (hits.length === 0) {
1493 console.log(
1494 `[memory] no-hits query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag}`,
1495 )
1496 return { messages: conversation, recalledChunkIds: [] }
1497 }
1498 if (!shouldInjectHits(hits, profile)) {
1499 console.log(
1500 `[memory] skipped query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag} reason=weak-match`,
1501 )
1502 return { messages: conversation, recalledChunkIds: [] }
1503 }
1504
1505 const extraPersons = Math.max(0, personCount - 1)
1506 const recallChars = MEMORY_RECALL_MAX_CHARS + extraPersons * MEMORY_RECALL_PER_EXTRA_PERSON_CHARS
1507 const recallContent = buildMemoryRecallMessage(hits, recallChars)
1508 console.log(
1509 `[memory] recalled query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag}\n${recallContent}`,
1510 )
1511
1512 recordMetric({
1513 type: "memory",
1514 query: memoryQuery,
1515 results: hits.map(toMemorySearchResult),
1516 })
1517
1518 const recallMessage: Message = {
1519 role: "user",
1520 content: recallContent,
1521 }
1522
1523 return {
1524 messages: [...conversation, recallMessage],
1525 recalledChunkIds: hits.map((hit) => hit.chunkId),
1526 }
1527}
1528
1529export function rememberRecalledMemoryChunks(
1530 cooldowns: Record<number, number>,
1531 injectedChunkIds: number[],
1532 currentTurn: number,
1533): Record<number, number> {
1534 const next = { ...cooldowns }
1535 for (const chunkId of injectedChunkIds) next[chunkId] = currentTurn
1536
1537 for (const [chunkId, turn] of Object.entries(next)) {
1538 if (currentTurn - turn >= MEMORY_RECALL_COOLDOWN_TURNS * 2) delete next[Number(chunkId)]
1539 }
1540
1541 return next
1542}
1543
1544export const __memoryTest = {
1545 latestMemoryRecallQuery,
1546 memoryQueryForUserMessage,
1547 memoryQueryToString,
1548 normalizeBodyText,
1549 searchTokens,
1550 buildSearchProfile,
1551 resolveAliases,
1552 normalizeHandle,
1553}
1554
1555export async function searchMemories(rawQuery: string, limit = 5): Promise<MemorySearchResult[]> {
1556 await syncMemoryIndex()
1557
1558 const profile = await buildSearchProfile({ sender: null, source: null, body: rawQuery })
1559 if (profile.tokens.length === 0) return []
1560
1561 let semanticSignal: SemanticQuerySignal | null = null
1562 try {
1563 semanticSignal = await semanticQuerySignal(rawQuery)
1564 } catch {
1565 semanticSignal = null
1566 }
1567
1568 const results = (await searchMemory(
1569 profile,
1570 {},
1571 Number.POSITIVE_INFINITY,
1572 Math.max(1, Math.min(limit, 10)),
1573 semanticSignal,
1574 )).map(toMemorySearchResult)
1575 recordMetric({
1576 type: "memory",
1577 query: rawQuery,
1578 results,
1579 })
1580 return results
1581}