hotsocket.fyi
This repository has no description
1import fs from "fs/promises"
2import path from "path"
3import { createHash } from "crypto"
4import { fileURLToPath } from "url"
5import type { Message } from "./types.js"
6import { getDb } from "./db.js"
7import { recordMetric } from "./metrics.js"
8
9const HOME_DIR = path.resolve(fileURLToPath(import.meta.url), "../../home")
10const MEMORIES_DIR = path.join(HOME_DIR, "memories")
11const JOURNAL_DIR = path.join(MEMORIES_DIR, "journal")
12const PEOPLE_DIR = path.join(MEMORIES_DIR, "people")
13const CORE_FILE = path.join(MEMORIES_DIR, "core.md")
14const ALIASES_FILE = path.join(MEMORIES_DIR, "aliases.json")
15
16const MEMORY_RECALL_HEADER = "[memory recall v1]"
17const MEMORY_RECALL_NOTE =
18 "Potentially relevant long-term notes. Use only if helpful; trust newer conversation details if anything conflicts."
19const MEMORY_RECALL_MAX_CHUNKS = 4
20const MEMORY_RECALL_MAX_CHUNKS_HARD_CAP = 8
21const MEMORY_RECALL_MAX_CHARS = 1_500
22const MEMORY_RECALL_PER_EXTRA_PERSON_CHARS = 400
23const MEMORY_QUERY_TOKEN_LIMIT = 12
24const MEMORY_RECALL_COOLDOWN_TURNS = 7
25const SCHEDULED_HEARTBEAT_CONTENT = "Scheduled heartbeat."
26const MEMORY_STOP_WORDS = new Set([
27 "a",
28 "an",
29 "and",
30 "are",
31 "as",
32 "at",
33 "be",
34 "been",
35 "but",
36 "by",
37 "for",
38 "from",
39 "had",
40 "has",
41 "have",
42 "he",
43 "her",
44 "hers",
45 "him",
46 "his",
47 "i",
48 "if",
49 "in",
50 "into",
51 "is",
52 "it",
53 "its",
54 "me",
55 "my",
56 "of",
57 "on",
58 "or",
59 "our",
60 "she",
61 "that",
62 "the",
63 "their",
64 "them",
65 "there",
66 "they",
67 "this",
68 "to",
69 "up",
70 "us",
71 "was",
72 "we",
73 "were",
74 "with",
75 "you",
76 "your",
77])
78
79type MemoryKind = "core" | "journal" | "people"
80
81type MemoryDocumentRow = {
82 id: number
83 path: string
84 content_hash: string
85 mtime_ms: number
86 kind?: string
87 title?: string
88}
89
90type MemoryChunkInput = {
91 title: string
92 headingPath: string | null
93 text: string
94 tags: string
95}
96
97type MemoryHit = {
98 chunkId: number
99 path: string
100 kind: MemoryKind
101 documentTitle: string
102 title: string
103 headingPath: string | null
104 text: string
105 rank: number
106}
107
108export type MemorySearchResult = {
109 chunkId: number
110 kind: MemoryKind
111 path: string
112 source: string
113 title: string
114 headingPath: string | null
115 preview: string
116}
117
118type MemoryQueryParts = {
119 sender: string | null
120 source: string | null
121 body: string
122}
123
124type MemorySearchProfile = {
125 normalized: string
126 sender: string | null
127 senderAliases: string[]
128 bodyTokens: string[]
129 bodyPeople: string[]
130 tokens: string[]
131 personQuery: boolean
132 eventQuery: boolean
133}
134
135type MemoryHitSignal = {
136 overlap: number
137 strongOverlap: boolean
138 bodyOverlap: number
139 senderMatch: boolean
140}
141
142export type AliasMap = Record<string, string[]>
143
144function normalizeText(value: string): string {
145 return value.replace(/\r\n/g, "\n").replace(/\s+/g, " ").trim()
146}
147
148function trimForPrompt(value: string, maxChars: number): string {
149 if (value.length <= maxChars) return value
150 if (maxChars <= 3) return ".".repeat(maxChars)
151 return `${value.slice(0, maxChars - 3).trimEnd()}...`
152}
153
154function basenameWithoutExt(filePath: string): string {
155 return path.basename(filePath, path.extname(filePath))
156}
157
158function titleFromPath(filePath: string, fallback: string): string {
159 const base = basenameWithoutExt(filePath).replace(/[-_]+/g, " ").trim()
160 return base ? base : fallback
161}
162
163function detectMemoryKind(filePath: string): MemoryKind | null {
164 if (filePath === CORE_FILE) return "core"
165 if (filePath.startsWith(`${JOURNAL_DIR}${path.sep}`)) return "journal"
166 if (filePath.startsWith(`${PEOPLE_DIR}${path.sep}`)) return "people"
167 return null
168}
169
170async function pathExists(target: string): Promise<boolean> {
171 try {
172 await fs.access(target)
173 return true
174 } catch {
175 return false
176 }
177}
178
179function normalizeHandle(handle: string): string {
180 return handle.trim().replace(/^@+/, "").toLowerCase()
181}
182
183let aliasCache: { mtimeMs: number; map: AliasMap } | null = null
184
185async function loadAliasMap(): Promise<AliasMap> {
186 try {
187 const stat = await fs.stat(ALIASES_FILE)
188 if (aliasCache && aliasCache.mtimeMs === stat.mtimeMs) return aliasCache.map
189 const raw = await fs.readFile(ALIASES_FILE, "utf-8")
190 const parsed = JSON.parse(raw) as unknown
191 const map: AliasMap = {}
192 if (parsed && typeof parsed === "object") {
193 for (const [key, value] of Object.entries(parsed as Record<string, unknown>)) {
194 const handle = normalizeHandle(key)
195 if (!handle) continue
196 const list = Array.isArray(value) ? value : [value]
197 const aliases = list
198 .map((v) => (typeof v === "string" ? normalizeHandle(v) : ""))
199 .filter((v) => v && v !== handle)
200 if (aliases.length > 0) map[handle] = Array.from(new Set(aliases))
201 }
202 }
203 aliasCache = { mtimeMs: stat.mtimeMs, map }
204 return map
205 } catch {
206 aliasCache = { mtimeMs: 0, map: {} }
207 return {}
208 }
209}
210
211async function writeAliasMap(map: AliasMap): Promise<void> {
212 await fs.mkdir(MEMORIES_DIR, { recursive: true })
213 const sorted: AliasMap = {}
214 for (const key of Object.keys(map).sort()) sorted[key] = [...map[key]!].sort()
215 await fs.writeFile(ALIASES_FILE, `${JSON.stringify(sorted, null, 2)}\n`, "utf-8")
216 aliasCache = null
217}
218
219function resolveAliases(handle: string | null, map: AliasMap): string[] {
220 if (!handle) return []
221 const seen = new Set<string>([handle])
222 const out: string[] = []
223 const queue = [handle]
224 while (queue.length > 0) {
225 const current = queue.shift()!
226 const next = map[current] ?? []
227 for (const alias of next) {
228 if (seen.has(alias)) continue
229 seen.add(alias)
230 out.push(alias)
231 queue.push(alias)
232 }
233 }
234 return out
235}
236
237export async function listAliases(): Promise<AliasMap> {
238 return loadAliasMap()
239}
240
241export async function setAlias(handle: string, canonical: string): Promise<AliasMap> {
242 const h = normalizeHandle(handle)
243 const c = normalizeHandle(canonical)
244 if (!h || !c) throw new Error("alias handle and canonical must be non-empty")
245 const map = await loadAliasMap()
246 if (h === c) return map
247 const existing = new Set(map[h] ?? [])
248 existing.add(c)
249 map[h] = Array.from(existing)
250 await writeAliasMap(map)
251 return map
252}
253
254export async function removeAlias(handle: string, canonical?: string): Promise<AliasMap> {
255 const h = normalizeHandle(handle)
256 if (!h) throw new Error("alias handle must be non-empty")
257 const map = await loadAliasMap()
258 if (!map[h]) return map
259 if (canonical) {
260 const c = normalizeHandle(canonical)
261 map[h] = map[h]!.filter((entry) => entry !== c)
262 if (map[h]!.length === 0) delete map[h]
263 } else {
264 delete map[h]
265 }
266 await writeAliasMap(map)
267 return map
268}
269
270async function walkMarkdownFiles(root: string): Promise<string[]> {
271 if (!(await pathExists(root))) return []
272
273 const found: string[] = []
274 const entries = await fs.readdir(root, { withFileTypes: true })
275 for (const entry of entries) {
276 const fullPath = path.join(root, entry.name)
277 if (entry.isDirectory()) {
278 found.push(...(await walkMarkdownFiles(fullPath)))
279 continue
280 }
281 if (entry.isFile() && entry.name.endsWith(".md")) found.push(fullPath)
282 }
283
284 return found.sort()
285}
286
287async function listMemoryFiles(): Promise<string[]> {
288 const files: string[] = []
289 if (await pathExists(CORE_FILE)) files.push(CORE_FILE)
290 files.push(...(await walkMarkdownFiles(JOURNAL_DIR)))
291 files.push(...(await walkMarkdownFiles(PEOPLE_DIR)))
292 return files
293}
294
295function contentHash(content: string): string {
296 return createHash("sha1").update(content).digest("hex")
297}
298
299function chunkLargeSection(text: string, maxChars = 900): string[] {
300 const paragraphs = text
301 .split(/\n\s*\n/g)
302 .map((part) => part.trim())
303 .filter(Boolean)
304
305 if (paragraphs.length === 0) return []
306
307 const chunks: string[] = []
308 let current = ""
309
310 for (const paragraph of paragraphs) {
311 const next = current ? `${current}\n\n${paragraph}` : paragraph
312 if (next.length <= maxChars || current.length === 0) {
313 current = next
314 continue
315 }
316 chunks.push(current)
317 current = paragraph
318 }
319
320 if (current) chunks.push(current)
321 return chunks
322}
323
324function parseMarkdownDocument(filePath: string, content: string): { title: string; chunks: MemoryChunkInput[] } {
325 const lines = content.replace(/\r\n/g, "\n").split("\n")
326 const h1 = lines.find((line) => /^#\s+/.test(line))
327 const title = h1 ? h1.replace(/^#\s+/, "").trim() : titleFromPath(filePath, "Memory")
328 const headingStack: string[] = []
329 let sectionLines: string[] = []
330 let sectionTitle = title
331 const chunks: MemoryChunkInput[] = []
332
333 const flushSection = () => {
334 const body = sectionLines.join("\n").trim()
335 if (!body) {
336 sectionLines = []
337 return
338 }
339
340 const headingPath = headingStack.length > 0 ? headingStack.join(" > ") : null
341 const tags = [basenameWithoutExt(filePath), ...headingStack].join(" ").trim()
342 for (const part of chunkLargeSection(body)) {
343 chunks.push({
344 title: sectionTitle || title,
345 headingPath,
346 text: part,
347 tags,
348 })
349 }
350 sectionLines = []
351 }
352
353 for (const line of lines) {
354 const headingMatch = line.match(/^(#{1,6})\s+(.*)$/)
355 if (!headingMatch) {
356 sectionLines.push(line)
357 continue
358 }
359
360 flushSection()
361
362 const level = headingMatch[1]!.length
363 const heading = headingMatch[2]!.trim()
364 if (level === 1) {
365 sectionTitle = heading || title
366 headingStack.length = 0
367 continue
368 }
369
370 while (headingStack.length >= level - 1) headingStack.pop()
371 headingStack.push(heading)
372 sectionTitle = heading || title
373 }
374
375 flushSection()
376
377 if (chunks.length === 0) {
378 const body = content.trim()
379 if (body) {
380 chunks.push({
381 title,
382 headingPath: null,
383 text: body,
384 tags: basenameWithoutExt(filePath),
385 })
386 }
387 }
388
389 return { title, chunks }
390}
391
392function memoryRecallCooldownTurns(kind: MemoryKind): number {
393 if (kind === "people" || kind === "journal") return MEMORY_RECALL_COOLDOWN_TURNS
394 return 0
395}
396
397function isCoolingDown(hit: MemoryHit, cooldowns: Record<number, number>, currentTurn: number): boolean {
398 const lastTurn = cooldowns[hit.chunkId]
399 if (typeof lastTurn !== "number") return false
400 return currentTurn - lastTurn < memoryRecallCooldownTurns(hit.kind)
401}
402
403function isMemoryRecallSkippedMessage(content: string): boolean {
404 if (content.startsWith(MEMORY_RECALL_HEADER)) return true
405 if (content.startsWith("[system]")) return true
406 if (content.includes("scan snapshot:") && !content.includes("[discord batch]")) return true
407 return false
408}
409
410function latestMemoryRecallQuery(conversation: Message[]): string | null {
411 for (let i = conversation.length - 1; i >= 0; i -= 1) {
412 const message = conversation[i]
413 if (!message || message.role !== "user") continue
414 if (typeof message.content !== "string") continue
415
416 const content = message.content.trim()
417 if (!content || isMemoryRecallSkippedMessage(content)) continue
418 if (content === SCHEDULED_HEARTBEAT_CONTENT) continue
419 return content
420 }
421 return null
422}
423
424function discordChannelLabel(channelId: string | null, fallbackContext: string | null, isDm: boolean): string {
425 if (isDm) return "DM"
426
427 const fallback = fallbackContext
428 ?.replace(/^context:\s*/i, "")
429 .replace(/\s*\(\d+\)\s*$/, "")
430 .trim()
431
432 if (channelId) {
433 try {
434 const row = getDb()
435 .prepare("select guild_id, guild_name, channel_name, is_dm from discord_channels where channel_id = ?")
436 .get(channelId) as
437 | {
438 guild_id: string | null
439 guild_name: string | null
440 channel_name: string | null
441 is_dm: number
442 }
443 | undefined
444
445 if (row?.is_dm) return "DM"
446 if (row) {
447 const guild = row.guild_name ?? row.guild_id
448 const channel = row.channel_name ?? channelId
449 if (guild && channel) return `${guild}/#${channel}`
450 if (channel) return `#${channel}`
451 }
452 } catch {
453 // If the main db is unavailable in tests or scripts, keep the parsed context.
454 }
455 }
456
457 if (fallback) return fallback
458 return channelId ? `#${channelId}` : "channel"
459}
460
461function conciseDiscordMemoryQuery(raw: string): MemoryQueryParts | null {
462 const withoutWakeEnvelope = raw.replace(/^\[(wake|incoming|harness restarted)[^\n]*\]\s*/gi, "").trim()
463 if (!/\[discord\/(?:dm|channel)\]/i.test(withoutWakeEnvelope)) return null
464
465 const blocks = withoutWakeEnvelope
466 .split(/\n\s*\n/g)
467 .map((block) => block.trim())
468 .filter(Boolean)
469 const headerBlock = blocks[0] ?? withoutWakeEnvelope
470 const message = blocks.length > 1 ? blocks.slice(1).join("\n\n").trim() : ""
471
472 const lines = headerBlock.split("\n").map((line) => line.trim()).filter(Boolean)
473 const discordLine = lines.find((line) => /^\[discord\/(?:dm|channel)\]/i.test(line)) ?? ""
474 const contextLine = lines.find((line) => /^context:\s*/i.test(line)) ?? null
475 const isDm = /\[discord\/dm\]/i.test(discordLine)
476 const author = discordLine.match(/@(\S+)/)?.[1] ?? null
477 const context = contextLine?.replace(/^context:\s*/i, "").trim() ?? ""
478 const dmChannelId = context.match(/^DM\s+(\d+)/i)?.[1] ?? null
479 const namedChannelId = context.match(/\((\d+)\)\s*$/)?.[1] ?? null
480 const channelId = dmChannelId ?? namedChannelId
481 const location = discordChannelLabel(channelId, contextLine, isDm)
482
483 if (!author && !location && !message) return null
484 return {
485 sender: author ? normalizeHandle(author) : null,
486 source: location || null,
487 body: message,
488 }
489}
490
491function extractBulletSection(raw: string, label: string): string[] {
492 const escapedLabel = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
493 const match = raw.match(new RegExp(`(?:^|\\n)${escapedLabel}:\\n([\\s\\S]*?)(?:\\n\\n[^\\n:]+:|$)`, "i"))
494 if (!match) return []
495
496 return match[1]
497 .split("\n")
498 .map((line) => line.trim())
499 .filter((line) => line.startsWith("- "))
500 .map((line) => line.slice(2).trim())
501 .filter((line) => line && line !== "(none)")
502}
503
504function conciseDiscordBatchMemoryQuery(raw: string): MemoryQueryParts | null {
505 const withoutWakeEnvelope = raw.replace(/^\[(wake|incoming|harness restarted)[^\n]*\]\s*/gi, "").trim()
506 if (!/\[discord batch\]/i.test(withoutWakeEnvelope)) return null
507
508 const recent = extractBulletSection(withoutWakeEnvelope, "recent messages")
509 const pending = extractBulletSection(withoutWakeEnvelope, "pending preview")
510 const selected = (recent.length > 0 ? recent : pending).slice(-3)
511 if (selected.length === 0) return null
512
513 const senders: string[] = []
514 const sources: string[] = []
515 const bodies: string[] = []
516 const entryPattern = /^\[([^\]]+)\]\s+\[[^\]]+\]\s+@([^:]+):\s*(.*)$/i
517 for (const entry of selected) {
518 const match = entry.match(entryPattern)
519 if (!match) {
520 bodies.push(entry)
521 continue
522 }
523 const [, location, author, body] = match
524 if (author) senders.push(normalizeHandle(author))
525 if (location) sources.push(location.trim())
526 if (body) bodies.push(body.trim())
527 }
528
529 const lastSender = senders.length > 0 ? senders[senders.length - 1]! : null
530 const lastSource = sources.length > 0 ? sources[sources.length - 1]! : null
531 const body = bodies.filter(Boolean).join("\n").trim()
532
533 if (!lastSender && !lastSource && !body) return null
534 return { sender: lastSender, source: lastSource, body }
535}
536
537function memoryQueryForUserMessage(raw: string): MemoryQueryParts {
538 return (
539 conciseDiscordMemoryQuery(raw) ??
540 conciseDiscordBatchMemoryQuery(raw) ??
541 { sender: null, source: null, body: raw }
542 )
543}
544
545function memoryQueryToString(parts: MemoryQueryParts): string {
546 const pieces = [
547 parts.sender ? `@${parts.sender}` : null,
548 parts.source,
549 parts.body || null,
550 ].filter((value): value is string => Boolean(value && value.trim()))
551 return pieces.join("\n")
552}
553
554function normalizeBodyText(raw: string): string {
555 return raw
556 .replace(/@([a-z0-9_.-]+)/gi, " $1 ")
557 .replace(/\b\d{6,}\b/g, " ")
558 .replace(/[^\p{L}\p{N}\s'-]+/gu, " ")
559 .toLowerCase()
560 .trim()
561}
562
563function tokensFromText(raw: string): string[] {
564 const clean = normalizeBodyText(raw)
565 const tokens = clean
566 .split(/\s+/)
567 .map((token) => token.replace(/^['-]+|['-]+$/g, ""))
568 .filter((token) => token.length >= 2 || /\d{2,}/.test(token))
569 .filter((token) => !MEMORY_STOP_WORDS.has(token))
570
571 const unique: string[] = []
572 const seen = new Set<string>()
573 for (const token of tokens) {
574 if (seen.has(token)) continue
575 seen.add(token)
576 unique.push(token)
577 if (unique.length >= MEMORY_QUERY_TOKEN_LIMIT) break
578 }
579 return unique
580}
581
582function searchTokens(raw: string): string[] {
583 return tokensFromText(raw)
584}
585
586async function knownPeopleHandles(aliasMap: AliasMap): Promise<Set<string>> {
587 const handles = new Set<string>()
588 if (await pathExists(PEOPLE_DIR)) {
589 const entries = await fs.readdir(PEOPLE_DIR, { withFileTypes: true })
590 for (const entry of entries) {
591 if (!entry.isFile() || !entry.name.endsWith(".md")) continue
592 const base = basenameWithoutExt(entry.name).toLowerCase()
593 if (base) handles.add(base)
594 }
595 }
596 for (const [key, values] of Object.entries(aliasMap)) {
597 handles.add(key)
598 for (const value of values) handles.add(value)
599 }
600 return handles
601}
602
603async function buildSearchProfile(parts: MemoryQueryParts): Promise<MemorySearchProfile> {
604 const aliasMap = await loadAliasMap()
605 const sender = parts.sender ? normalizeHandle(parts.sender) : null
606 const senderAliases = resolveAliases(sender, aliasMap)
607 const bodyTokens = parts.body ? tokensFromText(parts.body) : []
608
609 const known = await knownPeopleHandles(aliasMap)
610 const inlineMentions = parts.body
611 ? Array.from(parts.body.matchAll(/@([a-z0-9_.-]+)/gi)).map((match) => normalizeHandle(match[1]!))
612 : []
613 const bodyPeopleSet = new Set<string>()
614 const senderSet = new Set<string>([sender ?? "", ...senderAliases].filter(Boolean))
615 for (const token of [...bodyTokens, ...inlineMentions]) {
616 if (!token || senderSet.has(token)) continue
617 if (known.has(token)) bodyPeopleSet.add(token)
618 }
619 const bodyPeople = Array.from(bodyPeopleSet)
620 for (const person of [...bodyPeople]) {
621 for (const alias of resolveAliases(person, aliasMap)) {
622 if (!senderSet.has(alias)) bodyPeopleSet.add(alias)
623 }
624 }
625 const bodyPeopleResolved = Array.from(bodyPeopleSet)
626
627 const combined: string[] = []
628 const seen = new Set<string>()
629 const push = (value: string | null | undefined) => {
630 if (!value) return
631 const lower = value.toLowerCase()
632 if (seen.has(lower)) return
633 seen.add(lower)
634 combined.push(lower)
635 }
636 push(sender)
637 for (const alias of senderAliases) push(alias)
638 for (const person of bodyPeopleResolved) push(person)
639 for (const token of bodyTokens) push(token)
640
641 const normalized = [sender ? `@${sender}` : "", parts.source ?? "", parts.body ?? ""]
642 .filter(Boolean)
643 .join(" ")
644 .toLowerCase()
645
646 return {
647 normalized,
648 sender,
649 senderAliases,
650 bodyTokens,
651 bodyPeople: bodyPeopleResolved,
652 tokens: combined.slice(0, MEMORY_QUERY_TOKEN_LIMIT),
653 personQuery:
654 Boolean(sender) ||
655 bodyPeopleResolved.length > 0 ||
656 /\b(who is|who's|tell me about|about)\b/.test(normalized) ||
657 /\bname\b/.test(normalized) ||
658 /\bfriend\b/.test(normalized),
659 eventQuery:
660 /\b(what happened|when|yesterday|today|tonight|earlier|before|after|restart|restarted|session|wake)\b/.test(
661 normalized,
662 ) ||
663 /\b\d{4}-\d{2}-\d{2}\b/.test(normalized) ||
664 /\b\d{1,2}\/\d{1,2}(?:\/\d{2,4})?\b/.test(normalized) ||
665 /\b(january|february|march|april|may|june|july|august|september|october|november|december)\b/.test(
666 normalized,
667 ),
668 }
669}
670
671function buildSearchQuery(profile: MemorySearchProfile): string | null {
672 if (profile.tokens.length === 0) return null
673 return profile.tokens.map((token) => `"${token.replace(/"/g, '""')}"*`).join(" OR ")
674}
675
676async function readMemoryDocumentRows(): Promise<Map<string, MemoryDocumentRow>> {
677 const rows = getDb()
678 .prepare("select id, path, kind, title, content_hash, mtime_ms from memory_documents")
679 .all() as MemoryDocumentRow[]
680
681 return new Map(rows.map((row) => [row.path, row]))
682}
683
684export async function syncMemoryIndex(): Promise<void> {
685 const db = getDb()
686 const files = await listMemoryFiles()
687 const known = await readMemoryDocumentRows()
688 const present = new Set(files)
689
690 const deleteChunksByDocument = db.prepare("delete from memory_chunks where document_id = ?")
691 const insertDocument = db.prepare(`
692 insert into memory_documents (path, kind, title, mtime_ms, content_hash, updated_at)
693 values (@path, @kind, @title, @mtime_ms, @content_hash, datetime('now'))
694 on conflict(path) do update set
695 kind = excluded.kind,
696 title = excluded.title,
697 mtime_ms = excluded.mtime_ms,
698 content_hash = excluded.content_hash,
699 updated_at = datetime('now')
700 `)
701 const selectDocumentId = db.prepare("select id from memory_documents where path = ?")
702 const insertChunk = db.prepare(`
703 insert into memory_chunks (document_id, chunk_index, title, heading_path, chunk_text, tags)
704 values (?, ?, ?, ?, ?, ?)
705 `)
706 const deleteDocumentByPath = db.prepare("delete from memory_documents where path = ?")
707
708 const updates: Array<{
709 path: string
710 kind: MemoryKind
711 title: string
712 mtimeMs: number
713 hash: string
714 chunks: MemoryChunkInput[]
715 action: "inserted" | "updated"
716 }> = []
717
718 for (const filePath of files) {
719 const kind = detectMemoryKind(filePath)
720 if (!kind) continue
721
722 const [content, stat] = await Promise.all([fs.readFile(filePath, "utf-8"), fs.stat(filePath)])
723 const hash = contentHash(content)
724 const previous = known.get(filePath)
725 if (previous && previous.content_hash === hash && previous.mtime_ms === Math.floor(stat.mtimeMs)) continue
726
727 const parsed = parseMarkdownDocument(filePath, content)
728 updates.push({
729 path: filePath,
730 kind,
731 title: parsed.title,
732 mtimeMs: Math.floor(stat.mtimeMs),
733 hash,
734 chunks: parsed.chunks,
735 action: previous ? "updated" : "inserted",
736 })
737 }
738
739 const removedPaths: string[] = []
740
741 db.transaction(() => {
742 for (const item of updates) {
743 insertDocument.run({
744 path: item.path,
745 kind: item.kind,
746 title: item.title,
747 mtime_ms: item.mtimeMs,
748 content_hash: item.hash,
749 })
750 const row = selectDocumentId.get(item.path) as { id: number } | undefined
751 if (!row) continue
752
753 deleteChunksByDocument.run(row.id)
754 item.chunks.forEach((chunk, index) => {
755 insertChunk.run(row.id, index, chunk.title, chunk.headingPath, chunk.text, chunk.tags)
756 })
757
758 console.log(
759 `[memory] ${item.action} kind=${item.kind} chunks=${item.chunks.length} path=${path.relative(HOME_DIR, item.path)}`,
760 )
761 }
762
763 for (const filePath of known.keys()) {
764 if (present.has(filePath)) continue
765 deleteDocumentByPath.run(filePath)
766 removedPaths.push(filePath)
767 console.log(`[memory] removed path=${path.relative(HOME_DIR, filePath)}`)
768 }
769 })()
770
771 if (updates.length === 0 && removedPaths.length === 0) {
772 return
773 }
774}
775
776function senderHandles(profile: MemorySearchProfile): string[] {
777 const out: string[] = []
778 if (profile.sender) out.push(profile.sender)
779 for (const alias of profile.senderAliases) out.push(alias)
780 return out
781}
782
783function allPersonHandles(profile: MemorySearchProfile): string[] {
784 const seen = new Set<string>()
785 const out: string[] = []
786 for (const handle of [...senderHandles(profile), ...profile.bodyPeople]) {
787 if (!handle || seen.has(handle)) continue
788 seen.add(handle)
789 out.push(handle)
790 }
791 return out
792}
793
794function hitMatchesHandle(hit: MemoryHit, handle: string): boolean {
795 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
796 const pathHaystack = hit.path.toLowerCase()
797 return titleHaystack.includes(handle) || pathHaystack.includes(`/${handle}.md`)
798}
799
800function scoreMemoryHit(hit: MemoryHit, profile: MemorySearchProfile): number {
801 let score = hit.rank
802
803 if (profile.personQuery && !profile.eventQuery) {
804 if (hit.kind === "people") score -= 2
805 if (hit.kind === "core") score -= 0.5
806 if (hit.kind === "journal") score += 1.5
807 } else if (profile.eventQuery && !profile.personQuery) {
808 if (hit.kind === "journal") score -= 1.5
809 if (hit.kind === "people") score += 0.75
810 if (hit.kind === "core") score += 0.25
811 } else {
812 if (hit.kind === "people") score -= 0.5
813 if (hit.kind === "core") score -= 0.25
814 }
815
816 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
817 if (profile.bodyTokens.some((token) => titleHaystack.includes(token))) score -= 0.35
818
819 const handles = allPersonHandles(profile)
820 if (handles.length > 0) {
821 const fullHaystack = `${titleHaystack} ${hit.text.toLowerCase()}`
822 const pathHaystack = hit.path.toLowerCase()
823 if (handles.some((h) => titleHaystack.includes(h) || pathHaystack.includes(`/${h}.md`))) {
824 score -= 3
825 } else if (handles.some((h) => fullHaystack.includes(h))) {
826 score -= 1
827 }
828 }
829
830 return score
831}
832
833function hitSearchHaystack(hit: MemoryHit): string {
834 return `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)} ${hit.text}`.toLowerCase()
835}
836
837function memoryHitSignal(hit: MemoryHit, profile: MemorySearchProfile): MemoryHitSignal {
838 const haystack = hitSearchHaystack(hit)
839 let overlap = 0
840 let strongOverlap = false
841 let bodyOverlap = 0
842
843 for (const token of profile.tokens) {
844 if (!haystack.includes(token)) continue
845 overlap += 1
846 if (token.length >= 5 || /[0-9]/.test(token)) strongOverlap = true
847 }
848 for (const token of profile.bodyTokens) {
849 if (haystack.includes(token)) bodyOverlap += 1
850 }
851
852 const handles = allPersonHandles(profile)
853 const titleHaystack = `${hit.documentTitle} ${hit.title} ${hit.headingPath ?? ""} ${basenameWithoutExt(hit.path)}`.toLowerCase()
854 const pathHaystack = hit.path.toLowerCase()
855 const senderMatch =
856 handles.length > 0 &&
857 handles.some((h) => titleHaystack.includes(h) || pathHaystack.includes(`/${h}.md`) || haystack.includes(h))
858
859 return { overlap, strongOverlap, bodyOverlap, senderMatch }
860}
861
862function shouldInjectHits(hits: MemoryHit[], profile: MemorySearchProfile): boolean {
863 if (hits.length === 0) return false
864
865 const topSignal = memoryHitSignal(hits[0]!, profile)
866
867 if (profile.sender || profile.bodyPeople.length > 0) {
868 if (topSignal.senderMatch) return true
869 if (topSignal.bodyOverlap >= 2) return true
870 if (topSignal.bodyOverlap >= 1 && topSignal.strongOverlap) return true
871 return false
872 }
873
874 if (profile.eventQuery && !profile.personQuery) {
875 return topSignal.overlap >= 1 && hits.some((hit) => hit.kind === "journal")
876 }
877
878 if (profile.personQuery && !profile.eventQuery) {
879 return topSignal.overlap >= 1 && hits.some((hit) => hit.kind === "people" || hit.kind === "core")
880 }
881
882 if (topSignal.bodyOverlap >= 2) return true
883 if (topSignal.bodyOverlap >= 1 && topSignal.strongOverlap) return true
884 if (topSignal.overlap >= 2 && topSignal.strongOverlap) return true
885 return false
886}
887
888function searchMemory(
889 profile: MemorySearchProfile,
890 cooldowns: Record<number, number>,
891 currentTurn: number,
892 limit: number,
893): MemoryHit[] {
894 const db = getDb()
895 const query = buildSearchQuery(profile)
896 if (!query) return []
897 const rows = db
898 .prepare(`
899 select
900 c.id as chunkId,
901 d.path as path,
902 d.kind as kind,
903 d.title as documentTitle,
904 c.title as title,
905 c.heading_path as headingPath,
906 c.chunk_text as text,
907 bm25(memory_chunks_fts, 5.0, 2.0, 1.0, 0.5) as rank
908 from memory_chunks_fts
909 join memory_chunks c on c.id = memory_chunks_fts.rowid
910 join memory_documents d on d.id = c.document_id
911 where memory_chunks_fts match ?
912 order by rank
913 limit ?
914 `)
915 .all(query, Math.max(limit * 8, limit)) as MemoryHit[]
916
917 const ordered = rows.sort((a, b) => scoreMemoryHit(a, profile) - scoreMemoryHit(b, profile))
918 const pool =
919 profile.personQuery && !profile.eventQuery
920 ? (() => {
921 const structured = ordered.filter((row) => row.kind === "people" || row.kind === "core")
922 return structured.length > 0 ? structured : ordered
923 })()
924 : profile.eventQuery && !profile.personQuery
925 ? (() => {
926 const journal = ordered.filter((row) => row.kind === "journal")
927 return journal.length > 0 ? journal : ordered
928 })()
929 : ordered
930
931 const deduped: MemoryHit[] = []
932 const seenChunkIds = new Set<number>()
933 const seenPaths = new Set<string>()
934
935 const handles = allPersonHandles(profile)
936 if (handles.length >= 2) {
937 for (const handle of handles) {
938 if (deduped.length >= limit) break
939 const candidate = pool.find(
940 (row) =>
941 !seenChunkIds.has(row.chunkId) &&
942 !seenPaths.has(row.path) &&
943 !isCoolingDown(row, cooldowns, currentTurn) &&
944 hitMatchesHandle(row, handle),
945 )
946 if (!candidate) continue
947 seenChunkIds.add(candidate.chunkId)
948 seenPaths.add(candidate.path)
949 deduped.push(candidate)
950 }
951 }
952
953 for (const row of pool) {
954 if (deduped.length >= limit) break
955 if (seenChunkIds.has(row.chunkId)) continue
956 if (isCoolingDown(row, cooldowns, currentTurn)) continue
957 seenChunkIds.add(row.chunkId)
958 deduped.push(row)
959 }
960
961 return deduped
962}
963
964function formatMemorySource(hit: MemoryHit): string {
965 const relativePath = path.relative(HOME_DIR, hit.path)
966 const pieces = [`[${hit.kind}]`, hit.documentTitle]
967 if (hit.headingPath && hit.headingPath !== hit.documentTitle) pieces.push(hit.headingPath)
968 pieces.push(relativePath)
969 return pieces.join(" / ")
970}
971
972function toMemorySearchResult(hit: MemoryHit): MemorySearchResult {
973 return {
974 chunkId: hit.chunkId,
975 kind: hit.kind,
976 path: hit.path,
977 source: formatMemorySource(hit),
978 title: hit.title,
979 headingPath: hit.headingPath,
980 preview: trimForPrompt(normalizeText(hit.text), 280),
981 }
982}
983
984function buildMemoryRecallMessage(hits: MemoryHit[], maxChars: number): string {
985 const lines = [MEMORY_RECALL_HEADER, MEMORY_RECALL_NOTE, ""]
986 let usedChars = lines.join("\n").length
987
988 for (const hit of hits) {
989 const source = formatMemorySource(hit)
990 const remaining = Math.max(120, maxChars - usedChars - source.length - 10)
991 const body = trimForPrompt(normalizeText(hit.text), Math.min(280, remaining))
992 const block = `- ${source}\n ${body}`
993
994 if (usedChars + block.length > maxChars && lines.length > 3) break
995 lines.push(block)
996 usedChars += block.length + 1
997 }
998
999 return lines.join("\n").trim()
1000}
1001
1002export async function buildCompletionMessages(
1003 conversation: Message[],
1004 cooldowns: Record<number, number>,
1005 currentTurn: number,
1006): Promise<{ messages: Message[]; recalledChunkIds: number[] }> {
1007 const memoryQuerySource = latestMemoryRecallQuery(conversation)
1008 if (!memoryQuerySource) return { messages: conversation, recalledChunkIds: [] }
1009 const queryParts = memoryQueryForUserMessage(memoryQuerySource)
1010 const memoryQuery = memoryQueryToString(queryParts)
1011
1012 await syncMemoryIndex()
1013
1014 const profile = await buildSearchProfile(queryParts)
1015 if (profile.tokens.length === 0) return { messages: conversation, recalledChunkIds: [] }
1016
1017 const personCount = allPersonHandles(profile).length
1018 const recallLimit = Math.min(
1019 MEMORY_RECALL_MAX_CHUNKS_HARD_CAP,
1020 personCount >= 2 ? MEMORY_RECALL_MAX_CHUNKS + (personCount - 1) : MEMORY_RECALL_MAX_CHUNKS,
1021 )
1022 const hits = searchMemory(profile, cooldowns, currentTurn, recallLimit)
1023 const aliasInfo = profile.senderAliases.length > 0 ? ` aliases=${profile.senderAliases.join(",")}` : ""
1024 const peopleInfo = profile.bodyPeople.length > 0 ? ` people=${profile.bodyPeople.join(",")}` : ""
1025 const debugTag = `sender=${profile.sender ?? "-"}${aliasInfo}${peopleInfo} personQuery=${profile.personQuery} eventQuery=${profile.eventQuery}`
1026 if (hits.length === 0) {
1027 console.log(
1028 `[memory] no-hits query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag}`,
1029 )
1030 return { messages: conversation, recalledChunkIds: [] }
1031 }
1032 if (!shouldInjectHits(hits, profile)) {
1033 console.log(
1034 `[memory] skipped query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag} reason=weak-match`,
1035 )
1036 return { messages: conversation, recalledChunkIds: [] }
1037 }
1038
1039 const extraPersons = Math.max(0, personCount - 1)
1040 const recallChars = MEMORY_RECALL_MAX_CHARS + extraPersons * MEMORY_RECALL_PER_EXTRA_PERSON_CHARS
1041 const recallContent = buildMemoryRecallMessage(hits, recallChars)
1042 console.log(
1043 `[memory] recalled query=${JSON.stringify(trimForPrompt(normalizeText(memoryQuery), 120))} ${debugTag}\n${recallContent}`,
1044 )
1045
1046 recordMetric({
1047 type: "memory",
1048 query: memoryQuery,
1049 results: hits.map(toMemorySearchResult),
1050 })
1051
1052 const recallMessage: Message = {
1053 role: "user",
1054 content: recallContent,
1055 }
1056
1057 return {
1058 messages: [...conversation, recallMessage],
1059 recalledChunkIds: hits.map((hit) => hit.chunkId),
1060 }
1061}
1062
1063export function rememberRecalledMemoryChunks(
1064 cooldowns: Record<number, number>,
1065 injectedChunkIds: number[],
1066 currentTurn: number,
1067): Record<number, number> {
1068 const next = { ...cooldowns }
1069 for (const chunkId of injectedChunkIds) next[chunkId] = currentTurn
1070
1071 for (const [chunkId, turn] of Object.entries(next)) {
1072 if (currentTurn - turn >= MEMORY_RECALL_COOLDOWN_TURNS * 2) delete next[Number(chunkId)]
1073 }
1074
1075 return next
1076}
1077
1078export const __memoryTest = {
1079 latestMemoryRecallQuery,
1080 memoryQueryForUserMessage,
1081 memoryQueryToString,
1082 normalizeBodyText,
1083 searchTokens,
1084 buildSearchProfile,
1085 resolveAliases,
1086 normalizeHandle,
1087}
1088
1089export async function searchMemories(rawQuery: string, limit = 5): Promise<MemorySearchResult[]> {
1090 await syncMemoryIndex()
1091
1092 const profile = await buildSearchProfile({ sender: null, source: null, body: rawQuery })
1093 if (profile.tokens.length === 0) return []
1094
1095 const results = searchMemory(profile, {}, Number.POSITIVE_INFINITY, Math.max(1, Math.min(limit, 10))).map(toMemorySearchResult)
1096 recordMetric({
1097 type: "memory",
1098 query: rawQuery,
1099 results,
1100 })
1101 return results
1102}