/** * vault:analyze * * Parse Obsidian vault notes into structured carry data using a letta-code-sdk agent. * * Usage: * bun vault:analyze # process all vault notes * bun vault:analyze --note "Astera Institute" # process single note * bun vault:analyze --force # reprocess already-analyzed notes * bun vault:analyze --dry-run # show what would be extracted * bun vault:analyze --limit 5 # only process 5 notes */ import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; import { execFileSync } from 'node:child_process'; import { createAgent, createSession, type Session, type AnyAgentTool, type AgentToolResult, type SDKResultMessage, } from '@letta-ai/letta-code-sdk'; import { loadDotEnv } from './cli-utils.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); await loadDotEnv(resolve(__dirname, '..', '.env')); const VAULT_DIR = process.env.VAULT_DIR ?? '/home/nandi/vault'; const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault'; const PROVENANCE_DIR = resolve(CARRY_DIR, '.vault-analyze'); interface ExtractedItem { type: 'entity' | 'citation' | 'connection'; data: Record; } // --- carry_assert tool --- function carryAssert(type: string, data: Record): string { const fields = Object.entries(data) .filter(([, v]) => v && v.trim()) .map(([k, v]) => `${k}=${JSON.stringify(v)}`) .join(' '); const domain = type === 'entity' ? 'org.latha.entity' : type === 'citation' ? 'org.latha.citation' : 'org.latha.connection'; try { const out = execFileSync('carry', ['assert', domain, ...fields.split(' ')], { cwd: CARRY_DIR, encoding: 'utf-8', timeout: 10_000, }); return out.trim() || 'ok'; } catch (err: any) { return `error: ${err?.message ?? err}`; } } const carryAssertTool: AnyAgentTool = { label: 'carry_assert', name: 'carry_assert', description: `Assert structured data into carry. Call for each entity, citation, or connection you extract from the note. For entities: {type: "entity", data: {name, url, description}} For citations: {type: "citation", data: {url, title, takeaway, domain, confidence}} For connections: {type: "connection", data: {source, target, relation}} - Entity names should be canonical (e.g. "Ink and Switch" not "Ink & Switch") - URLs must be real HTTPS URLs found in the note - Connection source/target should be entity names, not URLs - Confidence should be one of: org.latha.island.defs#high, org.latha.island.defs#medium, org.latha.island.defs#low - Relation should be lowercase-hyphenated (e.g. "related-to", "director", "division-of", "member-of")`, parameters: { type: 'object', properties: { type: { type: 'string', enum: ['entity', 'citation', 'connection'], description: 'Type of carry record to assert', }, data: { type: 'object', description: 'Fields for the carry record', properties: { name: { type: 'string', description: 'Entity name' }, url: { type: 'string', description: 'HTTPS URL' }, description: { type: 'string', description: 'Short description' }, title: { type: 'string', description: 'Citation title' }, takeaway: { type: 'string', description: 'Key insight from citation' }, domain: { type: 'string', description: 'Domain/discipline' }, confidence: { type: 'string', description: 'Confidence level' }, source: { type: 'string', description: 'Connection source entity name' }, target: { type: 'string', description: 'Connection target entity name' }, relation: { type: 'string', description: 'Relationship type' }, }, }, }, required: ['type', 'data'], }, execute: async (_id, args): Promise> => { const { type, data } = args as { type: string; data: Record }; const result = carryAssert(type, data); return { content: [{ type: 'text', text: result }], }; }, }; // --- Note processing --- function readVaultNotes(): string[] { return readdirSync(VAULT_DIR) .filter((f) => f.endsWith('.md')) .sort(); } function readNote(filename: string): string { return readFileSync(resolve(VAULT_DIR, filename), 'utf-8'); } function isAlreadyProcessed(filename: string): boolean { const provenancePath = resolve(PROVENANCE_DIR, `${filename}.json`); return existsSync(provenancePath); } function saveProvenance(filename: string, items: ExtractedItem[]): void { mkdirSync(PROVENANCE_DIR, { recursive: true }); writeFileSync( resolve(PROVENANCE_DIR, `${filename}.json`), JSON.stringify({ filename, processedAt: new Date().toISOString(), items }, null, 2) + '\n', 'utf-8', ); } const EXTRACTION_PROMPT = `Analyze this Obsidian vault note and extract structured data into carry. For each note, extract: 1. **Entities**: People, organizations, concepts, tools mentioned. Include name, URL (if found in the note), and a short description from the note content. 2. **Citations**: Any referenced URLs with titles and key takeaways summarizing what the note says about that source. 3. **Connections**: Relationships between entities mentioned (e.g. "X is director of Y", "A is related-to B", "C is a division of D"). Rules: - Use carry_assert for each extraction - Entity names should be canonical and complete (e.g. "Ink and Switch" not "I&S") - Only include URLs that actually appear in the note text - Connection source/target should be entity names, not URLs - Be thorough — extract every named entity and every stated relationship - For wikilinks like [[Some Topic]], treat "Some Topic" as an entity name - Confidence for citations: use org.latha.island.defs#high for well-known sources, org.latha.island.defs#medium otherwise Note: {title} --- {content}`; async function processNote(session: Session, filename: string, dryRun: boolean): Promise { const content = readNote(filename); const title = filename.replace(/\.md$/, ''); const prompt = EXTRACTION_PROMPT.replace('{title}', title).replace('{content}', content); if (dryRun) { console.log(` [dry-run] Would send ${content.length} chars to agent`); return []; } const result: SDKResultMessage = await session.runTurn(prompt); if (!result.success) { console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`); return []; } // The agent calls carry_assert tools during the turn, so data is already written. // We just return an empty array here since the tool calls themselves do the writing. // In a more elaborate version, we'd collect the tool call args. return []; } async function main(): Promise { const args = process.argv.slice(2); const dryRun = args.includes('--dry-run'); const force = args.includes('--force'); const noteArg = args.find((a) => a.startsWith('--note=')); const noteName = noteArg ? noteArg.split('=')[1] : null; const limitArg = args.find((a) => a.startsWith('--limit=')); const limit = limitArg ? Number(limitArg.split('=')[1]) : Infinity; let notes = readVaultNotes(); if (noteName) { const match = notes.find((n) => n === noteName || n === `${noteName}.md`); if (!match) { console.error(`Note not found: ${noteName}`); process.exit(1); } notes = [match]; } if (!force) { notes = notes.filter((n) => !isAlreadyProcessed(n)); } notes = notes.slice(0, limit); if (notes.length === 0) { console.log('No notes to process.'); return; } console.log(`Processing ${notes.length} vault notes${dryRun ? ' (dry-run)' : ''}...`); // Create a fresh agent with the carry_assert tool const agentId = await createAgent({ persona: 'You are a research assistant that extracts structured entities, citations, and connections from Obsidian vault notes. You are thorough and precise, extracting every named entity and relationship. You always use the carry_assert tool to record your extractions.', human: 'vault-analyzer', tools: [carryAssertTool], permissionMode: 'bypassPermissions', memfs: false, systemInfoReminder: false, }); console.log(`Created agent: ${agentId}`); const session = createSession(agentId, { memfsStartup: 'skip', }); await session.initialize(); let processed = 0; let errors = 0; for (const filename of notes) { const title = filename.replace(/\.md$/, ''); console.log(`\nProcessing: ${title}`); try { const items = await processNote(session, filename, dryRun); if (!dryRun) { saveProvenance(filename, items); } processed++; } catch (err: any) { console.error(` Error: ${err?.message ?? err}`); errors++; } } console.log(`\nDone. Processed: ${processed}, Errors: ${errors}`); // Clean up session await session[Symbol.asyncDispose](); } main().catch((err) => { console.error('Fatal:', err instanceof Error ? err.message : String(err)); process.exit(1); });