nandi.uk
This repository has no description
9.3 kB
260 lines
1/**
2 * vault:analyze
3 *
4 * Parse Obsidian vault notes into structured carry data using a letta-code-sdk agent.
5 *
6 * Usage:
7 * bun vault:analyze # process all vault notes
8 * bun vault:analyze --note "Astera Institute" # process single note
9 * bun vault:analyze --force # reprocess already-analyzed notes
10 * bun vault:analyze --dry-run # show what would be extracted
11 * bun vault:analyze --limit 5 # only process 5 notes
12 */
13
14import { resolve, dirname } from 'node:path';
15import { fileURLToPath } from 'node:url';
16import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
17import { execFileSync } from 'node:child_process';
18import {
19 createAgent,
20 createSession,
21 type Session,
22 type AnyAgentTool,
23 type AgentToolResult,
24 type SDKResultMessage,
25} from '@letta-ai/letta-code-sdk';
26import { loadDotEnv } from './cli-utils.js';
27
28const __dirname = dirname(fileURLToPath(import.meta.url));
29await loadDotEnv(resolve(__dirname, '..', '.env'));
30
31const VAULT_DIR = process.env.VAULT_DIR ?? '/home/nandi/vault';
32const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault';
33const PROVENANCE_DIR = resolve(CARRY_DIR, '.vault-analyze');
34
35interface ExtractedItem {
36 type: 'entity' | 'citation' | 'connection';
37 data: Record<string, string>;
38}
39
40// --- carry_assert tool ---
41
42function carryAssert(type: string, data: Record<string, string>): string {
43 const fields = Object.entries(data)
44 .filter(([, v]) => v && v.trim())
45 .map(([k, v]) => `${k}=${JSON.stringify(v)}`)
46 .join(' ');
47 const domain = type === 'entity' ? 'org.latha.entity'
48 : type === 'citation' ? 'org.latha.citation'
49 : 'org.latha.connection';
50 try {
51 const out = execFileSync('carry', ['assert', domain, ...fields.split(' ')], {
52 cwd: CARRY_DIR,
53 encoding: 'utf-8',
54 timeout: 10_000,
55 });
56 return out.trim() || 'ok';
57 } catch (err: any) {
58 return `error: ${err?.message ?? err}`;
59 }
60}
61
62const carryAssertTool: AnyAgentTool = {
63 label: 'carry_assert',
64 name: 'carry_assert',
65 description: `Assert structured data into carry. Call for each entity, citation, or connection you extract from the note.
66
67For entities: {type: "entity", data: {name, url, description}}
68For citations: {type: "citation", data: {url, title, takeaway, domain, confidence}}
69For connections: {type: "connection", data: {source, target, relation}}
70
71- Entity names should be canonical (e.g. "Ink and Switch" not "Ink & Switch")
72- URLs must be real HTTPS URLs found in the note
73- Connection source/target should be entity names, not URLs
74- Confidence should be one of: org.latha.island.defs#high, org.latha.island.defs#medium, org.latha.island.defs#low
75- Relation should be lowercase-hyphenated (e.g. "related-to", "director", "division-of", "member-of")`,
76 parameters: {
77 type: 'object',
78 properties: {
79 type: {
80 type: 'string',
81 enum: ['entity', 'citation', 'connection'],
82 description: 'Type of carry record to assert',
83 },
84 data: {
85 type: 'object',
86 description: 'Fields for the carry record',
87 properties: {
88 name: { type: 'string', description: 'Entity name' },
89 url: { type: 'string', description: 'HTTPS URL' },
90 description: { type: 'string', description: 'Short description' },
91 title: { type: 'string', description: 'Citation title' },
92 takeaway: { type: 'string', description: 'Key insight from citation' },
93 domain: { type: 'string', description: 'Domain/discipline' },
94 confidence: { type: 'string', description: 'Confidence level' },
95 source: { type: 'string', description: 'Connection source entity name' },
96 target: { type: 'string', description: 'Connection target entity name' },
97 relation: { type: 'string', description: 'Relationship type' },
98 },
99 },
100 },
101 required: ['type', 'data'],
102 },
103 execute: async (_id, args): Promise<AgentToolResult<unknown>> => {
104 const { type, data } = args as { type: string; data: Record<string, string> };
105 const result = carryAssert(type, data);
106 return {
107 content: [{ type: 'text', text: result }],
108 };
109 },
110};
111
112// --- Note processing ---
113
114function readVaultNotes(): string[] {
115 return readdirSync(VAULT_DIR)
116 .filter((f) => f.endsWith('.md'))
117 .sort();
118}
119
120function readNote(filename: string): string {
121 return readFileSync(resolve(VAULT_DIR, filename), 'utf-8');
122}
123
124function isAlreadyProcessed(filename: string): boolean {
125 const provenancePath = resolve(PROVENANCE_DIR, `${filename}.json`);
126 return existsSync(provenancePath);
127}
128
129function saveProvenance(filename: string, items: ExtractedItem[]): void {
130 mkdirSync(PROVENANCE_DIR, { recursive: true });
131 writeFileSync(
132 resolve(PROVENANCE_DIR, `${filename}.json`),
133 JSON.stringify({ filename, processedAt: new Date().toISOString(), items }, null, 2) + '\n',
134 'utf-8',
135 );
136}
137
138const EXTRACTION_PROMPT = `Analyze this Obsidian vault note and extract structured data into carry.
139
140For each note, extract:
1411. **Entities**: People, organizations, concepts, tools mentioned. Include name, URL (if found in the note), and a short description from the note content.
1422. **Citations**: Any referenced URLs with titles and key takeaways summarizing what the note says about that source.
1433. **Connections**: Relationships between entities mentioned (e.g. "X is director of Y", "A is related-to B", "C is a division of D").
144
145Rules:
146- Use carry_assert for each extraction
147- Entity names should be canonical and complete (e.g. "Ink and Switch" not "I&S")
148- Only include URLs that actually appear in the note text
149- Connection source/target should be entity names, not URLs
150- Be thorough — extract every named entity and every stated relationship
151- For wikilinks like [[Some Topic]], treat "Some Topic" as an entity name
152- Confidence for citations: use org.latha.island.defs#high for well-known sources, org.latha.island.defs#medium otherwise
153
154Note: {title}
155---
156{content}`;
157
158async function processNote(session: Session, filename: string, dryRun: boolean): Promise<ExtractedItem[]> {
159 const content = readNote(filename);
160 const title = filename.replace(/\.md$/, '');
161 const prompt = EXTRACTION_PROMPT.replace('{title}', title).replace('{content}', content);
162
163 if (dryRun) {
164 console.log(` [dry-run] Would send ${content.length} chars to agent`);
165 return [];
166 }
167
168 const result: SDKResultMessage = await session.runTurn(prompt);
169
170 if (!result.success) {
171 console.error(` Agent failed: ${result.error ?? result.errorCode ?? 'unknown'}`);
172 return [];
173 }
174
175 // The agent calls carry_assert tools during the turn, so data is already written.
176 // We just return an empty array here since the tool calls themselves do the writing.
177 // In a more elaborate version, we'd collect the tool call args.
178 return [];
179}
180
181async function main(): Promise<void> {
182 const args = process.argv.slice(2);
183 const dryRun = args.includes('--dry-run');
184 const force = args.includes('--force');
185 const noteArg = args.find((a) => a.startsWith('--note='));
186 const noteName = noteArg ? noteArg.split('=')[1] : null;
187 const limitArg = args.find((a) => a.startsWith('--limit='));
188 const limit = limitArg ? Number(limitArg.split('=')[1]) : Infinity;
189
190 let notes = readVaultNotes();
191
192 if (noteName) {
193 const match = notes.find((n) => n === noteName || n === `${noteName}.md`);
194 if (!match) {
195 console.error(`Note not found: ${noteName}`);
196 process.exit(1);
197 }
198 notes = [match];
199 }
200
201 if (!force) {
202 notes = notes.filter((n) => !isAlreadyProcessed(n));
203 }
204
205 notes = notes.slice(0, limit);
206
207 if (notes.length === 0) {
208 console.log('No notes to process.');
209 return;
210 }
211
212 console.log(`Processing ${notes.length} vault notes${dryRun ? ' (dry-run)' : ''}...`);
213
214 // Create a fresh agent with the carry_assert tool
215 const agentId = await createAgent({
216 persona: 'You are a research assistant that extracts structured entities, citations, and connections from Obsidian vault notes. You are thorough and precise, extracting every named entity and relationship. You always use the carry_assert tool to record your extractions.',
217 human: 'vault-analyzer',
218 tools: [carryAssertTool],
219 permissionMode: 'bypassPermissions',
220 memfs: false,
221 systemInfoReminder: false,
222 });
223
224 console.log(`Created agent: ${agentId}`);
225
226 const session = createSession(agentId, {
227 memfsStartup: 'skip',
228 });
229
230 await session.initialize();
231
232 let processed = 0;
233 let errors = 0;
234
235 for (const filename of notes) {
236 const title = filename.replace(/\.md$/, '');
237 console.log(`\nProcessing: ${title}`);
238
239 try {
240 const items = await processNote(session, filename, dryRun);
241 if (!dryRun) {
242 saveProvenance(filename, items);
243 }
244 processed++;
245 } catch (err: any) {
246 console.error(` Error: ${err?.message ?? err}`);
247 errors++;
248 }
249 }
250
251 console.log(`\nDone. Processed: ${processed}, Errors: ${errors}`);
252
253 // Clean up session
254 await session[Symbol.asyncDispose]();
255}
256
257main().catch((err) => {
258 console.error('Fatal:', err instanceof Error ? err.message : String(err));
259 process.exit(1);
260});