nandi.uk
This repository has no description
refactor: use carry query for structured matching, drop vault note keyword overlap
The mapper was producing garbage connections (e.g. "checkin" daily note
matched to mental health vertices) because it used raw keyword overlap
on vault note content. Carry already exports structured semantic triples
via `carry query --format json`.
Changes:
- Install carry binary in container (patchelf for Nix→Debian compat)
- Run `carry query` during sync to export JSON (entities, citations,
connections) — always runs, not just after git pull
- Rewrite mapper to use carry structured data:
1. Carry connections: match by URL overlap with island vertices,
resolve entity names to URLs via carry entity index
2. Carry entities/citations: match by TF-IDF name similarity
3. D1 carry data: same structured matching
- Remove vault note content matching entirely
- Remove Letta URL discovery (carry records already have URLs)
- Add TF-IDF weighting with IDF computed across carry + island corpus
- Expand stop words to filter common vault noise
- Add connections to D1 getCarryData (worker reads connection records)
- Remove Letta env vars from worker/container
Performance: 301-vertex island analysis drops from 3.5min to 0.26s.
Quality: all connections are real semantic triples, no more noise.
Tested locally with `docker run -v carry-vault:/data/carry`.
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
nandi
co-author
Letta Code
date
(May 14, 2026, 4:38 PM UTC)
commit
584a09a8
584a09a8f5a5adc5765e02bc4d757da8af3f3025
parent
81ed8ca7
81ed8ca7858a4f26237b63ccae03d7a6a7c7fb40
+324
-308
+9
-1
container/Dockerfile
+9
-1
container/Dockerfile
···
1
1
FROM node:22-slim
2
2
3
-
# Install git and other deps
3
+
# Install git, curl, patchelf, and other deps
4
4
RUN apt-get update && apt-get install -y \
5
5
git \
6
6
curl \
7
7
ca-certificates \
8
+
patchelf \
8
9
&& rm -rf /var/lib/apt/lists/*
10
+
11
+
# Install carry (local-first semantic DB) — patch Nix-built binary for Debian
12
+
RUN curl -fsSL -o /usr/local/bin/carry \
13
+
https://github.com/tonk-labs/tonk/releases/download/latest/carry-linux-x86_64 \
14
+
&& chmod +x /usr/local/bin/carry \
15
+
&& patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 /usr/local/bin/carry \
16
+
&& apt-get purge -y patchelf && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
9
17
10
18
WORKDIR /app
11
19
+261
-297
container/analysis.ts
+261
-297
container/analysis.ts
···
6
6
// canonical URLs via web search.
7
7
// Returns structured analysis + new connection records to be created on PDS.
8
8
9
-
import { readdirSync, readFileSync, statSync } from "fs";
10
-
import { join, extname } from "path";
9
+
import { readFileSync, existsSync } from "fs";
10
+
import { join } from "path";
11
11
12
-
const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
13
12
const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
14
-
const LETTA_API_KEY = process.env.LETTA_API_KEY || "";
15
-
const LETTA_BASE_URL = process.env.LETTA_BASE_URL || "https://api.letta.com";
16
-
const LETTA_AGENT_ID = process.env.LETTA_AGENT_ID || "";
17
13
18
14
// ─── Types ──────────────────────────────────────────────────────────
19
15
20
-
interface VaultNote {
21
-
path: string;
22
-
content: string;
23
-
}
24
-
25
-
interface CarryNote {
26
-
path: string;
27
-
domain: string;
28
-
content: string;
29
-
}
30
-
31
16
interface VertexInfo {
32
17
uri: string;
33
18
title: string;
···
72
57
entities: CarryEntity[];
73
58
citations: CarryCitation[];
74
59
reasonings: CarryReasoning[];
60
+
connections: CarryConnection[];
61
+
}
62
+
63
+
// Structured carry connection — a semantic triple
64
+
interface CarryConnection {
65
+
sourceName: string; // e.g. "Khayame"
66
+
targetName: string; // e.g. "Vickers Appreciative Systems"
67
+
sourceUrl: string; // e.g. "https://link.springer.com/..."
68
+
targetUrl: string; // e.g. "https://pubadmin.institute/..."
69
+
relation: string; // e.g. "intellectual_influence"
70
+
context: string; // e.g. "Khayame draws on Vickers' appreciative systems..."
75
71
}
76
72
77
73
interface NewConnection {
···
104
100
export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
105
101
const { island, carryData } = input;
106
102
107
-
// 1. Read vault + carry from filesystem
108
-
const vaultNotes = readVault();
109
-
const carryNotes = readCarry();
103
+
// 1. Read carry JSON exports (from `carry query --format json` during sync)
104
+
const carryEntities = readCarryEntities();
105
+
const carryCitations = readCarryCitations();
106
+
const carryConnections = readCarryConnections();
107
+
const entityUrlIndex = buildEntityUrlIndex();
110
108
111
-
// 2. Build corpus from the entire island + precompute token sets
109
+
// 2. Build island URL set for fast lookup
112
110
const islandText = buildIslandText(island);
111
+
const islandUrls = new Set(island.vertices.map(v => v.uri));
113
112
const islandTokenSet = tokenize(islandText);
114
113
const vertexTokenSets = new Map<string, Set<string>>();
115
114
for (const v of island.vertices) {
116
115
vertexTokenSets.set(v.uri, tokenize(`${v.title} ${v.description || ""}`));
117
116
}
118
117
119
-
// 3. Extract themes from the island
118
+
// 3. Compute IDF for TF-IDF scoring
119
+
const allDocuments = [
120
+
...carryEntities.map(e => `${e.name} ${e.description || ""}`),
121
+
...carryCitations.map(c => `${c.title} ${c.takeaway || ""}`),
122
+
...island.vertices.map(v => `${v.title} ${v.description || ""}`),
123
+
];
124
+
const idf = computeIdf(allDocuments);
125
+
126
+
// 4. Extract themes
120
127
const themes = extractThemes(islandText);
121
128
122
-
// 4. Find carry touchpoints — semantic matches from D1 records
123
-
const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] }, vertexTokenSets);
129
+
// 5. Match carry data against island — structured semantic triples only
130
+
const newConnections = matchCarryData(
131
+
island, islandUrls, islandTokenSet, vertexTokenSets, idf,
132
+
carryEntities, carryCitations, carryConnections, entityUrlIndex,
133
+
carryData || { entities: [], citations: [], reasonings: [], connections: [] },
134
+
);
124
135
125
-
// 5. Find vault/carry touchpoints — semantic matches from filesystem notes
126
-
// (async — may call Letta API for URL discovery)
127
-
const fsConnections = await findFilesystemTouchpoints(island, vaultNotes, carryNotes, islandTokenSet, vertexTokenSets);
128
-
newConnections.push(...fsConnections);
136
+
// 6. Flag hot edges
137
+
const highlights = findHotEdges(islandText, island, newConnections);
129
138
130
-
// 6. Flag hot edges — existing edges that are semantically significant
131
-
const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections);
132
-
133
-
// 7. Find tensions from carry notes + reasonings
134
-
const tensions = [
135
-
...findTensionsFromNotes(islandText, carryNotes),
136
-
...findTensionsFromReasonings(islandText, carryData?.reasonings || []),
137
-
];
139
+
// 7. Find tensions from carry reasonings
140
+
const tensions = findTensionsFromReasonings(islandText, carryData?.reasonings || []);
138
141
139
142
// 8. Generate open questions
140
143
const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
···
154
157
return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) };
155
158
}
156
159
157
-
// ─── Filesystem readers ────────────────────────────────────────────
160
+
// ─── Carry JSON readers ────────────────────────────────────────────
161
+
// Reads carry data exported by `carry query --format json` during sync.
162
+
163
+
interface CarryEntityRecord {
164
+
id: string;
165
+
name: string;
166
+
url?: string;
167
+
type?: string;
168
+
description?: string;
169
+
}
158
170
159
-
function readVault(): VaultNote[] {
160
-
return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
161
-
path: relPath,
162
-
content,
163
-
}));
171
+
interface CarryCitationRecord {
172
+
id: string;
173
+
title: string;
174
+
url?: string;
175
+
takeaway?: string;
164
176
}
165
177
166
-
function readCarry(): CarryNote[] {
167
-
const notes: CarryNote[] = [];
178
+
interface CarryConnectionRecord {
179
+
id: string;
180
+
relation: string;
181
+
source?: string;
182
+
target?: string;
183
+
context?: string;
184
+
}
185
+
186
+
function readCarryJson<T>(filename: string): T[] {
187
+
const path = join(CARRY_PATH, filename);
168
188
try {
169
-
const domains = readdirSync(CARRY_PATH);
170
-
for (const domain of domains) {
171
-
const domainPath = join(CARRY_PATH, domain);
172
-
if (!statSync(domainPath).isDirectory()) continue;
173
-
if (domain.startsWith(".")) continue;
174
-
const files = readMarkdownTree(domainPath);
175
-
for (const { relPath, content } of files) {
176
-
notes.push({ path: relPath, domain, content });
177
-
}
178
-
}
189
+
if (!existsSync(path)) return [];
190
+
const data = readFileSync(path, "utf-8");
191
+
return JSON.parse(data) as T[];
179
192
} catch (e: any) {
180
-
console.error("Failed to read carry:", e.message);
193
+
console.error(`[carry] Failed to read ${filename}:`, e.message);
194
+
return [];
181
195
}
182
-
return notes;
196
+
}
197
+
198
+
function readCarryEntities(): CarryEntityRecord[] {
199
+
return readCarryJson<CarryEntityRecord>("org.latha.entity.json");
200
+
}
201
+
202
+
function readCarryCitations(): CarryCitationRecord[] {
203
+
return readCarryJson<CarryCitationRecord>("org.latha.citation.json");
183
204
}
184
205
185
-
function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
186
-
const results: Array<{ relPath: string; content: string }> = [];
187
-
try {
188
-
const entries = readdirSync(root, { withFileTypes: true });
189
-
for (const entry of entries) {
190
-
const fullPath = join(root, entry.name);
191
-
if (entry.name.startsWith(".")) continue;
192
-
if (entry.isDirectory()) {
193
-
results.push(...readMarkdownTree(fullPath));
194
-
} else if (extname(entry.name) === ".md") {
195
-
try {
196
-
const content = readFileSync(fullPath, "utf-8");
197
-
results.push({ relPath: fullPath.replace(root + "/", ""), content });
198
-
} catch {}
199
-
}
206
+
function readCarryConnections(): CarryConnectionRecord[] {
207
+
return readCarryJson<CarryConnectionRecord>("org.latha.connection.json");
208
+
}
209
+
210
+
// Build entity name → URL lookup from carry export
211
+
function buildEntityUrlIndex(): Map<string, string> {
212
+
const index = new Map<string, string>();
213
+
for (const e of readCarryEntities()) {
214
+
const url = e.url?.trim().replace(/^"|"$/g, "");
215
+
if (url && url.startsWith("http")) {
216
+
const name = e.name?.trim().replace(/^"|"$/g, "");
217
+
if (name) index.set(name, url);
200
218
}
201
-
} catch {}
202
-
return results;
219
+
}
220
+
return index;
203
221
}
204
222
205
223
// ─── Build island text corpus ──────────────────────────────────────
···
219
237
return parts.join(" ");
220
238
}
221
239
222
-
// ─── Carry touchpoint matching (D1 records) ────────────────────────
240
+
// ─── Core mapper: match carry semantic triples against island ──────
223
241
224
-
function findCarryTouchpoints(
242
+
function matchCarryData(
225
243
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
226
-
carryData: CarryData,
244
+
islandUrls: Set<string>,
245
+
islandTokenSet: Set<string>,
227
246
vertexTokenSets: Map<string, Set<string>>,
247
+
idf: Map<string, number>,
248
+
carryEntities: CarryEntityRecord[],
249
+
carryCitations: CarryCitationRecord[],
250
+
carryConnections: CarryConnectionRecord[],
251
+
entityUrlIndex: Map<string, string>,
252
+
d1CarryData: CarryData,
228
253
): NewConnection[] {
229
254
const connections: NewConnection[] = [];
230
255
231
-
// --- Match citations by URL ---
232
-
for (const cite of carryData.citations) {
233
-
if (!cite.url) continue; // Need an https:// URL as target
234
-
// Direct URL match: citation URL is a vertex URI
235
-
const vertexMatch = island.vertices.find(v => v.uri === cite.url);
236
-
if (vertexMatch) {
256
+
// ── 1. Carry connections (semantic triples) ──────────────────────
257
+
for (const conn of carryConnections) {
258
+
const srcName = conn.source || "";
259
+
const tgtName = conn.target || "";
260
+
const srcUrl = entityUrlIndex.get(srcName) || "";
261
+
const tgtUrl = entityUrlIndex.get(tgtName) || "";
262
+
263
+
const srcInIsland = srcUrl && islandUrls.has(srcUrl);
264
+
const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);
265
+
266
+
if (srcInIsland && tgtUrl) {
237
267
connections.push({
238
-
source: cite.url,
239
-
target: cite.url, // Self-referential — the citation IS about this vertex
240
-
connectionType: "network.cosmik.connection#annotates",
241
-
note: cite.takeaway.slice(0, 500),
268
+
source: srcUrl,
269
+
target: tgtUrl,
270
+
connectionType: `network.cosmik.connection#${conn.relation}`,
271
+
note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
242
272
});
243
-
continue;
244
273
}
245
274
246
-
// Semantic match: citation title/takeaway overlaps with island vertices
247
-
const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
248
-
const bestVertex = findBestVertexMatch(citeText, island.vertices, vertexTokenSets);
249
-
if (bestVertex) {
250
-
const overlap = keywordOverlapSets(tokenize(citeText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
251
-
if (overlap.length >= 2) {
252
-
connections.push({
253
-
source: bestVertex.uri,
254
-
target: cite.url,
255
-
connectionType: "network.cosmik.connection#annotates",
256
-
note: cite.takeaway.slice(0, 500),
257
-
});
258
-
}
275
+
if (tgtInIsland && srcUrl) {
276
+
connections.push({
277
+
source: tgtUrl,
278
+
target: srcUrl,
279
+
connectionType: `network.cosmik.connection#${conn.relation}`,
280
+
note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
281
+
});
259
282
}
260
-
}
283
+
284
+
// Neither in island — try TF-IDF name match
285
+
if (!srcInIsland && !tgtInIsland && (srcUrl || tgtUrl)) {
286
+
const nameToMatch = srcName || tgtName;
287
+
const urlToConnect = srcUrl || tgtUrl || "";
288
+
if (!urlToConnect) continue;
261
289
262
-
// --- Match entities by name/description ---
263
-
for (const entity of carryData.entities) {
264
-
if (!entity.url) continue; // Need an https:// URL as target
265
-
const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
266
-
const bestVertex = findBestVertexMatch(entityText, island.vertices, vertexTokenSets);
267
-
if (bestVertex) {
268
-
const overlap = keywordOverlapSets(tokenize(entityText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
269
-
if (overlap.length >= 2) {
290
+
const nameTerms = tokenize(nameToMatch);
291
+
const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 4.0);
292
+
if (bestVertex) {
270
293
connections.push({
271
294
source: bestVertex.uri,
272
-
target: entity.url,
273
-
connectionType: "network.cosmik.connection#relates",
274
-
note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
295
+
target: urlToConnect,
296
+
connectionType: `network.cosmik.connection#${conn.relation}`,
297
+
note: `${nameToMatch} (${conn.relation}) — carry connection`,
275
298
});
276
299
}
277
300
}
278
301
}
279
302
280
-
// --- Match reasonings by claim/evidence ---
281
-
for (const reasoning of carryData.reasonings) {
282
-
if (!reasoning.url) continue; // Need an https:// URL as target
283
-
const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
284
-
const bestVertex = findBestVertexMatch(reasoningText, island.vertices, vertexTokenSets);
303
+
// ── 2. Carry entities — match by URL or name ──────────────────────
304
+
for (const entity of carryEntities) {
305
+
const url = entity.url?.trim().replace(/^"|"$/g, "");
306
+
if (!url || !url.startsWith("http")) continue;
307
+
if (islandUrls.has(url)) continue;
308
+
309
+
const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`);
310
+
const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0);
285
311
if (bestVertex) {
286
-
const overlap = keywordOverlapSets(tokenize(reasoningText), vertexTokenSets.get(bestVertex.uri) || tokenize(`${bestVertex.title} ${bestVertex.description || ""}`));
287
-
if (overlap.length >= 2) {
288
-
const connType = reasoning.reasoningType?.includes("contradict")
289
-
? "network.cosmik.connection#contradicts"
290
-
: "network.cosmik.connection#extends";
291
-
connections.push({
292
-
source: bestVertex.uri,
293
-
target: reasoning.url,
294
-
connectionType: connType,
295
-
note: reasoning.claim.slice(0, 500),
296
-
});
297
-
}
312
+
connections.push({
313
+
source: bestVertex.uri,
314
+
target: url,
315
+
connectionType: "network.cosmik.connection#relates",
316
+
note: `Related: ${entity.name} (${entity.type || "entity"})`,
317
+
});
298
318
}
299
319
}
300
320
301
-
return connections;
302
-
}
303
-
304
-
// ─── Letta URL discovery ────────────────────────────────────────────
305
-
// For carry/vault notes that lack an https:// URL, ask the Letta agent
306
-
// to find the canonical URL via web search. Only called for notes that
307
-
// already matched the island (passed keyword overlap), so the volume
308
-
// is small. Results are cached in-memory for the lifetime of the container.
309
-
310
-
const discoveredUrls = new Map<string, string | null>();
311
-
312
-
async function discoverUrl(semanticDescription: string): Promise<string | null> {
313
-
if (!LETTA_API_KEY || !LETTA_AGENT_ID) return null;
314
-
315
-
// Check cache
316
-
const cacheKey = semanticDescription.slice(0, 200);
317
-
if (discoveredUrls.has(cacheKey)) return discoveredUrls.get(cacheKey)!;
318
-
319
-
try {
320
-
const resp = await fetch(`${LETTA_BASE_URL}/v1/agents/${LETTA_AGENT_ID}/messages`, {
321
-
method: "POST",
322
-
headers: {
323
-
"Authorization": `Bearer ${LETTA_API_KEY}`,
324
-
"Content-Type": "application/json",
325
-
},
326
-
signal: AbortSignal.timeout(15000), // 15s timeout per call
327
-
body: JSON.stringify({
328
-
messages: [{
329
-
role: "user",
330
-
content: `Find the canonical URL for: ${semanticDescription}`,
331
-
}],
332
-
}),
333
-
});
334
-
335
-
if (!resp.ok) {
336
-
console.error(`[letta] API error: ${resp.status}`);
337
-
discoveredUrls.set(cacheKey, null);
338
-
return null;
339
-
}
321
+
// ── 3. Carry citations — match by URL or title ───────────────────
322
+
for (const cite of carryCitations) {
323
+
const url = cite.url?.trim().replace(/^"|"$/g, "");
324
+
if (!url || !url.startsWith("http")) continue;
325
+
if (islandUrls.has(url)) continue;
340
326
341
-
const data = await resp.json() as { messages: Array<{ message_type: string; content?: string }> };
342
-
for (const msg of data.messages || []) {
343
-
if (msg.message_type === "assistant_message" && msg.content) {
344
-
const urlMatch = msg.content.match(/https?:\/\/[^\s)\]]+/);
345
-
if (urlMatch) {
346
-
discoveredUrls.set(cacheKey, urlMatch[0]);
347
-
return urlMatch[0];
348
-
}
349
-
}
327
+
const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`);
328
+
const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0);
329
+
if (bestVertex) {
330
+
connections.push({
331
+
source: bestVertex.uri,
332
+
target: url,
333
+
connectionType: "network.cosmik.connection#annotates",
334
+
note: cite.title.slice(0, 200),
335
+
});
350
336
}
351
-
discoveredUrls.set(cacheKey, null);
352
-
return null;
353
-
} catch (e: any) {
354
-
console.error("[letta] discoverUrl error:", e.message);
355
-
discoveredUrls.set(cacheKey, null);
356
-
return null;
357
337
}
358
-
}
359
-
360
-
// ─── Filesystem touchpoint matching (vault + carry notes) ───────────
361
338
362
-
function extractUrl(content: string): string | null {
363
-
// Pull the first https:// URL from the note content
364
-
const match = content.match(/https?:\/\/[^\s)\]]+/);
365
-
return match ? match[0] : null;
366
-
}
367
-
368
-
function extractTitle(content: string): string {
369
-
return content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || "";
370
-
}
339
+
// ── 4. D1 carry data (entities, citations, reasonings from PDS) ──
340
+
for (const entity of d1CarryData.entities) {
341
+
if (!entity.url || !entity.url.startsWith("http")) continue;
342
+
if (islandUrls.has(entity.url)) continue;
371
343
372
-
async function findFilesystemTouchpoints(
373
-
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
374
-
vaultNotes: VaultNote[],
375
-
carryNotes: CarryNote[],
376
-
islandTokenSet: Set<string>,
377
-
vertexTokenSets: Map<string, Set<string>>,
378
-
): Promise<NewConnection[]> {
379
-
const connections: NewConnection[] = [];
380
-
const MAX_LETTA_CALLS = 10;
381
-
let lettaCalls = 0;
382
-
383
-
// Match vault notes to island vertices
384
-
for (const note of vaultNotes) {
385
-
const noteLower = note.content.toLowerCase();
386
-
const overlap = keywordOverlap(islandTokenSet, noteLower);
387
-
if (overlap.length < 3) continue;
388
-
389
-
let noteUrl = extractUrl(note.content);
390
-
const title = extractTitle(note.content) || note.path;
391
-
392
-
if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) {
393
-
noteUrl = await discoverUrl(title || note.path);
394
-
lettaCalls++;
344
+
const nameTerms = tokenize(`${entity.name} ${entity.description || ""}`);
345
+
const bestVertex = findBestVertexMatch(nameTerms, island.vertices, vertexTokenSets, idf, 5.0);
346
+
if (bestVertex) {
347
+
connections.push({
348
+
source: bestVertex.uri,
349
+
target: entity.url,
350
+
connectionType: "network.cosmik.connection#relates",
351
+
note: `Related: ${entity.name} (${entity.entityType.replace("org.latha.strata.defs#", "")})`,
352
+
});
395
353
}
396
-
if (!noteUrl) continue;
354
+
}
355
+
356
+
for (const cite of d1CarryData.citations) {
357
+
if (!cite.url || !cite.url.startsWith("http")) continue;
358
+
if (islandUrls.has(cite.url)) continue;
397
359
398
-
const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets);
360
+
const citeTerms = tokenize(`${cite.title} ${cite.takeaway || ""}`);
361
+
const bestVertex = findBestVertexMatch(citeTerms, island.vertices, vertexTokenSets, idf, 5.0);
399
362
if (bestVertex) {
400
363
connections.push({
401
364
source: bestVertex.uri,
402
-
target: noteUrl,
365
+
target: cite.url,
403
366
connectionType: "network.cosmik.connection#annotates",
404
-
note: `Vault note: ${title.slice(0, 200)}`,
367
+
note: cite.takeaway?.slice(0, 200) || cite.title.slice(0, 200),
405
368
});
406
369
}
407
370
}
408
371
409
-
// Match carry notes to island vertices
410
-
for (const note of carryNotes) {
411
-
const noteLower = note.content.toLowerCase();
412
-
const overlap = keywordOverlap(islandTokenSet, noteLower);
413
-
if (overlap.length < 2) continue;
372
+
for (const reasoning of d1CarryData.reasonings) {
373
+
if (!reasoning.url || !reasoning.url.startsWith("http")) continue;
374
+
if (islandUrls.has(reasoning.url)) continue;
414
375
415
-
let noteUrl = extractUrl(note.content);
416
-
const title = extractTitle(note.content) || note.path;
417
-
418
-
if (!noteUrl && lettaCalls < MAX_LETTA_CALLS) {
419
-
noteUrl = await discoverUrl(`Carry [${note.domain}]: ${title}`);
420
-
lettaCalls++;
421
-
}
422
-
if (!noteUrl) continue;
423
-
424
-
const bestVertex = findBestVertexMatch(noteLower, island.vertices, vertexTokenSets);
376
+
const reasoningTerms = tokenize(`${reasoning.claim} ${reasoning.evidence}`);
377
+
const bestVertex = findBestVertexMatch(reasoningTerms, island.vertices, vertexTokenSets, idf, 4.0);
425
378
if (bestVertex) {
379
+
const connType = reasoning.reasoningType?.includes("contradict")
380
+
? "network.cosmik.connection#contradicts"
381
+
: "network.cosmik.connection#extends";
426
382
connections.push({
427
383
source: bestVertex.uri,
428
-
target: noteUrl,
429
-
connectionType: "network.cosmik.connection#relates",
430
-
note: `Carry [${note.domain}]: ${title.slice(0, 200)}`,
384
+
target: reasoning.url,
385
+
connectionType: connType,
386
+
note: reasoning.claim.slice(0, 500),
431
387
});
432
388
}
433
389
}
···
436
392
}
437
393
438
394
function findBestVertexMatch(
439
-
text: string,
395
+
noteTerms: Set<string>,
440
396
vertices: VertexInfo[],
441
-
vertexTokenSets?: Map<string, Set<string>>,
397
+
vertexTokenSets: Map<string, Set<string>>,
398
+
idf: Map<string, number>,
399
+
minScore: number = 3.0,
442
400
): VertexInfo | null {
443
401
let bestScore = 0;
444
402
let bestVertex: VertexInfo | null = null;
445
403
446
404
for (const v of vertices) {
447
-
let overlap: string[];
448
-
if (vertexTokenSets) {
449
-
const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`);
450
-
overlap = keywordOverlapSets(tokenize(text), vSet);
451
-
} else {
452
-
const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
453
-
overlap = keywordOverlap(tokenize(text), vertexText);
454
-
}
455
-
if (overlap.length > bestScore) {
456
-
bestScore = overlap.length;
405
+
const vSet = vertexTokenSets.get(v.uri) || tokenize(`${v.title} ${v.description || ""}`);
406
+
const score = tfidfScore(noteTerms, vSet, idf);
407
+
if (score > bestScore) {
408
+
bestScore = score;
457
409
bestVertex = v;
458
410
}
459
411
}
460
412
461
-
return bestScore >= 2 ? bestVertex : null;
413
+
return bestScore >= minScore ? bestVertex : null;
462
414
}
463
415
464
416
// ─── Analysis functions ─────────────────────────────────────────────
···
493
445
function findHotEdges(
494
446
islandText: string,
495
447
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
496
-
vaultNotes: VaultNote[],
497
-
carryNotes: CarryNote[],
498
448
newConnections: NewConnection[],
499
449
): string[] {
500
450
const highlights: string[] = [];
501
-
const lowerSet = tokenize(islandText);
502
451
503
452
// Score each edge by semantic significance
504
453
const edgeScores = new Map<string, number>();
505
454
506
455
for (const edge of island.edges) {
507
456
let score = 0;
508
-
const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
509
-
const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
510
-
511
-
// Check if source or target appears in vault notes
512
-
for (const note of vaultNotes) {
513
-
const noteLower = note.content.toLowerCase();
514
-
const overlap = keywordOverlap(lowerSet, noteLower);
515
-
if (overlap.length >= 2) {
516
-
if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
517
-
if (targetTitle && noteLower.includes(targetTitle)) score += 2;
518
-
}
519
-
}
520
-
521
-
// Check if source or target appears in carry notes
522
-
for (const note of carryNotes) {
523
-
const noteLower = note.content.toLowerCase();
524
-
if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
525
-
if (targetTitle && noteLower.includes(targetTitle)) score += 1;
526
-
}
527
457
528
458
// Edges with notes are more interesting
529
459
if (edge.note && edge.note.length > 10) score += 1;
···
552
482
return highlights;
553
483
}
554
484
555
-
function findTensionsFromNotes(
556
-
islandText: string,
557
-
carryNotes: CarryNote[],
558
-
): string[] {
559
-
const tensions: string[] = [];
560
-
const lowerSet = tokenize(islandText);
561
-
562
-
for (const note of carryNotes) {
563
-
const noteLower = note.content.toLowerCase();
564
-
if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
565
-
const overlap = keywordOverlap(lowerSet, noteLower);
566
-
if (overlap.length >= 1) {
567
-
tensions.push(
568
-
`Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
569
-
);
570
-
}
571
-
}
572
-
}
573
-
574
-
return tensions.slice(0, 5);
575
-
}
576
-
577
485
function findTensionsFromReasonings(
578
486
islandText: string,
579
487
reasonings: CarryReasoning[],
···
688
596
"before", "after", "above", "below", "between", "and", "but", "or",
689
597
"not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
690
598
"http", "https", "www", "com", "org",
599
+
// Common vault noise terms
600
+
"checkin", "check", "daily", "today", "yesterday", "tomorrow",
601
+
"morning", "evening", "week", "month", "year",
602
+
"also", "just", "like", "really", "very", "much", "some",
603
+
"more", "most", "other", "about", "which", "when", "what",
604
+
"there", "their", "these", "those", "each", "every", "all",
605
+
"make", "made", "thing", "things", "something", "anything",
606
+
"need", "want", "know", "think", "going", "come", "get",
607
+
"good", "great", "nice", "well", "still", "even", "back",
608
+
"first", "last", "new", "old", "own", "same", "another",
609
+
"many", "such", "only", "over", "than", "too", "very",
610
+
"because", "since", "while", "where", "how", "why", "who",
611
+
"work", "working", "time", "people", "using", "used", "use",
691
612
]);
692
613
693
614
function tokenize(text: string): Set<string> {
694
615
return new Set(
695
616
text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
696
617
);
618
+
}
619
+
620
+
function tokenizeArray(text: string): string[] {
621
+
return text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
622
+
}
623
+
624
+
// ─── TF-IDF scoring ────────────────────────────────────────────────
625
+
// Compute IDF across all notes, then score note-island matches by
626
+
// the sum of IDF weights of shared terms. This naturally downweights
627
+
// common terms like "checkin" and upweights distinctive terms like
628
+
// "stigmergy" or "nanopublication".
629
+
630
+
function computeIdf(documents: string[]): Map<string, number> {
631
+
const docCount = documents.length;
632
+
const df = new Map<string, number>(); // document frequency
633
+
634
+
for (const doc of documents) {
635
+
const terms = new Set(tokenizeArray(doc));
636
+
for (const t of terms) {
637
+
df.set(t, (df.get(t) || 0) + 1);
638
+
}
639
+
}
640
+
641
+
const idf = new Map<string, number>();
642
+
for (const [term, freq] of df) {
643
+
// Smooth IDF: log((N+1)/(df+1)) + 1
644
+
idf.set(term, Math.log((docCount + 1) / (freq + 1)) + 1);
645
+
}
646
+
return idf;
647
+
}
648
+
649
+
function tfidfScore(
650
+
noteTerms: Set<string>,
651
+
targetTerms: Set<string>,
652
+
idf: Map<string, number>,
653
+
): number {
654
+
let score = 0;
655
+
for (const t of noteTerms) {
656
+
if (targetTerms.has(t)) {
657
+
score += idf.get(t) || 0;
658
+
}
659
+
}
660
+
return score;
697
661
}
698
662
699
663
function keywordOverlap(setA: Set<string>, textB: string): string[] {
-2
container/server.ts
-2
container/server.ts
···
57
57
carryTokenSet: !!process.env.CARRY_GITHUB_TOKEN,
58
58
obsidianTokenSet: !!process.env.OBSIDIAN_AUTH_TOKEN,
59
59
obsidianConfigDir: existsSync("/root/.config/obsidian-headless/auth_token"),
60
-
lettaKeySet: !!process.env.LETTA_API_KEY,
61
-
lettaAgentId: process.env.LETTA_AGENT_ID || "not set",
62
60
}));
63
61
return;
64
62
}
+27
container/sync.ts
+27
container/sync.ts
···
119
119
}
120
120
}
121
121
122
+
function exportCarryJson(): void {
123
+
try {
124
+
const domains = [
125
+
{ domain: "org.latha.entity", fields: "name url type description" },
126
+
{ domain: "org.latha.citation", fields: "title url takeaway" },
127
+
{ domain: "org.latha.connection", fields: "relation source target context" },
128
+
];
129
+
130
+
for (const { domain, fields } of domains) {
131
+
const outPath = join(CARRY_PATH, `${domain}.json`);
132
+
const cmd = `carry query ${domain} ${fields} --format json`;
133
+
const result = execSync(cmd, {
134
+
cwd: CARRY_PATH,
135
+
encoding: "utf-8",
136
+
timeout: 30000,
137
+
});
138
+
writeFileSync(outPath, result);
139
+
const count = JSON.parse(result).length;
140
+
console.log(`[sync] exported ${domain}: ${count} records`);
141
+
}
142
+
} catch (e: any) {
143
+
console.error("[sync] carry export failed:", e.message?.slice(0, 200));
144
+
}
145
+
}
146
+
122
147
export async function syncAll(): Promise<void> {
123
148
await Promise.all([syncVault(), syncCarry()]);
149
+
// Always export carry JSON if carry vault exists — even without git sync
150
+
exportCarryJson();
124
151
}
+1
-5
src/container.ts
+1
-5
src/container.ts
···
6
6
OBSIDIAN_AUTH_TOKEN: string;
7
7
OBSIDIAN_ENCRYPTION_KEY: string;
8
8
OBSIDIAN_ENCRYPTION_SALT: string;
9
-
LETTA_API_KEY: string;
10
-
LETTA_AGENT_ID: string;
11
9
}
12
10
13
11
export class StrataContainer extends Container<ContainerEnv> {
···
28
26
OBSIDIAN_SYNC_HOST: "sync-53.obsidian.md",
29
27
...(env.OBSIDIAN_ENCRYPTION_KEY ? { OBSIDIAN_ENCRYPTION_KEY: env.OBSIDIAN_ENCRYPTION_KEY } : {}),
30
28
...(env.OBSIDIAN_ENCRYPTION_SALT ? { OBSIDIAN_ENCRYPTION_SALT: env.OBSIDIAN_ENCRYPTION_SALT } : {}),
31
-
...(env.LETTA_API_KEY ? { LETTA_API_KEY: env.LETTA_API_KEY } : {}),
32
-
...(env.LETTA_AGENT_ID ? { LETTA_AGENT_ID: env.LETTA_AGENT_ID } : {}),
33
29
};
34
30
}
35
31
36
32
async onStart() {
37
33
const has = (k: string) => !!this.envVars?.[k];
38
-
console.log("[strata-container] started, carry:", has("CARRY_GITHUB_TOKEN") ? "set" : "missing", "obsidian:", has("OBSIDIAN_AUTH_TOKEN") ? "set" : "missing", "letta:", has("LETTA_API_KEY") ? "set" : "missing");
34
+
console.log("[strata-container] started, carry:", has("CARRY_GITHUB_TOKEN") ? "set" : "missing", "obsidian:", has("OBSIDIAN_AUTH_TOKEN") ? "set" : "missing");
39
35
}
40
36
41
37
async onStop() {
+26
-3
src/worker.ts
+26
-3
src/worker.ts
···
19
19
OBSIDIAN_AUTH_TOKEN: string;
20
20
OBSIDIAN_ENCRYPTION_KEY: string;
21
21
OBSIDIAN_ENCRYPTION_SALT: string;
22
-
LETTA_API_KEY: string;
23
-
LETTA_AGENT_ID: string;
24
22
}
25
23
26
24
// ─── Contrail setup ─────────────────────────────────────────────────
···
700
698
entities: Array<{ atUri: string; name: string; entityType: string; description?: string; url?: string }>;
701
699
citations: Array<{ atUri: string; url: string; title: string; takeaway: string; confidence?: string }>;
702
700
reasonings: Array<{ atUri: string; claim: string; evidence: string; reasoningType?: string; url?: string }>;
701
+
connections: Array<{ atUri: string; source: string; target: string; relation: string; sourceUrl: string; targetUrl: string; context: string }>;
703
702
}> {
704
703
const entities: Array<{ atUri: string; name: string; entityType: string; description?: string; url?: string }> = [];
705
704
const citations: Array<{ atUri: string; url: string; title: string; takeaway: string; confidence?: string }> = [];
706
705
const reasonings: Array<{ atUri: string; claim: string; evidence: string; reasoningType?: string; url?: string }> = [];
706
+
const connections: Array<{ atUri: string; source: string; target: string; relation: string; sourceUrl: string; targetUrl: string; context: string }> = [];
707
707
708
708
try {
709
709
// Read entities
···
760
760
console.error("getCarryData error:", e.message);
761
761
}
762
762
763
-
return { entities, citations, reasonings };
763
+
// Read connections from D1
764
+
try {
765
+
const connRows = await db
766
+
.prepare("SELECT uri, record FROM records_connection ORDER BY time_us DESC LIMIT 100")
767
+
.all<{ uri: string; record: string }>();
768
+
for (const row of connRows.results || []) {
769
+
try {
770
+
const r = JSON.parse(row.record);
771
+
connections.push({
772
+
atUri: row.uri,
773
+
source: r.source || "",
774
+
target: r.target || "",
775
+
relation: r.relation || "",
776
+
sourceUrl: r.sourceUrl || "",
777
+
targetUrl: r.targetUrl || "",
778
+
context: r.context || "",
779
+
});
780
+
} catch {}
781
+
}
782
+
} catch (e: any) {
783
+
console.error("getCarryData connections error:", e.message);
784
+
}
785
+
786
+
return { entities, citations, reasonings, connections };
764
787
}
765
788
766
789
// Helper: call the strata container