nandi.uk
This repository has no description
feat: create connection records for carry touchpoints
Analysis now cross-references island vertices against carry data
(entities, citations, reasonings) from D1. When semantic matches are
found, new network.cosmik.connection records are created on PDS linking
the island vertex to the carry record.
Flow:
1. Worker reads carry records from D1 (records_entity, records_citation,
records_reasoning)
2. Passes carryData to container alongside island
3. Container matches carry records against island vertices by keyword
overlap (URL match, title overlap, takeaway overlap)
4. Returns newConnections array
5. Worker creates connection records on PDS via createRecord
6. New connection URIs added to highlights
No carry touchpoints found yet — carry data needs to be published to
AT Protocol first (entities, citations, reasonings) for contrail to
ingest into D1.
Removed carryRefs from frontend (replaced by real connection records).
Removed filesystem-based vault/carry reading from analysis.ts.
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
nandi
co-author
Letta Code
date
(May 14, 2026, 6:55 AM UTC)
commit
04852438
0485243837e8559d9967e4e97cf149a8500fbaea
parent
b8d6ac3d
b8d6ac3df50024e7badd44f4af4c2b462c53b71c
+311
-141
+184
-140
container/analysis.ts
+184
-140
container/analysis.ts
···
1
1
// ─── Strata Analysis Engine ─────────────────────────────────────────
2
-
// Reads vault (markdown) + carry (markdown) from local filesystem
3
-
// Cross-references an island (cluster of linked URLs) against local knowledge
4
-
// Returns structured analysis with carry cross-refs
5
-
//
6
-
// This runs inside a Cloudflare Container with synced vault + carry data.
7
-
8
-
import { readdirSync, readFileSync, statSync } from "fs";
9
-
import { join, extname } from "path";
10
-
11
-
const VAULT_PATH = process.env.VAULT_PATH || "/data/vault";
12
-
const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";
2
+
// Cross-references an island (cluster of linked URLs) against carry data
3
+
// (entities, citations, reasonings from D1) passed in by the worker.
4
+
// Returns structured analysis + new connection records to be created on PDS.
13
5
14
6
// ─── Types ──────────────────────────────────────────────────────────
15
7
16
-
interface VaultNote {
17
-
path: string;
18
-
content: string;
19
-
}
20
-
21
-
interface CarryNote {
22
-
path: string;
23
-
domain: string;
24
-
content: string;
25
-
}
26
-
27
8
interface VertexInfo {
28
9
uri: string;
29
10
title: string;
···
40
21
note: string;
41
22
}
42
23
43
-
interface CarryRef {
44
-
carryId: string;
24
+
interface CarryEntity {
25
+
atUri: string;
26
+
name: string;
27
+
entityType: string;
28
+
description?: string;
29
+
}
30
+
31
+
interface CarryCitation {
32
+
atUri: string;
33
+
url: string;
34
+
title: string;
35
+
takeaway: string;
36
+
confidence?: string;
37
+
}
38
+
39
+
interface CarryReasoning {
45
40
atUri: string;
46
-
summary: string;
41
+
claim: string;
42
+
evidence: string;
43
+
reasoningType?: string;
44
+
}
45
+
46
+
interface CarryData {
47
+
entities: CarryEntity[];
48
+
citations: CarryCitation[];
49
+
reasonings: CarryReasoning[];
50
+
}
51
+
52
+
interface NewConnection {
53
+
source: string; // island vertex URI
54
+
target: string; // carry record AT URI
55
+
connectionType: string; // e.g. "network.cosmik.connection#annotates"
56
+
note: string; // why this connection exists
47
57
}
48
58
49
59
interface AnalysisInput {
···
52
62
vertices: VertexInfo[];
53
63
edges: EdgeInfo[];
54
64
};
65
+
carryData?: CarryData;
55
66
}
56
67
57
68
interface AnalysisResult {
···
60
71
tensions: string[];
61
72
openQuestions: string[];
62
73
synthesis: string;
63
-
carryRefs: CarryRef[];
74
+
newConnections: NewConnection[];
64
75
}
65
76
66
77
// ─── Main analysis function ────────────────────────────────────────
67
78
68
79
export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
69
-
const { island } = input;
70
-
71
-
// 1. Read vault notes from filesystem
72
-
const vaultNotes = readVault();
73
-
74
-
// 2. Read carry notes from filesystem
75
-
const carryNotes = readCarry();
80
+
const { island, carryData } = input;
76
81
77
-
// 3. Build corpus from the entire island
82
+
// 1. Build corpus from the entire island
78
83
const islandText = buildIslandText(island);
79
84
80
-
// 4. Extract themes from the island
85
+
// 2. Extract themes from the island
81
86
const themes = extractThemes(islandText);
82
87
83
-
// 5. Flag hot edges — existing edges that are semantically significant
84
-
const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes);
85
-
86
-
// 6. Find tensions
87
-
const tensions = findTensions(islandText, carryNotes);
88
-
89
-
// 7. Generate open questions
90
-
const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions);
88
+
// 3. Find carry touchpoints — semantic matches between carry records and island vertices
89
+
const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] });
91
90
92
-
// 8. Synthesize prose
93
-
const synthesis = synthesize(island, themes, highlights, tensions, openQuestions);
91
+
// 4. Flag hot edges — existing edges that are semantically significant
92
+
const highlights = findHotEdges(islandText, island, newConnections);
94
93
95
-
// 9. Build carry cross-refs
96
-
const carryRefs = buildCarryRefs(highlights);
94
+
// 5. Find tensions from carry reasonings
95
+
const tensions = findTensions(islandText, carryData?.reasonings || []);
97
96
98
-
return { themes, highlights, tensions, openQuestions, synthesis, carryRefs };
99
-
}
97
+
// 6. Generate open questions
98
+
const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections);
100
99
101
-
// ─── Filesystem readers ────────────────────────────────────────────
100
+
// 7. Synthesize prose
101
+
const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections);
102
102
103
-
function readVault(): VaultNote[] {
104
-
return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({
105
-
path: relPath,
106
-
content,
107
-
}));
108
-
}
109
-
110
-
function readCarry(): CarryNote[] {
111
-
const notes: CarryNote[] = [];
112
-
try {
113
-
const domains = readdirSync(CARRY_PATH);
114
-
for (const domain of domains) {
115
-
const domainPath = join(CARRY_PATH, domain);
116
-
if (!statSync(domainPath).isDirectory()) continue;
117
-
if (domain.startsWith(".")) continue;
118
-
const files = readMarkdownTree(domainPath);
119
-
for (const { relPath, content } of files) {
120
-
notes.push({ path: relPath, domain, content });
121
-
}
122
-
}
123
-
} catch (e: any) {
124
-
console.error("Failed to read carry:", e.message);
125
-
}
126
-
return notes;
127
-
}
128
-
129
-
function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> {
130
-
const results: Array<{ relPath: string; content: string }> = [];
131
-
try {
132
-
const entries = readdirSync(root, { withFileTypes: true });
133
-
for (const entry of entries) {
134
-
const fullPath = join(root, entry.name);
135
-
if (entry.name.startsWith(".")) continue;
136
-
if (entry.isDirectory()) {
137
-
results.push(...readMarkdownTree(fullPath));
138
-
} else if (extname(entry.name) === ".md") {
139
-
try {
140
-
const content = readFileSync(fullPath, "utf-8");
141
-
results.push({ relPath: fullPath.replace(root + "/", ""), content });
142
-
} catch {}
143
-
}
144
-
}
145
-
} catch {}
146
-
return results;
103
+
return { themes, highlights, tensions, openQuestions, synthesis, newConnections };
147
104
}
148
105
149
106
// ─── Build island text corpus ──────────────────────────────────────
···
163
120
return parts.join(" ");
164
121
}
165
122
123
+
// ─── Carry touchpoint matching ──────────────────────────────────────
124
+
125
+
function findCarryTouchpoints(
126
+
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
127
+
carryData: CarryData,
128
+
): NewConnection[] {
129
+
const connections: NewConnection[] = [];
130
+
const islandText = buildIslandText(island).toLowerCase();
131
+
132
+
// Build a vertex index for fast lookup
133
+
const vertexByUri = new Map(island.vertices.map(v => [v.uri, v]));
134
+
135
+
// --- Match citations by URL ---
136
+
for (const cite of carryData.citations) {
137
+
// Direct URL match: citation URL is a vertex URI
138
+
if (vertexByUri.has(cite.url)) {
139
+
connections.push({
140
+
source: cite.url,
141
+
target: cite.atUri,
142
+
connectionType: "network.cosmik.connection#annotates",
143
+
note: cite.takeaway.slice(0, 500),
144
+
});
145
+
continue;
146
+
}
147
+
148
+
// Semantic match: citation title/takeaway overlaps with island vertices
149
+
const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase();
150
+
const bestVertex = findBestVertexMatch(citeText, island.vertices);
151
+
if (bestVertex) {
152
+
const overlap = keywordOverlap(citeText, bestVertex.title + " " + (bestVertex.description || ""));
153
+
if (overlap.length >= 2) {
154
+
connections.push({
155
+
source: bestVertex.uri,
156
+
target: cite.atUri,
157
+
connectionType: "network.cosmik.connection#annotates",
158
+
note: cite.takeaway.slice(0, 500),
159
+
});
160
+
}
161
+
}
162
+
}
163
+
164
+
// --- Match entities by name/description ---
165
+
for (const entity of carryData.entities) {
166
+
const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase();
167
+
const bestVertex = findBestVertexMatch(entityText, island.vertices);
168
+
if (bestVertex) {
169
+
const overlap = keywordOverlap(entityText, bestVertex.title + " " + (bestVertex.description || ""));
170
+
if (overlap.length >= 2) {
171
+
connections.push({
172
+
source: bestVertex.uri,
173
+
target: entity.atUri,
174
+
connectionType: "network.cosmik.connection#relates",
175
+
note: `Related to tracked ${entity.entityType.replace("org.latha.strata.defs#", "")}: ${entity.name}`,
176
+
});
177
+
}
178
+
}
179
+
}
180
+
181
+
// --- Match reasonings by claim/evidence ---
182
+
for (const reasoning of carryData.reasonings) {
183
+
const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase();
184
+
const bestVertex = findBestVertexMatch(reasoningText, island.vertices);
185
+
if (bestVertex) {
186
+
const overlap = keywordOverlap(reasoningText, bestVertex.title + " " + (bestVertex.description || ""));
187
+
if (overlap.length >= 2) {
188
+
const connType = reasoning.reasoningType?.includes("contradict")
189
+
? "network.cosmik.connection#contradicts"
190
+
: "network.cosmik.connection#extends";
191
+
connections.push({
192
+
source: bestVertex.uri,
193
+
target: reasoning.atUri,
194
+
connectionType: connType,
195
+
note: reasoning.claim.slice(0, 500),
196
+
});
197
+
}
198
+
}
199
+
}
200
+
201
+
// Deduplicate by (source, target) pair
202
+
const seen = new Set<string>();
203
+
return connections.filter(c => {
204
+
const key = `${c.source}|${c.target}`;
205
+
if (seen.has(key)) return false;
206
+
seen.add(key);
207
+
return true;
208
+
}).slice(0, 20);
209
+
}
210
+
211
+
function findBestVertexMatch(
212
+
text: string,
213
+
vertices: VertexInfo[],
214
+
): VertexInfo | null {
215
+
let bestScore = 0;
216
+
let bestVertex: VertexInfo | null = null;
217
+
218
+
for (const v of vertices) {
219
+
const vertexText = `${v.title} ${v.description || ""}`.toLowerCase();
220
+
const overlap = keywordOverlap(text, vertexText);
221
+
if (overlap.length > bestScore) {
222
+
bestScore = overlap.length;
223
+
bestVertex = v;
224
+
}
225
+
}
226
+
227
+
return bestScore >= 2 ? bestVertex : null;
228
+
}
229
+
166
230
// ─── Analysis functions ─────────────────────────────────────────────
167
231
168
232
function extractThemes(text: string): string[] {
···
195
259
function findHotEdges(
196
260
islandText: string,
197
261
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
198
-
vaultNotes: VaultNote[],
199
-
carryNotes: CarryNote[],
262
+
newConnections: NewConnection[],
200
263
): string[] {
201
264
const highlights: string[] = [];
202
-
const lower = islandText.toLowerCase();
203
265
204
-
// Score each edge by how much it connects to vault/carry knowledge
266
+
// Score each edge by semantic significance
205
267
const edgeScores = new Map<string, number>();
206
268
207
269
for (const edge of island.edges) {
208
270
let score = 0;
209
271
210
-
// Check if source or target appears in vault notes
211
-
for (const note of vaultNotes) {
212
-
const noteLower = note.content.toLowerCase();
213
-
const overlap = keywordOverlap(lower, noteLower);
214
-
if (overlap.length >= 2) {
215
-
// Does this edge's source or target title appear in the vault note?
216
-
const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
217
-
const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
218
-
if (sourceTitle && noteLower.includes(sourceTitle)) score += 2;
219
-
if (targetTitle && noteLower.includes(targetTitle)) score += 2;
220
-
}
221
-
}
222
-
223
-
// Check if source or target appears in carry notes
224
-
for (const note of carryNotes) {
225
-
const noteLower = note.content.toLowerCase();
226
-
const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || "";
227
-
const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || "";
228
-
if (sourceTitle && noteLower.includes(sourceTitle)) score += 1;
229
-
if (targetTitle && noteLower.includes(targetTitle)) score += 1;
230
-
}
231
-
232
272
// Edges with notes are more interesting
233
273
if (edge.note && edge.note.length > 10) score += 1;
234
274
···
236
276
const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
237
277
if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;
238
278
279
+
// Boost edges whose source or target is also connected to carry data
280
+
for (const nc of newConnections) {
281
+
if (edge.source === nc.source || edge.target === nc.source) score += 3;
282
+
if (edge.source === nc.target || edge.target === nc.target) score += 3;
283
+
}
284
+
239
285
if (score > 0) {
240
286
edgeScores.set(edge.uri, score);
241
287
}
···
252
298
253
299
function findTensions(
254
300
islandText: string,
255
-
carryNotes: CarryNote[],
301
+
reasonings: CarryReasoning[],
256
302
): string[] {
257
303
const tensions: string[] = [];
258
304
const lower = islandText.toLowerCase();
259
305
260
-
for (const note of carryNotes) {
261
-
const noteLower = note.content.toLowerCase();
262
-
// Look for tension markers in carry notes
263
-
if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) {
264
-
const overlap = keywordOverlap(lower, noteLower);
306
+
for (const r of reasonings) {
307
+
const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
308
+
// Look for contradiction markers
309
+
if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
310
+
const overlap = keywordOverlap(lower, rLower);
265
311
if (overlap.length >= 1) {
266
-
tensions.push(
267
-
`Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes`
268
-
);
312
+
tensions.push(r.claim.slice(0, 500));
269
313
}
270
314
}
271
315
}
···
278
322
island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
279
323
highlights: string[],
280
324
tensions: string[],
325
+
newConnections: NewConnection[],
281
326
): string[] {
282
327
const questions: string[] = [];
283
328
const lower = islandText.toLowerCase();
284
329
285
-
if (highlights.length > 0) {
330
+
if (newConnections.length > 0) {
286
331
questions.push(
287
-
`How does this island extend or challenge your existing knowledge? ${highlights.length} edges flagged as significant.`
332
+
`${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
288
333
);
289
334
}
290
335
···
325
370
highlights: string[],
326
371
tensions: string[],
327
372
openQuestions: string[],
373
+
newConnections: NewConnection[],
328
374
): string {
329
375
const vertexCount = island.vertices.length;
330
376
const edgeCount = island.edges.length;
···
336
382
? `Themes: ${themes.join(", ")}.`
337
383
: "";
338
384
385
+
const touchpointStr = newConnections.length > 0
386
+
? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
387
+
: "No carry touchpoints found.";
388
+
339
389
const highlightStr = highlights.length > 0
340
-
? `${highlights.length} edges flagged as semantically significant — connecting to your existing knowledge.`
341
-
: "No edges flagged as significant against your existing knowledge.";
390
+
? `${highlights.length} existing edges flagged as semantically significant.`
391
+
: "";
342
392
343
393
const tensionStr = tensions.length > 0
344
394
? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
···
348
398
? `Open questions: ${openQuestions.join(" ")}`
349
399
: "";
350
400
351
-
return [`Island (${clusterDesc}).`, themeStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
352
-
}
353
-
354
-
function buildCarryRefs(highlights: string[]): CarryRef[] {
355
-
// No carry-specific refs without the old connection objects
356
-
// Carry cross-refs are now implicit via the hot edge flags
357
-
return [];
401
+
return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
358
402
}
359
403
360
404
// ─── Utilities ──────────────────────────────────────────────────────
+127
-1
src/worker.ts
+127
-1
src/worker.ts
···
689
689
690
690
// ── Container-backed xrpc methods ────────────────────────────────
691
691
692
+
// Helper: read carry data from D1 for analysis
693
+
async function getCarryData(db: D1Database, did?: string): Promise<{
694
+
entities: Array<{ atUri: string; name: string; entityType: string; description?: string }>;
695
+
citations: Array<{ atUri: string; url: string; title: string; takeaway: string; confidence?: string }>;
696
+
reasonings: Array<{ atUri: string; claim: string; evidence: string; reasoningType?: string }>;
697
+
}> {
698
+
const entities: Array<{ atUri: string; name: string; entityType: string; description?: string }> = [];
699
+
const citations: Array<{ atUri: string; url: string; title: string; takeaway: string; confidence?: string }> = [];
700
+
const reasonings: Array<{ atUri: string; claim: string; evidence: string; reasoningType?: string }> = [];
701
+
702
+
try {
703
+
// Read entities
704
+
const entityRows = await db
705
+
.prepare("SELECT uri, record FROM records_entity ORDER BY time_us DESC LIMIT 100")
706
+
.all<{ uri: string; record: string }>();
707
+
for (const row of entityRows.results || []) {
708
+
try {
709
+
const r = JSON.parse(row.record);
710
+
entities.push({
711
+
atUri: row.uri,
712
+
name: r.name || "",
713
+
entityType: r.entityType || "",
714
+
description: r.description || "",
715
+
});
716
+
} catch {}
717
+
}
718
+
719
+
// Read citations
720
+
const citeRows = await db
721
+
.prepare("SELECT uri, record FROM records_citation ORDER BY time_us DESC LIMIT 100")
722
+
.all<{ uri: string; record: string }>();
723
+
for (const row of citeRows.results || []) {
724
+
try {
725
+
const r = JSON.parse(row.record);
726
+
citations.push({
727
+
atUri: row.uri,
728
+
url: r.url || "",
729
+
title: r.title || "",
730
+
takeaway: r.takeaway || "",
731
+
confidence: r.confidence || "",
732
+
});
733
+
} catch {}
734
+
}
735
+
736
+
// Read reasonings
737
+
const reasonRows = await db
738
+
.prepare("SELECT uri, record FROM records_reasoning ORDER BY time_us DESC LIMIT 100")
739
+
.all<{ uri: string; record: string }>();
740
+
for (const row of reasonRows.results || []) {
741
+
try {
742
+
const r = JSON.parse(row.record);
743
+
reasonings.push({
744
+
atUri: row.uri,
745
+
claim: r.claim || "",
746
+
evidence: r.evidence || "",
747
+
reasoningType: r.reasoningType || "",
748
+
});
749
+
} catch {}
750
+
}
751
+
} catch (e: any) {
752
+
console.error("getCarryData error:", e.message);
753
+
}
754
+
755
+
return { entities, citations, reasonings };
756
+
}
757
+
692
758
// Helper: call the strata container
693
759
async function callContainer(path: string, method: string, body?: any): Promise<any> {
694
760
const id = env.STRATA_CONTAINER.idFromName("strata");
···
756
822
}
757
823
758
824
try {
759
-
const result = await callContainer("/analyze", "POST", body);
825
+
// Read carry data from D1 for the user's DID
826
+
const carryData = await getCarryData(db, body.did);
827
+
828
+
// Pass carry data to container alongside island
829
+
const payload = { ...body, carryData };
830
+
const result = await callContainer("/analyze", "POST", payload);
831
+
832
+
// Create new connection records on PDS
833
+
const newConnections: Array<{ source: string; target: string; connectionType: string; note: string }> = result.newConnections || [];
834
+
const createdUris: string[] = [];
835
+
836
+
if (newConnections.length > 0) {
837
+
const appPassword = env.PDS_APP_PASSWORD;
838
+
if (appPassword && body.did) {
839
+
const pdsHost = "amanita.us-east.host.bsky.network";
840
+
const authResp = await fetch(`https://${pdsHost}/xrpc/com.atproto.server.createSession`, {
841
+
method: "POST",
842
+
headers: { "Content-Type": "application/json" },
843
+
body: JSON.stringify({ identifier: body.did, password: appPassword }),
844
+
});
845
+
if (authResp.ok) {
846
+
const { accessJwt } = await authResp.json() as { accessJwt: string };
847
+
848
+
for (const conn of newConnections) {
849
+
const rkey = crypto.randomUUID().replace(/-/g, "").slice(0, 13); // TID format
850
+
const record = {
851
+
"$type": "network.cosmik.connection",
852
+
source: conn.source,
853
+
target: conn.target,
854
+
connectionType: conn.connectionType,
855
+
note: conn.note,
856
+
createdAt: new Date().toISOString(),
857
+
};
858
+
859
+
const createResp = await fetch(`https://${pdsHost}/xrpc/com.atproto.repo.createRecord`, {
860
+
method: "POST",
861
+
headers: {
862
+
"Content-Type": "application/json",
863
+
"Authorization": `Bearer ${accessJwt}`,
864
+
},
865
+
body: JSON.stringify({
866
+
repo: body.did,
867
+
collection: "network.cosmik.connection",
868
+
rkey,
869
+
record,
870
+
}),
871
+
});
872
+
873
+
if (createResp.ok) {
874
+
const createResult = await createResp.json() as { uri: string };
875
+
createdUris.push(createResult.uri);
876
+
}
877
+
}
878
+
}
879
+
}
880
+
}
881
+
882
+
// Add new connection URIs to highlights
883
+
result.highlights = [...(result.highlights || []), ...createdUris];
884
+
result.createdConnections = createdUris;
885
+
760
886
return c.json(result);
761
887
} catch (e: any) {
762
888
return c.json({ error: "ContainerError", message: e.message }, 500);