feat: carry touchpoints create connection records + vault/carry sync · nandi.uk/stigmergic@45f0ad3

+203 -31

container/analysis.ts

··· 1 1 // ─── Strata Analysis Engine ───────────────────────────────────────── 2 - // Cross-references an island (cluster of linked URLs) against carry data 3 - // (entities, citations, reasonings from D1) passed in by the worker. 2 + // Cross-references an island (cluster of linked URLs) against: 3 + // 1. Vault + carry markdown from local filesystem (synced via ob/git) 4 + // 2. Structured carry records from D1 (entities, citations, reasonings) 4 5 // Returns structured analysis + new connection records to be created on PDS. 6 + 7 + import { readdirSync, readFileSync, statSync } from "fs"; 8 + import { join, extname } from "path"; 9 + 10 + const VAULT_PATH = process.env.VAULT_PATH || "/data/vault"; 11 + const CARRY_PATH = process.env.CARRY_PATH || "/data/carry"; 5 12 6 13 // ─── Types ────────────────────────────────────────────────────────── 14 + 15 + interface VaultNote { 16 + path: string; 17 + content: string; 18 + } 19 + 20 + interface CarryNote { 21 + path: string; 22 + domain: string; 23 + content: string; 24 + } 7 25 8 26 interface VertexInfo { 9 27 uri: string; ··· 79 97 export async function analyze(input: AnalysisInput): Promise<AnalysisResult> { 80 98 const { island, carryData } = input; 81 99 82 - // 1. Build corpus from the entire island 100 + // 1. Read vault + carry from filesystem 101 + const vaultNotes = readVault(); 102 + const carryNotes = readCarry(); 103 + 104 + // 2. Build corpus from the entire island 83 105 const islandText = buildIslandText(island); 84 106 85 - // 2. Extract themes from the island 107 + // 3. Extract themes from the island 86 108 const themes = extractThemes(islandText); 87 109 88 - // 3. Find carry touchpoints — semantic matches between carry records and island vertices 110 + // 4. Find carry touchpoints — semantic matches from D1 records 89 111 const newConnections = findCarryTouchpoints(island, carryData || { entities: [], citations: [], reasonings: [] }); 90 112 91 - // 4. Flag hot edges — existing edges that are semantically significant 92 - const highlights = findHotEdges(islandText, island, newConnections); 113 + // 5. Find vault/carry touchpoints — semantic matches from filesystem notes 114 + const fsConnections = findFilesystemTouchpoints(island, vaultNotes, carryNotes); 115 + newConnections.push(...fsConnections); 116 + 117 + // 6. Flag hot edges — existing edges that are semantically significant 118 + const highlights = findHotEdges(islandText, island, vaultNotes, carryNotes, newConnections); 93 119 94 - // 5. Find tensions from carry reasonings 95 - const tensions = findTensions(islandText, carryData?.reasonings || []); 120 + // 7. Find tensions from carry notes + reasonings 121 + const tensions = [ 122 + ...findTensionsFromNotes(islandText, carryNotes), 123 + ...findTensionsFromReasonings(islandText, carryData?.reasonings || []), 124 + ]; 96 125 97 - // 6. Generate open questions 126 + // 8. Generate open questions 98 127 const openQuestions = generateOpenQuestions(islandText, island, highlights, tensions, newConnections); 99 128 100 - // 7. Synthesize prose 129 + // 9. Synthesize prose 101 130 const synthesis = synthesize(island, themes, highlights, tensions, openQuestions, newConnections); 102 131 103 - return { themes, highlights, tensions, openQuestions, synthesis, newConnections }; 132 + // Deduplicate newConnections 133 + const seen = new Set<string>(); 134 + const deduped = newConnections.filter(c => { 135 + const key = `${c.source}|${c.target}`; 136 + if (seen.has(key)) return false; 137 + seen.add(key); 138 + return true; 139 + }); 140 + 141 + return { themes, highlights, tensions, openQuestions, synthesis, newConnections: deduped.slice(0, 20) }; 142 + } 143 + 144 + // ─── Filesystem readers ──────────────────────────────────────────── 145 + 146 + function readVault(): VaultNote[] { 147 + return readMarkdownTree(VAULT_PATH).map(({ relPath, content }) => ({ 148 + path: relPath, 149 + content, 150 + })); 151 + } 152 + 153 + function readCarry(): CarryNote[] { 154 + const notes: CarryNote[] = []; 155 + try { 156 + const domains = readdirSync(CARRY_PATH); 157 + for (const domain of domains) { 158 + const domainPath = join(CARRY_PATH, domain); 159 + if (!statSync(domainPath).isDirectory()) continue; 160 + if (domain.startsWith(".")) continue; 161 + const files = readMarkdownTree(domainPath); 162 + for (const { relPath, content } of files) { 163 + notes.push({ path: relPath, domain, content }); 164 + } 165 + } 166 + } catch (e: any) { 167 + console.error("Failed to read carry:", e.message); 168 + } 169 + return notes; 170 + } 171 + 172 + function readMarkdownTree(root: string): Array<{ relPath: string; content: string }> { 173 + const results: Array<{ relPath: string; content: string }> = []; 174 + try { 175 + const entries = readdirSync(root, { withFileTypes: true }); 176 + for (const entry of entries) { 177 + const fullPath = join(root, entry.name); 178 + if (entry.name.startsWith(".")) continue; 179 + if (entry.isDirectory()) { 180 + results.push(...readMarkdownTree(fullPath)); 181 + } else if (extname(entry.name) === ".md") { 182 + try { 183 + const content = readFileSync(fullPath, "utf-8"); 184 + results.push({ relPath: fullPath.replace(root + "/", ""), content }); 185 + } catch {} 186 + } 187 + } 188 + } catch {} 189 + return results; 104 190 } 105 191 106 192 // ─── Build island text corpus ────────────────────────────────────── ··· 120 206 return parts.join(" "); 121 207 } 122 208 123 - // ─── Carry touchpoint matching ────────────────────────────────────── 209 + // ─── Carry touchpoint matching (D1 records) ──────────────────────── 124 210 125 211 function findCarryTouchpoints( 126 212 island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 127 213 carryData: CarryData, 128 214 ): NewConnection[] { 129 215 const connections: NewConnection[] = []; 130 - const islandText = buildIslandText(island).toLowerCase(); 131 - 132 - // Build a vertex index for fast lookup 133 - const vertexByUri = new Map(island.vertices.map(v => [v.uri, v])); 134 216 135 217 // --- Match citations by URL --- 136 218 for (const cite of carryData.citations) { 137 219 // Direct URL match: citation URL is a vertex URI 138 - if (vertexByUri.has(cite.url)) { 220 + const vertexMatch = island.vertices.find(v => v.uri === cite.url); 221 + if (vertexMatch) { 139 222 connections.push({ 140 223 source: cite.url, 141 224 target: cite.atUri, ··· 149 232 const citeText = `${cite.title} ${cite.takeaway}`.toLowerCase(); 150 233 const bestVertex = findBestVertexMatch(citeText, island.vertices); 151 234 if (bestVertex) { 152 - const overlap = keywordOverlap(citeText, bestVertex.title + " " + (bestVertex.description || "")); 235 + const overlap = keywordOverlap(citeText, `${bestVertex.title} ${bestVertex.description || ""}`); 153 236 if (overlap.length >= 2) { 154 237 connections.push({ 155 238 source: bestVertex.uri, ··· 166 249 const entityText = `${entity.name} ${entity.description || ""}`.toLowerCase(); 167 250 const bestVertex = findBestVertexMatch(entityText, island.vertices); 168 251 if (bestVertex) { 169 - const overlap = keywordOverlap(entityText, bestVertex.title + " " + (bestVertex.description || "")); 252 + const overlap = keywordOverlap(entityText, `${bestVertex.title} ${bestVertex.description || ""}`); 170 253 if (overlap.length >= 2) { 171 254 connections.push({ 172 255 source: bestVertex.uri, ··· 183 266 const reasoningText = `${reasoning.claim} ${reasoning.evidence}`.toLowerCase(); 184 267 const bestVertex = findBestVertexMatch(reasoningText, island.vertices); 185 268 if (bestVertex) { 186 - const overlap = keywordOverlap(reasoningText, bestVertex.title + " " + (bestVertex.description || "")); 269 + const overlap = keywordOverlap(reasoningText, `${bestVertex.title} ${bestVertex.description || ""}`); 187 270 if (overlap.length >= 2) { 188 271 const connType = reasoning.reasoningType?.includes("contradict") 189 272 ? "network.cosmik.connection#contradicts" ··· 198 281 } 199 282 } 200 283 201 - // Deduplicate by (source, target) pair 202 - const seen = new Set<string>(); 203 - return connections.filter(c => { 204 - const key = `${c.source}|${c.target}`; 205 - if (seen.has(key)) return false; 206 - seen.add(key); 207 - return true; 208 - }).slice(0, 20); 284 + return connections; 285 + } 286 + 287 + // ─── Filesystem touchpoint matching (vault + carry notes) ─────────── 288 + 289 + function findFilesystemTouchpoints( 290 + island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 291 + vaultNotes: VaultNote[], 292 + carryNotes: CarryNote[], 293 + ): NewConnection[] { 294 + const connections: NewConnection[] = []; 295 + const islandText = buildIslandText(island).toLowerCase(); 296 + 297 + // Match vault notes to island vertices 298 + for (const note of vaultNotes) { 299 + const noteLower = note.content.toLowerCase(); 300 + const overlap = keywordOverlap(islandText, noteLower); 301 + if (overlap.length < 3) continue; 302 + 303 + // Find the best vertex for this vault note 304 + const bestVertex = findBestVertexMatch(noteLower, island.vertices); 305 + if (bestVertex) { 306 + // Extract title from first heading or first line 307 + const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path; 308 + connections.push({ 309 + source: bestVertex.uri, 310 + target: `vault://${note.path}`, 311 + connectionType: "network.cosmik.connection#annotates", 312 + note: `Vault note: ${title.slice(0, 200)}`, 313 + }); 314 + } 315 + } 316 + 317 + // Match carry notes to island vertices 318 + for (const note of carryNotes) { 319 + const noteLower = note.content.toLowerCase(); 320 + const overlap = keywordOverlap(islandText, noteLower); 321 + if (overlap.length < 2) continue; 322 + 323 + const bestVertex = findBestVertexMatch(noteLower, island.vertices); 324 + if (bestVertex) { 325 + const title = note.content.split("\n").find(l => l.startsWith("#"))?.replace(/^#+\s*/, "") || note.path; 326 + connections.push({ 327 + source: bestVertex.uri, 328 + target: `carry://${note.domain}/${note.path}`, 329 + connectionType: "network.cosmik.connection#relates", 330 + note: `Carry [${note.domain}]: ${title.slice(0, 200)}`, 331 + }); 332 + } 333 + } 334 + 335 + return connections; 209 336 } 210 337 211 338 function findBestVertexMatch( ··· 259 386 function findHotEdges( 260 387 islandText: string, 261 388 island: { vertices: VertexInfo[]; edges: EdgeInfo[] }, 389 + vaultNotes: VaultNote[], 390 + carryNotes: CarryNote[], 262 391 newConnections: NewConnection[], 263 392 ): string[] { 264 393 const highlights: string[] = []; 394 + const lower = islandText.toLowerCase(); 265 395 266 396 // Score each edge by semantic significance 267 397 const edgeScores = new Map<string, number>(); ··· 269 399 for (const edge of island.edges) { 270 400 let score = 0; 271 401 402 + // Check if source or target appears in vault notes 403 + for (const note of vaultNotes) { 404 + const noteLower = note.content.toLowerCase(); 405 + const overlap = keywordOverlap(lower, noteLower); 406 + if (overlap.length >= 2) { 407 + const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || ""; 408 + const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || ""; 409 + if (sourceTitle && noteLower.includes(sourceTitle)) score += 2; 410 + if (targetTitle && noteLower.includes(targetTitle)) score += 2; 411 + } 412 + } 413 + 414 + // Check if source or target appears in carry notes 415 + for (const note of carryNotes) { 416 + const noteLower = note.content.toLowerCase(); 417 + const sourceTitle = island.vertices.find(v => v.uri === edge.source)?.title?.toLowerCase() || ""; 418 + const targetTitle = island.vertices.find(v => v.uri === edge.target)?.title?.toLowerCase() || ""; 419 + if (sourceTitle && noteLower.includes(sourceTitle)) score += 1; 420 + if (targetTitle && noteLower.includes(targetTitle)) score += 1; 421 + } 422 + 272 423 // Edges with notes are more interesting 273 424 if (edge.note && edge.note.length > 10) score += 1; 274 425 ··· 296 447 return highlights; 297 448 } 298 449 299 - function findTensions( 450 + function findTensionsFromNotes( 451 + islandText: string, 452 + carryNotes: CarryNote[], 453 + ): string[] { 454 + const tensions: string[] = []; 455 + const lower = islandText.toLowerCase(); 456 + 457 + for (const note of carryNotes) { 458 + const noteLower = note.content.toLowerCase(); 459 + if (noteLower.includes("but") || noteLower.includes("however") || noteLower.includes("tension") || noteLower.includes("contradicts")) { 460 + const overlap = keywordOverlap(lower, noteLower); 461 + if (overlap.length >= 1) { 462 + tensions.push( 463 + `Carry note "${note.path}" (${note.domain}) notes a tension with this island's themes` 464 + ); 465 + } 466 + } 467 + } 468 + 469 + return tensions.slice(0, 5); 470 + } 471 + 472 + function findTensionsFromReasonings( 300 473 islandText: string, 301 474 reasonings: CarryReasoning[], 302 475 ): string[] { ··· 305 478 306 479 for (const r of reasonings) { 307 480 const rLower = `${r.claim} ${r.evidence}`.toLowerCase(); 308 - // Look for contradiction markers 309 481 if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) { 310 482 const overlap = keywordOverlap(lower, rLower); 311 483 if (overlap.length >= 1) {

+29 -2

container/server.ts

View file

··· 21 21 22 22 // ── Health ── 23 23 if (url.pathname === "/health" && req.method === "GET") { 24 - res.writeHead(200); 25 - res.end("ok"); 24 + // Include data status in health check 25 + const { readdirSync, statSync } = await import("fs"); 26 + const { join } = await import("path"); 27 + let vaultFiles = 0, carryFiles = 0, carryDomains: string[] = []; 28 + try { 29 + const walk = (dir: string): number => { 30 + let count = 0; 31 + for (const e of readdirSync(dir, { withFileTypes: true })) { 32 + if (e.name.startsWith(".")) continue; 33 + if (e.isDirectory()) count += walk(join(dir, e.name)); 34 + else if (e.name.endsWith(".md")) count++; 35 + } 36 + return count; 37 + }; 38 + vaultFiles = walk("/data/vault"); 39 + const carryBase = "/data/carry"; 40 + try { 41 + for (const d of readdirSync(carryBase)) { 42 + if (d.startsWith(".")) continue; 43 + const p = join(carryBase, d); 44 + if (statSync(p).isDirectory()) { 45 + carryDomains.push(d); 46 + carryFiles += walk(p); 47 + } 48 + } 49 + } catch {} 50 + } catch {} 51 + res.writeHead(200, { "Content-Type": "application/json" }); 52 + res.end(JSON.stringify({ status: "ok", vaultFiles, carryFiles, carryDomains, carryTokenSet: !!process.env.CARRY_GITHUB_TOKEN })); 26 53 return; 27 54 } 28 55

+15 -6

container/sync.ts

View file

··· 1 1 // ─── Sync: pull vault + carry into container ────────────────────── 2 2 // 3 - // vault: ob sync (Obsidian Sync) 4 - // carry: git pull from GitHub private repo 3 + // vault: ob sync (Obsidian CLI) 4 + // carry: git pull from GitHub private repo (auth via CARRY_GITHUB_TOKEN) 5 5 6 6 import { execSync } from "child_process"; 7 7 8 8 const VAULT_PATH = process.env.VAULT_PATH || "/data/vault"; 9 9 const CARRY_PATH = process.env.CARRY_PATH || "/data/carry"; 10 - const CARRY_REMOTE = process.env.CARRY_REMOTE || ""; 10 + const CARRY_GITHUB_TOKEN = process.env.CARRY_GITHUB_TOKEN || ""; 11 11 const CARRY_BRANCH = process.env.CARRY_BRANCH || "main"; 12 + const CARRY_REPO = "codegod100/carry-vault"; 13 + 14 + function getCarryRemote(): string { 15 + if (CARRY_GITHUB_TOKEN) { 16 + return `https://${CARRY_GITHUB_TOKEN}@github.com/${CARRY_REPO}.git`; 17 + } 18 + return ""; 19 + } 12 20 13 21 export async function syncVault(): Promise<void> { 14 22 try { ··· 20 28 } 21 29 22 30 export async function syncCarry(): Promise<void> { 23 - if (!CARRY_REMOTE) { 24 - console.error("[sync] CARRY_REMOTE not set, skipping carry sync"); 31 + const remote = getCarryRemote(); 32 + if (!remote) { 33 + console.error("[sync] CARRY_GITHUB_TOKEN not set, skipping carry sync"); 25 34 return; 26 35 } 27 36 ··· 37 46 console.log("[sync] carry pulled via git"); 38 47 } catch { 39 48 // Not a git repo — clone 40 - execSync(`git clone --branch ${CARRY_BRANCH} --single-branch ${CARRY_REMOTE} ${CARRY_PATH}`, { 49 + execSync(`git clone --branch ${CARRY_BRANCH} --single-branch ${remote} ${CARRY_PATH}`, { 41 50 stdio: "pipe", 42 51 timeout: 60000, 43 52 env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },

+17 -9

src/container.ts

View file

··· 1 1 import { Container } from "@cloudflare/containers"; 2 + import type { DurableObject } from "cloudflare:workers"; 2 3 3 - export class StrataContainer extends Container { 4 + interface ContainerEnv { 5 + CARRY_GITHUB_TOKEN: string; 6 + } 7 + 8 + export class StrataContainer extends Container<ContainerEnv> { 4 9 defaultPort = 8080; 5 10 sleepAfter = "5m"; 6 11 7 - // Env vars passed to the container process 8 - env: Record<string, string> = { 9 - VAULT_PATH: "/data/vault", 10 - CARRY_PATH: "/data/carry", 11 - CARRY_REMOTE: "", 12 - CARRY_BRANCH: "main", 13 - }; 12 + constructor(ctx: DurableObject["ctx"], env: ContainerEnv) { 13 + super(ctx, env); 14 + // Set env vars before container starts — inject secret from worker binding 15 + this.envVars = { 16 + VAULT_PATH: "/data/vault", 17 + CARRY_PATH: "/data/carry", 18 + CARRY_BRANCH: "main", 19 + ...(env.CARRY_GITHUB_TOKEN ? { CARRY_GITHUB_TOKEN: env.CARRY_GITHUB_TOKEN } : {}), 20 + }; 21 + } 14 22 15 23 async onStart() { 16 - console.log("[strata-container] started"); 24 + console.log("[strata-container] started, carry token:", this.envVars?.CARRY_GITHUB_TOKEN ? "set" : "missing"); 17 25 } 18 26 19 27 async onStop() {

+12 -1

src/frontend/app.ts

View file

··· 883 883 const res = await fetch("/xrpc/org.latha.strata.analyze", { 884 884 method: "POST", 885 885 headers: { "Content-Type": "application/json" }, 886 - body: JSON.stringify({ island: { vertices: island.vertices, edges: island.edges } }), 886 + body: JSON.stringify({ 887 + did: session?.did, 888 + island: { 889 + vertices: (island.vertices || []).map(uri => ({ 890 + uri, 891 + title: island.vertexMeta?.[uri]?.title || "", 892 + description: island.vertexMeta?.[uri]?.description || "", 893 + type: island.vertexMeta?.[uri]?.type || "url", 894 + })), 895 + edges: island.edges || [], 896 + }, 897 + }), 887 898 }); 888 899 if (!res.ok) throw new Error("Analysis failed"); 889 900 const result = await res.json();

+21

src/worker.ts

View file

··· 15 15 CARRY_BUCKET: R2Bucket; 16 16 STRATA_CONTAINER: DurableObjectNamespace; 17 17 PDS_APP_PASSWORD: string; 18 + CARRY_GITHUB_TOKEN: string; 18 19 } 19 20 20 21 // ─── Contrail setup ───────────────────────────────────────────────── ··· 773 774 throw new Error(`Container returned non-JSON: ${text.slice(0, 500)}`); 774 775 } 775 776 } 777 + 778 + // Sync container data (vault + carry) 779 + app.post("/xrpc/org.latha.strata.syncContainer", async (c) => { 780 + try { 781 + const result = await callContainer("/sync", "POST"); 782 + return c.json(result); 783 + } catch (e: any) { 784 + return c.json({ error: "ContainerError", message: e.message }, 500); 785 + } 786 + }); 787 + 788 + // Container health + data status 789 + app.get("/xrpc/org.latha.strata.containerHealth", async (c) => { 790 + try { 791 + const result = await callContainer("/health", "GET"); 792 + return c.json(result); 793 + } catch (e: any) { 794 + return c.json({ error: "ContainerError", message: e.message }, 500); 795 + } 796 + }); 776 797 777 798 // Derive islands — POST, delegates to container 778 799 // With seed: single island. Without: all islands.

Configure Feed

Configure Feed