nandi.uk
This repository has no description
94 kB
2296 lines
1import { Hono } from "hono";
2import { cors } from "hono/cors";
3import { createWorker } from "@atmo-dev/contrail/worker";
4import { Contrail } from "@atmo-dev/contrail";
5import { config } from "./contrail.config.js";
6import { lexicons } from "../lexicons/generated/index.js";
7import { LANDING_PAGE } from "./landing-page.js";
8// ─── Env ────────────────────────────────────────────────────────────
9
10interface Env {
11 DB: D1Database;
12 ASSETS: { fetch(request: Request): Promise<Response> };
13 VAULT_BUCKET: R2Bucket;
14 CARRY_BUCKET: R2Bucket;
15 LETTA_API_KEY?: string;
16 LETTA_AGENT_ID?: string;
17}
18
19interface CentroidDiscoveryResult {
20 title: string;
21 themes: string[];
22 highlights: string[];
23 tensions: string[];
24 openQuestions: string[];
25 synthesis: string;
26 edges: Array<{ source: string; target: string; connectionType: string; note?: string }>;
27}
28
29// ─── Contrail setup ─────────────────────────────────────────────────
30
31const contrailWorker = createWorker(config, { lexicons });
32const contrail = new Contrail(config);
33let contrailReady = false;
34
35async function ensureContrailReady(db: D1Database): Promise<void> {
36 if (contrailReady) return;
37 await contrail.init(db);
38 contrailReady = true;
39}
40
41async function resolveDidIdentity(did: string): Promise<{ did: string; handle: string | null; pds: string | null } | null> {
42 let doc: any;
43 if (did.startsWith("did:plc:")) {
44 const resp = await fetch(`https://plc.directory/${encodeURIComponent(did)}`);
45 if (!resp.ok) return null;
46 doc = await resp.json();
47 } else if (did.startsWith("did:web:")) {
48 const host = did.slice("did:web:".length).replace(/:/g, "/");
49 const resp = await fetch(`https://${host}/.well-known/did.json`);
50 if (!resp.ok) return null;
51 doc = await resp.json();
52 } else {
53 return null;
54 }
55
56 const aka = Array.isArray(doc.alsoKnownAs) ? doc.alsoKnownAs.find((v: string) => v.startsWith("at://")) : null;
57 const handle = aka ? aka.slice("at://".length) : null;
58 const svc = Array.isArray(doc.service)
59 ? doc.service.find((s: any) => s.type === "AtprotoPersonalDataServer" || s.id === "#atproto_pds")
60 : null;
61 const pds = typeof svc?.serviceEndpoint === "string" ? svc.serviceEndpoint.replace(/\/$/, "") : null;
62 return { did, handle, pds };
63}
64
65// ─── Resolve synthesisDoc AT URI pointers ───────────────────────────
66//
67// synthesisDoc used to be an inline pub.oxa.document object.
68// Now it's an AT URI pointing to a separate pub.oxa.document record.
69// This helper dereferences the pointer so the frontend always gets
70// the resolved OxaRecord object.
71
72const synthesisDocCache = new Map<string, any>();
73
74async function resolveSynthesisDoc(synthesisDoc: any): Promise<any> {
75 if (!synthesisDoc) return null;
76
77 // Already an inline OXA document object (legacy format)
78 if (typeof synthesisDoc === "object" && synthesisDoc.$type === "pub.oxa.document") {
79 return synthesisDoc;
80 }
81
82 // AT URI string — dereference it
83 if (typeof synthesisDoc === "string" && synthesisDoc.startsWith("at://")) {
84 // Check cache
85 if (synthesisDocCache.has(synthesisDoc)) {
86 return synthesisDocCache.get(synthesisDoc);
87 }
88
89 try {
90 // Parse AT URI: at://<did>/<collection>/<rkey>
91 const parts = synthesisDoc.replace("at://", "").split("/");
92 const did = parts[0];
93 const collection = parts[1];
94 const rkey = parts[2];
95 if (!did || !collection || !rkey) return null;
96
97 // Resolve PDS for this DID
98 const identity = await resolveDidIdentity(did);
99 if (!identity?.pds) return null;
100
101 const resp = await fetch(
102 `${identity.pds}/xrpc/com.atproto.repo.getRecord?repo=${did}&collection=${collection}&rkey=${rkey}`
103 );
104 if (!resp.ok) return null;
105
106 const data: any = await resp.json();
107 const doc = data.value;
108 if (doc?.$type === "pub.oxa.document") {
109 synthesisDocCache.set(synthesisDoc, doc);
110 return doc;
111 }
112 } catch {
113 return null;
114 }
115 }
116
117 return null;
118}
119
120// ─── Island detection (connected components) ────────────────────────
121//
122// Load all connections, build an undirected graph, find connected
123// components via BFS. Each component is an "island" — a cluster of
124// linked URLs that form a constellation in the knowledge graph.
125
126interface Island {
127 id: string;
128 centroid: string;
129 vertices: string[];
130 edges: Array<{
131 uri: string;
132 did: string;
133 source: string;
134 target: string;
135 connectionType: string;
136 note: string;
137 handle?: string | null;
138 }>;
139}
140
141/** Compute centroid: vertex with highest closeness centrality (smallest avg BFS distance). */
142function computeCentroid(adj: Map<string, Set<string>>, nodes: Set<string>): string {
143 if (nodes.size <= 1) return [...nodes][0] ?? '';
144
145 let bestNode = '';
146 let bestAvg = Infinity;
147
148 for (const start of nodes) {
149 const dist = new Map<string, number>();
150 dist.set(start, 0);
151 const queue = [start];
152 while (queue.length > 0) {
153 const current = queue.shift()!;
154 for (const neighbor of adj.get(current) || []) {
155 if (!dist.has(neighbor) && nodes.has(neighbor)) {
156 dist.set(neighbor, dist.get(current)! + 1);
157 queue.push(neighbor);
158 }
159 }
160 }
161
162 let totalDist = 0;
163 let reachable = 0;
164 for (const [node, d] of dist) {
165 if (nodes.has(node)) {
166 totalDist += d;
167 reachable++;
168 }
169 }
170
171 const avg = reachable > 0 ? totalDist / reachable : Infinity;
172 if (avg < bestAvg) {
173 bestAvg = avg;
174 bestNode = start;
175 }
176 }
177
178 return bestNode;
179}
180
181/** Hash a URL to a 12-char hex island ID (matches CLI's islandId()). */
182async function islandHash(url: string): Promise<string> {
183 const hash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(url));
184 return Array.from(new Uint8Array(hash)).map(b => b.toString(16).padStart(2, "0")).join("").slice(0, 12);
185}
186
187async function detectIslands(db: D1Database): Promise<Island[]> {
188 await ensureContrailReady(db);
189
190 // Load all connections
191 const rows = await db
192 .prepare(
193 `SELECT r.record, r.did, r.rkey, i.handle
194 FROM records_connection r
195 LEFT JOIN identities i ON r.did = i.did
196 ORDER BY r.time_us DESC`,
197 )
198 .all<{ record: string; did: string; rkey: string; handle: string | null }>();
199
200 // Build adjacency list
201 const adj = new Map<string, Set<string>>();
202 const edges: Island["edges"] = [];
203
204 for (const row of rows.results || []) {
205 try {
206 const value = JSON.parse(row.record);
207 const source = value.source as string;
208 const target = value.target as string;
209 if (!source || !target) continue;
210
211 if (!adj.has(source)) adj.set(source, new Set());
212 if (!adj.has(target)) adj.set(target, new Set());
213 adj.get(source)!.add(target);
214 adj.get(target)!.add(source);
215
216 edges.push({
217 uri: `at://${row.did}/network.cosmik.connection/${row.rkey}`,
218 did: row.did,
219 source,
220 target,
221 connectionType: value.connectionType || "relates",
222 note: value.note || "",
223 handle: row.handle,
224 });
225 } catch {}
226 }
227
228 // BFS to find connected components
229 const visited = new Set<string>();
230 const islands: Island[] = [];
231
232 for (const node of adj.keys()) {
233 if (visited.has(node)) continue;
234
235 const component: string[] = [];
236 const queue = [node];
237 while (queue.length > 0) {
238 const current = queue.shift()!;
239 if (visited.has(current)) continue;
240 visited.add(current);
241 component.push(current);
242
243 for (const neighbor of adj.get(current) || []) {
244 if (!visited.has(neighbor)) queue.push(neighbor);
245 }
246 }
247
248 // Only include edges where both endpoints are in this component
249 const componentSet = new Set(component);
250 const componentEdges = edges.filter(
251 e => componentSet.has(e.source) && componentSet.has(e.target),
252 );
253
254 // Stable ID: truncated SHA-256 of the centroid vertex (highest closeness centrality)
255 const centroid = computeCentroid(adj, componentSet);
256 const id = await islandHash(centroid);
257
258 islands.push({
259 id,
260 centroid,
261 vertices: component,
262 edges: componentEdges,
263 });
264 }
265
266 // Sort by size descending
267 islands.sort((a, b) => b.vertices.length - a.vertices.length);
268 return islands;
269}
270
271// ─── Resolve vertex metadata (batched) ──────────────────────────────
272//
273// Bulk queries instead of N+1 per vertex. Loads all cards and citations
274// matching the given URIs in a few chunked queries.
275
276async function resolveVertexMeta(
277 db: D1Database,
278 uris: string[],
279): Promise<Map<string, { title: string; description: string; type: string }>> {
280 const meta = new Map<string, { title: string; description: string; type: string }>();
281 for (const uri of uris) {
282 meta.set(uri, { title: uri, description: "", type: "unknown" });
283 }
284
285 const httpUris = uris.filter(u => u.startsWith("http"));
286 const atUris = uris.filter(u => u.startsWith("at://"));
287 const CHUNK = 50;
288
289 // Build all queries upfront, then batch-execute in one D1 round trip
290 const stmts: D1PreparedStatement[] = [];
291
292 // HTTP URI queries — cards
293 const httpChunks: string[][] = [];
294 for (let i = 0; i < httpUris.length; i += CHUNK) {
295 const chunk = httpUris.slice(i, i + CHUNK);
296 httpChunks.push(chunk);
297 const placeholders = chunk.map(() => "?").join(",");
298 stmts.push(
299 db.prepare(
300 `SELECT record FROM records_card WHERE json_extract(record, '$.content.url') IN (${placeholders})`
301 ).bind(...chunk)
302 );
303 }
304
305 // HTTP URI queries — citations
306 for (let i = 0; i < httpUris.length; i += CHUNK) {
307 const chunk = httpUris.slice(i, i + CHUNK);
308 const placeholders = chunk.map(() => "?").join(",");
309 stmts.push(
310 db.prepare(
311 `SELECT record FROM records_citation WHERE json_extract(record, '$.url') IN (${placeholders})`
312 ).bind(...chunk)
313 );
314 }
315
316 // AT URI queries — cards
317 const atChunks: string[][] = [];
318 for (let i = 0; i < atUris.length; i += CHUNK) {
319 const chunk = atUris.slice(i, i + CHUNK);
320 atChunks.push(chunk);
321 const placeholders = chunk.map(() => "?").join(",");
322 stmts.push(
323 db.prepare(`SELECT uri, record FROM records_card WHERE uri IN (${placeholders})`).bind(...chunk)
324 );
325 }
326
327 // AT URI queries — collections
328 for (let i = 0; i < atUris.length; i += CHUNK) {
329 const chunk = atUris.slice(i, i + CHUNK);
330 const placeholders = chunk.map(() => "?").join(",");
331 stmts.push(
332 db.prepare(`SELECT uri, record FROM records_collection WHERE uri IN (${placeholders})`).bind(...chunk)
333 );
334 }
335
336 // Execute all queries in one batch
337 const results = stmts.length > 0 ? await db.batch(stmts) : [];
338 let resultIdx = 0;
339
340 // Process card results (HTTP)
341 for (const chunk of httpChunks) {
342 const rows = (results[resultIdx++] as { results: { record: string }[] } | null)?.results || [];
343 for (const row of rows) {
344 try {
345 const card = JSON.parse(row.record);
346 const url = card?.content?.url;
347 if (!url || !meta.has(url)) continue;
348 const m = card.content.metadata;
349 if (m?.title || m?.description) {
350 meta.set(url, { title: m.title || url, description: m.description || "", type: card.type === "URL" ? "url" : "note" });
351 }
352 } catch {}
353 }
354 }
355
356 // Process citation results (HTTP) — overrides card data
357 for (let i = 0; i < httpChunks.length; i++) {
358 const rows = (results[resultIdx++] as { results: { record: string }[] } | null)?.results || [];
359 for (const row of rows) {
360 try {
361 const citation = JSON.parse(row.record);
362 const url = citation.url;
363 if (!url || !meta.has(url)) continue;
364 if (citation.title) {
365 const existing = meta.get(url)!;
366 meta.set(url, { title: citation.title, description: citation.takeaway || existing.description, type: "citation" });
367 }
368 } catch {}
369 }
370 }
371
372 // Process card results (AT)
373 for (const chunk of atChunks) {
374 const rows = (results[resultIdx++] as { results: { uri: string; record: string }[] } | null)?.results || [];
375 for (const row of rows) {
376 try {
377 const card = JSON.parse(row.record);
378 const m = card?.content?.metadata;
379 const url = card?.content?.url;
380 if (meta.has(row.uri)) {
381 meta.set(row.uri, { title: m?.title || url || row.uri, description: m?.description || "", type: "card" });
382 }
383 } catch {}
384 }
385 }
386
387 // Process collection results (AT)
388 for (let i = 0; i < atChunks.length; i++) {
389 const rows = (results[resultIdx++] as { results: { uri: string; record: string }[] } | null)?.results || [];
390 for (const row of rows) {
391 try {
392 const coll = JSON.parse(row.record);
393 if (meta.has(row.uri)) {
394 meta.set(row.uri, { title: coll.name || row.uri, description: coll.description || "", type: "collection" });
395 }
396 } catch {}
397 }
398 }
399
400 return meta;
401}
402
403// ─── Derive island from centroid ────────────────────────────────────
404//
405// Given a centroid vertex (canonical component reference), derive the
406// connected component by traversing network.cosmik.connection records.
407
408async function deriveIsland(
409 db: D1Database,
410 seed: string,
411): Promise<Island | null> {
412 await ensureContrailReady(db);
413
414 // BFS in application code — recursive CTE OOMs in D1 due to cross join
415 const visited = new Set<string>();
416 const queue = [seed];
417 const allEdges: Array<{
418 uri: string; did: string; rkey: string;
419 source: string; target: string; connectionType: string;
420 note: string; handle: string | null;
421 }> = [];
422
423 while (queue.length > 0) {
424 // Drain the current queue into a batch
425 const batch = queue.splice(0, 50);
426 const toQuery = batch.filter(v => !visited.has(v));
427 if (toQuery.length === 0) continue;
428
429 // Mark visited
430 for (const v of toQuery) visited.add(v);
431
432 // Query connections where any of these vertices are source or target
433 const placeholders = toQuery.map(() => "?").join(",");
434 const rows = await db
435 .prepare(
436 `SELECT r.record, r.did, r.rkey, i.handle
437 FROM records_connection r
438 LEFT JOIN identities i ON r.did = i.did
439 WHERE json_extract(r.record, '$.source') IN (${placeholders})
440 OR json_extract(r.record, '$.target') IN (${placeholders})`,
441 )
442 .bind(...toQuery, ...toQuery)
443 .all<{ record: string; did: string; rkey: string; handle: string | null }>();
444
445 for (const row of rows.results || []) {
446 try {
447 const value = JSON.parse(row.record);
448 const source = value.source as string;
449 const target = value.target as string;
450 if (!source || !target) continue;
451
452 // Collect edge
453 const edgeKey = `${source}|${target}|${row.rkey}`;
454 allEdges.push({
455 uri: `at://${row.did}/network.cosmik.connection/${row.rkey}`,
456 did: row.did,
457 rkey: row.rkey,
458 source,
459 target,
460 connectionType: value.connectionType || "relates",
461 note: value.note || "",
462 handle: row.handle,
463 });
464
465 // Enqueue unvisited neighbors
466 if (!visited.has(source)) queue.push(source);
467 if (!visited.has(target)) queue.push(target);
468 } catch {}
469 }
470 }
471
472 if (visited.size === 0) return null;
473
474 const vertices = [...visited];
475
476 // Filter edges to only those with both endpoints in the component
477 const vertexSet = new Set(vertices);
478 const edges: Island["edges"] = [];
479 const seenEdges = new Set<string>();
480 for (const e of allEdges) {
481 if (!vertexSet.has(e.source) || !vertexSet.has(e.target)) continue;
482 const edgeKey = `${e.source}|${e.target}|${e.rkey}`;
483 if (seenEdges.has(edgeKey)) continue;
484 seenEdges.add(edgeKey);
485 edges.push({
486 uri: e.uri,
487 did: e.did,
488 source: e.source,
489 target: e.target,
490 connectionType: e.connectionType,
491 note: e.note,
492 handle: e.handle,
493 });
494 }
495
496 // Compute stable ID from centroid (highest closeness centrality)
497 const adj = new Map<string, Set<string>>();
498 for (const e of edges) {
499 if (!adj.has(e.source)) adj.set(e.source, new Set());
500 if (!adj.has(e.target)) adj.set(e.target, new Set());
501 adj.get(e.source)!.add(e.target);
502 adj.get(e.target)!.add(e.source);
503 }
504 const centroid = computeCentroid(adj, vertexSet);
505 const id = await islandHash(centroid);
506
507 return { id, centroid, vertices, edges };
508}
509
510// ─── List islands from records_island ───────────────────────────
511
512// ─── Resolve edge URIs to full edge objects ──────────────────────
513
514/** Given a list of connections (AT URI strings, legacy {uri}, or legacy inline objects), resolve URIs to full edges from D1. */
515async function resolveEdgeUris(db: D1Database, connections: any[]): Promise<any[]> {
516 const resolved: any[] = [];
517 const uriIndices: Map<string, number[]> = new Map(); // uri → indices in connections that need it
518
519 for (let i = 0; i < connections.length; i++) {
520 const conn = connections[i];
521 if (typeof conn === "string") {
522 resolved.push(null);
523 if (!uriIndices.has(conn)) uriIndices.set(conn, []);
524 uriIndices.get(conn)!.push(i);
525 continue;
526 }
527 // Already has source/target — use as-is
528 if (typeof conn === "object" && conn.source && conn.target) {
529 resolved.push(conn);
530 continue;
531 }
532 // Just a URI reference — needs lookup
533 const uri = conn?.uri;
534 if (!uri) continue;
535 resolved.push(null); // placeholder
536 if (!uriIndices.has(uri)) uriIndices.set(uri, []);
537 uriIndices.get(uri)!.push(i);
538 }
539
540 // Batch-fetch connection records from D1 (for URI-only references)
541 if (uriIndices.size > 0) {
542 const uris = [...uriIndices.keys()];
543 const CHUNK = 100;
544 for (let i = 0; i < uris.length; i += CHUNK) {
545 const chunk = uris.slice(i, i + CHUNK);
546 const placeholders = chunk.map(() => "?").join(",");
547 const rows = await db
548 .prepare(
549 `SELECT r.uri, r.record, r.did, r.rkey, i.handle
550 FROM records_connection r
551 LEFT JOIN identities i ON r.did = i.did
552 WHERE r.uri IN (${placeholders})`,
553 )
554 .bind(...chunk)
555 .all<{ uri: string; record: string; did: string; rkey: string; handle: string | null }>();
556
557 for (const row of rows.results || []) {
558 try {
559 const value = JSON.parse(row.record);
560 const edge = {
561 uri: row.uri,
562 did: row.did,
563 source: value.source,
564 target: value.target,
565 connectionType: value.connectionType || "relates",
566 note: value.note || "",
567 handle: row.handle,
568 };
569 for (const idx of uriIndices.get(row.uri) || []) {
570 resolved[idx] = edge;
571 }
572 } catch {}
573 }
574 }
575 }
576
577 // Resolve handles for inline edges (those with source/target but no did/handle)
578 // Extract DID from AT URI (at://did:plc:.../collection/rkey) and batch-lookup handles
579 const inlineEdges = resolved.filter((e: any) => e && !e.did);
580 if (inlineEdges.length > 0) {
581 const dids = new Set<string>();
582 for (const edge of inlineEdges) {
583 const uri = (edge as any).uri as string | undefined;
584 if (!uri) continue;
585 const match = uri.match(/^at:\/\/(did:[^/]+)\//);
586 if (match) dids.add(match[1]);
587 }
588 if (dids.size > 0) {
589 const ph = [...dids].map(() => "?").join(",");
590 const rows = await db
591 .prepare(`SELECT did, handle FROM identities WHERE did IN (${ph})`)
592 .bind(...dids)
593 .all<{ did: string; handle: string | null }>();
594 const handleMap = new Map<string, string | null>();
595 for (const row of rows.results || []) {
596 handleMap.set(row.did, row.handle);
597 }
598 for (const edge of inlineEdges) {
599 const uri = (edge as any).uri as string | undefined;
600 if (!uri) continue;
601 const match = uri.match(/^at:\/\/(did:[^/]+)\//);
602 if (match) {
603 const did = match[1];
604 (edge as any).did = did;
605 (edge as any).handle = handleMap.get(did) ?? null;
606 }
607 }
608 }
609 }
610
611 return resolved.filter(Boolean);
612}
613
614/** Resolve at://inline:* URIs by looking up the forkOf record's inline edge data */
615async function resolveInlineFromForkOf(db: D1Database, record: any): Promise<any[]> {
616 const connections: any[] = Array.isArray(record.connections) ? record.connections : [];
617 const hasInlineUris = connections.some(c => typeof c === "string" && c.startsWith("at://inline:"));
618 if (!hasInlineUris || !record.forkOf) return [];
619
620 // Look up the forkOf record
621 const forkRow = await db
622 .prepare("SELECT record FROM records_island WHERE uri = ?")
623 .bind(record.forkOf)
624 .first<{ record: string }>();
625 if (!forkRow) return [];
626
627 const forkRec = JSON.parse(forkRow.record);
628 const forkConnections: any[] = Array.isArray(forkRec.connections) ? forkRec.connections : [];
629
630 // Build a map from inline URI → edge data
631 const inlineMap = new Map<string, any>();
632 for (const conn of forkConnections) {
633 if (typeof conn === "object" && conn.uri && conn.source && conn.target) {
634 inlineMap.set(conn.uri, conn);
635 }
636 }
637
638 // Resolve the current record's connections using the fork's inline data
639 const resolved: any[] = [];
640 for (const conn of connections) {
641 if (typeof conn === "string" && inlineMap.has(conn)) {
642 const edge = inlineMap.get(conn);
643 resolved.push({
644 uri: conn,
645 source: edge.source,
646 target: edge.target,
647 connectionType: edge.connectionType || "relates",
648 note: edge.note || "",
649 });
650 }
651 }
652 return resolved;
653}
654
655async function findSimilarIslands(db: D1Database, currentUri: string, currentVertices: Set<string>, currentConnUris?: Set<string>): Promise<any[]> {
656 if (currentVertices.size === 0) return [];
657 const domainOf = (url: string): string | null => {
658 try {
659 const u = new URL(url);
660 return u.hostname.replace(/^www\./, "").toLowerCase();
661 } catch { return null; }
662 };
663 const currentDomains = new Set([...currentVertices].map(domainOf).filter(Boolean) as string[]);
664 const rows = await db
665 .prepare("SELECT uri, record FROM records_island WHERE uri != ? AND uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT 500")
666 .bind(currentUri)
667 .all<{ uri: string; record: string }>();
668
669 // Phase 1: Fast pre-filter using connection URI overlap (no edge resolution)
670 const candidates: { uri: string; record: any; connOverlap: number; centroid: string | null }[] = [];
671 for (const row of rows.results || []) {
672 try {
673 const rec = JSON.parse(row.record);
674 const connections: any[] = Array.isArray(rec.connections) ? rec.connections : [];
675 const candidateConnUris = new Set(connections.filter(c => typeof c === "string"));
676 const centroid = rec.source?.uri || null;
677
678 // Fast overlap: shared connection AT URIs
679 let connOverlap = 0;
680 if (currentConnUris && currentConnUris.size > 0) {
681 for (const uri of candidateConnUris) {
682 if (currentConnUris.has(uri)) connOverlap++;
683 }
684 }
685
686 // Quick centroid/domain check for islands with no connection overlap
687 if (connOverlap === 0 && centroid) {
688 const centroidDomain = domainOf(centroid);
689 if (centroidDomain && currentDomains.has(centroidDomain)) connOverlap = -1; // domain match, low priority
690 }
691
692 if (connOverlap !== 0 || (centroid && currentVertices.has(centroid))) {
693 candidates.push({ uri: row.uri, record: rec, connOverlap, centroid });
694 }
695 } catch {}
696 }
697
698 // Phase 2: Resolve edges only for top candidates (max 20)
699 candidates.sort((a, b) => Math.abs(b.connOverlap) - Math.abs(a.connOverlap));
700 const topCandidates = candidates.slice(0, 20);
701
702 const similar: any[] = [];
703 for (const candidate of topCandidates) {
704 try {
705 const rec = candidate.record;
706 const connections: any[] = Array.isArray(rec.connections) ? rec.connections : [];
707 const edges = await resolveEdgeUris(db, connections);
708 const vertices = new Set<string>();
709 const centroid = rec.source?.uri || null;
710 if (centroid) vertices.add(centroid);
711 for (const edge of edges) {
712 if (edge?.source) vertices.add(edge.source);
713 if (edge?.target) vertices.add(edge.target);
714 }
715 const sharedVertices = [...vertices].filter(v => currentVertices.has(v));
716 const domains = new Set([...vertices].map(domainOf).filter(Boolean) as string[]);
717 const sharedDomains = [...domains].filter(d => currentDomains.has(d));
718 if (sharedVertices.length === 0 && sharedDomains.length < 2) continue;
719 const did = candidate.uri.split("/")[2];
720 const rkey = candidate.uri.split("/").pop() || candidate.uri;
721 similar.push({
722 id: rkey,
723 recordUri: candidate.uri,
724 handle: did === "did:plc:3kkhul7jznlb6ba7rprzawnj" ? "researcher.pds.latha.org" : did === "did:plc:ngokl2gnmpbvuvrfckja3g7p" ? "nandi.uk" : null,
725 title: rec.analysis?.title || null,
726 centroid,
727 sharedCount: sharedVertices.length,
728 sharedDomainCount: sharedDomains.length,
729 score: sharedVertices.length * 10 + sharedDomains.length,
730 vertexCount: vertices.size,
731 edgeCount: connections.length,
732 sharedVertices: sharedVertices.slice(0, 8),
733 sharedDomains: sharedDomains.slice(0, 8),
734 });
735 } catch {}
736 }
737
738 similar.sort((a, b) => (b.score - a.score) || (b.edgeCount - a.edgeCount));
739 return similar.slice(0, 6);
740}
741
742async function discoverCentroidWithLetta(env: Env, centroid: string): Promise<CentroidDiscoveryResult> {
743 const apiKey = env.LETTA_API_KEY;
744 const agentId = env.LETTA_AGENT_ID || "agent-bc21f9dc-83b1-4826-b9af-999f7c6b6053";
745 if (!apiKey) throw new Error("LETTA_API_KEY is not configured");
746
747 const prompt = `You are creating a stigmergic island from a centroid URL.
748
749Centroid: ${centroid}
750
751Use web research to discover 3-7 directly related HTTPS URLs. Prefer canonical sources: cited papers, project pages, docs, author pages, references, follow-up posts, or explanatory articles. Do not use broad domain matches. Every edge must use the centroid as source or target unless a cited/reference relationship is clearer. Then synthesize the island.
752
753Return ONLY compact JSON with this exact shape:
754{
755 "title": "short island title",
756 "themes": ["theme1", "theme2"],
757 "highlights": [],
758 "tensions": ["structural tension"],
759 "openQuestions": ["question?"],
760 "synthesis": "2-4 concise paragraphs explaining what binds the island",
761 "edges": [
762 {"source":"${centroid}","target":"https://example.com/real-related-url","connectionType":"network.cosmik.connection#related","note":"why this edge belongs"}
763 ]
764}
765
766Rules: JSON only, no markdown fences. Source/target must be canonical HTTPS URLs. Include at least one edge. Do not invent URLs.`;
767
768 const res = await fetch(`https://api.letta.com/v1/agents/${agentId}/messages`, {
769 method: "POST",
770 headers: {
771 "Authorization": `Bearer ${apiKey}`,
772 "Content-Type": "application/json",
773 },
774 body: JSON.stringify({ messages: [{ role: "user", content: prompt }] }),
775 });
776 if (!res.ok) throw new Error(`Letta API failed: ${res.status} ${await res.text()}`);
777
778 const data = await res.json() as { messages?: Array<{ message_type?: string; content?: string }> };
779 const content = [...(data.messages || [])].reverse().find(m => m.message_type === "assistant_message" && m.content)?.content;
780 if (!content) throw new Error("Letta API returned no assistant content");
781
782 const jsonText = content.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "");
783 const parsed = JSON.parse(jsonText) as CentroidDiscoveryResult;
784 const edges = (parsed.edges || [])
785 .filter(e => e.source?.startsWith("https://") && e.target?.startsWith("https://"))
786 .slice(0, 10);
787 if (edges.length === 0) throw new Error("Letta discovery returned no usable HTTPS edges");
788
789 return {
790 title: parsed.title || centroid,
791 themes: Array.isArray(parsed.themes) ? parsed.themes.slice(0, 8) : ["web-discovery"],
792 highlights: [],
793 tensions: Array.isArray(parsed.tensions) ? parsed.tensions.slice(0, 3) : [],
794 openQuestions: Array.isArray(parsed.openQuestions) ? parsed.openQuestions.slice(0, 5) : [],
795 synthesis: parsed.synthesis || `Island seeded from ${centroid}.`,
796 edges,
797 };
798}
799
800function countVerticesFromConnections(connections: any[], centroid?: string | null): number {
801 const vertices = new Set<string>();
802 if (centroid) vertices.add(centroid);
803 for (const conn of connections || []) {
804 if (conn?.source) vertices.add(conn.source);
805 if (conn?.target) vertices.add(conn.target);
806 }
807 return vertices.size;
808}
809
810async function loadHomepageIslands(db: D1Database, pageNum: number, pageSize: number): Promise<{ islands: any[]; total: number }> {
811 const rows = await db
812 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%'")
813 .all<{ uri: string; record: string }>();
814
815 // Phase 1: Parse all records and sort by edge count (no edge resolution)
816 const all: any[] = [];
817 for (const row of rows.results || []) {
818 try {
819 const rec = JSON.parse(row.record);
820 const centroid = rec.source?.uri;
821 if (!centroid) continue;
822 const id = row.uri.split("/").pop() || row.uri;
823 const did = row.uri.split("/")[2];
824 const connections: any[] = Array.isArray(rec.connections) ? rec.connections : [];
825 const edgeCount = connections.length;
826 const themes = Array.isArray(rec.analysis?.themes) ? rec.analysis.themes : [];
827
828 all.push({ row, rec, id, did, centroid, edgeCount, themes });
829 } catch {}
830 }
831
832 all.sort((a, b) => (b.edgeCount - a.edgeCount) || (a.id.localeCompare(b.id)));
833 const selected = all.slice((pageNum - 1) * pageSize, pageNum * pageSize);
834
835 // Phase 2: Build island summaries with graph SVG URLs
836 const islands = selected.map((item) => ({
837 id: item.id,
838 centroid: item.centroid,
839 title: item.rec.analysis?.title || null,
840 summary: item.rec.analysis?.title || null,
841 recordUri: item.row.uri,
842 handle: item.did === "did:plc:3kkhul7jznlb6ba7rprzawnj" ? "researcher.pds.latha.org" : item.did === "did:plc:ngokl2gnmpbvuvrfckja3g7p" ? "nandi.uk" : null,
843 vertexCount: 0,
844 edgeCount: item.edgeCount,
845 graphSvgUrl: `/api/graph/${encodeURIComponent(item.id)}`,
846 strata: item.rec.analysis?.title || item.themes.length ? { title: item.rec.analysis?.title || "", themes: item.themes, highlights: [], tensions: [], open_questions: [], synthesis: "", prose: "" } : null,
847 }));
848
849 return { islands, total: all.length };
850}
851
852// ── Server-side graph SVG generator ────────────────────────────────
853
854function generateGraphSVG(vertices: string[], edges: Array<{ source: string; target: string; connectionType: string }>, width = 400, height = 300): string {
855 if (vertices.length === 0) return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"></svg>`;
856
857 // Simple force-directed layout (pure JS, no canvas)
858 const cx = width / 2;
859 const cy = height / 2;
860 const r = Math.min(width, height) * 0.35;
861 const nodeMap = new Map<string, { x: number; y: number; vx: number; vy: number; id: string }>();
862
863 // Initialize nodes in a circle
864 for (let i = 0; i < vertices.length; i++) {
865 const angle = (2 * Math.PI * i) / vertices.length;
866 nodeMap.set(vertices[i], {
867 x: cx + Math.cos(angle) * r * 0.5,
868 y: cy + Math.sin(angle) * r * 0.5,
869 vx: 0, vy: 0, id: vertices[i],
870 });
871 }
872
873 // Run simulation (200 ticks)
874 let alpha = 1;
875 for (let tick = 0; tick < 200; tick++) {
876 alpha *= 0.99;
877 const nodes = [...nodeMap.values()];
878
879 // Center gravity
880 for (const n of nodes) {
881 n.vx += (cx - n.x) * 0.01 * alpha;
882 n.vy += (cy - n.y) * 0.01 * alpha;
883 }
884
885 // Repulsion (sampled for large graphs)
886 const repulsionSamples = Math.min(nodes.length, 50);
887 for (let i = 0; i < repulsionSamples; i++) {
888 for (let j = i + 1; j < repulsionSamples; j++) {
889 const a = nodes[i], b = nodes[j];
890 let dx = b.x - a.x, dy = b.y - a.y;
891 let dist = Math.sqrt(dx * dx + dy * dy) || 1;
892 const force = -300 * alpha / (dist * dist);
893 const fx = (dx / dist) * force, fy = (dy / dist) * force;
894 a.vx += fx; a.vy += fy;
895 b.vx -= fx; b.vy -= fy;
896 }
897 }
898
899 // Attraction (edges)
900 for (const e of edges) {
901 const a = nodeMap.get(e.source), b = nodeMap.get(e.target);
902 if (!a || !b) continue;
903 let dx = b.x - a.x, dy = b.y - a.y;
904 let dist = Math.sqrt(dx * dx + dy * dy) || 1;
905 const force = (dist - 60) * 0.05 * alpha;
906 const fx = (dx / dist) * force, fy = (dy / dist) * force;
907 a.vx += fx; a.vy += fy;
908 b.vx -= fx; b.vy -= fy;
909 }
910
911 // Apply velocity with damping
912 for (const n of nodes) {
913 n.vx *= 0.6; n.vy *= 0.6;
914 n.x += n.vx; n.y += n.vy;
915 const pad = 20;
916 n.x = Math.max(pad, Math.min(width - pad, n.x));
917 n.y = Math.max(pad, Math.min(height - pad, n.y));
918 }
919 }
920
921 // Fit to view
922 const nodes = [...nodeMap.values()];
923 let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
924 for (const n of nodes) {
925 if (n.x < minX) minX = n.x; if (n.y < minY) minY = n.y;
926 if (n.x > maxX) maxX = n.x; if (n.y > maxY) maxY = n.y;
927 }
928 const graphW = (maxX - minX) || 1, graphH = (maxY - minY) || 1;
929 const graphCX = (minX + maxX) / 2, graphCY = (minY + maxY) / 2;
930 const pad = 30;
931 const scale = Math.min((width - pad * 2) / graphW, (height - pad * 2) / graphH, 2.0);
932 for (const n of nodes) {
933 n.x = cx + (n.x - graphCX) * scale;
934 n.y = cy + (n.y - graphCY) * scale;
935 }
936
937 // Edge colors (matches frontend)
938 const edgeColors: Record<string, string> = {
939 relates: "#8b949e", related: "#8b949e", cites: "#58a6ff", contains: "#3fb950",
940 compares: "#d2a8ff", extends: "#f0883e", contradicts: "#f85149",
941 inspired: "#e3b341", helpful: "#3fb950", grounds: "#79c0ff",
942 annotates: "#d2a8ff", forks: "#f0883e", coauthor: "#a5d6ff",
943 };
944
945 // Build SVG
946 const edgeSvgs: string[] = [];
947 for (const e of edges) {
948 const a = nodeMap.get(e.source), b = nodeMap.get(e.target);
949 if (!a || !b) continue;
950 const color = edgeColors[e.connectionType?.replace(/^.*#/, "").toLowerCase()] || "#8b949e";
951 edgeSvgs.push(`<line x1="${a.x.toFixed(1)}" y1="${a.y.toFixed(1)}" x2="${b.x.toFixed(1)}" y2="${b.y.toFixed(1)}" stroke="${color}" stroke-opacity="0.3" stroke-width="1"/>`);
952 }
953
954 const nodeSvgs: string[] = [];
955 const radius = vertices.length <= 12 ? 3 : vertices.length <= 30 ? 2.5 : vertices.length <= 80 ? 1.8 : vertices.length <= 200 ? 1.4 : 1.2;
956 for (const n of nodes) {
957 nodeSvgs.push(`<circle cx="${n.x.toFixed(1)}" cy="${n.y.toFixed(1)}" r="${radius.toFixed(1)}" fill="#58a6ff" fill-opacity="0.8" stroke="#58a6ff" stroke-width="0.5"/>`);
958 }
959
960 return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" style="background:#0d1117">${edgeSvgs.join("")}${nodeSvgs.join("")}</svg>`;
961}
962
963// ─── Hono app ───────────────────────────────────────────────────────
964
965let app: Hono<{ Bindings: Env }> | null = null;
966
967function buildApp(env: Env): Hono<{ Bindings: Env }> {
968 const app = new Hono<{ Bindings: Env }>();
969 const db = env.DB;
970
971 app.use("*", cors());
972
973 // Landing page — strata records feed
974 app.get("/", async (c) => {
975 // Load only the fields we need — records are 100KB+ with embedded connections
976 // No ensureContrailReady needed — just D1
977 let islandsJson = "[]";
978 try {
979 const pageNum = Math.max(1, parseInt(c.req.query("page") || "1"));
980 const pageSize = Math.min(20, Math.max(1, parseInt(c.req.query("limit") || "10")));
981 const { islands, total } = await loadHomepageIslands(db, pageNum, pageSize);
982 islandsJson = JSON.stringify({ islands, page: pageNum, pageSize, total });
983 } catch (e) {
984 console.error("Failed to load islands:", e);
985 }
986
987 const page = LANDING_PAGE.replace(
988 "</head>",
989 `<script>window.__ISLANDS__=${islandsJson};</script></head>`,
990 );
991 return c.html(page);
992 });
993
994 app.get("/agents", (c) => {
995 const docs = `
996 <div class="agent-docs">
997 <a href="/" class="back-link">← Explore</a>
998 <h2>Agent Guide: Creating Stigmergic Islands</h2>
999 <p>Stigmergic is an AT Protocol appview for publishing and browsing knowledge-graph islands. An island is a compact, inspectable cluster of source URLs connected by typed edges and accompanied by a short synthesis.</p>
1000
1001 <div class="callout">
1002 <strong>Core invariant:</strong> islands point to actual <code>network.cosmik.connection</code> records. Do not put new edge data directly inside an island record. First publish real connection records, then include their AT URIs as strings in <code>connections</code>.
1003 </div>
1004
1005 <h2>Concepts</h2>
1006 <div class="grid">
1007 <div class="card"><h3>Island</h3><p>An <code>org.latha.island</code> record: centroid URL, connections, analysis, timestamps.</p></div>
1008 <div class="card"><h3>Centroid</h3><p>The seed/canonical URL for the island. The record rkey should be the first 12 hex chars of <code>sha256(centroid)</code>.</p></div>
1009 <div class="card"><h3>Connection</h3><p>An existing <code>network.cosmik.connection</code> record between two HTTPS URLs. Island <code>connections</code> is an array of AT URI strings for those records.</p></div>
1010 <div class="card"><h3>Synthesis</h3><p>Short-form abstract (≤2000 graphemes) in <code>analysis.synthesis</code>. Full rich-text synthesis lives in a separate <code>pub.oxa.document</code> record, linked by AT URI in <code>analysis.synthesisDoc</code>.</p></div>
1011 </div>
1012
1013 <h2>Record format</h2>
1014 <pre><code>{
1015 "$type": "org.latha.island",
1016 "source": {
1017 "uri": "https://example.org/canonical-source",
1018 "collection": "network.cosmik.connection"
1019 },
1020 "connections": [
1021 "at://did:plc:.../network.cosmik.connection/<connection-rkey>"
1022 ],
1023 "analysis": {
1024 "title": "Short descriptive title",
1025 "themes": ["theme"],
1026 "highlights": ["at://did:plc:.../network.cosmik.connection/<connection-rkey>"],
1027 "tensions": ["A real interpretive tension."],
1028 "openQuestions": ["A question this island raises?"],
1029 "synthesis": "Short abstract (≤2000 graphemes).",
1030 "synthesisDoc": "at://did:plc:.../pub.oxa.document/<doc-rkey>"
1031 },
1032 "createdAt": "2026-05-22T00:00:00.000Z",
1033 "updatedAt": "2026-05-22T00:00:00.000Z"
1034}</code></pre>
1035 <p>The <code>synthesis</code> field is a short plain-text abstract capped at 2000 graphemes. For the full rich-text synthesis (with headings, links, formatting), publish a <code>pub.oxa.document</code> record and put its AT URI in <code>synthesisDoc</code>. Stigmergic resolves the pointer server-side and renders the OXA document when present.</p>
1036
1037 <h2>Edge conventions</h2>
1038 <ul>
1039 <li><strong>Publish connections first.</strong> Each edge belongs in its own <code>network.cosmik.connection</code> record on a PDS.</li>
1040 <li><strong>Island connections are pointers.</strong> The island record's <code>connections</code> array should contain AT URI strings for those connection records.</li>
1041 <li><strong>Connection endpoints must be canonical HTTPS URLs.</strong> Inside the connection record, do not use AT URIs as <code>source</code> or <code>target</code>.</li>
1042 <li><strong>Use typed relations in connection records.</strong> Uppercase snake-case: <code>INFORMS</code>, <code>SUPPORTS</code>, <code>ANNOTATES</code>, <code>DERIVED_FROM</code>, <code>CONTAINS</code>, <code>RELATED</code>.</li>
1043 <li><strong>Notes matter.</strong> A connection record's <code>note</code> should explain the local reason for the edge, not summarize the whole island.</li>
1044 </ul>
1045
1046 <h2>How to create an island</h2>
1047 <ol>
1048 <li>Choose a centroid URL: the most central or canonical source.</li>
1049 <li>Compute <code>rkey = sha256(centroid).slice(0, 12)</code>.</li>
1050 <li>Gather 5–20 real sources. Use canonical URLs: DOI pages, publisher pages, project pages, documentation, primary artifacts.</li>
1051 <li>Publish <code>network.cosmik.connection</code> records for centroid edges and cross-edges. A useful island usually points to more connection records than it has vertices.</li>
1052 <li>Write analysis: title, themes, tensions, open questions, and a short synthesis abstract (≤2000 graphemes).</li>
1053 <li>If the synthesis is long, publish a <code>pub.oxa.document</code> record with the full rich-text synthesis and put its AT URI in <code>analysis.synthesisDoc</code>.</li>
1054 <li>Publish the island via <code>com.atproto.repo.putRecord</code> to collection <code>org.latha.island</code> using the centroid-hash rkey and the connection AT URIs.</li>
1055 <li>Notify this appview to ingest the update: <code>POST /xrpc/org.latha.island.notifyOfUpdate</code> with the publishing DID.</li>
1056 </ol>
1057
1058 <h2>Getting a PDS account on Rookery</h2>
1059 <p>The public agent PDS at <code>https://pds.latha.org</code> uses Rookery and the WelcomeMat enrollment protocol. Start with its discovery document:</p>
1060 <pre><code>GET https://pds.latha.org/.well-known/welcome.md</code></pre>
1061 <p>Rookery enrollment requires an RSA-4096 keypair. Keep the private key: it is your account key for later writes, and the operator cannot recover it. The flow is:</p>
1062 <ol>
1063 <li>Generate an RSA-4096 keypair and JWK thumbprint.</li>
1064 <li>Fetch <code>GET https://pds.latha.org/tos</code>.</li>
1065 <li>Sign the ToS text with your private key.</li>
1066 <li>Build a WelcomeMat <code>wm+jwt</code> token and DPoP proof as described in <code>/.well-known/welcome.md</code>.</li>
1067 <li>Call <code>POST https://pds.latha.org/api/signup</code>.</li>
1068 </ol>
1069
1070 <h2>Publishing example</h2>
1071 <pre><code>POST https://pds.example/xrpc/com.atproto.repo.putRecord
1072{
1073 "repo": "did:plc:...",
1074 "collection": "org.latha.island",
1075 "rkey": "<sha256-centroid-12hex>",
1076 "record": { ...islandRecord }
1077}</code></pre>
1078
1079 <p>Then notify Stigmergic:</p>
1080 <pre><code>POST https://stigmergic.latha.org/xrpc/org.latha.island.notifyOfUpdate
1081{
1082 "did": "did:plc:..."
1083}</code></pre>
1084
1085 <h2>Discovering existing islands</h2>
1086 <p>Before creating a new island, check whether one already exists for your URL or topic. Prefer updating or forking an existing island over publishing a near-duplicate.</p>
1087 <h3>Search by URL</h3>
1088 <pre><code>GET https://stigmergic.latha.org/xrpc/org.latha.island.searchIslands?url=https%3A%2F%2Fexample.org%2Fpaper</code></pre>
1089 <p>URL search is exact after light URL normalization. It does not fall back to broad domain matching, because large hosts like arxiv.org, biorxiv.org, nature.com, and openreview.net create false matches.</p>
1090 <h3>List indexed islands</h3>
1091 <pre><code>GET https://stigmergic.latha.org/xrpc/org.latha.island.getIslands?limit=50</code></pre>
1092 <p>This returns appview island summaries with resolved graph data when possible. It is the easiest way for agents to browse existing islands.</p>
1093 <h3>List recent islands</h3>
1094 <pre><code>GET https://stigmergic.latha.org/xrpc/org.latha.island.listRecords?limit=50</code></pre>
1095 <p>This returns raw indexed <code>org.latha.island</code> records.</p>
1096 <h3>Fetch an island record</h3>
1097 <pre><code>GET https://stigmergic.latha.org/xrpc/org.latha.island.getRecord?uri=at://did:plc:.../org.latha.island/<rkey></code></pre>
1098 <p>Use <code>getRecord</code> when you need the full record, resolved edges, vertex metadata, and the current analysis. Browser pages use the same record identity:</p>
1099 <pre><code>https://stigmergic.latha.org/island/<handle-or-did>/<rkey></code></pre>
1100
1101 <h2>Updating an existing island</h2>
1102 <p>There are two update modes: direct update when you own the record, and fork/update when another DID owns it.</p>
1103 <h3>If you own the record</h3>
1104 <ol>
1105 <li>Fetch the existing record from your PDS or from Stigmergic's <code>getRecord</code> endpoint.</li>
1106 <li>Preserve <code>source.uri</code>, <code>createdAt</code>, and the existing rkey.</li>
1107 <li>Publish any new <code>network.cosmik.connection</code> records first, then merge their AT URIs into <code>connections</code> and update <code>analysis</code>. Do not delete existing useful connection pointers unless they are wrong.</li>
1108 <li>Set <code>updatedAt</code> to the current ISO timestamp.</li>
1109 <li>Write it back with <code>com.atproto.repo.putRecord</code> using the same collection and rkey.</li>
1110 <li>Notify Stigmergic with your DID.</li>
1111 </ol>
1112 <pre><code>POST https://pds.example/xrpc/com.atproto.repo.putRecord
1113{
1114 "repo": "did:plc:...",
1115 "collection": "org.latha.island",
1116 "rkey": "<existing-rkey>",
1117 "record": { ...updatedIslandRecord }
1118}</code></pre>
1119 <h3>If another DID owns the record</h3>
1120 <p>Do not overwrite someone else's record. Publish a fork to your own PDS using the same centroid-hash rkey and include <code>forkOf</code> with the original AT URI.</p>
1121 <pre><code>{
1122 "$type": "org.latha.island",
1123 "forkOf": "at://did:plc:original/org.latha.island/<rkey>",
1124 "source": { "uri": "https://same-centroid.example", "collection": "network.cosmik.connection" },
1125 "connections": [ "at://did:plc:.../network.cosmik.connection/<connection-rkey>" ],
1126 "analysis": {
1127 "title": "Updated title",
1128 "themes": ["theme"],
1129 "synthesis": "Short abstract (≤2000 graphemes).",
1130 "synthesisDoc": "at://did:plc:.../pub.oxa.document/<doc-rkey>"
1131 },
1132 "createdAt": "2026-05-22T00:00:00.000Z",
1133 "updatedAt": "2026-05-22T00:00:00.000Z"
1134}</code></pre>
1135 <p>Forks should make a substantive improvement: additional real sources, stronger cross-edges, clearer synthesis, corrected tensions, or updated references.</p>
1136
1137 <h2>Edit and fork behavior</h2>
1138 <ul>
1139 <li>If you edit an island you own, Stigmergic saves with <code>putRecord</code> at the existing rkey.</li>
1140 <li>If you edit an island owned by another DID, Stigmergic forks it to your PDS using the same centroid-hash rkey and sets <code>forkOf</code> to the original AT URI.</li>
1141 <li>After saving, the page redirects to the saved record under the authoring handle.</li>
1142 </ul>
1143
1144 <h2>Similarity</h2>
1145 <p>Similar islands are computed from structured URL overlap: shared exact URLs first, shared hosts/domains as fallback. Avoid keyword-only matching; it creates false connections.</p>
1146
1147 <h2>Useful links</h2>
1148 <ul>
1149 <li><a href="/">Explore islands</a></li>
1150 <li><a href="/search">Search by URL</a></li>
1151 <li><a href="/connect">Create a connection preview</a></li>
1152 </ul>
1153 </div>`;
1154 const page = LANDING_PAGE
1155 .replace("<title>Stigmergic</title>", "<title>Agent Guide — Stigmergic</title>")
1156 .replace('<div id="page-content"></div>', `<div id="page-content">${docs}</div>`);
1157 return c.html(page);
1158 });
1159
1160 // ── Islands API ──────────────────────────────────────────────────
1161
1162 app.get("/xrpc/org.latha.island.getIslands", async (c) => {
1163 const limit = Math.min(parseInt(c.req.query("limit") || "50"), 100);
1164 const summary = c.req.query("summary") === "true";
1165 // No ensureContrailReady — just D1 queries
1166
1167 const rows = await db
1168 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT ?")
1169 .bind(limit)
1170 .all<{ uri: string; record: string }>();
1171
1172 const islands: any[] = [];
1173 for (const row of rows.results || []) {
1174 try {
1175 const rec = JSON.parse(row.record);
1176 const centroid = rec.source?.uri;
1177 if (!centroid) continue;
1178
1179 // ID is the record rkey — unique per strata record even if same graph
1180 const id = row.uri.split("/").pop() || row.uri;
1181
1182 // Build graph from connections — resolve URIs if edges are just {uri} references
1183 const connections: any[] = rec.connections || [];
1184 const resolvedEdges = await resolveEdgeUris(db, connections);
1185 const vertexSet = new Set<string>();
1186 const edges: any[] = [];
1187
1188 for (const conn of resolvedEdges) {
1189 if (conn.source && conn.target) {
1190 vertexSet.add(conn.source);
1191 vertexSet.add(conn.target);
1192 edges.push(conn);
1193 }
1194 }
1195
1196 const vertices = [...vertexSet].sort();
1197 const analysis = rec.analysis || {};
1198
1199 // Summary mode: minimal payload for explore page canvas
1200 // Skip vertexMeta (122KB) and trim edges to just source/target
1201 const vertexMeta = summary ? {} : Object.fromEntries(await resolveVertexMeta(db, vertices));
1202 const trimmedEdges = summary
1203 ? edges.map(e => ({ source: e.source, target: e.target, connectionType: e.connectionType }))
1204 : edges;
1205
1206 islands.push({
1207 id, centroid, vertices, edges: trimmedEdges, vertexMeta,
1208 title: analysis.title || null,
1209 strata: analysis.synthesis || analysis.synthesisDoc ? {
1210 prose: analysis.synthesis || "", title: analysis.title || "",
1211 themes: analysis.themes || [],
1212 relationships: [],
1213 highlights: analysis.highlights || [],
1214 tensions: analysis.tensions || [],
1215 open_questions: analysis.openQuestions || [],
1216 synthesis: analysis.synthesis || "",
1217 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
1218 } : null,
1219 recordUri: row.uri,
1220 handle: null as string | null,
1221 });
1222 } catch {}
1223 }
1224
1225 // Batch-resolve handles for all island DIDs
1226 const dids = new Set<string>();
1227 for (const island of islands) {
1228 const parts = island.recordUri.split("/");
1229 if (parts[2]?.startsWith("did:")) dids.add(parts[2]);
1230 }
1231 if (dids.size > 0) {
1232 const placeholders = [...dids].map(() => "?").join(",");
1233 const handleRows = await db
1234 .prepare(`SELECT did, handle FROM identities WHERE did IN (${placeholders})`)
1235 .bind(...dids)
1236 .all<{ did: string; handle: string | null }>();
1237 const handleMap = new Map<string, string>();
1238 for (const r of handleRows.results || []) {
1239 if (r.handle) handleMap.set(r.did, r.handle);
1240 }
1241 for (const island of islands) {
1242 const parts = island.recordUri.split("/");
1243 const did = parts[2];
1244 if (did?.startsWith("did:")) island.handle = handleMap.get(did) || null;
1245 }
1246 }
1247
1248 return c.json({ islands });
1249 });
1250
1251 // ── Get single island by ID or AT URI ────────────────────────────
1252
1253 app.get("/xrpc/org.latha.island.getIsland", async (c) => {
1254 const id = c.req.query("id");
1255 if (!id) {
1256 return c.json({ error: "MissingRequiredParameter", message: "id is required" }, 400);
1257 }
1258
1259 // Normalize handle-based AT URIs to DID-based URIs
1260 // e.g. at://nandi.latha.org/org.latha.island/rkey → at://did:plc:.../org.latha.island/rkey
1261 let normalizedId = id;
1262 let resolvedHandle: string | null = null;
1263 if (id.startsWith("at://")) {
1264 const match = id.match(/^at:\/\/([^/]+)\/(.+)$/);
1265 if (match && !match[1].startsWith("did:")) {
1266 const handle = match[1];
1267 const rest = match[2];
1268 const identity = await db
1269 .prepare("SELECT did, handle FROM identities WHERE handle = ?")
1270 .bind(handle)
1271 .first<{ did: string; handle: string }>();
1272 if (!identity) {
1273 return c.json({ error: "IdentityNotFound", message: `Handle ${handle} not found` }, 404);
1274 }
1275 normalizedId = `at://${identity.did}/${rest}`;
1276 resolvedHandle = identity.handle;
1277 }
1278 }
1279
1280 // If it's an AT URI, look up the strata record
1281 if (normalizedId.startsWith("at://")) {
1282 // Batch: fetch record + resolve handle in one D1 round trip
1283 const did = normalizedId.split("/")[2];
1284 const stmts: D1PreparedStatement[] = [
1285 db.prepare("SELECT uri, record FROM records_island WHERE uri = ?").bind(normalizedId),
1286 ];
1287 if (!resolvedHandle && did?.startsWith("did:")) {
1288 stmts.push(db.prepare("SELECT handle FROM identities WHERE did = ?").bind(did));
1289 }
1290 const batchResults = await db.batch(stmts);
1291
1292 const row = (batchResults[0] as { results: { uri: string; record: string }[] }).results?.[0];
1293 if (!row) {
1294 return c.json({ error: "RecordNotFound" }, 404);
1295 }
1296
1297 if (!resolvedHandle && batchResults[1]) {
1298 const identity = (batchResults[1] as { results: { handle: string }[] }).results?.[0];
1299 resolvedHandle = identity?.handle || null;
1300 }
1301
1302 const rec = JSON.parse(row.record);
1303 const centroid = rec.source?.uri;
1304 if (!centroid) {
1305 return c.json({ error: "InvalidRecord" }, 400);
1306 }
1307
1308 // Build graph from record's connections — no BFS needed
1309 const connections: any[] = rec.connections || [];
1310 const resolvedEdges = await resolveEdgeUris(db, connections);
1311 const vertexSet = new Set<string>();
1312 const edges: any[] = [];
1313 for (const conn of resolvedEdges) {
1314 if (conn.source && conn.target) {
1315 vertexSet.add(conn.source);
1316 vertexSet.add(conn.target);
1317 edges.push(conn);
1318 }
1319 }
1320 const vertices = [...vertexSet].sort();
1321
1322 // Compute centroid and island ID
1323 const adj = new Map<string, Set<string>>();
1324 for (const e of edges) {
1325 if (!adj.has(e.source)) adj.set(e.source, new Set());
1326 if (!adj.has(e.target)) adj.set(e.target, new Set());
1327 adj.get(e.source)!.add(e.target);
1328 adj.get(e.target)!.add(e.source);
1329 }
1330 const actualCentroid = computeCentroid(adj, vertexSet);
1331 const islandId = await islandHash(actualCentroid);
1332
1333 const meta = await resolveVertexMeta(db, vertices);
1334 const analysis = rec.analysis || {};
1335
1336 return c.json({
1337 island: {
1338 id: islandId,
1339 centroid: actualCentroid,
1340 vertices,
1341 edges,
1342 vertexMeta: Object.fromEntries(meta),
1343 summary: analysis.title || null,
1344 handle: resolvedHandle,
1345 rawRecord: rec,
1346 strata: analysis.synthesis ? {
1347 prose: analysis.synthesis || "",
1348 title: analysis.title || "",
1349 themes: analysis.themes || [],
1350 relationships: [],
1351 highlights: analysis.highlights || [],
1352 tensions: analysis.tensions || [],
1353 open_questions: analysis.openQuestions || [],
1354 synthesis: analysis.synthesis || "",
1355 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
1356 } : null,
1357 },
1358 recordUri: row.uri,
1359 });
1360 }
1361
1362 // ID is the record rkey — look up directly by URI
1363 const recordUri = `at://did:plc:ngokl2gnmpbvuvrfckja3g7p/org.latha.island/${id}`;
1364 const row = await db
1365 .prepare("SELECT uri, record FROM records_island WHERE uri = ?")
1366 .bind(recordUri)
1367 .first<{ uri: string; record: string }>();
1368
1369 if (row) {
1370 try {
1371 const rec = JSON.parse(row.record);
1372 const centroid = rec.source?.uri;
1373
1374 // Build graph from connections — resolve URIs if edges are just {uri} references
1375 const connections: any[] = rec.connections || [];
1376 const resolvedEdges = await resolveEdgeUris(db, connections);
1377 const vertexSet = new Set<string>();
1378 const edges: any[] = [];
1379 for (const conn of resolvedEdges) {
1380 if (conn.source && conn.target) {
1381 vertexSet.add(conn.source);
1382 vertexSet.add(conn.target);
1383 edges.push(conn);
1384 }
1385 }
1386 const vertices = [...vertexSet].sort();
1387 const meta = await resolveVertexMeta(db, vertices);
1388 const analysis = rec.analysis || {};
1389
1390 return c.json({
1391 island: {
1392 id,
1393 vertices,
1394 edges,
1395 centroid: centroid || vertices[0],
1396 vertexMeta: Object.fromEntries(meta),
1397 summary: analysis.title || null,
1398 strata: {
1399 prose: analysis.synthesis || "",
1400 title: analysis.title || "",
1401 themes: analysis.themes || [],
1402 relationships: [],
1403 highlights: analysis.highlights || [],
1404 tensions: analysis.tensions || [],
1405 open_questions: analysis.openQuestions || [],
1406 synthesis: analysis.synthesis || "",
1407 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
1408 },
1409 recordUri: row.uri,
1410 },
1411 });
1412 } catch {}
1413 }
1414
1415 // No strata record found — try deriving island by treating the ID as a centroid seed
1416 // If the ID is a 12-char hex string (centroid hash), scan strata records for a match
1417 if (/^[0-9a-f]{12}$/.test(id)) {
1418 const rows = await db
1419 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT 500")
1420 .all<{ uri: string; record: string }>();
1421 for (const row of rows.results || []) {
1422 try {
1423 const rec = JSON.parse(row.record);
1424 const centroid = rec.source?.uri;
1425 if (!centroid) continue;
1426 const hash = await islandHash(centroid);
1427 if (hash === id) {
1428 // Found the record — redirect to the AT URI path
1429 const connections: any[] = rec.connections || [];
1430 const resolvedEdges = await resolveEdgeUris(db, connections);
1431 const vertexSet = new Set<string>();
1432 const edges: any[] = [];
1433 for (const conn of resolvedEdges) {
1434 if (conn.source && conn.target) {
1435 vertexSet.add(conn.source);
1436 vertexSet.add(conn.target);
1437 edges.push(conn);
1438 }
1439 }
1440 const vertices = [...vertexSet].sort();
1441 const meta = await resolveVertexMeta(db, vertices);
1442 const analysis = rec.analysis || {};
1443 return c.json({
1444 island: {
1445 id,
1446 vertices,
1447 edges,
1448 centroid,
1449 vertexMeta: Object.fromEntries(meta),
1450 summary: analysis.title || null,
1451 strata: {
1452 prose: analysis.synthesis || "",
1453 title: analysis.title || "",
1454 themes: analysis.themes || [],
1455 relationships: [],
1456 highlights: analysis.highlights || [],
1457 tensions: analysis.tensions || [],
1458 open_questions: analysis.openQuestions || [],
1459 synthesis: analysis.synthesis || "",
1460 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
1461 },
1462 },
1463 recordUri: row.uri,
1464 });
1465 }
1466 } catch {}
1467 }
1468 }
1469
1470 // Fall back: treat ID as a vertex URL for BFS derivation
1471 const island = await deriveIsland(db, id);
1472 if (!island) {
1473 return c.json({ error: "IslandNotFound" }, 404);
1474 }
1475
1476 const meta = await resolveVertexMeta(db, island.vertices);
1477 return c.json({
1478 island: {
1479 ...island,
1480 centroid: island.centroid,
1481 vertexMeta: Object.fromEntries(meta),
1482 summary: null,
1483 strata: null,
1484 },
1485 recordUri: null,
1486 });
1487 });
1488
1489 // ── List strata records (AT URI-based) ──────────────────────────
1490
1491 app.get("/xrpc/org.latha.island.listRecords", async (c) => {
1492 const rows = await db
1493 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT 50")
1494 .all<{ uri: string; record: string }>();
1495
1496 const records = (rows.results || []).map(r => {
1497 try {
1498 const rec = JSON.parse(r.record);
1499 return {
1500 uri: r.uri,
1501 source: rec.source?.uri || "",
1502 title: rec.analysis?.title || "",
1503 themes: rec.analysis?.themes || [],
1504 createdAt: rec.createdAt || "",
1505 };
1506 } catch {
1507 return null;
1508 }
1509 }).filter(Boolean);
1510
1511 return c.json({ records });
1512 });
1513
1514 // ── Search islands by URL ────────────────────────────────────────
1515
1516 app.get("/xrpc/org.latha.island.searchIslands", async (c) => {
1517 const url = c.req.query("url");
1518 if (!url) {
1519 return c.json({ error: "MissingRequiredParameter", message: "url is required" }, 400);
1520 }
1521
1522 // Normalize the query URL
1523 let normalizedUrl: string;
1524 try {
1525 const u = new URL(url);
1526 u.hash = "";
1527 if (u.hostname.startsWith("www.")) u.hostname = u.hostname.slice(4);
1528 if (u.pathname.endsWith("/")) u.pathname = u.pathname.slice(0, -1);
1529 normalizedUrl = u.toString();
1530 } catch {
1531 return c.json({ error: "InvalidUrl", message: "Could not parse URL" }, 400);
1532 }
1533
1534 const limit = Math.min(parseInt(c.req.query("limit") || "50"), 100);
1535 const rows = await db
1536 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT ?")
1537 .bind(limit)
1538 .all<{ uri: string; record: string }>();
1539
1540 const results: any[] = [];
1541 for (const row of rows.results || []) {
1542 try {
1543 const rec = JSON.parse(row.record);
1544 const centroid = rec.source?.uri;
1545 if (!centroid) continue;
1546
1547 const connections: any[] = rec.connections || [];
1548 const resolvedEdges = await resolveEdgeUris(db, connections);
1549 const vertexSet = new Set<string>();
1550 const edges: any[] = [];
1551 for (const conn of resolvedEdges) {
1552 if (conn.source && conn.target) {
1553 vertexSet.add(conn.source);
1554 vertexSet.add(conn.target);
1555 edges.push(conn);
1556 }
1557 }
1558 const vertices = [...vertexSet].sort();
1559
1560 // Check for exact URL matches only. Domain matches are too broad for
1561 // hosts like preprint servers where many unrelated papers share a domain.
1562 let matchType: "exact" | null = null;
1563 const matchedUrls: string[] = [];
1564 for (const v of vertices) {
1565 let vNorm: string;
1566 try {
1567 const vu = new URL(v);
1568 vu.hash = "";
1569 if (vu.hostname.startsWith("www.")) vu.hostname = vu.hostname.slice(4);
1570 if (vu.pathname.endsWith("/")) vu.pathname = vu.pathname.slice(0, -1);
1571 vNorm = vu.toString();
1572 } catch { vNorm = v; }
1573 if (vNorm === normalizedUrl) {
1574 matchType = "exact";
1575 matchedUrls.push(v);
1576 }
1577 }
1578
1579 if (!matchType) continue;
1580
1581 const id = row.uri.split("/").pop() || row.uri;
1582 const analysis = rec.analysis || {};
1583 const did = row.uri.split("/")[2] || "";
1584 const handleRow = await db
1585 .prepare("SELECT handle FROM identities WHERE did = ?")
1586 .bind(did)
1587 .first<{ handle: string | null }>();
1588
1589 results.push({
1590 id,
1591 centroid,
1592 title: analysis.title || null,
1593 recordUri: row.uri,
1594 handle: handleRow?.handle || null,
1595 matchType,
1596 matchedUrls,
1597 nodeCount: vertices.length,
1598 edgeCount: edges.length,
1599 edges: edges.map(e => ({ source: e.source, target: e.target, connectionType: e.connectionType })),
1600 strata: analysis.synthesis ? {
1601 title: analysis.title || "",
1602 themes: analysis.themes || [],
1603 synthesis: analysis.synthesis || "",
1604 tensions: analysis.tensions || [],
1605 openQuestions: analysis.openQuestions || [],
1606 highlights: analysis.highlights || [],
1607 } : null,
1608 });
1609 } catch {}
1610 }
1611
1612 return c.json({ url, results });
1613 });
1614
1615 app.post("/xrpc/org.latha.island.discoverFromCentroid", async (c) => {
1616 const body = await c.req.json().catch(() => ({})) as { url?: string };
1617 const url = body.url;
1618 if (!url) {
1619 return c.json({ error: "MissingRequiredParameter", message: "url is required" }, 400);
1620 }
1621 try {
1622 const u = new URL(url);
1623 if (u.protocol !== "https:") {
1624 return c.json({ error: "InvalidUrl", message: "url must be https" }, 400);
1625 }
1626 u.hash = "";
1627 if (u.hostname.startsWith("www.")) u.hostname = u.hostname.slice(4);
1628 const centroid = u.toString();
1629 const discovery = await discoverCentroidWithLetta(c.env, centroid);
1630 const rkey = await islandHash(centroid);
1631 const connections = discovery.edges.map((edge, i) => ({
1632 uri: `inline:${rkey}:${i}`,
1633 source: edge.source,
1634 target: edge.target,
1635 connectionType: edge.connectionType || "network.cosmik.connection#related",
1636 note: edge.note || "Discovered by Letta researcher agent while building this island.",
1637 }));
1638 const highlightUris = connections.slice(0, 3).map(e => e.uri);
1639 return c.json({
1640 rkey,
1641 record: {
1642 $type: "org.latha.island",
1643 source: { uri: centroid, collection: "network.cosmik.connection" },
1644 connections,
1645 analysis: {
1646 title: discovery.title,
1647 themes: discovery.themes,
1648 highlights: highlightUris,
1649 tensions: discovery.tensions,
1650 openQuestions: discovery.openQuestions,
1651 synthesis: discovery.synthesis,
1652 },
1653 createdAt: new Date().toISOString(),
1654 updatedAt: new Date().toISOString(),
1655 },
1656 });
1657 } catch (e: any) {
1658 return c.json({ error: "DiscoveryFailed", message: e.message || String(e) }, 500);
1659 }
1660 });
1661
1662 // ── Strata record endpoint (AT URI-based) ──────────────────────
1663
1664 app.get("/xrpc/org.latha.island.getRecord", async (c) => {
1665 const uri = c.req.query("uri");
1666 if (!uri) {
1667 return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
1668 }
1669
1670 let normalizedUri = uri;
1671 if (uri.startsWith("at://")) {
1672 const match = uri.match(/^at:\/\/([^/]+)\/(.+)$/);
1673 if (match && !match[1].startsWith("did:")) {
1674 const identity = await db
1675 .prepare("SELECT did FROM identities WHERE handle = ?")
1676 .bind(match[1])
1677 .first<{ did: string }>();
1678 if (identity?.did) normalizedUri = `at://${identity.did}/${match[2]}`;
1679 }
1680 }
1681
1682 // Fetch the strata record from D1 (indexed by Contrail)
1683 const row = await db
1684 .prepare("SELECT record FROM records_island WHERE uri = ?")
1685 .bind(normalizedUri)
1686 .first<{ record: string }>();
1687
1688 if (!row) {
1689 return c.json({ error: "NotFound", message: "Strata record not found" }, 404);
1690 }
1691
1692 let strataRecord: any;
1693 try {
1694 strataRecord = JSON.parse(row.record);
1695 } catch {
1696 return c.json({ error: "InvalidRecord" }, 500);
1697 }
1698
1699 // Derive island from centroid (source.uri)
1700 const centroid = strataRecord.source?.uri;
1701 if (!centroid) {
1702 return c.json({ error: "MissingLexmin", message: "Strata record has no source.uri" }, 400);
1703 }
1704
1705 let island = await deriveIsland(db, centroid);
1706 let recordEdges = await resolveEdgeUris(db, Array.isArray(strataRecord.connections) ? strataRecord.connections : []);
1707 // Fallback: if edges are all at://inline:* URIs that didn't resolve, try forkOf
1708 if (recordEdges.length === 0) {
1709 recordEdges = await resolveInlineFromForkOf(db, strataRecord);
1710 }
1711 if (recordEdges.length > 0) {
1712 const vertexSet = new Set<string>();
1713 for (const edge of recordEdges) {
1714 if (edge?.source) vertexSet.add(edge.source);
1715 if (edge?.target) vertexSet.add(edge.target);
1716 }
1717 island = {
1718 id: normalizedUri.split("/").pop() || normalizedUri,
1719 centroid,
1720 vertices: [...vertexSet].sort(),
1721 edges: recordEdges,
1722 };
1723 }
1724 if (!island) {
1725 return c.json({ error: "IslandNotFound", message: "Could not derive island from centroid" }, 404);
1726 }
1727
1728 // Resolve vertex metadata
1729 const meta = await resolveVertexMeta(db, island.vertices);
1730
1731 // Build the response in the same format as the island cache
1732 const analysis = strataRecord.analysis || {};
1733 const strataData = {
1734 prose: analysis.synthesis || "",
1735 title: analysis.title || "",
1736 themes: analysis.themes || [],
1737 relationships: [],
1738 highlights: analysis.highlights || [],
1739 tensions: analysis.tensions || [],
1740 open_questions: analysis.openQuestions || [],
1741 synthesis: analysis.synthesis || "",
1742 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
1743 };
1744
1745 return c.json({
1746 island: {
1747 ...island,
1748 vertexMeta: Object.fromEntries(meta),
1749 summary: analysis.title || null,
1750 strata: strataData,
1751 rawRecord: strataRecord,
1752 },
1753 record: strataRecord,
1754 recordUri: normalizedUri,
1755 });
1756 });
1757
1758 // Notify appview of a newly published record — crawl the DID's PDS and upsert into D1
1759 app.post("/xrpc/org.latha.island.notifyOfUpdate", async (c) => {
1760 const body = await c.req.json().catch(() => ({})) as { did?: string; uri?: string };
1761 const did = body.did;
1762 if (!did) return c.json({ error: "InvalidRequest", message: "did is required" }, 400);
1763
1764 await ensureContrailReady(db);
1765
1766 // Look up the DID's PDS
1767 const identity = await db
1768 .prepare("SELECT pds FROM identities WHERE did = ?")
1769 .bind(did)
1770 .first<{ pds: string }>();
1771
1772 let pds = identity?.pds || null;
1773 if (!pds) {
1774 const resolved = await resolveDidIdentity(did);
1775 pds = resolved?.pds || null;
1776 if (!pds) return c.json({ error: "IdentityNotFound", message: `Could not resolve PDS for DID ${did}` }, 404);
1777 await db.prepare(
1778 `INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?)
1779 ON CONFLICT(did) DO UPDATE SET handle = excluded.handle, pds = excluded.pds, resolved_at = excluded.resolved_at`,
1780 ).bind(did, resolved?.handle || null, pds, Date.now() * 1000).run();
1781 }
1782
1783 const now = Date.now() * 1000; // time_us
1784 const indexedAt = Date.now();
1785 const islandStmt = db.prepare(
1786 `INSERT INTO records_island (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at`
1787 );
1788 const connectionStmt = db.prepare(
1789 `INSERT INTO records_connection (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at`
1790 );
1791
1792 const batch: D1PreparedStatement[] = [];
1793 const counts = { island: 0, connection: 0 };
1794 for (const [collection, stmt] of [
1795 ["org.latha.island", islandStmt],
1796 ["network.cosmik.connection", connectionStmt],
1797 ] as const) {
1798 // Paginate through all records (PDS limit is 100 per page)
1799 let cursor: string | undefined;
1800 do {
1801 const url = `${pds}/xrpc/com.atproto.repo.listRecords?repo=${did}&collection=${collection}&limit=100${cursor ? `&cursor=${cursor}` : ""}`;
1802 const resp = await fetch(url);
1803 if (!resp.ok) return c.json({ error: "PdsError", message: `PDS returned ${resp.status} for ${collection}` }, 502);
1804
1805 const data = await resp.json() as { records: Array<{ uri: string; cid: string; value: Record<string, unknown> }>; cursor?: string };
1806 for (const r of data.records || []) {
1807 // Skip strata records — they should not be indexed as islands
1808 if ((r.value as any)?.$type === "org.latha.strata") continue;
1809 const rkey = r.uri.split("/").pop() || "";
1810 const offset = counts.island + counts.connection;
1811 batch.push(stmt.bind(r.uri, did, rkey, r.cid || "", JSON.stringify(r.value), now + offset, indexedAt));
1812 counts[collection === "org.latha.island" ? "island" : "connection"]++;
1813 }
1814 cursor = data.cursor;
1815 } while (cursor);
1816 }
1817
1818 if (batch.length > 0) await db.batch(batch);
1819 return c.json({ ingested: counts.island, connections: counts.connection });
1820 });
1821
1822 // ── Similar islands (async) ─────────────────────────────────────
1823
1824 app.get("/api/similar-islands", async (c) => {
1825 const uri = c.req.query("uri");
1826 if (!uri) return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
1827
1828 let normalizedUri = uri;
1829 if (uri.startsWith("at://") && !uri.split("/")[2]?.startsWith("did:")) {
1830 const handle = uri.split("/")[2];
1831 const identity = await db
1832 .prepare("SELECT did FROM identities WHERE handle = ?")
1833 .bind(handle)
1834 .first<{ did: string }>();
1835 if (identity?.did) normalizedUri = `at://${identity.did}/${uri.split("/").slice(3).join("/")}`;
1836 }
1837
1838 const row = await db
1839 .prepare("SELECT record FROM records_island WHERE uri = ?")
1840 .bind(normalizedUri)
1841 .first<{ record: string }>();
1842 if (!row) return c.json({ similar: [] });
1843
1844 const rec = JSON.parse(row.record);
1845 const connections: any[] = Array.isArray(rec.connections) ? rec.connections : [];
1846 const currentConnUris = new Set(connections.filter(c => typeof c === "string"));
1847 const centroid = rec.source?.uri;
1848 if (centroid) { /* include centroid in vertex set */ }
1849 const edges = await resolveEdgeUris(db, connections);
1850 const currentVertices = new Set<string>();
1851 if (centroid) currentVertices.add(centroid);
1852 for (const edge of edges) {
1853 if (edge?.source) currentVertices.add(edge.source);
1854 if (edge?.target) currentVertices.add(edge.target);
1855 }
1856
1857 const similar = await findSimilarIslands(db, normalizedUri, currentVertices, currentConnUris);
1858 return c.json({ similar });
1859 });
1860
1861 // ── Graph SVG endpoint ─────────────────────────────────────────
1862
1863 app.get("/api/graph/:id", async (c) => {
1864 const id = c.req.param("id");
1865 const w = parseInt(c.req.query("w") || "400");
1866 const h = parseInt(c.req.query("h") || "300");
1867
1868 // Find the island record by rkey — scan all records and filter
1869 // D1 doesn't reliably support LIKE with leading wildcard
1870 let allRows: any[] = [];
1871 try {
1872 const result = await db
1873 .prepare("SELECT uri, record FROM records_island WHERE uri NOT LIKE '%org.latha.strata%'")
1874 .all<{ uri: string; record: string }>();
1875 allRows = result.results || [];
1876 } catch (e) {
1877 console.error("Graph SVG: D1 query failed:", e);
1878 return c.text("DB error", 500);
1879 }
1880 const row = allRows.find(r => r.uri.endsWith(`/org.latha.island/${id}`));
1881 if (!row) return c.text("Not found", 404);
1882
1883 try {
1884 const rec = JSON.parse(row.record);
1885 const connections: any[] = Array.isArray(rec.connections) ? rec.connections : [];
1886 const edges = await resolveEdgeUris(db, connections);
1887 const vertexSet = new Set<string>();
1888 const centroid = rec.source?.uri;
1889 if (centroid) vertexSet.add(centroid);
1890 for (const edge of edges) {
1891 if (edge?.source) vertexSet.add(edge.source);
1892 if (edge?.target) vertexSet.add(edge.target);
1893 }
1894
1895 const svg = generateGraphSVG([...vertexSet], edges.map(e => ({
1896 source: e.source, target: e.target, connectionType: e.connectionType || "relates",
1897 })), w, h);
1898
1899 return c.text(svg, 200, {
1900 "Content-Type": "image/svg+xml",
1901 "Cache-Control": "public, max-age=3600",
1902 });
1903 } catch (e) {
1904 console.error("Graph SVG: render failed:", e);
1905 return c.text("Render error", 500);
1906 }
1907 });
1908
1909 // ── R2 sync endpoints ───────────────────────────────────────────
1910
1911 app.put("/api/sync/vault/:did/*", async (c) => {
1912 const did = c.req.param("did");
1913 const path = c.req.param("path");
1914 if (!did || !path) {
1915 return c.json({ error: "Missing path" }, 400);
1916 }
1917
1918 const key = `${did}/${path}`;
1919 const body = await c.req.raw.arrayBuffer();
1920 await env.VAULT_BUCKET.put(key, body, {
1921 httpMetadata: { contentType: c.req.header("content-type") || "text/markdown" },
1922 });
1923
1924 return c.json({ ok: true, key });
1925 });
1926
1927 app.put("/api/sync/carry/:did/*", async (c) => {
1928 const did = c.req.param("did");
1929 const path = c.req.param("path");
1930 if (!did || !path) {
1931 return c.json({ error: "Missing path" }, 400);
1932 }
1933
1934 const key = `${did}/${path}`;
1935 const body = await c.req.raw.arrayBuffer();
1936 await env.CARRY_BUCKET.put(key, body, {
1937 httpMetadata: { contentType: c.req.header("content-type") || "application/json" },
1938 });
1939
1940 return c.json({ ok: true, key });
1941 });
1942
1943 // ── Graph neighborhood query ────────────────────────────────────
1944
1945 app.get("/xrpc/org.latha.island.connection.getGraph", async (c) => {
1946 const uri = c.req.query("uri");
1947 if (!uri) {
1948 return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
1949 }
1950
1951 const depth = Math.min(parseInt(c.req.query("depth") || "2"), 3);
1952 const limit = Math.min(parseInt(c.req.query("limit") || "50"), 100);
1953 const types = c.req.query("types")?.split(",").map((t) => t.trim());
1954
1955 // Reuse island detection for the subgraph
1956 const islands = await detectIslands(db);
1957 const island = islands.find(i => i.vertices.includes(uri));
1958
1959 if (!island) {
1960 return c.json({ connections: [], resources: [], depth, uri });
1961 }
1962
1963 // Convert to graph format
1964 const resources = island.vertices.map(v => ({
1965 uri: v,
1966 title: v,
1967 type: "unknown",
1968 }));
1969
1970 return c.json({
1971 connections: island.edges,
1972 resources,
1973 depth,
1974 uri,
1975 });
1976 });
1977
1978 // ── Validate record against lexicon schema ──────────────────────
1979
1980 app.get("/xrpc/org.latha.island.validateRecord", async (c) => {
1981 const uri = c.req.query("uri");
1982 if (!uri) {
1983 return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
1984 }
1985
1986 // Normalize handle-based URIs
1987 let normalizedUri = uri;
1988 const handleMatch = uri.match(/^at:\/\/([^/.]+)\./);
1989 if (handleMatch && !uri.includes("did:plc:") && !uri.includes("did:web:")) {
1990 const handle = uri.split("/")[2];
1991 try {
1992 const didRes = await fetch(`https://plc.directory/${handle}`);
1993 if (didRes.ok) {
1994 const didDoc = await didRes.json() as { id: string };
1995 normalizedUri = uri.replace(`at://${handle}`, `at://${didDoc.id}`);
1996 }
1997 } catch { /* use original */ }
1998 }
1999
2000 // Fetch the record from D1
2001 const recordRow = await db.prepare("SELECT uri, record FROM records_island WHERE uri = ?").bind(normalizedUri).first() as { uri: string; record: string } | null;
2002 if (!recordRow) {
2003 return c.json({ error: "RecordNotFound", message: "Island record not found in index" }, 404);
2004 }
2005
2006 const rec = JSON.parse(recordRow.record || "{}");
2007 const issues: Array<{ field: string; issue: string; detail?: string }> = [];
2008
2009 // Check $type
2010 if (rec.$type !== "org.latha.island") {
2011 issues.push({ field: "$type", issue: "missing_or_wrong", detail: `Expected "org.latha.island", got "${rec.$type || "missing"}"` });
2012 }
2013
2014 // Check source
2015 if (!rec.source || typeof rec.source !== "object") {
2016 issues.push({ field: "source", issue: "missing_or_wrong_type", detail: "source must be an object with uri" });
2017 } else if (!rec.source.uri || typeof rec.source.uri !== "string") {
2018 issues.push({ field: "source.uri", issue: "missing", detail: "source.uri (canonical HTTPS URL) is required" });
2019 }
2020
2021 // Check connections
2022 if (!Array.isArray(rec.connections)) {
2023 issues.push({ field: "connections", issue: "missing_or_wrong_type", detail: "connections must be an array" });
2024 } else {
2025 const emptyConns = rec.connections.filter((c: any) => typeof c === "string" && c.trim() === "");
2026 const inlineConns = rec.connections.filter((c: any) => typeof c === "string" && c.startsWith("at://inline:"));
2027 const objConns = rec.connections.filter((c: any) => typeof c === "object" && c !== null);
2028 const nonAtUriConns = rec.connections.filter((c: any) => typeof c === "string" && c.trim() !== "" && !c.startsWith("at://"));
2029
2030 if (emptyConns.length > 0) {
2031 issues.push({ field: "connections", issue: "empty_strings", detail: `${emptyConns.length} empty connection string(s). Connections must be AT URI strings pointing to network.cosmik.connection records.` });
2032 }
2033 if (inlineConns.length > 0) {
2034 issues.push({ field: "connections", issue: "inline_uris", detail: `${inlineConns.length} inline synthetic URI(s) (at://inline:*). These are not real records and should be replaced with actual network.cosmik.connection AT URIs.` });
2035 }
2036 if (objConns.length > 0) {
2037 issues.push({ field: "connections", issue: "object_format", detail: `${objConns.length} object-format connection(s). Connections must be AT URI strings, not objects.` });
2038 }
2039 if (nonAtUriConns.length > 0) {
2040 issues.push({ field: "connections", issue: "non_at_uri", detail: `${nonAtUriConns.length} non-AT-URI string(s). Connections must be AT URI format.` });
2041 }
2042 }
2043
2044 // Check analysis
2045 if (!rec.analysis || typeof rec.analysis !== "object") {
2046 issues.push({ field: "analysis", issue: "missing_or_wrong_type", detail: "analysis object is required" });
2047 } else {
2048 if (!Array.isArray(rec.analysis.themes)) {
2049 issues.push({ field: "analysis.themes", issue: "missing_or_wrong_type", detail: "analysis.themes array is required" });
2050 }
2051 }
2052
2053 // Check createdAt
2054 if (!rec.createdAt || typeof rec.createdAt !== "string") {
2055 issues.push({ field: "createdAt", issue: "missing", detail: "createdAt datetime string is required" });
2056 }
2057
2058 // Check synthesisDoc format
2059 if (rec.analysis?.synthesisDoc && typeof rec.analysis.synthesisDoc === "object") {
2060 issues.push({ field: "analysis.synthesisDoc", issue: "wrong_type", detail: "synthesisDoc must be an AT URI string, not an inline object. Publish the OXA document as a pub.oxa.document record and reference it by AT URI." });
2061 }
2062
2063 return c.json({
2064 uri: normalizedUri,
2065 valid: issues.length === 0,
2066 issues,
2067 });
2068 });
2069
2070 // ── OAuth client metadata ──────────────────────────────────────
2071
2072 app.get("/oauth-client-metadata.json", (c) => {
2073 const host = new URL(c.req.url).host;
2074 return c.json({
2075 client_id: `https://${host}/oauth-client-metadata.json`,
2076 client_name: "Stigmergic",
2077 client_uri: `https://${host}`,
2078 redirect_uris: [`https://${host}/`, `https://${host}/oauth/callback`],
2079 scope: "atproto transition:generic",
2080 grant_types: ["authorization_code", "refresh_token"],
2081 response_types: ["code"],
2082 token_endpoint_auth_method: "none",
2083 application_type: "web",
2084 dpop_bound_access_tokens: true,
2085 });
2086 });
2087
2088 app.get("/oauth/callback", (c) => c.html(landingPage()));
2089
2090 app.all("/debug/echo/*", (c) => c.json({ path: c.req.path, url: c.req.url }));
2091 app.all("/debug/echo", (c) => c.json({ path: c.req.path, url: c.req.url }));
2092
2093 // Static assets are files only. App routing stays in this Worker.
2094 app.get("/app.js", (c) => c.env.ASSETS.fetch(c.req.raw));
2095 app.get("/app.js.map", (c) => c.env.ASSETS.fetch(c.req.raw));
2096
2097 // ── SPA fallback: serve landing page for client-side routes ──────
2098
2099 // /island/<handle>/<rkey> — inject island data for instant hydration
2100 app.get("/island/:repo/:rkey", async (c) => {
2101 let repo = c.req.param("repo");
2102 const rkey = c.req.param("rkey");
2103 const collection = c.req.query("collection") || "org.latha.island";
2104
2105 let islandsJson = "[]";
2106 let islandDataJson = "null";
2107 try {
2108 // Batch: island list + specific island record
2109 const atUri = repo.startsWith("did:")
2110 ? `at://${repo}/${collection}/${rkey}`
2111 : `at://${repo}/${collection}/${rkey}`;
2112
2113 // Resolve handle → DID if needed
2114 let did = repo;
2115 let handle: string | null = null;
2116 if (!repo.startsWith("did:")) {
2117 const identity = await db
2118 .prepare("SELECT did, handle FROM identities WHERE handle = ?")
2119 .bind(repo)
2120 .first<{ did: string; handle: string }>();
2121 if (identity) {
2122 did = identity.did;
2123 handle = identity.handle;
2124 }
2125 } else {
2126 const identity = await db
2127 .prepare("SELECT handle FROM identities WHERE did = ?")
2128 .bind(repo)
2129 .first<{ handle: string }>();
2130 handle = identity?.handle || null;
2131 }
2132
2133 const fullAtUri = `at://${did}/${collection}/${rkey}`;
2134
2135 // Batch: fetch record + island list
2136 const stmts = [
2137 db.prepare("SELECT uri, record FROM records_island WHERE uri = ?").bind(fullAtUri),
2138 db.prepare("SELECT uri, json_extract(record, '$.source.uri') as centroid, json_extract(record, '$.analysis.title') as title FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT 50"),
2139 ];
2140 const [recordResult, listResult] = await db.batch(stmts);
2141
2142 // Build island list
2143 const listRows = (listResult as { results: { uri: string; centroid: string | null; title: string | null }[] }).results || [];
2144 const islands: any[] = [];
2145 for (const row of listRows) {
2146 if (!row.centroid) continue;
2147 const id = row.uri.split("/").pop() || row.uri;
2148 const listDid = row.uri.split("/")[2];
2149 // Resolve handle for list items lazily — skip for now, use DID
2150 islands.push({ id, centroid: row.centroid, title: row.title || null, recordUri: row.uri, handle: listDid === did ? handle : null });
2151 }
2152 islandsJson = JSON.stringify(islands);
2153
2154 // Build island data from record
2155 const recordRow = (recordResult as { results: { uri: string; record: string }[] }).results?.[0];
2156 if (recordRow) {
2157 const rec = JSON.parse(recordRow.record);
2158 const centroid = rec.source?.uri;
2159 if (centroid) {
2160 const connections: any[] = rec.connections || [];
2161 // Resolve only the island's referenced edges. This is bounded by the
2162 // record size and avoids full component BFS / vertex metadata lookups,
2163 // but still gives the server-rendered page the real graph size.
2164 let edges: any[] = await resolveEdgeUris(db, connections);
2165 // Fallback: if edges are all at://inline:* URIs that didn't resolve, try forkOf
2166 if (edges.length === 0) {
2167 edges = await resolveInlineFromForkOf(db, rec);
2168 }
2169 let vertexSet = new Set<string>([centroid]);
2170 for (const edge of edges) {
2171 if (edge.source) vertexSet.add(edge.source);
2172 if (edge.target) vertexSet.add(edge.target);
2173 }
2174
2175 // Fallback: if record connections are empty/unresolvable, derive from BFS
2176 if (edges.length === 0) {
2177 const derived = await deriveIsland(db, centroid);
2178 if (derived) {
2179 edges = derived.edges;
2180 vertexSet = new Set(derived.vertices);
2181 }
2182 }
2183
2184 const vertices = [...vertexSet].sort();
2185 const islandId = rkey;
2186 const analysis = rec.analysis || {};
2187 islandDataJson = JSON.stringify({
2188 island: {
2189 id: islandId,
2190 centroid,
2191 vertices,
2192 edges,
2193 vertexMeta: {},
2194 summary: analysis.title || null,
2195 handle,
2196 strata: analysis.synthesis || analysis.synthesisDoc ? {
2197 prose: analysis.synthesis || "",
2198 title: analysis.title || "",
2199 themes: analysis.themes || [],
2200 relationships: [],
2201 highlights: analysis.highlights || [],
2202 tensions: analysis.tensions || [],
2203 open_questions: analysis.openQuestions || [],
2204 synthesis: analysis.synthesis || "",
2205 synthesisDoc: await resolveSynthesisDoc(analysis.synthesisDoc),
2206 } : null,
2207 rawRecord: rec,
2208 },
2209 recordUri: recordRow.uri,
2210 });
2211 }
2212 }
2213 } catch (e) {
2214 console.error("Failed to load island data:", e);
2215 }
2216
2217 // Build OG tags from island data
2218 let ogTags = "";
2219 try {
2220 const islandData = JSON.parse(islandDataJson);
2221 const title = islandData?.island?.strata?.title || islandData?.island?.summary || "Island";
2222 const description = islandData?.island?.strata?.synthesis?.slice(0, 200) || islandData?.island?.strata?.tensions?.[0] || "";
2223 const nodeCount = islandData?.island?.vertices?.length || 0;
2224 const edgeCount = islandData?.island?.edges?.length || 0;
2225 const fullTitle = `${title} — Stigmergic`;
2226 ogTags = `<meta property="og:title" content="${fullTitle.replace(/"/g, """)}">
2227<meta property="og:description" content="${description.replace(/"/g, """)}">
2228<meta property="og:type" content="article">
2229<meta name="twitter:card" content="summary">
2230<meta name="twitter:title" content="${fullTitle.replace(/"/g, """)}">
2231<meta name="twitter:description" content="${description.replace(/"/g, """)}">`;
2232 } catch {}
2233
2234 const page = LANDING_PAGE.replace(
2235 "</head>",
2236 `${ogTags}<script>window.__ISLANDS__=${islandsJson};window.__ISLAND_DATA__=${islandDataJson};</script></head>`,
2237 ).replace("<title>Stigmergic</title>", `<title>${(JSON.parse(islandDataJson)?.island?.strata?.title || JSON.parse(islandDataJson)?.island?.summary || "Island")} — Stigmergic</title>`);
2238 return c.html(page);
2239 });
2240
2241 // Other SPA routes — just inject island list
2242 for (const path of ["/island", "/strata", "/connect", "/search"]) {
2243 app.get(path, async (c) => {
2244 let islandsJson = "[]";
2245 try {
2246 const rows = await db
2247 .prepare("SELECT uri, json_extract(record, '$.source.uri') as centroid, json_extract(record, '$.analysis.title') as title FROM records_island WHERE uri NOT LIKE '%org.latha.strata%' ORDER BY time_us DESC LIMIT 50")
2248 .all<{ uri: string; centroid: string | null; title: string | null }>();
2249 const islands: any[] = [];
2250 for (const row of rows.results || []) {
2251 if (!row.centroid) continue;
2252 const id = row.uri.split("/").pop() || row.uri;
2253 islands.push({ id, centroid: row.centroid, title: row.title || null, recordUri: row.uri });
2254 }
2255 islandsJson = JSON.stringify(islands);
2256 } catch {}
2257
2258 const page = LANDING_PAGE.replace(
2259 "</head>",
2260 `<script>window.__ISLANDS__=${islandsJson};</script></head>`,
2261 );
2262 return c.html(page);
2263 });
2264 }
2265
2266 // ── All other routes pass through to contrail ──────────────────
2267
2268 app.all("*", async (c) => {
2269 const response = await contrailWorker.fetch(
2270 c.req.raw,
2271 c.env as unknown as Record<string, unknown>,
2272 );
2273 return response;
2274 });
2275
2276 return app;
2277}
2278
2279// ─── Export ─────────────────────────────────────────────────────────
2280
2281export { StrataContainer };
2282
2283export default {
2284 fetch(request: Request, env: Env): Response | Promise<Response> {
2285 app ??= buildApp(env);
2286 return app.fetch(request, env);
2287 },
2288 async scheduled(event: ScheduledEvent, env: Env, ctx: ExecutionContext) {
2289 // Run Contrail indexing
2290 await contrailWorker.scheduled(
2291 event,
2292 env as unknown as Record<string, unknown>,
2293 ctx,
2294 );
2295 },
2296};