nandi.uk
This repository has no description
24 kB
740 lines
1import { Hono } from "hono";
2import { cors } from "hono/cors";
3import { createWorker } from "@atmo-dev/contrail/worker";
4import { Contrail } from "@atmo-dev/contrail";
5import { config } from "./contrail.config.js";
6import { lexicons } from "../lexicons/generated/index.js";
7import { LANDING_PAGE } from "./landing-page.js";
8import { StrataContainer } from "./container.js";
9
10// ─── Env ────────────────────────────────────────────────────────────
11
12interface Env {
13 DB: D1Database;
14 VAULT_BUCKET: R2Bucket;
15 CARRY_BUCKET: R2Bucket;
16 STRATA_CONTAINER: DurableObjectNamespace;
17}
18
19// ─── Contrail setup ─────────────────────────────────────────────────
20
21const contrailWorker = createWorker(config, { lexicons });
22const contrail = new Contrail(config);
23let contrailReady = false;
24
25async function ensureContrailReady(db: D1Database): Promise<void> {
26 if (contrailReady) return;
27 await contrail.init(db);
28 contrailReady = true;
29}
30
31// ─── Island detection (connected components) ────────────────────────
32//
33// Load all connections, build an undirected graph, find connected
34// components via BFS. Each component is an "island" — a cluster of
35// linked URLs that form a constellation in the knowledge graph.
36
37interface Island {
38 id: string;
39 vertices: string[];
40 edges: Array<{
41 uri: string;
42 did: string;
43 source: string;
44 target: string;
45 connectionType: string;
46 note: string;
47 handle?: string | null;
48 }>;
49}
50
51async function detectIslands(db: D1Database): Promise<Island[]> {
52 await ensureContrailReady(db);
53
54 // Load all connections
55 const rows = await db
56 .prepare(
57 `SELECT r.record, r.did, r.rkey, i.handle
58 FROM records_connection r
59 LEFT JOIN identities i ON r.did = i.did
60 ORDER BY r.time_us DESC`,
61 )
62 .all<{ record: string; did: string; rkey: string; handle: string | null }>();
63
64 // Build adjacency list
65 const adj = new Map<string, Set<string>>();
66 const edges: Island["edges"] = [];
67
68 for (const row of rows.results || []) {
69 try {
70 const value = JSON.parse(row.record);
71 const source = value.source as string;
72 const target = value.target as string;
73 if (!source || !target) continue;
74
75 if (!adj.has(source)) adj.set(source, new Set());
76 if (!adj.has(target)) adj.set(target, new Set());
77 adj.get(source)!.add(target);
78 adj.get(target)!.add(source);
79
80 edges.push({
81 uri: `at://${row.did}/network.cosmik.connection/${row.rkey}`,
82 did: row.did,
83 source,
84 target,
85 connectionType: value.connectionType || "relates",
86 note: value.note || "",
87 handle: row.handle,
88 });
89 } catch {}
90 }
91
92 // BFS to find connected components
93 const visited = new Set<string>();
94 const islands: Island[] = [];
95
96 for (const node of adj.keys()) {
97 if (visited.has(node)) continue;
98
99 const component: string[] = [];
100 const queue = [node];
101 while (queue.length > 0) {
102 const current = queue.shift()!;
103 if (visited.has(current)) continue;
104 visited.add(current);
105 component.push(current);
106
107 for (const neighbor of adj.get(current) || []) {
108 if (!visited.has(neighbor)) queue.push(neighbor);
109 }
110 }
111
112 // Only include edges where both endpoints are in this component
113 const componentSet = new Set(component);
114 const componentEdges = edges.filter(
115 e => componentSet.has(e.source) && componentSet.has(e.target),
116 );
117
118 // Stable ID: truncated SHA-256 of the lexicographic minimum vertex
119 const lexmin = component.sort()[0];
120 const lexminHash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(lexmin));
121 const id = Array.from(new Uint8Array(lexminHash)).map(b => b.toString(16).padStart(2, "0")).join("").slice(0, 16);
122
123 islands.push({
124 id,
125 vertices: component,
126 edges: componentEdges,
127 });
128 }
129
130 // Sort by size descending
131 islands.sort((a, b) => b.vertices.length - a.vertices.length);
132 return islands;
133}
134
135// ─── Resolve vertex metadata (batched) ──────────────────────────────
136//
137// Bulk queries instead of N+1 per vertex. Loads all cards and citations
138// matching the given URIs in a few chunked queries.
139
140async function resolveVertexMeta(
141 db: D1Database,
142 uris: string[],
143): Promise<Map<string, { title: string; description: string; type: string }>> {
144 const meta = new Map<string, { title: string; description: string; type: string }>();
145 for (const uri of uris) {
146 meta.set(uri, { title: uri, description: "", type: "unknown" });
147 }
148
149 const httpUris = uris.filter(u => u.startsWith("http"));
150 const atUris = uris.filter(u => u.startsWith("at://"));
151 const CHUNK = 50;
152
153 // ── Resolve HTTP URIs (existing logic) ──
154
155 // Batch card lookup
156 for (let i = 0; i < httpUris.length; i += CHUNK) {
157 const chunk = httpUris.slice(i, i + CHUNK);
158 const placeholders = chunk.map(() => "?").join(",");
159 const rows = await db
160 .prepare(
161 `SELECT record FROM records_card
162 WHERE json_extract(record, '$.content.url') IN (${placeholders})`,
163 )
164 .bind(...chunk)
165 .all<{ record: string }>();
166
167 for (const row of rows.results || []) {
168 try {
169 const card = JSON.parse(row.record);
170 const url = card?.content?.url;
171 if (!url || !meta.has(url)) continue;
172 const m = card.content.metadata;
173 if (m?.title || m?.description) {
174 meta.set(url, {
175 title: m.title || url,
176 description: m.description || "",
177 type: card.type === "URL" ? "url" : "note",
178 });
179 }
180 } catch {}
181 }
182 }
183
184 // Batch citation lookup (overrides card data if found)
185 for (let i = 0; i < httpUris.length; i += CHUNK) {
186 const chunk = httpUris.slice(i, i + CHUNK);
187 const placeholders = chunk.map(() => "?").join(",");
188 const rows = await db
189 .prepare(
190 `SELECT record FROM records_citation
191 WHERE json_extract(record, '$.url') IN (${placeholders})`,
192 )
193 .bind(...chunk)
194 .all<{ record: string }>();
195
196 for (const row of rows.results || []) {
197 try {
198 const citation = JSON.parse(row.record);
199 const url = citation.url;
200 if (!url || !meta.has(url)) continue;
201 if (citation.title) {
202 const existing = meta.get(url)!;
203 meta.set(url, {
204 title: citation.title,
205 description: citation.takeaway || existing.description,
206 type: "citation",
207 });
208 }
209 } catch {}
210 }
211 }
212
213 // ── Resolve AT URIs ──
214
215 // Batch card lookup by AT URI
216 for (let i = 0; i < atUris.length; i += CHUNK) {
217 const chunk = atUris.slice(i, i + CHUNK);
218 const placeholders = chunk.map(() => "?").join(",");
219 const rows = await db
220 .prepare(`SELECT uri, record FROM records_card WHERE uri IN (${placeholders})`)
221 .bind(...chunk)
222 .all<{ uri: string; record: string }>();
223
224 for (const row of rows.results || []) {
225 try {
226 const card = JSON.parse(row.record);
227 const m = card?.content?.metadata;
228 const url = card?.content?.url;
229 if (meta.has(row.uri)) {
230 meta.set(row.uri, {
231 title: m?.title || url || row.uri,
232 description: m?.description || "",
233 type: "card",
234 });
235 }
236 } catch {}
237 }
238 }
239
240 // Batch collection lookup by AT URI
241 for (let i = 0; i < atUris.length; i += CHUNK) {
242 const chunk = atUris.slice(i, i + CHUNK);
243 const placeholders = chunk.map(() => "?").join(",");
244 const rows = await db
245 .prepare(`SELECT uri, record FROM records_collection WHERE uri IN (${placeholders})`)
246 .bind(...chunk)
247 .all<{ uri: string; record: string }>();
248
249 for (const row of rows.results || []) {
250 try {
251 const coll = JSON.parse(row.record);
252 if (meta.has(row.uri)) {
253 meta.set(row.uri, {
254 title: coll.name || row.uri,
255 description: coll.description || "",
256 type: "collection",
257 });
258 }
259 } catch {}
260 }
261 }
262
263 return meta;
264}
265
266// ─── Derive island from lexmin ────────────────────────────────────
267//
268// Given a lexmin vertex (canonical component reference), derive the
269// connected component by traversing network.cosmik.connection records.
270
271async function deriveIsland(
272 db: D1Database,
273 seed: string,
274): Promise<Island | null> {
275 await ensureContrailReady(db);
276
277 // BFS in application code — recursive CTE OOMs in D1 due to cross join
278 const visited = new Set<string>();
279 const queue = [seed];
280 const allEdges: Array<{
281 uri: string; did: string; rkey: string;
282 source: string; target: string; connectionType: string;
283 note: string; handle: string | null;
284 }> = [];
285
286 while (queue.length > 0) {
287 // Drain the current queue into a batch
288 const batch = queue.splice(0, 50);
289 const toQuery = batch.filter(v => !visited.has(v));
290 if (toQuery.length === 0) continue;
291
292 // Mark visited
293 for (const v of toQuery) visited.add(v);
294
295 // Query connections where any of these vertices are source or target
296 const placeholders = toQuery.map(() => "?").join(",");
297 const rows = await db
298 .prepare(
299 `SELECT r.record, r.did, r.rkey, i.handle
300 FROM records_connection r
301 LEFT JOIN identities i ON r.did = i.did
302 WHERE json_extract(r.record, '$.source') IN (${placeholders})
303 OR json_extract(r.record, '$.target') IN (${placeholders})`,
304 )
305 .bind(...toQuery, ...toQuery)
306 .all<{ record: string; did: string; rkey: string; handle: string | null }>();
307
308 for (const row of rows.results || []) {
309 try {
310 const value = JSON.parse(row.record);
311 const source = value.source as string;
312 const target = value.target as string;
313 if (!source || !target) continue;
314
315 // Collect edge
316 const edgeKey = `${source}|${target}|${row.rkey}`;
317 allEdges.push({
318 uri: `at://${row.did}/network.cosmik.connection/${row.rkey}`,
319 did: row.did,
320 rkey: row.rkey,
321 source,
322 target,
323 connectionType: value.connectionType || "relates",
324 note: value.note || "",
325 handle: row.handle,
326 });
327
328 // Enqueue unvisited neighbors
329 if (!visited.has(source)) queue.push(source);
330 if (!visited.has(target)) queue.push(target);
331 } catch {}
332 }
333 }
334
335 if (visited.size === 0) return null;
336
337 const vertices = [...visited];
338
339 // Filter edges to only those with both endpoints in the component
340 const vertexSet = new Set(vertices);
341 const edges: Island["edges"] = [];
342 const seenEdges = new Set<string>();
343 for (const e of allEdges) {
344 if (!vertexSet.has(e.source) || !vertexSet.has(e.target)) continue;
345 const edgeKey = `${e.source}|${e.target}|${e.rkey}`;
346 if (seenEdges.has(edgeKey)) continue;
347 seenEdges.add(edgeKey);
348 edges.push({
349 uri: e.uri,
350 did: e.did,
351 source: e.source,
352 target: e.target,
353 connectionType: e.connectionType,
354 note: e.note,
355 handle: e.handle,
356 });
357 }
358
359 // Compute stable ID from lexmin
360 const sortedVertices = [...vertices].sort();
361 const actualLexmin = sortedVertices[0];
362 const lexminHash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(actualLexmin));
363 const id = Array.from(new Uint8Array(lexminHash)).map(b => b.toString(16).padStart(2, "0")).join("").slice(0, 16);
364
365 return { id, vertices, edges };
366}
367
368// ─── List strata islands from records_strata ──────────────────────
369
370async function listStrataIslands(db: D1Database): Promise<any[]> {
371 await ensureContrailReady(db);
372
373 const rows = await db
374 .prepare("SELECT uri, record FROM records_strata ORDER BY time_us DESC LIMIT 50")
375 .all<{ uri: string; record: string }>();
376
377 const results: any[] = [];
378 for (const row of rows.results || []) {
379 try {
380 const rec = JSON.parse(row.record);
381 const lexmin = rec.source?.uri;
382 if (!lexmin) continue;
383
384 const island = await deriveIsland(db, lexmin);
385 if (!island) continue;
386
387 const meta = await resolveVertexMeta(db, island.vertices);
388 const analysis = rec.analysis || {};
389
390 results.push({
391 ...island,
392 vertexMeta: Object.fromEntries(meta),
393 summary: analysis.title || null,
394 strata: {
395 prose: analysis.synthesis || "",
396 title: analysis.title || "",
397 themes: analysis.themes || [],
398 relationships: (analysis.connections || []).map((c: any) => c.description || ""),
399 tensions: analysis.tensions || [],
400 open_questions: analysis.openQuestions || [],
401 synthesis: analysis.synthesis || "",
402 },
403 recordUri: row.uri,
404 });
405 } catch {}
406 }
407
408 return results;
409}
410
411// ─── Island cache ──────────────────────────────────────────────────
412//
413// ─── Hono app ───────────────────────────────────────────────────────
414
415let app: Hono<{ Bindings: Env }> | null = null;
416
417function buildApp(env: Env): Hono<{ Bindings: Env }> {
418 const app = new Hono<{ Bindings: Env }>();
419 const db = env.DB;
420
421 app.use("*", cors());
422
423 // Landing page — strata records feed
424 app.get("/", async (c) => {
425 let islandsJson = "[]";
426 try {
427 const islands = await listStrataIslands(db);
428 islandsJson = JSON.stringify(islands);
429 } catch (e) {
430 console.error("Failed to load islands:", e);
431 }
432
433 const page = LANDING_PAGE.replace(
434 "</head>",
435 `<script>window.__ISLANDS__=${islandsJson};</script></head>`,
436 );
437 return c.html(page);
438 });
439
440 // ── Islands API ──────────────────────────────────────────────────
441
442 app.get("/xrpc/org.latha.strata.getIslands", async (c) => {
443 const islands = await listStrataIslands(db);
444 return c.json({ islands });
445 });
446
447 // ── List strata records (AT URI-based) ──────────────────────────
448
449 app.get("/xrpc/org.latha.strata.listRecords", async (c) => {
450 const rows = await db
451 .prepare("SELECT uri, record FROM records_strata ORDER BY time_us DESC LIMIT 50")
452 .all<{ uri: string; record: string }>();
453
454 const records = (rows.results || []).map(r => {
455 try {
456 const rec = JSON.parse(r.record);
457 return {
458 uri: r.uri,
459 source: rec.source?.uri || "",
460 title: rec.analysis?.title || "",
461 themes: rec.analysis?.themes || [],
462 createdAt: rec.createdAt || "",
463 };
464 } catch {
465 return null;
466 }
467 }).filter(Boolean);
468
469 return c.json({ records });
470 });
471
472 // ── Strata record endpoint (AT URI-based) ──────────────────────
473
474 app.get("/xrpc/org.latha.strata.getRecord", async (c) => {
475 const uri = c.req.query("uri");
476 if (!uri) {
477 return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
478 }
479
480 // Fetch the strata record from D1 (indexed by Contrail)
481 const row = await db
482 .prepare("SELECT record FROM records_strata WHERE uri = ?")
483 .bind(uri)
484 .first<{ record: string }>();
485
486 if (!row) {
487 return c.json({ error: "NotFound", message: "Strata record not found" }, 404);
488 }
489
490 let strataRecord: any;
491 try {
492 strataRecord = JSON.parse(row.record);
493 } catch {
494 return c.json({ error: "InvalidRecord" }, 500);
495 }
496
497 // Derive island from lexmin (source.uri)
498 const lexmin = strataRecord.source?.uri;
499 if (!lexmin) {
500 return c.json({ error: "MissingLexmin", message: "Strata record has no source.uri" }, 400);
501 }
502
503 const island = await deriveIsland(db, lexmin);
504 if (!island) {
505 return c.json({ error: "IslandNotFound", message: "Could not derive island from lexmin" }, 404);
506 }
507
508 // Resolve vertex metadata
509 const meta = await resolveVertexMeta(db, island.vertices);
510
511 // Build the response in the same format as the island cache
512 const analysis = strataRecord.analysis || {};
513 const strataData = {
514 prose: analysis.synthesis || "",
515 title: analysis.title || "",
516 themes: analysis.themes || [],
517 relationships: (analysis.connections || []).map((c: any) => c.description || ""),
518 tensions: analysis.tensions || [],
519 open_questions: analysis.openQuestions || [],
520 synthesis: analysis.synthesis || "",
521 };
522
523 return c.json({
524 island: {
525 ...island,
526 vertexMeta: Object.fromEntries(meta),
527 summary: analysis.title || null,
528 strata: strataData,
529 },
530 record: strataRecord,
531 recordUri: uri,
532 });
533 });
534
535 // ── Container-backed xrpc methods ────────────────────────────────
536
537 // Helper: call the strata container
538 async function callContainer(path: string, method: string, body?: any): Promise<any> {
539 const id = env.STRATA_CONTAINER.idFromName("strata");
540 const stub = env.STRATA_CONTAINER.get(id);
541 const url = new URL(path, "http://container");
542 const init: RequestInit = { method };
543 if (body) {
544 init.body = JSON.stringify(body);
545 init.headers = { "Content-Type": "application/json" };
546 }
547 const resp = await stub.fetch(new Request(url.toString(), init));
548 return resp.json();
549 }
550
551 // Derive islands — delegates to container
552 // With seed: single island. Without: all islands.
553 app.get("/xrpc/org.latha.strata.deriveIsland", async (c) => {
554 const seed = c.req.query("seed");
555
556 // Load all connections from D1 for the container to traverse
557 await ensureContrailReady(db);
558 const rows = await db
559 .prepare(
560 `SELECT r.record, r.did, r.rkey, i.handle
561 FROM records_connection r
562 LEFT JOIN identities i ON r.did = i.did`,
563 )
564 .all<{ record: string; did: string; rkey: string; handle: string | null }>();
565
566 const connections = (rows.results || []).map(r => {
567 try {
568 const value = JSON.parse(r.record);
569 return {
570 source: value.source,
571 target: value.target,
572 connectionType: value.connectionType || "relates",
573 note: value.note || "",
574 did: r.did,
575 rkey: r.rkey,
576 };
577 } catch {
578 return null;
579 }
580 }).filter(Boolean);
581
582 try {
583 const result = await callContainer("/deriveIsland", "POST", { connections, seed: seed || undefined });
584 return c.json(result);
585 } catch (e: any) {
586 return c.json({ error: "ContainerError", message: e.message }, 500);
587 }
588 });
589
590 // Run strata analysis — delegates to container
591 app.post("/xrpc/org.latha.strata.analyze", async (c) => {
592 const body = await c.req.json().catch(() => ({}));
593 if (!body.island) {
594 return c.json({ error: "MissingRequiredField", message: "island is required" }, 400);
595 }
596
597 try {
598 const result = await callContainer("/analyze", "POST", body);
599 return c.json(result);
600 } catch (e: any) {
601 return c.json({ error: "ContainerError", message: e.message }, 500);
602 }
603 });
604
605 // ── R2 sync endpoints ───────────────────────────────────────────
606
607 app.put("/api/sync/vault/:did/*", async (c) => {
608 const did = c.req.param("did");
609 const path = c.req.param("path");
610 if (!did || !path) {
611 return c.json({ error: "Missing path" }, 400);
612 }
613
614 const key = `${did}/${path}`;
615 const body = await c.req.raw.arrayBuffer();
616 await env.VAULT_BUCKET.put(key, body, {
617 httpMetadata: { contentType: c.req.header("content-type") || "text/markdown" },
618 });
619
620 return c.json({ ok: true, key });
621 });
622
623 app.put("/api/sync/carry/:did/*", async (c) => {
624 const did = c.req.param("did");
625 const path = c.req.param("path");
626 if (!did || !path) {
627 return c.json({ error: "Missing path" }, 400);
628 }
629
630 const key = `${did}/${path}`;
631 const body = await c.req.raw.arrayBuffer();
632 await env.CARRY_BUCKET.put(key, body, {
633 httpMetadata: { contentType: c.req.header("content-type") || "application/json" },
634 });
635
636 return c.json({ ok: true, key });
637 });
638
639 // ── Graph neighborhood query ────────────────────────────────────
640
641 app.get("/xrpc/org.latha.strata.connection.getGraph", async (c) => {
642 const uri = c.req.query("uri");
643 if (!uri) {
644 return c.json({ error: "MissingRequiredParameter", message: "uri is required" }, 400);
645 }
646
647 const depth = Math.min(parseInt(c.req.query("depth") || "2"), 3);
648 const limit = Math.min(parseInt(c.req.query("limit") || "50"), 100);
649 const types = c.req.query("types")?.split(",").map((t) => t.trim());
650
651 // Reuse island detection for the subgraph
652 const islands = await detectIslands(db);
653 const island = islands.find(i => i.vertices.includes(uri));
654
655 if (!island) {
656 return c.json({ connections: [], resources: [], depth, uri });
657 }
658
659 // Convert to graph format
660 const resources = island.vertices.map(v => ({
661 uri: v,
662 title: v,
663 type: "unknown",
664 }));
665
666 return c.json({
667 connections: island.edges,
668 resources,
669 depth,
670 uri,
671 });
672 });
673
674 // ── OAuth client metadata ──────────────────────────────────────
675
676 app.get("/oauth-client-metadata.json", (c) => {
677 const host = new URL(c.req.url).host;
678 return c.json({
679 client_id: `https://${host}/oauth-client-metadata.json`,
680 client_name: "Stigmergic",
681 client_uri: `https://${host}`,
682 redirect_uris: [`https://${host}/`],
683 scope: "atproto transition:generic",
684 grant_types: ["authorization_code", "refresh_token"],
685 response_types: ["code"],
686 token_endpoint_auth_method: "none",
687 application_type: "web",
688 dpop_bound_access_tokens: true,
689 });
690 });
691
692 // ── SPA fallback: serve landing page for client-side routes ──────
693 for (const path of ["/island", "/strata", "/connect"]) {
694 app.get(path, async (c) => {
695 let islandsJson = "[]";
696 try {
697 const islands = await listStrataIslands(db);
698 islandsJson = JSON.stringify(islands);
699 } catch {}
700
701 const page = LANDING_PAGE.replace(
702 "</head>",
703 `<script>window.__ISLANDS__=${islandsJson};</script></head>`,
704 );
705 return c.html(page);
706 });
707 }
708
709 // ── All other routes pass through to contrail ──────────────────
710
711 app.all("*", async (c) => {
712 const response = await contrailWorker.fetch(
713 c.req.raw,
714 c.env as unknown as Record<string, unknown>,
715 );
716 return response;
717 });
718
719 return app;
720}
721
722// ─── Export ─────────────────────────────────────────────────────────
723
724export { StrataContainer };
725
726export default {
727 fetch(request: Request, env: Env): Response | Promise<Response> {
728 app ??= buildApp(env);
729 return app.fetch(request, env);
730 },
731 async scheduled(event: ScheduledEvent, env: Env, ctx: ExecutionContext) {
732 // Run Contrail indexing
733 await contrailWorker.scheduled(
734 event,
735 env as unknown as Record<string, unknown>,
736 ctx,
737 );
738 },
739};
740