+2 -1

lex.config.js

View file Reviewed

··· 1 1 import { defineLexiconConfig } from "@atcute/lex-cli"; 2 2 3 3 export default defineLexiconConfig({ 4 4 - files: ["lexicons/org/**/*.json", "lexicons/pulled/**/*.json"], 4 4 + files: ["lexicons/custom/**/*.json", "lexicons/pulled/**/*.json", "lexicons/generated/**/*.json"], 5 5 outdir: "src/lexicon-types/", 6 6 imports: ["@atcute/atproto"], 7 7 pull: { ··· 17 17 "network.cosmik.connection", 18 18 "network.cosmik.defs", 19 19 "network.cosmik.follow", 20 20 + "org.latha.island", 20 21 "org.latha.island.citation", 21 22 "org.latha.island.entity", 22 23 "org.latha.island.reasoning"

+1

package.json

View file Reviewed

··· 14 14 "island:analyze": "bun run src/island-analyze.ts", 15 15 "connection:detect": "bun run src/connection-detect.ts", 16 16 "island:publish": "bun run src/island-publish.ts", 17 17 + "island:cleanup": "bun run src/island-cleanup.ts", 17 18 "vault:analyze": "bun run src/vault-analyze.ts" 18 19 }, 19 20 "dependencies": {

+109 -123

src/frontend/app.ts

View file Reviewed

··· 28 28 interface Island { 29 29 id: string; 30 30 islandHash?: string; 31 31 - lexmin?: string; 31 31 + centroid?: string; 32 32 vertices?: string[]; 33 33 edges?: IslandEdge[]; 34 34 vertexMeta?: Record<string, VertexMeta>; ··· 36 36 title?: string | null; 37 37 strata: StrataData | null; 38 38 recordUri?: string | null; 39 39 + handle?: string | null; 39 40 } 40 41 41 42 interface StrataData { ··· 377 378 378 379 // ── Rendering helpers ────────────────────────────────────────────────────── 379 380 381 381 + // Compute island hash from centroid: first 12 hex chars of SHA-256 382 382 + async function computeIslandHash(centroid: string): Promise<string> { 383 383 + const data = new TextEncoder().encode(centroid); 384 384 + const hashBuffer = await crypto.subtle.digest("SHA-256", data); 385 385 + const hashArray = Array.from(new Uint8Array(hashBuffer)); 386 386 + return hashArray.map(b => b.toString(16).padStart(2, "0")).join("").slice(0, 12); 387 387 + } 388 388 + 389 389 + // Populate islandHash on all islands that have centroid but no hash 390 390 + async function enrichIslandHashes(list: Island[]): Promise<void> { 391 391 + for (const i of list) { 392 392 + if (!i.islandHash && i.centroid) { 393 393 + i.islandHash = await computeIslandHash(i.centroid); 394 394 + } 395 395 + } 396 396 + } 397 397 + 398 398 + function islandPath(island: Island): string { 399 399 + if (island.recordUri) { 400 400 + const parts = island.recordUri.split("/"); 401 401 + const rkey = parts[parts.length - 1]; 402 402 + const repo = island.handle || parts[2]; 403 403 + return `/island/${repo}/${rkey}`; 404 404 + } 405 405 + // Fallback to hash 406 406 + return `/island/${encodeURIComponent(island.islandHash || island.id)}`; 407 407 + } 408 408 + 380 409 function escHtml(s: string): string { 381 410 return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;"); 382 411 } ··· 481 510 const islandId = target.dataset.islandId; 482 511 if (!islandId) return; 483 512 const island = islands.find(i => i.id === islandId); 484 484 - window.location.href = `/island/${encodeURIComponent(island?.islandHash || islandId)}`; 513 513 + window.location.href = islandPath(island); 485 514 }); 486 515 }); 487 516 } ··· 615 644 } 616 645 617 646 document.getElementById("island-strata-btn")?.addEventListener("click", async () => { 618 618 - window.location.href = `/island/${encodeURIComponent(island.islandHash || islandId)}`; 647 647 + window.location.href = islandPath(island); 619 648 }); 620 649 } 621 650 622 651 // ── Strata page — prose with arrows ─────────────────────────────────────── 623 652 653 653 + async function showStrataByRecord(repo: string, rkey: string): Promise<void> { 654 654 + const atUri = `at://${repo}/org.latha.island/${rkey}`; 655 655 + currentPage = "strata"; 656 656 + const content = document.getElementById("page-content"); 657 657 + if (!content) return; 658 658 + 659 659 + try { 660 660 + const data = await apiGet<{ island: Island; recordUri: string }>(`/xrpc/org.latha.island.getIsland?id=${encodeURIComponent(atUri)}`); 661 661 + await renderStrataPage(data.island, data.recordUri); 662 662 + } catch { 663 663 + content.innerHTML = '<div class="empty">Island record not found.</div>'; 664 664 + } 665 665 + } 666 666 + 667 667 + async function showStrataByLegacyId(legacyId: string): Promise<void> { 668 668 + // Resolve legacy 12-char hash to a record, then redirect to canonical path 669 669 + currentPage = "strata"; 670 670 + const content = document.getElementById("page-content"); 671 671 + if (!content) return; 672 672 + 673 673 + try { 674 674 + const data = await apiGet<{ island: Island; recordUri: string }>(`/xrpc/org.latha.island.getIsland?id=${encodeURIComponent(legacyId)}`); 675 675 + if (data.recordUri) { 676 676 + const parts = data.recordUri.split("/"); 677 677 + const recordRkey = parts[parts.length - 1]; 678 678 + const repo = data.island.handle || parts[2]; 679 679 + window.history.replaceState(null, "", `/island/${repo}/${recordRkey}`); 680 680 + } 681 681 + await renderStrataPage(data.island, data.recordUri); 682 682 + } catch { 683 683 + content.innerHTML = '<div class="empty">Island not found.</div>'; 684 684 + } 685 685 + } 686 686 + 687 687 + async function renderStrataPage(island: Island, recordUri?: string): Promise<void> { 688 688 + if (recordUri) island.recordUri = recordUri; 689 689 + 690 690 + if (!island.strata) { 691 691 + const content = document.getElementById("page-content"); 692 692 + if (!content) return; 693 693 + content.innerHTML = ` 694 694 + <div class="strata-page"> 695 695 + <a href="/" class="back-link">&larr; Explore</a> 696 696 + <h2>Strata</h2> 697 697 + <div class="empty">No strata analysis yet for this island. Run <code>scripts/strata-analyze.py</code> to generate.</div> 698 698 + </div> 699 699 + `; 700 700 + return; 701 701 + } 702 702 + 703 703 + renderStrata(island); 704 704 + } 705 705 + 624 706 async function showStrata(islandId: string): Promise<void> { 625 707 currentPage = "strata"; 626 708 const content = document.getElementById("page-content"); 627 709 if (!content) return; 628 710 629 711 let island: Island | null = null; 712 712 + let recordUri: string | undefined; 630 713 631 631 - // If the ID is an AT URI, fetch from the strata record endpoint 714 714 + // If the ID is an AT URI, fetch from the island endpoint 632 715 if (islandId.startsWith("at://")) { 633 716 try { 634 634 - const data = await apiGet<{ island: Island; recordUri: string }>(`/xrpc/org.latha.island.getRecord?uri=${encodeURIComponent(islandId)}`); 717 717 + const data = await apiGet<{ island: Island; recordUri: string }>(`/xrpc/org.latha.island.getIsland?id=${encodeURIComponent(islandId)}`); 635 718 island = data.island; 636 636 - island.recordUri = data.recordUri; 719 719 + recordUri = data.recordUri; 637 720 } catch { 638 721 content.innerHTML = '<div class="empty">Island record not found.</div>'; 639 722 return; 640 723 } 641 724 } else { 642 725 // Island hash or legacy ID — find from loaded islands 643 643 - island = islands.find(i => i.islandHash === islandId || i.id === islandId) || null; 726 726 + await enrichIslandHashes(islands); 727 727 + const matchIsland = (list: Island[]): Island | undefined => 728 728 + list.find(i => i.islandHash === islandId || i.id === islandId); 729 729 + 730 730 + island = matchIsland(islands) || null; 644 731 645 732 // If not in memory, fetch from API 646 733 if (!island) { 647 734 try { 648 735 const data = await apiGet<{ islands: Island[] }>("/xrpc/org.latha.island.getIslands"); 649 649 - island = data.islands.find(i => i.islandHash === islandId || i.id === islandId) || null; 736 736 + await enrichIslandHashes(data.islands); 737 737 + island = matchIsland(data.islands) || null; 650 738 if (island) islands = data.islands; 651 739 } catch {} 652 740 } ··· 657 745 return; 658 746 } 659 747 660 660 - if (!island.strata) { 661 661 - content.innerHTML = ` 662 662 - <div class="strata-page"> 663 663 - <a href="/" class="back-link">&larr; Explore</a> 664 664 - <h2>Strata</h2> 665 665 - <div class="empty">No strata analysis yet for this island. Run <code>scripts/strata-analyze.py</code> to generate.</div> 666 666 - </div> 667 667 - `; 668 668 - return; 669 669 - } 670 670 - 671 671 - renderStrata(island); 748 748 + await renderStrataPage(island, recordUri); 672 749 } 673 750 674 751 function renderStrata(island: Island): void { ··· 717 794 </div> 718 795 719 796 <div class="strata-actions"> 720 720 - <button class="strata-btn regenerate-btn" id="regenerate-btn">Regenerate</button> 721 797 <button class="strata-btn save-btn hidden" id="save-btn">Save to PDS</button> 722 798 <span class="save-status" id="save-status"></span> 723 799 </div> ··· 874 950 } 875 951 }); 876 952 877 877 - // ── Regenerate ───────────────────────────────────────────────── 878 878 - const regenBtn = document.getElementById("regenerate-btn"); 879 879 - regenBtn?.addEventListener("click", async () => { 880 880 - regenBtn.textContent = "Generating..."; 881 881 - regenBtn.setAttribute("disabled", "true"); 882 882 - 883 883 - try { 884 884 - const res = await fetch("/xrpc/org.latha.island.analyze", { 885 885 - method: "POST", 886 886 - headers: { "Content-Type": "application/json" }, 887 887 - body: JSON.stringify({ 888 888 - did: session?.did, 889 889 - island: { 890 890 - vertices: (island.vertices || []).map(uri => ({ 891 891 - uri, 892 892 - title: island.vertexMeta?.[uri]?.title || "", 893 893 - description: island.vertexMeta?.[uri]?.description || "", 894 894 - type: island.vertexMeta?.[uri]?.type || "url", 895 895 - })), 896 896 - edges: island.edges || [], 897 897 - }, 898 898 - }), 899 899 - }); 900 900 - if (!res.ok) throw new Error("Analysis failed"); 901 901 - const result = await res.json(); 902 902 - 903 903 - // Diff-highlight: mark changed fields 904 904 - const oldAnalysis = collectAnalysis(); 905 905 - const newAnalysis = result as Record<string, any>; 906 906 - 907 907 - // Update fields with animation 908 908 - const titleEl = editContainer.querySelector('[data-field="title"]') as HTMLElement; 909 909 - const synthesisEl = editContainer.querySelector('[data-field="synthesis"]') as HTMLElement; 910 910 - 911 911 - if (titleEl && newAnalysis.title !== oldAnalysis.title) { 912 912 - titleEl.textContent = newAnalysis.title; 913 913 - titleEl.classList.add("field-changed"); 914 914 - setTimeout(() => titleEl.classList.remove("field-changed"), 2000); 915 915 - } 916 916 - if (synthesisEl && newAnalysis.synthesis !== oldAnalysis.synthesis) { 917 917 - synthesisEl.textContent = newAnalysis.synthesis; 918 918 - synthesisEl.classList.add("field-changed"); 919 919 - setTimeout(() => synthesisEl.classList.remove("field-changed"), 2000); 920 920 - } 921 921 - 922 922 - // Update themes 923 923 - const themesContainer = editContainer.querySelector('[data-field="themes"]'); 924 924 - if (themesContainer) { 925 925 - themesContainer.querySelectorAll(".theme-tag").forEach(t => t.remove()); 926 926 - const addBtn = themesContainer.querySelector(".theme-add"); 927 927 - for (const theme of newAnalysis.themes || []) { 928 928 - const tag = document.createElement("span"); 929 929 - tag.className = "theme-tag field-changed"; 930 930 - tag.dataset.value = theme; 931 931 - tag.innerHTML = `${escHtml(theme)}<span class="theme-remove">&times;</span>`; 932 932 - tag.querySelector(".theme-remove")!.addEventListener("click", () => { tag.remove(); markDirty(); }); 933 933 - (addBtn as ChildNode)?.before(tag); 934 934 - setTimeout(() => tag.classList.remove("field-changed"), 2000); 935 935 - } 936 936 - } 937 937 - 938 938 - // Update tensions 939 939 - const tensionsContainer = editContainer.querySelector('[data-field="tensions"]'); 940 940 - if (tensionsContainer) { 941 941 - tensionsContainer.querySelectorAll(".editable").forEach(t => t.remove()); 942 942 - const addBtn = tensionsContainer.querySelector(".list-add"); 943 943 - for (const t of newAnalysis.tensions || []) { 944 944 - const div = document.createElement("div"); 945 945 - div.className = "analysis-tension editable field-changed"; 946 946 - div.setAttribute("contenteditable", "true"); 947 947 - div.textContent = t; 948 948 - div.addEventListener("input", markDirty); 949 949 - (addBtn as ChildNode)?.before(div); 950 950 - setTimeout(() => div.classList.remove("field-changed"), 2000); 951 951 - } 952 952 - } 953 953 - 954 954 - // Update questions 955 955 - const questionsContainer = editContainer.querySelector('[data-field="openQuestions"]'); 956 956 - if (questionsContainer) { 957 957 - questionsContainer.querySelectorAll(".editable").forEach(q => q.remove()); 958 958 - const addBtn = questionsContainer.querySelector(".list-add"); 959 959 - for (const q of newAnalysis.openQuestions || []) { 960 960 - const div = document.createElement("div"); 961 961 - div.className = "analysis-question editable field-changed"; 962 962 - div.setAttribute("contenteditable", "true"); 963 963 - div.textContent = q; 964 964 - div.addEventListener("input", markDirty); 965 965 - (addBtn as ChildNode)?.before(div); 966 966 - setTimeout(() => div.classList.remove("field-changed"), 2000); 967 967 - } 968 968 - } 969 969 - 970 970 - markDirty(); // Regenerated = needs save 971 971 - } catch (e: any) { 972 972 - saveStatus!.textContent = `Regenerate error: ${e.message}`; 973 973 - } finally { 974 974 - regenBtn.textContent = "Regenerate"; 975 975 - regenBtn.removeAttribute("disabled"); 976 976 - } 977 977 - }); 978 953 } 979 954 980 955 // ── Connect page ─────────────────────────────────────────────────────────── ··· 1066 1041 const serverData = (window as any).__ISLANDS__; 1067 1042 if (serverData) islands = serverData; 1068 1043 } 1044 1044 + await enrichIslandHashes(islands); 1069 1045 1070 1046 renderExplore(); 1071 1047 } ··· 1079 1055 if (path === "/island") { 1080 1056 return { page: "island", params: { id: url.searchParams.get("id") || "" } }; 1081 1057 } 1058 1058 + // /island/<repo>/<rkey> — AT Protocol record path 1059 1059 + const repoRkeyMatch = path.match(/^\/island\/([^/]+)\/([^/]+)$/); 1060 1060 + if (repoRkeyMatch) { 1061 1061 + return { page: "strata", params: { repo: repoRkeyMatch[1], rkey: repoRkeyMatch[2] } }; 1062 1062 + } 1063 1063 + // Legacy: /island/<12-char-hash> — redirect to record path 1082 1064 const islandMatch = path.match(/^\/island\/([0-9a-f]{12})$/); 1083 1065 if (islandMatch) { 1084 1084 - return { page: "strata", params: { id: islandMatch[1] } }; 1066 1066 + return { page: "strata", params: { legacyId: islandMatch[1] } }; 1085 1067 } 1086 1068 if (path === "/strata") { 1087 1069 // Legacy redirect ··· 1103 1085 1104 1086 if (page === "island" && params.id) { 1105 1087 await showIsland(params.id); 1088 1088 + } else if (page === "strata" && params.repo && params.rkey) { 1089 1089 + await showStrataByRecord(params.repo, params.rkey); 1090 1090 + } else if (page === "strata" && params.legacyId) { 1091 1091 + await showStrataByLegacyId(params.legacyId); 1106 1092 } else if (page === "strata" && params.id) { 1107 1093 await showStrata(params.id); 1108 1094 } else if (page === "connect") {

+314

src/island-cleanup.ts

View file Reviewed

··· 1 1 + /** 2 2 + * island:cleanup 3 3 + * 4 4 + * Detect and remove low-quality edges from the island-detect SQLite DB. 5 5 + * Runs in dry-run mode by default — use --apply to actually clean. 6 6 + * 7 7 + * Categories: 8 8 + * at-uri — source or target is an at:// URI (not HTTPS) 9 9 + * null-relation — relation is NULL or empty 10 10 + * case-variant — relation is lowercase variant of a known type 11 11 + * bsky-app-noise — bsky.app/profile/HANDLE/post/RKEY URLs (post permalinks, not content) 12 12 + * same-domain-trivial — source and target share a registered domain, one is a subpage of the other 13 13 + * 14 14 + * Usage: 15 15 + * bun island:cleanup # dry-run: report what would be cleaned 16 16 + * bun island:cleanup --apply # actually apply the cleanup 17 17 + * bun island:cleanup --apply --json # apply + JSON output 18 18 + */ 19 19 + 20 20 + import { resolve, dirname } from 'node:path'; 21 21 + import { fileURLToPath } from 'node:url'; 22 22 + import { Database } from 'bun:sqlite'; 23 23 + import { loadDotEnv } from './cli-utils.js'; 24 24 + import { domainFromUrl } from './island-shared.js'; 25 25 + 26 26 + const __dirname = dirname(fileURLToPath(import.meta.url)); 27 27 + await loadDotEnv(resolve(__dirname, '..', '.env')); 28 28 + 29 29 + const CARRY_DIR = process.env.CARRY_DIR ?? '/home/nandi/.local/share/carry-vault'; 30 30 + const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db'); 31 31 + 32 32 + // --- Edge classification --- 33 33 + 34 34 + interface Edge { 35 35 + source: string; 36 36 + target: string; 37 37 + relation: string | null; 38 38 + did: string; 39 39 + } 40 40 + 41 41 + interface CleanupReport { 42 42 + atUri: { count: number; samples: Edge[] }; 43 43 + nullRelation: { count: number; samples: Edge[] }; 44 44 + caseVariant: { count: number; samples: Edge[]; normalized: Map<string, string> }; 45 45 + bskyAppNoise: { count: number; samples: Edge[] }; 46 46 + sameDomainTrivial: { count: number; samples: Edge[] }; 47 47 + totalEdges: number; 48 48 + } 49 49 + 50 50 + const BSky_POST_RE = /^https:\/\/bsky\.app\/profile\/[^/]+\/post\//; 51 51 + const CANONICAL_RELATIONS = new Set([ 52 52 + 'RELATED', 'LEADS_TO', 'SUPPORTS', 'SUPPLEMENT', 'SUPPLEMENTS', 53 53 + 'EXPLAINER', 'HELPFUL', 'ADDRESSES', 'ANNOTATES', 'OPPOSES', 54 54 + 'CONTAINS', 'RELATES_TO', 55 55 + ]); 56 56 + 57 57 + /** Extract registered domain: strip www., take hostname. */ 58 58 + function registeredDomain(url: string): string { 59 59 + try { 60 60 + return new URL(url).hostname.replace(/^www\./, ''); 61 61 + } catch { 62 62 + return url; 63 63 + } 64 64 + } 65 65 + 66 66 + /** Check if one URL is a trivial same-domain link (subpage → root, www → non-www). */ 67 67 + function isTrivialSameDomain(source: string, target: string): boolean { 68 68 + const sDomain = registeredDomain(source); 69 69 + const tDomain = registeredDomain(target); 70 70 + if (sDomain !== tDomain) return false; 71 71 + 72 72 + // Normalize scheme and www only (keep query params — different ?v= means different content) 73 73 + const normalize = (u: string) => { 74 74 + try { 75 75 + const parsed = new URL(u); 76 76 + parsed.hostname = parsed.hostname.replace(/^www\./, ''); 77 77 + // Force https 78 78 + if (parsed.protocol === 'http:') parsed.protocol = 'https:'; 79 79 + parsed.hash = ''; 80 80 + return parsed.toString().replace(/\/+$/, ''); 81 81 + } catch { 82 82 + return u.replace(/\/+$/, ''); 83 83 + } 84 84 + }; 85 85 + 86 86 + const s = normalize(source); 87 87 + const t = normalize(target); 88 88 + 89 89 + // Exact duplicate after scheme/www normalization 90 90 + if (s === t) return true; 91 91 + 92 92 + // One is the root (no path beyond /) and the other is a subpage 93 93 + // This catches: attie.ai/login → attera.org/ but NOT: youtube.com/watch?v=X → youtube.com/watch?v=Y 94 94 + const sUrl = new URL(s); 95 95 + const tUrl = new URL(t); 96 96 + const sPath = sUrl.pathname.replace(/\/+$/, ''); 97 97 + const tPath = tUrl.pathname.replace(/\/+$/, ''); 98 98 + 99 99 + // Only trivial if one has an empty path (root) and the other doesn't 100 100 + if ((sPath === '' || sPath === '/') && tPath !== '' && tPath !== '/') return true; 101 101 + if ((tPath === '' || tPath === '/') && sPath !== '' && sPath !== '/') return true; 102 102 + 103 103 + return false; 104 104 + } 105 105 + 106 106 + function classifyEdges(edges: Edge[]): CleanupReport { 107 107 + const report: CleanupReport = { 108 108 + atUri: { count: 0, samples: [] }, 109 109 + nullRelation: { count: 0, samples: [] }, 110 110 + caseVariant: { count: 0, samples: [], normalized: new Map() }, 111 111 + bskyAppNoise: { count: 0, samples: [] }, 112 112 + sameDomainTrivial: { count: 0, samples: [] }, 113 113 + totalEdges: edges.length, 114 114 + }; 115 115 + 116 116 + const addSample = (arr: Edge[], e: Edge) => { 117 117 + if (arr.length < 5) arr.push(e); 118 118 + }; 119 119 + 120 120 + for (const e of edges) { 121 121 + // at-uri: source or target starts with at:// 122 122 + if (e.source.startsWith('at://') || e.target.startsWith('at://')) { 123 123 + report.atUri.count++; 124 124 + addSample(report.atUri.samples, e); 125 125 + continue; // don't double-count 126 126 + } 127 127 + 128 128 + // null-relation 129 129 + if (!e.relation || e.relation.trim() === '') { 130 130 + report.nullRelation.count++; 131 131 + addSample(report.nullRelation.samples, e); 132 132 + continue; 133 133 + } 134 134 + 135 135 + // case-variant: relation != UPPER(relation) and UPPER is a known type 136 136 + const upper = e.relation.toUpperCase(); 137 137 + if (e.relation !== upper && CANONICAL_RELATIONS.has(upper)) { 138 138 + report.caseVariant.count++; 139 139 + report.caseVariant.normalized.set(e.relation, upper); 140 140 + addSample(report.caseVariant.samples, e); 141 141 + continue; 142 142 + } 143 143 + 144 144 + // bsky-app-noise: post permalinks 145 145 + if (BSky_POST_RE.test(e.source) || BSky_POST_RE.test(e.target)) { 146 146 + report.bskyAppNoise.count++; 147 147 + addSample(report.bskyAppNoise.samples, e); 148 148 + continue; 149 149 + } 150 150 + 151 151 + // same-domain-trivial 152 152 + if (isTrivialSameDomain(e.source, e.target)) { 153 153 + report.sameDomainTrivial.count++; 154 154 + addSample(report.sameDomainTrivial.samples, e); 155 155 + } 156 156 + } 157 157 + 158 158 + return report; 159 159 + } 160 160 + 161 161 + // --- Formatting --- 162 162 + 163 163 + function formatEdge(e: Edge): string { 164 164 + const rel = e.relation ?? '(null)'; 165 165 + return ` ${e.source.slice(0, 55).padEnd(58)} → ${e.target.slice(0, 55)} [${rel}]`; 166 166 + } 167 167 + 168 168 + function printReport(report: CleanupReport): void { 169 169 + console.log(`\nEdge cleanup report (${report.totalEdges} total edges)\n`); 170 170 + console.log('─'.repeat(72)); 171 171 + 172 172 + const categories: Array<{ label: string; count: number; action: string; samples: Edge[] }> = [ 173 173 + { label: 'at-uri', count: report.atUri.count, action: 'delete', samples: report.atUri.samples }, 174 174 + { label: 'null-relation', count: report.nullRelation.count, action: 'delete', samples: report.nullRelation.samples }, 175 175 + { label: 'case-variant', count: report.caseVariant.count, action: 'normalize', samples: report.caseVariant.samples }, 176 176 + { label: 'bsky-app-noise', count: report.bskyAppNoise.count, action: 'delete', samples: report.bskyAppNoise.samples }, 177 177 + { label: 'same-domain-trivial', count: report.sameDomainTrivial.count, action: 'delete', samples: report.sameDomainTrivial.samples }, 178 178 + ]; 179 179 + 180 180 + for (const cat of categories) { 181 181 + if (cat.count === 0) continue; 182 182 + console.log(`\n${cat.label}: ${cat.count} edges (${cat.action})`); 183 183 + for (const e of cat.samples) console.log(formatEdge(e)); 184 184 + if (cat.count > cat.samples.length) console.log(` ... and ${cat.count - cat.samples.length} more`); 185 185 + } 186 186 + 187 187 + // Show case-variant normalization map 188 188 + if (report.caseVariant.count > 0) { 189 189 + console.log('\nNormalization map:'); 190 190 + for (const [from, to] of report.caseVariant.normalized) { 191 191 + console.log(` ${from} → ${to}`); 192 192 + } 193 193 + } 194 194 + 195 195 + const totalDelete = report.atUri.count + report.nullRelation.count + report.bskyAppNoise.count + report.sameDomainTrivial.count; 196 196 + const totalNormalize = report.caseVariant.count; 197 197 + console.log('\n' + '─'.repeat(72)); 198 198 + console.log(`Summary: ${totalDelete} edges to delete, ${totalNormalize} relations to normalize`); 199 199 + console.log(`Remaining: ${report.totalEdges - totalDelete} edges`); 200 200 + console.log('\nRun with --apply to execute.'); 201 201 + } 202 202 + 203 203 + // --- Apply cleanup --- 204 204 + 205 205 + function applyCleanup(db: Database, report: CleanupReport): { deleted: number; normalized: number } { 206 206 + let deleted = 0; 207 207 + let normalized = 0; 208 208 + 209 209 + // Delete at-uri edges 210 210 + if (report.atUri.count > 0) { 211 211 + const result = db.run("DELETE FROM edges WHERE source LIKE 'at://%' OR target LIKE 'at://%'"); 212 212 + deleted += result.changes; 213 213 + } 214 214 + 215 215 + // Delete null-relation edges 216 216 + if (report.nullRelation.count > 0) { 217 217 + const result = db.run("DELETE FROM edges WHERE relation IS NULL OR relation = ''"); 218 218 + deleted += result.changes; 219 219 + } 220 220 + 221 221 + // Delete bsky-app-noise edges (post permalinks) 222 222 + if (report.bskyAppNoise.count > 0) { 223 223 + const rows = db.query("SELECT source, target, did FROM edges WHERE source LIKE 'https://bsky.app/profile/%/post/%' OR target LIKE 'https://bsky.app/profile/%/post/%'").all() as Edge[]; 224 224 + const stmt = db.query('DELETE FROM edges WHERE source = ? AND target = ? AND did = ?'); 225 225 + for (const r of rows) { 226 226 + stmt.run(r.source, r.target, r.did); 227 227 + } 228 228 + deleted += rows.length; 229 229 + } 230 230 + 231 231 + // Delete same-domain-trivial edges 232 232 + if (report.sameDomainTrivial.count > 0) { 233 233 + // Re-scan to find exact rows to delete 234 234 + const allEdges = db.query('SELECT source, target, relation, did FROM edges').all() as Edge[]; 235 235 + const stmt = db.query('DELETE FROM edges WHERE source = ? AND target = ? AND did = ?'); 236 236 + for (const e of allEdges) { 237 237 + if (isTrivialSameDomain(e.source, e.target)) { 238 238 + stmt.run(e.source, e.target, e.did); 239 239 + deleted++; 240 240 + } 241 241 + } 242 242 + } 243 243 + 244 244 + // Normalize case-variant relations 245 245 + if (report.caseVariant.count > 0) { 246 246 + for (const [from, to] of report.caseVariant.normalized) { 247 247 + const result = db.query('UPDATE edges SET relation = ? WHERE relation = ?').run(to, from); 248 248 + normalized += result.changes; 249 249 + } 250 250 + } 251 251 + 252 252 + return { deleted, normalized }; 253 253 + } 254 254 + 255 255 + // --- Main --- 256 256 + 257 257 + async function main(): Promise<void> { 258 258 + const args = process.argv.slice(2); 259 259 + const apply = args.includes('--apply'); 260 260 + const wantJson = args.includes('--json'); 261 261 + 262 262 + let db: Database; 263 263 + try { 264 264 + db = new Database(DB_PATH); 265 265 + db.exec('PRAGMA journal_mode = WAL'); 266 266 + db.exec('PRAGMA synchronous = NORMAL'); 267 267 + const count = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0; 268 268 + if (count === 0) { 269 269 + console.error('No edges found. Run `bun island:discover` first.'); 270 270 + db.close(); 271 271 + process.exit(1); 272 272 + } 273 273 + } catch { 274 274 + console.error('Database not found. Run `bun island:discover` first.'); 275 275 + process.exit(1); 276 276 + } 277 277 + 278 278 + const edges = db.query('SELECT source, target, relation, did FROM edges').all() as Edge[]; 279 279 + const report = classifyEdges(edges); 280 280 + 281 281 + if (wantJson) { 282 282 + const jsonReport = { 283 283 + totalEdges: report.totalEdges, 284 284 + atUri: report.atUri.count, 285 285 + nullRelation: report.nullRelation.count, 286 286 + caseVariant: report.caseVariant.count, 287 287 + caseVariantMap: Object.fromEntries(report.caseVariant.normalized), 288 288 + bskyAppNoise: report.bskyAppNoise.count, 289 289 + sameDomainTrivial: report.sameDomainTrivial.count, 290 290 + }; 291 291 + if (apply) { 292 292 + const result = applyCleanup(db, report); 293 293 + console.log(JSON.stringify({ ...jsonReport, applied: result }, null, 2)); 294 294 + } else { 295 295 + console.log(JSON.stringify(jsonReport, null, 2)); 296 296 + } 297 297 + } else { 298 298 + printReport(report); 299 299 + 300 300 + if (apply) { 301 301 + const result = applyCleanup(db, report); 302 302 + console.log(`\nApplied: ${result.deleted} edges deleted, ${result.normalized} relations normalized`); 303 303 + const remaining = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c; 304 304 + console.log(`Remaining edges: ${remaining}`); 305 305 + } 306 306 + } 307 307 + 308 308 + db.close(); 309 309 + } 310 310 + 311 311 + main().catch((err) => { 312 312 + console.error('Fatal:', err instanceof Error ? err.message : String(err)); 313 313 + process.exit(1); 314 314 + });

+5 -5

src/island-explore.ts

View file Reviewed

··· 5 5 * fringe nodes, and emits endofunctor suggestions for re-attaching them to the island. 6 6 * 7 7 * Accepts either an island hash (12 hex chars, e.g. a1b2c3d4e5f6) or an AT URI. 8 8 - * When given an island hash, resolves it to the component's lexmin and looks up 8 8 + * When given an island hash, resolves it to the component's centroid and looks up 9 9 * the published record on the PDS. 10 10 * 11 11 * Scoring is carry-backed: crawled page text is matched against the carry entity ··· 284 284 let aturi: string; 285 285 const ISLAND_HASH_RE = /^[0-9a-f]{12}$/; 286 286 if (ISLAND_HASH_RE.test(input)) { 287 287 - // Island hash — resolve to component lexmin, then find published record 287 287 + // Island hash — resolve to component centroid, then find published record 288 288 const DB_PATH = process.env.ISLAND_DETECT_DB ?? resolve(CARRY_DIR, '.island-detect.db'); 289 289 let db: Database; 290 290 try { ··· 295 295 const component = findComponentById(db, input); 296 296 db.close(); 297 297 if (!component) throw new Error(`Island ${input} not found in discovery DB. Run \`bun island:discover\` first.`); 298 298 - console.error(`Resolved island ${input} → lexmin ${component.lexmin}`); 298 298 + console.error(`Resolved island ${input} → centroid ${component.centroid}`); 299 299 300 300 // Look for a published org.latha.island record on our PDS 301 301 const agent = new AtpAgent({ service: ATPROTO_SERVICE }); ··· 313 313 if (!res.ok) break; 314 314 const data = await res.json() as { records: Array<{ uri: string; value: { source?: { uri?: string } } }>; cursor?: string }; 315 315 for (const r of data.records) { 316 316 - if (r.value.source?.uri === component.lexmin) { 316 316 + if (r.value.source?.uri === component.centroid) { 317 317 foundUri = r.uri; 318 318 break; 319 319 } ··· 322 322 cursor = data.cursor ?? ''; 323 323 if (!cursor) break; 324 324 } 325 325 - if (!foundUri) throw new Error(`No published org.latha.island record found for lexmin ${component.lexmin}. Run \`bun island:publish ${input}\` first.`); 325 325 + if (!foundUri) throw new Error(`No published org.latha.island record found for centroid ${component.centroid}. Run \`bun island:publish ${input}\` first.`); 326 326 aturi = foundUri; 327 327 console.error(`Found record: ${aturi}`); 328 328 } else {

+49 -5

src/island-shared.ts

View file Reviewed

··· 25 25 atUri: string | null; 26 26 } 27 27 28 28 - // --- Canonical island ID: first 12 hex chars of SHA-256(lexmin) --- 28 28 + // --- Canonical island ID: first 12 hex chars of SHA-256(centroid) --- 29 29 30 30 - export function islandId(lexmin: string): string { 31 31 - return createHash('sha256').update(lexmin).digest('hex').slice(0, 12); 30 30 + export function islandId(centroid: string): string { 31 31 + return createHash('sha256').update(centroid).digest('hex').slice(0, 12); 32 32 + } 33 33 + 34 34 + // --- Centroid: vertex with highest closeness centrality (smallest avg distance) --- 35 35 + 36 36 + export function computeCentroid(adj: Map<string, Set<string>>, nodes: Set<string>): string { 37 37 + if (nodes.size <= 1) return [...nodes][0] ?? ''; 38 38 + 39 39 + let bestNode = ''; 40 40 + let bestAvg = Infinity; 41 41 + 42 42 + for (const start of nodes) { 43 43 + // BFS from start 44 44 + const dist = new Map<string, number>(); 45 45 + dist.set(start, 0); 46 46 + const queue = [start]; 47 47 + while (queue.length > 0) { 48 48 + const current = queue.shift()!; 49 49 + for (const neighbor of adj.get(current) ?? []) { 50 50 + if (!dist.has(neighbor) && nodes.has(neighbor)) { 51 51 + dist.set(neighbor, dist.get(current)! + 1); 52 52 + queue.push(neighbor); 53 53 + } 54 54 + } 55 55 + } 56 56 + 57 57 + // Average distance to all reachable nodes 58 58 + let totalDist = 0; 59 59 + let reachable = 0; 60 60 + for (const [node, d] of dist) { 61 61 + if (nodes.has(node)) { 62 62 + totalDist += d; 63 63 + reachable++; 64 64 + } 65 65 + } 66 66 + 67 67 + const avg = reachable > 0 ? totalDist / reachable : Infinity; 68 68 + if (avg < bestAvg) { 69 69 + bestAvg = avg; 70 70 + bestNode = start; 71 71 + } 72 72 + } 73 73 + 74 74 + return bestNode; 32 75 } 33 76 34 77 // --- SQLite --- ··· 76 119 } 77 120 } 78 121 const lexmin = [...compNodes].sort()[0]; 79 79 - components.push({ nodes: compNodes, dids: compDids, lexmin, islandId: islandId(lexmin) }); 122 122 + const centroid = computeCentroid(adj, compNodes); 123 123 + components.push({ nodes: compNodes, dids: compDids, lexmin, centroid, islandId: islandId(centroid) }); 80 124 } 81 125 82 126 return components.sort((a, b) => b.nodes.size - a.nodes.size); 83 127 } 84 128 85 85 - /** Find a component by its island ID (lexmin hash). */ 129 129 + /** Find a component by its island ID (centroid hash). */ 86 130 export function findComponentById(db: Database, targetId: string): Component | undefined { 87 131 return findComponents(db).find((c) => c.islandId === targetId); 88 132 }

+60 -2

src/worker.ts

View file Reviewed

··· 580 580 synthesis: analysis.synthesis || "", 581 581 } : null, 582 582 recordUri: row.uri, 583 583 + handle: null as string | null, 583 584 }); 584 585 } catch {} 585 586 } 586 587 588 588 + // Batch-resolve handles for all island DIDs 589 589 + const dids = new Set<string>(); 590 590 + for (const island of islands) { 591 591 + const parts = island.recordUri.split("/"); 592 592 + if (parts[2]?.startsWith("did:")) dids.add(parts[2]); 593 593 + } 594 594 + if (dids.size > 0) { 595 595 + const placeholders = [...dids].map(() => "?").join(","); 596 596 + const handleRows = await db 597 597 + .prepare(`SELECT did, handle FROM identities WHERE did IN (${placeholders})`) 598 598 + .bind(...dids) 599 599 + .all<{ did: string; handle: string | null }>(); 600 600 + const handleMap = new Map<string, string>(); 601 601 + for (const r of handleRows.results || []) { 602 602 + if (r.handle) handleMap.set(r.did, r.handle); 603 603 + } 604 604 + for (const island of islands) { 605 605 + const parts = island.recordUri.split("/"); 606 606 + const did = parts[2]; 607 607 + if (did?.startsWith("did:")) island.handle = handleMap.get(did) || null; 608 608 + } 609 609 + } 610 610 + 587 611 return c.json({ islands }); 588 612 }); 589 613 ··· 597 621 598 622 await ensureContrailReady(db); 599 623 600 600 - // If it's an AT URI, look up the strata record and derive from its centroid 624 624 + // Normalize handle-based AT URIs to DID-based URIs 625 625 + // e.g. at://nandi.latha.org/org.latha.island/rkey → at://did:plc:.../org.latha.island/rkey 626 626 + let normalizedId = id; 627 627 + let resolvedHandle: string | null = null; 601 628 if (id.startsWith("at://")) { 629 629 + const match = id.match(/^at:\/\/([^/]+)\/(.+)$/); 630 630 + if (match && !match[1].startsWith("did:")) { 631 631 + const handle = match[1]; 632 632 + const rest = match[2]; 633 633 + const identity = await db 634 634 + .prepare("SELECT did, handle FROM identities WHERE handle = ?") 635 635 + .bind(handle) 636 636 + .first<{ did: string; handle: string }>(); 637 637 + if (!identity) { 638 638 + return c.json({ error: "IdentityNotFound", message: `Handle ${handle} not found` }, 404); 639 639 + } 640 640 + normalizedId = `at://${identity.did}/${rest}`; 641 641 + resolvedHandle = identity.handle; 642 642 + } 643 643 + } 644 644 + 645 645 + // If it's an AT URI, look up the strata record and derive from its centroid 646 646 + if (normalizedId.startsWith("at://")) { 602 647 const row = await db 603 648 .prepare("SELECT uri, record FROM records_strata WHERE uri = ?") 604 604 - .bind(id) 649 649 + .bind(normalizedId) 605 650 .first<{ uri: string; record: string }>(); 606 651 607 652 if (!row) { 608 653 return c.json({ error: "RecordNotFound" }, 404); 654 654 + } 655 655 + 656 656 + // Resolve handle if not already known 657 657 + if (!resolvedHandle) { 658 658 + const did = normalizedId.split("/")[2]; 659 659 + if (did?.startsWith("did:")) { 660 660 + const identity = await db 661 661 + .prepare("SELECT handle FROM identities WHERE did = ?") 662 662 + .bind(did) 663 663 + .first<{ handle: string }>(); 664 664 + resolvedHandle = identity?.handle || null; 665 665 + } 609 666 } 610 667 611 668 const rec = JSON.parse(row.record); ··· 628 685 centroid: island.centroid, 629 686 vertexMeta: Object.fromEntries(meta), 630 687 summary: analysis.title || null, 688 688 + handle: resolvedHandle, 631 689 strata: { 632 690 prose: analysis.synthesis || "", 633 691 title: analysis.title || "",