nandi.uk
This repository has no description
11 kB
335 lines
1/**
2 * island:search
3 *
4 * Search discovered islands by URL or free-text query.
5 * Searches both the local discover cache and published islands via the stigmergic API.
6 *
7 * URL mode (detected automatically):
8 * bun island:search https://example.com/paper --top 3
9 * bun island:search https://example.com/paper --json
10 *
11 * Free-text mode:
12 * bun island:search "inlay web tiles"
13 * bun island:search "collective sensemaking" --top 10
14 * bun island:search "atproto ui" --json
15 */
16
17import { openDb, findComponents, domainFromUrl, fetchComponentEdges, resolveDidHandles } from './island-shared.js';
18import { loadDotEnv } from './cli-utils.js';
19import { resolve, dirname } from 'node:path';
20import { fileURLToPath } from 'node:url';
21
22const __dirname = dirname(fileURLToPath(import.meta.url));
23await loadDotEnv(resolve(__dirname, '..', '.env'));
24
25const STIGMERGIC_URL = process.env.STIGMERGIC_URL ?? 'https://stigmergic.latha.org';
26
27interface SearchResult {
28 islandId: string;
29 centroid: string;
30 nodeCount: number;
31 edgeCount: number;
32 didCount: number;
33 score: number;
34 matchType: 'exact' | 'token';
35 source: 'cache' | 'published';
36 recordUri?: string;
37 handle?: string;
38 matchedTerms: string[];
39 matchedUrls: string[];
40 sampleEdges: string[];
41}
42
43function usage(): never {
44 console.error('Usage: bun island:search <url or search terms> [--top N|--top=N] [--json]');
45 process.exit(1);
46}
47
48function isUrl(s: string): boolean {
49 return /^https?:\/\//.test(s);
50}
51
52function normalizeUrl(url: string): string {
53 try {
54 const u = new URL(url);
55 u.hash = '';
56 if (u.hostname.startsWith('www.')) u.hostname = u.hostname.slice(4);
57 if (u.pathname.endsWith('/')) u.pathname = u.pathname.slice(0, -1);
58 return u.toString();
59 } catch {
60 return url;
61 }
62}
63
64function tokenize(s: string): string[] {
65 return [...new Set(
66 s.toLowerCase()
67 .replace(/https?:\/\//g, ' ')
68 .replace(/[^a-z0-9.\-]+/g, ' ')
69 .split(/\s+/)
70 .map(t => t.trim())
71 .filter(t => t.length >= 2)
72 )];
73}
74
75function urlText(url: string): string {
76 try {
77 const u = new URL(url);
78 return `${u.hostname.replace(/^www\./, '')} ${u.pathname.replace(/[\/_\-]+/g, ' ')}`.toLowerCase();
79 } catch {
80 return url.toLowerCase();
81 }
82}
83
84function scoreUrl(url: string, terms: string[]): { score: number; matched: string[] } {
85 const text = urlText(url);
86 const host = domainFromUrl(url).toLowerCase();
87 let score = 0;
88 const matched: string[] = [];
89 for (const term of terms) {
90 if (host === term || host.startsWith(`${term}.`) || host.includes(`.${term}.`)) {
91 score += 6;
92 matched.push(term);
93 } else if (host.includes(term)) {
94 score += 4;
95 matched.push(term);
96 } else if (text.includes(term)) {
97 score += 2;
98 matched.push(term);
99 }
100 }
101 return { score, matched };
102}
103
104function parseArgs(): { query: string; topN: number; json: boolean } {
105 const args = process.argv.slice(2);
106 const json = args.includes('--json');
107 const topEq = args.find(a => a.startsWith('--top='));
108 const topIdx = args.indexOf('--top');
109 const topN = topEq ? Number(topEq.split('=')[1]) : topIdx >= 0 ? Number(args[topIdx + 1]) : 5;
110 const query = args.filter((a, i) => a !== '--json' && !a.startsWith('--top=') && a !== '--top' && (topIdx < 0 || i !== topIdx + 1)).join(' ').trim();
111 if (!query) usage();
112 return { query, topN: Number.isFinite(topN) && topN > 0 ? topN : 5, json };
113}
114
115interface PublishedIsland {
116 id: string;
117 centroid: string;
118 vertices: string[];
119 edges: Array<{ source: string; target: string; connectionType: string; handle?: string }>;
120 title?: string;
121 recordUri?: string;
122 handle?: string;
123}
124
125async function fetchPublishedIslands(): Promise<PublishedIsland[]> {
126 try {
127 const res = await fetch(`${STIGMERGIC_URL}/xrpc/org.latha.island.getIslands?limit=100&summary=true`);
128 if (!res.ok) return [];
129 const data = await res.json() as { islands: any[] };
130 return (data.islands || []).map((i: any) => ({
131 id: i.id,
132 centroid: i.centroid,
133 vertices: i.vertices || [],
134 edges: (i.edges || []).map((e: any) => ({
135 source: e.source,
136 target: e.target,
137 connectionType: e.connectionType || 'relates',
138 handle: e.handle,
139 })),
140 title: i.title || i.strata?.title || null,
141 recordUri: i.recordUri,
142 handle: i.handle,
143 }));
144 } catch {
145 return [];
146 }
147}
148
149async function searchPublishedIslands(query: string, normalizedQuery: string): Promise<SearchResult[]> {
150 const published = await fetchPublishedIslands();
151 const results: SearchResult[] = [];
152
153 for (const island of published) {
154 let bestMatchType: 'exact' | null = null;
155 const matchedUrls: string[] = [];
156
157 for (const vertex of island.vertices) {
158 const vertexNorm = normalizeUrl(vertex);
159 if (vertexNorm === normalizedQuery) {
160 bestMatchType = 'exact';
161 matchedUrls.push(vertex);
162 }
163 }
164
165 if (!bestMatchType) continue;
166
167 const score = 100;
168 const topEdges = island.edges.slice(0, 5).map(e =>
169 `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.connectionType}]${e.handle ? ` by @${e.handle}` : ''}`
170 );
171
172 results.push({
173 islandId: island.id,
174 centroid: island.centroid,
175 nodeCount: island.vertices.length,
176 edgeCount: island.edges.length,
177 didCount: 0,
178 score,
179 matchType: bestMatchType,
180 source: 'published',
181 recordUri: island.recordUri,
182 handle: island.handle || undefined,
183 matchedTerms: [bestMatchType],
184 matchedUrls,
185 sampleEdges: topEdges,
186 });
187 }
188
189 return results;
190}
191
192async function main(): Promise<void> {
193 const { query, topN, json } = parseArgs();
194 const urlMode = isUrl(query);
195 const normalizedQuery = urlMode ? normalizeUrl(query) : null;
196 const terms = urlMode ? [] : tokenize(query);
197 if (!urlMode && terms.length === 0) usage();
198
199 const db = openDb();
200 const edgeCount = (db.query('SELECT COUNT(*) as c FROM edges').get() as { c: number } | null)?.c ?? 0;
201 if (edgeCount === 0) {
202 console.error('No island cache found. Run `bun island:discover` first.');
203 process.exit(1);
204 }
205
206 const components = findComponents(db);
207 const results: SearchResult[] = [];
208
209 // ── Search discover cache ──────────────────────────────────────
210 for (const comp of components) {
211 if (urlMode) {
212 let bestMatchType: 'exact' | null = null;
213 const matchedUrls: string[] = [];
214
215 for (const node of comp.nodes) {
216 const nodeNorm = normalizeUrl(node);
217 if (nodeNorm === normalizedQuery) {
218 bestMatchType = 'exact';
219 matchedUrls.push(node);
220 }
221 }
222
223 if (!bestMatchType) continue;
224
225 const score = 100;
226 const edges = fetchComponentEdges(db, comp);
227 const handles = resolveDidHandles(db, comp.dids);
228 const topEdges = edges.slice(0, 5).map(e => `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}]${handles.get(e.did) ? ` by @${handles.get(e.did)}` : ''}`);
229
230 results.push({
231 islandId: comp.islandId,
232 centroid: comp.centroid,
233 nodeCount: comp.nodes.size,
234 edgeCount: edges.length,
235 didCount: comp.dids.size,
236 score,
237 matchType: bestMatchType,
238 source: 'cache',
239 matchedTerms: [bestMatchType],
240 matchedUrls,
241 sampleEdges: topEdges,
242 });
243 } else {
244 let score = 0;
245 const matchedTerms = new Set<string>();
246 const matchedUrls: Array<{ url: string; score: number }> = [];
247
248 for (const node of comp.nodes) {
249 const s = scoreUrl(node, terms);
250 if (s.score > 0) {
251 score += s.score;
252 for (const t of s.matched) matchedTerms.add(t);
253 matchedUrls.push({ url: node, score: s.score });
254 }
255 }
256
257 if (score === 0) continue;
258
259 const normalized = score / Math.log2(comp.nodes.size + 2);
260 const edges = fetchComponentEdges(db, comp);
261 const handles = resolveDidHandles(db, comp.dids);
262 const topEdges = edges.slice(0, 5).map(e => `${domainFromUrl(e.source)} → ${domainFromUrl(e.target)} [${e.relation || 'relates'}]${handles.get(e.did) ? ` by @${handles.get(e.did)}` : ''}`);
263
264 results.push({
265 islandId: comp.islandId,
266 centroid: comp.centroid,
267 nodeCount: comp.nodes.size,
268 edgeCount: edges.length,
269 didCount: comp.dids.size,
270 score: normalized,
271 matchType: 'token',
272 source: 'cache',
273 matchedTerms: [...matchedTerms].sort(),
274 matchedUrls: matchedUrls.sort((a, b) => b.score - a.score).slice(0, 10).map(m => m.url),
275 sampleEdges: topEdges,
276 });
277 }
278 }
279
280 // ── Search published islands (URL mode only) ──────────────────
281 if (urlMode) {
282 const publishedResults = await searchPublishedIslands(query, normalizedQuery!);
283 // Deduplicate by centroid: prefer published over cache (more edges, has record URI)
284 const centroidToIdx = new Map<string, number>();
285 for (let i = 0; i < results.length; i++) {
286 centroidToIdx.set(normalizeUrl(results[i].centroid), i);
287 }
288 for (const pr of publishedResults) {
289 const prCentroid = normalizeUrl(pr.centroid);
290 const existingIdx = centroidToIdx.get(prCentroid);
291 if (existingIdx !== undefined) {
292 // Replace cache result with published
293 results[existingIdx] = pr;
294 } else {
295 results.push(pr);
296 centroidToIdx.set(prCentroid, results.length - 1);
297 }
298 }
299 }
300
301 results.sort((a, b) => b.score - a.score);
302 const top = results.slice(0, topN);
303
304 const totalIslands = components.length + (urlMode ? `+published` : '');
305
306 if (json) {
307 console.log(JSON.stringify({ query, urlMode, terms, componentCount: components.length, resultCount: results.length, results: top }, null, 2));
308 } else {
309 if (urlMode) {
310 console.log(`\nIsland search: ${query}`);
311 console.log(`Matched ${results.length} islands\n`);
312 } else {
313 console.log(`\nIsland search: "${query}" (${terms.join(', ')})`);
314 console.log(`Matched ${results.length} of ${components.length} islands\n`);
315 }
316 for (const [i, r] of top.entries()) {
317 const matchLabel = r.matchType === 'exact' ? 'EXACT' : `score=${r.score.toFixed(2)}`;
318 const srcLabel = r.source === 'published' ? 'published' : 'cache';
319 console.log(`${i + 1}. ${r.islandId} ${matchLabel} ${srcLabel} nodes=${r.nodeCount} edges=${r.edgeCount}${r.handle ? ` by @${r.handle}` : ''}`);
320 if (r.recordUri) console.log(` record: ${r.recordUri}`);
321 console.log(` centroid: ${r.centroid}`);
322 if (r.matchType === 'token') console.log(` matched terms: ${r.matchedTerms.join(', ')}`);
323 for (const url of r.matchedUrls.slice(0, 5)) console.log(` - ${url}`);
324 for (const edge of r.sampleEdges.slice(0, 3)) console.log(` ${edge}`);
325 console.log('');
326 }
327 }
328
329 db.close();
330}
331
332main().catch((e) => {
333 console.error(e);
334 process.exit(1);
335});