feat(site): /recommended endpoint + leaderboard page · zzstoatzz.io/pub-search@2037a84

+86

backend/src/server.zig

··· 94 94 try handleTimelineApi(request, target); 95 95 } else if (mem.eql(u8, path, "/api/latency")) { 96 96 try handleLatencyApi(request, target); 97 + } else if (mem.eql(u8, path, "/recommended")) { 98 + try handleRecommended(request); 97 99 } else if (mem.startsWith(u8, path, "/similar")) { 98 100 try handleSimilar(request, target, io); 99 101 } else if (mem.eql(u8, path, "/activity")) { ··· 318 320 try jw.beginArray(); 319 321 while (rows.next()) |row| { 320 322 try jw.write(PopularJson{ .query = row.text(0), .count = row.int(1) }); 323 + } 324 + try jw.endArray(); 325 + return try output.toOwnedSlice(); 326 + } 327 + 328 + const RecommendedJson = struct { 329 + type: []const u8, 330 + uri: []const u8, 331 + did: []const u8, 332 + title: []const u8, 333 + createdAt: []const u8, 334 + rkey: []const u8, 335 + basePath: []const u8, 336 + platform: []const u8, 337 + path: []const u8, 338 + publicationName: []const u8, 339 + url: []const u8, 340 + recommendCount: i64, 341 + }; 342 + 343 + // Recommends live only on Turso (not the local SQLite replica), so this 344 + // handler queries Turso directly. Volume is small (~500 records at launch); 345 + // page is not on a hot path. If usage grows, plumb recommends into LocalDb. 346 + const RECOMMENDED_SQL = 347 + \\SELECT d.uri, d.did, d.title, COALESCE(d.created_at, '') AS created_at, 348 + \\ d.rkey, COALESCE(d.base_path, '') AS base_path, d.platform, 349 + \\ COALESCE(d.path, '') AS path, d.has_publication, 350 + \\ COALESCE(p.name, '') AS publication_name, 351 + \\ COUNT(r.uri) AS recommend_count 352 + \\FROM documents d 353 + \\LEFT JOIN publications p ON d.publication_uri = p.uri 354 + \\JOIN recommends r ON r.document_uri = d.uri 355 + \\GROUP BY d.uri 356 + \\ORDER BY recommend_count DESC, d.created_at DESC 357 + \\LIMIT 50 358 + ; 359 + 360 + fn handleRecommended(request: *http.Server.Request) !void { 361 + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); 362 + defer arena.deinit(); 363 + const alloc = arena.allocator(); 364 + 365 + const span = logfire.span("http.recommended", .{}); 366 + defer span.end(); 367 + 368 + try sendJson(request, try getRecommended(alloc)); 369 + } 370 + 371 + fn getRecommended(alloc: Allocator) ![]const u8 { 372 + const c = db.getClient() orelse return error.NotInitialized; 373 + 374 + var output: std.Io.Writer.Allocating = .init(alloc); 375 + errdefer output.deinit(); 376 + 377 + var res = c.query(RECOMMENDED_SQL, &.{}) catch { 378 + try output.writer.writeAll("{\"error\":\"failed to fetch recommended\"}"); 379 + return try output.toOwnedSlice(); 380 + }; 381 + defer res.deinit(); 382 + 383 + var jw: json.Stringify = .{ .writer = &output.writer }; 384 + try jw.beginArray(); 385 + for (res.rows) |row| { 386 + const did = row.text(1); 387 + const rkey = row.text(4); 388 + const base_path = row.text(5); 389 + const platform = row.text(6); 390 + const path = row.text(7); 391 + const has_publication = row.int(8) != 0; 392 + const doc_type: []const u8 = if (has_publication) "article" else "looseleaf"; 393 + try jw.write(RecommendedJson{ 394 + .type = doc_type, 395 + .uri = row.text(0), 396 + .did = did, 397 + .title = row.text(2), 398 + .createdAt = row.text(3), 399 + .rkey = rkey, 400 + .basePath = base_path, 401 + .platform = platform, 402 + .path = path, 403 + .publicationName = row.text(9), 404 + .url = search.buildDocUrl(alloc, doc_type, platform, base_path, path, rkey, did), 405 + .recommendCount = row.int(10), 406 + }); 321 407 } 322 408 try jw.endArray(); 323 409 return try output.toOwnedSlice();

+1 -1

backend/src/server/search.zig

View file

··· 108 108 109 109 /// Build canonical URL for a document/publication from its fields. 110 110 /// Mirrors the frontend's buildDocUrl logic (site/index.html:1174). 111 - fn buildDocUrl(alloc: Allocator, doc_type: []const u8, platform: []const u8, base_path: []const u8, path: []const u8, rkey: []const u8, did: []const u8) []const u8 { 111 + pub fn buildDocUrl(alloc: Allocator, doc_type: []const u8, platform: []const u8, base_path: []const u8, path: []const u8, rkey: []const u8, did: []const u8) []const u8 { 112 112 // publication → https://{basePath} 113 113 if (std.mem.eql(u8, doc_type, "publication") and base_path.len > 0) { 114 114 return std.fmt.allocPrint(alloc, "https://{s}", .{base_path}) catch "";

+298

scripts/backfill-content-type

View file

··· 1 + #!/usr/bin/env -S uv run --script --quiet 2 + # /// script 3 + # requires-python = ">=3.12" 4 + # dependencies = ["httpx", "pydantic-settings"] 5 + # /// 6 + """ 7 + Backfill `documents.content_type` from PDS records. 8 + 9 + Reads `content.$type` from each record on its PDS and writes it back to Turso. 10 + Skips whitewind (no content.$type) and rows already populated. 11 + 12 + Does NOT bump `indexed_at` — content_type isn't in the local replica schema yet, 13 + so we don't want to force a full re-sync of every touched row. 14 + 15 + Usage: 16 + TURSO_URL=libsql://... TURSO_TOKEN=... ./scripts/backfill-content-type 17 + ./scripts/backfill-content-type --dry-run 18 + ./scripts/backfill-content-type --limit 100 # cap distinct (did, collection) groups 19 + ./scripts/backfill-content-type --workers 10 # PDS concurrency 20 + """ 21 + 22 + from __future__ import annotations 23 + 24 + import argparse 25 + import os 26 + import sys 27 + import time 28 + from concurrent.futures import ThreadPoolExecutor, as_completed 29 + from dataclasses import dataclass 30 + 31 + import httpx 32 + from pydantic_settings import BaseSettings, SettingsConfigDict 33 + 34 + 35 + class Settings(BaseSettings): 36 + model_config = SettingsConfigDict( 37 + env_file=os.environ.get("ENV_FILE", ".env"), extra="ignore" 38 + ) 39 + 40 + turso_url: str 41 + turso_token: str 42 + 43 + @property 44 + def turso_host(self) -> str: 45 + url = self.turso_url 46 + if url.startswith("libsql://"): 47 + url = url[len("libsql://") :] 48 + return url 49 + 50 + 51 + # ---- Turso HTTP pipeline ----------------------------------------------------- 52 + 53 + 54 + def turso_pipeline(settings: Settings, requests: list[dict]) -> list[dict]: 55 + """POST a batch of statements; return the per-statement results.""" 56 + last_err: Exception | None = None 57 + for attempt in range(3): 58 + try: 59 + resp = httpx.post( 60 + f"https://{settings.turso_host}/v2/pipeline", 61 + headers={ 62 + "Authorization": f"Bearer {settings.turso_token}", 63 + "Content-Type": "application/json", 64 + }, 65 + json={"requests": [*requests, {"type": "close"}]}, 66 + timeout=180, 67 + ) 68 + resp.raise_for_status() 69 + return resp.json()["results"] 70 + except (httpx.ReadTimeout, httpx.ConnectError, httpx.RemoteProtocolError) as e: 71 + last_err = e 72 + print(f" turso pipeline attempt {attempt + 1} failed: {type(e).__name__}; retrying...", file=sys.stderr) 73 + time.sleep(2 * (attempt + 1)) 74 + assert last_err is not None 75 + raise last_err 76 + 77 + 78 + def turso_query(settings: Settings, sql: str, args: list | None = None) -> list[list]: 79 + stmt: dict = {"sql": sql} 80 + if args: 81 + stmt["args"] = [ 82 + {"type": "null"} if a is None else {"type": "text", "value": str(a)} 83 + for a in args 84 + ] 85 + results = turso_pipeline(settings, [{"type": "execute", "stmt": stmt}]) 86 + r = results[0] 87 + if r["type"] == "error": 88 + raise RuntimeError(f"Turso query error: {r['error']}") 89 + rows = r["response"]["result"]["rows"] 90 + return [[c["value"] if isinstance(c, dict) and "value" in c else None for c in row] for row in rows] 91 + 92 + 93 + def turso_batch_update(settings: Settings, updates: list[tuple[str, str, str]]) -> int: 94 + """Apply UPDATE documents SET content_type=? WHERE did=? AND rkey=? in batches.""" 95 + BATCH = 50 96 + written = 0 97 + for i in range(0, len(updates), BATCH): 98 + chunk = updates[i : i + BATCH] 99 + reqs = [ 100 + { 101 + "type": "execute", 102 + "stmt": { 103 + "sql": "UPDATE documents SET content_type = ? WHERE did = ? AND rkey = ?", 104 + "args": [ 105 + {"type": "text", "value": ct}, 106 + {"type": "text", "value": did}, 107 + {"type": "text", "value": rkey}, 108 + ], 109 + }, 110 + } 111 + for (ct, did, rkey) in chunk 112 + ] 113 + results = turso_pipeline(settings, reqs) 114 + for r in results: 115 + if r["type"] == "error": 116 + print(f" turso UPDATE error: {r['error']}", file=sys.stderr) 117 + continue 118 + response = r.get("response") or {} 119 + if response.get("type") != "execute": 120 + continue # skip the trailing close response 121 + written += int(response.get("result", {}).get("affected_row_count", 0)) 122 + return written 123 + 124 + 125 + # ---- PDS resolution + fetch -------------------------------------------------- 126 + 127 + 128 + _pds_cache: dict[str, str | None] = {} 129 + 130 + 131 + def resolve_pds(did: str) -> str | None: 132 + """Return PDS endpoint URL for a DID, or None if unresolvable.""" 133 + if did in _pds_cache: 134 + return _pds_cache[did] 135 + url: str | None = None 136 + try: 137 + if did.startswith("did:plc:"): 138 + resp = httpx.get(f"https://plc.directory/{did}", timeout=15) 139 + resp.raise_for_status() 140 + for svc in resp.json().get("service", []): 141 + if svc.get("type") == "AtprotoPersonalDataServer": 142 + url = svc["serviceEndpoint"] 143 + break 144 + elif did.startswith("did:web:"): 145 + host = did[len("did:web:") :].replace(":", "/") 146 + resp = httpx.get(f"https://{host}/.well-known/did.json", timeout=15, follow_redirects=True) 147 + resp.raise_for_status() 148 + for svc in resp.json().get("service", []): 149 + if svc.get("type") == "AtprotoPersonalDataServer": 150 + url = svc["serviceEndpoint"] 151 + break 152 + except Exception: 153 + url = None 154 + _pds_cache[did] = url 155 + return url 156 + 157 + 158 + def list_records(pds: str, did: str, collection: str) -> list[dict]: 159 + records: list[dict] = [] 160 + cursor: str | None = None 161 + while True: 162 + params: dict = {"repo": did, "collection": collection, "limit": 100} 163 + if cursor: 164 + params["cursor"] = cursor 165 + resp = httpx.get( 166 + f"{pds}/xrpc/com.atproto.repo.listRecords", params=params, timeout=30 167 + ) 168 + resp.raise_for_status() 169 + data = resp.json() 170 + records.extend(data.get("records", [])) 171 + cursor = data.get("cursor") 172 + if not cursor: 173 + break 174 + return records 175 + 176 + 177 + # ---- Backfill -------------------------------------------------------------- 178 + 179 + 180 + @dataclass 181 + class Group: 182 + did: str 183 + collection: str 184 + expected: int # how many rows we expect to update in this group 185 + 186 + 187 + def gather_groups(settings: Settings, limit: int | None) -> list[Group]: 188 + sql = ( 189 + "SELECT did, source_collection, COUNT(*) " 190 + "FROM documents " 191 + "WHERE content_type IS NULL " 192 + " AND source_collection NOT LIKE 'com.whtwnd.%' " 193 + "GROUP BY did, source_collection " 194 + "ORDER BY COUNT(*) DESC" 195 + ) 196 + if limit: 197 + sql += f" LIMIT {int(limit)}" 198 + rows = turso_query(settings, sql) 199 + return [Group(did=r[0], collection=r[1], expected=int(r[2])) for r in rows] 200 + 201 + 202 + def fetch_group(group: Group) -> tuple[Group, list[tuple[str, str, str]] | None, str]: 203 + """Return (group, [(content_type, did, rkey), ...] | None, status).""" 204 + pds = resolve_pds(group.did) 205 + if not pds: 206 + return group, None, "no-pds" 207 + try: 208 + records = list_records(pds, group.did, group.collection) 209 + except httpx.HTTPStatusError as e: 210 + if e.response.status_code == 400: 211 + return group, [], "empty" 212 + return group, None, f"pds-{e.response.status_code}" 213 + except Exception as e: 214 + return group, None, f"pds-err:{type(e).__name__}" 215 + 216 + updates: list[tuple[str, str, str]] = [] 217 + for rec in records: 218 + uri = rec.get("uri", "") 219 + rkey = uri.rsplit("/", 1)[-1] if uri else "" 220 + if not rkey: 221 + continue 222 + value = rec.get("value") or {} 223 + content = value.get("content") 224 + content_type = "" 225 + if isinstance(content, dict): 226 + ct = content.get("$type") 227 + if isinstance(ct, str): 228 + content_type = ct 229 + updates.append((content_type, group.did, rkey)) 230 + return group, updates, "ok" 231 + 232 + 233 + def main() -> int: 234 + parser = argparse.ArgumentParser(description="Backfill documents.content_type from PDS records") 235 + parser.add_argument("--dry-run", action="store_true", help="Fetch from PDSes but don't write") 236 + parser.add_argument("--limit", type=int, default=None, help="Cap distinct (did, collection) groups (debug)") 237 + parser.add_argument("--workers", type=int, default=8, help="PDS concurrency") 238 + args = parser.parse_args() 239 + 240 + try: 241 + settings = Settings() # type: ignore 242 + except Exception as e: 243 + print(f"error loading settings: {e}", file=sys.stderr) 244 + print("required env vars: TURSO_URL, TURSO_TOKEN", file=sys.stderr) 245 + return 1 246 + 247 + print("loading groups...") 248 + groups = gather_groups(settings, args.limit) 249 + total_expected = sum(g.expected for g in groups) 250 + print(f" {len(groups)} (did, collection) groups, {total_expected} rows to attempt") 251 + 252 + if not groups: 253 + print("nothing to backfill") 254 + return 0 255 + 256 + pending_writes: list[tuple[str, str, str]] = [] 257 + started = time.time() 258 + done = 0 259 + status_counts: dict[str, int] = {} 260 + 261 + with ThreadPoolExecutor(max_workers=args.workers) as pool: 262 + futures = {pool.submit(fetch_group, g): g for g in groups} 263 + for fut in as_completed(futures): 264 + group, updates, status = fut.result() 265 + status_counts[status] = status_counts.get(status, 0) + 1 266 + done += 1 267 + if updates: 268 + pending_writes.extend(updates) 269 + 270 + if done % 50 == 0 or done == len(groups): 271 + elapsed = time.time() - started 272 + rate = done / elapsed if elapsed > 0 else 0 273 + print( 274 + f" [{done}/{len(groups)}] {rate:.1f} groups/s, " 275 + f"{len(pending_writes)} pending writes, statuses={status_counts}" 276 + ) 277 + 278 + print(f"\nfetched {len(pending_writes)} record-level updates") 279 + if args.dry_run: 280 + print("dry-run: skipping writes") 281 + # show a sample 282 + by_ct: dict[str, int] = {} 283 + for ct, _, _ in pending_writes: 284 + by_ct[ct] = by_ct.get(ct, 0) + 1 285 + print("content_type distribution (dry-run):") 286 + for ct, n in sorted(by_ct.items(), key=lambda kv: -kv[1])[:20]: 287 + print(f" {n:>6} {ct!r}") 288 + return 0 289 + 290 + print("writing...") 291 + written = turso_batch_update(settings, pending_writes) 292 + print(f" wrote {written} rows") 293 + 294 + return 0 295 + 296 + 297 + if __name__ == "__main__": 298 + sys.exit(main())

+325

scripts/backfill-recommends

View file

··· 1 + #!/usr/bin/env -S uv run --script --quiet 2 + # /// script 3 + # requires-python = ">=3.12" 4 + # dependencies = [ 5 + # "httpx", 6 + # "pydantic-settings", 7 + # "atproto", 8 + # ] 9 + # /// 10 + """ 11 + Backfill site.standard.graph.recommend records via CAR walk. 12 + 13 + Pattern (cribbed from prefect-pack/collection_creators): 14 + lightrail listReposByCollection -> enumerate DIDs holding the collection 15 + slingshot resolveMiniDoc -> map DID -> PDS endpoint 16 + com.atproto.sync.getRepo -> fetch full repo as CAR 17 + walk MST locally -> read each key+value with prefix 18 + UPSERT into recommends -> idempotent on (uri) 19 + 20 + Why not tap? Tap is for streaming new records going forward. For a one-shot 21 + backfill, a CAR walk is one HTTP call per DID and gives ground truth. 22 + 23 + Usage: 24 + TURSO_URL=libsql://... TURSO_TOKEN=... ./scripts/backfill-recommends 25 + ./scripts/backfill-recommends --dry-run 26 + """ 27 + 28 + from __future__ import annotations 29 + 30 + import argparse 31 + import os 32 + import sys 33 + import time 34 + from concurrent.futures import ThreadPoolExecutor, as_completed 35 + from dataclasses import dataclass 36 + 37 + import httpx 38 + from atproto_core.car import CAR 39 + from atproto_core.cid import CID 40 + from pydantic_settings import BaseSettings, SettingsConfigDict 41 + 42 + SLINGSHOT = "https://slingshot.microcosm.blue" 43 + COLLECTION = "site.standard.graph.recommend" 44 + 45 + # Union DIDs across multiple relays — lightrail's index lags the Bluesky 46 + # reference relay by ~15% for niche collections (216 vs 254 at last check). 47 + RELAY_ENDPOINTS = [ 48 + "https://relay1.us-east.bsky.network", 49 + "https://lightrail.microcosm.blue", 50 + "https://relay.waow.tech", 51 + ] 52 + 53 + 54 + class Settings(BaseSettings): 55 + model_config = SettingsConfigDict( 56 + env_file=os.environ.get("ENV_FILE", ".env"), extra="ignore" 57 + ) 58 + turso_url: str 59 + turso_token: str 60 + 61 + @property 62 + def turso_host(self) -> str: 63 + url = self.turso_url 64 + if url.startswith("libsql://"): 65 + url = url[len("libsql://") :] 66 + return url 67 + 68 + 69 + # ---- Turso ---------------------------------------------------------------- 70 + 71 + 72 + def turso_pipeline(settings: Settings, requests: list[dict]) -> list[dict]: 73 + last_err: Exception | None = None 74 + for attempt in range(3): 75 + try: 76 + resp = httpx.post( 77 + f"https://{settings.turso_host}/v2/pipeline", 78 + headers={ 79 + "Authorization": f"Bearer {settings.turso_token}", 80 + "Content-Type": "application/json", 81 + }, 82 + json={"requests": [*requests, {"type": "close"}]}, 83 + timeout=180, 84 + ) 85 + resp.raise_for_status() 86 + return resp.json()["results"] 87 + except (httpx.ReadTimeout, httpx.ConnectError, httpx.RemoteProtocolError) as e: 88 + last_err = e 89 + print( 90 + f" turso pipeline attempt {attempt + 1} failed: {type(e).__name__}; retrying", 91 + file=sys.stderr, 92 + ) 93 + time.sleep(2 * (attempt + 1)) 94 + assert last_err is not None 95 + raise last_err 96 + 97 + 98 + def turso_upsert_recommends(settings: Settings, rows: list[tuple[str, str, str, str, str]]) -> int: 99 + """rows: (uri, did, rkey, document_uri, created_at).""" 100 + BATCH = 50 101 + written = 0 102 + for i in range(0, len(rows), BATCH): 103 + chunk = rows[i : i + BATCH] 104 + reqs = [ 105 + { 106 + "type": "execute", 107 + "stmt": { 108 + "sql": ( 109 + "INSERT INTO recommends (uri, did, rkey, document_uri, created_at, indexed_at) " 110 + "VALUES (?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%S','now')) " 111 + "ON CONFLICT(uri) DO UPDATE SET " 112 + " did = excluded.did, " 113 + " rkey = excluded.rkey, " 114 + " document_uri = excluded.document_uri, " 115 + " created_at = excluded.created_at, " 116 + " indexed_at = strftime('%Y-%m-%dT%H:%M:%S','now')" 117 + ), 118 + "args": [ 119 + {"type": "text", "value": uri}, 120 + {"type": "text", "value": did}, 121 + {"type": "text", "value": rkey}, 122 + {"type": "text", "value": doc_uri}, 123 + {"type": "text", "value": created_at}, 124 + ], 125 + }, 126 + } 127 + for (uri, did, rkey, doc_uri, created_at) in chunk 128 + ] 129 + results = turso_pipeline(settings, reqs) 130 + for r in results: 131 + if r["type"] == "error": 132 + print(f" upsert error: {r['error']}", file=sys.stderr) 133 + continue 134 + response = r.get("response") or {} 135 + if response.get("type") != "execute": 136 + continue 137 + written += int(response.get("result", {}).get("affected_row_count", 0)) 138 + return written 139 + 140 + 141 + # ---- Enumeration + resolution ------------------------------------------- 142 + 143 + 144 + def _enumerate_from_relay(client: httpx.Client, base: str, collection: str) -> list[str]: 145 + dids: list[str] = [] 146 + cursor: str | None = None 147 + while True: 148 + params: dict = {"collection": collection, "limit": 1000} 149 + if cursor: 150 + params["cursor"] = cursor 151 + try: 152 + r = client.get( 153 + f"{base}/xrpc/com.atproto.sync.listReposByCollection", 154 + params=params, 155 + timeout=30, 156 + ) 157 + r.raise_for_status() 158 + except Exception as e: 159 + print(f" {base}: relay error {type(e).__name__}", file=sys.stderr) 160 + return dids 161 + body = r.json() 162 + for repo in body.get("repos", []): 163 + dids.append(repo["did"]) 164 + cursor = body.get("cursor") 165 + if not cursor: 166 + return dids 167 + 168 + 169 + def enumerate_creator_dids(collection: str) -> list[str]: 170 + all_dids: set[str] = set() 171 + with httpx.Client() as client: 172 + for base in RELAY_ENDPOINTS: 173 + from_relay = _enumerate_from_relay(client, base, collection) 174 + print(f" {base}: {len(from_relay)} DIDs") 175 + all_dids.update(from_relay) 176 + return sorted(all_dids) 177 + 178 + 179 + def resolve_pds(client: httpx.Client, did: str) -> str | None: 180 + try: 181 + r = client.get( 182 + f"{SLINGSHOT}/xrpc/blue.microcosm.identity.resolveMiniDoc", 183 + params={"identifier": did}, 184 + timeout=15, 185 + ) 186 + r.raise_for_status() 187 + return r.json().get("pds") 188 + except Exception: 189 + return None 190 + 191 + 192 + # ---- CAR walk ------------------------------------------------------------ 193 + 194 + 195 + def _walk_mst(car: CAR, collection: str) -> list[tuple[str, dict]]: 196 + """Return [(rkey, decoded_value_dict), ...] for every entry whose key 197 + starts with f"{collection}/". 198 + 199 + MST nodes are nested via CIDs in `l` (left subtree) and `e[i].t` (right 200 + subtree after entry i). Keys are byte-encoded with prefix compression 201 + (`p` = chars shared with previous key, `k` = unique suffix). 202 + """ 203 + blocks = car.blocks 204 + root = blocks[car.root] 205 + data_cid = CID.decode(root["data"]) 206 + 207 + prefix_bytes = f"{collection}/".encode() 208 + out: list[tuple[str, dict]] = [] 209 + stack = [data_cid] 210 + while stack: 211 + node = blocks[stack.pop()] 212 + if (left := node.get("l")) is not None: 213 + stack.append(CID.decode(left)) 214 + prev_key = b"" 215 + for entry in node["e"]: 216 + key = prev_key[: entry["p"]] + entry["k"] 217 + prev_key = key 218 + if key.startswith(prefix_bytes): 219 + value_cid = CID.decode(entry["v"]) 220 + value = blocks[value_cid] 221 + rkey = key[len(prefix_bytes) :].decode("utf-8", errors="replace") 222 + out.append((rkey, value)) 223 + if (t := entry.get("t")) is not None: 224 + stack.append(CID.decode(t)) 225 + return out 226 + 227 + 228 + # ---- Fetch one DID's recommends ------------------------------------------ 229 + 230 + 231 + @dataclass 232 + class FetchResult: 233 + did: str 234 + rows: list[tuple[str, str, str, str, str]] 235 + status: str # "ok", "unresolved", "no-pds", "fetch-err:<type>", "parse-err" 236 + 237 + 238 + def fetch_recommends_for_did(client: httpx.Client, did: str) -> FetchResult: 239 + pds = resolve_pds(client, did) 240 + if not pds: 241 + return FetchResult(did, [], "no-pds") 242 + try: 243 + r = client.get( 244 + f"{pds}/xrpc/com.atproto.sync.getRepo", 245 + params={"did": did}, 246 + timeout=60, 247 + ) 248 + r.raise_for_status() 249 + car = CAR.from_bytes(r.content) 250 + except httpx.HTTPStatusError as e: 251 + return FetchResult(did, [], f"fetch-err:{e.response.status_code}") 252 + except Exception as e: 253 + return FetchResult(did, [], f"fetch-err:{type(e).__name__}") 254 + try: 255 + entries = _walk_mst(car, COLLECTION) 256 + except Exception: 257 + return FetchResult(did, [], "parse-err") 258 + 259 + rows: list[tuple[str, str, str, str, str]] = [] 260 + for rkey, value in entries: 261 + uri = f"at://{did}/{COLLECTION}/{rkey}" 262 + document_uri = value.get("document") if isinstance(value, dict) else None 263 + created_at = value.get("createdAt") if isinstance(value, dict) else None 264 + if not isinstance(document_uri, str): 265 + continue # spec requires document; skip malformed 266 + rows.append((uri, did, rkey, document_uri, str(created_at) if created_at else "")) 267 + return FetchResult(did, rows, "ok") 268 + 269 + 270 + # ---- Main --------------------------------------------------------------- 271 + 272 + 273 + def main() -> int: 274 + parser = argparse.ArgumentParser(description="Backfill site.standard.graph.recommend records via CAR walk") 275 + parser.add_argument("--dry-run", action="store_true", help="Don't write to Turso") 276 + parser.add_argument("--workers", type=int, default=8, help="Concurrent PDS fetches") 277 + parser.add_argument("--limit", type=int, default=None, help="Cap DIDs (debug)") 278 + args = parser.parse_args() 279 + 280 + try: 281 + settings = Settings() # type: ignore 282 + except Exception as e: 283 + print(f"error loading settings: {e}", file=sys.stderr) 284 + print("required env vars: TURSO_URL, TURSO_TOKEN", file=sys.stderr) 285 + return 1 286 + 287 + print("enumerating creator DIDs via lightrail...") 288 + dids = enumerate_creator_dids(COLLECTION) 289 + if args.limit: 290 + dids = dids[: args.limit] 291 + print(f" {len(dids)} DIDs") 292 + 293 + if not dids: 294 + return 0 295 + 296 + started = time.time() 297 + done = 0 298 + statuses: dict[str, int] = {} 299 + rows_acc: list[tuple[str, str, str, str, str]] = [] 300 + 301 + with httpx.Client() as client, ThreadPoolExecutor(max_workers=args.workers) as pool: 302 + futures = {pool.submit(fetch_recommends_for_did, client, did): did for did in dids} 303 + for fut in as_completed(futures): 304 + res = fut.result() 305 + statuses[res.status] = statuses.get(res.status, 0) + 1 306 + rows_acc.extend(res.rows) 307 + done += 1 308 + if done % 25 == 0 or done == len(dids): 309 + elapsed = time.time() - started 310 + rate = done / elapsed if elapsed > 0 else 0 311 + print(f" [{done}/{len(dids)}] {rate:.1f} dids/s, {len(rows_acc)} records, statuses={statuses}") 312 + 313 + print(f"\ncollected {len(rows_acc)} records from {statuses.get('ok', 0)} DIDs") 314 + if args.dry_run: 315 + print("dry-run: skipping writes") 316 + return 0 317 + 318 + print("upserting...") 319 + written = turso_upsert_recommends(settings, rows_acc) 320 + print(f" wrote {written} rows") 321 + return 0 322 + 323 + 324 + if __name__ == "__main__": 325 + sys.exit(main())

+1 -1

site/index.html

View file

··· 1779 1779 fetch(`${API_URL}/stats`) 1780 1780 .then(r => r.json()) 1781 1781 .then(data => { 1782 - statsDiv.innerHTML = `${data.documents} documents, ${data.publications} publications · <a href="/dashboard.html">stats</a> · <a href="/atlas.html">atlas</a>`; 1782 + statsDiv.innerHTML = `${data.documents} documents, ${data.publications} publications · <a href="/dashboard.html">stats</a> · <a href="/atlas.html">atlas</a> · <a href="/recommended.html">recommended</a>`; 1783 1783 const headerStats = document.getElementById('header-stats'); 1784 1784 headerStats.href = '/dashboard.html'; 1785 1785 headerStats.textContent = `[${data.documents.toLocaleString()} docs]`;

+261

site/recommended.html

View file

··· 1 + <!DOCTYPE html> 2 + <html lang="en"> 3 + <head> 4 + <meta charset="UTF-8"> 5 + <meta name="viewport" content="width=device-width, initial-scale=1.0"> 6 + <title>pub search / recommended</title> 7 + <meta name="description" content="most-recommended posts across the atproto standard.site network"> 8 + <meta property="og:title" content="pub search / recommended"> 9 + <meta property="og:description" content="most-recommended posts across the atproto standard.site network"> 10 + <meta property="og:type" content="website"> 11 + <meta property="og:image" content="https://pub-search.waow.tech/og-image"> 12 + <meta property="og:image:width" content="1200"> 13 + <meta property="og:image:height" content="630"> 14 + <meta name="twitter:card" content="summary_large_image"> 15 + <meta name="twitter:title" content="pub search / recommended"> 16 + <meta name="twitter:description" content="most-recommended posts across the atproto standard.site network"> 17 + <meta name="twitter:image" content="https://pub-search.waow.tech/og-image"> 18 + <link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><circle cx='16' cy='16' r='2' fill='%231B7340'/><circle cx='8' cy='10' r='1.5' fill='%231B7340' opacity='.6'/><circle cx='24' cy='8' r='1' fill='%231B7340' opacity='.4'/><circle cx='22' cy='22' r='1.5' fill='%231B7340' opacity='.5'/><circle cx='6' cy='24' r='1' fill='%231B7340' opacity='.3'/></svg>"> 19 + <script> 20 + (function() { 21 + var t = localStorage.getItem('theme') || 'dark'; 22 + if (t === 'system') t = matchMedia('(prefers-color-scheme: light)').matches ? 'light' : 'dark'; 23 + document.documentElement.setAttribute('data-theme', t); 24 + })(); 25 + </script> 26 + <link rel="stylesheet" href="components.css"> 27 + <style> 28 + :root, [data-theme="dark"] { 29 + --bg: #0a0a0a; 30 + --bg-subtle: #111; 31 + --bg-hover: #151515; 32 + --border: #222; 33 + --border-subtle: #252525; 34 + --border-focus: #333; 35 + --text: #ccc; 36 + --text-bright: #fff; 37 + --text-secondary: #888; 38 + --text-dim: #555; 39 + --text-muted: #444; 40 + --result-hover: #111; 41 + --row-border: #1a1a1a; 42 + } 43 + [data-theme="light"] { 44 + --bg: #f5f5f0; 45 + --bg-subtle: #eee; 46 + --bg-hover: #e4e4e0; 47 + --border: #ddd; 48 + --border-subtle: #ccc; 49 + --border-focus: #bbb; 50 + --text: #333; 51 + --text-bright: #111; 52 + --text-secondary: #555; 53 + --text-dim: #888; 54 + --text-muted: #999; 55 + --result-hover: #eee; 56 + --row-border: #e0e0e0; 57 + } 58 + 59 + * { box-sizing: border-box; margin: 0; padding: 0; } 60 + 61 + body { 62 + font-family: monospace; 63 + background: var(--bg); 64 + color: var(--text); 65 + min-height: 100vh; 66 + padding: 1rem; 67 + font-size: 14px; 68 + line-height: 1.6; 69 + } 70 + 71 + .container { max-width: 720px; margin: 0 auto; } 72 + 73 + a { color: #1B7340; text-decoration: none; } 74 + a:hover { color: #2a9d5c; } 75 + 76 + h1 { 77 + font-size: 12px; 78 + font-weight: normal; 79 + margin-bottom: 0.75rem; 80 + display: flex; 81 + align-items: center; 82 + flex-wrap: wrap; 83 + gap: 0.4rem; 84 + } 85 + h1 a.title { color: var(--text-secondary); } 86 + h1 a.title:hover { color: var(--text-bright); } 87 + h1 .dim { color: var(--text-dim); } 88 + 89 + .intro { 90 + font-size: 11px; 91 + color: var(--text-dim); 92 + margin-bottom: 1.5rem; 93 + line-height: 1.7; 94 + } 95 + .intro code { 96 + color: var(--text-secondary); 97 + font-family: monospace; 98 + } 99 + 100 + .status { 101 + font-size: 11px; 102 + color: var(--text-dim); 103 + padding: 1rem 0; 104 + } 105 + 106 + /* one row per recommended doc. flex row that adapts: 107 + * desktop: rank | title+source (flex 1) | count 108 + * mobile : rank+count on first line, title and source stack below */ 109 + .row { 110 + display: grid; 111 + grid-template-columns: 2.5rem 1fr auto; 112 + column-gap: 0.75rem; 113 + align-items: baseline; 114 + padding: 0.75rem 0; 115 + border-bottom: 1px solid var(--row-border); 116 + } 117 + .row:hover { background: var(--result-hover); margin: 0 -0.5rem; padding: 0.75rem 0.5rem; } 118 + 119 + .rank { 120 + color: var(--text-dim); 121 + font-variant-numeric: tabular-nums; 122 + font-size: 12px; 123 + } 124 + 125 + .body { min-width: 0; } 126 + .title-line { 127 + color: var(--text-bright); 128 + word-break: break-word; 129 + line-height: 1.4; 130 + } 131 + .title-line a { color: inherit; } 132 + .title-line a:hover { color: #2a9d5c; } 133 + .source { 134 + font-size: 11px; 135 + color: var(--text-dim); 136 + margin-top: 0.15rem; 137 + } 138 + 139 + .count { 140 + color: var(--text-secondary); 141 + font-variant-numeric: tabular-nums; 142 + font-size: 12px; 143 + white-space: nowrap; 144 + } 145 + .count .label { 146 + color: var(--text-dim); 147 + font-size: 10px; 148 + margin-left: 0.25rem; 149 + } 150 + 151 + .nav-row { 152 + margin-bottom: 1.5rem; 153 + font-size: 11px; 154 + color: var(--text-dim); 155 + } 156 + .nav-row a { color: var(--text-secondary); } 157 + .nav-row a:hover { color: var(--text-bright); } 158 + 159 + /* mobile: same grid shape (rank | body | count) but tighter columns 160 + * and slightly smaller title. count stays on the first text line; if 161 + * title wraps, source flows under it inside `body`. */ 162 + @media (max-width: 480px) { 163 + .row { 164 + grid-template-columns: 2rem 1fr auto; 165 + column-gap: 0.5rem; 166 + padding: 0.6rem 0; 167 + align-items: start; 168 + } 169 + .title-line { font-size: 13px; } 170 + .count .label { display: none; } /* just the number on tight screens */ 171 + } 172 + </style> 173 + </head> 174 + <body> 175 + <div class="container"> 176 + <h1> 177 + <a href="/" class="title">pub search</a> 178 + <span class="dim">/ recommended</span> 179 + <button class="lf-settings-btn" id="settings-btn" aria-label="settings" title="settings"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><circle cx="12" cy="12" r="3"></circle><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 1 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 1 1-4 0v-.09a1.65 1.65 0 0 0-1-1.51 1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 1 1-2.83-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 1 1 0-4h.09a1.65 1.65 0 0 0 1.51-1 1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 1 1 2.83-2.83l.06.06a1.65 1.65 0 0 0 1.82.33h.01A1.65 1.65 0 0 0 10 4.6V3a2 2 0 1 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 1 1 2.83 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 1 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"></path></svg></button> 180 + </h1> 181 + 182 + <div class="intro"> 183 + most-recommended posts across atproto. counts come from <code>site.standard.graph.recommend</code> records. 184 + </div> 185 + 186 + <div class="nav-row"> 187 + « <a href="/">back to search</a> 188 + </div> 189 + 190 + <div id="status" class="status">loading...</div> 191 + <div id="list"></div> 192 + </div> 193 + 194 + <script> 195 + var API_BASE = (location.hostname === 'localhost' || location.hostname === '127.0.0.1') 196 + ? 'http://localhost:3000' 197 + : 'https://leaflet-search-backend.fly.dev'; 198 + 199 + function escapeHtml(s) { 200 + return String(s == null ? '' : s) 201 + .replace(/&/g, '&') 202 + .replace(/</g, '<') 203 + .replace(/>/g, '>') 204 + .replace(/"/g, '"') 205 + .replace(/'/g, '''); 206 + } 207 + 208 + function render(rows) { 209 + var status = document.getElementById('status'); 210 + var list = document.getElementById('list'); 211 + if (!rows || rows.length === 0) { 212 + status.textContent = 'no recommends indexed yet.'; 213 + list.innerHTML = ''; 214 + return; 215 + } 216 + status.textContent = ''; 217 + var html = ''; 218 + rows.forEach(function(doc, i) { 219 + var rank = String(i + 1).padStart(2, '0'); 220 + var title = escapeHtml(doc.title || 'Untitled'); 221 + var url = doc.url || ''; 222 + var source = doc.basePath || doc.publicationName || doc.platform || ''; 223 + var titleHtml = url 224 + ? '<a href="' + escapeHtml(url) + '" target="_blank" rel="noopener">' + title + '</a>' 225 + : title; 226 + html += '<div class="row">' + 227 + '<div class="rank">' + rank + '</div>' + 228 + '<div class="body">' + 229 + '<div class="title-line">' + titleHtml + '</div>' + 230 + (source ? '<div class="source">' + escapeHtml(source) + '</div>' : '') + 231 + '</div>' + 232 + '<div class="count">' + doc.recommendCount + 233 + '<span class="label">rec' + (doc.recommendCount === 1 ? '' : 's') + '</span>' + 234 + '</div>' + 235 + '</div>'; 236 + }); 237 + list.innerHTML = html; 238 + } 239 + 240 + fetch(API_BASE + '/recommended') 241 + .then(function(r) { 242 + if (!r.ok) throw new Error('http ' + r.status); 243 + return r.json(); 244 + }) 245 + .then(function(data) { 246 + if (data && data.error) throw new Error(data.error); 247 + render(data); 248 + }) 249 + .catch(function(err) { 250 + document.getElementById('status').textContent = 'failed to load: ' + err.message; 251 + }); 252 + 253 + // settings gear opens the preferred-client picker; index.html owns the 254 + // actual modal, so on this page the button just navigates there with a 255 + // hash that index.html could pick up later. 256 + document.getElementById('settings-btn').addEventListener('click', function() { 257 + location.href = '/#settings'; 258 + }); 259 + </script> 260 + </body> 261 + </html>

Configure Feed

Configure Feed