nandi.uk / papers
This repository has no description
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

Poll followed accounts for paper links, fix D1 indexing

- pollFollowedAccounts() fetches recent posts from all 39 followed accounts
- Extracts paper URLs from post text and embeds
- Merges with Paper Skygest feed results (deduped)
- Fixed contrail.notify error (replaced with direct D1 insert)
- Fixed </script> in JSON breaking SSR
- Enriched all 18 new records via Letta API
- 26 total summaries now indexed

👾 Generated with [Letta Code](https://letta.com)

Co-Authored-By: Letta Code <noreply@letta.com>

author
Nandi co-author
Letta Code date (May 17, 2026, 9:51 PM -0700) commit c5be4bc6 parent c5288201
+183 -9
+121
src/feed-watcher.ts
··· 146 146 147 147 return { posts: allPosts, newCursor: cursor }; 148 148 } 149 + 150 + // --- Followed accounts feed --- 151 + 152 + const RESEARCHER_DID = "did:plc:3kkhul7jznlb6ba7rprzawnj"; 153 + const PDS_ORIGIN = "https://pds.latha.org"; 154 + 155 + async function getFollows(): Promise<Array<{ did: string; handle: string }>> { 156 + const follows: Array<{ did: string; handle: string }> = []; 157 + let cursor: string | undefined; 158 + 159 + for (let i = 0; i < 10; i++) { 160 + const params = new URLSearchParams({ 161 + repo: RESEARCHER_DID, 162 + collection: "app.bsky.graph.follow", 163 + limit: "100", 164 + }); 165 + if (cursor) params.set("cursor", cursor); 166 + 167 + const res = await fetch(`${PDS_ORIGIN}/xrpc/com.atproto.repo.listRecords?${params}`); 168 + if (!res.ok) break; 169 + 170 + const data = await res.json() as { 171 + cursor?: string; 172 + records: Array<{ value: { subject: string } }>; 173 + }; 174 + 175 + for (const rec of data.records) { 176 + follows.push({ did: rec.value.subject, handle: "" }); 177 + } 178 + 179 + if (!data.cursor || data.cursor === cursor) break; 180 + cursor = data.cursor; 181 + if (data.records.length < 100) break; 182 + } 183 + 184 + // Resolve DIDs to handles 185 + for (const f of follows) { 186 + try { 187 + const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.actor.getProfile?actor=${f.did}`); 188 + if (res.ok) { 189 + const profile = await res.json() as { handle: string }; 190 + f.handle = profile.handle; 191 + } 192 + } catch { 193 + // skip 194 + } 195 + } 196 + 197 + return follows; 198 + } 199 + 200 + export async function pollFollowedAccounts(limit = 5): Promise<PaperPost[]> { 201 + const follows = await getFollows(); 202 + const allPosts: PaperPost[] = []; 203 + const seenUris = new Set<string>(); 204 + 205 + for (const follow of follows) { 206 + if (!follow.handle) continue; 207 + 208 + try { 209 + const params = new URLSearchParams({ 210 + actor: follow.handle, 211 + limit: String(limit), 212 + }); 213 + 214 + const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.feed.getAuthorFeed?${params}`); 215 + if (!res.ok) continue; 216 + 217 + const data = await res.json() as { 218 + feed: Array<{ 219 + post: { 220 + uri: string; 221 + cid: string; 222 + record: { text?: string }; 223 + author: { did: string; handle: string }; 224 + embed?: any; 225 + indexedAt: string; 226 + }; 227 + }>; 228 + }; 229 + 230 + for (const item of data.feed) { 231 + const { uri, cid, author, embed, indexedAt } = item.post; 232 + const text = item.post.record?.text || ""; 233 + 234 + let paperUrl = extractFromEmbed(embed); 235 + if (!paperUrl) { 236 + const textUrls = extractPaperUrls(text); 237 + if (textUrls.length > 0) paperUrl = textUrls[0]; 238 + } 239 + 240 + if (!paperUrl) continue; 241 + if (seenUris.has(paperUrl)) continue; 242 + seenUris.add(paperUrl); 243 + 244 + let embedTitle: string | undefined; 245 + let embedDescription: string | undefined; 246 + if (embed?.$type === "app.bsky.embed.external#view" && embed.external) { 247 + embedTitle = embed.external.title; 248 + embedDescription = embed.external.description; 249 + } 250 + 251 + allPosts.push({ 252 + postUri: uri, 253 + postCid: cid, 254 + paperUrl, 255 + posterDid: author.did, 256 + posterHandle: author.handle, 257 + postText: text, 258 + embedTitle, 259 + embedDescription, 260 + indexedAt, 261 + }); 262 + } 263 + } catch { 264 + // skip this account 265 + } 266 + } 267 + 268 + return allPosts; 269 + }
+62 -9
src/worker.ts
··· 11 11 import { Contrail } from "@atmo-dev/contrail"; 12 12 import { lexicons } from "../lexicons/generated/index.js"; 13 13 import { config } from "./contrail.config.js"; 14 - import { pollNewPapers } from "./feed-watcher.js"; 14 + import { pollNewPapers, pollFollowedAccounts, type PaperPost } from "./feed-watcher.js"; 15 15 import { buildSummary, enrichWithLetta } from "./paper-summarizer.js"; 16 16 import { writeSummaryToPds } from "./pds-writer.js"; 17 17 import { LANDING_PAGE } from "./landing-page.js"; ··· 80 80 81 81 const page = LANDING_PAGE.replace( 82 82 "</head>", 83 - `<script>window.__SUMMARIES__=${JSON.stringify(summaries)};</script></head>` 83 + `<script>window.__SUMMARIES__=${JSON.stringify(summaries).replace(/<\//g, "<\\/")};</script></head>` 84 84 ); 85 85 return c.html(page); 86 86 }); ··· 168 168 // first run 169 169 } 170 170 171 - const { posts, newCursor } = await pollNewPapers(lastCursor); 171 + const { posts: feedPosts, newCursor } = await pollNewPapers(lastCursor); 172 + 173 + // Also poll followed accounts for paper links 174 + let followedPosts: PaperPost[] = []; 175 + try { 176 + followedPosts = await pollFollowedAccounts(5); 177 + } catch (err: any) { 178 + console.error(`Followed accounts poll failed: ${err?.message ?? err}`); 179 + } 180 + 181 + // Merge and deduplicate 182 + const seenPaperUrls = new Set<string>(); 183 + const posts: PaperPost[] = []; 184 + for (const p of [...feedPosts, ...followedPosts]) { 185 + if (!seenPaperUrls.has(p.paperUrl)) { 186 + seenPaperUrls.add(p.paperUrl); 187 + posts.push(p); 188 + } 189 + } 172 190 173 191 // Deduplicate against existing records 174 192 const existingUrls = new Set<string>(); ··· 189 207 const newPosts = posts.filter(p => !existingUrls.has(p.paperUrl)); 190 208 let written = 0; 191 209 let errors = 0; 210 + const errorDetails: string[] = []; 192 211 193 212 for (const post of newPosts) { 194 213 try { 195 214 const summary = await buildSummary(post); 196 215 const result = await writeSummaryToPds(post, summary); 197 216 198 - // Notify contrail to index the new record 199 - await contrail.notify(result.uri); 217 + // Index the new record into D1 directly 218 + const rkey = result.uri.split("/").pop() || ""; 219 + await db 220 + .prepare("INSERT OR REPLACE INTO records_summary (uri, cid, did, rkey, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?)") 221 + .bind(result.uri, result.cid, "did:plc:3kkhul7jznlb6ba7rprzawnj", rkey, JSON.stringify({ 222 + $type: "org.latha.papers.summary", 223 + ...summary, 224 + sourceUri: post.postUri, 225 + posterDid: post.posterDid, 226 + posterHandle: post.posterHandle, 227 + postText: post.postText, 228 + indexedAt: new Date().toISOString(), 229 + }), Date.now() * 1000, Date.now() * 1000) 230 + .run(); 200 231 written++; 201 232 } catch (err: any) { 202 - console.error(`Failed to write summary for ${post.paperUrl}: ${err?.message ?? err}`); 233 + const msg = `${post.paperUrl}: ${err?.message ?? err}`; 234 + console.error(`Failed to write summary for ${msg}`); 235 + if (errors < 3) errorDetails.push(msg.substring(0, 200)); 203 236 errors++; 204 237 } 205 238 } ··· 214 247 215 248 return c.json({ 216 249 polled: posts.length, 250 + fromFeed: feedPosts.length, 251 + fromFollows: followedPosts.length, 217 252 new: newPosts.length, 218 253 written, 219 254 errors, 255 + lastError: errors > 0 ? errorDetails[0] : undefined, 220 256 }); 221 257 }); 222 258 ··· 232 268 } catch { 233 269 // table might not exist yet 234 270 } 235 - return c.json({ ok: true, summaries: count }); 271 + const hasKey = !!process.env.RESEARCHER_KEY_PEM; 272 + return c.json({ ok: true, summaries: count, hasPdsKey: hasKey }); 236 273 }); 237 274 238 275 // Enrich existing summaries using Letta API ··· 396 433 // first run 397 434 } 398 435 399 - const { posts, newCursor } = await pollNewPapers(lastCursor); 436 + const { posts: feedPosts, newCursor } = await pollNewPapers(lastCursor); 400 437 401 - // Deduplicate 438 + // Also poll followed accounts 439 + let followedPosts: PaperPost[] = []; 440 + try { 441 + followedPosts = await pollFollowedAccounts(5); 442 + } catch {} 443 + 444 + // Merge and deduplicate 445 + const seenPaperUrls = new Set<string>(); 446 + const posts: PaperPost[] = []; 447 + for (const p of [...feedPosts, ...followedPosts]) { 448 + if (!seenPaperUrls.has(p.paperUrl)) { 449 + seenPaperUrls.add(p.paperUrl); 450 + posts.push(p); 451 + } 452 + } 453 + 454 + // Deduplicate against existing records 402 455 const existingUrls = new Set<string>(); 403 456 if (posts.length > 0) { 404 457 const placeholders = posts.map(() => "?").join(",");