nandi.uk
This repository has no description
1/**
2 * Feed watcher — polls the Paper Skygest feed and extracts paper URLs.
3 *
4 * The feed is a Bluesky custom feed generator at:
5 * at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest
6 *
7 * We poll it via the public API and extract paper URLs from:
8 * - embed.external.uri (most reliable)
9 * - Post text (arxiv.org, doi.org, nber.org, openreview.net, etc.)
10 */
11
12const FEED_URI = "at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest";
13const BSKY_PUBLIC_API = "https://public.api.bsky.app";
14
15const PAPER_URL_PATTERNS = [
16 /https?:\/\/arxiv\.org\/abs\/\d+\.\d+/,
17 /https?:\/\/arxiv\.org\/abs\/\d{4}\.\d{4,5}/,
18 /https?:\/\/doi\.org\/10\.\d{4,}\/[^\s]+/,
19 /https?:\/\/www\.nber\.org\/papers\/w\d+/,
20 /https?:\/\/openreview\.net\/forum\?id=[^\s]+/,
21 /https?:\/\/papers\.ssrn\.com\/sol3\/papers\.cfm\?abstract_id=\d+/,
22 /https?:\/\/proceedings\.mlr\.press\/v\d+\//,
23 /https?:\/\/www\.science\.org\/doi\/[^\s]+/,
24 /https?:\/\/www\.nature\.com\/articles\/[^\s]+/,
25 /https?:\/\/journals\.plos\.org\/[^\s]+\/article\?id=[^\s]+/,
26 /https?:\/\/www\.biorxiv\.org\/content\/[^\s]+/,
27 /https?:\/\/www\.medrxiv\.org\/content\/[^\s]+/,
28 /https?:\/\/aclanthology\.org\/[^\s]+/,
29 /https?:\/\/proceedings\.neurips\.cc\/paper_files\/[^\s]+/,
30 /https?:\/\/openaccess\.thecvf\.com\/[^\s]+/,
31];
32
33export interface PaperPost {
34 postUri: string;
35 postCid: string;
36 paperUrl: string;
37 posterDid: string;
38 posterHandle: string;
39 postText: string;
40 embedTitle?: string;
41 embedDescription?: string;
42 indexedAt: string;
43}
44
45function extractPaperUrls(text: string): string[] {
46 const urls: string[] = [];
47 for (const pattern of PAPER_URL_PATTERNS) {
48 const match = text.match(pattern);
49 if (match) urls.push(match[0]);
50 }
51 return urls;
52}
53
54function extractFromEmbed(embed: any): string | null {
55 if (!embed) return null;
56 if (embed.$type === "app.bsky.embed.external#view" && embed.external?.uri) {
57 const uri = embed.external.uri as string;
58 for (const pattern of PAPER_URL_PATTERNS) {
59 if (pattern.test(uri)) return uri;
60 }
61 if (/https?:\/\/(arxiv|doi|nber|openreview|ssrn|mlr|science|nature|plos|biorxiv|medrxiv|aclanthology|neurips|thecvf)\./.test(uri)) {
62 return uri;
63 }
64 }
65 return null;
66}
67
68export async function pollFeed(cursor?: string, limit = 50): Promise<{ posts: PaperPost[]; cursor?: string }> {
69 const params = new URLSearchParams({
70 feed: FEED_URI,
71 limit: String(limit),
72 });
73 if (cursor) params.set("cursor", cursor);
74
75 const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.feed.getFeed?${params}`);
76 if (!res.ok) {
77 const body = await res.text();
78 throw new Error(`Feed poll failed (${res.status}): ${body}`);
79 }
80
81 const data = await res.json() as {
82 cursor?: string;
83 feed: Array<{
84 post: {
85 uri: string;
86 cid: string;
87 record: { text?: string };
88 author: { did: string; handle: string };
89 embed?: any;
90 indexedAt: string;
91 };
92 }>;
93 };
94
95 const posts: PaperPost[] = [];
96 const seenUris = new Set<string>();
97
98 for (const item of data.feed) {
99 const { uri, cid, author, embed, indexedAt } = item.post;
100 const text = item.post.record?.text || "";
101
102 let paperUrl = extractFromEmbed(embed);
103 if (!paperUrl) {
104 const textUrls = extractPaperUrls(text);
105 if (textUrls.length > 0) paperUrl = textUrls[0];
106 }
107
108 if (!paperUrl) continue;
109 if (seenUris.has(paperUrl)) continue;
110 seenUris.add(paperUrl);
111
112 let embedTitle: string | undefined;
113 let embedDescription: string | undefined;
114 if (embed?.$type === "app.bsky.embed.external#view" && embed.external) {
115 embedTitle = embed.external.title;
116 embedDescription = embed.external.description;
117 }
118
119 posts.push({
120 postUri: uri,
121 postCid: cid,
122 paperUrl,
123 posterDid: author.did,
124 posterHandle: author.handle,
125 postText: text,
126 embedTitle,
127 embedDescription,
128 indexedAt,
129 });
130 }
131
132 return { posts, cursor: data.cursor };
133}
134
135export async function pollNewPapers(lastCursor?: string): Promise<{ posts: PaperPost[]; newCursor?: string }> {
136 const allPosts: PaperPost[] = [];
137 let cursor = lastCursor;
138
139 for (let i = 0; i < 3; i++) {
140 const result = await pollFeed(cursor, 50);
141 allPosts.push(...result.posts);
142 if (!result.cursor || result.cursor === cursor) break;
143 cursor = result.cursor;
144 if (result.posts.length < 50) break;
145 }
146
147 return { posts: allPosts, newCursor: cursor };
148}
149
150// --- Followed accounts feed ---
151
152const RESEARCHER_DID = "did:plc:3kkhul7jznlb6ba7rprzawnj";
153const PDS_ORIGIN = "https://pds.latha.org";
154
155async function getFollows(): Promise<Array<{ did: string; handle: string }>> {
156 const follows: Array<{ did: string; handle: string }> = [];
157 let cursor: string | undefined;
158
159 for (let i = 0; i < 10; i++) {
160 const params = new URLSearchParams({
161 repo: RESEARCHER_DID,
162 collection: "app.bsky.graph.follow",
163 limit: "100",
164 });
165 if (cursor) params.set("cursor", cursor);
166
167 const res = await fetch(`${PDS_ORIGIN}/xrpc/com.atproto.repo.listRecords?${params}`);
168 if (!res.ok) break;
169
170 const data = await res.json() as {
171 cursor?: string;
172 records: Array<{ value: { subject: string } }>;
173 };
174
175 for (const rec of data.records) {
176 follows.push({ did: rec.value.subject, handle: "" });
177 }
178
179 if (!data.cursor || data.cursor === cursor) break;
180 cursor = data.cursor;
181 if (data.records.length < 100) break;
182 }
183
184 // Resolve DIDs to handles
185 for (const f of follows) {
186 try {
187 const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.actor.getProfile?actor=${f.did}`);
188 if (res.ok) {
189 const profile = await res.json() as { handle: string };
190 f.handle = profile.handle;
191 }
192 } catch {
193 // skip
194 }
195 }
196
197 return follows;
198}
199
200export async function pollFollowedAccounts(limit = 5): Promise<PaperPost[]> {
201 const follows = await getFollows();
202 const allPosts: PaperPost[] = [];
203 const seenUris = new Set<string>();
204
205 for (const follow of follows) {
206 if (!follow.handle) continue;
207
208 try {
209 const params = new URLSearchParams({
210 actor: follow.handle,
211 limit: String(limit),
212 });
213
214 const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.feed.getAuthorFeed?${params}`);
215 if (!res.ok) continue;
216
217 const data = await res.json() as {
218 feed: Array<{
219 post: {
220 uri: string;
221 cid: string;
222 record: { text?: string };
223 author: { did: string; handle: string };
224 embed?: any;
225 indexedAt: string;
226 };
227 }>;
228 };
229
230 for (const item of data.feed) {
231 const { uri, cid, author, embed, indexedAt } = item.post;
232 const text = item.post.record?.text || "";
233
234 let paperUrl = extractFromEmbed(embed);
235 if (!paperUrl) {
236 const textUrls = extractPaperUrls(text);
237 if (textUrls.length > 0) paperUrl = textUrls[0];
238 }
239
240 if (!paperUrl) continue;
241 if (seenUris.has(paperUrl)) continue;
242 seenUris.add(paperUrl);
243
244 let embedTitle: string | undefined;
245 let embedDescription: string | undefined;
246 if (embed?.$type === "app.bsky.embed.external#view" && embed.external) {
247 embedTitle = embed.external.title;
248 embedDescription = embed.external.description;
249 }
250
251 allPosts.push({
252 postUri: uri,
253 postCid: cid,
254 paperUrl,
255 posterDid: author.did,
256 posterHandle: author.handle,
257 postText: text,
258 embedTitle,
259 embedDescription,
260 indexedAt,
261 });
262 }
263 } catch {
264 // skip this account
265 }
266 }
267
268 return allPosts;
269}