nandi.uk
This repository has no description
1/**
2 * Feed watcher — polls the Paper Skygest feed and extracts paper URLs.
3 *
4 * The feed is a Bluesky custom feed generator at:
5 * at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest
6 *
7 * We poll it via the public API and extract paper URLs from:
8 * - embed.external.uri (most reliable)
9 * - Post text (arxiv.org, doi.org, nber.org, openreview.net, etc.)
10 */
11
12const FEED_URI = "at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest";
13const BSKY_PUBLIC_API = "https://public.api.bsky.app";
14
15const PAPER_URL_PATTERNS = [
16 /https?:\/\/arxiv\.org\/abs\/\d+\.\d+/,
17 /https?:\/\/arxiv\.org\/abs\/\d{4}\.\d{4,5}/,
18 /https?:\/\/doi\.org\/10\.\d{4,}\/[^\s]+/,
19 /https?:\/\/www\.nber\.org\/papers\/w\d+/,
20 /https?:\/\/openreview\.net\/forum\?id=[^\s]+/,
21 /https?:\/\/papers\.ssrn\.com\/sol3\/papers\.cfm\?abstract_id=\d+/,
22 /https?:\/\/proceedings\.mlr\.press\/v\d+\//,
23 /https?:\/\/www\.science\.org\/doi\/[^\s]+/,
24 /https?:\/\/www\.nature\.com\/articles\/[^\s]+/,
25 /https?:\/\/journals\.plos\.org\/[^\s]+\/article\?id=[^\s]+/,
26 /https?:\/\/www\.biorxiv\.org\/content\/[^\s]+/,
27 /https?:\/\/www\.medrxiv\.org\/content\/[^\s]+/,
28 /https?:\/\/aclanthology\.org\/[^\s]+/,
29 /https?:\/\/proceedings\.neurips\.cc\/paper_files\/[^\s]+/,
30 /https?:\/\/openaccess\.thecvf\.com\/[^\s]+/,
31];
32
33export interface PaperPost {
34 postUri: string;
35 postCid: string;
36 paperUrl: string;
37 posterDid: string;
38 posterHandle: string;
39 postText: string;
40 embedTitle?: string;
41 embedDescription?: string;
42 indexedAt: string;
43}
44
45function extractPaperUrls(text: string): string[] {
46 const urls: string[] = [];
47 for (const pattern of PAPER_URL_PATTERNS) {
48 const match = text.match(pattern);
49 if (match) urls.push(match[0]);
50 }
51 return urls;
52}
53
54function extractFromEmbed(embed: any): string | null {
55 if (!embed) return null;
56 if (embed.$type === "app.bsky.embed.external#view" && embed.external?.uri) {
57 const uri = embed.external.uri as string;
58 for (const pattern of PAPER_URL_PATTERNS) {
59 if (pattern.test(uri)) return uri;
60 }
61 if (/https?:\/\/(arxiv|doi|nber|openreview|ssrn|mlr|science|nature|plos|biorxiv|medrxiv|aclanthology|neurips|thecvf)\./.test(uri)) {
62 return uri;
63 }
64 }
65 return null;
66}
67
68export async function pollFeed(cursor?: string, limit = 50): Promise<{ posts: PaperPost[]; cursor?: string }> {
69 const params = new URLSearchParams({
70 feed: FEED_URI,
71 limit: String(limit),
72 });
73 if (cursor) params.set("cursor", cursor);
74
75 const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.feed.getFeed?${params}`);
76 if (!res.ok) {
77 const body = await res.text();
78 throw new Error(`Feed poll failed (${res.status}): ${body}`);
79 }
80
81 const data = await res.json() as {
82 cursor?: string;
83 feed: Array<{
84 post: {
85 uri: string;
86 cid: string;
87 record: { text?: string };
88 author: { did: string; handle: string };
89 embed?: any;
90 indexedAt: string;
91 };
92 }>;
93 };
94
95 const posts: PaperPost[] = [];
96 const seenUris = new Set<string>();
97
98 for (const item of data.feed) {
99 const { uri, cid, author, embed, indexedAt } = item.post;
100 const text = item.post.record?.text || "";
101
102 let paperUrl = extractFromEmbed(embed);
103 if (!paperUrl) {
104 const textUrls = extractPaperUrls(text);
105 if (textUrls.length > 0) paperUrl = textUrls[0];
106 }
107
108 if (!paperUrl) continue;
109 if (seenUris.has(paperUrl)) continue;
110 seenUris.add(paperUrl);
111
112 let embedTitle: string | undefined;
113 let embedDescription: string | undefined;
114 if (embed?.$type === "app.bsky.embed.external#view" && embed.external) {
115 embedTitle = embed.external.title;
116 embedDescription = embed.external.description;
117 }
118
119 posts.push({
120 postUri: uri,
121 postCid: cid,
122 paperUrl,
123 posterDid: author.did,
124 posterHandle: author.handle,
125 postText: text,
126 embedTitle,
127 embedDescription,
128 indexedAt,
129 });
130 }
131
132 return { posts, cursor: data.cursor };
133}
134
135export async function pollNewPapers(lastCursor?: string): Promise<{ posts: PaperPost[]; newCursor?: string }> {
136 const allPosts: PaperPost[] = [];
137 let cursor = lastCursor;
138
139 for (let i = 0; i < 3; i++) {
140 const result = await pollFeed(cursor, 50);
141 allPosts.push(...result.posts);
142 if (!result.cursor || result.cursor === cursor) break;
143 cursor = result.cursor;
144 if (result.posts.length < 50) break;
145 }
146
147 return { posts: allPosts, newCursor: cursor };
148}