nandi.uk
This repository has no description
1/**
2 * Paper summarizer — fetches paper content and generates a structured summary.
3 *
4 * Two modes:
5 * 1. Worker mode: heuristic extraction from embed + OpenGraph enrichment
6 * 2. Letta enrichment: calls Letta API for rich LLM-generated summaries
7 */
8
9import { mystTextToOxaRecord } from "@nandithebull/myst-oxa";
10import type { PaperPost } from "./feed-watcher.js";
11
12export interface PaperSummary {
13 paperUrl: string;
14 title: string;
15 authors: string[];
16 abstract: string;
17 summary: string;
18 summaryDoc?: any;
19 domains: string[];
20 venue: string;
21 year: number | null;
22 publishedDate: string | null;
23}
24
25export function extractFromPost(post: PaperPost): Partial<PaperSummary> {
26 const result: Partial<PaperSummary> = { paperUrl: post.paperUrl };
27
28 if (post.embedTitle) result.title = post.embedTitle;
29 if (post.embedDescription) result.abstract = post.embedDescription;
30
31 const authorMatch = post.postText.match(/from\s+(.+?)(?:\s+https?:\/\/|$)/);
32 if (authorMatch) {
33 result.authors = authorMatch[1]
34 .split(/,\s*|\s+and\s+/)
35 .map(a => a.trim())
36 .filter(a => a.length > 0);
37 }
38
39 if (post.paperUrl.includes("arxiv.org")) result.venue = "arXiv";
40 else if (post.paperUrl.includes("nber.org")) result.venue = "NBER";
41 else if (post.paperUrl.includes("openreview.net")) result.venue = "OpenReview";
42 else if (post.paperUrl.includes("ssrn.com")) result.venue = "SSRN";
43 else if (post.paperUrl.includes("biorxiv.org")) result.venue = "bioRxiv";
44 else if (post.paperUrl.includes("medrxiv.org")) result.venue = "medRxiv";
45 else if (post.paperUrl.includes("neurips.cc")) result.venue = "NeurIPS";
46 else if (post.paperUrl.includes("mlr.press")) result.venue = "JMLR/PMLR";
47 else if (post.paperUrl.includes("aclanthology.org")) result.venue = "ACL";
48 else if (post.paperUrl.includes("thecvf.com")) result.venue = "CVPR/ICCV/ECCV";
49
50 // Extract year from URL — venue-specific patterns to avoid matching IDs
51 const yearPatterns: Array<[RegExp, number]> = [
52 [/arxiv\.org\/abs\/(\d{2})(\d{2})\./, 2], // arXiv: YYMM -> 20YY
53 [/biorxiv\.org\/content\/.*?(\d{4})\./, 1], // bioRxiv: YYYY.MM
54 [/medrxiv\.org\/content\/.*?(\d{4})\./, 1], // medRxiv: YYYY.MM
55 [/nature\.com\/.*?(\d{4})/, 1], // Nature: embedded in article ID
56 [/nber\.org\/papers\/w(\d{2})/, 1], // NBER: wYYnnn
57 [/20(\d{2})/, 0], // Fallback: 20XX anywhere
58 ];
59 for (const [pattern, groupIdx] of yearPatterns) {
60 const m = post.paperUrl.match(pattern);
61 if (m) {
62 const raw = m[groupIdx + 1] || m[1];
63 const y = raw.length === 2 ? 2000 + parseInt(raw) : parseInt(raw);
64 if (y >= 1990 && y <= 2030) { result.year = y; break; }
65 }
66 }
67
68 if (post.embedDescription) {
69 const domainKeywords = extractDomainKeywords(post.embedDescription);
70 if (domainKeywords.length > 0) result.domains = domainKeywords;
71 }
72
73 return result;
74}
75
76const DOMAIN_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
77 { pattern: /\bmachine learning\b/i, domain: "machine learning" },
78 { pattern: /\bdeep learning\b/i, domain: "deep learning" },
79 { pattern: /\bnatural language processing\b|\bNLP\b/i, domain: "NLP" },
80 { pattern: /\bcomputer vision\b/i, domain: "computer vision" },
81 { pattern: /\breinforcement learning\b/i, domain: "reinforcement learning" },
82 { pattern: /\bgenerative AI\b|\bgenerative model/i, domain: "generative AI" },
83 { pattern: /\bneural network\b/i, domain: "neural networks" },
84 { pattern: /\btransformer\b/i, domain: "transformers" },
85 { pattern: /\blanguage model\b|\bLLM\b/i, domain: "language models" },
86 { pattern: /\brobotics\b/i, domain: "robotics" },
87 { pattern: /\bbioinformatics\b/i, domain: "bioinformatics" },
88 { pattern: /\bneuroscience\b/i, domain: "neuroscience" },
89 { pattern: /\bclimate\b/i, domain: "climate science" },
90 { pattern: /\beconomics?\b/i, domain: "economics" },
91 { pattern: /\bpsychology\b/i, domain: "psychology" },
92 { pattern: /\bsociology\b/i, domain: "sociology" },
93 { pattern: /\bphilosophy\b/i, domain: "philosophy" },
94 { pattern: /\bepidemiol/i, domain: "epidemiology" },
95 { pattern: /\bgenomics?\b/i, domain: "genomics" },
96 { pattern: /\bquantum\b/i, domain: "quantum computing" },
97];
98
99function extractDomainKeywords(text: string): string[] {
100 const domains: string[] = [];
101 for (const { pattern, domain } of DOMAIN_PATTERNS) {
102 if (pattern.test(text) && !domains.includes(domain)) {
103 domains.push(domain);
104 }
105 }
106 return domains;
107}
108
109export function generateHeuristicSummary(post: PaperPost, metadata: Partial<PaperSummary>): string {
110 const parts: string[] = [];
111 if (metadata.title) parts.push(metadata.title);
112 if (metadata.abstract) {
113 const sentences = metadata.abstract.split(/\.\s+/).slice(0, 2);
114 parts.push(sentences.join(". ") + ".");
115 }
116 if (metadata.authors?.length) {
117 const authorStr = metadata.authors.length <= 3
118 ? metadata.authors.join(", ")
119 : `${metadata.authors[0]} et al.`;
120 parts.push(`By ${authorStr}.`);
121 }
122 if (metadata.venue) {
123 parts.push(`Published in ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}.`);
124 }
125 return parts.join(" ");
126}
127
128export async function fetchOpenGraph(paperUrl: string): Promise<{ title?: string; description?: string }> {
129 try {
130 const res = await fetch(paperUrl, {
131 headers: { "User-Agent": "papers-appview/0.1 (mailto:researcher@pds.latha.org)" },
132 signal: AbortSignal.timeout(10000),
133 });
134 if (!res.ok) return {};
135 const html = await res.text();
136 const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]+)"/)?.[1];
137 const ogDesc = html.match(/<meta\s+property="og:description"\s+content="([^"]+)"/)?.[1];
138 const titleTag = html.match(/<title>([^<]+)<\/title>/)?.[1];
139 return {
140 title: ogTitle || titleTag || undefined,
141 description: ogDesc || undefined,
142 };
143 } catch {
144 return {};
145 }
146}
147
148export async function buildSummary(post: PaperPost): Promise<PaperSummary> {
149 const metadata = extractFromPost(post);
150
151 if (!metadata.title || !metadata.abstract) {
152 const og = await fetchOpenGraph(post.paperUrl);
153 if (!metadata.title && og.title) metadata.title = og.title;
154 if (!metadata.abstract && og.description) metadata.abstract = og.description;
155 }
156
157 const summary = generateHeuristicSummary(post, metadata);
158
159 const mystText = [
160 metadata.title ? `# ${metadata.title}` : null,
161 metadata.authors?.length ? `**Authors:** ${metadata.authors.join(", ")}` : null,
162 metadata.venue ? `**Venue:** ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}` : null,
163 "",
164 metadata.abstract || "",
165 "",
166 "## Summary",
167 "",
168 summary,
169 ].filter(Boolean).join("\n");
170
171 let summaryDoc;
172 try {
173 summaryDoc = mystTextToOxaRecord(mystText);
174 } catch {
175 // OXA conversion is best-effort
176 }
177
178 return {
179 paperUrl: post.paperUrl,
180 title: metadata.title || "Untitled",
181 authors: metadata.authors || [],
182 abstract: metadata.abstract || "",
183 summary,
184 summaryDoc,
185 domains: metadata.domains || [],
186 venue: metadata.venue || "",
187 year: metadata.year || null,
188 };
189}
190
191// --- Letta-powered enrichment ---
192
193const LETTA_API_BASE = "https://api.letta.com/v1";
194
195const ENRICHMENT_PROMPT = `You are a research paper summarizer. Given a paper's URL, title, abstract, and existing metadata, produce a concise, insightful summary.
196
197Return ONLY a JSON object with these fields (no markdown, no code fences):
198- "title": the paper's real title. If the current title is "Untitled" or placeholder, derive it from the URL or abstract.
199- "summary": 2-3 sentence summary in your own words. Focus on what the paper does and why it matters. Not a restatement of the abstract.
200- "domains": array of 1-3 research domain strings (e.g. ["machine learning", "optimization"])
201- "takeaway": one sentence — the single most important thing a reader should remember
202- "publishedDate": the publication date in YYYY-MM-DD format. Derive from the URL (e.g. biorxiv.org URLs contain YYYY.MM.DD, arXiv URLs contain YYMM), abstract, or title. If only month is known, use YYYY-MM-01. If only year, use YYYY-01-01.
203
204Paper: {title}
205URL: {paperUrl}
206Abstract: {abstract}
207Existing domains: {domains}
208Venue: {venue}`;
209
210export interface LettaEnrichment {
211 title?: string;
212 summary: string;
213 domains: string[];
214 takeaway: string;
215 year?: number;
216 publishedDate?: string;
217}
218
219export async function enrichWithLetta(
220 paper: { title: string; paperUrl: string; abstract: string; domains: string[]; venue: string },
221 apiKey: string,
222 agentId: string,
223): Promise<LettaEnrichment> {
224 const prompt = ENRICHMENT_PROMPT
225 .replace("{title}", paper.title)
226 .replace("{paperUrl}", paper.paperUrl)
227 .replace("{abstract}", paper.abstract || "(no abstract available)")
228 .replace("{domains}", paper.domains.join(", ") || "unknown")
229 .replace("{venue}", paper.venue || "unknown");
230
231 const res = await fetch(`${LETTA_API_BASE}/agents/${agentId}/messages`, {
232 method: "POST",
233 headers: {
234 "Authorization": `Bearer ${apiKey}`,
235 "Content-Type": "application/json",
236 "Accept": "application/json",
237 },
238 body: JSON.stringify({
239 messages: [{ role: "user", content: prompt }],
240 }),
241 });
242
243 if (!res.ok) {
244 const body = await res.text();
245 throw new Error(`Letta API error (${res.status}): ${body}`);
246 }
247
248 const data = await res.json() as {
249 messages: Array<{ message_type: string; content?: string }>;
250 };
251
252 // Find the assistant message
253 const assistantMsg = data.messages.find(
254 (m) => m.message_type === "assistant_message" && m.content,
255 );
256
257 if (!assistantMsg?.content) {
258 throw new Error("No assistant message in Letta response");
259 }
260
261 // Parse JSON from the response — handle possible markdown code fences
262 let content = assistantMsg.content;
263 const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
264 if (jsonMatch) content = jsonMatch[1].trim();
265
266 let parsed: LettaEnrichment;
267 try {
268 parsed = JSON.parse(content);
269 } catch {
270 // If JSON parse fails, use the raw content as summary
271 parsed = { summary: content, domains: paper.domains, takeaway: "" };
272 }
273
274 return {
275 title: parsed.title || undefined,
276 summary: parsed.summary || content,
277 domains: parsed.domains || paper.domains,
278 takeaway: parsed.takeaway || "",
279 year: parsed.year || undefined,
280 publishedDate: parsed.publishedDate || undefined,
281 };
282}