nandi.uk
This repository has no description
1/**
2 * Paper summarizer — fetches paper content and generates a structured summary.
3 *
4 * Two modes:
5 * 1. Worker mode: heuristic extraction from embed + OpenGraph enrichment
6 * 2. Letta enrichment: calls Letta API for rich LLM-generated summaries
7 */
8
9import { mystTextToOxaRecord } from "@nandithebull/myst-oxa";
10import type { PaperPost } from "./feed-watcher.js";
11
12export interface PaperSummary {
13 paperUrl: string;
14 title: string;
15 authors: string[];
16 abstract: string;
17 summary: string;
18 summaryDoc?: any;
19 domains: string[];
20 venue: string;
21 year: number | null;
22}
23
24export function extractFromPost(post: PaperPost): Partial<PaperSummary> {
25 const result: Partial<PaperSummary> = { paperUrl: post.paperUrl };
26
27 if (post.embedTitle) result.title = post.embedTitle;
28 if (post.embedDescription) result.abstract = post.embedDescription;
29
30 const authorMatch = post.postText.match(/from\s+(.+?)(?:\s+https?:\/\/|$)/);
31 if (authorMatch) {
32 result.authors = authorMatch[1]
33 .split(/,\s*|\s+and\s+/)
34 .map(a => a.trim())
35 .filter(a => a.length > 0);
36 }
37
38 if (post.paperUrl.includes("arxiv.org")) result.venue = "arXiv";
39 else if (post.paperUrl.includes("nber.org")) result.venue = "NBER";
40 else if (post.paperUrl.includes("openreview.net")) result.venue = "OpenReview";
41 else if (post.paperUrl.includes("ssrn.com")) result.venue = "SSRN";
42 else if (post.paperUrl.includes("biorxiv.org")) result.venue = "bioRxiv";
43 else if (post.paperUrl.includes("medrxiv.org")) result.venue = "medRxiv";
44 else if (post.paperUrl.includes("neurips.cc")) result.venue = "NeurIPS";
45 else if (post.paperUrl.includes("mlr.press")) result.venue = "JMLR/PMLR";
46 else if (post.paperUrl.includes("aclanthology.org")) result.venue = "ACL";
47 else if (post.paperUrl.includes("thecvf.com")) result.venue = "CVPR/ICCV/ECCV";
48
49 const yearMatch = post.paperUrl.match(/(\d{4})/);
50 if (yearMatch) {
51 const y = parseInt(yearMatch[1]);
52 if (y >= 1990 && y <= 2030) result.year = y;
53 }
54
55 if (post.embedDescription) {
56 const domainKeywords = extractDomainKeywords(post.embedDescription);
57 if (domainKeywords.length > 0) result.domains = domainKeywords;
58 }
59
60 return result;
61}
62
63const DOMAIN_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
64 { pattern: /\bmachine learning\b/i, domain: "machine learning" },
65 { pattern: /\bdeep learning\b/i, domain: "deep learning" },
66 { pattern: /\bnatural language processing\b|\bNLP\b/i, domain: "NLP" },
67 { pattern: /\bcomputer vision\b/i, domain: "computer vision" },
68 { pattern: /\breinforcement learning\b/i, domain: "reinforcement learning" },
69 { pattern: /\bgenerative AI\b|\bgenerative model/i, domain: "generative AI" },
70 { pattern: /\bneural network\b/i, domain: "neural networks" },
71 { pattern: /\btransformer\b/i, domain: "transformers" },
72 { pattern: /\blanguage model\b|\bLLM\b/i, domain: "language models" },
73 { pattern: /\brobotics\b/i, domain: "robotics" },
74 { pattern: /\bbioinformatics\b/i, domain: "bioinformatics" },
75 { pattern: /\bneuroscience\b/i, domain: "neuroscience" },
76 { pattern: /\bclimate\b/i, domain: "climate science" },
77 { pattern: /\beconomics?\b/i, domain: "economics" },
78 { pattern: /\bpsychology\b/i, domain: "psychology" },
79 { pattern: /\bsociology\b/i, domain: "sociology" },
80 { pattern: /\bphilosophy\b/i, domain: "philosophy" },
81 { pattern: /\bepidemiol/i, domain: "epidemiology" },
82 { pattern: /\bgenomics?\b/i, domain: "genomics" },
83 { pattern: /\bquantum\b/i, domain: "quantum computing" },
84];
85
86function extractDomainKeywords(text: string): string[] {
87 const domains: string[] = [];
88 for (const { pattern, domain } of DOMAIN_PATTERNS) {
89 if (pattern.test(text) && !domains.includes(domain)) {
90 domains.push(domain);
91 }
92 }
93 return domains;
94}
95
96export function generateHeuristicSummary(post: PaperPost, metadata: Partial<PaperSummary>): string {
97 const parts: string[] = [];
98 if (metadata.title) parts.push(metadata.title);
99 if (metadata.abstract) {
100 const sentences = metadata.abstract.split(/\.\s+/).slice(0, 2);
101 parts.push(sentences.join(". ") + ".");
102 }
103 if (metadata.authors?.length) {
104 const authorStr = metadata.authors.length <= 3
105 ? metadata.authors.join(", ")
106 : `${metadata.authors[0]} et al.`;
107 parts.push(`By ${authorStr}.`);
108 }
109 if (metadata.venue) {
110 parts.push(`Published in ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}.`);
111 }
112 return parts.join(" ");
113}
114
115export async function fetchOpenGraph(paperUrl: string): Promise<{ title?: string; description?: string }> {
116 try {
117 const res = await fetch(paperUrl, {
118 headers: { "User-Agent": "papers-appview/0.1 (mailto:researcher@pds.latha.org)" },
119 signal: AbortSignal.timeout(10000),
120 });
121 if (!res.ok) return {};
122 const html = await res.text();
123 const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]+)"/)?.[1];
124 const ogDesc = html.match(/<meta\s+property="og:description"\s+content="([^"]+)"/)?.[1];
125 const titleTag = html.match(/<title>([^<]+)<\/title>/)?.[1];
126 return {
127 title: ogTitle || titleTag || undefined,
128 description: ogDesc || undefined,
129 };
130 } catch {
131 return {};
132 }
133}
134
135export async function buildSummary(post: PaperPost): Promise<PaperSummary> {
136 const metadata = extractFromPost(post);
137
138 if (!metadata.title || !metadata.abstract) {
139 const og = await fetchOpenGraph(post.paperUrl);
140 if (!metadata.title && og.title) metadata.title = og.title;
141 if (!metadata.abstract && og.description) metadata.abstract = og.description;
142 }
143
144 const summary = generateHeuristicSummary(post, metadata);
145
146 const mystText = [
147 metadata.title ? `# ${metadata.title}` : null,
148 metadata.authors?.length ? `**Authors:** ${metadata.authors.join(", ")}` : null,
149 metadata.venue ? `**Venue:** ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}` : null,
150 "",
151 metadata.abstract || "",
152 "",
153 "## Summary",
154 "",
155 summary,
156 ].filter(Boolean).join("\n");
157
158 let summaryDoc;
159 try {
160 summaryDoc = mystTextToOxaRecord(mystText);
161 } catch {
162 // OXA conversion is best-effort
163 }
164
165 return {
166 paperUrl: post.paperUrl,
167 title: metadata.title || "Untitled",
168 authors: metadata.authors || [],
169 abstract: metadata.abstract || "",
170 summary,
171 summaryDoc,
172 domains: metadata.domains || [],
173 venue: metadata.venue || "",
174 year: metadata.year || null,
175 };
176}
177
178// --- Letta-powered enrichment ---
179
180const LETTA_API_BASE = "https://api.letta.com/v1";
181
182const ENRICHMENT_PROMPT = `You are a research paper summarizer. Given a paper's URL, title, abstract, and existing metadata, produce a concise, insightful summary.
183
184Return ONLY a JSON object with these fields (no markdown, no code fences):
185- "title": the paper's real title. If the current title is "Untitled" or placeholder, derive it from the URL or abstract.
186- "summary": 2-3 sentence summary in your own words. Focus on what the paper does and why it matters. Not a restatement of the abstract.
187- "domains": array of 1-3 research domain strings (e.g. ["machine learning", "optimization"])
188- "takeaway": one sentence — the single most important thing a reader should remember
189
190Paper: {title}
191URL: {paperUrl}
192Abstract: {abstract}
193Existing domains: {domains}
194Venue: {venue}`;
195
196export interface LettaEnrichment {
197 title?: string;
198 summary: string;
199 domains: string[];
200 takeaway: string;
201}
202
203export async function enrichWithLetta(
204 paper: { title: string; paperUrl: string; abstract: string; domains: string[]; venue: string },
205 apiKey: string,
206 agentId: string,
207): Promise<LettaEnrichment> {
208 const prompt = ENRICHMENT_PROMPT
209 .replace("{title}", paper.title)
210 .replace("{paperUrl}", paper.paperUrl)
211 .replace("{abstract}", paper.abstract || "(no abstract available)")
212 .replace("{domains}", paper.domains.join(", ") || "unknown")
213 .replace("{venue}", paper.venue || "unknown");
214
215 const res = await fetch(`${LETTA_API_BASE}/agents/${agentId}/messages`, {
216 method: "POST",
217 headers: {
218 "Authorization": `Bearer ${apiKey}`,
219 "Content-Type": "application/json",
220 "Accept": "application/json",
221 },
222 body: JSON.stringify({
223 messages: [{ role: "user", content: prompt }],
224 }),
225 });
226
227 if (!res.ok) {
228 const body = await res.text();
229 throw new Error(`Letta API error (${res.status}): ${body}`);
230 }
231
232 const data = await res.json() as {
233 messages: Array<{ message_type: string; content?: string }>;
234 };
235
236 // Find the assistant message
237 const assistantMsg = data.messages.find(
238 (m) => m.message_type === "assistant_message" && m.content,
239 );
240
241 if (!assistantMsg?.content) {
242 throw new Error("No assistant message in Letta response");
243 }
244
245 // Parse JSON from the response — handle possible markdown code fences
246 let content = assistantMsg.content;
247 const jsonMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
248 if (jsonMatch) content = jsonMatch[1].trim();
249
250 let parsed: LettaEnrichment;
251 try {
252 parsed = JSON.parse(content);
253 } catch {
254 // If JSON parse fails, use the raw content as summary
255 parsed = { summary: content, domains: paper.domains, takeaway: "" };
256 }
257
258 return {
259 title: parsed.title || undefined,
260 summary: parsed.summary || content,
261 domains: parsed.domains || paper.domains,
262 takeaway: parsed.takeaway || "",
263 };
264}