nandi.uk
This repository has no description
1/**
2 * Paper summarizer — fetches paper content and generates a structured summary.
3 *
4 * Two modes:
5 * 1. Worker mode: heuristic extraction from embed + OpenGraph enrichment
6 * 2. CLI mode: uses letta-code-sdk for rich OXA-backed summaries
7 */
8
9import { mystTextToOxaRecord } from "@nandithebull/myst-oxa";
10import type { PaperPost } from "./feed-watcher.js";
11
12export interface PaperSummary {
13 paperUrl: string;
14 title: string;
15 authors: string[];
16 abstract: string;
17 summary: string;
18 summaryDoc?: any;
19 domains: string[];
20 venue: string;
21 year: number | null;
22}
23
24export function extractFromPost(post: PaperPost): Partial<PaperSummary> {
25 const result: Partial<PaperSummary> = { paperUrl: post.paperUrl };
26
27 if (post.embedTitle) result.title = post.embedTitle;
28 if (post.embedDescription) result.abstract = post.embedDescription;
29
30 const authorMatch = post.postText.match(/from\s+(.+?)(?:\s+https?:\/\/|$)/);
31 if (authorMatch) {
32 result.authors = authorMatch[1]
33 .split(/,\s*|\s+and\s+/)
34 .map(a => a.trim())
35 .filter(a => a.length > 0);
36 }
37
38 if (post.paperUrl.includes("arxiv.org")) result.venue = "arXiv";
39 else if (post.paperUrl.includes("nber.org")) result.venue = "NBER";
40 else if (post.paperUrl.includes("openreview.net")) result.venue = "OpenReview";
41 else if (post.paperUrl.includes("ssrn.com")) result.venue = "SSRN";
42 else if (post.paperUrl.includes("biorxiv.org")) result.venue = "bioRxiv";
43 else if (post.paperUrl.includes("medrxiv.org")) result.venue = "medRxiv";
44 else if (post.paperUrl.includes("neurips.cc")) result.venue = "NeurIPS";
45 else if (post.paperUrl.includes("mlr.press")) result.venue = "JMLR/PMLR";
46 else if (post.paperUrl.includes("aclanthology.org")) result.venue = "ACL";
47 else if (post.paperUrl.includes("thecvf.com")) result.venue = "CVPR/ICCV/ECCV";
48
49 const yearMatch = post.paperUrl.match(/(\d{4})/);
50 if (yearMatch) {
51 const y = parseInt(yearMatch[1]);
52 if (y >= 1990 && y <= 2030) result.year = y;
53 }
54
55 if (post.embedDescription) {
56 const domainKeywords = extractDomainKeywords(post.embedDescription);
57 if (domainKeywords.length > 0) result.domains = domainKeywords;
58 }
59
60 return result;
61}
62
63const DOMAIN_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
64 { pattern: /\bmachine learning\b/i, domain: "machine learning" },
65 { pattern: /\bdeep learning\b/i, domain: "deep learning" },
66 { pattern: /\bnatural language processing\b|\bNLP\b/i, domain: "NLP" },
67 { pattern: /\bcomputer vision\b/i, domain: "computer vision" },
68 { pattern: /\breinforcement learning\b/i, domain: "reinforcement learning" },
69 { pattern: /\bgenerative AI\b|\bgenerative model/i, domain: "generative AI" },
70 { pattern: /\bneural network\b/i, domain: "neural networks" },
71 { pattern: /\btransformer\b/i, domain: "transformers" },
72 { pattern: /\blanguage model\b|\bLLM\b/i, domain: "language models" },
73 { pattern: /\brobotics\b/i, domain: "robotics" },
74 { pattern: /\bbioinformatics\b/i, domain: "bioinformatics" },
75 { pattern: /\bneuroscience\b/i, domain: "neuroscience" },
76 { pattern: /\bclimate\b/i, domain: "climate science" },
77 { pattern: /\beconomics?\b/i, domain: "economics" },
78 { pattern: /\bpsychology\b/i, domain: "psychology" },
79 { pattern: /\bsociology\b/i, domain: "sociology" },
80 { pattern: /\bphilosophy\b/i, domain: "philosophy" },
81 { pattern: /\bepidemiol/i, domain: "epidemiology" },
82 { pattern: /\bgenomics?\b/i, domain: "genomics" },
83 { pattern: /\bquantum\b/i, domain: "quantum computing" },
84];
85
86function extractDomainKeywords(text: string): string[] {
87 const domains: string[] = [];
88 for (const { pattern, domain } of DOMAIN_PATTERNS) {
89 if (pattern.test(text) && !domains.includes(domain)) {
90 domains.push(domain);
91 }
92 }
93 return domains;
94}
95
96export function generateHeuristicSummary(post: PaperPost, metadata: Partial<PaperSummary>): string {
97 const parts: string[] = [];
98 if (metadata.title) parts.push(metadata.title);
99 if (metadata.abstract) {
100 const sentences = metadata.abstract.split(/\.\s+/).slice(0, 2);
101 parts.push(sentences.join(". ") + ".");
102 }
103 if (metadata.authors?.length) {
104 const authorStr = metadata.authors.length <= 3
105 ? metadata.authors.join(", ")
106 : `${metadata.authors[0]} et al.`;
107 parts.push(`By ${authorStr}.`);
108 }
109 if (metadata.venue) {
110 parts.push(`Published in ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}.`);
111 }
112 return parts.join(" ");
113}
114
115export async function fetchOpenGraph(paperUrl: string): Promise<{ title?: string; description?: string }> {
116 try {
117 const res = await fetch(paperUrl, {
118 headers: { "User-Agent": "papers-appview/0.1 (mailto:researcher@pds.latha.org)" },
119 signal: AbortSignal.timeout(10000),
120 });
121 if (!res.ok) return {};
122 const html = await res.text();
123 const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]+)"/)?.[1];
124 const ogDesc = html.match(/<meta\s+property="og:description"\s+content="([^"]+)"/)?.[1];
125 const titleTag = html.match(/<title>([^<]+)<\/title>/)?.[1];
126 return {
127 title: ogTitle || titleTag || undefined,
128 description: ogDesc || undefined,
129 };
130 } catch {
131 return {};
132 }
133}
134
135export async function buildSummary(post: PaperPost): Promise<PaperSummary> {
136 const metadata = extractFromPost(post);
137
138 if (!metadata.title || !metadata.abstract) {
139 const og = await fetchOpenGraph(post.paperUrl);
140 if (!metadata.title && og.title) metadata.title = og.title;
141 if (!metadata.abstract && og.description) metadata.abstract = og.description;
142 }
143
144 const summary = generateHeuristicSummary(post, metadata);
145
146 const mystText = [
147 metadata.title ? `# ${metadata.title}` : null,
148 metadata.authors?.length ? `**Authors:** ${metadata.authors.join(", ")}` : null,
149 metadata.venue ? `**Venue:** ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}` : null,
150 "",
151 metadata.abstract || "",
152 "",
153 "## Summary",
154 "",
155 summary,
156 ].filter(Boolean).join("\n");
157
158 let summaryDoc;
159 try {
160 summaryDoc = mystTextToOxaRecord(mystText);
161 } catch {
162 // OXA conversion is best-effort
163 }
164
165 return {
166 paperUrl: post.paperUrl,
167 title: metadata.title || "Untitled",
168 authors: metadata.authors || [],
169 abstract: metadata.abstract || "",
170 summary,
171 summaryDoc,
172 domains: metadata.domains || [],
173 venue: metadata.venue || "",
174 year: metadata.year || null,
175 };
176}