nandi.uk
This repository has no description
Papers appview: AT Protocol feed watcher + summarizer
Contrail appview that watches the Paper Skygest preprint digest feed,
detects linked papers, generates OXA-backed summaries, and writes
org.latha.papers.summary records to researcher.pds.latha.org via DPoP auth.
- Feed watcher polls Paper Skygest feed via public API
- Paper URL extraction from embeds and post text (arxiv, doi, nber, etc.)
- Heuristic summarizer with OpenGraph enrichment
- OXA document generation via @nandithebull/myst-oxa
- PDS write-back via Welcome Mat v1.0 DPoP auth
- Landing page and summary detail page at papers.latha.org
- Contrail XRPC endpoints for querying summaries
- Cron trigger every 5 minutes for automated feed polling
- CLI script for manual poll-and-summarize
Hosted at papers.latha.org on Cloudflare Workers + D1.
👾 Generated with [Letta Code](https://letta.com)
Co-Authored-By: Letta Code <noreply@letta.com>
author
Nandi
co-author
Letta Code
date
(May 17, 2026, 8:58 PM -0700)
commit
003b56b7
003b56b7f29fa6543e46041cfeedef09364acc37
+1483
+11
.gitignore
+11
.gitignore
+47
.letta/.lettaignore
+47
.letta/.lettaignore
···
1
+
# .lettaignore — Letta Code file index exclusions
2
+
#
3
+
# Files and directories matching these patterns are excluded from the @ file
4
+
# search index and disk scan fallback. Comment out or remove a line to bring
5
+
# it back into search results. Add new patterns to exclude more.
6
+
#
7
+
# Syntax: one pattern per line, supports globs (e.g. *.log, src/generated/**)
8
+
# Lines starting with # are comments.
9
+
#
10
+
# --- Dependency directories ---
11
+
node_modules
12
+
bower_components
13
+
vendor
14
+
15
+
# --- Build outputs ---
16
+
dist
17
+
build
18
+
out
19
+
coverage
20
+
target
21
+
.next
22
+
.nuxt
23
+
24
+
# --- Python ---
25
+
venv
26
+
.venv
27
+
__pycache__
28
+
.tox
29
+
30
+
# --- Version control & tooling ---
31
+
.git
32
+
.cache
33
+
.letta
34
+
35
+
# --- Lock files ---
36
+
package-lock.json
37
+
yarn.lock
38
+
pnpm-lock.yaml
39
+
poetry.lock
40
+
Cargo.lock
41
+
42
+
# --- Logs ---
43
+
*.log
44
+
45
+
# --- OS artifacts ---
46
+
.DS_Store
47
+
Thumbs.db
+13
.letta/settings.local.json
+13
.letta/settings.local.json
···
1
+
{
2
+
"lastAgent": "agent-bc21f9dc-83b1-4826-b9af-999f7c6b6053",
3
+
"sessionsByServer": {
4
+
"api.letta.com": {
5
+
"agentId": "agent-bc21f9dc-83b1-4826-b9af-999f7c6b6053",
6
+
"conversationId": "conv-381a0d2a-82b2-42ff-a65e-838d58918bb1"
7
+
}
8
+
},
9
+
"lastSession": {
10
+
"agentId": "agent-bc21f9dc-83b1-4826-b9af-999f7c6b6053",
11
+
"conversationId": "conv-381a0d2a-82b2-42ff-a65e-838d58918bb1"
12
+
}
13
+
}
+12
build-frontend.sh
+12
build-frontend.sh
+79
lexicons/custom/org.latha.papers.summary.json
+79
lexicons/custom/org.latha.papers.summary.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "org.latha.papers.summary",
4
+
"defs": {
5
+
"main": {
6
+
"type": "record",
7
+
"description": "A summary of a research paper, generated from a Bluesky post linking to the paper.",
8
+
"key": "tid",
9
+
"record": {
10
+
"type": "object",
11
+
"properties": {
12
+
"sourceUri": {
13
+
"type": "string",
14
+
"format": "at-uri",
15
+
"description": "AT URI of the Bluesky post that linked to the paper"
16
+
},
17
+
"paperUrl": {
18
+
"type": "string",
19
+
"format": "uri",
20
+
"description": "Canonical URL of the paper (arxiv, DOI, journal, etc.)"
21
+
},
22
+
"title": {
23
+
"type": "string",
24
+
"description": "Paper title"
25
+
},
26
+
"authors": {
27
+
"type": "array",
28
+
"items": { "type": "string" },
29
+
"description": "Author names"
30
+
},
31
+
"abstract": {
32
+
"type": "string",
33
+
"description": "Paper abstract or first few paragraphs"
34
+
},
35
+
"summary": {
36
+
"type": "string",
37
+
"description": "Short plain-text summary of the paper's contribution"
38
+
},
39
+
"summaryDoc": {
40
+
"type": "ref",
41
+
"ref": "pub.oxa.document",
42
+
"description": "Rich OXA document with structured summary"
43
+
},
44
+
"domains": {
45
+
"type": "array",
46
+
"items": { "type": "string" },
47
+
"description": "Research domains / fields"
48
+
},
49
+
"venue": {
50
+
"type": "string",
51
+
"description": "Publication venue (journal, conference, preprint server)"
52
+
},
53
+
"year": {
54
+
"type": "integer",
55
+
"description": "Publication year"
56
+
},
57
+
"posterDid": {
58
+
"type": "string",
59
+
"description": "DID of the Bluesky user who posted about the paper"
60
+
},
61
+
"posterHandle": {
62
+
"type": "string",
63
+
"description": "Handle of the Bluesky user who posted about the paper"
64
+
},
65
+
"postText": {
66
+
"type": "string",
67
+
"description": "Original text of the Bluesky post"
68
+
},
69
+
"indexedAt": {
70
+
"type": "string",
71
+
"format": "datetime",
72
+
"description": "When this summary was created"
73
+
}
74
+
},
75
+
"required": ["sourceUri", "paperUrl", "title", "summary", "indexedAt"]
76
+
}
77
+
}
78
+
}
79
+
}
+9
migrations/0001_initial.sql
+9
migrations/0001_initial.sql
+35
package.json
+35
package.json
···
1
+
{
2
+
"name": "papers-appview",
3
+
"version": "0.0.1",
4
+
"private": true,
5
+
"type": "module",
6
+
"scripts": {
7
+
"build:frontend": "bash build-frontend.sh",
8
+
"dev": "pnpm build:frontend && wrangler dev",
9
+
"deploy": "pnpm build:frontend && CLOUDFLARE_API_TOKEN=$CLOUDFLARE_API_TOKEN wrangler deploy",
10
+
"typecheck": "tsc --noEmit",
11
+
"backfill": "pnpm contrail backfill",
12
+
"backfill:remote": "pnpm contrail backfill --remote",
13
+
"lex": "contrail-lex all",
14
+
"lex:publish": "contrail-lex publish"
15
+
},
16
+
"dependencies": {
17
+
"@atmo-dev/contrail": "^0.6.0",
18
+
"@nandithebull/myst-oxa": "^0.1.8",
19
+
"hono": "^4.7.0"
20
+
},
21
+
"devDependencies": {
22
+
"@atcute/lex-cli": "^2.8.1",
23
+
"@atmo-dev/contrail-lexicons": "^0.4.6",
24
+
"@cloudflare/workers-types": "^4.20250124.0",
25
+
"esbuild": "^0.28.0",
26
+
"typescript": "^5.7.3",
27
+
"wrangler": "^4.63.0"
28
+
},
29
+
"pnpm": {
30
+
"onlyBuiltDependencies": [
31
+
"esbuild",
32
+
"workerd"
33
+
]
34
+
}
35
+
}
public/favicon.svg
public/favicon.svg
This is a binary file and will not be displayed.
+57
scripts/poll-and-summarize.ts
+57
scripts/poll-and-summarize.ts
···
1
+
/**
2
+
* CLI script: poll the Paper Skygest feed and generate summaries.
3
+
*
4
+
* Usage:
5
+
* bun scripts/poll-and-summarize.ts # poll and write to PDS
6
+
* bun scripts/poll-and-summarize.ts --dry-run # show what would be processed
7
+
* bun scripts/poll-and-summarize.ts --limit 5 # only process 5 papers
8
+
*/
9
+
10
+
import { pollNewPapers } from "../src/feed-watcher.js";
11
+
import { buildSummary } from "../src/paper-summarizer.js";
12
+
import { writeSummaryToPds } from "../src/pds-writer.js";
13
+
14
+
const args = process.argv.slice(2);
15
+
const dryRun = args.includes("--dry-run");
16
+
const limitArg = args.find(a => a.startsWith("--limit="));
17
+
const limit = limitArg ? parseInt(limitArg.split("=")[1]) : Infinity;
18
+
19
+
async function main() {
20
+
console.log("Polling Paper Skygest feed...");
21
+
const { posts, newCursor } = await pollNewPapers();
22
+
23
+
console.log(`Found ${posts.length} paper posts`);
24
+
25
+
const toProcess = posts.slice(0, limit);
26
+
console.log(`Processing ${toProcess.length} papers${dryRun ? " (dry-run)" : ""}...`);
27
+
28
+
let written = 0;
29
+
let errors = 0;
30
+
31
+
for (const post of toProcess) {
32
+
console.log(`\n ${post.paperUrl}`);
33
+
console.log(` Title: ${post.embedTitle || "(from post text)"}`);
34
+
console.log(` Poster: @${post.posterHandle}`);
35
+
36
+
if (dryRun) continue;
37
+
38
+
try {
39
+
const summary = await buildSummary(post);
40
+
console.log(` Summary: ${summary.summary.substring(0, 100)}...`);
41
+
const result = await writeSummaryToPds(post, summary);
42
+
console.log(` Written: ${result.uri}`);
43
+
written++;
44
+
} catch (err: any) {
45
+
console.error(` Error: ${err?.message ?? err}`);
46
+
errors++;
47
+
}
48
+
}
49
+
50
+
console.log(`\nDone. Written: ${written}, Errors: ${errors}`);
51
+
if (newCursor) console.log(`Next cursor: ${newCursor}`);
52
+
}
53
+
54
+
main().catch(err => {
55
+
console.error("Fatal:", err instanceof Error ? err.message : String(err));
56
+
process.exit(1);
57
+
});
+148
src/auth.ts
+148
src/auth.ts
···
1
+
/**
2
+
* Rookery DPoP auth (Welcome Mat v1.0) for researcher.pds.latha.org
3
+
*
4
+
* Flow:
5
+
* 1. Load RSA private key from vault
6
+
* 2. Derive public JWK and thumbprint
7
+
* 3. Fetch ToS, build wm+jwt access token
8
+
* 4. Create session via com.atproto.server.createSession with DPoP
9
+
* 5. Use session JWT + DPoP for subsequent PDS writes
10
+
*/
11
+
12
+
import crypto from "node:crypto";
13
+
import { readFile } from "node:fs/promises";
14
+
15
+
const PDS_ORIGIN = "https://pds.latha.org";
16
+
const RESEARCHER_DID = "did:plc:3kkhul7jznlb6ba7rprzawnj";
17
+
const RESEARCHER_HANDLE = "researcher.pds.latha.org";
18
+
19
+
function base64url(input: Buffer | Uint8Array): string {
20
+
return Buffer.from(input).toString("base64url");
21
+
}
22
+
23
+
function pemToJwk(pem: string): { kty: string; n: string; e: string } {
24
+
const key = crypto.createPublicKey(pem);
25
+
const jwk = key.export({ format: "jwk" });
26
+
return { kty: jwk.kty as string, n: jwk.n as string, e: jwk.e as string };
27
+
}
28
+
29
+
function computeThumbprint(jwk: { kty: string; n: string; e: string }): string {
30
+
return base64url(
31
+
crypto.createHash("sha256")
32
+
.update(JSON.stringify({ e: jwk.e, kty: "RSA", n: jwk.n }))
33
+
.digest(),
34
+
);
35
+
}
36
+
37
+
function createJwt(header: object, payload: object, privateKeyPem: string): string {
38
+
const encode = (obj: object) => base64url(Buffer.from(JSON.stringify(obj)));
39
+
const signingInput = `${encode(header)}.${encode(payload)}`;
40
+
const sig = crypto.createSign("SHA256");
41
+
sig.update(signingInput);
42
+
return `${signingInput}.${base64url(sig.sign(privateKeyPem))}`;
43
+
}
44
+
45
+
function createDpopProof(
46
+
method: string,
47
+
url: string,
48
+
accessToken: string,
49
+
publicJwk: { kty: string; n: string; e: string },
50
+
privateKeyPem: string,
51
+
): string {
52
+
const ath = base64url(crypto.createHash("sha256").update(accessToken).digest());
53
+
return createJwt(
54
+
{ typ: "dpop+jwt", alg: "RS256", jwk: publicJwk },
55
+
{
56
+
jti: crypto.randomUUID(),
57
+
htm: method,
58
+
htu: url,
59
+
iat: Math.floor(Date.now() / 1000),
60
+
ath,
61
+
},
62
+
privateKeyPem,
63
+
);
64
+
}
65
+
66
+
export interface PdsSession {
67
+
did: string;
68
+
handle: string;
69
+
accessJwt: string;
70
+
wmJwt: string;
71
+
privateKeyPem: string;
72
+
publicJwk: { kty: string; n: string; e: string };
73
+
}
74
+
75
+
export async function createPdsSession(privateKeyPem: string): Promise<PdsSession> {
76
+
const publicJwk = pemToJwk(privateKeyPem);
77
+
const thumbprint = computeThumbprint(publicJwk);
78
+
79
+
// Fetch ToS and build wm+jwt access token
80
+
const tosRes = await fetch(`${PDS_ORIGIN}/tos`);
81
+
if (!tosRes.ok) throw new Error(`Failed to fetch ToS: ${tosRes.status}`);
82
+
const tosText = await tosRes.text();
83
+
const tosHash = base64url(crypto.createHash("sha256").update(tosText).digest());
84
+
85
+
const wmJwt = createJwt(
86
+
{ typ: "wm+jwt", alg: "RS256" },
87
+
{
88
+
tos_hash: tosHash,
89
+
aud: PDS_ORIGIN,
90
+
cnf: { jkt: thumbprint },
91
+
iat: Math.floor(Date.now() / 1000),
92
+
},
93
+
privateKeyPem,
94
+
);
95
+
96
+
// Create session
97
+
const sessionUrl = `${PDS_ORIGIN}/xrpc/com.atproto.server.createSession`;
98
+
const dpop = createDpopProof("POST", sessionUrl, wmJwt, publicJwk, privateKeyPem);
99
+
100
+
const res = await fetch(sessionUrl, {
101
+
method: "POST",
102
+
headers: {
103
+
"Content-Type": "application/json",
104
+
Authorization: `DPoP ${wmJwt}`,
105
+
DPoP: dpop,
106
+
},
107
+
body: JSON.stringify({}),
108
+
});
109
+
110
+
if (!res.ok) {
111
+
const body = await res.text();
112
+
throw new Error(`createSession failed (${res.status}): ${body}`);
113
+
}
114
+
115
+
const data = await res.json() as { did: string; handle: string; accessJwt: string };
116
+
return {
117
+
did: data.did || RESEARCHER_DID,
118
+
handle: data.handle || RESEARCHER_HANDLE,
119
+
accessJwt: data.accessJwt || wmJwt,
120
+
wmJwt,
121
+
privateKeyPem,
122
+
publicJwk,
123
+
};
124
+
}
125
+
126
+
export async function dpopFetch(
127
+
url: string,
128
+
init: RequestInit,
129
+
session: PdsSession,
130
+
): Promise<Response> {
131
+
const method = init.method || "GET";
132
+
const accessToken = session.wmJwt;
133
+
const dpop = createDpopProof(method, url, accessToken, session.publicJwk, session.privateKeyPem);
134
+
135
+
const headers = new Headers(init.headers);
136
+
headers.set("Authorization", `DPoP ${accessToken}`);
137
+
headers.set("DPoP", dpop);
138
+
if (!headers.has("Content-Type")) {
139
+
headers.set("Content-Type", "application/json");
140
+
}
141
+
142
+
return fetch(url, { ...init, headers });
143
+
}
144
+
145
+
export async function loadPrivateKey(): Promise<string> {
146
+
const keyPath = process.env.RESEARCHER_KEY_PATH || "/home/nandi/vault/.credentials/researcher-pds-latha-org.pem";
147
+
return readFile(keyPath, "utf-8");
148
+
}
+20
src/contrail.config.ts
+20
src/contrail.config.ts
···
1
+
import type { ContrailConfig } from "@atmo-dev/contrail";
2
+
3
+
export const config: ContrailConfig = {
4
+
namespace: "org.latha.papers",
5
+
collections: {
6
+
summary: {
7
+
collection: "org.latha.papers.summary",
8
+
queryable: {
9
+
paperUrl: {},
10
+
venue: {},
11
+
year: { type: "range" },
12
+
posterDid: {},
13
+
},
14
+
searchable: ["title", "summary", "abstract"],
15
+
methods: ["listRecords", "getRecord"],
16
+
},
17
+
},
18
+
profiles: ["app.bsky.actor.profile"],
19
+
notify: true,
20
+
};
+148
src/feed-watcher.ts
+148
src/feed-watcher.ts
···
1
+
/**
2
+
* Feed watcher — polls the Paper Skygest feed and extracts paper URLs.
3
+
*
4
+
* The feed is a Bluesky custom feed generator at:
5
+
* at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest
6
+
*
7
+
* We poll it via the public API and extract paper URLs from:
8
+
* - embed.external.uri (most reliable)
9
+
* - Post text (arxiv.org, doi.org, nber.org, openreview.net, etc.)
10
+
*/
11
+
12
+
const FEED_URI = "at://did:plc:uaadt6f5bbda6cycbmatcm3z/app.bsky.feed.generator/preprintdigest";
13
+
const BSKY_PUBLIC_API = "https://public.api.bsky.app";
14
+
15
+
const PAPER_URL_PATTERNS = [
16
+
/https?:\/\/arxiv\.org\/abs\/\d+\.\d+/,
17
+
/https?:\/\/arxiv\.org\/abs\/\d{4}\.\d{4,5}/,
18
+
/https?:\/\/doi\.org\/10\.\d{4,}\/[^\s]+/,
19
+
/https?:\/\/www\.nber\.org\/papers\/w\d+/,
20
+
/https?:\/\/openreview\.net\/forum\?id=[^\s]+/,
21
+
/https?:\/\/papers\.ssrn\.com\/sol3\/papers\.cfm\?abstract_id=\d+/,
22
+
/https?:\/\/proceedings\.mlr\.press\/v\d+\//,
23
+
/https?:\/\/www\.science\.org\/doi\/[^\s]+/,
24
+
/https?:\/\/www\.nature\.com\/articles\/[^\s]+/,
25
+
/https?:\/\/journals\.plos\.org\/[^\s]+\/article\?id=[^\s]+/,
26
+
/https?:\/\/www\.biorxiv\.org\/content\/[^\s]+/,
27
+
/https?:\/\/www\.medrxiv\.org\/content\/[^\s]+/,
28
+
/https?:\/\/aclanthology\.org\/[^\s]+/,
29
+
/https?:\/\/proceedings\.neurips\.cc\/paper_files\/[^\s]+/,
30
+
/https?:\/\/openaccess\.thecvf\.com\/[^\s]+/,
31
+
];
32
+
33
+
export interface PaperPost {
34
+
postUri: string;
35
+
postCid: string;
36
+
paperUrl: string;
37
+
posterDid: string;
38
+
posterHandle: string;
39
+
postText: string;
40
+
embedTitle?: string;
41
+
embedDescription?: string;
42
+
indexedAt: string;
43
+
}
44
+
45
+
function extractPaperUrls(text: string): string[] {
46
+
const urls: string[] = [];
47
+
for (const pattern of PAPER_URL_PATTERNS) {
48
+
const match = text.match(pattern);
49
+
if (match) urls.push(match[0]);
50
+
}
51
+
return urls;
52
+
}
53
+
54
+
function extractFromEmbed(embed: any): string | null {
55
+
if (!embed) return null;
56
+
if (embed.$type === "app.bsky.embed.external#view" && embed.external?.uri) {
57
+
const uri = embed.external.uri as string;
58
+
for (const pattern of PAPER_URL_PATTERNS) {
59
+
if (pattern.test(uri)) return uri;
60
+
}
61
+
if (/https?:\/\/(arxiv|doi|nber|openreview|ssrn|mlr|science|nature|plos|biorxiv|medrxiv|aclanthology|neurips|thecvf)\./.test(uri)) {
62
+
return uri;
63
+
}
64
+
}
65
+
return null;
66
+
}
67
+
68
+
export async function pollFeed(cursor?: string, limit = 50): Promise<{ posts: PaperPost[]; cursor?: string }> {
69
+
const params = new URLSearchParams({
70
+
feed: FEED_URI,
71
+
limit: String(limit),
72
+
});
73
+
if (cursor) params.set("cursor", cursor);
74
+
75
+
const res = await fetch(`${BSKY_PUBLIC_API}/xrpc/app.bsky.feed.getFeed?${params}`);
76
+
if (!res.ok) {
77
+
const body = await res.text();
78
+
throw new Error(`Feed poll failed (${res.status}): ${body}`);
79
+
}
80
+
81
+
const data = await res.json() as {
82
+
cursor?: string;
83
+
feed: Array<{
84
+
post: {
85
+
uri: string;
86
+
cid: string;
87
+
record: { text?: string };
88
+
author: { did: string; handle: string };
89
+
embed?: any;
90
+
indexedAt: string;
91
+
};
92
+
}>;
93
+
};
94
+
95
+
const posts: PaperPost[] = [];
96
+
const seenUris = new Set<string>();
97
+
98
+
for (const item of data.feed) {
99
+
const { uri, cid, author, embed, indexedAt } = item.post;
100
+
const text = item.post.record?.text || "";
101
+
102
+
let paperUrl = extractFromEmbed(embed);
103
+
if (!paperUrl) {
104
+
const textUrls = extractPaperUrls(text);
105
+
if (textUrls.length > 0) paperUrl = textUrls[0];
106
+
}
107
+
108
+
if (!paperUrl) continue;
109
+
if (seenUris.has(paperUrl)) continue;
110
+
seenUris.add(paperUrl);
111
+
112
+
let embedTitle: string | undefined;
113
+
let embedDescription: string | undefined;
114
+
if (embed?.$type === "app.bsky.embed.external#view" && embed.external) {
115
+
embedTitle = embed.external.title;
116
+
embedDescription = embed.external.description;
117
+
}
118
+
119
+
posts.push({
120
+
postUri: uri,
121
+
postCid: cid,
122
+
paperUrl,
123
+
posterDid: author.did,
124
+
posterHandle: author.handle,
125
+
postText: text,
126
+
embedTitle,
127
+
embedDescription,
128
+
indexedAt,
129
+
});
130
+
}
131
+
132
+
return { posts, cursor: data.cursor };
133
+
}
134
+
135
+
export async function pollNewPapers(lastCursor?: string): Promise<{ posts: PaperPost[]; newCursor?: string }> {
136
+
const allPosts: PaperPost[] = [];
137
+
let cursor = lastCursor;
138
+
139
+
for (let i = 0; i < 3; i++) {
140
+
const result = await pollFeed(cursor, 50);
141
+
allPosts.push(...result.posts);
142
+
if (!result.cursor || result.cursor === cursor) break;
143
+
cursor = result.cursor;
144
+
if (result.posts.length < 50) break;
145
+
}
146
+
147
+
return { posts: allPosts, newCursor: cursor };
148
+
}
+2
src/frontend/app.ts
+2
src/frontend/app.ts
+138
src/landing-page.ts
+138
src/landing-page.ts
···
1
+
export const LANDING_PAGE = `<!DOCTYPE html>
2
+
<html lang="en">
3
+
<head>
4
+
<meta charset="utf-8">
5
+
<meta name="viewport" content="width=device-width, initial-scale=1">
6
+
<title>Papers — Research Paper Summaries</title>
7
+
<style>
8
+
:root {
9
+
--bg: #0f0f0f;
10
+
--surface: #1a1a1a;
11
+
--text: #e0e0e0;
12
+
--dim: #888;
13
+
--accent: #6d9eeb;
14
+
--border: #333;
15
+
--green: #4caf50;
16
+
--mauve: #ce93d8;
17
+
}
18
+
* { box-sizing: border-box; margin: 0; padding: 0; }
19
+
body {
20
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
21
+
background: var(--bg);
22
+
color: var(--text);
23
+
line-height: 1.6;
24
+
padding: 2rem;
25
+
max-width: 960px;
26
+
margin: 0 auto;
27
+
}
28
+
header { margin-bottom: 2rem; }
29
+
header h1 { font-size: 1.75rem; font-weight: 700; }
30
+
header p { color: var(--dim); margin-top: 0.25rem; }
31
+
.actions { margin-bottom: 1.5rem; display: flex; gap: 0.75rem; }
32
+
.actions button {
33
+
padding: 0.5rem 1rem;
34
+
background: var(--accent);
35
+
color: var(--bg);
36
+
border: none;
37
+
border-radius: 6px;
38
+
font-weight: 600;
39
+
cursor: pointer;
40
+
}
41
+
.actions button:hover { opacity: 0.9; }
42
+
.actions button.secondary {
43
+
background: var(--surface);
44
+
color: var(--text);
45
+
border: 1px solid var(--border);
46
+
}
47
+
.paper-list { display: flex; flex-direction: column; gap: 0.75rem; }
48
+
.paper-card {
49
+
background: var(--surface);
50
+
border: 1px solid var(--border);
51
+
border-radius: 8px;
52
+
padding: 1rem 1.25rem;
53
+
cursor: pointer;
54
+
transition: border-color 0.15s;
55
+
text-decoration: none;
56
+
color: inherit;
57
+
display: block;
58
+
}
59
+
.paper-card:hover { border-color: var(--accent); }
60
+
.paper-card .title { font-weight: 600; margin-bottom: 0.25rem; }
61
+
.paper-card .meta { font-size: 0.8125rem; color: var(--dim); }
62
+
.paper-card .summary { font-size: 0.875rem; margin-top: 0.5rem; color: var(--text); }
63
+
.paper-card .domains { margin-top: 0.5rem; display: flex; gap: 0.375rem; flex-wrap: wrap; }
64
+
.paper-card .domain {
65
+
font-size: 0.6875rem;
66
+
padding: 0.125rem 0.375rem;
67
+
border-radius: 4px;
68
+
background: var(--border);
69
+
color: var(--dim);
70
+
}
71
+
.paper-card .venue-badge {
72
+
font-size: 0.6875rem;
73
+
padding: 0.125rem 0.375rem;
74
+
border-radius: 4px;
75
+
background: var(--mauve);
76
+
color: var(--bg);
77
+
font-weight: 600;
78
+
}
79
+
.empty { color: var(--dim); text-align: center; padding: 3rem; }
80
+
#poll-status { font-size: 0.8125rem; color: var(--dim); margin-left: 0.75rem; }
81
+
</style>
82
+
</head>
83
+
<body>
84
+
<header>
85
+
<h1>Papers</h1>
86
+
<p>Research paper summaries from the Paper Skygest feed</p>
87
+
</header>
88
+
<div class="actions">
89
+
<button onclick="pollFeed()">Poll Feed</button>
90
+
<span id="poll-status"></span>
91
+
</div>
92
+
<div id="papers" class="paper-list"></div>
93
+
<script>
94
+
const summaries = window.__SUMMARIES__ || [];
95
+
const container = document.getElementById('papers');
96
+
const pollStatus = document.getElementById('poll-status');
97
+
98
+
function render(summaries) {
99
+
if (summaries.length === 0) {
100
+
container.innerHTML = '<div class="empty">No papers indexed yet. Click "Poll Feed" to start.</div>';
101
+
return;
102
+
}
103
+
container.innerHTML = summaries.map(s => {
104
+
const domains = (s.domains || []).map(d => '<span class="domain">' + d + '</span>').join('');
105
+
const venue = s.venue ? '<span class="venue-badge">' + s.venue + '</span> ' : '';
106
+
const year = s.year ? ' (' + s.year + ')' : '';
107
+
const authors = (s.authors || []).length > 0
108
+
? s.authors.length <= 3 ? s.authors.join(', ') : s.authors[0] + ' et al.'
109
+
: '';
110
+
return '<a class="paper-card" href="/paper/' + s.rkey + '">'
111
+
+ '<div class="title">' + venue + escHtml(s.title) + year + '</div>'
112
+
+ '<div class="meta">' + (authors ? escHtml(authors) + ' · ' : '') + escHtml(s.posterHandle) + ' · ' + s.paperUrl + '</div>'
113
+
+ '<div class="summary">' + escHtml(s.summary).substring(0, 200) + '</div>'
114
+
+ (domains ? '<div class="domains">' + domains + '</div>' : '')
115
+
+ '</a>';
116
+
}).join('');
117
+
}
118
+
119
+
function escHtml(s) {
120
+
return (s || '').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
121
+
}
122
+
123
+
async function pollFeed() {
124
+
pollStatus.textContent = 'Polling...';
125
+
try {
126
+
const res = await fetch('/api/poll', { method: 'POST' });
127
+
const data = await res.json();
128
+
pollStatus.textContent = 'Polled ' + data.polled + ', new: ' + data.new + ', written: ' + data.written + (data.errors ? ', errors: ' + data.errors : '');
129
+
if (data.written > 0) setTimeout(() => location.reload(), 1500);
130
+
} catch (err) {
131
+
pollStatus.textContent = 'Error: ' + err.message;
132
+
}
133
+
}
134
+
135
+
render(summaries);
136
+
</script>
137
+
</body>
138
+
</html>`;
+176
src/paper-summarizer.ts
+176
src/paper-summarizer.ts
···
1
+
/**
2
+
* Paper summarizer — fetches paper content and generates a structured summary.
3
+
*
4
+
* Two modes:
5
+
* 1. Worker mode: heuristic extraction from embed + OpenGraph enrichment
6
+
* 2. CLI mode: uses letta-code-sdk for rich OXA-backed summaries
7
+
*/
8
+
9
+
import { mystTextToOxaRecord } from "@nandithebull/myst-oxa";
10
+
import type { PaperPost } from "./feed-watcher.js";
11
+
12
+
export interface PaperSummary {
13
+
paperUrl: string;
14
+
title: string;
15
+
authors: string[];
16
+
abstract: string;
17
+
summary: string;
18
+
summaryDoc?: any;
19
+
domains: string[];
20
+
venue: string;
21
+
year: number | null;
22
+
}
23
+
24
+
export function extractFromPost(post: PaperPost): Partial<PaperSummary> {
25
+
const result: Partial<PaperSummary> = { paperUrl: post.paperUrl };
26
+
27
+
if (post.embedTitle) result.title = post.embedTitle;
28
+
if (post.embedDescription) result.abstract = post.embedDescription;
29
+
30
+
const authorMatch = post.postText.match(/from\s+(.+?)(?:\s+https?:\/\/|$)/);
31
+
if (authorMatch) {
32
+
result.authors = authorMatch[1]
33
+
.split(/,\s*|\s+and\s+/)
34
+
.map(a => a.trim())
35
+
.filter(a => a.length > 0);
36
+
}
37
+
38
+
if (post.paperUrl.includes("arxiv.org")) result.venue = "arXiv";
39
+
else if (post.paperUrl.includes("nber.org")) result.venue = "NBER";
40
+
else if (post.paperUrl.includes("openreview.net")) result.venue = "OpenReview";
41
+
else if (post.paperUrl.includes("ssrn.com")) result.venue = "SSRN";
42
+
else if (post.paperUrl.includes("biorxiv.org")) result.venue = "bioRxiv";
43
+
else if (post.paperUrl.includes("medrxiv.org")) result.venue = "medRxiv";
44
+
else if (post.paperUrl.includes("neurips.cc")) result.venue = "NeurIPS";
45
+
else if (post.paperUrl.includes("mlr.press")) result.venue = "JMLR/PMLR";
46
+
else if (post.paperUrl.includes("aclanthology.org")) result.venue = "ACL";
47
+
else if (post.paperUrl.includes("thecvf.com")) result.venue = "CVPR/ICCV/ECCV";
48
+
49
+
const yearMatch = post.paperUrl.match(/(\d{4})/);
50
+
if (yearMatch) {
51
+
const y = parseInt(yearMatch[1]);
52
+
if (y >= 1990 && y <= 2030) result.year = y;
53
+
}
54
+
55
+
if (post.embedDescription) {
56
+
const domainKeywords = extractDomainKeywords(post.embedDescription);
57
+
if (domainKeywords.length > 0) result.domains = domainKeywords;
58
+
}
59
+
60
+
return result;
61
+
}
62
+
63
+
const DOMAIN_PATTERNS: Array<{ pattern: RegExp; domain: string }> = [
64
+
{ pattern: /\bmachine learning\b/i, domain: "machine learning" },
65
+
{ pattern: /\bdeep learning\b/i, domain: "deep learning" },
66
+
{ pattern: /\bnatural language processing\b|\bNLP\b/i, domain: "NLP" },
67
+
{ pattern: /\bcomputer vision\b/i, domain: "computer vision" },
68
+
{ pattern: /\breinforcement learning\b/i, domain: "reinforcement learning" },
69
+
{ pattern: /\bgenerative AI\b|\bgenerative model/i, domain: "generative AI" },
70
+
{ pattern: /\bneural network\b/i, domain: "neural networks" },
71
+
{ pattern: /\btransformer\b/i, domain: "transformers" },
72
+
{ pattern: /\blanguage model\b|\bLLM\b/i, domain: "language models" },
73
+
{ pattern: /\brobotics\b/i, domain: "robotics" },
74
+
{ pattern: /\bbioinformatics\b/i, domain: "bioinformatics" },
75
+
{ pattern: /\bneuroscience\b/i, domain: "neuroscience" },
76
+
{ pattern: /\bclimate\b/i, domain: "climate science" },
77
+
{ pattern: /\beconomics?\b/i, domain: "economics" },
78
+
{ pattern: /\bpsychology\b/i, domain: "psychology" },
79
+
{ pattern: /\bsociology\b/i, domain: "sociology" },
80
+
{ pattern: /\bphilosophy\b/i, domain: "philosophy" },
81
+
{ pattern: /\bepidemiol/i, domain: "epidemiology" },
82
+
{ pattern: /\bgenomics?\b/i, domain: "genomics" },
83
+
{ pattern: /\bquantum\b/i, domain: "quantum computing" },
84
+
];
85
+
86
+
function extractDomainKeywords(text: string): string[] {
87
+
const domains: string[] = [];
88
+
for (const { pattern, domain } of DOMAIN_PATTERNS) {
89
+
if (pattern.test(text) && !domains.includes(domain)) {
90
+
domains.push(domain);
91
+
}
92
+
}
93
+
return domains;
94
+
}
95
+
96
+
export function generateHeuristicSummary(post: PaperPost, metadata: Partial<PaperSummary>): string {
97
+
const parts: string[] = [];
98
+
if (metadata.title) parts.push(metadata.title);
99
+
if (metadata.abstract) {
100
+
const sentences = metadata.abstract.split(/\.\s+/).slice(0, 2);
101
+
parts.push(sentences.join(". ") + ".");
102
+
}
103
+
if (metadata.authors?.length) {
104
+
const authorStr = metadata.authors.length <= 3
105
+
? metadata.authors.join(", ")
106
+
: `${metadata.authors[0]} et al.`;
107
+
parts.push(`By ${authorStr}.`);
108
+
}
109
+
if (metadata.venue) {
110
+
parts.push(`Published in ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}.`);
111
+
}
112
+
return parts.join(" ");
113
+
}
114
+
115
+
export async function fetchOpenGraph(paperUrl: string): Promise<{ title?: string; description?: string }> {
116
+
try {
117
+
const res = await fetch(paperUrl, {
118
+
headers: { "User-Agent": "papers-appview/0.1 (mailto:researcher@pds.latha.org)" },
119
+
signal: AbortSignal.timeout(10000),
120
+
});
121
+
if (!res.ok) return {};
122
+
const html = await res.text();
123
+
const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]+)"/)?.[1];
124
+
const ogDesc = html.match(/<meta\s+property="og:description"\s+content="([^"]+)"/)?.[1];
125
+
const titleTag = html.match(/<title>([^<]+)<\/title>/)?.[1];
126
+
return {
127
+
title: ogTitle || titleTag || undefined,
128
+
description: ogDesc || undefined,
129
+
};
130
+
} catch {
131
+
return {};
132
+
}
133
+
}
134
+
135
+
export async function buildSummary(post: PaperPost): Promise<PaperSummary> {
136
+
const metadata = extractFromPost(post);
137
+
138
+
if (!metadata.title || !metadata.abstract) {
139
+
const og = await fetchOpenGraph(post.paperUrl);
140
+
if (!metadata.title && og.title) metadata.title = og.title;
141
+
if (!metadata.abstract && og.description) metadata.abstract = og.description;
142
+
}
143
+
144
+
const summary = generateHeuristicSummary(post, metadata);
145
+
146
+
const mystText = [
147
+
metadata.title ? `# ${metadata.title}` : null,
148
+
metadata.authors?.length ? `**Authors:** ${metadata.authors.join(", ")}` : null,
149
+
metadata.venue ? `**Venue:** ${metadata.venue}${metadata.year ? ` (${metadata.year})` : ""}` : null,
150
+
"",
151
+
metadata.abstract || "",
152
+
"",
153
+
"## Summary",
154
+
"",
155
+
summary,
156
+
].filter(Boolean).join("\n");
157
+
158
+
let summaryDoc;
159
+
try {
160
+
summaryDoc = mystTextToOxaRecord(mystText);
161
+
} catch {
162
+
// OXA conversion is best-effort
163
+
}
164
+
165
+
return {
166
+
paperUrl: post.paperUrl,
167
+
title: metadata.title || "Untitled",
168
+
authors: metadata.authors || [],
169
+
abstract: metadata.abstract || "",
170
+
summary,
171
+
summaryDoc,
172
+
domains: metadata.domains || [],
173
+
venue: metadata.venue || "",
174
+
year: metadata.year || null,
175
+
};
176
+
}
+68
src/pds-writer.ts
+68
src/pds-writer.ts
···
1
+
/**
2
+
* PDS writer — creates org.latha.papers.summary records on the researcher's PDS.
3
+
*/
4
+
5
+
import { createPdsSession, dpopFetch, loadPrivateKey, type PdsSession } from "./auth.js";
6
+
import type { PaperSummary } from "./paper-summarizer.js";
7
+
import type { PaperPost } from "./feed-watcher.js";
8
+
9
+
const PDS_ORIGIN = "https://pds.latha.org";
10
+
const COLLECTION = "org.latha.papers.summary";
11
+
12
+
function generateRkey(): string {
13
+
const chars = "234567abcdefghijklmnopqrstuvwxyz";
14
+
const now = Date.now() * 1000;
15
+
let tid = "";
16
+
let n = now;
17
+
while (n > 0) {
18
+
tid = chars[n % 32] + tid;
19
+
n = Math.floor(n / 32);
20
+
}
21
+
return tid.padStart(13, "2");
22
+
}
23
+
24
+
export async function writeSummaryToPds(
25
+
post: PaperPost,
26
+
summary: PaperSummary,
27
+
): Promise<{ uri: string; cid: string }> {
28
+
const privateKeyPem = await loadPrivateKey();
29
+
const session = await createPdsSession(privateKeyPem);
30
+
31
+
const rkey = generateRkey();
32
+
const record: Record<string, any> = {
33
+
$type: COLLECTION,
34
+
sourceUri: post.postUri,
35
+
paperUrl: summary.paperUrl,
36
+
title: summary.title,
37
+
summary: summary.summary,
38
+
indexedAt: new Date().toISOString(),
39
+
};
40
+
41
+
if (summary.authors.length > 0) record.authors = summary.authors;
42
+
if (summary.abstract) record.abstract = summary.abstract;
43
+
if (summary.summaryDoc) record.summaryDoc = summary.summaryDoc;
44
+
if (summary.domains.length > 0) record.domains = summary.domains;
45
+
if (summary.venue) record.venue = summary.venue;
46
+
if (summary.year) record.year = summary.year;
47
+
record.posterDid = post.posterDid;
48
+
record.posterHandle = post.posterHandle;
49
+
record.postText = post.postText;
50
+
51
+
const url = `${PDS_ORIGIN}/xrpc/com.atproto.repo.createRecord`;
52
+
const res = await dpopFetch(url, {
53
+
method: "POST",
54
+
body: JSON.stringify({
55
+
repo: session.did,
56
+
collection: COLLECTION,
57
+
record,
58
+
}),
59
+
}, session);
60
+
61
+
if (!res.ok) {
62
+
const body = await res.text();
63
+
throw new Error(`putRecord failed (${res.status}): ${body}`);
64
+
}
65
+
66
+
const data = await res.json() as { uri: string; cid: string };
67
+
return { uri: data.uri, cid: data.cid };
68
+
}
+121
src/summary-page.ts
+121
src/summary-page.ts
···
1
+
export const SUMMARY_PAGE = `<!DOCTYPE html>
2
+
<html lang="en">
3
+
<head>
4
+
<meta charset="utf-8">
5
+
<meta name="viewport" content="width=device-width, initial-scale=1">
6
+
<title>Paper Summary</title>
7
+
<style>
8
+
:root {
9
+
--bg: #0f0f0f;
10
+
--surface: #1a1a1a;
11
+
--text: #e0e0e0;
12
+
--dim: #888;
13
+
--accent: #6d9eeb;
14
+
--border: #333;
15
+
--mauve: #ce93d8;
16
+
}
17
+
* { box-sizing: border-box; margin: 0; padding: 0; }
18
+
body {
19
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
20
+
background: var(--bg);
21
+
color: var(--text);
22
+
line-height: 1.6;
23
+
padding: 2rem;
24
+
max-width: 760px;
25
+
margin: 0 auto;
26
+
}
27
+
a { color: var(--accent); text-decoration: none; }
28
+
a:hover { text-decoration: underline; }
29
+
header { margin-bottom: 2rem; }
30
+
header .back { font-size: 0.8125rem; color: var(--dim); margin-bottom: 0.5rem; }
31
+
h1 { font-size: 1.5rem; font-weight: 700; margin-bottom: 0.5rem; }
32
+
.meta { color: var(--dim); font-size: 0.875rem; margin-bottom: 1rem; }
33
+
.meta .venue-badge {
34
+
font-size: 0.6875rem;
35
+
padding: 0.125rem 0.375rem;
36
+
border-radius: 4px;
37
+
background: var(--mauve);
38
+
color: var(--bg);
39
+
font-weight: 600;
40
+
}
41
+
.domains { margin-bottom: 1rem; display: flex; gap: 0.375rem; flex-wrap: wrap; }
42
+
.domain {
43
+
font-size: 0.6875rem;
44
+
padding: 0.125rem 0.375rem;
45
+
border-radius: 4px;
46
+
background: var(--border);
47
+
color: var(--dim);
48
+
}
49
+
.abstract {
50
+
background: var(--surface);
51
+
border: 1px solid var(--border);
52
+
border-radius: 8px;
53
+
padding: 1rem 1.25rem;
54
+
margin-bottom: 1.5rem;
55
+
font-size: 0.9375rem;
56
+
}
57
+
.abstract h2 { font-size: 1rem; margin-bottom: 0.5rem; }
58
+
.summary-section h2 { font-size: 1.125rem; margin-bottom: 0.5rem; }
59
+
.summary-section p { margin-bottom: 0.75rem; }
60
+
.post-link { margin-top: 2rem; padding-top: 1rem; border-top: 1px solid var(--border); }
61
+
.post-link .label { font-size: 0.8125rem; color: var(--dim); }
62
+
.at-uri { font-family: monospace; font-size: 0.75rem; color: var(--dim); word-break: break-all; }
63
+
</style>
64
+
</head>
65
+
<body>
66
+
<header>
67
+
<div class="back"><a href="/">← All Papers</a></div>
68
+
<h1 id="title"></h1>
69
+
<div class="meta" id="meta"></div>
70
+
<div class="domains" id="domains"></div>
71
+
</header>
72
+
<div class="abstract" id="abstract-section" style="display:none">
73
+
<h2>Abstract</h2>
74
+
<p id="abstract"></p>
75
+
</div>
76
+
<div class="summary-section">
77
+
<h2>Summary</h2>
78
+
<p id="summary"></p>
79
+
</div>
80
+
<div class="post-link" id="post-link-section" style="display:none">
81
+
<div class="label">Source Post</div>
82
+
<div class="at-uri" id="source-uri"></div>
83
+
</div>
84
+
<script>
85
+
const s = window.__SUMMARY__;
86
+
if (s) {
87
+
document.getElementById('title').textContent = s.title || 'Untitled';
88
+
document.title = s.title || 'Paper Summary';
89
+
90
+
const metaParts = [];
91
+
if (s.venue) metaParts.push('<span class="venue-badge">' + esc(s.venue) + '</span>');
92
+
if (s.year) metaParts.push(s.year);
93
+
if (s.authors && s.authors.length) {
94
+
const a = s.authors.length <= 3 ? s.authors.join(', ') : s.authors[0] + ' et al.';
95
+
metaParts.push(esc(a));
96
+
}
97
+
if (s.posterHandle) metaParts.push('via @' + esc(s.posterHandle));
98
+
if (s.paperUrl) metaParts.push('<a href="' + esc(s.paperUrl) + '">' + esc(s.paperUrl) + '</a>');
99
+
document.getElementById('meta').innerHTML = metaParts.join(' · ');
100
+
101
+
if (s.domains && s.domains.length) {
102
+
document.getElementById('domains').innerHTML = s.domains.map(d => '<span class="domain">' + esc(d) + '</span>').join('');
103
+
}
104
+
105
+
if (s.abstract) {
106
+
document.getElementById('abstract').textContent = s.abstract;
107
+
document.getElementById('abstract-section').style.display = '';
108
+
}
109
+
110
+
document.getElementById('summary').textContent = s.summary || '';
111
+
112
+
if (s.sourceUri) {
113
+
document.getElementById('source-uri').textContent = s.sourceUri;
114
+
document.getElementById('post-link-section').style.display = '';
115
+
}
116
+
}
117
+
118
+
function esc(s) { return (s||'').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"'); }
119
+
</script>
120
+
</body>
121
+
</html>`;
+357
src/worker.ts
+357
src/worker.ts
···
1
+
/**
2
+
* Papers Appview — watches the Paper Skygest feed, detects linked papers,
3
+
* generates summaries, and writes org.latha.papers.summary records to PDS.
4
+
*
5
+
* Hosted at papers.latha.org
6
+
*/
7
+
8
+
import { Hono } from "hono";
9
+
import { cors } from "hono/cors";
10
+
import { createWorker } from "@atmo-dev/contrail/worker";
11
+
import { Contrail } from "@atmo-dev/contrail";
12
+
import { lexicons } from "../lexicons/generated/index.js";
13
+
import { config } from "./contrail.config.js";
14
+
import { pollNewPapers } from "./feed-watcher.js";
15
+
import { buildSummary } from "./paper-summarizer.js";
16
+
import { writeSummaryToPds } from "./pds-writer.js";
17
+
import { LANDING_PAGE } from "./landing-page.js";
18
+
import { SUMMARY_PAGE } from "./summary-page.js";
19
+
20
+
interface Env {
21
+
DB: D1Database;
22
+
FEED_CURSOR?: string;
23
+
}
24
+
25
+
const contrailWorker = createWorker(config, { lexicons });
26
+
const contrail = new Contrail(config);
27
+
let contrailReady = false;
28
+
29
+
async function ensureContrailReady(db: D1Database): Promise<void> {
30
+
if (contrailReady) return;
31
+
await contrail.init(db);
32
+
contrailReady = true;
33
+
}
34
+
35
+
let app: Hono<{ Bindings: Env }> | null = null;
36
+
37
+
function buildApp(env: Env): Hono<{ Bindings: Env }> {
38
+
const app = new Hono<{ Bindings: Env }>();
39
+
const db = env.DB;
40
+
41
+
app.use("*", cors());
42
+
43
+
// Landing page — browse all indexed summaries
44
+
app.get("/", async (c) => {
45
+
await ensureContrailReady(db);
46
+
47
+
let summaries: any[] = [];
48
+
try {
49
+
const rows = await db
50
+
.prepare(
51
+
"SELECT uri, record, did, rkey, time_us FROM records_summary ORDER BY time_us DESC LIMIT 100"
52
+
)
53
+
.all();
54
+
summaries = (rows.results || []).map((r: any) => {
55
+
try {
56
+
const value = JSON.parse(r.record);
57
+
return {
58
+
uri: r.uri,
59
+
rkey: r.rkey,
60
+
title: value?.title || "Untitled",
61
+
paperUrl: value?.paperUrl || "",
62
+
venue: value?.venue || "",
63
+
year: value?.year || null,
64
+
authors: value?.authors || [],
65
+
summary: value?.summary || "",
66
+
domains: value?.domains || [],
67
+
posterHandle: value?.posterHandle || "",
68
+
indexedAt: value?.indexedAt || "",
69
+
};
70
+
} catch {
71
+
return null;
72
+
}
73
+
}).filter(Boolean);
74
+
} catch {
75
+
// empty list is fine
76
+
}
77
+
78
+
const page = LANDING_PAGE.replace(
79
+
"</head>",
80
+
`<script>window.__SUMMARIES__=${JSON.stringify(summaries)};</script></head>`
81
+
);
82
+
return c.html(page);
83
+
});
84
+
85
+
// Summary detail page
86
+
app.get("/paper/:rkey", async (c) => {
87
+
const rkey = c.req.param("rkey");
88
+
await ensureContrailReady(db);
89
+
90
+
let summary: any = null;
91
+
try {
92
+
const row = await db
93
+
.prepare("SELECT uri, record, did, rkey FROM records_summary WHERE rkey = ? LIMIT 1")
94
+
.bind(rkey)
95
+
.first<{ uri: string; record: string; did: string; rkey: string }>();
96
+
if (row) {
97
+
const value = JSON.parse(row.record);
98
+
summary = {
99
+
uri: row.uri,
100
+
rkey: row.rkey,
101
+
...value,
102
+
};
103
+
}
104
+
} catch {
105
+
// not found
106
+
}
107
+
108
+
if (!summary) {
109
+
return c.html("<h1>Not Found</h1>", 404);
110
+
}
111
+
112
+
const page = SUMMARY_PAGE.replace(
113
+
"</head>",
114
+
`<script>window.__SUMMARY__=${JSON.stringify(summary)};</script></head>`
115
+
);
116
+
return c.html(page);
117
+
});
118
+
119
+
// API: list summaries as JSON
120
+
app.get("/api/summaries", async (c) => {
121
+
await ensureContrailReady(db);
122
+
const limit = Math.min(Number(c.req.query("limit") || 50), 200);
123
+
const cursor = c.req.query("cursor");
124
+
125
+
let rows: D1Result<any>;
126
+
if (cursor) {
127
+
rows = await db
128
+
.prepare("SELECT uri, record, did, rkey, time_us FROM records_summary WHERE time_us < ? ORDER BY time_us DESC LIMIT ?")
129
+
.bind(cursor, limit)
130
+
.all();
131
+
} else {
132
+
rows = await db
133
+
.prepare("SELECT uri, record, did, rkey, time_us FROM records_summary ORDER BY time_us DESC LIMIT ?")
134
+
.bind(limit)
135
+
.all();
136
+
}
137
+
138
+
const summaries = (rows.results || []).map((r: any) => {
139
+
try {
140
+
const value = JSON.parse(r.record);
141
+
return { uri: r.uri, rkey: r.rkey, ...value };
142
+
} catch {
143
+
return null;
144
+
}
145
+
}).filter(Boolean);
146
+
147
+
const lastRow = rows.results?.[rows.results.length - 1];
148
+
const nextCursor = lastRow?.time_us || undefined;
149
+
150
+
return c.json({ summaries, cursor: nextCursor });
151
+
});
152
+
153
+
// API: trigger feed poll + summary generation
154
+
app.post("/api/poll", async (c) => {
155
+
await ensureContrailReady(db);
156
+
157
+
// Get last cursor from D1
158
+
let lastCursor: string | undefined;
159
+
try {
160
+
const row = await db
161
+
.prepare("SELECT value FROM instance_settings WHERE key = 'feed_cursor' LIMIT 1")
162
+
.first<{ value: string }>();
163
+
lastCursor = row?.value || undefined;
164
+
} catch {
165
+
// first run
166
+
}
167
+
168
+
const { posts, newCursor } = await pollNewPapers(lastCursor);
169
+
170
+
// Deduplicate against existing records
171
+
const existingUrls = new Set<string>();
172
+
if (posts.length > 0) {
173
+
const placeholders = posts.map(() => "?").join(",");
174
+
const urls = posts.map(p => p.paperUrl);
175
+
const rows = await db
176
+
.prepare(
177
+
`SELECT DISTINCT json_extract(record, '$.paperUrl') as paperUrl FROM records_summary WHERE paperUrl IN (${placeholders})`
178
+
)
179
+
.bind(...urls)
180
+
.all<{ paperUrl: string }>();
181
+
for (const r of rows.results || []) {
182
+
if (r.paperUrl) existingUrls.add(r.paperUrl);
183
+
}
184
+
}
185
+
186
+
const newPosts = posts.filter(p => !existingUrls.has(p.paperUrl));
187
+
let written = 0;
188
+
let errors = 0;
189
+
190
+
for (const post of newPosts) {
191
+
try {
192
+
const summary = await buildSummary(post);
193
+
const result = await writeSummaryToPds(post, summary);
194
+
195
+
// Notify contrail to index the new record
196
+
await contrail.notify(result.uri);
197
+
written++;
198
+
} catch (err: any) {
199
+
console.error(`Failed to write summary for ${post.paperUrl}: ${err?.message ?? err}`);
200
+
errors++;
201
+
}
202
+
}
203
+
204
+
// Save cursor
205
+
if (newCursor) {
206
+
await db
207
+
.prepare("INSERT OR REPLACE INTO instance_settings (key, value) VALUES ('feed_cursor', ?)")
208
+
.bind(newCursor)
209
+
.run();
210
+
}
211
+
212
+
return c.json({
213
+
polled: posts.length,
214
+
new: newPosts.length,
215
+
written,
216
+
errors,
217
+
});
218
+
});
219
+
220
+
// Health check
221
+
app.get("/api/health", async (c) => {
222
+
await ensureContrailReady(db);
223
+
let count = 0;
224
+
try {
225
+
const row = await db
226
+
.prepare("SELECT COUNT(*) as cnt FROM records_summary")
227
+
.first<{ cnt: number }>();
228
+
count = row?.cnt || 0;
229
+
} catch {
230
+
// table might not exist yet
231
+
}
232
+
return c.json({ ok: true, summaries: count });
233
+
});
234
+
235
+
// Crawl endpoint: register a DID and trigger immediate ingest
236
+
app.post("/xrpc/com.atproto.sync.requestCrawl", async (c) => {
237
+
let body: { did?: string; hostname?: string };
238
+
try {
239
+
body = await c.req.json();
240
+
} catch {
241
+
body = {};
242
+
}
243
+
const did = body.did;
244
+
if (!did) {
245
+
return c.json({ error: "BadRequest", message: "Provide did" }, 400);
246
+
}
247
+
248
+
await ensureContrailReady(db);
249
+
try {
250
+
// Ensure identities table has this DID
251
+
await db.exec(
252
+
"CREATE TABLE IF NOT EXISTS identities (did TEXT PRIMARY KEY, handle TEXT, time_us INTEGER)"
253
+
);
254
+
await db
255
+
.prepare("INSERT OR IGNORE INTO identities (did, handle, time_us) VALUES (?, ?, ?)")
256
+
.bind(did, "researcher.pds.latha.org", Date.now() * 1000)
257
+
.run();
258
+
259
+
// Ensure records_summary table exists
260
+
await db.exec(
261
+
"CREATE TABLE IF NOT EXISTS records_summary (uri TEXT PRIMARY KEY, cid TEXT, did TEXT, rkey TEXT, record TEXT, time_us INTEGER)"
262
+
);
263
+
264
+
// Fetch records from PDS and insert directly
265
+
const pdsUrl = `https://pds.latha.org/xrpc/com.atproto.repo.listRecords?repo=${did}&collection=org.latha.papers.summary&limit=100`;
266
+
const res = await fetch(pdsUrl);
267
+
if (res.ok) {
268
+
const data = await res.json() as {
269
+
records: Array<{ uri: string; cid: string; value: any }>;
270
+
};
271
+
for (const rec of data.records) {
272
+
const rkey = rec.uri.split("/").pop() || "";
273
+
await db
274
+
.prepare(
275
+
"INSERT OR REPLACE INTO records_summary (uri, cid, did, rkey, record, time_us) VALUES (?, ?, ?, ?, ?, ?)"
276
+
)
277
+
.bind(rec.uri, rec.cid, did, rkey, JSON.stringify(rec.value), Date.now() * 1000)
278
+
.run();
279
+
}
280
+
}
281
+
} catch (err: any) {
282
+
console.error(`Crawl error: ${err?.message ?? err}`);
283
+
}
284
+
285
+
return c.json({ ok: true, did });
286
+
});
287
+
288
+
// All other routes pass through to contrail
289
+
app.all("*", async (c) => {
290
+
const response = await contrailWorker.fetch(
291
+
c.req.raw,
292
+
c.env as unknown as Record<string, unknown>
293
+
);
294
+
return response;
295
+
});
296
+
297
+
return app;
298
+
}
299
+
300
+
export default {
301
+
fetch(request: Request, env: Env): Response | Promise<Response> {
302
+
app ??= buildApp(env);
303
+
return app.fetch(request, env);
304
+
},
305
+
async scheduled(event: ScheduledEvent, env: Env, ctx: ExecutionContext) {
306
+
// Cron: poll feed and generate summaries
307
+
const db = env.DB;
308
+
await ensureContrailReady(db);
309
+
310
+
let lastCursor: string | undefined;
311
+
try {
312
+
const row = await db
313
+
.prepare("SELECT value FROM instance_settings WHERE key = 'feed_cursor' LIMIT 1")
314
+
.first<{ value: string }>();
315
+
lastCursor = row?.value || undefined;
316
+
} catch {
317
+
// first run
318
+
}
319
+
320
+
const { posts, newCursor } = await pollNewPapers(lastCursor);
321
+
322
+
// Deduplicate
323
+
const existingUrls = new Set<string>();
324
+
if (posts.length > 0) {
325
+
const placeholders = posts.map(() => "?").join(",");
326
+
const urls = posts.map(p => p.paperUrl);
327
+
const rows = await db
328
+
.prepare(
329
+
`SELECT DISTINCT json_extract(record, '$.paperUrl') as paperUrl FROM records_summary WHERE paperUrl IN (${placeholders})`
330
+
)
331
+
.bind(...urls)
332
+
.all<{ paperUrl: string }>();
333
+
for (const r of rows.results || []) {
334
+
if (r.paperUrl) existingUrls.add(r.paperUrl);
335
+
}
336
+
}
337
+
338
+
const newPosts = posts.filter(p => !existingUrls.has(p.paperUrl));
339
+
340
+
for (const post of newPosts) {
341
+
try {
342
+
const summary = await buildSummary(post);
343
+
const result = await writeSummaryToPds(post, summary);
344
+
await contrail.notify(result.uri);
345
+
} catch (err: any) {
346
+
console.error(`Cron: failed to write summary for ${post.paperUrl}: ${err?.message ?? err}`);
347
+
}
348
+
}
349
+
350
+
if (newCursor) {
351
+
await db
352
+
.prepare("INSERT OR REPLACE INTO instance_settings (key, value) VALUES ('feed_cursor', ?)")
353
+
.bind(newCursor)
354
+
.run();
355
+
}
356
+
},
357
+
};
+14
tsconfig.json
+14
tsconfig.json
···
1
+
{
2
+
"compilerOptions": {
3
+
"target": "ES2022",
4
+
"module": "ES2022",
5
+
"moduleResolution": "bundler",
6
+
"strict": true,
7
+
"skipLibCheck": true,
8
+
"resolveJsonModule": true,
9
+
"types": ["@cloudflare/workers-types"],
10
+
"outDir": "dist"
11
+
},
12
+
"include": ["src/**/*.ts", "lexicons/**/*.ts", "lexicons/**/*.json"],
13
+
"exclude": ["src/frontend/**", "node_modules"]
14
+
}
+28
wrangler.jsonc
+28
wrangler.jsonc
···
1
+
{
2
+
"name": "papers-appview",
3
+
"main": "src/worker.ts",
4
+
"compatibility_date": "2025-12-25",
5
+
"compatibility_flags": ["nodejs_compat"],
6
+
"observability": {
7
+
"enabled": true
8
+
},
9
+
"assets": {
10
+
"directory": "public"
11
+
},
12
+
"d1_databases": [
13
+
{
14
+
"binding": "DB",
15
+
"database_name": "papers-appview",
16
+
"database_id": "5cefa152-8b9d-41c2-ba36-3af71032283d"
17
+
}
18
+
],
19
+
"triggers": {
20
+
"crons": ["*/5 * * * *"]
21
+
},
22
+
"routes": [
23
+
{
24
+
"pattern": "papers.latha.org",
25
+
"custom_domain": true
26
+
}
27
+
]
28
+
}