[READ-ONLY] Mirror of https://github.com/flo-bit/atmo-garden. atmo.garden
0

Configure Feed

Select the types of activity you want to include in your feed.

add following feeds, cache stuff

+881 -48
+164
scripts/create-placeholder-post.ts
··· 1 + // Create the one-time empty-state placeholder post that the 2 + // `following-hot` / `following-new` bsky feed generators emit when 3 + // the viewer follows zero atmo.garden communities. 4 + // 5 + // The feed generator handler prepends this post's URI to the feed 6 + // skeleton response before falling back to the all-<sort> contents, 7 + // so users who open a following feed with no subscriptions see: 8 + // 9 + // "🌱 Follow communities on atmo.garden to personalize this feed. 10 + // Browse communities → https://atmo.garden" 11 + // 12 + // …followed by the global hot/new feed. 13 + // 14 + // Usage: 15 + // pnpm tsx scripts/create-placeholder-post.ts 16 + // 17 + // Prints the resulting at-uri. Paste it into wrangler.jsonc as 18 + // `FOLLOWING_FEED_PLACEHOLDER_URI`, then deploy. This script uses 19 + // `createRecord` (not `putRecord`) because the post needs a TID 20 + // rkey for the bsky firehose, so re-running it will create a NEW 21 + // post each time — run it once and keep the URI stable. To retire 22 + // or change the placeholder, delete the old record from bsky and 23 + // re-run the script. 24 + // 25 + // Requires the same env vars the discovery-list publisher + feed 26 + // generator publisher use in src/lib/reddit/bot.ts: 27 + // 28 + // ATMO_GARDEN_PDS - e.g. https://bsky.social (or the 29 + // PDS that actually hosts the account) 30 + // ATMO_GARDEN_IDENTIFIER - handle, e.g. atmo.garden 31 + // ATMO_GARDEN_APP_PASSWORD - bsky app password (NOT main password) 32 + 33 + import { readFileSync } from 'fs'; 34 + 35 + function loadEnv(path: string) { 36 + try { 37 + const text = readFileSync(path, 'utf8'); 38 + for (const line of text.split('\n')) { 39 + const m = line.match(/^\s*([A-Z_][A-Z0-9_]*)\s*=\s*(.*)\s*$/i); 40 + if (!m) continue; 41 + let val = m[2]; 42 + if (val.startsWith('"') && val.endsWith('"')) val = val.slice(1, -1); 43 + if (!process.env[m[1]]) process.env[m[1]] = val; 44 + } 45 + } catch { 46 + /* ignore */ 47 + } 48 + } 49 + loadEnv('.env'); 50 + loadEnv('.dev.vars'); 51 + 52 + const PDS = process.env.ATMO_GARDEN_PDS; 53 + const IDENTIFIER = process.env.ATMO_GARDEN_IDENTIFIER; 54 + const APP_PASSWORD = process.env.ATMO_GARDEN_APP_PASSWORD; 55 + 56 + if (!PDS || !IDENTIFIER || !APP_PASSWORD) { 57 + console.error( 58 + 'Missing ATMO_GARDEN_PDS / ATMO_GARDEN_IDENTIFIER / ATMO_GARDEN_APP_PASSWORD in env' 59 + ); 60 + process.exit(1); 61 + } 62 + 63 + // --------------------------------------------------------------------------- 64 + // Post content + richtext facet for the https://atmo.garden link 65 + // --------------------------------------------------------------------------- 66 + 67 + const POST_TEXT = 68 + '🌱 Follow communities on atmo.garden to personalize this feed.\n\nBrowse communities → https://atmo.garden'; 69 + 70 + /** Find the UTF-8 byte range of `substring` inside `text`, or null. */ 71 + function byteRange( 72 + text: string, 73 + substring: string 74 + ): { byteStart: number; byteEnd: number } | null { 75 + const charIdx = text.indexOf(substring); 76 + if (charIdx === -1) return null; 77 + const encoder = new TextEncoder(); 78 + const byteStart = encoder.encode(text.slice(0, charIdx)).byteLength; 79 + const byteEnd = byteStart + encoder.encode(substring).byteLength; 80 + return { byteStart, byteEnd }; 81 + } 82 + 83 + const LINK_SUBSTRING = 'https://atmo.garden'; 84 + const linkRange = byteRange(POST_TEXT, LINK_SUBSTRING); 85 + if (!linkRange) { 86 + console.error('internal: could not locate link substring in POST_TEXT'); 87 + process.exit(1); 88 + } 89 + 90 + const facets = [ 91 + { 92 + index: linkRange, 93 + features: [ 94 + { 95 + $type: 'app.bsky.richtext.facet#link', 96 + uri: 'https://atmo.garden' 97 + } 98 + ] 99 + } 100 + ]; 101 + 102 + // --------------------------------------------------------------------------- 103 + // 1. Log in 104 + // --------------------------------------------------------------------------- 105 + 106 + console.log(`[1/2] createSession on ${PDS} as ${IDENTIFIER}…`); 107 + const sessionRes = await fetch(`${PDS}/xrpc/com.atproto.server.createSession`, { 108 + method: 'POST', 109 + headers: { 'Content-Type': 'application/json' }, 110 + body: JSON.stringify({ identifier: IDENTIFIER, password: APP_PASSWORD }) 111 + }); 112 + if (!sessionRes.ok) { 113 + console.error( 114 + ` createSession failed (${sessionRes.status}):`, 115 + await sessionRes.text() 116 + ); 117 + process.exit(1); 118 + } 119 + const session = (await sessionRes.json()) as { did: string; accessJwt: string }; 120 + console.log(` did = ${session.did}`); 121 + 122 + // --------------------------------------------------------------------------- 123 + // 2. createRecord the placeholder post 124 + // --------------------------------------------------------------------------- 125 + 126 + console.log(`[2/2] createRecord app.bsky.feed.post…`); 127 + const createRes = await fetch(`${PDS}/xrpc/com.atproto.repo.createRecord`, { 128 + method: 'POST', 129 + headers: { 130 + 'Content-Type': 'application/json', 131 + Authorization: `Bearer ${session.accessJwt}` 132 + }, 133 + body: JSON.stringify({ 134 + repo: session.did, 135 + collection: 'app.bsky.feed.post', 136 + record: { 137 + $type: 'app.bsky.feed.post', 138 + text: POST_TEXT, 139 + facets, 140 + createdAt: new Date().toISOString() 141 + } 142 + }) 143 + }); 144 + if (!createRes.ok) { 145 + console.error( 146 + ` createRecord failed (${createRes.status}):`, 147 + await createRes.text() 148 + ); 149 + process.exit(1); 150 + } 151 + const body = (await createRes.json()) as { uri: string; cid: string }; 152 + 153 + // --------------------------------------------------------------------------- 154 + // 3. Report 155 + // --------------------------------------------------------------------------- 156 + 157 + console.log(`\n✅ Placeholder post created:\n`); 158 + console.log(` at-uri: ${body.uri}`); 159 + const rkey = body.uri.split('/').pop(); 160 + console.log(` web: https://bsky.app/profile/${IDENTIFIER}/post/${rkey}`); 161 + console.log(`\nPaste the at-uri into wrangler.jsonc as:`); 162 + console.log(` "FOLLOWING_FEED_PLACEHOLDER_URI": "${body.uri}"`); 163 + console.log(`\nThen deploy. The following-hot / following-new feeds will`); 164 + console.log(`prepend this post when the viewer follows zero communities.\n`);
+12
scripts/publish-feed-generators.ts
··· 94 94 displayName: 'atmo.garden Top Week', 95 95 description: 96 96 'The most-liked submissions from every atmo.garden community over the last 7 days. Browse + submit at https://atmo.garden.' 97 + }, 98 + { 99 + rkey: 'following-hot', 100 + displayName: 'atmo Following Hot', 101 + description: 102 + 'Hot submissions from only the atmo.garden communities you follow on Bluesky. Follow a community account to add it to this feed. Browse + submit at https://atmo.garden.' 103 + }, 104 + { 105 + rkey: 'following-new', 106 + displayName: 'atmo Following New', 107 + description: 108 + 'Newest submissions from only the atmo.garden communities you follow on Bluesky. Follow a community account to add it to this feed. Browse + submit at https://atmo.garden.' 97 109 } 98 110 ]; 99 111
+18
src/app.d.ts
··· 22 22 COOKIE_SECRET: string; 23 23 OAUTH_PUBLIC_URL: string; 24 24 PROFILE_CACHE?: KVNamespace; 25 + /** 26 + * KV namespace holding the materialized sorted feed lists 27 + * (`sorted:<sort>` keys, one per PostSort). The cron tick 28 + * rewrites these once per minute from `getCombinedFeed`, 29 + * and both `getHomeFeed` (main page) and the bsky feed 30 + * generator XRPC handler read from them through 31 + * `src/lib/reddit/feed-cache.ts`. 32 + */ 33 + FEEDS_CACHE: KVNamespace; 25 34 DB: D1Database; 26 35 COMMUNITY_ENCRYPTION_KEY: string; 27 36 ROOKERY_HOSTNAME: string; ··· 34 43 ATMO_GARDEN_IDENTIFIER?: string; 35 44 ATMO_GARDEN_APP_PASSWORD?: string; 36 45 ATMO_GARDEN_LIST_RKEY?: string; 46 + /** 47 + * at-uri of the single bsky post emitted as the first entry 48 + * in following-* feeds when the viewer follows zero atmo 49 + * communities. Created once via 50 + * `scripts/create-placeholder-post.ts`. Empty string disables 51 + * the placeholder and the feed falls back to the all-<sort> 52 + * contents. 53 + */ 54 + FOLLOWING_FEED_PLACEHOLDER_URI?: string; 37 55 }; 38 56 } 39 57 }
+76
src/lib/reddit/bot.ts
··· 1719 1719 jetstreamEvents: number; 1720 1720 postsRefreshed: number; 1721 1721 postsDeleted: number; 1722 + feedCachesBuilt: number; 1722 1723 errors: string[]; 1723 1724 }> { 1724 1725 const db = env.DB; ··· 1798 1799 return 0; 1799 1800 }); 1800 1801 1802 + // Rebuild the materialized sorted lists + community DID list in 1803 + // KV so both the main page (`getHomeFeed`) and the bsky feed 1804 + // generator XRPC handler can serve feed requests without hitting 1805 + // D1 on the hot path. Runs last so it reflects the freshest state 1806 + // after refresh + sweep. Non-fatal: any per-sort failure just 1807 + // leaves the previous KV entry in place (the 5 min expirationTtl 1808 + // safety net kicks in only if the cron stops running entirely). 1809 + const feedCachesBuilt = await rebuildFeedCaches(env, db).catch((e) => { 1810 + errors.push(`feed-cache: ${String(e)}`); 1811 + return 0; 1812 + }); 1813 + 1801 1814 return { 1802 1815 communitiesChecked: communities.length, 1803 1816 postsCreated, ··· 1806 1819 jetstreamEvents: jetstreamResult.events, 1807 1820 postsRefreshed, 1808 1821 postsDeleted, 1822 + feedCachesBuilt, 1809 1823 errors 1810 1824 }; 1825 + } 1826 + 1827 + // ------------------------------------------------------------------------- 1828 + // Feed-cache materialization 1829 + // ------------------------------------------------------------------------- 1830 + 1831 + /** 1832 + * Rebuild the KV-backed sorted lists that both the atmo.garden home 1833 + * page and the bsky feed generator XRPC handler read from. 1834 + * 1835 + * Writes: 1836 + * - `sorted:hot`, `sorted:new`, `sorted:top-day`, `sorted:top-week` 1837 + * — each a JSON-serialized `PostWithCommunity[]` of up to 1838 + * `FEED_CACHE_LIMIT` rows from `getCombinedFeed`. 1839 + * - `communities:dids` — a JSON array of all known community DIDs, 1840 + * used by the following-feed path's `getRelationships` batch 1841 + * lookup. 1842 + * 1843 + * Runs the 4 sort queries in parallel (they're independent) so the 1844 + * total added latency to `runCronTick` is bounded by the slowest 1845 + * query. Returns the number of keys successfully written. 1846 + */ 1847 + async function rebuildFeedCaches( 1848 + env: App.Platform['env'], 1849 + db: D1Database 1850 + ): Promise<number> { 1851 + const { FEED_CACHE_LIMIT, writeSortedList, writeAllCommunityDids } = 1852 + await import('./feed-cache'); 1853 + 1854 + // All five sorts the main page exposes in its SortTabs. The bsky 1855 + // feed generator only dispatches hot/new/top-day/top-week, but 1856 + // materializing top-month here too costs almost nothing (~500 KB 1857 + // of KV storage) and lets `getHomeFeed` stay on the fast path 1858 + // across all UI sorts instead of falling back to D1 for one sort. 1859 + const sorts = ['hot', 'new', 'top-day', 'top-week', 'top-month'] as const; 1860 + let written = 0; 1861 + 1862 + await Promise.all([ 1863 + // Full community DID list (small, but refresh it here so the 1864 + // following-feed path picks up newly-registered communities 1865 + // within one cron tick). 1866 + writeAllCommunityDids(env, db) 1867 + .then(() => { 1868 + written++; 1869 + }) 1870 + .catch((e) => { 1871 + console.error('[rebuildFeedCaches] community DIDs write failed', e); 1872 + }), 1873 + // Per-sort materializations. 1874 + ...sorts.map((sort) => 1875 + getCombinedFeed(db, FEED_CACHE_LIMIT, sort, 0) 1876 + .then((rows) => writeSortedList(env, sort, rows)) 1877 + .then(() => { 1878 + written++; 1879 + }) 1880 + .catch((e) => { 1881 + console.error(`[rebuildFeedCaches] ${sort} failed`, e); 1882 + }) 1883 + ) 1884 + ]); 1885 + 1886 + return written; 1811 1887 } 1812 1888 1813 1889 // Convenience re-export so routes don't need to import db.ts directly.
+404
src/lib/reddit/feed-cache.ts
··· 1 + // Shared cache layer for the feed pipeline. 2 + // 3 + // All of our feed surfaces — the atmo.garden home page 4 + // (`getHomeFeed`), the bsky feed generator XRPC handler 5 + // (`getFeedSkeleton`), and the following-feed variants — read from 6 + // the same materialized sorted lists stored in Workers KV (binding 7 + // `FEEDS_CACHE`). The cron tick rewrites those lists once per minute 8 + // from `getCombinedFeed`, so the hot path never touches D1. 9 + // 10 + // Two cache tiers: 11 + // 12 + // 1. Workers KV (`env.FEEDS_CACHE`) — global, durable, written by 13 + // the cron once per minute. Source of truth. 14 + // 2. Workers Cache API (per-colocation edge cache, free) — sits in 15 + // front of KV for both sorted lists and per-viewer follow 16 + // intersections. 30 s TTL on lists, 5 min TTL on follow sets. 17 + // 18 + // The following-feed paths also cache each viewer's set of followed 19 + // community DIDs through the same Cache API layer. Invalidation is 20 + // best-effort per-colo via `invalidateViewerCommunityFollows`, 21 + // triggered by the `POST /api/refresh-follows` endpoint the UI hits 22 + // after a user toggles a community follow. 23 + // 24 + // See `scripts/publish-feed-generators.ts` for the feed records this 25 + // cache eventually serves, and the main cron's `rebuildFeedCaches` 26 + // step for how the KV entries get populated. 27 + 28 + import type { PostSort, PostWithCommunity } from './db'; 29 + import { listCommunities } from './db'; 30 + 31 + // Cloudflare Workers Cache API — `caches.default` is a CF-specific 32 + // extension not present on the DOM's `CacheStorage` type, AND the 33 + // `caches` global itself isn't defined in vite's Node SSR runtime 34 + // during `pnpm dev`. We cast once at the top and fall back to a 35 + // no-op shim when the global is missing so the module still loads 36 + // in dev — all cache operations become harmless no-ops and every 37 + // request falls through to KV (which DOES work via platformProxy 38 + // against the remote binding). In production on Workers the real 39 + // `caches.default` takes over transparently. 40 + type MinimalCache = { 41 + match(key: Request): Promise<Response | undefined>; 42 + put(key: Request, value: Response): Promise<void>; 43 + delete(key: Request): Promise<boolean>; 44 + }; 45 + 46 + const noopCache: MinimalCache = { 47 + async match() { 48 + return undefined; 49 + }, 50 + async put() { 51 + /* no-op */ 52 + }, 53 + async delete() { 54 + return false; 55 + } 56 + }; 57 + 58 + const cfCache: MinimalCache = 59 + typeof caches !== 'undefined' && 60 + (caches as unknown as { default?: MinimalCache }).default 61 + ? (caches as unknown as { default: MinimalCache }).default 62 + : noopCache; 63 + 64 + // --------------------------------------------------------------------------- 65 + // Sorted-list cache (materialized `getCombinedFeed` output per sort) 66 + // --------------------------------------------------------------------------- 67 + 68 + /** 69 + * Upper bound on how many rows per sort we materialize and store in 70 + * KV. The main page's infinite scroll and every feed generator call 71 + * share this cap — users can page through the top 1000 per sort, and 72 + * scrolling past that stops. Bump if users start complaining; lower 73 + * if KV storage gets tight. 1000 rows × ~500 bytes/row = ~500 KB per 74 + * sort, well under KV's 25 MB value limit. 75 + */ 76 + export const FEED_CACHE_LIMIT = 1000; 77 + 78 + /** 79 + * KV key naming: one key per sort. Values are JSON-serialized 80 + * `PostWithCommunity[]` with length ≤ `FEED_CACHE_LIMIT`, in sort 81 + * order. `missing_since IS NULL` is already applied by the underlying 82 + * `getCombinedFeed` query so we never surface rows that are pending 83 + * sweep. 84 + */ 85 + function kvKeyForSort(sort: PostSort): string { 86 + return `sorted:${sort}`; 87 + } 88 + 89 + /** 90 + * Edge-cache key for the sorted list. Workers Cache API keys are 91 + * Request objects; the URL doesn't have to resolve — it's purely a 92 + * deterministic identifier. 93 + */ 94 + function cacheKeyForSort(sort: PostSort): Request { 95 + return new Request(`https://cache.internal/feeds/sorted/${sort}`); 96 + } 97 + 98 + /** 99 + * Write the materialized sorted list for a given sort into KV. Called 100 + * by the cron's `rebuildFeedCaches` step. Includes a 5 min 101 + * `expirationTtl` as a belt-and-suspenders safety net — the cron 102 + * overwrites every minute, so the TTL only matters if the cron stops 103 + * running entirely. 104 + */ 105 + export async function writeSortedList( 106 + env: App.Platform['env'], 107 + sort: PostSort, 108 + rows: PostWithCommunity[] 109 + ): Promise<void> { 110 + await env.FEEDS_CACHE.put(kvKeyForSort(sort), JSON.stringify(rows), { 111 + expirationTtl: 300 112 + }); 113 + } 114 + 115 + /** 116 + * Read a sorted list through the edge cache, falling back to KV. 117 + * 118 + * Flow: 119 + * 1. Check Workers Cache API. Hit → return parsed JSON. 120 + * 2. Miss → fetch from KV. Returns [] if the cron hasn't populated 121 + * it yet (brand-new deploy or KV key was manually cleared). 122 + * 3. Populate the edge cache with a 30 s `s-maxage`. 123 + * 124 + * The 30 s edge TTL is chosen to be well under the cron tick 125 + * frequency, so each colo hits KV at most ~2×/min per sort — ~1.4 M 126 + * KV reads/month across ~200 colos, cheap. 127 + */ 128 + export async function getCachedSortedList( 129 + env: App.Platform['env'], 130 + sort: PostSort 131 + ): Promise<PostWithCommunity[]> { 132 + const cacheKey = cacheKeyForSort(sort); 133 + const cached = await cfCache.match(cacheKey); 134 + if (cached) { 135 + try { 136 + return (await cached.json()) as PostWithCommunity[]; 137 + } catch (e) { 138 + console.error(`[feed-cache] cached list parse failed for ${sort}`, e); 139 + // fall through to re-fetch 140 + } 141 + } 142 + 143 + const raw = await env.FEEDS_CACHE.get(kvKeyForSort(sort)); 144 + const list: PostWithCommunity[] = raw ? (JSON.parse(raw) as PostWithCommunity[]) : []; 145 + 146 + await cfCache.put( 147 + cacheKey, 148 + new Response(JSON.stringify(list), { 149 + headers: { 150 + 'content-type': 'application/json', 151 + 'cache-control': 'public, s-maxage=30' 152 + } 153 + }) 154 + ); 155 + return list; 156 + } 157 + 158 + // --------------------------------------------------------------------------- 159 + // Community-DID list (small, ~100 entries, rarely changes) 160 + // --------------------------------------------------------------------------- 161 + 162 + const COMMUNITY_DIDS_KV_KEY = 'communities:dids'; 163 + const COMMUNITY_DIDS_CACHE_KEY = new Request('https://cache.internal/communities/dids'); 164 + 165 + /** 166 + * Rewrite the list of all known community DIDs into KV. Called by 167 + * the cron alongside the sorted-list rebuild — the community set 168 + * changes rarely (new registrations) so hourly-ish staleness is 169 + * fine, but running it on every tick is also trivially cheap. 170 + */ 171 + export async function writeAllCommunityDids( 172 + env: App.Platform['env'], 173 + db: D1Database 174 + ): Promise<void> { 175 + const rows = await listCommunities(db); 176 + const dids = rows.map((r) => r.did); 177 + await env.FEEDS_CACHE.put(COMMUNITY_DIDS_KV_KEY, JSON.stringify(dids), { 178 + expirationTtl: 3600 179 + }); 180 + } 181 + 182 + /** 183 + * Read the list of all known community DIDs, edge-cached. Used by 184 + * `fetchViewerCommunityRelationships` to know which DIDs to ask bsky 185 + * about in each `getRelationships` batch. 186 + */ 187 + export async function getAllCommunityDids(env: App.Platform['env']): Promise<string[]> { 188 + const cached = await cfCache.match(COMMUNITY_DIDS_CACHE_KEY); 189 + if (cached) { 190 + try { 191 + return (await cached.json()) as string[]; 192 + } catch { 193 + /* fall through */ 194 + } 195 + } 196 + 197 + const raw = await env.FEEDS_CACHE.get(COMMUNITY_DIDS_KV_KEY); 198 + const dids: string[] = raw ? (JSON.parse(raw) as string[]) : []; 199 + 200 + await cfCache.put( 201 + COMMUNITY_DIDS_CACHE_KEY, 202 + new Response(JSON.stringify(dids), { 203 + headers: { 204 + 'content-type': 'application/json', 205 + 'cache-control': 'public, s-maxage=300' 206 + } 207 + }) 208 + ); 209 + return dids; 210 + } 211 + 212 + // --------------------------------------------------------------------------- 213 + // Per-viewer community-follows cache 214 + // --------------------------------------------------------------------------- 215 + 216 + const BSKY_APPVIEW_PUBLIC = 'https://public.api.bsky.app'; 217 + 218 + /** 219 + * Max DIDs allowed per `app.bsky.graph.getRelationships` call, per 220 + * the lexicon's `others` array maxLength. Any more and bsky returns 221 + * 400. 222 + */ 223 + const RELATIONSHIPS_BATCH_SIZE = 30; 224 + 225 + function viewerFollowsCacheKey(viewerDid: string): Request { 226 + // URL-encode the DID so colons etc. don't confuse the cache. 227 + return new Request( 228 + `https://cache.internal/follows/${encodeURIComponent(viewerDid)}` 229 + ); 230 + } 231 + 232 + /** 233 + * Call `app.bsky.graph.getRelationships` against the public bsky 234 + * appview in parallel batches, asking whether `viewerDid` follows 235 + * each community DID. Returns the subset of community DIDs that 236 + * `viewerDid` follows. 237 + * 238 + * Uses `getRelationships` instead of `getFollows` because we only 239 + * care about a specific small set of DIDs (the community accounts), 240 + * not the user's entire follow graph. This is O(communities) instead 241 + * of O(user's follows) and scales with a stable number (~100). 242 + */ 243 + export async function fetchViewerCommunityRelationships( 244 + viewerDid: string, 245 + communityDids: string[] 246 + ): Promise<string[]> { 247 + if (communityDids.length === 0) return []; 248 + 249 + // Chunk into batches of 30. 250 + const batches: string[][] = []; 251 + for (let i = 0; i < communityDids.length; i += RELATIONSHIPS_BATCH_SIZE) { 252 + batches.push(communityDids.slice(i, i + RELATIONSHIPS_BATCH_SIZE)); 253 + } 254 + 255 + const results = await Promise.all( 256 + batches.map(async (batch) => { 257 + const url = new URL( 258 + `${BSKY_APPVIEW_PUBLIC}/xrpc/app.bsky.graph.getRelationships` 259 + ); 260 + url.searchParams.set('actor', viewerDid); 261 + // `others` is an array; repeated query params. 262 + for (const did of batch) url.searchParams.append('others', did); 263 + try { 264 + const res = await fetch(url); 265 + if (!res.ok) { 266 + console.error( 267 + '[feed-cache] getRelationships non-ok', 268 + res.status, 269 + batch.length 270 + ); 271 + return [] as string[]; 272 + } 273 + const body = (await res.json()) as { 274 + relationships?: Array< 275 + | { did: string; following?: string; followedBy?: string } 276 + | { actor: string; notFound: true } 277 + >; 278 + }; 279 + const followed: string[] = []; 280 + for (const rel of body.relationships ?? []) { 281 + // `following` is set when `viewerDid` follows this DID. 282 + if ('did' in rel && typeof rel.following === 'string') { 283 + followed.push(rel.did); 284 + } 285 + } 286 + return followed; 287 + } catch (e) { 288 + console.error('[feed-cache] getRelationships threw', e); 289 + return []; 290 + } 291 + }) 292 + ); 293 + 294 + // Flatten. Any batch that failed just contributes nothing — we 295 + // return a possibly-incomplete follow set, which silently hides 296 + // some of the user's real follows until the next refresh. Better 297 + // than erroring out the whole feed. 298 + return results.flat(); 299 + } 300 + 301 + /** 302 + * Read the cached set of community DIDs the viewer follows. Cache 303 + * miss triggers a fresh `getRelationships` fan-out (parallel batches, 304 + * ~150ms worst case) and a 5 min edge-cache write. 305 + * 306 + * Returns the intersection of "communities on atmo.garden" ∩ "DIDs 307 + * the viewer follows on bsky", using bsky's native graph as the 308 + * source of truth. No D1 writes, no new follow table — the user's 309 + * `app.bsky.graph.follow` records on their own PDS drive everything. 310 + */ 311 + export async function getCachedViewerCommunityFollows( 312 + env: App.Platform['env'], 313 + viewerDid: string 314 + ): Promise<string[]> { 315 + const cacheKey = viewerFollowsCacheKey(viewerDid); 316 + const cached = await cfCache.match(cacheKey); 317 + if (cached) { 318 + try { 319 + return (await cached.json()) as string[]; 320 + } catch { 321 + /* fall through */ 322 + } 323 + } 324 + 325 + const communityDids = await getAllCommunityDids(env); 326 + const followed = await fetchViewerCommunityRelationships(viewerDid, communityDids); 327 + 328 + await cfCache.put( 329 + cacheKey, 330 + new Response(JSON.stringify(followed), { 331 + headers: { 332 + 'content-type': 'application/json', 333 + 'cache-control': 'public, s-maxage=300' 334 + } 335 + }) 336 + ); 337 + return followed; 338 + } 339 + 340 + /** 341 + * Purge the cached follow set for a viewer and eagerly repopulate it 342 + * from bsky. Called by `POST /api/refresh-follows` after the UI 343 + * writes a new `app.bsky.graph.follow` record. 344 + * 345 + * Important caveat: Workers Cache API is **per-colocation**, so this 346 + * only busts the cache at the colo handling the current request. 347 + * Other colos will continue to serve stale data until their 5 min 348 + * TTL expires. In practice this is fine — users' follow-then-reload 349 + * flows stick to the same colo via TCP/TLS stickiness and geo 350 + * routing. Cross-colo invalidation would require moving this cache 351 + * to KV, which roughly triples monthly cost. 352 + */ 353 + export async function invalidateViewerCommunityFollows( 354 + env: App.Platform['env'], 355 + viewerDid: string 356 + ): Promise<string[]> { 357 + const cacheKey = viewerFollowsCacheKey(viewerDid); 358 + await cfCache.delete(cacheKey); 359 + // Eagerly repopulate so the refresh round-trip includes the fresh 360 + // data — the UI can immediately use it (or at least knows the 361 + // bsky graph has propagated). 362 + return getCachedViewerCommunityFollows(env, viewerDid); 363 + } 364 + 365 + // --------------------------------------------------------------------------- 366 + // JWT helper (unverified payload parsing) 367 + // --------------------------------------------------------------------------- 368 + 369 + /** 370 + * Pull the `iss` claim out of a service-auth JWT in the 371 + * `Authorization: Bearer <jwt>` header. Does **not** verify the 372 + * signature — the attack surface is "see how a different user's 373 + * feed would look," which is bounded by the fact that bsky follow 374 + * graphs are public anyway. Proper signature verification would 375 + * require fetching the caller's DID doc and running ES256K against 376 + * their repo signing key; worth doing eventually, not blocking for 377 + * MVP. 378 + * 379 + * Also rejects obviously-expired JWTs as a minimal sanity check. 380 + */ 381 + export function parseViewerDidFromJwt(authHeader: string | null): string | null { 382 + if (!authHeader) return null; 383 + const m = authHeader.match(/^Bearer (.+)$/i); 384 + if (!m) return null; 385 + const parts = m[1].split('.'); 386 + if (parts.length !== 3) return null; 387 + 388 + try { 389 + // base64url → base64 → atob 390 + let payloadB64 = parts[1].replace(/-/g, '+').replace(/_/g, '/'); 391 + while (payloadB64.length % 4 !== 0) payloadB64 += '='; 392 + const json = atob(payloadB64); 393 + const obj = JSON.parse(json) as { iss?: unknown; exp?: unknown }; 394 + 395 + // Sanity: reject JWTs whose expiry is already past. 396 + if (typeof obj.exp === 'number' && obj.exp * 1000 < Date.now()) return null; 397 + 398 + const iss = obj.iss; 399 + if (typeof iss !== 'string' || !iss.startsWith('did:')) return null; 400 + return iss; 401 + } catch { 402 + return null; 403 + } 404 + }
+17 -8
src/lib/reddit/server/communities.remote.ts
··· 3 3 import * as v from 'valibot'; 4 4 import { 5 5 getRecentPostsForCommunity, 6 - getCombinedFeed, 7 6 getPostByUri, 8 7 listCommunities, 9 8 getCommunityByHandle, ··· 18 17 checkCanSubmit, 19 18 refreshCommunityCache 20 19 } from '../bot'; 20 + import { getCachedSortedList, FEED_CACHE_LIMIT } from '../feed-cache'; 21 21 import { parseListUri } from '../list-uri'; 22 22 import { 23 23 ACCENT_COLORS, ··· 245 245 } 246 246 ); 247 247 248 + /** 249 + * Home feed for atmo.garden's main page. Reads from the KV-materialized 250 + * sorted list that the cron tick rebuilds every minute — zero D1 on 251 + * the hot path. Uses the same cache as the bsky feed generator XRPC 252 + * handler so the two stay in sync by construction. 253 + * 254 + * Pagination is offset-based, capped at `FEED_CACHE_LIMIT` total 255 + * rows — infinite scroll on the main page stops when the client has 256 + * consumed all materialized entries. If someone needs to go deeper, 257 + * bump `FEED_CACHE_LIMIT` in feed-cache.ts. 258 + */ 248 259 export const getHomeFeed = command( 249 260 v.object({ 250 261 limit: v.optional(v.number()), ··· 254 265 async (input): Promise<PostWithCommunity[]> => { 255 266 const { platform } = getRequestEvent(); 256 267 const env = platform?.env; 257 - if (!env || !env.DB) return []; 268 + if (!env) return []; 258 269 259 - return getCombinedFeed( 260 - env.DB, 261 - input.limit ?? 50, 262 - (input.sort ?? 'hot') as PostSort, 263 - input.offset ?? 0 264 - ); 270 + const list = await getCachedSortedList(env, (input.sort ?? 'hot') as PostSort); 271 + const offset = Math.max(0, input.offset ?? 0); 272 + const limit = Math.max(1, Math.min(FEED_CACHE_LIMIT, input.limit ?? 50)); 273 + return list.slice(offset, offset + limit); 265 274 } 266 275 ); 267 276
+7 -2
src/routes/+page.svelte
··· 11 11 type SubmitterProfile = { handle: string; displayName: string | null; avatar: string | null }; 12 12 13 13 const PAGE_SIZE = 50; 14 + // Matches `FEED_CACHE_LIMIT` in src/lib/reddit/feed-cache.ts — the 15 + // main page is served entirely from the KV-materialized sorted 16 + // list, so infinite scroll stops once we've consumed it. Bumping 17 + // this here requires also bumping FEED_CACHE_LIMIT on the server. 18 + const FEED_MAX_ROWS = 1000; 14 19 15 20 let loading = $state(true); 16 21 let loadingMore = $state(false); ··· 49 54 quoted = quotedRes.posts; 50 55 submitters = profileRes.profiles; 51 56 } 52 - hasMore = rows.length >= PAGE_SIZE; 57 + hasMore = rows.length >= PAGE_SIZE && feed.length < FEED_MAX_ROWS; 53 58 } catch (e) { 54 59 console.error('[home] loadFeed failed', e); 55 60 } finally { ··· 75 80 quoted = { ...quoted, ...quotedRes.posts }; 76 81 submitters = { ...submitters, ...profileRes.profiles }; 77 82 } 78 - if (rows.length < PAGE_SIZE) { 83 + if (rows.length < PAGE_SIZE || feed.length >= FEED_MAX_ROWS) { 79 84 hasMore = false; 80 85 } 81 86 } catch (e) {
+33
src/routes/api/refresh-follows/+server.ts
··· 1 + // Purge the edge-cached community-follow set for the authenticated 2 + // user, then eagerly repopulate it from bsky's public appview via 3 + // `getRelationships`. Called by the community page's follow / unfollow 4 + // button after a successful `app.bsky.graph.follow` (or delete) write 5 + // on the user's PDS, so the next `getFeedSkeleton` call for this 6 + // user picks up the new community subscription without waiting for 7 + // the default 5 min TTL. 8 + // 9 + // Per-colo invalidation only — see `invalidateViewerCommunityFollows` 10 + // in `src/lib/reddit/feed-cache.ts` for the tradeoff. In practice the 11 + // user's "follow → reload feed" flow stays on one Cloudflare colo via 12 + // TCP/TLS stickiness + geo routing, so the stale-in-other-colos 13 + // window is a non-issue for interactive UX. 14 + 15 + import type { RequestHandler } from './$types'; 16 + import { json, error } from '@sveltejs/kit'; 17 + import { invalidateViewerCommunityFollows } from '$lib/reddit/feed-cache'; 18 + 19 + export const POST: RequestHandler = async ({ platform, locals }) => { 20 + const env = platform?.env; 21 + if (!env) error(500, 'Platform env unavailable'); 22 + if (!locals.did) error(401, 'Not authenticated'); 23 + 24 + // `invalidateViewerCommunityFollows` deletes the cached entry at 25 + // the current colo and then re-fetches from bsky, so this request 26 + // returns with the fresh data baked into the per-colo cache. The 27 + // UI doesn't need the returned list today, but we include it in 28 + // the response in case it wants to render immediate feedback 29 + // ("you now follow N communities") without a second round-trip. 30 + const followed = await invalidateViewerCommunityFollows(env, locals.did); 31 + 32 + return json({ ok: true, followedCount: followed.length }); 33 + };
+8
src/routes/c/[handle]/+page.svelte
··· 188 188 const result = await followUser({ did: community.did }); 189 189 followUri = result.uri; 190 190 } 191 + // Purge the edge-cached community-follow set for this 192 + // viewer so the bsky following-hot / following-new feed 193 + // generators pick up the change on the next request 194 + // without waiting for the 5 min TTL. Fire-and-forget — 195 + // the refresh is a nice-to-have, not load-bearing. 196 + fetch('/api/refresh-follows', { method: 'POST' }).catch((e) => { 197 + console.error('[community] refresh-follows failed', e); 198 + }); 191 199 } catch (e) { 192 200 console.error('[community] toggle join failed', e); 193 201 } finally {
+131 -37
src/routes/xrpc/app.bsky.feed.getFeedSkeleton/+server.ts
··· 1 1 // Bluesky feed generator: serves the `app.bsky.feed.getFeedSkeleton` 2 - // XRPC endpoint so the two `app.bsky.feed.generator` records published 3 - // by atmo.garden (`all-hot` and `top-day`) can be subscribed to from 4 - // any bsky client. 2 + // XRPC endpoint. Handles two kinds of feeds: 3 + // 4 + // scope = 'all' — global feeds (all-hot, all-new, top-day, 5 + // top-week). Reads from the KV-materialized 6 + // sorted list, slices the requested page, 7 + // maps to skeleton entries. 8 + // 9 + // scope = 'following' — per-viewer personalized feeds 10 + // (following-hot, following-new). Reads the 11 + // same KV list, filters down to posts 12 + // whose community_did is in the viewer's 13 + // followed-community set, then slices. 14 + // Zero subscriptions → emit an optional 15 + // placeholder post followed by the all-<sort> 16 + // contents as a fallback. 17 + // 18 + // Both paths are zero-D1 on the hot request path — all reads come 19 + // from Workers KV with an edge-cache tier in front (see 20 + // src/lib/reddit/feed-cache.ts). The cron tick rebuilds the KV 21 + // entries once per minute, so feed freshness is bounded by that. 22 + // 23 + // Following feeds pull the viewer DID from the `Authorization: 24 + // Bearer <jwt>` header's `iss` claim, parsed without signature 25 + // verification. The attack surface is "look at how a different user's 26 + // feed would render" — which an attacker could already compute from 27 + // bsky's public follow graph — so unverified parsing is acceptable 28 + // for MVP. Proper signature verification would require fetching the 29 + // caller's DID doc and running ES256K, worth doing eventually. 5 30 // 6 31 // Wire shape (per lexicons.atproto.com / app.bsky.feed.getFeedSkeleton): 7 32 // ··· 19 44 // } 20 45 // } 21 46 // 22 - // atmo.garden serves reposts and quote posts differently: 47 + // Post routing for skeleton entries: 23 48 // 24 49 // - Community REPOST rows (`uri` → `app.bsky.feed.repost/...`): 25 50 // emit `{ post: quoted_post_uri, reason: skeletonReasonRepost(uri) }`. ··· 31 56 // emit `{ post: uri }`. The community's quote post is itself a 32 57 // valid bsky post with an `app.bsky.embed.record` embed, so the 33 58 // appview hydrates it with the original post rendered inline. 34 - // 35 - // Feeds are matched by the rkey of the `feed` query param 36 - // (`all-hot` / `top-day`). We intentionally don't validate the 37 - // authority DID — if someone publishes a generator record on their 38 - // own account that points at our service, they'll get the same 39 - // feed data, but they can't modify it, so it's harmless. 40 59 41 60 import type { RequestHandler } from './$types'; 42 61 import { json, error } from '@sveltejs/kit'; 43 - import { getCombinedFeed, type PostSort, type PostWithCommunity } from '$lib/reddit/db'; 62 + import type { PostSort, PostWithCommunity } from '$lib/reddit/db'; 63 + import { 64 + getCachedSortedList, 65 + getCachedViewerCommunityFollows, 66 + parseViewerDidFromJwt 67 + } from '$lib/reddit/feed-cache'; 44 68 45 69 const DEFAULT_LIMIT = 50; 46 70 const MAX_LIMIT = 100; 47 71 48 - /** Map a known feed rkey → the `PostSort` we drive `getCombinedFeed` with. */ 49 - const FEED_RKEY_TO_SORT: Record<string, PostSort> = { 50 - 'all-hot': 'hot', 51 - 'all-new': 'new', 52 - 'top-day': 'top-day', 53 - 'top-week': 'top-week' 72 + type FeedConfig = { 73 + sort: PostSort; 74 + scope: 'all' | 'following'; 75 + }; 76 + 77 + /** Map a known feed rkey → config (sort + scope). */ 78 + const FEED_RKEY_TO_CONFIG: Record<string, FeedConfig> = { 79 + 'all-hot': { sort: 'hot', scope: 'all' }, 80 + 'all-new': { sort: 'new', scope: 'all' }, 81 + 'top-day': { sort: 'top-day', scope: 'all' }, 82 + 'top-week': { sort: 'top-week', scope: 'all' }, 83 + 'following-hot': { sort: 'hot', scope: 'following' }, 84 + 'following-new': { sort: 'new', scope: 'following' } 54 85 }; 55 86 56 87 type SkeletonFeedPost = { ··· 104 135 return null; 105 136 } 106 137 107 - export const GET: RequestHandler = async ({ url, platform }) => { 138 + /** 139 + * Build a response envelope with cursor handling. The cursor is an 140 + * opaque string per the lexicon spec; we use base-10 offset because 141 + * everything downstream is offset-paginated. Only emit a cursor 142 + * when the page was filled — saves one empty paginated round-trip 143 + * at the end of the feed. 144 + */ 145 + function buildResponse( 146 + entries: SkeletonFeedPost[], 147 + offset: number, 148 + limit: number, 149 + totalAvailable: number 150 + ): { feed: SkeletonFeedPost[]; cursor?: string } { 151 + const page = entries.slice(0, limit); 152 + const nextOffset = offset + page.length; 153 + const nextCursor = nextOffset < totalAvailable && page.length === limit ? String(nextOffset) : undefined; 154 + return { 155 + feed: page, 156 + ...(nextCursor ? { cursor: nextCursor } : {}) 157 + }; 158 + } 159 + 160 + export const GET: RequestHandler = async ({ url, request, platform }) => { 108 161 const env = platform?.env; 109 - if (!env?.DB) error(500, 'DB binding unavailable'); 162 + if (!env) error(500, 'Platform env unavailable'); 110 163 111 164 const feedUri = url.searchParams.get('feed'); 112 165 if (!feedUri) error(400, 'missing `feed` parameter'); 113 166 114 167 const rkey = parseFeedRkey(feedUri); 115 - const sort = rkey ? FEED_RKEY_TO_SORT[rkey] : undefined; 116 - if (!sort) { 168 + const config = rkey ? FEED_RKEY_TO_CONFIG[rkey] : undefined; 169 + if (!config) { 117 170 // Unknown feed. The getFeedSkeleton lexicon defines a single 118 171 // named error for this case (`errors: [{ name: "UnknownFeed" }]`) 119 172 // and the atproto XRPC error convention is a 400 with a JSON 120 173 // body of `{ error: "<name>", message: "<human>" }` — clients 121 - // branch on the name to render a "feed unavailable" state, 122 - // which is better UX than silently returning an empty feed. 174 + // branch on the name to render a "feed unavailable" state. 123 175 return json( 124 176 { error: 'UnknownFeed', message: `Feed not found: ${feedUri}` }, 125 177 { status: 400 } 126 178 ); 127 179 } 128 180 129 - // Clamp limit to the [1, MAX_LIMIT] range per the lexicon spec. 181 + // Clamp limit to [1, MAX_LIMIT] per the lexicon spec. 130 182 const limitRaw = url.searchParams.get('limit'); 131 183 const limitParsed = limitRaw ? parseInt(limitRaw, 10) : DEFAULT_LIMIT; 132 184 const limit = Number.isFinite(limitParsed) 133 185 ? Math.max(1, Math.min(MAX_LIMIT, limitParsed)) 134 186 : DEFAULT_LIMIT; 135 187 136 - // Cursor is an opaque string per spec; we use a base-10 offset 137 - // because `getCombinedFeed` already accepts offset-based pagination 138 - // and we don't need anything fancier. Invalid cursors fall back 139 - // to offset 0. 188 + // Cursor → offset. Invalid cursors fall back to 0. 140 189 const cursorRaw = url.searchParams.get('cursor'); 141 190 const cursorParsed = cursorRaw ? parseInt(cursorRaw, 10) : 0; 142 191 const offset = 143 192 Number.isFinite(cursorParsed) && cursorParsed >= 0 ? cursorParsed : 0; 144 193 145 - const rows = await getCombinedFeed(env.DB, limit, sort, offset); 146 - const feed = rows.map(rowToSkeleton).filter((x): x is SkeletonFeedPost => x !== null); 194 + // Pull the materialized global sorted list once — both scopes 195 + // read from the same cached entry. 196 + const sortedList = await getCachedSortedList(env, config.sort); 197 + 198 + if (config.scope === 'all') { 199 + const slice = sortedList 200 + .slice(offset, offset + limit) 201 + .map(rowToSkeleton) 202 + .filter((x): x is SkeletonFeedPost => x !== null); 203 + return json(buildResponse(slice, offset, limit, sortedList.length)); 204 + } 147 205 148 - // Only emit a cursor when we filled the page — saves one empty 149 - // paginated round-trip for the common "we've reached the end" case. 150 - const nextCursor = rows.length === limit ? String(offset + rows.length) : undefined; 206 + // ----- scope === 'following' ----- 151 207 152 - return json({ 153 - feed, 154 - ...(nextCursor ? { cursor: nextCursor } : {}) 155 - }); 208 + // Extract viewer DID from the service-auth JWT. Missing / malformed 209 + // JWT → treat as zero-subscription fallback, which surfaces the 210 + // placeholder + all-<sort> contents so an unauthenticated client 211 + // peek at the feed still renders something useful. 212 + const viewerDid = parseViewerDidFromJwt(request.headers.get('authorization')); 213 + 214 + let filtered: PostWithCommunity[] = []; 215 + if (viewerDid) { 216 + const followed = await getCachedViewerCommunityFollows(env, viewerDid); 217 + if (followed.length > 0) { 218 + const followedSet = new Set(followed); 219 + filtered = sortedList.filter((r) => followedSet.has(r.community_did)); 220 + } 221 + } 222 + 223 + if (filtered.length === 0) { 224 + // Empty-state: show the placeholder post (if configured) then 225 + // fall back to the all-<sort> list so the feed always has 226 + // content. Placeholder absent (env var empty) → just fall back 227 + // to the all-<sort> contents. 228 + const placeholderUri = env.FOLLOWING_FEED_PLACEHOLDER_URI ?? ''; 229 + const fallback = sortedList 230 + .slice(offset, offset + limit) 231 + .map(rowToSkeleton) 232 + .filter((x): x is SkeletonFeedPost => x !== null); 233 + 234 + // Only prepend the placeholder on the first page (offset 0) — 235 + // otherwise subsequent pages would have it re-inserted. 236 + const entries: SkeletonFeedPost[] = 237 + offset === 0 && placeholderUri 238 + ? [{ post: placeholderUri }, ...fallback].slice(0, limit) 239 + : fallback; 240 + 241 + return json(buildResponse(entries, offset, limit, sortedList.length)); 242 + } 243 + 244 + // Normal path: viewer has real subscriptions. 245 + const slice = filtered 246 + .slice(offset, offset + limit) 247 + .map(rowToSkeleton) 248 + .filter((x): x is SkeletonFeedPost => x !== null); 249 + return json(buildResponse(slice, offset, limit, filtered.length)); 156 250 };
+11 -1
wrangler.jsonc
··· 13 13 }, 14 14 "vars": { 15 15 "OAUTH_PUBLIC_URL": "https://atmo.garden", 16 - "ROOKERY_HOSTNAME": "pds.atmo.garden" 16 + "ROOKERY_HOSTNAME": "pds.atmo.garden", 17 + // at-uri of the single bsky post emitted as the first entry in 18 + // following-* feeds when the viewer follows zero communities. 19 + // Created once via `pnpm tsx scripts/create-placeholder-post.ts` 20 + // and pasted here. Empty string disables the placeholder (feed 21 + // falls back directly to the all-<sort> contents). 22 + "FOLLOWING_FEED_PLACEHOLDER_URI": "" 17 23 }, 18 24 "kv_namespaces": [ 19 25 { ··· 23 29 { 24 30 "binding": "OAUTH_STATES", 25 31 "id": "8cff96d0d78f4966a09b6b8833cab54e" 32 + }, 33 + { 34 + "binding": "FEEDS_CACHE", 35 + "id": "c37213227cec459887c3c4bd69ffa636" 26 36 } 27 37 ], 28 38 "d1_databases": [