burrito.space
asdfas
1// Live round-trip verification for the CID-exact Cloudflare R2 transport (roadmap item 6): closes
2// the handoff's "Item 6: no real R2 round-trip" gap by driving the REAL `r2BucketPut` transport
3// (packages/publish) through the REAL `raslBucketProducer` seam against a genuine R2 bucket, then
4// reading every block back over the bucket's public custom domain to prove it serves under EXACTLY
5// our CID at the exact RASL key `.well-known/rasl/<cid>`.
6//
7// What this proves that the unit tests can't:
8// 1. SigV4 signing is accepted by real R2 (auth, canonical request, credential scope all correct).
9// 2. The *signed* `x-amz-content-sha256` header round-trips: R2 stores the object only if the body
10// hashes to that value, so a 2xx is R2's own attestation the bytes are CID-exact (see the
11// CID-EXACTNESS invariant in packages/publish/src/r2-bucket-put.ts).
12// 3. The public custom domain serves `https://<host>/.well-known/rasl/<cid>` — the exact URL the
13// gateway's `raslWellKnownUrl` races — and the bytes re-hash to our CID (gateway fail-closed §9.2).
14// 4. The >256 KiB blob is stored as a single CID-exact object (no RASL codec/chunking trap; that
15// trap is IPFS-only, this confirms it empirically for the CDN path too).
16// 5. The bucket's CORS policy would let the SHIPPING in-browser publisher PUT (this script pushes
17// from Node, which has no origin and never triggers CORS — so the preflight is checked directly).
18//
19// Like scripts/ipfs-roundtrip.mjs this lives OUTSIDE the workspace packages and needs NO extra deps
20// (it reuses the already-built dist/ of core+publish and `@atcute/cid`, which is a workspace dep).
21// It runs the SAME code path publish() takes — r2BucketPut is transport, raslBucketProducer is the seam.
22//
23// Setup — the user's Cloudflare R2 details, either as env vars or in a gitignored
24// `scripts/.r2-env` (KEY=VALUE lines; loaded automatically, never committed, never logged):
25// R2_ACCOUNT_ID Cloudflare account id (forms <id>.r2.cloudflarestorage.com)
26// R2_BUCKET target bucket name
27// R2_ACCESS_KEY_ID R2 API token access key id (Object Read & Write on the bucket)
28// R2_SECRET_ACCESS_KEY R2 API token secret access key
29// R2_PUBLIC_HOST the bucket's PUBLIC custom domain, bare host, e.g. cdn.example.com
30// (an R2 custom domain or public r2.dev subdomain; the S3 endpoint won't serve reads)
31// R2_PUBLISHER_ORIGIN optional; origin the CORS preflight is checked for (default https://szzt.app)
32// Prereqs on the Cloudflare side, all doable with wrangler except the token:
33// npx wrangler r2 bucket create <bucket>
34// npx wrangler r2 bucket dev-url enable <bucket> -y # public read → R2_PUBLIC_HOST
35// npx wrangler r2 bucket cors set <bucket> --file scripts/r2-cors.json
36// The R2 API token (Object Read & Write) must come from the dashboard — wrangler has no token
37// subcommand and its OAuth token is not authorized to mint one (verified: HTTP 403, code 9109).
38// Run:
39// pnpm -r build # build core + publish dist/ the script imports
40// node scripts/r2-roundtrip.mjs # reads scripts/.r2-env if present
41// Exit code 0 on full success, 1 on any mismatch/failure.
42//
43// Objects it writes are content-addressed (`.well-known/rasl/<cid>`) and harmless; it does not delete
44// them afterward (R2 read is free, storage is negligible). The big blob is seeded with the current
45// time so each run pushes a FRESH CID — making the pre-push 404 check ("R2 did not already have it")
46// meaningful rather than a cache hit from a prior run.
47import { readFileSync } from "node:fs";
48import { join, dirname } from "node:path";
49import { fileURLToPath } from "node:url";
50
51import { create, toString as cidToString } from "@atcute/cid";
52
53const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
54
55// Load creds from a gitignored `scripts/.r2-env` (KEY=VALUE lines) when not already in the
56// environment — keeps R2 secrets out of the shell history and out of git. Env vars set on the
57// command line still win. Silently skipped if the file is absent.
58try {
59 const envFile = readFileSync(join(repoRoot, "scripts/.r2-env"), "utf8");
60 for (const raw of envFile.split("\n")) {
61 const line = raw.trim();
62 if (!line || line.startsWith("#")) continue;
63 const eq = line.indexOf("=");
64 if (eq === -1) continue;
65 const key = line.slice(0, eq).trim();
66 let val = line.slice(eq + 1).trim();
67 if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
68 val = val.slice(1, -1);
69 }
70 if (process.env[key] === undefined) process.env[key] = val;
71 }
72} catch (err) {
73 if (err?.code !== "ENOENT") throw err;
74}
75const { build } = await import(join(repoRoot, "packages/core/dist/index.js"));
76const { r2BucketPut, raslBucketProducer, R2PutError, raslKey } = await import(
77 join(repoRoot, "packages/publish/dist/index.js")
78);
79
80const CODEC_CODE = { raw: 0x55, "dag-cbor": 0x71 };
81
82function fail(msg) {
83 console.error(`\n✗ FAIL: ${msg}`);
84 process.exitCode = 1;
85}
86
87// --- Read the user's R2 config from the environment ------------------------------------------------
88const env = process.env;
89const creds = {
90 accountId: env.R2_ACCOUNT_ID,
91 bucket: env.R2_BUCKET,
92 accessKeyId: env.R2_ACCESS_KEY_ID,
93 secretAccessKey: env.R2_SECRET_ACCESS_KEY,
94};
95const publicHost = env.R2_PUBLIC_HOST;
96const missing = [
97 ["R2_ACCOUNT_ID", creds.accountId],
98 ["R2_BUCKET", creds.bucket],
99 ["R2_ACCESS_KEY_ID", creds.accessKeyId],
100 ["R2_SECRET_ACCESS_KEY", creds.secretAccessKey],
101 ["R2_PUBLIC_HOST", publicHost],
102].filter(([, v]) => !v).map(([k]) => k);
103if (missing.length) {
104 console.error(`Missing required env var(s): ${missing.join(", ")}`);
105 console.error("See the header of scripts/r2-roundtrip.mjs for what each one is.");
106 process.exit(2);
107}
108// Guard against a full URL slipping into the bare-host field (the gateway expects a bare host).
109const bareHost = publicHost.replace(/^https?:\/\//, "").replace(/\/.*$/, "");
110
111// --- Build a mini-site: text + json + a >256 KiB binary blob (the RASL codec-trap case) -----------
112const big = new Uint8Array(300 * 1024);
113const seed = Date.now() & 0xffff; // fresh content each run → fresh CID → meaningful pre-push 404
114for (let i = 0; i < big.length; i++) big[i] = (i * 31 + 7 + seed) & 0xff;
115const files = [
116 { path: "index.html", bytes: new TextEncoder().encode(`<!doctype html><h1>r2 round-trip ${seed}</h1>`) },
117 { path: "data.json", bytes: new TextEncoder().encode(JSON.stringify({ hello: "world", seed })) },
118 { path: "big.bin", bytes: big },
119];
120
121const built = await build(files, { name: "r2 round-trip" });
122console.log(`built rootCid: ${built.rootCid} (${built.blobs.length} blobs + 1 manifest)`);
123console.log(`R2 endpoint: ${creds.accountId}.r2.cloudflarestorage.com/${creds.bucket}`);
124console.log(`public host: ${bareHost}\n`);
125
126// The exact set of blocks publish()'s pushToMirror() would push: every blob (raw) + manifest (dag-cbor).
127const blocks = [
128 ...built.blobs.map((b) => ({
129 cid: b.cid,
130 bytes: b.bytes,
131 codec: "raw",
132 ...(b.blobMimeType !== undefined ? { contentType: b.blobMimeType } : {}),
133 })),
134 { cid: built.rootCid, bytes: built.encodedManifest, codec: "dag-cbor" },
135];
136
137// Independent CID re-derivation from arbitrary bytes+codec — the gateway's own fail-closed check.
138async function rederive(bytes, codec) {
139 return cidToString(await create(CODEC_CODE[codec], bytes));
140}
141
142function raslUrl(cid) {
143 return `https://${bareHost}/${raslKey(cid)}`;
144}
145
146const producer = raslBucketProducer({ host: bareHost, put: r2BucketPut(creds) });
147
148try {
149 // 0) Pre-push: the fresh big.bin CID must NOT already be served (proves the push, not a prior run).
150 const freshCid = blocks.find((b) => b.bytes.length > 256 * 1024)?.cid;
151 if (freshCid) {
152 const pre = await fetch(raslUrl(freshCid), { method: "HEAD" }).catch((e) => ({ ok: false, status: `network:${e.code ?? e}` }));
153 if (pre.ok) console.warn(` ! pre-push: ${freshCid} already served (HTTP ${pre.status}); unexpected for a fresh CID`);
154 else console.log(` pre-push: fresh ${freshCid} not yet served (HTTP ${pre.status}) — good`);
155 }
156
157 // 1) Push every block through the REAL producer/transport (same call publish() makes).
158 for (const block of blocks) {
159 try {
160 await producer.pushBlock(block);
161 console.log(` pushed ${block.codec.padEnd(8)} ${block.cid}`);
162 } catch (err) {
163 if (err instanceof R2PutError) fail(`R2 rejected ${block.cid}: HTTP ${err.status} ${err.body?.slice(0, 300) ?? ""}`);
164 else fail(`push ${block.cid} threw: ${err?.stack ?? err}`);
165 throw err; // abort — a failed push means no verifiable round-trip
166 }
167 }
168
169 // 2) Read every block back over the PUBLIC custom domain at the exact RASL URL and verify
170 // byte-exact + re-hash to our CID (this is what a gateway does on read).
171 for (const block of blocks) {
172 const url = raslUrl(block.cid);
173 let res;
174 try {
175 res = await fetch(url);
176 } catch (err) {
177 fail(`GET ${url} network error: ${err?.code ?? err}`);
178 continue;
179 }
180 if (!res.ok) {
181 fail(`GET ${url} → HTTP ${res.status} (block not served under its RASL key)`);
182 continue;
183 }
184 const served = new Uint8Array(await res.arrayBuffer());
185 if (served.length !== block.bytes.length || Buffer.compare(Buffer.from(served), Buffer.from(block.bytes)) !== 0) {
186 fail(`served bytes for ${block.cid} differ (${served.length} vs ${block.bytes.length})`);
187 continue;
188 }
189 const rd = await rederive(served, block.codec);
190 if (rd !== block.cid) {
191 fail(`served bytes re-hash to ${rd}, not ${block.cid}`);
192 continue;
193 }
194 console.log(` verified ${block.cid} (served byte-exact, re-hashes to same CID)`);
195 }
196
197 // 3) The producer advertises exactly the bucket's public host as a bare-host hint on full success.
198 const hints = producer.hints();
199 if (JSON.stringify(hints) !== JSON.stringify([bareHost])) fail(`unexpected hints: ${JSON.stringify(hints)}`);
200 else console.log(` hints: ${JSON.stringify(hints)}`);
201
202 // 4) CORS preflight from the publisher's browser origin. This harness pushes from Node, which has
203 // no origin and so never triggers CORS — but the SHIPPING path is an in-browser PUT from the
204 // publisher, which a browser blocks unless the bucket's CORS policy allows the exact method and
205 // the exact headers r2BucketPut signs. Checking the preflight here catches a misconfigured
206 // bucket without needing browser automation. Apply the policy with:
207 // npx wrangler r2 bucket cors set <bucket> --file scripts/r2-cors.json
208 const publisherOrigin = env.R2_PUBLISHER_ORIGIN ?? "https://szzt.app";
209 const signedHeaders = ["authorization", "content-type", "x-amz-content-sha256", "x-amz-date"];
210 const flight = await fetch(
211 `https://${creds.accountId}.r2.cloudflarestorage.com/${creds.bucket}/${raslKey("cors-preflight-probe")}`,
212 {
213 method: "OPTIONS",
214 headers: {
215 origin: publisherOrigin,
216 "access-control-request-method": "PUT",
217 "access-control-request-headers": signedHeaders.join(","),
218 },
219 },
220 ).catch((e) => ({ ok: false, status: `network:${e.code ?? e}`, headers: new Headers() }));
221
222 const allowOrigin = flight.headers.get("access-control-allow-origin");
223 const allowMethods = flight.headers.get("access-control-allow-methods") ?? "";
224 const allowHeaders = (flight.headers.get("access-control-allow-headers") ?? "").toLowerCase();
225 if (!flight.ok) {
226 fail(`CORS preflight from ${publisherOrigin} → HTTP ${flight.status} (browser PUT would be blocked)`);
227 } else if (allowOrigin !== publisherOrigin && allowOrigin !== "*") {
228 fail(`CORS allow-origin is ${allowOrigin ?? "(absent)"}, not ${publisherOrigin} — browser PUT blocked`);
229 } else if (!/PUT/i.test(allowMethods)) {
230 fail(`CORS allow-methods is "${allowMethods}" — must include PUT`);
231 } else {
232 const missing = signedHeaders.filter((h) => !allowHeaders.includes(h));
233 if (missing.length) fail(`CORS allow-headers missing signed header(s): ${missing.join(", ")}`);
234 else console.log(` CORS: ${publisherOrigin} may PUT with all ${signedHeaders.length} signed headers`);
235 }
236} catch (err) {
237 if (process.exitCode === undefined) fail(`round-trip aborted: ${err?.stack ?? err}`);
238}
239
240if (process.exitCode) console.error("\n=== R2 ROUND-TRIP FAILED ===");
241else console.log("\n=== R2 ROUND-TRIP OK — real r2BucketPut stored & the public domain served every block CID-exactly ===");