mary.my.id
streaming RAR extractor
jsr.io/@mary/rar
jsr
1import { crc32 } from '@mary/crc32';
2
3import {
4 ChecksumMismatchError,
5 InvalidSignatureError,
6 MalformedArchiveError,
7 MissingPasswordError,
8 MissingVolumeError,
9 TruncatedArchiveError,
10} from '../errors.ts';
11import { concat, decodeUtf8From, read } from '../utils/buffer.ts';
12import {
13 crc32Verifier,
14 type DataSegment,
15 deferStream,
16 readBytes,
17 streamSegments,
18 verifyingPassthrough,
19} from '../utils/segments.ts';
20import { StreamReader } from '../utils/stream-reader.ts';
21
22import type { UnrarOptions, WalkOptions } from '../options.ts';
23import { fromUint8Array } from '../reader/common.ts';
24import type { Reader } from '../reader/types.ts';
25import type { FileBlock, Rar3Block } from './block.ts';
26import { parseBlock } from './block.ts';
27import { CompressMethod, FileFlag, MainFlag, RAR3_SIGNATURE } from './constants.ts';
28import { decryptRar3HeadersIfNeeded, decryptRar3Stream } from './crypto.ts';
29import { Rar3Decoder } from './unpack.ts';
30
31/**
32 * @module
33 *
34 * sequential extraction of RAR3 (RAR1.5 - RAR4) archives. {@link unrar3} walks
35 * a volume's block headers and yields a {@link Rar3Entry} for each file it
36 * contains, stitching split entries across volumes when an ordered list of
37 * readers is supplied.
38 *
39 * stored entries (method 0x30) extract verbatim. compressed entries flow
40 * through the {@link Rar3Decoder} LZSS engine; the engine throws on PPMd
41 * blocks and on VM filter symbols, both of which are not yet supported.
42 * password-protected entries also throw at read time.
43 */
44
45// #region helpers
46
47const COMMON_HEADER_SIZE = 7;
48// the probe is sized to cover almost every block we'll meet in one
49// reader.read() — RAR3 file/main/sub headers in the wild fit easily under
50// 512 bytes (32-byte fixed prefix + a short filename + a few extras).
51// reading more bytes than the header turns into a no-op subarray; reading
52// fewer would force a second round trip to fetch the rest, which is
53// exactly what we're avoiding by oversizing the probe.
54const HEADER_PROBE_SIZE = 512;
55
56// #endregion
57
58// #region archive walking
59
60/**
61 * a parsed block paired with the volume reader it came from.
62 */
63export interface Rar3VolumeBlock {
64 reader: Reader;
65 block: Rar3Block;
66}
67
68// reads the header bytes of one block starting at `offset`. RAR3 headers are
69// at most 7 + 4 = 11 bytes for the common+add_size prefix; the parser slices
70// out exactly the declared headerSize from there.
71const readHeaderBytes = async (
72 reader: Reader,
73 offset: number,
74): Promise<Uint8Array<ArrayBuffer>> => {
75 const probeLen = Math.min(HEADER_PROBE_SIZE, reader.length - offset);
76 const probe = await readBytes(reader, offset, probeLen);
77 if (probe.length < COMMON_HEADER_SIZE) {
78 throw new TruncatedArchiveError(`RAR3 archive truncated: near 0x${offset.toString(16)}`);
79 }
80 const headerSize = probe[5] | (probe[6] << 8);
81 if (headerSize < COMMON_HEADER_SIZE) {
82 throw new MalformedArchiveError(
83 `invalid RAR3 header size: ${headerSize} (at 0x${offset.toString(16)})`,
84 );
85 }
86 if (offset + headerSize > reader.length) {
87 throw new TruncatedArchiveError(
88 `RAR3 block truncated: declares ${headerSize} bytes,` +
89 ` ${reader.length - offset} available (at 0x${offset.toString(16)})`,
90 );
91 }
92 if (headerSize <= probe.length) {
93 return probe.subarray(0, headerSize) as Uint8Array<ArrayBuffer>;
94 }
95 // rare: a header bigger than HEADER_PROBE_SIZE. read the full span fresh.
96 return await readBytes(reader, offset, headerSize);
97};
98
99// walks every block of a single RAR3 volume, yielding each parsed header
100// paired with the reader it came from.
101async function* walkVolume(reader: Reader): AsyncGenerator<Rar3VolumeBlock> {
102 if (reader.length < RAR3_SIGNATURE.length) {
103 throw new TruncatedArchiveError(`not a RAR3 archive: file shorter than the signature`);
104 }
105 const signature = await readBytes(reader, 0, RAR3_SIGNATURE.length);
106 for (let idx = 0; idx < RAR3_SIGNATURE.length; idx++) {
107 if (signature[idx] !== RAR3_SIGNATURE[idx]) {
108 throw new InvalidSignatureError(`not a RAR3 archive: signature not found`);
109 }
110 }
111
112 let offset = RAR3_SIGNATURE.length;
113 while (offset + COMMON_HEADER_SIZE <= reader.length) {
114 const headerBytes = await readHeaderBytes(reader, offset);
115 const block = parseBlock(headerBytes, offset);
116 yield { block, reader };
117
118 if (block.kind === 'endArc') {
119 break;
120 }
121 offset = block.dataOffset + block.dataSize;
122 }
123}
124
125// walks every volume in order, yielding all blocks tagged with their reader.
126async function* walkVolumes(readers: readonly Reader[]): AsyncGenerator<Rar3VolumeBlock> {
127 for (const reader of readers) {
128 yield* walkVolume(reader);
129 }
130}
131
132// #endregion
133
134// #region entry
135
136/**
137 * a single file entry within a RAR3 archive.
138 *
139 * metadata is available immediately; the entry's content is read lazily when
140 * {@link Rar3Entry.body} or one of the convenience readers is consumed.
141 * PPMd-compressed entries throw at read time pending the PPMd decoder.
142 */
143export class Rar3Entry {
144 readonly #head: FileBlock;
145 readonly #segments: DataSegment[];
146 readonly #finalCrc: number;
147 readonly #decoder?: Rar3Decoder;
148 readonly #password?: string;
149
150 #chunks?: Promise<Uint8Array<ArrayBuffer>[]>;
151
152 /** @ignore */
153 constructor(
154 head: FileBlock,
155 segments: DataSegment[],
156 finalCrc: number,
157 decoder?: Rar3Decoder,
158 password?: string,
159 ) {
160 this.#head = head;
161 this.#segments = segments;
162 this.#finalCrc = finalCrc;
163 this.#decoder = decoder;
164 this.#password = password;
165 }
166
167 /** the entry's path within the archive, with forward slashes as separators. */
168 get filename(): string {
169 return this.#head.name.replaceAll('\\', '/');
170 }
171
172 /** the uncompressed size of the entry. */
173 get size(): number {
174 return this.#head.unpackedSize;
175 }
176
177 /** the size the entry occupies across the archive's data area, summed over volumes. */
178 get compressedSize(): number {
179 let total = 0;
180 for (const segment of this.#segments) {
181 total += segment.size;
182 }
183 return total;
184 }
185
186 /** the compression method byte (0x30 stored, 0x33 normal, 0x35 best). */
187 get compressionMethod(): number {
188 return this.#head.method;
189 }
190
191 /** whether the entry is a directory. */
192 get isDirectory(): boolean {
193 return (this.#head.fileFlags & FileFlag.dictMask) === FileFlag.directory;
194 }
195
196 /** whether the entry is a unix symlink. */
197 get isSymlink(): boolean {
198 return this.#head.hostOs === 3 && (this.#head.attributes & 0xf000) === 0xa000;
199 }
200
201 /** the symlink target, when stored inline. for RAR3 the target is the file body. */
202 get linkTarget(): string | undefined {
203 return undefined;
204 }
205
206 /** whether the entry's data is encrypted. */
207 get isEncrypted(): boolean {
208 return (this.#head.fileFlags & FileFlag.password) !== 0;
209 }
210
211 /** whether the entry is part of a solid block. */
212 get isSolid(): boolean {
213 return (this.#head.fileFlags & FileFlag.solid) !== 0;
214 }
215
216 /** whether the entry's data spans multiple volumes. */
217 get isSplit(): boolean {
218 return this.#segments.length > 1;
219 }
220
221 /** the data CRC32 stored for this entry; final-part CRC for a split entry. */
222 get crc32(): number {
223 return this.#finalCrc;
224 }
225
226 /**
227 * the last modification time, milliseconds since the unix epoch, or `NaN`
228 * when the archive stored an unset / invalid DOS time. RAR3 timestamps are
229 * local time; this getter treats them as UTC so the displayed wall-clock
230 * matches `date_time` lines in dumprar.
231 */
232 get mtime(): number {
233 const ext = this.#head.extMtime;
234 const time = ext?.time ?? this.#head.mtime;
235 // DOS month/day are 1-based; some encoders write 0 for "unknown".
236 // surface that as NaN rather than silently rolling into the previous
237 // year via Date.UTC(year, -1, 0, ...).
238 if (time.month < 1 || time.month > 12 || time.day < 1 || time.day > 31) {
239 return Number.NaN;
240 }
241 return Date.UTC(time.year, time.month - 1, time.day, time.hour, time.minute, time.second) +
242 Math.floor((ext?.nanos ?? 0) / 1e6);
243 }
244
245 /** the OS-specific attribute bits (unix st_mode, Windows file attributes). */
246 get mode(): number {
247 return this.#head.attributes;
248 }
249
250 #materialize(): Promise<Uint8Array<ArrayBuffer>[]> {
251 if (this.#chunks === undefined) {
252 this.#chunks = this.#decode();
253 }
254 return this.#chunks;
255 }
256
257 async #decode(): Promise<Uint8Array<ArrayBuffer>[]> {
258 if (this.isEncrypted && (this.#password === undefined || this.#head.salt === undefined)) {
259 throw new MissingPasswordError(
260 `encrypted RAR3 entry: password required` +
261 ` (entry ${this.filename}, header must carry a salt)`,
262 );
263 }
264
265 const expected = this.#head.unpackedSize;
266
267 if (this.#head.method === CompressMethod.stored) {
268 // stored entries flow straight from the segment readers into a
269 // pre-allocated result buffer; no chunk-list bookkeeping, no final
270 // concat. for an encrypted stored entry, AES-CBC pads the ciphertext
271 // up to a 16-byte block so the source may deliver a few trailing
272 // bytes past `expected`; the read-loop stops at `expected` and
273 // cancels the rest. multi-segment entries still go through the
274 // segment stitcher.
275 const source = this.isEncrypted
276 ? await decryptRar3Stream(
277 this.#password!,
278 this.#head.salt!,
279 streamSegments(this.#segments),
280 )
281 : this.#segments.length === 1
282 ? await this.#segments[0].reader.read(
283 this.#segments[0].offset,
284 this.#segments[0].size,
285 )
286 : streamSegments(this.#segments);
287 const chunks = [await read(source, expected)];
288 this.#verifyCrc32(chunks);
289 return chunks;
290 }
291
292 const ciphertext = streamSegments(this.#segments);
293 const source = this.isEncrypted
294 ? await decryptRar3Stream(this.#password!, this.#head.salt!, ciphertext)
295 : ciphertext;
296
297 if (this.#decoder === undefined) {
298 throw new MalformedArchiveError(
299 `solid RAR3 entry has no preceding run head:` +
300 ` ${this.filename} (compression method 0x${this.#head.method.toString(16)})`,
301 );
302 }
303
304 // compressed: hand the segments to the shared LZSS decoder. a 0-byte
305 // entry still goes through decode so the LZSS end-of-entry marker
306 // (which the encoder writes for every entry, regardless of size) is
307 // consumed — otherwise the next solid entry would read those bits as
308 // its own first symbol.
309 const chunks: Uint8Array<ArrayBuffer>[] = [];
310 await using reader = new StreamReader(source);
311 await this.#decoder.decode(reader, expected, (chunk) => {
312 chunks.push(chunk);
313 });
314 this.#verifyCrc32(chunks);
315 return chunks;
316 }
317
318 // verifies the stored CRC32 against the decoded plaintext. for a split
319 // entry RAR3 records the full file CRC in the final part's header (the
320 // value carried as #finalCrc), so a running CRC across every chunk lines
321 // up with what we expect.
322 #verifyCrc32(chunks: readonly Uint8Array<ArrayBuffer>[]): void {
323 let computed = 0;
324 for (const chunk of chunks) {
325 computed = crc32(chunk, computed);
326 }
327 if (computed !== this.#finalCrc) {
328 throw new ChecksumMismatchError(
329 `CRC32 mismatch: ` +
330 `stored 0x${this.#finalCrc.toString(16).padStart(8, '0')}, ` +
331 `computed 0x${computed.toString(16).padStart(8, '0')}` +
332 ` (entry ${this.filename})`,
333 );
334 }
335 }
336
337 /**
338 * a readable stream of the entry's content. decoding (or any underlying
339 * read) is deferred until the stream is pulled, so an unconsumed access
340 * does no I/O. unencrypted stored entries stream segment by segment without
341 * buffering once pulled; encrypted and LZSS-compressed entries first
342 * materialize through {@link #decode} and subsequent consumers replay from
343 * a cached copy.
344 */
345 get body(): ReadableStream<Uint8Array> {
346 // fast path: an unencrypted stored entry that has not yet been consumed
347 // elsewhere can stream straight from segments without holding a cache.
348 // the passthrough wraps the raw stream so a streamed consumer still
349 // catches a CRC32 / size mismatch — bytes() catches it via #verifyCrc32.
350 const cacheable = this.isEncrypted || this.#head.method !== CompressMethod.stored;
351 if (!cacheable && this.#chunks === undefined) {
352 return verifyingPassthrough(
353 streamSegments(this.#segments),
354 crc32Verifier(this.filename, this.#head.unpackedSize, this.#finalCrc),
355 );
356 }
357
358 return deferStream(async () => {
359 const chunks = await this.#materialize();
360 let idx = 0;
361 return new ReadableStream({
362 pull(controller) {
363 if (idx < chunks.length) {
364 controller.enqueue(chunks[idx++]);
365 } else {
366 controller.close();
367 }
368 },
369 });
370 });
371 }
372
373 /** reads the entry content as bytes. */
374 async bytes(): Promise<Uint8Array<ArrayBuffer>> {
375 const chunks = await this.#materialize();
376 if (chunks.length === 1) {
377 return chunks[0];
378 }
379 return concat(chunks, this.size);
380 }
381
382 /** reads the entry content as an ArrayBuffer. */
383 async arrayBuffer(): Promise<ArrayBuffer> {
384 const bytes = await this.bytes();
385 if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
386 return bytes.buffer;
387 }
388 return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
389 }
390
391 /** reads the entry content as a Blob. */
392 async blob(): Promise<Blob> {
393 return new Blob([await this.bytes()]);
394 }
395
396 /** reads the entry content as UTF-8 text. */
397 async text(): Promise<string> {
398 return decodeUtf8From(await this.bytes());
399 }
400
401 /** reads the entry content as UTF-8 text and parses it as JSON. */
402 async json(): Promise<unknown> {
403 return JSON.parse(await this.text());
404 }
405
406 /**
407 * forces the entry's body to be decoded if it has not already been. used by
408 * the iterator to advance a shared solid-run decoder past this entry before
409 * yielding the next one.
410 * @ignore
411 */
412 async _drain(): Promise<void> {
413 if (this.#decoder !== undefined) {
414 await this.#materialize();
415 }
416 }
417}
418
419// #endregion
420
421// #region unrar3
422
423// reads a reader's full bytes and returns a fresh Uint8Array. used when an
424// hpsw volume needs to be header-decrypted in memory before walking.
425const slurpReader = async (reader: Reader): Promise<Uint8Array> => {
426 const stream = await reader.read(0, reader.length);
427 return await read(stream, reader.length);
428};
429
430// determines whether a volume's main header sets MAIN_PASSWORD by peeking
431// the first two block headers (mark + main). returns false on any read or
432// parse failure so the caller can defer error handling to the walker.
433const hasEncryptedHeaders = async (reader: Reader): Promise<boolean> => {
434 const probeLen = Math.min(reader.length, RAR3_SIGNATURE.length + COMMON_HEADER_SIZE);
435 if (probeLen < RAR3_SIGNATURE.length + COMMON_HEADER_SIZE) {
436 return false;
437 }
438 try {
439 const probe = await readBytes(reader, 0, probeLen);
440 const mainOffset = RAR3_SIGNATURE.length;
441 const mainType = probe[mainOffset + 2];
442 const mainFlags = probe[mainOffset + 3] | (probe[mainOffset + 4] << 8);
443 return mainType === 0x73 /* main */ && (mainFlags & MainFlag.password) !== 0;
444 } catch {
445 return false;
446 }
447};
448
449// for each input reader: if its main header is encrypted, slurp + decrypt
450// and return a synthetic in-memory reader; otherwise pass through unchanged.
451const prepareReaders = async (
452 inputReaders: readonly Reader[],
453 password: string | undefined,
454): Promise<readonly Reader[]> => {
455 const out: Reader[] = [];
456 for (const reader of inputReaders) {
457 if (!(await hasEncryptedHeaders(reader))) {
458 out.push(reader);
459 continue;
460 }
461 if (password === undefined) {
462 throw new MissingPasswordError(
463 'RAR3 archive has encrypted headers: password required',
464 );
465 }
466 const raw = await slurpReader(reader);
467 const decrypted = await decryptRar3HeadersIfNeeded(password, raw);
468 // fromUint8Array requires Uint8Array<ArrayBuffer>, which is what
469 // decryptRar3HeadersIfNeeded returns (it always allocates fresh).
470 out.push(fromUint8Array(decrypted as Uint8Array<ArrayBuffer>));
471 }
472 return out;
473};
474
475/**
476 * walks every parsed block of a RAR3 archive (or multi-volume set), yielding
477 * each block paired with the reader it came from. only header bytes are read;
478 * data areas are skipped. encrypted-header volumes are decrypted on the fly
479 * when `options.password` is supplied.
480 *
481 * @param source one reader for a single-volume archive, or the ordered set of
482 * volume readers for a multi-volume archive
483 * @param options optional settings such as the password for encrypted headers
484 * @returns async generator yielding every block tagged with its volume reader
485 * @throws if a volume is not a RAR3 archive, has encrypted headers without a
486 * password, or a block header overruns the file
487 */
488export async function* walkRar3Archive(
489 source: Reader | readonly Reader[],
490 options?: WalkOptions,
491): AsyncGenerator<Rar3VolumeBlock> {
492 const inputReaders: readonly Reader[] = Array.isArray(source) ? source : [source];
493 const readers = await prepareReaders(inputReaders, options?.password);
494 yield* walkVolumes(readers);
495}
496
497/**
498 * extracts entries from a RAR3 (RAR1.5 - RAR4) archive.
499 *
500 * entries are yielded in archive order; split entries are yielded once after
501 * their final segment, and superseded versions are omitted. PPMd blocks and
502 * custom VM filter programs throw at read time pending implementations.
503 *
504 * @param source one reader for a single-volume archive, or the ordered set of
505 * volume readers for a multi-volume archive
506 * @param options optional settings such as a password for encrypted entries
507 * @returns async generator yielding the archive's file entries
508 * @throws if a volume is not a RAR3 archive, a header overruns the file, or a
509 * split file's continuation is missing
510 */
511export async function* unrar3(
512 source: Reader | readonly Reader[],
513 options?: UnrarOptions,
514): AsyncGenerator<Rar3Entry> {
515 const inputReaders: readonly Reader[] = Array.isArray(source) ? source : [source];
516 const password = options?.password;
517
518 // hpsw archives: any volume whose main header sets MAIN_PASSWORD needs to
519 // be slurped into memory and header-decrypted in one pass before the
520 // streaming walker can read it. unencrypted volumes pass through with no
521 // extra allocation.
522 const readers = await prepareReaders(inputReaders, password);
523
524 let openHead: FileBlock | undefined;
525 let openSegments: DataSegment[] = [];
526
527 let decoder: Rar3Decoder | undefined;
528 let pending: Rar3Entry | undefined;
529
530 for await (const { reader, block } of walkVolumes(readers)) {
531 if (block.kind !== 'file') {
532 continue;
533 }
534 if ((block.fileFlags & FileFlag.version) !== 0) {
535 continue;
536 }
537
538 const splitBefore = (block.fileFlags & FileFlag.splitBefore) !== 0;
539 const splitAfter = (block.fileFlags & FileFlag.splitAfter) !== 0;
540
541 if (splitBefore) {
542 if (openHead === undefined || openHead.name !== block.name) {
543 throw new MissingVolumeError(
544 `split continuation without a preceding head: ${block.name}` +
545 ` (first volume may be missing)`,
546 );
547 }
548 openSegments.push({ offset: block.dataOffset, reader, size: block.dataSize });
549 } else {
550 if (openHead !== undefined) {
551 throw new MissingVolumeError(
552 `unterminated split entry: ${openHead.name} interrupted by ${block.name}` +
553 ` (continuation volume may be missing)`,
554 );
555 }
556 openHead = block;
557 openSegments = [{ offset: block.dataOffset, reader, size: block.dataSize }];
558 }
559
560 if (splitAfter) {
561 continue;
562 }
563
564 const head = openHead;
565 const segments = openSegments;
566 const finalCrc = block.crc;
567 openHead = undefined;
568 openSegments = [];
569
570 const method = head.method;
571 const solid = (head.fileFlags & FileFlag.solid) !== 0;
572 const dictSize = dictionarySizeFromFlags(head.fileFlags);
573
574 let entry: Rar3Entry;
575 if (method === CompressMethod.stored) {
576 // stored entries skip the LZSS decoder, but the decoder state has to
577 // keep up if a later solid run depends on this entry's bytes — RAR3
578 // does not write stored entries into a solid run, so we can leave
579 // the decoder alone here.
580 entry = new Rar3Entry(head, segments, finalCrc, undefined, password);
581 } else if (!solid) {
582 // the head of a (potentially solid) compressed run; subsequent
583 // solid entries reuse the decoder. when the previous run's
584 // decoder already owns a window large enough for this entry we
585 // reset its state in place — the (1-4 MiB) window allocation
586 // dominates the cost of decoding small entries in archives that
587 // share a dict size across many file headers.
588 if (decoder !== undefined && decoder.windowSize >= dictSize) {
589 decoder.reset();
590 } else {
591 decoder = new Rar3Decoder(dictSize);
592 }
593 entry = new Rar3Entry(head, segments, finalCrc, decoder, password);
594 } else if (decoder !== undefined) {
595 // a solid continuation: the previous entry must finish decoding so
596 // the shared tables and dictionary are in the right state.
597 if (pending !== undefined) {
598 await pending._drain();
599 }
600 entry = new Rar3Entry(head, segments, finalCrc, decoder, password);
601 } else {
602 // solid entry without a preceding run head; report at read time so
603 // callers can choose to skip it without aborting the iteration.
604 entry = new Rar3Entry(head, segments, finalCrc, undefined, password);
605 }
606
607 pending = entry;
608 yield entry;
609 }
610
611 if (openHead !== undefined) {
612 throw new MissingVolumeError(
613 `unterminated split entry: ${openHead.name} (final volume may be missing)`,
614 );
615 }
616}
617
618// returns the dictionary size in bytes implied by an entry's file flags.
619const dictionarySizeFromFlags = (fileFlags: number): number => {
620 const dict = fileFlags & FileFlag.dictMask;
621 if (dict === FileFlag.directory) {
622 return 64 * 1024;
623 }
624 return 64 * 1024 << (dict >> 5);
625};
626
627// #endregion