mary.my.id
streaming 7-Zip archive extractor
1/**
2 * @module
3 *
4 * the public extraction surface: {@link unsevenzip} walks an archive's
5 * file entries in order, yielding a {@link SevenZipEntry} for each one
6 * whose body can be read on demand.
7 *
8 * solid blocks (multiple files in one folder) share a decode cache so
9 * the folder's coder chain runs at most once across its constituent
10 * entries.
11 */
12
13import { crc32 } from '@mary/crc32';
14
15import { type DispatchOptions, rewrapDecodeError, runFolder } from './codec/folder.ts';
16import { readContainer } from './container.ts';
17import { ChecksumMismatchError, MalformedArchiveError } from './errors.ts';
18import type { ArchiveHeader, FileEntry, Folder } from './header.ts';
19import { concatReaders } from './reader/concat.ts';
20import type { Reader } from './reader/types.ts';
21import { concat, decodeUtf8From } from './utils/buffer.ts';
22
23/** options recognised by {@link unsevenzip}. */
24export interface UnsevenzipOptions {
25 /** password used to decrypt AES-encrypted entries. */
26 password?: string;
27}
28
29// #region folder cache
30
31// caches the decoded output of each folder so entries that share a folder
32// (i.e. live in the same solid block) pay the decode cost once.
33class FolderCache {
34 readonly #reader: Reader;
35 readonly #header: ArchiveHeader;
36 readonly #packBase: number;
37 readonly #packOffsets: number[];
38 readonly #pending = new Map<number, Promise<Uint8Array<ArrayBuffer>>>();
39 readonly #options: DispatchOptions;
40
41 constructor(reader: Reader, header: ArchiveHeader, packBase: number, options: DispatchOptions) {
42 this.#reader = reader;
43 this.#header = header;
44 this.#packBase = packBase;
45 this.#options = options;
46 // running sum of pack sizes to get each pack stream's offset within
47 // the data area; folders consume pack streams in order.
48 this.#packOffsets = [];
49 let acc = 0;
50 for (const size of header.pack.packSizes) {
51 this.#packOffsets.push(acc);
52 acc += size;
53 }
54 }
55
56 get(folderIdx: number): Promise<Uint8Array<ArrayBuffer>> {
57 const cached = this.#pending.get(folderIdx);
58 if (cached !== undefined) {
59 return cached;
60 }
61 const p = this.#decode(folderIdx);
62 this.#pending.set(folderIdx, p);
63 return p;
64 }
65
66 async #decode(folderIdx: number): Promise<Uint8Array<ArrayBuffer>> {
67 const folder = this.#header.folders[folderIdx];
68 const firstPack = this.#firstPackStreamIndex(folderIdx);
69 const packOffsets: number[] = [];
70 const packSizes: number[] = [];
71 for (let i = 0; i < folder.packedStreams.length; i++) {
72 packOffsets.push(this.#packBase + this.#packOffsets[firstPack + i]);
73 packSizes.push(this.#header.pack.packSizes[firstPack + i]);
74 }
75 const unpackSize = folderUnpackSize(folder);
76
77 try {
78 const chunks: Uint8Array<ArrayBuffer>[] = [];
79 let total = 0;
80 for await (
81 const chunk of runFolder(
82 this.#reader,
83 folder,
84 packOffsets,
85 packSizes,
86 unpackSize,
87 this.#options,
88 )
89 ) {
90 chunks.push(chunk);
91 total += chunk.length;
92 }
93 if (total !== unpackSize) {
94 throw new MalformedArchiveError(
95 `folder output size mismatch: folder ${folderIdx}, expected ${unpackSize} bytes, got ${total}`,
96 );
97 }
98 const merged = concat(chunks, total);
99
100 // optional folder-level CRC
101 if (folder.crc !== undefined) {
102 const computed = crc32(merged);
103 if (computed !== folder.crc) {
104 throw new ChecksumMismatchError(
105 `folder CRC32 mismatch: folder ${folderIdx}` +
106 `, stored 0x${folder.crc.toString(16).padStart(8, '0')}` +
107 `, computed 0x${computed.toString(16).padStart(8, '0')}`,
108 );
109 }
110 }
111 return merged;
112 } catch (err) {
113 throw rewrapDecodeError(err, folder, this.#options);
114 }
115 }
116
117 #firstPackStreamIndex(folderIdx: number): number {
118 let idx = 0;
119 for (let f = 0; f < folderIdx; f++) {
120 idx += this.#header.folders[f].packedStreams.length;
121 }
122 return idx;
123 }
124}
125
126const folderUnpackSize = (folder: Folder): number => {
127 for (let i = folder.unpackSizes.length - 1; i >= 0; i--) {
128 if (!folder.bindPairs.some((bp) => bp.out === i)) {
129 return folder.unpackSizes[i];
130 }
131 }
132 return 0;
133};
134
135// #endregion
136
137// #region entry
138
139/**
140 * a single file entry within a 7z archive. metadata is available immediately;
141 * content is decoded on demand when {@link body}, {@link bytes}, or one of the
142 * convenience readers is called.
143 */
144export class SevenZipEntry {
145 readonly #file: FileEntry;
146 readonly #folderIdx: number | undefined;
147 readonly #subStreamOffset: number;
148 readonly #cache: FolderCache;
149
150 /** @ignore */
151 constructor(
152 file: FileEntry,
153 folderIdx: number | undefined,
154 subStreamOffset: number,
155 cache: FolderCache,
156 ) {
157 this.#file = file;
158 this.#folderIdx = folderIdx;
159 this.#subStreamOffset = subStreamOffset;
160 this.#cache = cache;
161 }
162
163 /** the entry's path within the archive, forward-slash separated. */
164 get filename(): string {
165 return this.#file.name.replaceAll('\\', '/');
166 }
167
168 /** the uncompressed size in bytes. */
169 get size(): number {
170 return this.#file.size;
171 }
172
173 /** whether the entry is a directory. */
174 get isDirectory(): boolean {
175 return this.#file.isDirectory;
176 }
177
178 /**
179 * whether the entry is an anti-item — a deletion marker used by 7z's
180 * differential update format. callers that aren't applying updates can
181 * typically skip these.
182 */
183 get isAntiItem(): boolean {
184 return this.#file.isAntiItem;
185 }
186
187 /** the stored CRC32 over the plaintext content, when present. */
188 get crc32(): number | undefined {
189 return this.#file.crc;
190 }
191
192 /** the modification time in unix-epoch ms, or `NaN` when unset. */
193 get mtime(): number {
194 return ntTimeToMillis(this.#file.mTime);
195 }
196
197 /** the creation time in unix-epoch ms, or `NaN` when unset. */
198 get ctime(): number {
199 return ntTimeToMillis(this.#file.cTime);
200 }
201
202 /** the access time in unix-epoch ms, or `NaN` when unset. */
203 get atime(): number {
204 return ntTimeToMillis(this.#file.aTime);
205 }
206
207 /** Windows file attribute bits, when stored. */
208 get winAttributes(): number | undefined {
209 return this.#file.winAttributes;
210 }
211
212 /**
213 * a readable stream of the entry's plaintext content. each call returns
214 * a fresh stream.
215 */
216 get body(): ReadableStream<Uint8Array> {
217 const size = this.#file.size;
218 const folderIdx = this.#folderIdx;
219 const offset = this.#subStreamOffset;
220 const cache = this.#cache;
221 const expectedCrc = this.#file.crc;
222 const name = this.filename;
223
224 return new ReadableStream({
225 async start(controller) {
226 try {
227 if (folderIdx === undefined) {
228 // empty / directory entry — emit nothing.
229 controller.close();
230 return;
231 }
232 const decoded = await cache.get(folderIdx);
233 const slice = decoded.subarray(offset, offset + size);
234 if (expectedCrc !== undefined) {
235 const computed = crc32(slice);
236 if (computed !== expectedCrc) {
237 controller.error(
238 new ChecksumMismatchError(
239 `entry CRC32 mismatch: ${name}` +
240 `, stored 0x${expectedCrc.toString(16).padStart(8, '0')}` +
241 `, computed 0x${computed.toString(16).padStart(8, '0')}`,
242 ),
243 );
244 return;
245 }
246 }
247 controller.enqueue(slice);
248 controller.close();
249 } catch (error) {
250 controller.error(error);
251 }
252 },
253 });
254 }
255
256 /** reads the entry as bytes. */
257 async bytes(): Promise<Uint8Array<ArrayBuffer>> {
258 const reader = this.body.getReader();
259 const chunks: Uint8Array[] = [];
260 let total = 0;
261 while (true) {
262 const { done, value } = await reader.read();
263 if (done) {
264 break;
265 }
266 chunks.push(value);
267 total += value.length;
268 }
269 if (chunks.length === 1) {
270 const only = chunks[0];
271 if (
272 only instanceof Uint8Array && only.buffer instanceof ArrayBuffer &&
273 only.byteLength === only.buffer.byteLength
274 ) {
275 return only as Uint8Array<ArrayBuffer>;
276 }
277 const fresh = new Uint8Array(only.length);
278 fresh.set(only);
279 return fresh;
280 }
281 return concat(chunks, total);
282 }
283
284 /** reads the entry as an ArrayBuffer. */
285 async arrayBuffer(): Promise<ArrayBuffer> {
286 const bytes = await this.bytes();
287 if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
288 return bytes.buffer;
289 }
290 return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
291 }
292
293 /** reads the entry as a Blob. */
294 async blob(): Promise<Blob> {
295 return new Blob([await this.bytes()]);
296 }
297
298 /** reads the entry as UTF-8 text. */
299 async text(): Promise<string> {
300 return decodeUtf8From(await this.bytes());
301 }
302
303 /** reads the entry as UTF-8 text and parses it as JSON. */
304 async json(): Promise<unknown> {
305 return JSON.parse(await this.text());
306 }
307}
308
309// #endregion
310
311// #region unsevenzip
312
313/**
314 * extracts entries from a 7z archive. yielded entries can be read on demand.
315 *
316 * @param source a single reader, or an ordered list of readers for a
317 * multi-volume archive
318 * @param options optional extraction settings, including the decryption
319 * password
320 * @returns async generator yielding the archive's file entries
321 */
322export async function* unsevenzip(
323 source: Reader | readonly Reader[],
324 options: UnsevenzipOptions = {},
325): AsyncGenerator<SevenZipEntry> {
326 const dispatchOptions: DispatchOptions = { password: options.password };
327 const reader: Reader = Array.isArray(source)
328 ? concatReaders(source as readonly Reader[])
329 : source as Reader;
330 const container = await readContainer(reader, dispatchOptions);
331 const cache = new FolderCache(reader, container.header, container.packBase, dispatchOptions);
332
333 // substream offsets within each folder, in archive order
334 const subStreamOffsets = computeSubStreamOffsets(container.header);
335
336 for (let i = 0; i < container.header.files.length; i++) {
337 const file = container.header.files[i];
338 const folderIdx = container.header.fileFolderIndex[i];
339 const subStreamIdx = container.header.fileSubStreamIndex[i];
340 let offset = 0;
341 if (folderIdx !== undefined && subStreamIdx !== undefined) {
342 offset = subStreamOffsets[folderIdx][subStreamIdx];
343 }
344 yield new SevenZipEntry(file, folderIdx, offset, cache);
345 }
346}
347
348/**
349 * returns offsets[folderIdx][subStreamIdx] = byte offset within the
350 * folder's decoded output where that substream begins.
351 */
352const computeSubStreamOffsets = (header: ArchiveHeader): number[][] => {
353 const out: number[][] = [];
354 let flat = 0;
355 for (let f = 0; f < header.folders.length; f++) {
356 const count = header.subStreams.numUnpackStreams[f];
357 const offsets: number[] = [];
358 let acc = 0;
359 for (let s = 0; s < count; s++) {
360 offsets.push(acc);
361 acc += header.subStreams.unpackSizes[flat + s];
362 }
363 out.push(offsets);
364 flat += count;
365 }
366 return out;
367};
368
369// #endregion
370
371// #region time
372
373// NT epoch is 1601-01-01 UTC; unix epoch is 1970-01-01 UTC. delta in ms:
374const NT_TO_UNIX_DELTA_MS = 11_644_473_600_000;
375
376const ntTimeToMillis = (value: bigint | undefined): number => {
377 if (value === undefined) {
378 return Number.NaN;
379 }
380 // NT time counts 100-ns intervals; divide by 10_000 to get ms.
381 const ms = Number(value / 10000n) - NT_TO_UNIX_DELTA_MS;
382 return ms;
383};
384
385// #endregion