mary.my.id
streaming 7-Zip archive extractor
feat: LZMA2 decoder and EncodedHeader decoding
LZMA2 wraps LZMA with chunk framing: a 1-byte control selects
end/uncompressed/LZMA, encodes uncompressed and compressed sizes for
each chunk, and signals dict/state/props resets at the boundaries.
decodeLzma2Stream walks those chunks, flushing the LZ window between
chunks so the run can exceed the dictionary size.
with a one-coder dispatcher in lib/codec/folder.ts (Copy, LZMA,
LZMA2 — multi-coder chains and BCJ filters land later), the
container layer can now decode EncodedHeaders. bla.7z's
LZMA-compressed end-header now round-trips through readContainer
and recovers the original test1.xml entry; the previous "throws on
EncodedHeader" guard is replaced with the round-trip assertion.
author
Mary
date
(May 28, 2026, 12:05 PM +0700)
commit
34ed98a2
34ed98a294b9f62b33dd535efca0a10d9fc09dc0
parent
3a4261eb
3a4261ebe4bbd03d68be11f4f120a81275e3ff52
+276
-25
+113
lib/codec/folder.ts
+113
lib/codec/folder.ts
···
1
+
/**
2
+
* @module
3
+
*
4
+
* runs a {@link Folder}'s coder chain over a packed byte stream and yields
5
+
* the decoded result. only single-coder chains are supported at this layer
6
+
* (Copy, LZMA, LZMA2); BCJ / BCJ2 / Delta filters and multi-coder chains
7
+
* land in later passes.
8
+
*/
9
+
10
+
import { UnsupportedFeatureError } from '../errors.ts';
11
+
import type { Coder, Folder } from '../header.ts';
12
+
import type { Reader } from '../reader/types.ts';
13
+
import { Method } from '../constants.ts';
14
+
import { StreamReader } from '../utils/stream-reader.ts';
15
+
16
+
import {
17
+
type ByteSource,
18
+
decodeLzma2Stream,
19
+
decodeLzmaStream,
20
+
parseLzma2Properties,
21
+
parseLzmaProperties,
22
+
} from './lzma.ts';
23
+
24
+
// #region method id
25
+
26
+
/** packs a coder's variable-length method ID bytes into a single number. */
27
+
const methodIdToNumber = (methodId: Uint8Array): number => {
28
+
let n = 0;
29
+
for (const byte of methodId) {
30
+
n = (n * 0x100) + byte;
31
+
}
32
+
return n;
33
+
};
34
+
35
+
// #endregion
36
+
37
+
// #region single-coder runner
38
+
39
+
/**
40
+
* decodes a single-coder folder by streaming `packedSize` bytes from
41
+
* `reader` at `packedOffset`, running the coder, and yielding decoded
42
+
* chunks until `unpackedSize` bytes have been produced.
43
+
*
44
+
* for an EncodedHeader the caller already has the folder description from
45
+
* {@link readNextHeader}; for a data folder the caller pulls it from the
46
+
* parsed header.
47
+
*
48
+
* @throws {@link UnsupportedFeatureError} when the folder uses a coder
49
+
* chain (more than one coder) or a coder this layer doesn't know about
50
+
*/
51
+
export async function* runFolder(
52
+
reader: Reader,
53
+
folder: Folder,
54
+
packedOffset: number,
55
+
packedSize: number,
56
+
unpackedSize: number,
57
+
): AsyncGenerator<Uint8Array<ArrayBuffer>> {
58
+
if (folder.coders.length !== 1) {
59
+
throw new UnsupportedFeatureError(
60
+
`coder chains are not yet supported (folder has ${folder.coders.length} coders)`,
61
+
);
62
+
}
63
+
const coder = folder.coders[0];
64
+
const method = methodIdToNumber(coder.methodId);
65
+
66
+
const stream = await reader.read(packedOffset, packedSize);
67
+
await using sr = new StreamReader(stream);
68
+
69
+
switch (method) {
70
+
case Method.copy: {
71
+
yield* streamCopy(sr, unpackedSize);
72
+
return;
73
+
}
74
+
case Method.lzma: {
75
+
const props = parseLzmaProperties(coder.properties);
76
+
yield* decodeLzmaStream(sr, props, unpackedSize);
77
+
return;
78
+
}
79
+
case Method.lzma2: {
80
+
const props = parseLzma2Properties(coder.properties);
81
+
yield* decodeLzma2Stream(sr, props.dictSize, unpackedSize);
82
+
return;
83
+
}
84
+
default: {
85
+
throw new UnsupportedFeatureError(`coder method 0x${method.toString(16)} is not supported`);
86
+
}
87
+
}
88
+
}
89
+
90
+
const streamCopy = async function* (
91
+
source: ByteSource,
92
+
size: number,
93
+
): AsyncGenerator<Uint8Array<ArrayBuffer>> {
94
+
let produced = 0;
95
+
while (produced < size) {
96
+
const chunkSize = Math.min(size - produced, 64 * 1024);
97
+
await source.require(chunkSize);
98
+
const chunk = source.take(chunkSize);
99
+
const owned = new Uint8Array(chunk.length);
100
+
owned.set(chunk);
101
+
yield owned;
102
+
produced += chunkSize;
103
+
}
104
+
};
105
+
106
+
// #endregion
107
+
108
+
// #region coder metadata helpers
109
+
110
+
/** the numeric method id this layer recognises. */
111
+
export const coderMethod = (coder: Coder): number => methodIdToNumber(coder.methodId);
112
+
113
+
// #endregion
+132
lib/codec/lzma.ts
+132
lib/codec/lzma.ts
···
810
810
}
811
811
812
812
// #endregion
813
+
814
+
// #region lzma2
815
+
816
+
const LZMA2_CHUNK_COMPRESSED_MAX = 1 << 16;
817
+
818
+
/**
819
+
* decodes a raw LZMA2 packed stream and yields decoded chunks until either
820
+
* the expected uncompressed size is delivered or the 0x00 end-marker chunk
821
+
* is encountered.
822
+
*
823
+
* LZMA2 wraps LZMA with chunk framing: every chunk carries a 1-byte control
824
+
* byte that tells the decoder whether the next chunk is uncompressed or
825
+
* LZMA-compressed and whether to reset the dictionary / LZMA state / props.
826
+
* the actual LZMA payload of each compressed chunk is at most 64 KiB.
827
+
*
828
+
* @param input byte source for the packed stream
829
+
* @param dictSize dictionary size from the LZMA2 properties byte
830
+
* @param uncompressedSize the expected plaintext byte count
831
+
*/
832
+
export async function* decodeLzma2Stream(
833
+
input: ByteSource,
834
+
dictSize: number,
835
+
uncompressedSize: number,
836
+
): AsyncGenerator<Uint8Array<ArrayBuffer>> {
837
+
const lz = new LzWindow(dictSize);
838
+
let lzma: LzmaDecoder | undefined;
839
+
let needDictReset = true;
840
+
let needProps = true;
841
+
let produced = 0;
842
+
843
+
while (produced < uncompressedSize) {
844
+
await input.require(1);
845
+
const control = input.take(1)[0];
846
+
if (control === 0) {
847
+
break;
848
+
}
849
+
if (control === 0x01 || control >= 0xe0) {
850
+
lz.reset();
851
+
needDictReset = false;
852
+
needProps = true;
853
+
} else if (needDictReset) {
854
+
throw new MalformedArchiveError('LZMA2 dictionary reset required at chunk boundary');
855
+
}
856
+
857
+
if (control >= 0x80) {
858
+
// LZMA chunk
859
+
await input.require(4);
860
+
const meta = input.take(4);
861
+
const uncomp = (((control & 0x1f) << 16) | (meta[0] << 8) | meta[1]) + 1;
862
+
const compSize = ((meta[2] << 8) | meta[3]) + 1;
863
+
if (control >= 0xc0) {
864
+
await input.require(1);
865
+
const propsByte = input.take(1)[0];
866
+
if (propsByte > (4 * 5 + 4) * 9 + 8) {
867
+
throw new MalformedArchiveError(`invalid LZMA2 props byte: 0x${propsByte.toString(16)}`);
868
+
}
869
+
const pb = Math.floor(propsByte / (9 * 5));
870
+
const after = propsByte - pb * 9 * 5;
871
+
const lp = Math.floor(after / 9);
872
+
const lc = after - lp * 9;
873
+
if (lc + lp > 4) {
874
+
throw new MalformedArchiveError(`invalid LZMA2 lc=${lc} lp=${lp}`);
875
+
}
876
+
lzma = new LzmaDecoder({ lc, lp, pb });
877
+
needProps = false;
878
+
} else if (needProps) {
879
+
throw new MalformedArchiveError('LZMA2 chunk needs props but none have been seen');
880
+
} else if (control >= 0xa0) {
881
+
lzma!.reset();
882
+
}
883
+
if (compSize > LZMA2_CHUNK_COMPRESSED_MAX) {
884
+
throw new MalformedArchiveError(`LZMA2 compressed chunk too large: ${compSize}`);
885
+
}
886
+
887
+
await input.require(compSize);
888
+
const packed = input.take(compSize);
889
+
const rc = new RangeDecoder();
890
+
rc.feed(packed instanceof Uint8Array ? packed : new Uint8Array(packed), true);
891
+
rc.initStream();
892
+
893
+
// the chunk may exceed the remaining dict-window space, in which case
894
+
// we flush mid-chunk and resume.
895
+
let chunkRemaining = uncomp;
896
+
while (chunkRemaining > 0) {
897
+
const span = Math.min(chunkRemaining, lz.size - lz.position);
898
+
lz.setLimit(span);
899
+
lzma!.decode(lz, rc);
900
+
const out = lz.flush();
901
+
if (out.length === 0) {
902
+
throw new MalformedArchiveError('LZMA2 LZMA chunk stalled mid-decode');
903
+
}
904
+
chunkRemaining -= out.length;
905
+
produced += out.length;
906
+
yield out;
907
+
}
908
+
if (!rc.finished) {
909
+
throw new MalformedArchiveError(
910
+
`LZMA2 chunk did not consume all ${compSize} compressed bytes`,
911
+
);
912
+
}
913
+
} else if (control === 0x01 || control === 0x02) {
914
+
// uncompressed chunk
915
+
await input.require(2);
916
+
const lenBytes = input.take(2);
917
+
const len = ((lenBytes[0] << 8) | lenBytes[1]) + 1;
918
+
let copied = 0;
919
+
while (copied < len) {
920
+
const span = Math.min(len - copied, lz.size - lz.position);
921
+
lz.setLimit(span);
922
+
await input.require(span);
923
+
const raw = input.take(span);
924
+
lz.copyRaw(raw instanceof Uint8Array ? raw : new Uint8Array(raw));
925
+
const out = lz.flush();
926
+
if (out.length > 0) {
927
+
produced += out.length;
928
+
yield out;
929
+
}
930
+
copied += span;
931
+
}
932
+
} else {
933
+
throw new MalformedArchiveError(`invalid LZMA2 control byte: 0x${control.toString(16)}`);
934
+
}
935
+
}
936
+
937
+
if (produced < uncompressedSize) {
938
+
throw new TruncatedArchiveError(
939
+
`LZMA2 stream ended at ${produced} of ${uncompressedSize} bytes`,
940
+
);
941
+
}
942
+
}
943
+
944
+
// #endregion
+26
-13
lib/container.ts
+26
-13
lib/container.ts
···
14
14
15
15
import { crc32 } from '@mary/crc32';
16
16
17
+
import { runFolder } from './codec/folder.ts';
17
18
import { SIGNATURE, SIGNATURE_HEADER_SIZE } from './constants.ts';
18
19
import {
19
20
ChecksumMismatchError,
···
24
25
import { type ArchiveHeader, type Folder, parseHeader, parseStreamsInfo } from './header.ts';
25
26
import { Property } from './constants.ts';
26
27
import type { Reader } from './reader/types.ts';
27
-
import { readBytes } from './utils/buffer.ts';
28
+
import { concat, readBytes } from './utils/buffer.ts';
28
29
import { Cursor } from './utils/cursor.ts';
29
30
30
31
// #region signature header
···
196
197
197
198
/**
198
199
* reads an archive's container layer: signature, StartHeader, NextHeader.
199
-
* fails with {@link MalformedArchiveError} when the NextHeader is encoded —
200
-
* decoding requires the codec layer.
201
-
*
202
-
* once codec support lands, this function will gain a decode path; until
203
-
* then, callers wanting to test container parsing on the
204
-
* commons-compress mhc-off fixtures (which store the NextHeader
205
-
* uncompressed) should use the entry points here.
200
+
* when the NextHeader is encoded (the common case), its single-folder
201
+
* coder chain is run through the codec layer to recover the plain header
202
+
* bytes before parsing.
206
203
*
207
204
* @param reader random-access archive source
208
205
* @returns the parsed container
209
-
* @throws {@link MalformedArchiveError} when the NextHeader is encoded
210
206
*/
211
207
export const readContainer = async (reader: Reader): Promise<ArchiveContainer> => {
212
208
const start = await readSignatureHeader(reader);
213
209
const raw = await readNextHeader(reader, start);
214
-
if (raw.kind !== 'header') {
210
+
const headerBytes = raw.kind === 'header' ? raw.bytes : await decodeEncodedHeader(reader, raw);
211
+
const header = parseHeader(headerBytes);
212
+
return { header, packBase: SIGNATURE_HEADER_SIZE + header.pack.packPos, start };
213
+
};
214
+
215
+
const decodeEncodedHeader = async (
216
+
reader: Reader,
217
+
raw: Extract<NextHeaderRaw, { kind: 'encoded' }>,
218
+
): Promise<Uint8Array<ArrayBuffer>> => {
219
+
const packedOffset = SIGNATURE_HEADER_SIZE + raw.packPos;
220
+
const chunks: Uint8Array<ArrayBuffer>[] = [];
221
+
let total = 0;
222
+
for await (
223
+
const chunk of runFolder(reader, raw.folder, packedOffset, raw.packSize, raw.unpackSize)
224
+
) {
225
+
chunks.push(chunk);
226
+
total += chunk.length;
227
+
}
228
+
if (total !== raw.unpackSize) {
215
229
throw new MalformedArchiveError(
216
-
'NextHeader is compressed (EncodedHeader); decoder support is not yet wired',
230
+
`EncodedHeader produced ${total} bytes, expected ${raw.unpackSize}`,
217
231
);
218
232
}
219
-
const header = parseHeader(raw.bytes);
220
-
return { header, packBase: SIGNATURE_HEADER_SIZE + header.pack.packPos, start };
233
+
return concat(chunks, total);
221
234
};
222
235
223
236
// #endregion
+5
-12
tests/container.test.ts
+5
-12
tests/container.test.ts
···
1
1
import { assert, assertEquals, assertRejects } from '@std/assert';
2
2
3
-
import {
4
-
ChecksumMismatchError,
5
-
InvalidSignatureError,
6
-
MalformedArchiveError,
7
-
TruncatedArchiveError,
8
-
} from '../lib/errors.ts';
3
+
import { ChecksumMismatchError, InvalidSignatureError, TruncatedArchiveError } from '../lib/errors.ts';
9
4
import { readContainer, readNextHeader, readSignatureHeader } from '../lib/container.ts';
10
5
import { fromUint8Array } from '../lib/reader/common.ts';
11
6
···
106
101
assertEquals(raw.kind, 'encoded');
107
102
});
108
103
109
-
Deno.test('readContainer: bla.7z (encoded header) throws until decoder is wired', async () => {
104
+
Deno.test('readContainer: bla.7z decodes its encoded header', async () => {
110
105
const reader = fromUint8Array(loadFixture('bla.7z'));
111
-
await assertRejects(
112
-
() => readContainer(reader),
113
-
MalformedArchiveError,
114
-
'EncodedHeader',
115
-
);
106
+
const container = await readContainer(reader);
107
+
assert(container.header.files.length > 0);
108
+
assertEquals(container.header.files[0].name, 'test1.xml');
116
109
});
117
110
118
111
Deno.test('readContainer: mhc-off-empty has one zero-byte file and no folders', async () => {