mary.my.id
read and write tar archives
initial commit
author
Mary
date
(Mar 2, 2024, 10:37 AM +0700)
commit
0bf8a6b2
0bf8a6b2075723848009218a5997c91df33e0388
+388
+17
LICENSE
+17
LICENSE
···
1
+
Permission is hereby granted, free of charge, to any person obtaining a copy
2
+
of this software and associated documentation files (the "Software"), to deal
3
+
in the Software without restriction, including without limitation the rights
4
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
5
+
copies of the Software, and to permit persons to whom the Software is
6
+
furnished to do so, subject to the following conditions:
7
+
8
+
The above copyright notice and this permission notice shall be included in all
9
+
copies or substantial portions of the Software.
10
+
11
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
12
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
13
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
14
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
15
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
16
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17
+
SOFTWARE.
+26
README.md
+26
README.md
···
1
+
# tar
2
+
3
+
Read and write tar archives.
4
+
5
+
```ts
6
+
// `writeTarEntry` is designed to be very simplistic, as the name suggests, it
7
+
// only spits out a buffer for one file entry. Good for streaming writes, but
8
+
// means you'd have to concatenate it yourself if you're doing it in one go.
9
+
10
+
const buffer = writeTarEntry({
11
+
name: 'README.md',
12
+
data: `Hello, **world**!`,
13
+
});
14
+
15
+
// `untar` expects a Reader interface, the package `@mary/reader` provides this
16
+
// by letting you convert a Uint8Array async iterable to one.
17
+
18
+
for await (const entry of untar(reader)) {
19
+
// If it's a file in the blobs directory...
20
+
if (entry.name.startsWith('blobs/')) {
21
+
// Read the contents...
22
+
const buffer = new Uint8Array(entry.size);
23
+
await entry.read(buffer);
24
+
}
25
+
}
26
+
```
+15
deno.json
+15
deno.json
+136
tar.ts
+136
tar.ts
···
1
+
import { getChecksum, RECORD_SIZE } from './utils.ts';
2
+
3
+
export interface TarFileAttributes {
4
+
/** @default 0o664 */
5
+
mode?: number;
6
+
/** @default 1000 */
7
+
uid?: number;
8
+
/** @default 1000 */
9
+
gid?: number;
10
+
/** @default Date.now() */
11
+
mtime?: number;
12
+
/** @default "" */
13
+
user?: string;
14
+
/** @default "" */
15
+
group?: string;
16
+
}
17
+
18
+
export interface TarFileEntry {
19
+
/** Entry name */
20
+
filename: string;
21
+
/** Entry data */
22
+
data: string | Uint8Array | ArrayBuffer;
23
+
/** Entry file attributes */
24
+
attrs?: TarFileAttributes;
25
+
}
26
+
27
+
const encoder = new TextEncoder();
28
+
29
+
const DEFAULT_ATTRS: TarFileAttributes = {};
30
+
31
+
/**
32
+
* Writes a single file entry in tar format, returns an array buffer which can
33
+
* then be concatenated or written to a stream.
34
+
*/
35
+
export function writeTarEntry(entry: TarFileEntry): ArrayBuffer {
36
+
const { filename, data, attrs = DEFAULT_ATTRS } = entry;
37
+
38
+
let name = filename;
39
+
let prefix = '';
40
+
41
+
if (name.length > 100) {
42
+
let i = 0;
43
+
while (name.length > 100) {
44
+
i = filename.indexOf('/', i);
45
+
46
+
if (i === -1) {
47
+
break;
48
+
}
49
+
50
+
prefix = filename.slice(0, i);
51
+
name = filename.slice(i + 1);
52
+
}
53
+
54
+
if (name.length > 100 || prefix.length > 155) {
55
+
const total_length = (prefix.length && prefix.length + 1) + name.length;
56
+
throw new Error(`filename is too long (${total_length})`);
57
+
}
58
+
}
59
+
60
+
const data_buf = normalizeData(data);
61
+
const data_size = data_buf.byteLength;
62
+
63
+
const padding_size = RECORD_SIZE - (data_size % RECORD_SIZE || RECORD_SIZE);
64
+
65
+
const buf = new ArrayBuffer(512 + data_size + padding_size);
66
+
67
+
// File name
68
+
writeString(buf, name, 0, 100);
69
+
70
+
// File mode
71
+
writeString(buf, formatOctal(attrs.mode ?? 0o664, 7), 100, 8);
72
+
73
+
// UID
74
+
writeString(buf, formatOctal(attrs.uid ?? 1000, 7), 108, 8);
75
+
76
+
// GID
77
+
writeString(buf, formatOctal(attrs.gid ?? 1000, 7), 116, 8);
78
+
79
+
// File size
80
+
writeString(buf, formatOctal(data_size, 11), 124, 12);
81
+
82
+
// Modified time
83
+
writeString(buf, formatOctal(attrs.mtime ?? Date.now(), 11), 136, 12);
84
+
85
+
// File type
86
+
writeString(buf, '0', 156, 12);
87
+
88
+
// Ustar
89
+
writeString(buf, 'ustar00', 257, 8);
90
+
91
+
// User ownership
92
+
writeString(buf, attrs.user ?? '', 265, 32);
93
+
94
+
// User group
95
+
writeString(buf, attrs.group ?? '', 297, 32);
96
+
97
+
// File prefix
98
+
writeString(buf, prefix, 345, 155);
99
+
100
+
// Checksum
101
+
{
102
+
const header = new Uint8Array(buf, 0, 512);
103
+
const chksum = getChecksum(header);
104
+
105
+
writeString(buf, formatOctal(chksum, 8), 148, 8);
106
+
}
107
+
108
+
// Actual data
109
+
{
110
+
const dest = new Uint8Array(buf, 512, data_size);
111
+
dest.set(data_buf, 0);
112
+
}
113
+
114
+
return buf;
115
+
}
116
+
117
+
function normalizeData(data: string | ArrayBuffer | Uint8Array): Uint8Array {
118
+
if (typeof data === 'string') {
119
+
return encoder.encode(data);
120
+
}
121
+
122
+
if (data instanceof ArrayBuffer) {
123
+
return new Uint8Array(data);
124
+
}
125
+
126
+
return data;
127
+
}
128
+
129
+
function writeString(buf: ArrayBuffer, str: string, offset: number, size: number) {
130
+
const view = new Uint8Array(buf, offset, size);
131
+
encoder.encodeInto(str, view);
132
+
}
133
+
134
+
function formatOctal(input: number, length: number) {
135
+
return input.toString(8).padStart(length, '0');
136
+
}
+170
untar.ts
+170
untar.ts
···
1
+
import { getChecksum, INITIAL_CHKSUM, RECORD_SIZE } from './utils.ts';
2
+
3
+
/**
4
+
* Provides the ability to read variably-sized buffers
5
+
*/
6
+
export interface Reader {
7
+
/**
8
+
* Reads up to `p.byteLength` into `p`
9
+
*/
10
+
read(p: Uint8Array): Promise<number | null>;
11
+
/**
12
+
* Skip reading `n` amount of bytes
13
+
*/
14
+
seek(n: number): Promise<number>;
15
+
}
16
+
17
+
const decoder = new TextDecoder();
18
+
19
+
const FILE_TYPES: Record<number, string> = {
20
+
0: 'file',
21
+
1: 'link',
22
+
2: 'symlink',
23
+
3: 'character_device',
24
+
4: 'block_device',
25
+
5: 'directory',
26
+
6: 'fifo',
27
+
7: 'contiguous_file',
28
+
};
29
+
30
+
export async function* untar(reader: Reader): AsyncGenerator<TarEntry> {
31
+
const header = new Uint8Array(512);
32
+
33
+
let entry: TarEntry | undefined;
34
+
35
+
while (true) {
36
+
if (entry) {
37
+
await entry.discard();
38
+
}
39
+
40
+
const res = await reader.read(header);
41
+
42
+
if (res === null) {
43
+
break;
44
+
}
45
+
46
+
// validate checksum
47
+
{
48
+
const expected = readOctal(header, 148, 8);
49
+
const actual = getChecksum(header);
50
+
if (expected !== actual) {
51
+
if (actual === INITIAL_CHKSUM) {
52
+
break;
53
+
}
54
+
55
+
throw new Error(`invalid checksum, expected ${expected} got ${actual}`);
56
+
}
57
+
}
58
+
59
+
// validate magic
60
+
{
61
+
const magic = readString(header, 257, 8);
62
+
if (!magic.startsWith('ustar')) {
63
+
throw new Error(`unsupported archive format: ${magic}`);
64
+
}
65
+
}
66
+
67
+
entry = new TarEntry(header, reader);
68
+
yield entry;
69
+
}
70
+
}
71
+
72
+
class TarEntry {
73
+
#reader: Reader;
74
+
#read: number = 0;
75
+
76
+
readonly name: string;
77
+
readonly mode: number;
78
+
readonly uid: number;
79
+
readonly gid: number;
80
+
readonly size: number;
81
+
readonly mtime: number;
82
+
readonly type: string;
83
+
readonly linkName: string;
84
+
readonly owner: string;
85
+
readonly group: string;
86
+
readonly entrySize: number;
87
+
88
+
constructor(header: Uint8Array, reader: Reader) {
89
+
const name = readString(header, 0, 100);
90
+
const mode = readOctal(header, 100, 8);
91
+
const uid = readOctal(header, 108, 8);
92
+
const gid = readOctal(header, 116, 8);
93
+
const size = readOctal(header, 124, 12);
94
+
const mtime = readOctal(header, 136, 12);
95
+
const type = readOctal(header, 156, 1);
96
+
const link_name = readString(header, 157, 100);
97
+
const owner = readString(header, 265, 32);
98
+
const group = readString(header, 297, 32);
99
+
const prefix = readString(header, 345, 155);
100
+
101
+
this.name = prefix.length > 0 ? prefix + '/' + name : name;
102
+
this.mode = mode;
103
+
this.uid = uid;
104
+
this.gid = gid;
105
+
this.size = size;
106
+
this.mtime = mtime;
107
+
this.type = FILE_TYPES[type] ?? '' + type;
108
+
this.linkName = link_name;
109
+
this.owner = owner;
110
+
this.group = group;
111
+
this.entrySize = Math.ceil(this.size / RECORD_SIZE) * RECORD_SIZE;
112
+
113
+
this.#reader = reader;
114
+
}
115
+
116
+
async read(p: Uint8Array): Promise<number | null> {
117
+
const remaining = this.size - this.#read;
118
+
119
+
if (remaining <= 0) {
120
+
return null;
121
+
}
122
+
123
+
if (p.byteLength <= remaining) {
124
+
this.#read += p.byteLength;
125
+
return await this.#reader.read(p);
126
+
}
127
+
128
+
// User exceeded the remaining size of this entry, we can't fulfill that
129
+
// directly because it means reading partially into the next entry
130
+
this.#read += remaining;
131
+
132
+
const block = new Uint8Array(remaining);
133
+
const n = await this.#reader.read(block);
134
+
135
+
p.set(block, 0);
136
+
return n;
137
+
}
138
+
139
+
async discard(): Promise<void> {
140
+
const remaining = this.entrySize - this.#read;
141
+
142
+
if (remaining <= 0) {
143
+
return;
144
+
}
145
+
146
+
this.#read += remaining;
147
+
148
+
await this.#reader.seek(remaining);
149
+
}
150
+
}
151
+
152
+
function readString(arr: Uint8Array, offset: number, size: number): string {
153
+
let input = arr.subarray(offset, offset + size);
154
+
155
+
for (let idx = 0, len = input.length; idx < len; idx++) {
156
+
const code = input[idx];
157
+
158
+
if (code === 0) {
159
+
input = input.subarray(0, idx);
160
+
break;
161
+
}
162
+
}
163
+
164
+
return decoder.decode(input);
165
+
}
166
+
167
+
function readOctal(arr: Uint8Array, offset: number, size: number): number {
168
+
const res = readString(arr, offset, size);
169
+
return res ? parseInt(res, 8) : 0;
170
+
}
+19
utils.ts
+19
utils.ts
···
1
+
/** Initial checksum value, includes the 8 bytes in the checksum field itself */
2
+
export const INITIAL_CHKSUM = 8 ** 32;
3
+
export const RECORD_SIZE = 512;
4
+
5
+
/** Calculates the checksum from the first 512 bytes of a buffer */
6
+
export function getChecksum(buf: Uint8Array): number {
7
+
let checksum = INITIAL_CHKSUM;
8
+
9
+
for (let i = 0; i < RECORD_SIZE; i++) {
10
+
// Ignore own checksum field
11
+
if (i >= 148 && i < 156) {
12
+
continue;
13
+
}
14
+
15
+
checksum += buf[i];
16
+
}
17
+
18
+
return checksum;
19
+
}