jakelazaroff.com
a simpler package manager for no-build websites
Refactor vendor
author
Jake Lazaroff
date
(Jun 9, 2026, 10:59 PM -0400)
commit
e10bbd77
e10bbd7786190404f215126b4253f8ea7dcdfa58
parent
72382412
72382412f989c98d788901cc31d614097d5ef5c1
+523
-438
+24
cmd/unpm/unpm_test.go
+24
cmd/unpm/unpm_test.go
···
786
786
}
787
787
}
788
788
789
+
func TestBinary_FailedVendorDoesNotCleanOldFiles(t *testing.T) {
790
+
srv := newTestServer(map[string]testFile{})
791
+
defer srv.Close()
792
+
793
+
dir := t.TempDir()
794
+
vendorDir := filepath.Join(dir, "vendor")
795
+
staleDir := filepath.Join(vendorDir, "old-host.com")
796
+
staleFile := filepath.Join(staleDir, "stale.js")
797
+
os.MkdirAll(staleDir, 0o755)
798
+
os.WriteFile(staleFile, []byte(`export const stale = 1;`), 0o644)
799
+
800
+
configPath := writeConfig(t, dir, map[string]any{
801
+
"imports": map[string]string{"missing": srv.URL + "/missing.js"},
802
+
})
803
+
804
+
_, _, code := runCmd(binary, "vendor", "--config", configPath)
805
+
if code == 0 {
806
+
t.Fatal("expected vendor to fail")
807
+
}
808
+
if _, err := os.Stat(staleFile); err != nil {
809
+
t.Fatalf("stale file should be preserved after failed vendor: %v", err)
810
+
}
811
+
}
812
+
789
813
// --- check and why tests ---
790
814
791
815
func setupCheckDir(t *testing.T) (dir, vendorDir string) {
+255
internal/vendor/crawl.go
+255
internal/vendor/crawl.go
···
1
+
// This Source Code Form is subject to the terms of the Mozilla Public
2
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+
//
5
+
// Copyright (c) 2026 Jake Lazaroff https://tangled.org/jakelazaroff.com/unpm
6
+
7
+
package vendor
8
+
9
+
import (
10
+
"errors"
11
+
"fmt"
12
+
"io"
13
+
"mime"
14
+
"net/http"
15
+
"net/url"
16
+
"path"
17
+
"slices"
18
+
"strings"
19
+
20
+
"tangled.org/jakelazaroff.com/unpm/internal/imports"
21
+
"tangled.org/jakelazaroff.com/unpm/internal/layout"
22
+
)
23
+
24
+
type crawler struct {
25
+
files map[string]*file // canonical URL -> fetched file
26
+
aliases map[string]*file // requested URL -> fetched file
27
+
misses map[string]error // requested URL -> error, to skip retrying known-bad URLs
28
+
warnings []string
29
+
}
30
+
31
+
func newCrawler() *crawler {
32
+
return &crawler{
33
+
files: make(map[string]*file),
34
+
aliases: make(map[string]*file),
35
+
misses: make(map[string]error),
36
+
}
37
+
}
38
+
39
+
// errSkip signals that a URL was reachable but served content we don't vendor
40
+
// (typically text/html from a wrong URL). Callers should treat it as a soft
41
+
// skip rather than a hard failure.
42
+
var errSkip = errors.New("unsupported response")
43
+
44
+
var knownContentTypes = map[string][]string{
45
+
".js": {"text/javascript", "application/javascript"},
46
+
".mjs": {"text/javascript", "application/javascript"},
47
+
".cjs": {"text/javascript", "application/javascript"},
48
+
".map": {"application/json"},
49
+
}
50
+
51
+
// file is the cached result of pass 1 for a single URL.
52
+
type file struct {
53
+
content []byte
54
+
path layout.RelativePath // location relative to the output directory (see layout.Rel)
55
+
deps map[string]*file // import spec as written in source -> dep file
56
+
sourceMap *file // referenced source map file
57
+
types *file // the file's types
58
+
}
59
+
60
+
// filename returns the base name of the file's vendored path.
61
+
func (f *file) filename() string { return f.path.Base() }
62
+
63
+
// fetch downloads a URL, follows x-esm-path shims, caches the bytes in memory,
64
+
// and recursively enumerates the URLs referenced by the file. It returns the
65
+
// fetched file.
66
+
func (c *crawler) fetch(rawURL string) (*file, error) {
67
+
var err error
68
+
69
+
if f, ok := c.aliases[rawURL]; ok {
70
+
return f, nil
71
+
}
72
+
if cached, ok := c.misses[rawURL]; ok {
73
+
return nil, cached
74
+
}
75
+
defer func() {
76
+
if err != nil {
77
+
c.misses[rawURL] = err
78
+
}
79
+
}()
80
+
81
+
u, err := url.Parse(rawURL)
82
+
if err != nil {
83
+
return nil, fmt.Errorf("parsing URL %s: %w", rawURL, err)
84
+
}
85
+
86
+
resp, err := http.Get(rawURL)
87
+
if err != nil {
88
+
return nil, fmt.Errorf("fetching %s: %w", rawURL, err)
89
+
}
90
+
defer resp.Body.Close()
91
+
92
+
if resp.StatusCode != http.StatusOK {
93
+
return nil, fmt.Errorf("fetching %s: status %d", rawURL, resp.StatusCode)
94
+
}
95
+
96
+
// warn if the content-type header does not match the file extension
97
+
// (e.g. a .js file served as text/html)
98
+
ct, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
99
+
if strings.HasSuffix(strings.ToLower(u.Path), ".d.ts") {
100
+
if ct != "text/plain" && ct != "application/typescript" {
101
+
c.warnings = append(c.warnings, fmt.Sprintf("%s: content-type %q does not match extension", rawURL, ct))
102
+
}
103
+
} else if expected, ok := knownContentTypes[strings.ToLower(path.Ext(u.Path))]; ok && !slices.Contains(expected, ct) {
104
+
c.warnings = append(c.warnings, fmt.Sprintf("%s: content-type %q does not match extension", rawURL, ct))
105
+
}
106
+
107
+
// esm.sh returns x-esm-path with the canonical resolved path;
108
+
// skip the shim and fetch the resolved module directly
109
+
if esmPath := resp.Header.Get("x-esm-path"); esmPath != "" {
110
+
canonURL := u.Scheme + "://" + u.Host + esmPath
111
+
f, err := c.fetch(canonURL)
112
+
if err != nil {
113
+
return nil, fmt.Errorf("downloading canonical path for %s: %w", rawURL, err)
114
+
}
115
+
c.aliases[rawURL] = f
116
+
117
+
// x-typescript-types is on this response, not the canonical one
118
+
c.attachTypesHeader(u, resp, rawURL, f)
119
+
120
+
return f, nil
121
+
}
122
+
123
+
body, err := io.ReadAll(resp.Body)
124
+
if err != nil {
125
+
return nil, fmt.Errorf("reading %s: %w", rawURL, err)
126
+
}
127
+
128
+
canon := u.String()
129
+
f := &file{
130
+
content: body,
131
+
path: layout.Rel(u),
132
+
deps: make(map[string]*file),
133
+
}
134
+
c.files[canon] = f
135
+
c.aliases[rawURL] = f
136
+
137
+
c.discover(u, f)
138
+
139
+
c.attachTypesHeader(u, resp, rawURL, f)
140
+
141
+
// if the response didn't already point us at types, try a "sidecar" .d.ts at
142
+
// the same URL with the extension replaced (e.g. foo.mjs -> foo.d.ts). 404s
143
+
// are expected and silent.
144
+
if f.types == nil {
145
+
if sURL := sidecarURL(u, f.filename()); sURL != "" {
146
+
if sf, err := c.fetch(sURL); err == nil {
147
+
f.types = sf
148
+
}
149
+
}
150
+
}
151
+
152
+
return f, nil
153
+
}
154
+
155
+
// discover enumerates the URLs a fetched file references — its imports and any
156
+
// source map — fetching each and recording the edges on f. Source maps are
157
+
// skipped; all other files use regex-based import scanning.
158
+
func (c *crawler) discover(u *url.URL, f *file) {
159
+
if !isSourceMap(f.filename()) {
160
+
c.discoverRegex(u, f)
161
+
}
162
+
c.discoverSourceMap(u, f)
163
+
}
164
+
165
+
// discoverSourceMap follows a //# sourceMappingURL= comment, fetching the
166
+
// referenced map and recording it on f.sourceMap. Inline data: URLs are skipped.
167
+
func (c *crawler) discoverSourceMap(u *url.URL, f *file) {
168
+
for _, mapPath := range imports.ScanSourceMaps(string(f.content)) {
169
+
ref, err := url.Parse(mapPath)
170
+
if err != nil {
171
+
continue
172
+
}
173
+
mapURL := u.ResolveReference(ref).String()
174
+
if sm, err := c.fetch(mapURL); err != nil {
175
+
c.warnings = append(c.warnings, fmt.Sprintf("failed to download source map %s: %v", mapURL, err))
176
+
} else {
177
+
f.sourceMap = sm
178
+
}
179
+
}
180
+
}
181
+
182
+
// isSourceMap reports whether filename is a .map sidecar.
183
+
func isSourceMap(filename string) bool {
184
+
return strings.HasSuffix(strings.ToLower(filename), ".map")
185
+
}
186
+
187
+
// attachTypesHeader follows the x-typescript-types header on resp and records
188
+
// the result on f.types. The header is authoritative, so it overwrites any
189
+
// sidecar already found. Errors are non-fatal — a warning is emitted.
190
+
func (c *crawler) attachTypesHeader(u *url.URL, resp *http.Response, rawURL string, f *file) {
191
+
typesURL := resp.Header.Get("x-typescript-types")
192
+
if typesURL == "" {
193
+
return
194
+
}
195
+
if strings.HasPrefix(typesURL, "/") {
196
+
typesURL = u.Scheme + "://" + u.Host + typesURL
197
+
}
198
+
tf, err := c.fetch(typesURL)
199
+
if err != nil {
200
+
c.warnings = append(c.warnings, fmt.Sprintf("failed to download types for %s: %v", rawURL, err))
201
+
return
202
+
}
203
+
if f != nil {
204
+
f.types = tf
205
+
}
206
+
}
207
+
208
+
// isScript reports whether filename is a script file (JS or TS, but not .d.ts).
209
+
func isScript(filename string) bool {
210
+
f := strings.ToLower(filename)
211
+
if strings.HasSuffix(f, ".d.ts") {
212
+
return false
213
+
}
214
+
switch path.Ext(f) {
215
+
case ".ts", ".mts", ".tsx", ".jsx", ".js", ".mjs":
216
+
return true
217
+
}
218
+
return false
219
+
}
220
+
221
+
// sidecarURL returns the URL of the .d.ts file that may live alongside a script
222
+
// file (e.g. foo.mjs -> foo.d.ts), or "" if no sidecar lookup is appropriate.
223
+
func sidecarURL(u *url.URL, filename string) string {
224
+
if !isScript(filename) {
225
+
return ""
226
+
}
227
+
ext := path.Ext(u.Path)
228
+
if ext == "" {
229
+
return ""
230
+
}
231
+
s := *u
232
+
s.Path = strings.TrimSuffix(u.Path, ext) + ".d.ts"
233
+
return s.String()
234
+
}
235
+
236
+
// discoverRegex enumerates imports by scanning the source with a regex.
237
+
// Bare specifiers are skipped because imports.Resolve returns "" for anything
238
+
// without a scheme or a "/", "./", "../" prefix.
239
+
func (c *crawler) discoverRegex(u *url.URL, f *file) {
240
+
for _, spec := range imports.Scan(string(f.content)) {
241
+
if _, seen := f.deps[spec]; seen {
242
+
continue
243
+
}
244
+
depURL := imports.Resolve(u, spec)
245
+
if depURL == "" {
246
+
continue
247
+
}
248
+
df, err := c.fetch(depURL)
249
+
if err != nil {
250
+
c.warnings = append(c.warnings, fmt.Sprintf("failed to download %s: %v", depURL, err))
251
+
continue
252
+
}
253
+
f.deps[spec] = df
254
+
}
255
+
}
+24
-438
internal/vendor/vendor.go
+24
-438
internal/vendor/vendor.go
···
7
7
package vendor
8
8
9
9
import (
10
-
"encoding/json"
11
10
"errors"
12
11
"fmt"
13
-
"io"
14
-
"io/fs"
15
-
"mime"
16
-
"net/http"
17
-
"net/url"
18
-
"os"
19
-
"path"
20
-
"path/filepath"
21
-
"slices"
22
-
"sort"
23
12
"strings"
24
-
"syscall"
25
13
26
14
"tangled.org/jakelazaroff.com/unpm/internal/cfg"
27
-
"tangled.org/jakelazaroff.com/unpm/internal/imports"
28
15
"tangled.org/jakelazaroff.com/unpm/internal/inspect"
29
16
"tangled.org/jakelazaroff.com/unpm/internal/layout"
30
17
)
31
18
32
-
type vendorer struct {
33
-
config *cfg.Config
34
-
l layout.Layout
35
-
files map[string]*file // canonical URL -> fetched file
36
-
aliases map[string]*file // requested URL -> fetched file
37
-
misses map[string]error // requested URL -> error, to skip retrying known-bad URLs
38
-
warnings []string
39
-
}
40
-
41
-
// errSkip signals that a URL was reachable but served content we don't vendor
42
-
// (typically text/html from a wrong URL). Callers should treat it as a soft
43
-
// skip rather than a hard failure.
44
-
var errSkip = errors.New("unsupported response")
45
-
46
-
var knownContentTypes = map[string][]string{
47
-
".js": {"text/javascript", "application/javascript"},
48
-
".mjs": {"text/javascript", "application/javascript"},
49
-
".cjs": {"text/javascript", "application/javascript"},
50
-
".map": {"application/json"},
51
-
}
52
-
53
-
// file is the cached result of pass 1 for a single URL.
54
-
type file struct {
55
-
content []byte
56
-
path layout.RelativePath // location relative to the output directory (see layout.Rel)
57
-
deps map[string]*file // import spec as written in source -> dep file
58
-
sourceMap *file // referenced source map file
59
-
types *file // the file's types
60
-
}
61
-
62
-
// filename returns the base name of the file's vendored path.
63
-
func (f *file) filename() string { return f.path.Base() }
64
-
65
19
func Vendor(c *cfg.Config) ([]string, error) {
66
-
// clean the output directory, preserving pinned files
67
-
if _, err := clean(c, c.Unpm.Out); err != nil && !errors.Is(err, fs.ErrNotExist) {
68
-
return []string{}, err
69
-
}
70
-
71
-
v := &vendorer{
72
-
config: c,
73
-
l: layout.Layout{Out: c.Unpm.Out, Root: c.Unpm.Root},
74
-
files: make(map[string]*file),
75
-
aliases: make(map[string]*file),
76
-
misses: make(map[string]error),
77
-
}
20
+
l := layout.Layout{Out: c.Unpm.Out, Root: c.Unpm.Root}
21
+
cr := newCrawler()
78
22
79
23
// pass 1: fetch every reachable URL and discover dep edges in memory
80
24
entries := make(map[string]*file) // import key -> entry file
81
25
for key, rawURL := range c.Imports {
82
-
f, err := v.fetch(rawURL)
26
+
f, err := cr.fetch(rawURL)
83
27
if err != nil {
84
28
if errors.Is(err, errSkip) {
85
29
continue
86
30
}
87
-
return v.warnings, fmt.Errorf("fetching %q: %w", key, err)
31
+
return cr.warnings, fmt.Errorf("fetching %q: %w", key, err)
88
32
}
89
33
90
34
entries[key] = f
···
97
41
if !ok {
98
42
continue
99
43
}
100
-
tf, err := v.fetch(typesURL)
44
+
tf, err := cr.fetch(typesURL)
101
45
if err != nil {
102
-
v.warnings = append(v.warnings, fmt.Sprintf("failed to download types for %q: %v", key, err))
46
+
cr.warnings = append(cr.warnings, fmt.Sprintf("failed to download types for %q: %v", key, err))
103
47
continue
104
48
}
105
49
if f != nil {
···
109
53
110
54
// pass 2: write each fetched file to disk at host/dir/filename, rewriting
111
55
// imports to relative paths between vendor locations
112
-
for canon := range v.files {
113
-
if err := v.writeOne(canon); err != nil {
114
-
return v.warnings, fmt.Errorf("writing %s: %w", canon, err)
56
+
w := &writer{
57
+
config: c,
58
+
l: l,
59
+
written: make(map[layout.RelativePath]bool),
60
+
}
61
+
for canon, f := range cr.files {
62
+
if err := w.writeOne(canon, f); err != nil {
63
+
return cr.warnings, fmt.Errorf("writing %s: %w", canon, err)
115
64
}
116
65
}
117
66
···
121
70
if f == nil {
122
71
continue
123
72
}
124
-
importMap[key] = v.l.WebPath(f.path)
73
+
importMap[key] = l.WebPath(f.path)
125
74
126
75
// types: explicit config entry / x-typescript-types header / sidecar .d.ts > entry file itself
127
76
if f.types != nil && f.types != f {
···
131
80
}
132
81
}
133
82
134
-
if err := writeImportMap(c.Unpm.Out, importMap, c.Unpm.Verbose); err != nil {
135
-
return v.warnings, err
136
-
}
137
-
138
-
if err := writeTypesDts(c.Unpm.Out, typesMap, c.Unpm.Verbose); err != nil {
139
-
return v.warnings, err
140
-
}
141
-
142
-
for spec, files := range inspect.FindBareImports(c.Unpm.Out, c.Imports) {
143
-
fmt.Printf("\033[33mwarning:\033[0m %q is imported by %s but missing from import map\n", spec, strings.Join(files, ", "))
144
-
}
145
-
146
-
return v.warnings, nil
147
-
}
148
-
149
-
// recursively remove a directory's children while leaving pinned files,
150
-
// returning whether the cleaned directory should be deleted
151
-
func clean(config *cfg.Config, path string) (bool, error) {
152
-
// if the file is pinned, don't clean it
153
-
relPath, _ := filepath.Rel(config.Unpm.Out, path)
154
-
if config.IsPinned(filepath.ToSlash(relPath)) {
155
-
return false, nil
156
-
}
157
-
158
-
// if the path is a leaf file, it should be deleted
159
-
entries, err := os.ReadDir(path)
160
-
if err != nil {
161
-
if errors.Is(err, syscall.ENOTDIR) {
162
-
return true, nil
163
-
}
164
-
165
-
return false, err
166
-
}
167
-
168
-
// iterate through the directory, removing any files and now-empty child directories
169
-
empty := true
170
-
for _, entry := range entries {
171
-
child := filepath.Join(path, entry.Name())
172
-
delete, err := clean(config, child)
173
-
if err != nil {
174
-
return false, err
175
-
}
176
-
177
-
empty = empty && delete
178
-
if delete {
179
-
if err := os.Remove(child); err != nil {
180
-
return false, err
181
-
}
182
-
}
183
-
}
184
-
185
-
return empty, nil
186
-
}
187
-
188
-
// fetch downloads a URL, follows x-esm-path shims, caches the bytes in memory,
189
-
// and recursively enumerates the URLs referenced by the file. It returns the
190
-
// fetched file.
191
-
func (v *vendorer) fetch(rawURL string) (*file, error) {
192
-
var err error
193
-
194
-
if f, ok := v.aliases[rawURL]; ok {
195
-
return f, nil
196
-
}
197
-
if cached, ok := v.misses[rawURL]; ok {
198
-
return nil, cached
199
-
}
200
-
defer func() {
201
-
if err != nil {
202
-
v.misses[rawURL] = err
203
-
}
204
-
}()
205
-
206
-
u, err := url.Parse(rawURL)
207
-
if err != nil {
208
-
return nil, fmt.Errorf("parsing URL %s: %w", rawURL, err)
209
-
}
210
-
211
-
resp, err := http.Get(rawURL)
212
-
if err != nil {
213
-
return nil, fmt.Errorf("fetching %s: %w", rawURL, err)
214
-
}
215
-
defer resp.Body.Close()
216
-
217
-
if resp.StatusCode != http.StatusOK {
218
-
return nil, fmt.Errorf("fetching %s: status %d", rawURL, resp.StatusCode)
219
-
}
220
-
221
-
// warn if the content-type header does not match the file extension
222
-
// (e.g. a .js file served as text/html)
223
-
ct, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
224
-
if strings.HasSuffix(strings.ToLower(u.Path), ".d.ts") {
225
-
if ct != "text/plain" && ct != "application/typescript" {
226
-
v.warnings = append(v.warnings, fmt.Sprintf("%s: content-type %q does not match extension", rawURL, ct))
227
-
}
228
-
} else if expected, ok := knownContentTypes[strings.ToLower(path.Ext(u.Path))]; ok && !slices.Contains(expected, ct) {
229
-
v.warnings = append(v.warnings, fmt.Sprintf("%s: content-type %q does not match extension", rawURL, ct))
230
-
}
231
-
232
-
// esm.sh returns x-esm-path with the canonical resolved path;
233
-
// skip the shim and fetch the resolved module directly
234
-
if esmPath := resp.Header.Get("x-esm-path"); esmPath != "" {
235
-
canonURL := u.Scheme + "://" + u.Host + esmPath
236
-
f, err := v.fetch(canonURL)
237
-
if err != nil {
238
-
return nil, fmt.Errorf("downloading canonical path for %s: %w", rawURL, err)
239
-
}
240
-
v.aliases[rawURL] = f
241
-
242
-
// x-typescript-types is on this response, not the canonical one
243
-
v.attachTypesHeader(u, resp, rawURL, f)
244
-
245
-
return f, nil
246
-
}
247
-
248
-
body, err := io.ReadAll(resp.Body)
249
-
if err != nil {
250
-
return nil, fmt.Errorf("reading %s: %w", rawURL, err)
251
-
}
252
-
253
-
canon := u.String()
254
-
f := &file{
255
-
content: body,
256
-
path: layout.Rel(u),
257
-
deps: make(map[string]*file),
258
-
}
259
-
v.files[canon] = f
260
-
v.aliases[rawURL] = f
261
-
262
-
v.discover(u, f)
263
-
264
-
v.attachTypesHeader(u, resp, rawURL, f)
265
-
266
-
// if the response didn't already point us at types, try a "sidecar" .d.ts at
267
-
// the same URL with the extension replaced (e.g. foo.mjs -> foo.d.ts). 404s
268
-
// are expected and silent.
269
-
if f.types == nil {
270
-
if sURL := sidecarURL(u, f.filename()); sURL != "" {
271
-
if sf, err := v.fetch(sURL); err == nil {
272
-
f.types = sf
273
-
}
274
-
}
275
-
}
276
-
277
-
return f, nil
278
-
}
279
-
280
-
// discover enumerates the URLs a fetched file references — its imports and any
281
-
// source map — fetching each and recording the edges on f. Source maps are
282
-
// skipped; all other files use regex-based import scanning.
283
-
func (v *vendorer) discover(u *url.URL, f *file) {
284
-
if !isSourceMap(f.filename()) {
285
-
v.discoverRegex(u, f)
286
-
}
287
-
v.discoverSourceMap(u, f)
288
-
}
289
-
290
-
// discoverSourceMap follows a //# sourceMappingURL= comment, fetching the
291
-
// referenced map and recording it on f.sourceMap. Inline data: URLs are skipped.
292
-
func (v *vendorer) discoverSourceMap(u *url.URL, f *file) {
293
-
for _, mapPath := range imports.ScanSourceMaps(string(f.content)) {
294
-
ref, err := url.Parse(mapPath)
295
-
if err != nil {
296
-
continue
297
-
}
298
-
mapURL := u.ResolveReference(ref).String()
299
-
if sm, err := v.fetch(mapURL); err != nil {
300
-
v.warnings = append(v.warnings, fmt.Sprintf("failed to download source map %s: %v", mapURL, err))
301
-
} else {
302
-
f.sourceMap = sm
303
-
}
304
-
}
305
-
}
306
-
307
-
// isSourceMap reports whether filename is a .map sidecar.
308
-
func isSourceMap(filename string) bool {
309
-
return strings.HasSuffix(strings.ToLower(filename), ".map")
310
-
}
311
-
312
-
// attachTypesHeader follows the x-typescript-types header on resp and records
313
-
// the result on f.types. The header is authoritative, so it overwrites any
314
-
// sidecar already found. Errors are non-fatal — a warning is emitted.
315
-
func (v *vendorer) attachTypesHeader(u *url.URL, resp *http.Response, rawURL string, f *file) {
316
-
typesURL := resp.Header.Get("x-typescript-types")
317
-
if typesURL == "" {
318
-
return
319
-
}
320
-
if strings.HasPrefix(typesURL, "/") {
321
-
typesURL = u.Scheme + "://" + u.Host + typesURL
322
-
}
323
-
tf, err := v.fetch(typesURL)
324
-
if err != nil {
325
-
v.warnings = append(v.warnings, fmt.Sprintf("failed to download types for %s: %v", rawURL, err))
326
-
return
327
-
}
328
-
if f != nil {
329
-
f.types = tf
330
-
}
331
-
}
332
-
333
-
// isScript reports whether filename is a script file (JS or TS, but not .d.ts).
334
-
func isScript(filename string) bool {
335
-
f := strings.ToLower(filename)
336
-
if strings.HasSuffix(f, ".d.ts") {
337
-
return false
338
-
}
339
-
switch path.Ext(f) {
340
-
case ".ts", ".mts", ".tsx", ".jsx", ".js", ".mjs":
341
-
return true
83
+
if err := w.writeImportMap(importMap); err != nil {
84
+
return cr.warnings, err
342
85
}
343
-
return false
344
-
}
345
86
346
-
// sidecarURL returns the URL of the .d.ts file that may live alongside a script
347
-
// file (e.g. foo.mjs -> foo.d.ts), or "" if no sidecar lookup is appropriate.
348
-
func sidecarURL(u *url.URL, filename string) string {
349
-
if !isScript(filename) {
350
-
return ""
351
-
}
352
-
ext := path.Ext(u.Path)
353
-
if ext == "" {
354
-
return ""
355
-
}
356
-
s := *u
357
-
s.Path = strings.TrimSuffix(u.Path, ext) + ".d.ts"
358
-
return s.String()
359
-
}
360
-
361
-
// discoverRegex enumerates imports by scanning the source with a regex.
362
-
// Bare specifiers are skipped because imports.Resolve returns "" for anything
363
-
// without a scheme or a "/", "./", "../" prefix.
364
-
func (v *vendorer) discoverRegex(u *url.URL, f *file) {
365
-
for _, spec := range imports.Scan(string(f.content)) {
366
-
if _, seen := f.deps[spec]; seen {
367
-
continue
368
-
}
369
-
depURL := imports.Resolve(u, spec)
370
-
if depURL == "" {
371
-
continue
372
-
}
373
-
df, err := v.fetch(depURL)
374
-
if err != nil {
375
-
v.warnings = append(v.warnings, fmt.Sprintf("failed to download %s: %v", depURL, err))
376
-
continue
377
-
}
378
-
f.deps[spec] = df
87
+
if err := w.writeTypesDts(typesMap); err != nil {
88
+
return cr.warnings, err
379
89
}
380
-
}
381
90
382
-
func (v *vendorer) writeOne(canon string) error {
383
-
f := v.files[canon]
384
-
if f == nil {
385
-
return nil
91
+
if err := w.clean(); err != nil {
92
+
return cr.warnings, err
386
93
}
387
94
388
-
var content []byte
389
-
if isSourceMap(f.filename()) {
390
-
content = f.content
391
-
} else {
392
-
content = v.rewrite(f)
95
+
for spec, files := range inspect.FindBareImports(c.Unpm.Out, c.Imports) {
96
+
fmt.Printf("\033[33mwarning:\033[0m %q is imported by %s but missing from import map\n", spec, strings.Join(files, ", "))
393
97
}
394
-
return v.writeFile(canon, f.path, content)
395
-
}
396
98
397
-
func (v *vendorer) writeFile(canon string, rel layout.RelativePath, content []byte) error {
398
-
if v.config.IsPinned(string(rel)) {
399
-
if v.config.Unpm.Verbose {
400
-
fmt.Printf("%s -> %s (pinned)\n", canon, rel)
401
-
}
402
-
return nil
403
-
}
404
-
dest := v.l.DiskPath(rel)
405
-
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
406
-
return fmt.Errorf("creating directory %s: %w", filepath.Dir(dest), err)
407
-
}
408
-
if err := os.WriteFile(dest, content, 0o644); err != nil {
409
-
return fmt.Errorf("writing %s: %w", dest, err)
410
-
}
411
-
if v.config.Unpm.Verbose {
412
-
fmt.Printf("%s -> %s\n", canon, rel)
413
-
}
414
-
return nil
415
-
}
416
-
417
-
// rewrite substitutes import specs and the sourceMappingURL in f.content with
418
-
// paths relative to f's vendored location using the vendor locations of f's deps.
419
-
func (v *vendorer) rewrite(f *file) []byte {
420
-
rewrites := make(map[string]string, len(f.deps))
421
-
for spec, dep := range f.deps {
422
-
rewrites[spec] = layout.Relative(f.path, dep.path)
423
-
}
424
-
425
-
result := imports.Rewrite(string(f.content), func(spec string) (string, bool) {
426
-
rewritten, ok := rewrites[spec]
427
-
return rewritten, ok
428
-
})
429
-
430
-
if f.sourceMap != nil {
431
-
result = imports.RewriteSourceMap(result, func(string) string {
432
-
return layout.Relative(f.path, f.sourceMap.path)
433
-
})
434
-
}
435
-
436
-
return []byte(result)
437
-
}
438
-
439
-
func writeImportMap(outDir string, rewritten map[string]string, verbose bool) error {
440
-
// Build the imports object as JS key-value pairs
441
-
var entries []string
442
-
for key, val := range rewritten {
443
-
entries = append(entries, fmt.Sprintf(" %q: %q", key, val))
444
-
}
445
-
sort.Strings(entries)
446
-
447
-
js := fmt.Sprintf(`const importmap = document.createElement("script");
448
-
importmap.type = "importmap";
449
-
importmap.textContent = JSON.stringify({
450
-
imports: {
451
-
%s,
452
-
},
453
-
});
454
-
document.currentScript.after(importmap);
455
-
`, strings.Join(entries, ",\n"))
456
-
457
-
dest := filepath.Join(outDir, "importmap.js")
458
-
if err := os.WriteFile(dest, []byte(js), 0o644); err != nil {
459
-
return fmt.Errorf("writing importmap.js: %w", err)
460
-
}
461
-
462
-
if verbose {
463
-
fmt.Printf("wrote %s\n", dest)
464
-
}
465
-
466
-
// Also write importmap.json for use with Node or SSR
467
-
jsonData, err := json.MarshalIndent(map[string]any{
468
-
"imports": rewritten,
469
-
}, "", " ")
470
-
if err != nil {
471
-
return fmt.Errorf("marshaling importmap.json: %w", err)
472
-
}
473
-
474
-
jsonDest := filepath.Join(outDir, "importmap.json")
475
-
if err := os.WriteFile(jsonDest, append(jsonData, '\n'), 0o644); err != nil {
476
-
return fmt.Errorf("writing importmap.json: %w", err)
477
-
}
478
-
479
-
if verbose {
480
-
fmt.Printf("wrote %s\n", jsonDest)
481
-
}
482
-
return nil
483
-
}
484
-
485
-
func writeTypesDts(dir string, types map[string]string, verbose bool) error {
486
-
if len(types) == 0 {
487
-
return nil
488
-
}
489
-
490
-
paths := make(map[string][]string)
491
-
for key, path := range types {
492
-
paths[key] = []string{path}
493
-
}
494
-
495
-
data, err := json.MarshalIndent(map[string]any{
496
-
"compilerOptions": map[string]any{
497
-
"paths": paths,
498
-
},
499
-
"exclude": []string{"."},
500
-
}, "", " ")
501
-
if err != nil {
502
-
return fmt.Errorf("marshaling jsconfig.json: %w", err)
503
-
}
504
-
505
-
dest := filepath.Join(dir, "jsconfig.json")
506
-
if err := os.WriteFile(dest, append(data, '\n'), 0o644); err != nil {
507
-
return fmt.Errorf("writing jsconfig.json: %w", err)
508
-
}
509
-
510
-
if verbose {
511
-
fmt.Printf("wrote %s\n", dest)
512
-
}
513
-
return nil
99
+
return cr.warnings, nil
514
100
}
+220
internal/vendor/write.go
+220
internal/vendor/write.go
···
1
+
// This Source Code Form is subject to the terms of the Mozilla Public
2
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+
//
5
+
// Copyright (c) 2026 Jake Lazaroff https://tangled.org/jakelazaroff.com/unpm
6
+
7
+
package vendor
8
+
9
+
import (
10
+
"encoding/json"
11
+
"errors"
12
+
"fmt"
13
+
"io/fs"
14
+
"os"
15
+
"path/filepath"
16
+
"sort"
17
+
"strings"
18
+
19
+
"tangled.org/jakelazaroff.com/unpm/internal/cfg"
20
+
"tangled.org/jakelazaroff.com/unpm/internal/imports"
21
+
"tangled.org/jakelazaroff.com/unpm/internal/layout"
22
+
)
23
+
24
+
type writer struct {
25
+
config *cfg.Config
26
+
l layout.Layout
27
+
written map[layout.RelativePath]bool
28
+
}
29
+
30
+
func (w *writer) writeOne(canon string, f *file) error {
31
+
var content []byte
32
+
if isSourceMap(f.filename()) {
33
+
content = f.content
34
+
} else {
35
+
content = w.rewrite(f)
36
+
}
37
+
return w.writeFile(canon, f.path, content)
38
+
}
39
+
40
+
func (w *writer) writeFile(canon string, rel layout.RelativePath, content []byte) error {
41
+
if w.config.IsPinned(string(rel)) {
42
+
if w.config.Unpm.Verbose {
43
+
fmt.Printf("%s -> %s (pinned)\n", canon, rel)
44
+
}
45
+
return nil
46
+
}
47
+
dest := w.l.DiskPath(rel)
48
+
if err := writeBytes(dest, content); err != nil {
49
+
return fmt.Errorf("writing %s: %w", dest, err)
50
+
}
51
+
w.written[rel] = true
52
+
if w.config.Unpm.Verbose {
53
+
fmt.Printf("%s -> %s\n", canon, rel)
54
+
}
55
+
return nil
56
+
}
57
+
58
+
// rewrite substitutes import specs and the sourceMappingURL in f.content with
59
+
// paths relative to f's vendored location using the vendor locations of f's deps.
60
+
func (w *writer) rewrite(f *file) []byte {
61
+
rewrites := make(map[string]string, len(f.deps))
62
+
for spec, dep := range f.deps {
63
+
rewrites[spec] = layout.Relative(f.path, dep.path)
64
+
}
65
+
66
+
result := imports.Rewrite(string(f.content), func(spec string) (string, bool) {
67
+
rewritten, ok := rewrites[spec]
68
+
return rewritten, ok
69
+
})
70
+
71
+
if f.sourceMap != nil {
72
+
result = imports.RewriteSourceMap(result, func(string) string {
73
+
return layout.Relative(f.path, f.sourceMap.path)
74
+
})
75
+
}
76
+
77
+
return []byte(result)
78
+
}
79
+
80
+
func (w *writer) writeImportMap(rewritten map[string]string) error {
81
+
// Build the imports object as JS key-value pairs
82
+
var entries []string
83
+
for key, val := range rewritten {
84
+
entries = append(entries, fmt.Sprintf(" %q: %q", key, val))
85
+
}
86
+
sort.Strings(entries)
87
+
88
+
js := fmt.Sprintf(`const importmap = document.createElement("script");
89
+
importmap.type = "importmap";
90
+
importmap.textContent = JSON.stringify({
91
+
imports: {
92
+
%s,
93
+
},
94
+
});
95
+
document.currentScript.after(importmap);
96
+
`, strings.Join(entries, ",\n"))
97
+
98
+
if err := w.writeGenerated("importmap.js", []byte(js)); err != nil {
99
+
return fmt.Errorf("writing importmap.js: %w", err)
100
+
}
101
+
102
+
if w.config.Unpm.Verbose {
103
+
fmt.Printf("wrote %s\n", w.l.DiskPath("importmap.js"))
104
+
}
105
+
106
+
// Also write importmap.json for use with Node or SSR
107
+
jsonData, err := json.MarshalIndent(map[string]any{
108
+
"imports": rewritten,
109
+
}, "", " ")
110
+
if err != nil {
111
+
return fmt.Errorf("marshaling importmap.json: %w", err)
112
+
}
113
+
114
+
if err := w.writeGenerated("importmap.json", append(jsonData, '\n')); err != nil {
115
+
return fmt.Errorf("writing importmap.json: %w", err)
116
+
}
117
+
118
+
if w.config.Unpm.Verbose {
119
+
fmt.Printf("wrote %s\n", w.l.DiskPath("importmap.json"))
120
+
}
121
+
return nil
122
+
}
123
+
124
+
func (w *writer) writeTypesDts(types map[string]string) error {
125
+
if len(types) == 0 {
126
+
return nil
127
+
}
128
+
129
+
paths := make(map[string][]string)
130
+
for key, path := range types {
131
+
paths[key] = []string{path}
132
+
}
133
+
134
+
data, err := json.MarshalIndent(map[string]any{
135
+
"compilerOptions": map[string]any{
136
+
"paths": paths,
137
+
},
138
+
"exclude": []string{"."},
139
+
}, "", " ")
140
+
if err != nil {
141
+
return fmt.Errorf("marshaling jsconfig.json: %w", err)
142
+
}
143
+
144
+
if err := w.writeGenerated("jsconfig.json", append(data, '\n')); err != nil {
145
+
return fmt.Errorf("writing jsconfig.json: %w", err)
146
+
}
147
+
148
+
if w.config.Unpm.Verbose {
149
+
fmt.Printf("wrote %s\n", w.l.DiskPath("jsconfig.json"))
150
+
}
151
+
return nil
152
+
}
153
+
154
+
func (w *writer) writeGenerated(rel layout.RelativePath, content []byte) error {
155
+
if err := writeBytes(w.l.DiskPath(rel), content); err != nil {
156
+
return err
157
+
}
158
+
w.written[rel] = true
159
+
return nil
160
+
}
161
+
162
+
func writeBytes(dest string, content []byte) error {
163
+
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
164
+
return fmt.Errorf("creating directory %s: %w", filepath.Dir(dest), err)
165
+
}
166
+
return os.WriteFile(dest, content, 0o644)
167
+
}
168
+
169
+
// clean removes files that were not written by this run, preserving pinned
170
+
// files, then prunes any empty directories left behind.
171
+
func (w *writer) clean() error {
172
+
out := w.config.Unpm.Out
173
+
if _, err := os.Stat(out); errors.Is(err, fs.ErrNotExist) {
174
+
return nil
175
+
}
176
+
177
+
if err := filepath.WalkDir(out, func(p string, d fs.DirEntry, err error) error {
178
+
if err != nil || d.IsDir() {
179
+
return err
180
+
}
181
+
rel, err := filepath.Rel(out, p)
182
+
if err != nil {
183
+
return err
184
+
}
185
+
relPath := layout.RelativePath(filepath.ToSlash(rel))
186
+
if w.written[relPath] || w.config.IsPinned(string(relPath)) {
187
+
return nil
188
+
}
189
+
return os.Remove(p)
190
+
}); err != nil {
191
+
return err
192
+
}
193
+
194
+
return pruneEmptyDirs(out)
195
+
}
196
+
197
+
func pruneEmptyDirs(root string) error {
198
+
var dirs []string
199
+
if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
200
+
if err != nil {
201
+
return err
202
+
}
203
+
if d.IsDir() && p != root {
204
+
dirs = append(dirs, p)
205
+
}
206
+
return nil
207
+
}); err != nil {
208
+
return err
209
+
}
210
+
211
+
for i := len(dirs) - 1; i >= 0; i-- {
212
+
entries, err := os.ReadDir(dirs[i])
213
+
if err == nil && len(entries) == 0 {
214
+
if err := os.Remove(dirs[i]); err != nil {
215
+
return err
216
+
}
217
+
}
218
+
}
219
+
return nil
220
+
}