nandi.uk
Nix packages for third-party projects that don't ship their own flakes
1#!/usr/bin/env bash
2# Update packages/* from their upstream sources.
3# Intended for local use and weekly CI (see .github/workflows/update-packages.yml).
4#
5# Usage:
6# scripts/update-packages.sh # update all packages that have upstream.json
7# scripts/update-packages.sh vit # update only vit
8# scripts/update-packages.sh --check # report outdated packages; exit 1 if any
9set -euo pipefail
10
11ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
12cd "$ROOT"
13
14FAKE_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
15CHECK_ONLY=0
16PACKAGES=()
17
18log() { printf '==> %s\n' "$*"; }
19warn() { printf 'warning: %s\n' "$*" >&2; }
20die() { printf 'error: %s\n' "$*" >&2; exit 1; }
21
22need() {
23 command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
24}
25
26for arg in "$@"; do
27 case "$arg" in
28 --check) CHECK_ONLY=1 ;;
29 -h|--help)
30 sed -n '2,10p' "$0"
31 exit 0
32 ;;
33 -*)
34 die "unknown flag: $arg"
35 ;;
36 *)
37 PACKAGES+=("$arg")
38 ;;
39 esac
40done
41
42need curl
43need nix
44need nix-prefetch-url
45need python3
46
47if [[ ${#PACKAGES[@]} -eq 0 ]]; then
48 while IFS= read -r d; do
49 name="$(basename "$d")"
50 [[ -f "$d/upstream.json" && -f "$d/default.nix" ]] || continue
51 PACKAGES+=("$name")
52 done < <(find "$ROOT/packages" -mindepth 1 -maxdepth 1 -type d | sort)
53fi
54
55[[ ${#PACKAGES[@]} -gt 0 ]] || die "no packages to update"
56
57read_field() {
58 # read_field <file> <json-key>
59 python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get(sys.argv[2],""))' "$1" "$2"
60}
61
62current_version() {
63 python3 -c '
64import re, sys
65text = open(sys.argv[1]).read()
66m = re.search(r"version\s*=\s*\"([^\"]+)\"", text)
67if not m:
68 sys.exit("version not found in " + sys.argv[1])
69print(m.group(1))
70' "$1"
71}
72
73set_field_string() {
74 # set_field_string <file> <attr> <value> — replaces first attr = "..."
75 local file="$1" attr="$2" value="$3"
76 python3 -c '
77import re, sys
78path, attr, value = sys.argv[1], sys.argv[2], sys.argv[3]
79text = open(path).read()
80pat = re.compile(rf"({re.escape(attr)}\s*=\s*\")([^\"]*)(\")", re.M)
81new, n = pat.subn(rf"\g<1>{value}\g<3>", text, count=1)
82if n != 1:
83 sys.exit(f"failed to set {attr} in {path} (matches={n})")
84open(path, "w").write(new)
85' "$file" "$attr" "$value"
86}
87
88set_npm_deps_hash() {
89 local file="$1" value="$2"
90 python3 -c '
91import re, sys
92path, value = sys.argv[1], sys.argv[2]
93text = open(path).read()
94pat = re.compile(r"(npmDepsHash\s*=\s*)([^;]+)(;)")
95new, n = pat.subn(rf"\g<1>\"{value}\"\g<3>", text, count=1)
96if n != 1:
97 sys.exit(f"failed to set npmDepsHash in {path} (matches={n})")
98open(path, "w").write(new)
99' "$file" "$value"
100}
101
102sri_from_url_unpack() {
103 local url="$1"
104 local base32
105 base32="$(nix-prefetch-url --unpack "$url" 2>/dev/null)"
106 nix hash convert --hash-algo sha256 --to sri "$base32"
107}
108
109latest_github_tag() {
110 # latest_github_tag <owner/repo> <prefix>
111 local repo="$1" prefix="$2"
112 local url="https://api.github.com/repos/${repo}/tags?per_page=30"
113 local args=(-fsSL)
114 if [[ -n "${GITHUB_TOKEN:-}" ]]; then
115 args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
116 fi
117 curl "${args[@]}" "$url" | python3 -c '
118import json, re, sys
119prefix = sys.argv[1]
120tags = json.load(sys.stdin)
121# Prefer semver-ish tags matching prefix
122pat = re.compile(r"^" + re.escape(prefix) + r"(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)$")
123versions = []
124for t in tags:
125 name = t.get("name") or ""
126 m = pat.match(name)
127 if m:
128 versions.append((m.group(1), name))
129if not versions:
130 sys.exit("no matching tags for prefix " + prefix)
131# sort by version components
132def key(v):
133 ver = re.split(r"[+-]", v[0], maxsplit=1)[0]
134 return [int(x) for x in ver.split(".")]
135versions.sort(key=key)
136print(versions[-1][0])
137' "$prefix"
138}
139
140version_gt() {
141 # version_gt a b => true if a > b (semver-ish, numeric)
142 python3 -c '
143import re, sys
144def parts(v):
145 core = re.split(r"[+-]", v, maxsplit=1)[0]
146 return [int(x) for x in core.split(".")]
147a, b = sys.argv[1], sys.argv[2]
148sys.exit(0 if parts(a) > parts(b) else 1)
149' "$1" "$2"
150}
151
152capture_npm_deps_hash() {
153 # Build with fake hash; parse the "got:" line from the FOD mismatch.
154 local attr="$1"
155 local log
156 log="$(mktemp)"
157 set +e
158 nix build "path:${ROOT}#${attr}" --print-build-logs >"$log" 2>&1
159 local status=$?
160 set -e
161 local got
162 got="$(python3 -c '
163import re, sys
164text = open(sys.argv[1]).read()
165# Prefer the last "got:" line (npm-deps FOD).
166matches = re.findall(r"got:\s+(sha256-[A-Za-z0-9+/=]+)", text)
167if not matches:
168 sys.exit(1)
169print(matches[-1])
170' "$log" 2>/dev/null || true)"
171 if [[ -z "$got" ]]; then
172 warn "failed to capture npmDepsHash for ${attr}; build log:"
173 tail -n 80 "$log" >&2 || true
174 rm -f "$log"
175 return 1
176 fi
177 rm -f "$log"
178 printf '%s\n' "$got"
179 return 0
180}
181
182verify_build() {
183 local attr="$1"
184 log "verifying nix build .#${attr}"
185 nix build "path:${ROOT}#${attr}" --print-build-logs
186}
187
188regenerate_npm_lock() {
189 # regenerate_npm_lock <owner/repo> <version> <dest-lock>
190 local repo="$1" version="$2" dest="$3"
191 need npm
192 local tmp archive dir
193 tmp="$(mktemp -d)"
194 archive="${tmp}/src.tar.gz"
195 dir="${tmp}/src"
196 curl -fsSL "https://github.com/${repo}/archive/refs/tags/v${version}.tar.gz" -o "$archive"
197 mkdir -p "$dir"
198 tar -xzf "$archive" -C "$dir" --strip-components=1
199 (
200 cd "$dir"
201 rm -f package-lock.json bun.lock
202 npm install --package-lock-only --ignore-scripts
203 )
204 cp "$dir/package-lock.json" "$dest"
205 rm -rf "$tmp"
206}
207
208set_fetch_from_github_field() {
209 # set_fetch_from_github_field <file> <attr> <value>
210 # Replaces attr = "..." inside the first fetchFromGitHub { ... } block.
211 local file="$1" attr="$2" value="$3"
212 python3 -c '
213import re, sys
214path, attr, value = sys.argv[1], sys.argv[2], sys.argv[3]
215text = open(path).read()
216pat = re.compile(
217 rf"(fetchFromGitHub\s*\{{[^}}]*?{re.escape(attr)}\s*=\s*\")([^\"]+)(\")",
218 re.S,
219)
220new, n = pat.subn(rf"\g<1>{value}\g<3>", text, count=1)
221if n != 1:
222 sys.exit(f"failed to set fetchFromGitHub.{attr} in {path} (matches={n})")
223open(path, "w").write(new)
224' "$file" "$attr" "$value"
225}
226
227current_rev() {
228 python3 -c '
229import re, sys
230text = open(sys.argv[1]).read()
231m = re.search(r"fetchFromGitHub\s*\{[^}]*?rev\s*=\s*\"([^\"]+)\"", text, re.S)
232if not m:
233 sys.exit("rev not found in " + sys.argv[1])
234print(m.group(1))
235' "$1"
236}
237
238latest_github_commit() {
239 # latest_github_commit <owner/repo> <branch>
240 # prints: <sha> <YYYY-MM-DD>
241 local repo="$1" branch="$2"
242 local url="https://api.github.com/repos/${repo}/commits/${branch}"
243 local args=(-fsSL)
244 if [[ -n "${GITHUB_TOKEN:-}" ]]; then
245 args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
246 fi
247 curl "${args[@]}" "$url" | python3 -c '
248import json, sys
249c = json.load(sys.stdin)
250sha = c["sha"]
251date = c["commit"]["committer"]["date"][:10]
252print(sha, date)
253'
254}
255
256sri_from_url_file() {
257 local url="$1"
258 # nix-prefetch-url (no --unpack) for plain files (gems, etc.)
259 local base32
260 base32="$(nix-prefetch-url "$url" 2>/dev/null)"
261 nix hash convert --hash-algo sha256 --to sri "$base32"
262}
263
264refresh_vendored_gems() {
265 # refresh_vendored_gems <default.nix> <owner/repo> <rev>
266 # If the package vendors prism/rbs gems, re-read versions from upstream
267 # Makefile at that rev and refresh fetchurl hashes when they change.
268 local default_nix="$1" repo="$2" rev="$3"
269 python3 -c '
270import re, sys
271text = open(sys.argv[1]).read()
272sys.exit(0 if re.search(r"prismVersion\s*=", text) else 1)
273' "$default_nix" 2>/dev/null || return 0
274
275 local makefile
276 makefile="$(mktemp)"
277 local args=(-fsSL)
278 if [[ -n "${GITHUB_TOKEN:-}" ]]; then
279 args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
280 fi
281 curl "${args[@]}" \
282 "https://raw.githubusercontent.com/${repo}/${rev}/Makefile" \
283 -o "$makefile"
284
285 local prism_ver rbs_ver
286 prism_ver="$(python3 -c '
287import re, sys
288text = open(sys.argv[1]).read()
289m = re.search(r"PRISM_VERSION\s*\?=\s*(\S+)", text)
290print(m.group(1) if m else "")
291' "$makefile")"
292 rbs_ver="$(python3 -c '
293import re, sys
294text = open(sys.argv[1]).read()
295m = re.search(r"RBS_VERSION\s*\?=\s*(\S+)", text)
296print(m.group(1) if m else "")
297' "$makefile")"
298 rm -f "$makefile"
299
300 [[ -n "$prism_ver" && -n "$rbs_ver" ]] || {
301 warn "could not parse PRISM_VERSION/RBS_VERSION from upstream Makefile"
302 return 0
303 }
304
305 local cur_prism cur_rbs
306 cur_prism="$(python3 -c '
307import re, sys
308m = re.search(r"prismVersion\s*=\s*\"([^\"]+)\"", open(sys.argv[1]).read())
309print(m.group(1) if m else "")
310' "$default_nix")"
311 cur_rbs="$(python3 -c '
312import re, sys
313m = re.search(r"rbsVersion\s*=\s*\"([^\"]+)\"", open(sys.argv[1]).read())
314print(m.group(1) if m else "")
315' "$default_nix")"
316
317 if [[ "$prism_ver" != "$cur_prism" ]]; then
318 log " prism ${cur_prism} -> ${prism_ver}"
319 set_field_string "$default_nix" prismVersion "$prism_ver"
320 local prism_hash
321 prism_hash="$(sri_from_url_file "https://rubygems.org/gems/prism-${prism_ver}.gem")"
322 python3 -c '
323import re, sys
324path, ver, h = sys.argv[1], sys.argv[2], sys.argv[3]
325text = open(path).read()
326# first fetchurl after prismGem =
327pat = re.compile(
328 r"(prismGem\s*=\s*fetchurl\s*\{[^}]*?hash\s*=\s*\")([^\"]+)(\")",
329 re.S,
330)
331new, n = pat.subn(rf"\g<1>{h}\g<3>", text, count=1)
332if n != 1:
333 sys.exit(f"failed to set prismGem hash (matches={n})")
334open(path, "w").write(new)
335' "$default_nix" "$prism_ver" "$prism_hash"
336 log " prism gem hash ${prism_hash}"
337 fi
338
339 if [[ "$rbs_ver" != "$cur_rbs" ]]; then
340 log " rbs ${cur_rbs} -> ${rbs_ver}"
341 set_field_string "$default_nix" rbsVersion "$rbs_ver"
342 local rbs_hash
343 rbs_hash="$(sri_from_url_file "https://rubygems.org/gems/rbs-${rbs_ver}.gem")"
344 python3 -c '
345import re, sys
346path, h = sys.argv[1], sys.argv[2]
347text = open(path).read()
348pat = re.compile(
349 r"(rbsGem\s*=\s*fetchurl\s*\{[^}]*?hash\s*=\s*\")([^\"]+)(\")",
350 re.S,
351)
352new, n = pat.subn(rf"\g<1>{h}\g<3>", text, count=1)
353if n != 1:
354 sys.exit(f"failed to set rbsGem hash (matches={n})")
355open(path, "w").write(new)
356' "$default_nix" "$rbs_hash"
357 log " rbs gem hash ${rbs_hash}"
358 fi
359}
360
361update_npm_github() {
362 local name="$1"
363 local pkg_dir="$ROOT/packages/${name}"
364 local default_nix="$pkg_dir/default.nix"
365 local upstream="$pkg_dir/upstream.json"
366 local repo tag_prefix
367 repo="$(read_field "$upstream" github)"
368 tag_prefix="$(read_field "$upstream" tag_prefix)"
369 [[ -n "$repo" ]] || die "${name}: upstream.json missing github"
370 tag_prefix="${tag_prefix:-v}"
371
372 local cur latest
373 cur="$(current_version "$default_nix")"
374 log "${name}: current=${cur}"
375 latest="$(latest_github_tag "$repo" "$tag_prefix")"
376 log "${name}: latest upstream=${latest}"
377
378 if ! version_gt "$latest" "$cur"; then
379 log "${name}: already up to date"
380 return 0
381 fi
382
383 if [[ "$CHECK_ONLY" -eq 1 ]]; then
384 log "${name}: OUTDATED (${cur} -> ${latest})"
385 return 10
386 fi
387
388 log "${name}: updating ${cur} -> ${latest}"
389 set_field_string "$default_nix" version "$latest"
390
391 local src_url src_hash
392 src_url="https://github.com/${repo}/archive/refs/tags/${tag_prefix}${latest}.tar.gz"
393 log "${name}: prefetching src ${src_url}"
394 src_hash="$(sri_from_url_unpack "$src_url")"
395 set_fetch_from_github_field "$default_nix" hash "$src_hash"
396 log "${name}: src hash ${src_hash}"
397
398 if [[ -f "$pkg_dir/package-lock.json" ]]; then
399 log "${name}: regenerating package-lock.json"
400 regenerate_npm_lock "$repo" "$latest" "$pkg_dir/package-lock.json"
401 fi
402
403 log "${name}: computing npmDepsHash"
404 set_npm_deps_hash "$default_nix" "$FAKE_HASH"
405 local npm_hash
406 npm_hash="$(capture_npm_deps_hash "$name")"
407 set_npm_deps_hash "$default_nix" "$npm_hash"
408 log "${name}: npmDepsHash ${npm_hash}"
409
410 verify_build "$name"
411 log "${name}: updated successfully to ${latest}"
412}
413
414latest_github_tag_with_suffix() {
415 # latest_github_tag_with_suffix <owner/repo> <prefix> <suffix> [prerelease]
416 # e.g. prefix=v suffix=-pre => matches v1.12.0-pre, prints 1.12.0-pre
417 # When prerelease=1, match any vX.Y.Z-<pre> tag (rc1, beta, …) and ignore suffix.
418 local repo="$1" prefix="$2" suffix="$3" prerelease="${4:-0}"
419 local url="https://api.github.com/repos/${repo}/releases?per_page=30"
420 local args=(-fsSL)
421 if [[ -n "${GITHUB_TOKEN:-}" ]]; then
422 args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
423 fi
424 curl "${args[@]}" "$url" | python3 -c '
425import json, re, sys
426prefix, suffix, prerelease = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
427releases = json.load(sys.stdin)
428if prerelease:
429 # Any prerelease/build tag: v1.18.0-rc1, v1.18.0-beta.2, …
430 pat = re.compile(
431 r"^" + re.escape(prefix) + r"(\d+\.\d+\.\d+[-+][0-9A-Za-z.-]+)$"
432 )
433else:
434 pat = re.compile(
435 r"^" + re.escape(prefix) + r"(\d+\.\d+\.\d+)" + re.escape(suffix) + r"$"
436 )
437versions = []
438for r in releases:
439 if r.get("draft"):
440 continue
441 name = r.get("tag_name") or ""
442 m = pat.match(name)
443 if m:
444 if prerelease:
445 versions.append(m.group(1))
446 else:
447 versions.append(m.group(1) + suffix)
448if not versions:
449 kind = "prerelease" if prerelease else "prefix=%r suffix=%r" % (prefix, suffix)
450 sys.exit("no matching release tags for " + kind)
451def key(v):
452 m = re.match(r"^(\d+\.\d+\.\d+)(?:-([0-9A-Za-z.-]+))?(?:\+.*)?$", v)
453 core = [int(x) for x in m.group(1).split(".")]
454 pre = m.group(2) or ""
455 # Prefer higher core; among same core, sort pre parts (rc2 > rc1).
456 pre_parts = re.findall(r"\d+|[A-Za-z]+", pre)
457 pre_key = [int(p) if p.isdigit() else p.lower() for p in pre_parts]
458 return (core, pre_key)
459versions.sort(key=key)
460print(versions[-1])
461' "$prefix" "$suffix" "$prerelease"
462}
463
464set_fetchurl_hash() {
465 # set_fetchurl_hash <file> <value>
466 # Replaces hash = "..." inside the first fetchurl { ... } block.
467 local file="$1" value="$2"
468 python3 -c '
469import re, sys
470path, value = sys.argv[1], sys.argv[2]
471text = open(path).read()
472pat = re.compile(
473 r"(fetchurl\s*\{[^}]*?hash\s*=\s*\")([^\"]+)(\")",
474 re.S,
475)
476new, n = pat.subn(rf"\g<1>{value}\g<3>", text, count=1)
477if n != 1:
478 sys.exit(f"failed to set fetchurl.hash in {path} (matches={n})")
479open(path, "w").write(new)
480' "$file" "$value"
481}
482
483update_github_release_binary() {
484 # Official prebuilt release assets.
485 # version in default.nix is the tag without the leading "v" (e.g. 1.12.0-pre).
486 #
487 # Single-asset (zed-preview):
488 # asset: "zed-linux-x86_64.tar.gz"
489 # Multi-platform (whetuu) — rewrites sources = { … } like url-manifest-binary:
490 # platforms: { "x86_64-linux": "whetuu-v{version}-x86_64-linux-musl.tar.gz", … }
491 # {version} is substituted with the bare version (no tag prefix).
492 local name="$1"
493 local pkg_dir="$ROOT/packages/${name}"
494 local default_nix="$pkg_dir/default.nix"
495 local upstream="$pkg_dir/upstream.json"
496 local repo tag_prefix tag_suffix asset
497 repo="$(read_field "$upstream" github)"
498 tag_prefix="$(read_field "$upstream" tag_prefix)"
499 tag_suffix="$(read_field "$upstream" tag_suffix)"
500 asset="$(read_field "$upstream" asset)"
501 [[ -n "$repo" ]] || die "${name}: upstream.json missing github"
502 tag_prefix="${tag_prefix:-v}"
503 tag_suffix="${tag_suffix:-}"
504
505 local has_platforms=0
506 if python3 -c '
507import json, sys
508u = json.load(open(sys.argv[1]))
509sys.exit(0 if u.get("platforms") else 1)
510' "$upstream" 2>/dev/null; then
511 has_platforms=1
512 fi
513
514 local tag_prerelease=0
515 if python3 -c '
516import json, sys
517u = json.load(open(sys.argv[1]))
518sys.exit(0 if u.get("tag_prerelease") else 1)
519' "$upstream" 2>/dev/null; then
520 tag_prerelease=1
521 fi
522
523 if [[ "$has_platforms" -eq 0 && -z "$asset" ]]; then
524 die "${name}: upstream.json needs either asset or platforms"
525 fi
526
527 local cur latest
528 cur="$(current_version "$default_nix")"
529 log "${name}: current=${cur}"
530 latest="$(latest_github_tag_with_suffix "$repo" "$tag_prefix" "$tag_suffix" "$tag_prerelease")"
531 log "${name}: latest upstream=${latest}"
532
533 if [[ "$latest" == "$cur" ]]; then
534 log "${name}: already up to date"
535 return 0
536 fi
537
538 # Semver compare on the numeric core (ignore -pre etc.)
539 local cur_core latest_core
540 cur_core="${cur%%-*}"; cur_core="${cur_core%%+*}"
541 latest_core="${latest%%-*}"; latest_core="${latest_core%%+*}"
542 if ! version_gt "$latest_core" "$cur_core" && [[ "$latest" != "$cur" ]]; then
543 # Same core version but different suffix, or non-monotonic tag: still update
544 # when the full tag string differs (handled above). If latest core is older,
545 # skip to avoid downgrades.
546 if ! version_gt "$latest_core" "$cur_core" && [[ "$latest_core" != "$cur_core" ]]; then
547 log "${name}: latest core ${latest_core} is not newer than ${cur_core}; skipping"
548 return 0
549 fi
550 fi
551
552 if [[ "$CHECK_ONLY" -eq 1 ]]; then
553 log "${name}: OUTDATED (${cur} -> ${latest})"
554 return 10
555 fi
556
557 log "${name}: updating ${cur} -> ${latest}"
558 set_field_string "$default_nix" version "$latest"
559
560 if [[ "$has_platforms" -eq 1 ]]; then
561 local updated_sources
562 updated_sources="$(python3 -c '
563import json, re, subprocess, sys
564
565upstream = json.load(open(sys.argv[1]))
566repo = upstream["github"]
567prefix = upstream.get("tag_prefix") or "v"
568version = sys.argv[2]
569platforms = upstream["platforms"]
570if not platforms:
571 sys.exit("platforms map is empty")
572
573out = {}
574for nix_system, asset_tmpl in platforms.items():
575 asset = asset_tmpl.replace("{version}", version)
576 url = f"https://github.com/{repo}/releases/download/{prefix}{version}/{asset}"
577 base32 = subprocess.check_output(
578 ["nix-prefetch-url", url], text=True
579 ).strip().splitlines()[-1]
580 sri = subprocess.check_output(
581 ["nix", "hash", "convert", "--hash-algo", "sha256", "--to", "sri", base32],
582 text=True,
583 ).strip()
584 out[nix_system] = {"url": url, "hash": sri}
585 print(f" {nix_system}: {sri}", file=sys.stderr)
586print(json.dumps(out))
587' "$upstream" "$latest")"
588
589 python3 -c '
590import json, re, sys
591
592path, sources_json = sys.argv[1], sys.argv[2]
593sources = json.loads(sources_json)
594text = open(path).read()
595
596order = ["x86_64-linux", "aarch64-linux", "x86_64-darwin", "aarch64-darwin"]
597keys = [k for k in order if k in sources] + sorted(k for k in sources if k not in order)
598
599def fmt_entry(sysname):
600 e = sources[sysname]
601 return (
602 f" {sysname} = {{\n"
603 f" url = \"{e['url']}\";\n"
604 f" hash = \"{e['hash']}\";\n"
605 f" }};"
606 )
607
608block = "sources = {\n" + "\n".join(fmt_entry(k) for k in keys) + "\n };"
609pat = re.compile(r"sources\s*=\s*\{.*?\n \};", re.S)
610new, n = pat.subn(block, text, count=1)
611if n != 1:
612 sys.exit(f"failed to rewrite sources block in {path} (matches={n})")
613open(path, "w").write(new)
614' "$default_nix" "$updated_sources"
615 log "${name}: sources refreshed"
616 else
617 local src_url src_hash
618 src_url="https://github.com/${repo}/releases/download/${tag_prefix}${latest}/${asset}"
619 log "${name}: prefetching ${src_url}"
620 src_hash="$(sri_from_url_file "$src_url")"
621 set_fetchurl_hash "$default_nix" "$src_hash"
622 log "${name}: src hash ${src_hash}"
623 fi
624
625 verify_build "$name"
626 log "${name}: updated successfully to ${latest}"
627}
628
629update_url_manifest_binary() {
630 # Prebuilt multi-platform binaries discovered via a version manifest URL.
631 # upstream.json:
632 # type: url-manifest-binary
633 # manifest_url: "https://…/latest-{platform}.json" ({platform} substituted)
634 # platforms: { "x86_64-linux": "linux-amd64", … }
635 #
636 # default.nix is expected to declare:
637 # version = "…";
638 # sources = { <nix-system> = { url = "…"; hash = "…"; }; … };
639 local name="$1"
640 local pkg_dir="$ROOT/packages/${name}"
641 local default_nix="$pkg_dir/default.nix"
642 local upstream="$pkg_dir/upstream.json"
643 local manifest_url
644 manifest_url="$(read_field "$upstream" manifest_url)"
645 [[ -n "$manifest_url" ]] || die "${name}: upstream.json missing manifest_url"
646
647 local cur latest
648 cur="$(current_version "$default_nix")"
649 log "${name}: current=${cur}"
650
651 # Fetch all platform manifests; require a single shared version.
652 local platform_data
653 platform_data="$(python3 -c '
654import json, os, re, subprocess, sys, urllib.request
655
656upstream = json.load(open(sys.argv[1]))
657manifest_tmpl = upstream["manifest_url"]
658platforms = upstream["platforms"]
659if not platforms:
660 sys.exit("platforms map is empty")
661
662entries = {}
663versions = set()
664for nix_system, platform in platforms.items():
665 url = manifest_tmpl.replace("{platform}", platform)
666 with urllib.request.urlopen(url) as r:
667 m = json.load(r)
668 ver = m.get("version") or ""
669 bin_url = m.get("url") or ""
670 if not ver or not bin_url:
671 sys.exit(f"invalid manifest for {platform}: {m!r}")
672 versions.add(ver)
673 entries[nix_system] = {"url": bin_url, "platform": platform}
674if len(versions) != 1:
675 sys.exit(f"platform versions disagree: {sorted(versions)}")
676version = versions.pop()
677print(json.dumps({"version": version, "entries": entries}))
678' "$upstream")"
679
680 latest="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["version"])' <<<"$platform_data")"
681 log "${name}: latest upstream=${latest}"
682
683 if [[ "$latest" == "$cur" ]]; then
684 # Still refresh hashes if any platform URL/hash drifted without a version bump.
685 # Compare by re-prefetching only when check mode is off and we force-check
686 # via full equality of the sources block after a dry rewrite.
687 if [[ "$CHECK_ONLY" -eq 1 ]]; then
688 log "${name}: already up to date"
689 return 0
690 fi
691 # Fall through only if sources need rewriting (handled below with hash refresh).
692 fi
693
694 if [[ "$latest" != "$cur" ]]; then
695 if [[ "$CHECK_ONLY" -eq 1 ]]; then
696 log "${name}: OUTDATED (${cur} -> ${latest})"
697 return 10
698 fi
699 log "${name}: updating ${cur} -> ${latest}"
700 set_field_string "$default_nix" version "$latest"
701 elif [[ "$CHECK_ONLY" -eq 1 ]]; then
702 log "${name}: already up to date"
703 return 0
704 fi
705
706 # Prefetch each platform binary and rewrite the sources = { … } block.
707 local updated_sources
708 updated_sources="$(python3 -c '
709import json, re, subprocess, sys
710
711platform_data = json.loads(sys.argv[1])
712entries = platform_data["entries"]
713out = {}
714for nix_system, e in entries.items():
715 url = e["url"]
716 base32 = subprocess.check_output(
717 ["nix-prefetch-url", url], text=True
718 ).strip().splitlines()[-1]
719 sri = subprocess.check_output(
720 ["nix", "hash", "convert", "--hash-algo", "sha256", "--to", "sri", base32],
721 text=True,
722 ).strip()
723 out[nix_system] = {"url": url, "hash": sri}
724 print(f" {nix_system}: {sri}", file=sys.stderr)
725print(json.dumps(out))
726' "$platform_data")"
727
728 python3 -c '
729import json, re, sys
730
731path, sources_json = sys.argv[1], sys.argv[2]
732sources = json.loads(sources_json)
733text = open(path).read()
734
735# Prefer a stable system order.
736order = ["x86_64-linux", "aarch64-linux", "x86_64-darwin", "aarch64-darwin"]
737keys = [k for k in order if k in sources] + sorted(k for k in sources if k not in order)
738
739def fmt_entry(sysname):
740 e = sources[sysname]
741 return (
742 f" {sysname} = {{\n"
743 f" url = \"{e['url']}\";\n"
744 f" hash = \"{e['hash']}\";\n"
745 f" }};"
746 )
747
748block = "sources = {\n" + "\n".join(fmt_entry(k) for k in keys) + "\n };"
749pat = re.compile(r"sources\s*=\s*\{.*?\n \};", re.S)
750new, n = pat.subn(block, text, count=1)
751if n != 1:
752 sys.exit(f"failed to rewrite sources block in {path} (matches={n})")
753open(path, "w").write(new)
754' "$default_nix" "$updated_sources"
755
756 log "${name}: sources refreshed"
757
758 verify_build "$name"
759 log "${name}: updated successfully to ${latest}"
760}
761
762update_github_unstable() {
763 # Track tip of a branch for projects without release tags.
764 # version format: 0-unstable-YYYY-MM-DD
765 local name="$1"
766 local pkg_dir="$ROOT/packages/${name}"
767 local default_nix="$pkg_dir/default.nix"
768 local upstream="$pkg_dir/upstream.json"
769 local repo branch
770 repo="$(read_field "$upstream" github)"
771 branch="$(read_field "$upstream" branch)"
772 [[ -n "$repo" ]] || die "${name}: upstream.json missing github"
773 branch="${branch:-master}"
774
775 local cur_rev cur_ver latest_sha latest_date latest_ver
776 cur_rev="$(current_rev "$default_nix")"
777 cur_ver="$(current_version "$default_nix")"
778 log "${name}: current=${cur_ver} (rev ${cur_rev:0:12})"
779
780 read -r latest_sha latest_date < <(latest_github_commit "$repo" "$branch")
781 latest_ver="0-unstable-${latest_date}"
782 log "${name}: latest ${branch}=${latest_sha:0:12} (${latest_date})"
783
784 if [[ "$latest_sha" == "$cur_rev" ]]; then
785 log "${name}: already up to date"
786 return 0
787 fi
788
789 if [[ "$CHECK_ONLY" -eq 1 ]]; then
790 log "${name}: OUTDATED (${cur_rev:0:12} -> ${latest_sha:0:12})"
791 return 10
792 fi
793
794 log "${name}: updating ${cur_rev:0:12} -> ${latest_sha:0:12}"
795 set_field_string "$default_nix" version "$latest_ver"
796 set_fetch_from_github_field "$default_nix" rev "$latest_sha"
797
798 local src_url src_hash
799 src_url="https://github.com/${repo}/archive/${latest_sha}.tar.gz"
800 log "${name}: prefetching src ${src_url}"
801 src_hash="$(sri_from_url_unpack "$src_url")"
802 set_fetch_from_github_field "$default_nix" hash "$src_hash"
803 log "${name}: src hash ${src_hash}"
804
805 refresh_vendored_gems "$default_nix" "$repo" "$latest_sha"
806
807 verify_build "$name"
808 log "${name}: updated successfully to ${latest_ver} (${latest_sha:0:12})"
809}
810
811UPDATED=0
812OUTDATED=0
813
814for name in "${PACKAGES[@]}"; do
815 pkg_dir="$ROOT/packages/${name}"
816 [[ -d "$pkg_dir" ]] || die "unknown package: ${name}"
817 [[ -f "$pkg_dir/upstream.json" ]] || die "${name}: missing packages/${name}/upstream.json"
818 [[ -f "$pkg_dir/default.nix" ]] || die "${name}: missing packages/${name}/default.nix"
819
820 type="$(read_field "$pkg_dir/upstream.json" type)"
821 status=0
822 case "$type" in
823 npm-github)
824 update_npm_github "$name" || status=$?
825 ;;
826 github-unstable)
827 update_github_unstable "$name" || status=$?
828 ;;
829 github-release-binary)
830 update_github_release_binary "$name" || status=$?
831 ;;
832 url-manifest-binary)
833 update_url_manifest_binary "$name" || status=$?
834 ;;
835 *)
836 die "${name}: unsupported upstream type '${type}'"
837 ;;
838 esac
839 if [[ $status -eq 10 ]]; then
840 OUTDATED=$((OUTDATED + 1))
841 elif [[ $status -ne 0 ]]; then
842 exit "$status"
843 else
844 # detect if files changed for this package
845 if [[ "$CHECK_ONLY" -eq 0 ]] && ! git -C "$ROOT" diff --quiet -- "packages/${name}" 2>/dev/null; then
846 UPDATED=$((UPDATED + 1))
847 fi
848 fi
849done
850
851if [[ "$CHECK_ONLY" -eq 1 ]]; then
852 if [[ "$OUTDATED" -gt 0 ]]; then
853 log "${OUTDATED} package(s) outdated"
854 exit 1
855 fi
856 log "all packages up to date"
857 exit 0
858fi
859
860log "done (${UPDATED} package dir(s) with local changes)"