utensil.tngl.sh
Utensil's Zettelkasten-style forest of evergreen notes on math and tech.
utensil.tngl.sh/forest/
1engine: nixery
2
3# Forest render on tangled Spindle: pull the three pre-built closures from
4# R2 (tectonic + forester + wasm-pkgs), unpack/import, run `bash ./build.sh`,
5# push `./output/forest/` to the `site` branch → utensil.tngl.sh/forest.
6#
7# build-depot.yml uploads the closures + manifest.json under R2's forest/
8# subdir; this consumer hardcodes only R2_PUBLIC_URL + uses jq to discover
9# everything else. Same pattern as blog/.tangled/workflows/render.yml.
10when:
11 - event: ["push"]
12 branch: main
13
14dependencies:
15 nixpkgs:
16 - bash
17 - coreutils
18 - gnugrep
19 - findutils
20 - gawk
21 - gnused # sed used by build.sh + the dvisvgm shim writer
22 - mupdf # iter 45 — provides `mutool` binary; dvisvgm 3.6's
23 # PDFHandler uses mutool as a PDF backend when libgs's
24 # version is rejected by its allowlist (nixery's gs
25 # 10.07 is > 10.01). With mutool on PATH,
26 # PDFHandler::available() returns true, dvisvgm skips
27 # the gs check, and produces SVG with text/fonts
28 # (preserves --font-format=woff for tikz figures).
29 - poppler-utils # pdftocairo — kept as a redundant fallback path
30 # (was the primary in iter 41e–44g; superseded by
31 # mupdf in iter 45 because pdftocairo strips text).
32 - fontconfig # XeTeX (tectonic uses xetex internally) calls fontconfig
33 # for font lookup; without it lize.sh's tectonic step
34 # prints "Fontconfig error: Cannot load default config file"
35 # and subsequent package loading (fontspec → todonotes)
36 # silently fails to define \todostyle. iter 43.
37 - git
38 - openssh
39 - cacert
40 - gcc
41 - curl
42 - gnutar
43 - gzip
44 - xz # tar -J for tlnet archive todonotes.tar.xz (iter 44c)
45 - patchelf
46 - glibc
47 - zstd # decompress the closure NARs / tar
48 - nix # nix-store --import for tectonic + forester closures
49 - jq # parse manifest.json from R2
50 - unzip # extract CTAN package zip (pseudo.sty override)
51 - bun # build.sh::bun_build for css/ts/tsx
52 - just # bash ./build.sh shells out to `just forest`, `just assets`
53 - biome # ./chk.sh (lint) — keep parity with gh-pages.yml even if unused
54 - nodejs # bunx xslt3 needs node
55 - libxslt # provides xsltproc (used by convert_xml.sh) — top-level pkg name in nixpkgs is libxslt, not xsltproc
56 - python3 # fize.py + manifest jq alt
57 # dvisvgm ships inside the forest-tectonic NAR closure (nixery's
58 # docker-name rules reject the bare `dvisvgm` pkg name).
59 # No texlive bundle — operator directive: tectonic-only + latex shim
60 # (see render command for the `latex` shim that translates forester's
61 # `latex job.tex` into `tectonic --outfmt xdv`).
62
63steps:
64 - name: render forest LIVE (cached toolchain from R2) + publish to site
65 command: |
66 set -euo pipefail
67 export SSL_CERT_FILE="${SSL_CERT_FILE:-${NIX_SSL_CERT_FILE:-/etc/ssl/certs/ca-bundle.crt}}"
68 libstdcpp_dir="$(dirname "$(find /nix/store -maxdepth 4 -name 'libstdc++.so.6' 2>/dev/null | head -1)")"
69 export LD_LIBRARY_PATH="${libstdcpp_dir}:${LD_LIBRARY_PATH:-}"
70 export HOME="${HOME:-/tmp/cihome}"
71
72 echo "::group::discover assets via R2 manifest.json"
73 R2_PUBLIC_URL=https://pub-5462b73a50594a06b85663123d74ab23.r2.dev
74 curl -fsSL "$R2_PUBLIC_URL/forest/manifest.json" -o /tmp/manifest.json
75 jq . /tmp/manifest.json
76 TECTONIC_NAR=$(jq -r '.assets.tectonic.nar' /tmp/manifest.json)
77 TECTONIC_PATH=$(jq -r '.assets.tectonic.store_path' /tmp/manifest.json)
78 FORESTER_NAR=$(jq -r '.assets.forester.nar' /tmp/manifest.json)
79 FORESTER_PATH=$(jq -r '.assets.forester.store_path' /tmp/manifest.json)
80 WASM_TAR=$(jq -r '.assets.wasm_pkgs.tar' /tmp/manifest.json)
81 WASM_PATH=$(jq -r '.assets.wasm_pkgs.store_path' /tmp/manifest.json)
82 # iter 95 — texlive re-added: forest-texlive provides a proper
83 # TEXMF tree at runtime so dvisvgm's kpathsea can resolve LM OTF
84 # native font references. May be absent on first deploy until
85 # build-depot re-runs.
86 TEXLIVE_NAR=$(jq -r '.assets.texlive.nar // "missing"' /tmp/manifest.json)
87 TEXLIVE_PATH=$(jq -r '.assets.texlive.store_path // ""' /tmp/manifest.json)
88 echo "::endgroup::"
89
90 echo "::group::import tectonic + forester + texlive closures (nix-store --import)"
91 # The .nar.zst archives carry every store path referenced by the
92 # forester / tectonic / texlive binaries (libev, gmp, ocaml-runtime,
93 # etc.) so the binaries' ELF interpreter refs resolve at runtime.
94 # Plain tar of just $out was insufficient — bin/forester died with
95 # 'required file not found' on its loader.
96 _pairs=("tectonic:$TECTONIC_NAR:$TECTONIC_PATH" \
97 "forester:$FORESTER_NAR:$FORESTER_PATH")
98 if [ "$TEXLIVE_NAR" != "missing" ] && [ -n "$TEXLIVE_PATH" ]; then
99 _pairs+=("texlive:$TEXLIVE_NAR:$TEXLIVE_PATH")
100 else
101 echo "::warning::texlive asset not in manifest yet (build-depot may need to re-run for iter 95+)"
102 fi
103 for pair in "${_pairs[@]}"; do
104 name="${pair%%:*}"; rest="${pair#*:}"
105 nar="${rest%%:*}"; path="${rest#*:}"
106 echo "::group::fetch + nix-store --import $name"
107 curl -fsSL "$R2_PUBLIC_URL/forest/$nar" -o "/tmp/$nar"
108 zstd -d -T0 "/tmp/$nar" -o "/tmp/${nar%.zst}"
109 nix-store --import < "/tmp/${nar%.zst}"
110 rm -f "/tmp/$nar" "/tmp/${nar%.zst}"
111 test -e "$path/bin" && echo "✓ $name binary present" || echo "(no bin at $path)"
112 du -sh "$path" || true
113 echo "::endgroup::"
114 done
115 echo "::endgroup::"
116
117 echo "::group::extract wasm-pkgs (plain tar — no closure)"
118 if mkdir -p "/nix/store/.write-test-$$" 2>/dev/null; then
119 rmdir "/nix/store/.write-test-$$"
120 else
121 echo "::error::/nix/store not writable in nixery"; exit 1
122 fi
123 curl -fsSL "$R2_PUBLIC_URL/forest/$WASM_TAR" -o "/tmp/$WASM_TAR"
124 zstd -d -T0 "/tmp/$WASM_TAR" -o "/tmp/${WASM_TAR%.zst}"
125 tar -C /nix/store -xf "/tmp/${WASM_TAR%.zst}"
126 rm -f "/tmp/$WASM_TAR" "/tmp/${WASM_TAR%.zst}"
127 test -d "$WASM_PATH" || { echo "::error::wasm-pkgs dir absent: $WASM_PATH"; exit 1; }
128 du -sh "$WASM_PATH"
129 echo "::endgroup::"
130
131 echo "::group::wire tectonic + forester paths + drop wasm pkgs into lib/"
132 # NOTE: env.sh files from the closures call `mkdir` before PATH is
133 # established (issue: $out/bin only has the one binary, so prepending
134 # it replaces PATH if ${PATH:-} happened to be empty — and on the very
135 # first command of the source, before subsequent commands restore PATH,
136 # `mkdir` isn't reachable). Do the equivalent inline so PATH stays
137 # additive against the nixery deps.
138 export PATH="$TECTONIC_PATH/bin:$FORESTER_PATH/bin:$PATH"
139 # iter 95 — if forest-texlive was loaded, add its bin (kpsewhich,
140 # mktexlsr, etc.) and put its texmf-dist on the kpathsea search
141 # path so dvisvgm can find LM OTF/PFB at runtime via the real
142 # kpathsea infrastructure (vs the iter 86-92 cwd-symlink hacks).
143 if [ -n "${TEXLIVE_PATH:-}" ] && [ -d "$TEXLIVE_PATH/bin" ]; then
144 export PATH="$TEXLIVE_PATH/bin:$PATH"
145 # texlive.combine puts the merged texmf-dist tree at $out/share
146 # (a symlink to the texCombo's share dir). kpathsea picks it up
147 # via TEXMF env var or texmf.cnf; we just add it to TEXMFAUXTREES
148 # for ADDITIVE search.
149 # iter 97 — verify the actual texmf-dist location since iter 96
150 # showed forest-texlive loaded but dvisvgm STILL nf-fallbacks.
151 # texlive.combine output structure differs from naive expectation.
152 for cand in "$TEXLIVE_PATH/share/texmf-dist" \
153 "$TEXLIVE_PATH/share/texmf" \
154 "$TEXLIVE_PATH/texmf-dist" \
155 "$TEXLIVE_PATH/share"; do
156 if [ -d "$cand/fonts/opentype/public/lm" ]; then
157 export TEXLIVE_TEXMF="$cand"
158 echo "✓ forest-texlive: TEXLIVE_TEXMF=$TEXLIVE_TEXMF"
159 ls -la "$TEXLIVE_TEXMF/fonts/opentype/public/lm/" 2>/dev/null | head -5
160 break
161 fi
162 done
163 if [ -z "${TEXLIVE_TEXMF:-}" ]; then
164 echo "::warning::forest-texlive loaded but no texmf-dist tree found under $TEXLIVE_PATH/share/"
165 ls -la "$TEXLIVE_PATH/share/" 2>/dev/null | head -10
166 fi
167 # Also tell kpsewhich to use this as the primary TEXMF (more
168 # aggressive than TEXMFAUXTREES).
169 if [ -n "${TEXLIVE_TEXMF:-}" ]; then
170 export OSFONTDIR="$TEXLIVE_TEXMF/fonts/opentype:$TEXLIVE_TEXMF/fonts/truetype:${OSFONTDIR:-}"
171 # iter 97 — try kpsewhich from forest-texlive's bin to see
172 # if it can find LM fonts now.
173 echo " kpsewhich lmroman10-italic.otf: $(kpsewhich lmroman10-italic.otf 2>/dev/null || echo 'NOT FOUND')"
174 echo " kpsewhich lmromancaps10-regular.otf: $(kpsewhich lmromancaps10-regular.otf 2>/dev/null || echo 'NOT FOUND')"
175 echo " kpsewhich --format='opentype fonts' lmroman10-italic.otf: $(kpsewhich --format='opentype fonts' lmroman10-italic.otf 2>/dev/null || echo 'NOT FOUND')"
176 fi
177 fi
178 # forest-tectonic stages dvips .pro headers (tex.pro, texps.pro,
179 # special.pro, color.pro, finclude.pro) directly at $out/share/dvips/
180 # base/ (copied at flake build time from texlive.combine — symlinking
181 # the texmf-dist tree didn't survive the NAR closure boundary). The
182 # latex shim symlinks them into each per-snippet cwd so dvisvgm picks
183 # them up via PSHEADERS=. — no kpathsea config needed.
184 export DVIPS_BASE_DIR="$TECTONIC_PATH/share/dvips/base"
185 # Use ls -l on the dir (always succeeds if dir exists). Avoid
186 # `ls *.pro | head` — non-matching glob exits ls(2), and `set -e
187 # pipefail` then kills the whole render.
188 echo "DVIPS_BASE_DIR=$DVIPS_BASE_DIR"
189 if [ -d "$DVIPS_BASE_DIR" ]; then
190 n_pro=$(find "$DVIPS_BASE_DIR" -maxdepth 1 -name '*.pro' -type f 2>/dev/null | wc -l)
191 echo " .pro file count: $n_pro"
192 find "$DVIPS_BASE_DIR" -maxdepth 1 -name '*.pro' -type f 2>/dev/null | head -10 | sed 's/^/ /' || true
193 else
194 echo "::warning::DVIPS_BASE_DIR dir missing"
195 fi
196 export TECTONIC_CACHE_DIR="$HOME/.cache/Tectonic"
197 mkdir -p "$TECTONIC_CACHE_DIR"
198 if [ -d "$TECTONIC_PATH/share/tectonic-cache" ]; then
199 cp -rn "$TECTONIC_PATH/share/tectonic-cache/." "$TECTONIC_CACHE_DIR/" 2>/dev/null || true
200 fi
201 # Mirror what build.sh::prep_wasm would have populated had it run:
202 # lib/<name>/<…inner-path…>/pkg/* — but since the rendered build.sh
203 # only reads `lib/<lib_path>/pkg/`, we just stage `lib/wgputoy/pkg/`,
204 # `lib/egglog/web-demo/pkg/`, `lib/rhaiscript/pkg/`.
205 mkdir -p lib/wgputoy/pkg lib/egglog/web-demo/pkg lib/rhaiscript/pkg
206 cp -r "$WASM_PATH/pkg/wgputoy/." lib/wgputoy/pkg/
207 cp -r "$WASM_PATH/pkg/egglog/." lib/egglog/web-demo/pkg/
208 cp -r "$WASM_PATH/pkg/rhaiscript/." lib/rhaiscript/pkg/
209 # prep_wasm only rebuilds when lib/<name>/.git is absent — write a
210 # placeholder so it skips fetching upstream sources too. The hash file
211 # tells prep_wasm the pkg is fresh (matches the desired hash).
212 for triple in "wgputoy:wgputoy:60d0bec4bd912a54d5049f2c28c1bd6a0916e5ec" \
213 "egglog:egglog/web-demo:8d9b10ec712106b21d10b7bf45d10c0f9d1d09c7" \
214 "rhaiscript:rhaiscript:9fa80661bc9eb69363ac86879826dcd8ccb604af"; do
215 lib_name="${triple%%:*}"; rest="${triple#*:}"
216 lib_path="${rest%:*}"; hash="${rest##*:}"
217 mkdir -p "lib/$lib_name/.git" "lib/$lib_path/pkg"
218 # bare .git marker — prep_wasm's `[ ! -d lib/$lib_name/.git ]` check
219 # is just for "no clone present", any non-empty marker satisfies it.
220 echo "$hash" > "lib/$lib_path/pkg/.commit_hash"
221 done
222 ls lib/wgputoy/pkg lib/egglog/web-demo/pkg lib/rhaiscript/pkg | head -20
223 echo "::endgroup::"
224
225 echo "::group::theme submodule"
226 git submodule update --init --recursive 2>/dev/null || true
227 echo "::endgroup::"
228
229 echo "::group::tectonic cache hydration"
230 # env.sh seeds ~/.cache/Tectonic from $TECTONIC_PATH/share/tectonic-cache.
231 ls -la "${TECTONIC_CACHE_DIR:-$HOME/.cache/Tectonic}" 2>&1 | head -5 || true
232 echo "::endgroup::"
233
234 echo "::group::opam shim (justfile calls 'opam exec -- forester …')"
235 # build.sh delegates to 'just forest', which is 'opam exec -- forester build'
236 # — opam is the env wrapper GH Actions uses because forester is installed in
237 # an opam switch there. Our forester closure has the binary directly on
238 # PATH, so just unwrap and exec.
239 mkdir -p /tmp/bin
240 # Heredoc inside a YAML `|` block would carry leading-whitespace into
241 # the shebang line and break execution. printf with explicit newlines
242 # keeps the file's first byte at column 0.
243 printf '%s\n' \
244 '#!/bin/sh' \
245 'if [ "$1" = "exec" ] && [ "$2" = "--" ]; then' \
246 ' shift 2; exec "$@"' \
247 'fi' \
248 'echo "opam shim: unhandled invocation: $*" >&2' \
249 'exit 1' \
250 > /tmp/bin/opam
251 chmod +x /tmp/bin/opam
252 export PATH="/tmp/bin:$PATH"
253 opam exec -- forester --version || true
254 echo "::endgroup::"
255
256 echo "::group::latex shim (forester shells out to plain 'latex'; route to tectonic — iter 41 PDF mode)"
257 # iter 41 — pivot to PDF mode per operator's "try 1 then 2" directive.
258 # After 6 iters trying to wire kpathsea/PSHEADERS so dvisvgm-3.6 could
259 # find dvips `.pro` headers, kpathsea SELFAUTOLOC kept pointing at the
260 # dvisvgm-3.6 store path and bypassing our $TECTONIC_PATH/share staging.
261 # Pivot: tectonic --outfmt pdf → dvisvgm --pdf. PDF mode parses pages
262 # natively, bypasses dvips PostScript headers entirely.
263 #
264 # Shim discards latex-specific args; passes the .tex through to tectonic.
265 printf '%s\n' \
266 '#!/bin/sh' \
267 'set -e' \
268 'tex=""' \
269 'for arg; do case "$arg" in *.tex) tex="$arg" ;; esac; done' \
270 '[ -z "$tex" ] && { echo "latex shim: no .tex in args: $*" >&2; exit 1; }' \
271 'base=$(basename "${tex%.tex}")' \
272 'dir=$(dirname "$tex")' \
273 'cd "$dir"' \
274 '# tectonic ignores TEXINPUTS — symlink the project tex/ files into cwd' \
275 '# so `\input{diagrams}` etc. (relative refs) resolve against them.' \
276 'if [ -d "$FOREST_TEX_DIR" ]; then' \
277 ' # iter 51 — symlink BOTH .tex/.sty AND font payloads (tfm/otf/pfb/vf/afm/enc/map/ttf/pfm/fd/def/cfg/clo)' \
278 ' # so dvisvgm/kpathsea finds them in cwd. iter 48 staged them into FOREST_TEX_DIR' \
279 ' # but the loop only symlinked *.tex/*.sty, so the fonts never reached per-snippet cwd.' \
280 ' for ext in tex sty tfm otf pfb vf afm enc map ttf pfm fd def cfg clo; do' \
281 ' for f in "$FOREST_TEX_DIR"/*."$ext"; do [ -e "$f" ] && ln -sf "$f" .; done' \
282 ' done' \
283 'fi' \
284 '# dvips PostScript headers in cwd so kpathsea finds them via PSHEADERS=.' \
285 'if [ -n "$DVIPS_BASE_DIR" ] && [ -d "$DVIPS_BASE_DIR" ]; then' \
286 ' for f in "$DVIPS_BASE_DIR"/*.pro; do [ -e "$f" ] && ln -sf "$f" .; done' \
287 'fi' \
288 '# iter 47 — revert to XDV mode. PDF mode (iter 41–46) lost tikz' \
289 '# vector content (paths/gradients/clip-paths) on every PDF→SVG' \
290 '# backend tried (pdftocairo strips text, mutool drops paths, libgs' \
291 '# 10.07 produced empty output). XDV is the format dvisvgm natively' \
292 '# understands; with .pro files symlinked into cwd above and' \
293 '# PSHEADERS=. (via dvisvgm shim env), the kpathsea cascade resolves.' \
294 'tectonic -Z shell-escape-cwd="$(pwd)" --outfmt xdv --keep-logs --print --bundle "${TECTONIC_BUNDLE:-https://bebopbamf-tex.syd1.cdn.digitaloceanspaces.com/texlive2024-0312.ttb}" "${base}.tex" 2>&1 || { echo "::error::tectonic FAILED for ${base}.tex (cwd=$(pwd))" >&2; cat "${base}.log" 2>/dev/null | tail -40 >&2 || true; exit 1; }' \
295 '# Forester chains dvisvgm right after latex; it expects ${base}.dvi.' \
296 '# Rename the XDV. dvisvgm detects XDV from file magic regardless of extension.' \
297 '[ -f "${base}.xdv" ] && mv -f "${base}.xdv" "${base}.dvi"' \
298 > /tmp/bin/latex
299 chmod +x /tmp/bin/latex
300
301 # dvisvgm shim — iter 41b: forester invokes dvisvgm via stdin/stdout,
302 # not a file arg. Per ocaml-forester LaTeX_pipeline.ml:
303 # dvisvgm --exact --clipjoin --font-format=woff --bbox=papersize \
304 # --zoom=1.5 --stdin --stdout
305 # with job.dvi (now a renamed PDF after the latex shim) piped into stdin.
306 # dvisvgm's `--stdin` defaults to DVI-mode, so PDF magic causes
307 # "invalid DVI file (missing preamble) at position 1".
308 #
309 # The shim:
310 # 1. If invoked with --stdin: buffer stdin to a temp file,
311 # sniff first 4 bytes.
312 # 2. If %PDF: rebuild args dropping --stdin, exec real with
313 # `--pdf` + the temp file as positional. `--stdout` stays.
314 # 3. If DVI (\xf7 magic): exec real with original args, piping
315 # the buffered temp file back to stdin.
316 # 4. If invoked without --stdin (file-based): unchanged from v1
317 # (sniff file arg's magic, prepend --pdf if PDF).
318 # We resolve the real binary by stripping /tmp/bin from PATH first.
319 DVISVGM_REAL=$(PATH=$(echo "$PATH" | tr ':' '\n' | grep -v '^/tmp/bin$' | paste -sd: -) command -v dvisvgm)
320 cat > /tmp/bin/dvisvgm <<EOF
321 #!/usr/bin/env bash
322 set -e
323 real='$DVISVGM_REAL'
324
325 has_stdin=0
326 for arg in "\$@"; do
327 [ "\$arg" = "--stdin" ] && has_stdin=1 && break
328 done
329
330 if [ "\$has_stdin" -eq 0 ]; then
331 # File-arg path: sniff first non-flag arg's magic.
332 for arg in "\$@"; do
333 case "\$arg" in
334 -*) ;;
335 *) if [ -f "\$arg" ]; then
336 if head -c 4 "\$arg" 2>/dev/null | grep -q "^%PDF"; then
337 exec "\$real" --pdf "\$@"
338 fi
339 break
340 fi ;;
341 esac
342 done
343 exec "\$real" "\$@"
344 fi
345
346 # --stdin path: buffer stdin, sniff magic, dispatch.
347 tmp=\$(mktemp)
348 trap "rm -f '\$tmp'" EXIT
349 cat > "\$tmp"
350
351 # iter 69 — neutralize the 'pdf:pagesize' XDV \\special that
352 # crashes dvisvgm 3.6's preprocessing with
353 # ERROR: basic_string::substr: __pos (6) > size (3)
354 # ag-0018 / uts-001F / ca-000J tikz-cd all emit this special via
355 # tectonic's XDV driver. iter 68 confirmed --bbox=min still crashes
356 # because the preprocessor parses pdf:pagesize unconditionally.
357 # Replace 'pdf:pagesize' with 'xyz:pagesize' (same byte length, so
358 # DVI offsets stay intact); dvisvgm's SpecialManager then ignores
359 # the unknown xyz: scheme and proceeds. Only run on DVI/XDV
360 # inputs (first byte == 0xf7); PDF path (below) is unaffected.
361 first_byte=\$(od -An -N1 -tx1 "\$tmp" 2>/dev/null | tr -d ' \n')
362 if [ "\$first_byte" = "f7" ]; then
363 python3 -c '
364 import sys
365 p = sys.argv[1]
366 with open(p, "rb") as f: d = f.read()
367 d2 = d.replace(b"pdf:pagesize", b"xyz:pagesize")
368 if d2 != d:
369 with open(p, "wb") as f: f.write(d2)
370 ' "\$tmp" 2>/dev/null || true
371 fi
372
373 # iter 47 — for XDV/DVI inputs dvisvgm needs dvips .pro headers
374 # (tex.pro, texps.pro, special.pro, color.pro, finclude.pro).
375 # The latex shim symlinks them into cwd; PSHEADERS=. tells kpathsea
376 # to look in cwd first.
377 export PSHEADERS=".:\${PSHEADERS:-}"
378 export TEXPSHEADERS="\$PSHEADERS"
379 export DVIPSHEADERS="\$PSHEADERS"
380 # iter 93 — tectonic emits native font references in XDV as bare
381 # filenames (lmroman10-italic, lmromancaps10-regular). dvisvgm
382 # routes those through kpathsea's OPENTYPEFONTS / TTFONTS search
383 # paths, NOT through psfonts.map (that's only for type1). Per
384 # iter 86 kpsewhich diagnosis, kpathsea couldn't find anything
385 # by default; the FOREST_TEX_DIR / cwd hack works for type1 PFB
386 # files because the path is in psfonts.map. For OTF, we need
387 # to point kpathsea at the dirs that hold the OTFs.
388 export OPENTYPEFONTS=".:\${FOREST_TEX_DIR:-}:\${FOREST_TEXMF:-}/fonts/opentype//:\${OPENTYPEFONTS:-}"
389 export TTFONTS=".:\${FOREST_TEX_DIR:-}:\${FOREST_TEXMF:-}/fonts/truetype//:\${TTFONTS:-}"
390 export TYPE1FONTS=".:\${FOREST_TEX_DIR:-}:\${FOREST_TEXMF:-}/fonts/type1//:\${TYPE1FONTS:-}"
391 # iter 54 added kpathsea font env vars (TFMFONTS/TYPE1FONTS/...)
392 # pointing at FOREST_TEX_DIR — that regressed working figures from
393 # 3 placeholders to 7. Reverted in iter 55. Some kpathsea cascade
394 # must already work via cwd/SELFAUTOLOC for the figures that do
395 # render; overriding the search paths broke that.
396 # iter 57 — instead, point TEXMFAUXTREES at FOREST_TEXMF (a proper
397 # TEXMF tree we built alongside FOREST_TEX_DIR's flat copy).
398 # TEXMFAUXTREES is documented as an ADDITIVE search path: kpathsea
399 # appends it to the default cascade rather than overriding, so the
400 # working figures' resolution stays intact.
401 if [ -n "\${FOREST_TEXMF:-}" ]; then
402 # trailing '//' tells kpathsea to recurse
403 export TEXMFAUXTREES="\${FOREST_TEXMF}//,\${TEXMFAUXTREES:-}"
404 fi
405 # iter 95 — also add forest-texlive's texmf-dist (real kpathsea-
406 # indexed TEXMF tree from nixpkgs's texlive.combine, with ls-R).
407 if [ -n "\${TEXLIVE_TEXMF:-}" ]; then
408 export TEXMFAUXTREES="\${TEXLIVE_TEXMF}//,\${TEXMFAUXTREES:-}"
409 fi
410
411 if head -c 4 "\$tmp" 2>/dev/null | grep -q "^%PDF"; then
412 # PDF input — iter 46: force dvisvgm to use libgs (Ghostscript)
413 # via the DVISVGM_PDF_PROC=gs env var (see PDFToSVG.cpp:30-34).
414 #
415 # iter 45 used mutool (the default for gs >= 10.01), which preserves
416 # text/fonts but mutool's 'draw -F trace' output drops tikz vector
417 # primitives — tikz figures with circles/paths/radialGradients
418 # ended up as text-only SVGs (0 paths, vs gh-pages' dvisvgm-on-XDV
419 # output with 17 paths + the full sphere drawing).
420 #
421 # libgs renders the PDF via Ghostscript's PostScript interpreter,
422 # preserving paths, gradients, clip paths — same fidelity as the
423 # XDV path. nixery's gs 10.07 fails dvisvgm's version-allowlist
424 # check by default, but DVISVGM_PDF_PROC=gs forces _useGS=true and
425 # bypasses the check entirely.
426 export DVISVGM_PDF_PROC=gs
427 new_args=()
428 for arg in "\$@"; do
429 [ "\$arg" = "--stdin" ] && continue
430 new_args+=("\$arg")
431 done
432 exec "\$real" --pdf "\${new_args[@]}" "\$tmp"
433 fi
434
435 # Real DVI/XDV: pass args unchanged, replay stdin from the temp file.
436 # iter 47b — tolerate dvisvgm errors on stubborn docs (e.g. macros.tree
437 # which calls \\tex{}{} with an empty body and trips font lookup that
438 # we can't satisfy from the bundle). Forester treats a non-zero exit
439 # as fatal for the whole build. Capture stderr; if dvisvgm exits
440 # non-zero, emit a minimal valid SVG on stdout and exit 0 so the
441 # build proceeds. The error goes to stderr (visible in CI logs) but
442 # doesn't kill render.
443 # iter 53 — also pass --fontmap=\$FOREST_TEX_DIR/psfonts.map explicitly
444 # so dvisvgm finds the aggregated TL font map (kpathsea cascade didn't
445 # find it via cwd alone — see iter 52b warnings).
446 err=\$(mktemp)
447 trap "rm -f '\$tmp' '\$err'" EXIT
448 fontmap_arg=""
449 if [ -f "\${FOREST_TEX_DIR:-}/psfonts.map" ]; then
450 fontmap_arg="--fontmap=\${FOREST_TEX_DIR}/psfonts.map"
451 fi
452 # iter 91 — bump capture from 20 to ALL DVIs (max 200), and hash
453 # each by its content MD5 so I can correlate with the SVG output
454 # filename (forester hashes the .tex content not the DVI, but the
455 # MD5 of the buffered DVI still uniquely identifies the figure).
456 mkdir -p /tmp/dvi-samples 2>/dev/null
457 _n=\$(ls /tmp/dvi-samples 2>/dev/null | wc -l | tr -d ' ')
458 if [ "\${_n:-0}" -lt 200 ]; then
459 _hash=\$(md5sum "\$tmp" 2>/dev/null | awk '{print \$1}' | head -c 8)
460 cp "\$tmp" "/tmp/dvi-samples/dvi-\$(printf '%03d' \$_n)-\${_hash}.bin" 2>/dev/null || true
461 fi
462 if "\$real" \$fontmap_arg "\$@" 2>"\$err" < "\$tmp"; then
463 # iter 60 — even on success, log any WARNING lines (missing fonts
464 # in particular) so we can see what fonts are silently missing
465 # from figures that LOOK fine sizewise but render with blank
466 # glyphs for the missing fonts.
467 if grep -q "WARNING\|font file" "\$err" 2>/dev/null; then
468 {
469 echo "==== dvisvgm WARN at \$(date -u +%H:%M:%SZ) ===="
470 grep -E "WARNING|font file" "\$err"
471 echo "----"
472 } >> /tmp/dvisvgm-warnings.log 2>/dev/null || true
473 fi
474 exit 0
475 fi
476 # iter 68 — retry on basic_string::substr crash with --bbox=min.
477 # dvisvgm 3.6's pdf:pagesize special handler trips substr OOB on
478 # tectonic's XDV output for \minialg / tikz-cd / pseudo blocks.
479 # Switching from --bbox=papersize to --bbox=min skips pdf:pagesize
480 # parsing entirely (uses content tight-fit bbox instead).
481 if grep -q "basic_string::substr" "\$err" 2>/dev/null; then
482 new_args=()
483 for arg in "\$@"; do
484 case "\$arg" in
485 --bbox=papersize) new_args+=("--bbox=min") ;;
486 *) new_args+=("\$arg") ;;
487 esac
488 done
489 err2=\$(mktemp)
490 if "\$real" \$fontmap_arg "\${new_args[@]}" 2>"\$err2" < "\$tmp"; then
491 echo "::notice::dvisvgm substr crash on --bbox=papersize, retry with --bbox=min succeeded" >&2
492 {
493 echo "==== dvisvgm substr-retry RECOVERED at \$(date -u +%H:%M:%SZ) ===="
494 echo "argv: \$*"
495 echo "retry-argv: \${new_args[*]}"
496 echo "----"
497 } >> /tmp/dvisvgm-retry-recovered.log 2>/dev/null || true
498 rm -f "\$err2"
499 exit 0
500 fi
501 # retry also failed; fall through to placeholder + log both errors
502 cat "\$err2" >> "\$err" 2>/dev/null || true
503 rm -f "\$err2"
504 fi
505 echo "::warning::dvisvgm failed for stdin input; emitting empty SVG (see stderr)" >&2
506 cat "\$err" >&2
507 # iter 50 — also tee the dvisvgm error to a known file so we can see
508 # warnings post-hoc (the SSH TUI's scrollback is too short to grep).
509 {
510 echo "==== dvisvgm failure at \$(date -u +%H:%M:%SZ) ===="
511 echo "argv: \$*"
512 cat "\$err"
513 echo "---- first 200 bytes of input ----"
514 head -c 200 "\$tmp" | od -c | head -10
515 echo "----"
516 } >> /tmp/dvisvgm-failures.log 2>/dev/null || true
517 printf '%s\n' \\
518 '<?xml version="1.0" encoding="UTF-8"?>' \\
519 '<svg xmlns="http://www.w3.org/2000/svg" width="1" height="1" viewBox="0 0 1 1"/>'
520 exit 0
521 EOF
522 sed -i 's/^ //' /tmp/bin/dvisvgm
523 chmod +x /tmp/bin/dvisvgm
524
525 # Sanity: shim takes precedence over any latex/dvisvgm that nixery might provide.
526 echo "latex resolves to: $(command -v latex)"
527 echo "dvisvgm resolves to: $(command -v dvisvgm)"
528 echo "dvisvgm real binary: $DVISVGM_REAL"
529 echo "::endgroup::"
530
531 echo "::group::build forest"
532 export CI=1 # build.sh::bun_build skips bun install branch without this; also gates prep_wasm
533 export TEC=1 # build.sh::lize uses tectonic instead of lualatex
534 # latex shim symlinks files from this dir into each _tmp/<N>/ cwd so
535 # forester's `\input{diagrams}` etc. resolve against the project's
536 # tex/ sources (tectonic ignores TEXINPUTS for reproducibility).
537 export FOREST_TEX_DIR="$PWD/tex"
538
539 echo "::group::pick tectonic bundle with auto-fallback"
540 # Primary: community-hosted TL2024 .ttb (per tectonic#1269 comment).
541 # Fallback: tectonic's stable default bundle (TL2022 redirect).
542 PRIMARY_BUNDLE="${TECTONIC_BUNDLE_PRIMARY:-https://bebopbamf-tex.syd1.cdn.digitaloceanspaces.com/texlive2024-0312.ttb}"
543 FALLBACK_BUNDLE="${TECTONIC_BUNDLE_FALLBACK:-https://relay.fullyjustified.net/default_bundle_v33.tar}"
544 if curl -fsSI --max-time 10 "$PRIMARY_BUNDLE" >/dev/null 2>&1; then
545 export TECTONIC_BUNDLE="$PRIMARY_BUNDLE"
546 echo "✓ TL2024 bundle reachable: $TECTONIC_BUNDLE"
547 else
548 export TECTONIC_BUNDLE="$FALLBACK_BUNDLE"
549 echo "::warning::TL2024 bundle unavailable; falling back to TL2022 default ($TECTONIC_BUNDLE). Forest's algorithms.tex needs newer pseudo.sty — the CTAN override below covers that for the TL2022 case."
550 fi
551 echo "::endgroup::"
552
553 # tectonic's default bundle is TL2022-frozen — its pseudo.sty doesn't
554 # auto-register the /tcb/pseudo/* keys that forest's algorithms.tex
555 # expects (those land in pseudo.sty newer than TL2022). Fetch the
556 # whole CTAN pseudo package (zip), extract every payload file
557 # (*.sty / *.tex / *.def / *.fd / *.cfg / *.cls) into FOREST_TEX_DIR
558 # so the latex shim's per-_tmp symlink stages them; tectonic's
559 # cwd-first lookup wins over the bundled stale versions.
560 tmpdir=$(mktemp -d)
561 curl -fsSL --max-time 60 \
562 https://mirror.ctan.org/macros/latex/contrib/pseudo.zip \
563 -o "$tmpdir/pseudo.zip"
564 ( cd "$tmpdir" && unzip -q pseudo.zip )
565 # Copy in every potentially-loadable TeX payload file (skip docs/tests).
566 find "$tmpdir" -type f \( \
567 -name '*.sty' -o -name '*.tex' -o -name '*.def' \
568 -o -name '*.fd' -o -name '*.cfg' -o -name '*.cls' \) \
569 -not -path '*/doc/*' -not -path '*/test/*' -not -path '*/example*/*' \
570 -exec cp -v {} "$FOREST_TEX_DIR/" \;
571 rm -rf "$tmpdir"
572 grep -q '/tcb/pseudo' "$FOREST_TEX_DIR/pseudo.sty" \
573 && echo "✓ pseudo.sty (with /tcb/pseudo keys) staged from CTAN zip" \
574 || { echo "::error::pseudo.sty from CTAN zip missing /tcb/pseudo keys"; exit 1; }
575
576 # iter 44d — TL bundle missing several modern package versions.
577 # Generic helper: stage a list of TL packages from tlnet archive.
578 # Each tar.xz contains tex/latex/<pkg>/*.sty (and optionally .tex
579 # / .def / .fd / .cfg / .cls). Drop the payload files into
580 # FOREST_TEX_DIR; cwd-first lookup wins over the stale bundle.
581 stage_tl_pkg() {
582 local pkg="$1"
583 local tmpdir
584 tmpdir=$(mktemp -d)
585 echo "::group::stage tlnet pkg: $pkg"
586 # iter 77 — bump --max-time to 240 (cm-super.tar.xz is ~64MB and
587 # was silently timing out at 60s; that's why iter 75's cm/cm-super
588 # additions appeared to be no-ops).
589 if ! curl -fsSL --max-time 240 \
590 "https://mirror.ctan.org/systems/texlive/tlnet/archive/$pkg.tar.xz" \
591 -o "$tmpdir/$pkg.tar.xz"; then
592 echo "::warning::tlnet $pkg.tar.xz download failed"
593 rm -rf "$tmpdir"; echo "::endgroup::"; return 0
594 fi
595 ( cd "$tmpdir" && tar xJf "$pkg.tar.xz" )
596 local n
597 # iter 48 — extend to font files (afm/tfm/otf/pfb/vf/enc/map/ttf)
598 # so kpathsea can resolve font lookups for newpx (PagellaX) and
599 # lm (Latin Modern) etc. tlnet's font packages put files in
600 # fonts/<type>/public/<pkg>/; flat-copy them all into
601 # FOREST_TEX_DIR (which the latex shim symlinks into per-snippet
602 # cwd, where kpathsea looks first).
603 n=$(find "$tmpdir" -type f \( \
604 -name '*.sty' -o -name '*.tex' -o -name '*.def' \
605 -o -name '*.fd' -o -name '*.cfg' -o -name '*.cls' \
606 -o -name '*.clo' -o -name '*.tikz' \
607 -o -name '*.afm' -o -name '*.tfm' -o -name '*.otf' \
608 -o -name '*.pfb' -o -name '*.vf' -o -name '*.enc' \
609 -o -name '*.map' -o -name '*.ttf' -o -name '*.pfm' \) \
610 -not -path '*/doc/*' -not -path '*/test/*' -not -path '*/example*/*' \
611 -exec cp {} "$FOREST_TEX_DIR/" \; -print | wc -l)
612 # iter 57 — also stage with original TEXMF tree structure preserved
613 # under FOREST_TEXMF so kpathsea can find files via the canonical
614 # path (e.g. fonts/type1/public/lm/lmr10.pfb). cwd-only / env-var
615 # tricks aren't enough because dvisvgm runs in forester's MAIN cwd
616 # (not the per-snippet tmp dir).
617 if [ -n "${FOREST_TEXMF:-}" ] && [ -d "$tmpdir" ]; then
618 ( cd "$tmpdir" && find . -type d -path './fonts*' -o -path './tex*' -o -path './scripts*' 2>/dev/null \
619 | xargs -I {} mkdir -p "$FOREST_TEXMF/{}" 2>/dev/null ) || true
620 find "$tmpdir" -type f \( -name '*.sty' -o -name '*.tex' -o -name '*.def' \
621 -o -name '*.fd' -o -name '*.cfg' -o -name '*.cls' -o -name '*.clo' \
622 -o -name '*.afm' -o -name '*.tfm' -o -name '*.otf' -o -name '*.pfb' \
623 -o -name '*.vf' -o -name '*.enc' -o -name '*.map' -o -name '*.ttf' \
624 -o -name '*.pfm' \) -not -path '*/doc/*' -not -path '*/test/*' \
625 -not -path '*/example*/*' | while IFS= read -r src; do
626 rel="${src#$tmpdir/}"
627 dst="$FOREST_TEXMF/$rel"
628 mkdir -p "$(dirname "$dst")" 2>/dev/null
629 cp "$src" "$dst" 2>/dev/null || true
630 done
631 fi
632 rm -rf "$tmpdir"
633 echo " ✓ $pkg: $n files staged"
634 echo "::endgroup::"
635 }
636
637 # iter 57 — also stage a parallel TEXMF tree (FOREST_TEXMF) preserving
638 # the original fonts/<type>/public/<pkg>/ structure. dvisvgm runs in
639 # forester's MAIN cwd (not per-snippet tmp), so symlinks via the latex
640 # shim don't help dvisvgm. A TEXMF tree at a fixed path that we
641 # advertise via TEXMFAUXTREES lets kpathsea search it regardless of cwd.
642 export FOREST_TEXMF="$PWD/forest-texmf"
643 mkdir -p "$FOREST_TEXMF"
644
645 # iter 58 — also symlink the staged font files into forester's MAIN cwd
646 # ($PWD = workspace root). dvisvgm runs there per forester's source,
647 # so anything findable in `.` is findable to dvisvgm. Cheap belt+
648 # suspenders alongside TEXMFAUXTREES. Will be populated by stage_tl_pkg's
649 # cp into FOREST_TEX_DIR; we symlink the contents AFTER all packages
650 # are staged.
651
652 # tectonic's BebopBamf TL2024 bundle ships older versions of several
653 # packages that forest preamble.tex relies on. Stage modern versions
654 # from tlnet — but ONLY ones that don't trigger a format-file mismatch.
655 #
656 # Iter 44e found that staging newer l3kernel/l3packages/l3backend
657 # collides with the bundle's precompiled format file:
658 # "L3 programming layer in the LaTeX format is dated 2024-02-20,
659 # but in your TeX tree the files require at least 2026-05-26."
660 # The format is precompiled from the bundle's old l3kernel, so newer
661 # l3kernel .sty files in cwd refuse to load.
662 #
663 # Compromise (iter 44g): stage l3keys2e (for \ProcessKeysOptions),
664 # todonotes (for \todostyle), AND utfsym (so its \ProcessKeysOptions
665 # call lines up with the staged l3keys2e; bundle's utfsym at line 38
666 # has a different undefined macro). Skip l3kernel/l3packages/l3backend
667 # so the bundle's precompiled format stays consistent.
668 #
669 # iter 48 — add font packages: newpx (PagellaX, TeX Gyre Pagella),
670 # lm (Latin Modern: lmb10/lmbx etc.), so dvisvgm's kpathsea finds
671 # the actual .otf/.tfm/.pfb/.enc/.map files referenced by macros.tree
672 # + ag-000G + ~110 other trees that currently get the empty-SVG
673 # placeholder from the dvisvgm-tolerance shim.
674 # iter 49 — add math/symbol packages that 3 remaining placeholders
675 # (010adfc, 9ce7359, b95c9498) likely reference: stmaryrd (St Mary's
676 # Road math), physics (physics conventions + fonts), newpxmath (newpx
677 # math font tables), pgfplots (plotting; ag-001H uses pgfplots),
678 # tipa (phonetic; sometimes loaded transitively).
679 # iter 56 — try staging mathpazo (Pazo/Pagella math font, includes
680 # fplmb.pfb that newpx's vf chain might point at) and psnfss
681 # (PostScript NFSS — standard PostScript fonts incl. symbol psyr).
682 # Also try lm-math (Latin Modern math, latinmodern-math.otf for
683 # math glyphs that might be referenced).
684 # iter 61 — also stage amsfonts (msbm10 blackboard math) for tt-0014 etc.
685 # iter 62 — add newtx (stxscr, ntxsym fonts used by newpx via VF chain).
686 # iter 64 — add pxfonts (the original PX Fonts) — newpx's zplbmi.vf
687 # and zplexx.vf chain to pxbmi/pxexa from pxfonts.
688 # iter 75 added cm + cm-super; appeared to regress but turned out to
689 # be a measurement error on my side (head -10 truncated the font list).
690 # Reverted needlessly in iter 76. iter 77 re-adds them WITH the
691 # max-time fix (cm-super.tar.xz is ~64MB and was silently timing out
692 # at 60s), plus verification echoes below to prove the staging
693 # actually placed the relevant TFM + PFB + map files.
694 for pkg in l3keys2e todonotes utfsym newpx lm \
695 newpxmath stmaryrd physics pgfplots tipa \
696 mathpazo psnfss lm-math amsfonts newtx pxfonts \
697 cm cm-super; do
698 stage_tl_pkg "$pkg"
699 done
700
701 # iter 77 — verify cmcsc10 / cmti10 / sfcsc10 staging landed and is
702 # discoverable by the psfonts.map aggregator. ag-0018 / uts-001F SVG
703 # diff shows tngl uses nf2/nf3/nf5/nf7 dvisvgm fallbacks where
704 # gh-pages uses cmcsc10 + cmti10; this is closes #2 if all three of:
705 # - cmcsc10.tfm + cmti10.tfm present in FOREST_TEX_DIR
706 # - sfcsc10.pfb (or equivalent Type1) present
707 # - psfonts.map maps cmcsc10 -> some PFB path
708 # …are confirmed. Print to CI log for evidence.
709 # iter 85 — the actual root cause for ag-0018's overlap (per iter 84
710 # warnings) is dvisvgm's "can't find rm-lmb10.tfm" / rm-lmr*.tfm /
711 # rm-lmri*.tfm errors during the VF chain expansion. Those rm-*.tfm
712 # files ARE in lm.tar.xz at fonts/tfm/public/lm/ — stage_tl_pkg's
713 # flat copy puts them in FOREST_TEX_DIR (already verified) AND
714 # FOREST_TEXMF (also already). So they're on disk but kpathsea
715 # somehow can't find them.
716 #
717 # Hypothesis: ls-R cache wasn't built when forester first ran (iter
718 # 83's rebuild happens AFTER psfonts.map agg but maybe AFTER the
719 # build runs too?). Rebuild ls-R BEFORE the build step, and dump
720 # it to /tmp for inspection.
721 {
722 echo "=== iter 86 staging verification ==="
723 echo " FOREST_TEX_DIR: $FOREST_TEX_DIR"
724 echo " FOREST_TEXMF: $FOREST_TEXMF"
725 echo "--- iter 83 carryover: cm*.pfb mirror + ls-R ---"
726 if [ -d "$FOREST_TEXMF/fonts/type1/public/amsfonts/cm" ]; then
727 mkdir -p "$FOREST_TEXMF/fonts/type1/public/cm"
728 cp -n "$FOREST_TEXMF/fonts/type1/public/amsfonts/cm/"*.pfb \
729 "$FOREST_TEXMF/fonts/type1/public/cm/" 2>/dev/null || true
730 fi
731 if command -v mktexlsr >/dev/null 2>&1; then
732 mktexlsr "$FOREST_TEXMF" 2>&1 | head -5
733 else
734 ( cd "$FOREST_TEXMF" && ls -R > ls-R ) 2>/dev/null
735 fi
736 echo "--- iter 86: kpsewhich what dvisvgm runtime actually finds ---"
737 echo " TEXMFAUXTREES exported in dvisvgm shim only — kpsewhich here may differ from shim runtime."
738 echo " Test BOTH with and without TEXMFAUXTREES set explicitly here:"
739 echo "--- (a) bare environment ---"
740 echo " kpsewhich cmcsc10.pfb: $(kpsewhich cmcsc10.pfb 2>/dev/null || echo 'NOT FOUND')"
741 echo " kpsewhich cmcsc10.tfm: $(kpsewhich cmcsc10.tfm 2>/dev/null || echo 'NOT FOUND')"
742 echo " kpsewhich rm-lmb10.tfm: $(kpsewhich rm-lmb10.tfm 2>/dev/null || echo 'NOT FOUND')"
743 echo " kpsewhich cmr10.pfb (sanity): $(kpsewhich cmr10.pfb 2>/dev/null || echo 'NOT FOUND')"
744 echo "--- (b) with TEXMFAUXTREES set ---"
745 export TEXMFAUXTREES="${FOREST_TEXMF}//"
746 echo " TEXMFAUXTREES=$TEXMFAUXTREES"
747 echo " kpsewhich cmcsc10.pfb: $(kpsewhich cmcsc10.pfb 2>/dev/null || echo 'NOT FOUND')"
748 echo " kpsewhich cmcsc10.tfm: $(kpsewhich cmcsc10.tfm 2>/dev/null || echo 'NOT FOUND')"
749 echo " kpsewhich rm-lmb10.tfm: $(kpsewhich rm-lmb10.tfm 2>/dev/null || echo 'NOT FOUND')"
750 echo " kpsewhich cmr10.pfb: $(kpsewhich cmr10.pfb 2>/dev/null || echo 'NOT FOUND')"
751 echo " kpsewhich lmr10.pfb (this one works in real renders): $(kpsewhich lmr10.pfb 2>/dev/null || echo 'NOT FOUND')"
752 echo "--- (c) dvisvgm config (if it has one we can dump) ---"
753 echo " dvisvgm --list-fonts | head: $(dvisvgm --list-fonts 2>/dev/null | head -3 || echo 'no --list-fonts support')"
754 echo " dvisvgm --version: $(dvisvgm --version 2>/dev/null | head -2 || echo 'no version')"
755 } | tee /tmp/iter78-staging.log
756
757 # iter 52b — concatenate all staged .map files into a single
758 # psfonts.map so dvisvgm's kpathsea finds it. updmap normally does
759 # this aggregation after package install; we do it inline here.
760 # dvisvgm's "WARNING: none of the default map files could be found"
761 # is exactly this miss — fonts/.tfm are present in cwd but without
762 # psfonts.map, kpathsea can't resolve TeX font name → file mapping.
763 # Use find -exec to avoid `cat *.map` failing on zero matches under set -e.
764 map_count=$(find "$FOREST_TEX_DIR" -maxdepth 1 -name '*.map' -type f 2>/dev/null | wc -l)
765 {
766 echo "% concatenated from $map_count staged .map files"
767 find "$FOREST_TEX_DIR" -maxdepth 1 -name '*.map' -type f -exec cat {} + 2>/dev/null || true
768 } > "$FOREST_TEX_DIR/psfonts.map"
769 echo " psfonts.map lines: $(wc -l < "$FOREST_TEX_DIR/psfonts.map")"
770 head -3 "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || true
771 # iter 89 — REVERT iter 84's surgical sed strip. iter 88's evidence
772 # showed cmcsc10.pfb + cmti10.pfb + their TFMs ARE in PWD (verified
773 # via stashed _debug-staging.log) but dvisvgm still renders nf-fonts.
774 # Hypothesis: with iter 84 stripping the LM-rep entry, dvisvgm has
775 # ONLY the direct cmcsc10 -> cmcsc10.pfb mapping. Something about
776 # that direct path fails silently (no warning). cmr10 works because
777 # it KEEPS its LM-rep entry that points at lmr10.pfb (with
778 # lm-rep-cmrm.enc which is staged in lm package). Restore the
779 # LM-rep entries for cmcsc10/cmti10 AND add the matching .enc files
780 # to the targeted symlink list (next block).
781 # sed -i '/^cmcsc10 *LMRomanCaps10/d; /^cmti10 *LMRoman10-Italic/d' "$FOREST_TEX_DIR/psfonts.map" # ← iter 84 strip, disabled in iter 89
782 echo " psfonts.map lines (iter 89 revert): $(wc -l < "$FOREST_TEX_DIR/psfonts.map")"
783 echo " cmcsc10 entries: $(grep -c '^cmcsc10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || echo 0)"
784 grep '^cmcsc10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null
785 echo " cmti10 entries: $(grep -c '^cmti10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || echo 0)"
786 grep '^cmti10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null
787 # iter 80 — append psfonts.map cmcsc10/cmti10 grep to the SAME
788 # staging log file that iter 78 stashes to deployed site/. This
789 # tells me whether dvisvgm has a mapping line at all for the
790 # missing fonts. Files are present (per iter 79) — if the .map
791 # entries are also there, dvisvgm runtime lookup is the issue;
792 # if not, the aggregator is the issue.
793 {
794 echo "--- iter 80: psfonts.map content checks ---"
795 echo " total psfonts.map lines: $(wc -l < "$FOREST_TEX_DIR/psfonts.map")"
796 echo " cmcsc10 entries: $(grep -c '^cmcsc10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || echo 0)"
797 grep '^cmcsc10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null | head -3
798 echo " cmti10 entries: $(grep -c '^cmti10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || echo 0)"
799 grep '^cmti10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null | head -3
800 echo " cmr10 (sanity, should be there): $(grep -c '^cmr10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null || echo 0)"
801 grep '^cmr10' "$FOREST_TEX_DIR/psfonts.map" 2>/dev/null | head -3
802 } >> /tmp/iter78-staging.log
803 grep -q 'todostyle' "$FOREST_TEX_DIR/todonotes.sty" 2>/dev/null \
804 && echo "✓ todonotes.sty has \\todostyle" \
805 || echo "::warning::staged todonotes.sty missing \\todostyle"
806 grep -q 'ProcessKeysOptions' "$FOREST_TEX_DIR/l3keys2e.sty" 2>/dev/null \
807 && echo "✓ l3keys2e.sty has \\ProcessKeysOptions" \
808 || echo "::warning::staged l3keys2e.sty missing \\ProcessKeysOptions"
809
810 # iter 58 (NARROWED in iter 59) — symlink only the SPECIFIC font files
811 # the 3 placeholders need (TeXGyrePagellaX-Regular.otf, pxsys.pfb,
812 # zplmi*, lmr10*, lmr7*). iter 58 symlinked all extensions wholesale,
813 # which fixed the 3 originals but triggered
814 # 'basic_string::substr __pos (6) > size (3)' crashes on 4 OTHER
815 # figures (probably stale font tables in our staged set conflicting
816 # with what dvisvgm parsed against the bundle's versions).
817 echo "::group::symlink targeted fonts into forester main cwd ($PWD)"
818 # iter 61 — extended set per iter 60's warning log. Direct file
819 # misses (TFM/PFB) and font-name misses (resolved via psfonts.map
820 # to actual files in the staged archives).
821 n=0
822 for pat in TeXGyrePagellaX-Regular.otf TeXGyrePagellaX-Italic.otf \
823 TeXGyrePagellaX-Bold.otf TeXGyrePagellaX-BoldItalic.otf \
824 pxsys.pfb pxsys.tfm \
825 zplmi.tfm zplmi.pfb zplmi.vf zplmia.tfm zplmia.pfb zplmi1.tfm \
826 zplsy.tfm zplsy.pfb zplsyc.tfm zplsyc.pfb \
827 zplsym.tfm zplsym.vf \
828 zplbmi.tfm zplbmi.pfb \
829 zplexx.tfm zplexx.pfb \
830 NewPXMI.pfb NewPXMI_gnu.pfb \
831 lmr5.tfm lmr5.pfb lmr6.tfm lmr6.pfb \
832 lmr7.tfm lmr7.pfb lmr8.tfm lmr8.pfb \
833 lmr9.tfm lmr9.pfb lmr10.tfm lmr10.pfb \
834 lmr12.tfm lmr12.pfb lmr17.tfm lmr17.pfb \
835 lmri7.tfm lmri7.pfb lmri10.tfm lmri10.pfb \
836 lmri12.tfm lmri12.pfb \
837 lmb10.tfm lmb10.pfb lmbx10.tfm lmbx10.pfb \
838 lmbx12.tfm lmbx12.pfb lmbx7.tfm lmbx7.pfb \
839 lmmi5.tfm lmmi5.pfb lmmi7.tfm lmmi7.pfb \
840 lmmi9.tfm lmmi9.pfb lmmi10.tfm lmmi10.pfb \
841 lmmi12.tfm lmmi12.pfb \
842 lmsy5.tfm lmsy5.pfb lmsy7.tfm lmsy7.pfb \
843 lmsy10.tfm lmsy10.pfb \
844 lmbsy7.tfm lmbsy7.pfb lmbsy10.tfm lmbsy10.pfb \
845 lmcsc10.tfm lmcsc10.pfb \
846 lmtt10.tfm lmtt10.pfb \
847 lmmono10-regular.otf lmmono8-regular.otf \
848 lmroman10-regular.otf lmroman10-bold.otf lmroman10-italic.otf \
849 lmroman9-regular.otf lmromancaps10-regular.otf \
850 lmromandemi10-regular.otf \
851 stxscr.tfm stxscr.pfb \
852 ntxsym.tfm ntxsym.pfb ntxsyc.tfm ntxsyc.pfb \
853 fplmb.pfb fplmbb.pfb fplmbi.pfb fplmr.pfb fplmri.pfb \
854 msbm10.tfm msbm10.pfb \
855 zplbmi0.tfm zplbmi0.vf zplbmi1.tfm zplbmi1.vf \
856 zplbmia.tfm zplbmia.vf \
857 zplmia.vf zplmi0.tfm zplmi0.vf zplmi1.vf \
858 zplsyc.vf zplsym.tfm \
859 pxmiaX.tfm pxmiaX.pfb pxmiaX.afm \
860 txmiaSTbb.tfm txmiaSTbb.pfb \
861 txmiaX.tfm txmiaX.pfb \
862 txsym.tfm txsym.pfb \
863 pxbmia.pfb pxbsyc.pfb pxbsy.pfb pxbsya.pfb pxbsyb.pfb \
864 pxbex.pfb pxbexa.pfb \
865 pxmia.pfb pxsy.pfb pxsya.pfb pxsyb.pfb pxsyc.pfb \
866 pxex.pfb pxexa.pfb \
867 rpxb.pfb rpxbi.pfb rpxbmi.pfb rpxbmia.pfb \
868 rpxr.pfb rpxi.pfb rpxbsl.pfb rpxsl.pfb \
869 cmcsc10.pfb cmcsc10.tfm \
870 cmti10.pfb cmti10.tfm \
871 cmr10.pfb cmr10.tfm cmr7.pfb cmr7.tfm \
872 cmbx10.pfb cmbx10.tfm \
873 cmmi10.pfb cmmi10.tfm cmmi7.pfb cmmi7.tfm \
874 cmsy10.pfb cmsy10.tfm cmsy7.pfb cmsy7.tfm \
875 rm-lmb10.tfm rm-lmbo10.tfm rm-lmbx10.tfm \
876 rm-lmr10.tfm rm-lmr5.tfm rm-lmr6.tfm rm-lmr7.tfm rm-lmr8.tfm rm-lmr9.tfm \
877 rm-lmri10.tfm rm-lmri7.tfm rm-lmri12.tfm \
878 rm-lmbx5.tfm rm-lmbx6.tfm rm-lmbx7.tfm rm-lmbx8.tfm rm-lmbx9.tfm rm-lmbx12.tfm \
879 lm-rep-cmrm.enc lm-rep-cmsc.enc lm-rep-cmit.enc lm-rep-cmitt.enc lm-rep-cmtt.enc lm-rep-cmin.enc; do
880 # iter 87 — add CM Type1 fonts (cmcsc10/cmti10/cmr10/cmbx10/etc.)
881 # and rm-lm* VF shim TFMs to the targeted symlink list.
882 # kpsewhich diagnosis (iter 86) showed kpathsea couldn't
883 # find ANYTHING (even lmr10.pfb which renders fine), so
884 # dvisvgm's runtime font lookup goes through $PWD (cwd),
885 # not kpathsea. The previous list pulled in lm fonts but
886 # not their cm aliases or the rm-* VF shims. Adding them.
887 # iter 65 added NewPX*/NewTX*/ntx* PFBs but they regressed
888 # the same 4 trees (21bc037, 2330c77, 5d08d7c, e62c50c) to
889 # placeholders via the iter-58 substr crash pattern.
890 # Reverted in iter 66.
891 f="$FOREST_TEX_DIR/$pat"
892 if [ -e "$f" ] && [ ! -e "$PWD/$pat" ]; then
893 ln -sf "$f" "$PWD/$pat" 2>/dev/null && n=$((n+1))
894 fi
895 done
896 echo " ✓ $n targeted font files symlinked into $PWD"
897 # iter 94 — tectonic emits XDV native font names as BARE FILENAMES
898 # without extension (e.g. 'lmroman10-italic'). dvisvgm's registerFont
899 # calls FileSystem::exists(fname) which checks for the literal name;
900 # no .otf/.ttf auto-append. Create extensionless symlinks pointing
901 # to the .otf files in PWD so the bare-name lookup succeeds.
902 for pat in lmroman10-bold lmroman10-italic lmroman10-regular \
903 lmromancaps10-regular lmromandemi10-regular \
904 lmroman9-regular; do
905 if [ -e "$PWD/$pat.otf" ] && [ ! -e "$PWD/$pat" ]; then
906 ln -sf "$pat.otf" "$PWD/$pat" 2>/dev/null && \
907 echo " + extensionless: $pat -> $pat.otf" || true
908 fi
909 done
910 # iter 92 — also tee verification of LM OTF symlinks (these are
911 # what tectonic actually emits in XDV as native font names —
912 # lmroman10-italic, lmromancaps10-regular, etc.). DVI sample
913 # capture (iter 91, dvi-001-1cb0845c.bin) confirmed the pseudo
914 # block's font_def includes those 4 LM names, which dvisvgm
915 # fallback-emits as nf2/3/5/7. Files SHOULD be in PWD per iter 87,
916 # but verify.
917 {
918 echo "--- iter 92: cm-fallback + LM-OTF symlink verification in PWD ---"
919 echo " total targeted symlinks created: $n"
920 for f in cmcsc10.pfb cmcsc10.tfm cmti10.pfb cmti10.tfm \
921 cmr10.pfb cmr10.tfm cmbx10.pfb cmbx10.tfm \
922 rm-lmb10.tfm rm-lmr10.tfm rm-lmri10.tfm \
923 lmroman10-bold.otf lmroman10-italic.otf lmroman10-regular.otf \
924 lmromancaps10-regular.otf lmromandemi10-regular.otf \
925 lmroman9-regular.otf; do
926 echo " $f in PWD: $(ls -la "$PWD/$f" 2>/dev/null | head -1 || echo MISSING)"
927 if [ ! -e "$PWD/$f" ] && [ -e "$FOREST_TEX_DIR/$f" ]; then
928 echo " NB: present in FOREST_TEX_DIR but NOT symlinked into PWD"
929 elif [ ! -e "$FOREST_TEX_DIR/$f" ]; then
930 echo " NB: NOT staged in FOREST_TEX_DIR either"
931 fi
932 done
933 } >> /tmp/iter78-staging.log
934 echo "::endgroup::"
935
936 # lize.sh's symlink loop only covers $PROJECT_ROOT/tex/*; the CTAN
937 # overrides land in $FOREST_TEX_DIR which is the same dir. Good.
938 forester --version || true
939 tectonic --version || true
940 bash ./build.sh
941 test -d output/forest
942 du -sh output/forest
943 # iter 50 — dump dvisvgm failure log so we can pinpoint the missing
944 # fonts for the 3 (now possibly fewer) placeholder SVGs.
945 echo "::group::dvisvgm failure log"
946 if [ -f /tmp/dvisvgm-failures.log ]; then
947 echo " $(wc -l < /tmp/dvisvgm-failures.log) lines of failures captured:"
948 cat /tmp/dvisvgm-failures.log
949 else
950 echo " no failures logged ✓"
951 fi
952 echo "::endgroup::"
953 # iter 60 — also dump the warnings log (per-success WARNING lines).
954 echo "::group::dvisvgm warning log"
955 if [ -f /tmp/dvisvgm-warnings.log ]; then
956 echo " $(wc -l < /tmp/dvisvgm-warnings.log) lines of warnings captured:"
957 # only dump first 200 lines to avoid log explosion
958 head -200 /tmp/dvisvgm-warnings.log
959 echo " ... (truncated to first 200 lines if longer)"
960 echo " unique font misses:"
961 grep -oE "font file '[^']*'|no font file found for '[^']*'" /tmp/dvisvgm-warnings.log | sort -u | head -50
962 else
963 echo " no warnings logged ✓"
964 fi
965 echo "::endgroup::"
966 # iter 67 — copy dvisvgm logs INTO the deployed site branch so we can
967 # diagnose persistent placeholders without rerunning. Operator's per-tree
968 # feedback identified 4 broken trees (ag-0018, uts-001F, ca-0001/spin-0001
969 # via ca-000J tikz-cd, and a typst one) — need the actual stderr to
970 # target fixes instead of guessing fonts.
971 echo "::group::iter 67 — stash dvisvgm logs in deploy dir"
972 [ -f /tmp/dvisvgm-failures.log ] && cp /tmp/dvisvgm-failures.log output/forest/_debug-failures.log || echo " no failures.log to stash"
973 [ -f /tmp/dvisvgm-warnings.log ] && cp /tmp/dvisvgm-warnings.log output/forest/_debug-warnings.log || echo " no warnings.log to stash"
974 # iter 78 — also stash the staging verification log so I can read it
975 # without needing the spindle TUI.
976 [ -f /tmp/iter78-staging.log ] && cp /tmp/iter78-staging.log output/forest/_debug-staging.log || echo " no staging.log to stash"
977 # iter 90 — stash the first 20 DVI byte streams so I can read the
978 # cmdXFontDef opcodes locally. Tarball to one artifact to keep
979 # the deploy commit small.
980 if [ -d /tmp/dvi-samples ] && [ "$(ls /tmp/dvi-samples 2>/dev/null | wc -l)" -gt 0 ]; then
981 tar czf output/forest/_debug-dvi-samples.tar.gz -C /tmp dvi-samples 2>/dev/null && \
982 echo " ✓ stashed $(ls /tmp/dvi-samples | wc -l) DVI samples" || \
983 echo " ! failed to tar DVI samples"
984 fi
985 ls -la output/forest/_debug-* 2>/dev/null || true
986 echo "::endgroup::"
987 echo "::endgroup::"
988
989 echo "::group::publish output/forest → site branch"
990 : "${DEPLOY_SSH_KEY_B64:?DEPLOY_SSH_KEY_B64 secret not set on tangled forest repo}"
991 mkdir -p "$HOME/.ssh"; chmod 700 "$HOME/.ssh"
992 grep -q ':0:0:' /etc/passwd || echo "root:x:0:0:root:$HOME:/bin/sh" >> /etc/passwd 2>/dev/null || true
993 printf '%s' "$DEPLOY_SSH_KEY_B64" | base64 -d > "$HOME/.ssh/deploy"; chmod 600 "$HOME/.ssh/deploy"
994 export GIT_SSH_COMMAND="ssh -i $HOME/.ssh/deploy -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HOME/.ssh/known_hosts"
995 export GIT_AUTHOR_NAME=tangled-ci GIT_AUTHOR_EMAIL=ci@utensil.tngl.sh
996 export GIT_COMMITTER_NAME=tangled-ci GIT_COMMITTER_EMAIL=ci@utensil.tngl.sh
997 cd output/forest && git init -q && git checkout -q -b site && git add -A
998 git commit -q -m "render forest $(date -u +%Y-%m-%dT%H:%M:%SZ)"
999 git push --force "git@tangled.org:utensil.tngl.sh/forest" site
1000 echo "pushed rendered forest → branch site"
1001 echo "::endgroup::"
1002
1003 echo "✅ forest rendered + published"