tangled.org
Monorepo for Tangled
tangled.org
1#!/usr/bin/env bash
2set -euo pipefail
3# dont export big captures like `out=$(run_vm ...)` into the environment.
4set +a
5# extglob enables the *(...) pattern strip_ansi uses for pure-bash CSI stripping
6shopt -s extglob
7# note: needs `sudo modprobe vhost_vsock`!
8
9log() {
10 printf "\n\033[1;36m>>> %s\033[0m\n" "$*"
11}
12
13_strip_ansi_stream() {
14 local line
15 while IFS= read -r line || [ -n "$line" ]; do
16 line=${line//$'\e'\[*([0-9;])[a-zA-Z]/}
17 line=${line//$'\e'\([a-zA-Z]/}
18 printf '%s\n' "$line"
19 done
20}
21
22strip_ansi() {
23 if [ "$#" -gt 0 ]; then
24 local f
25 for f in "$@"; do
26 [ -r "$f" ] && _strip_ansi_stream < "$f"
27 done
28 else
29 _strip_ansi_stream
30 fi
31}
32
33# check_needles OUT NEEDLES...
34# each needle is an extended regex matched per line (mirrors `grep -qE`).
35check_needles() {
36 local out="$1"
37 shift
38
39 local -a lines
40 mapfile -t lines <<< "$out"
41
42 # strip ANSI per line (each line is short, so this stays cheap)
43 local i l
44 for i in "${!lines[@]}"; do
45 l=${lines[i]}
46 l=${l//$'\e'\[*([0-9;])[a-zA-Z]/}
47 l=${l//$'\e'\([a-zA-Z]/}
48 lines[i]=$l
49 done
50
51 local needle missing=0 line hit
52 for needle in "$@"; do
53 hit=0
54 for line in "${lines[@]}"; do
55 if [[ $line =~ $needle ]]; then
56 hit=1
57 break
58 fi
59 done
60 if [ "$hit" -eq 0 ]; then
61 echo "error: output missing '$needle'" >&2
62 missing=1
63 fi
64 done
65
66 if [ "$missing" -ne 0 ]; then
67 printf '%s\n' "${lines[@]}" >&2
68 return 1
69 fi
70}
71
72declare -a TEST_NAMES=()
73declare -a TEST_STATUSES=()
74declare -a TEST_TIMES=()
75SKIP_TEST_RC=200
76
77get_time_ms() {
78 local t="${EPOCHREALTIME:-}"
79 if [[ "$t" == *.* ]]; then
80 local secs="${t%.*}"
81 local subs="${t#*.}"
82 subs="${subs:0:3}"
83 while [ "${#subs}" -lt 3 ]; do
84 subs="${subs}0"
85 done
86 echo "${secs}${subs}"
87 else
88 echo "$(date +%s)000"
89 fi
90}
91
92format_duration() {
93 local ms=$1
94 local secs=$((ms / 1000))
95 local rem=$((ms % 1000))
96 printf "%d.%03ds" "$secs" "$rem"
97}
98
99print_summary() {
100 if [ "${#TEST_NAMES[@]}" -eq 0 ]; then
101 return
102 fi
103 printf "\n"
104 log "test summary"
105 echo "========================================="
106 local passed_count=0
107 local skipped_count=0
108 local failed_count=0
109 local total_time=0
110 for i in "${!TEST_NAMES[@]}"; do
111 local name="${TEST_NAMES[$i]}"
112 local status="${TEST_STATUSES[$i]}"
113 local duration_ms="${TEST_TIMES[$i]}"
114 local duration_str
115 duration_str=$(format_duration "$duration_ms")
116
117 local status_color="\033[0;32m"
118 if [ "$status" = "Failed" ]; then
119 status_color="\033[0;31m"
120 failed_count=$((failed_count + 1))
121 elif [ "$status" = "Skipped" ]; then
122 status_color="\033[0;33m"
123 skipped_count=$((skipped_count + 1))
124 else
125 passed_count=$((passed_count + 1))
126 fi
127 total_time=$((total_time + duration_ms))
128
129 printf " %-30s %b%-8b\033[0m %s\n" "$name" "$status_color" "$status" "$duration_str"
130 done
131 echo "-----------------------------------------"
132 local total_tests="${#TEST_NAMES[@]}"
133 local total_time_str
134 total_time_str=$(format_duration "$total_time")
135 printf " total: %d tests, %d passed, %d skipped, %d failed\n" "$total_tests" "$passed_count" "$skipped_count" "$failed_count"
136 printf " total execution time: %s\n" "$total_time_str"
137 echo "========================================="
138}
139
140skip_test() {
141 echo "skipped: $*"
142 return "$SKIP_TEST_RC"
143}
144
145host_is_nixos() {
146 [ -e /etc/NIXOS ] && return 0
147 [ -r /etc/os-release ] && grep -q '^ID=nixos$' /etc/os-release
148}
149
150JOBS="${JOBS:-4}"
151while [[ $# -gt 0 ]]; do
152 case "$1" in
153 -j | --jobs)
154 JOBS="$2"
155 shift 2
156 ;;
157 --jobs=*)
158 JOBS="${1#*=}"
159 shift
160 ;;
161 --only)
162 TEST_ONLY="$2"
163 shift 2
164 ;;
165 --only=*)
166 TEST_ONLY="${1#*=}"
167 shift
168 ;;
169 *)
170 echo "unknown argument: $1" >&2
171 echo "usage: $0 [-j N|--jobs N] [--only TEST]" >&2
172 exit 1
173 ;;
174 esac
175done
176if ! [[ "$JOBS" =~ ^[0-9]+$ ]] || [ "$JOBS" -lt 1 ]; then
177 echo "error: --jobs must be a positive integer (got '$JOBS')" >&2
178 exit 1
179fi
180
181pick_free_port() {
182 local port
183 for _ in $(seq 1 50); do
184 port=$(((RANDOM % 16384) + 20000))
185 if ! (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then
186 echo "$port"
187 return 0
188 fi
189 done
190 echo "error: could not find a free port for the cache" >&2
191 return 1
192}
193
194SUCCESS=0
195rm -rf /tmp/test-spindle-microvm-logs
196
197log "setup local cache & temp environment"
198TEMP_DIR=$(mktemp -d -t test-spindle-microvm-XXXXXX)
199
200if [ ! -e /dev/vsock ]; then
201 echo "error: /dev/vsock is missing; run sudo modprobe vhost_vsock" >&2
202 exit 1
203fi
204
205log "build spindle & microvm image tarball"
206nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run
207TARBALL_PATH=$(nix build .#spindle-nixos-image-tarball --no-link --print-out-paths)
208mkdir -p "$TEMP_DIR/image"
209tar -C "$TEMP_DIR/image" -xzf "$TARBALL_PATH"
210IMAGE_SPEC_JSON="$TEMP_DIR/image/spec.json"
211
212log "build alpine microvm image tarball"
213ALPINE_TARBALL_PATH=$(nix build .#spindle-alpine-image-tarball --no-link --print-out-paths)
214mkdir -p "$TEMP_DIR/alpine-image"
215tar -C "$TEMP_DIR/alpine-image" -xzf "$ALPINE_TARBALL_PATH"
216ALPINE_IMAGE_SPEC_JSON="$TEMP_DIR/alpine-image/spec.json"
217
218kill_temp_dir_procs() {
219 if [ -f "$TEMP_DIR/ncps.pid" ]; then
220 kill "$(cat "$TEMP_DIR/ncps.pid")" 2>/dev/null || true
221 fi
222 pkill -TERM -f "$TEMP_DIR" 2>/dev/null || true
223 local i
224 for i in $(seq 1 20); do
225 pgrep -f "$TEMP_DIR" >/dev/null 2>&1 || break
226 sleep 0.25
227 done
228 pkill -KILL -f "$TEMP_DIR" 2>/dev/null || true
229}
230
231collect_logs() {
232 echo "test failed. copying logs to /tmp/test-spindle-microvm-logs"
233 mkdir -p /tmp/test-spindle-microvm-logs
234 local f
235 for f in "$TEMP_DIR"/*; do
236 [ -f "$f" ] && cp "$f" /tmp/test-spindle-microvm-logs/
237 done
238 local work logf
239 for work in "$TEMP_DIR"/work-*; do
240 [ -d "$work" ] || continue
241 for logf in "$work"/*.log; do
242 [ -f "$logf" ] || continue
243 strip_ansi "$logf" > "/tmp/test-spindle-microvm-logs/$(basename "$work")-$(basename "$logf")"
244 done
245 done
246}
247
248CLEANED=0
249cleanup() {
250 [ "$CLEANED" -eq 1 ] && return
251 CLEANED=1
252
253 print_summary
254 log "cleaning up..."
255
256 local jobs_pids
257 jobs_pids=$(jobs -p)
258 [ -n "$jobs_pids" ] && kill $jobs_pids 2>/dev/null || true
259
260 kill_temp_dir_procs
261
262 [ "$SUCCESS" -ne 1 ] && collect_logs
263
264 chmod -R +w "$TEMP_DIR" 2>/dev/null || true
265 rm -rf "$TEMP_DIR"
266 echo "done"
267}
268trap cleanup EXIT
269# route signals through the EXIT trap so an interrupt still tears down VMs.
270trap 'exit 130' INT
271trap 'exit 143' TERM
272
273CACHE_PORT=$(pick_free_port)
274./spindle/engines/microvm/start-test-cache.sh "$TEMP_DIR" "$CACHE_PORT"
275source "$TEMP_DIR/env.sh"
276
277run_vm() {
278 local name=""
279 local timeout="60s"
280 local upload=0
281 local upload_url=""
282 local activate=""
283 local no_cache=0
284 local db=""
285 local spec="$IMAGE_SPEC_JSON"
286
287 while [[ $# -gt 0 ]]; do
288 case "$1" in
289 --spec)
290 spec="$2"
291 shift 2
292 ;;
293 --name)
294 name="$2"
295 shift 2
296 ;;
297 --timeout)
298 timeout="$2"
299 shift 2
300 ;;
301 --upload)
302 upload=1
303 shift
304 ;;
305 --upload-url)
306 upload=1
307 upload_url="$2"
308 shift 2
309 ;;
310 --activate)
311 activate="$2"
312 shift 2
313 ;;
314 --no-cache)
315 no_cache=1
316 shift
317 ;;
318 --db)
319 db="$2"
320 shift 2
321 ;;
322 --)
323 shift
324 break
325 ;;
326 *)
327 echo "unknown argument: $1" >&2
328 exit 1
329 ;;
330 esac
331 done
332
333 local work_dir="$TEMP_DIR/work-${name}"
334 mkdir -p "$work_dir"
335
336 local args=(
337 --image-spec "$spec"
338 --work-dir "$work_dir"
339 --exec-timeout "$timeout"
340 --port "${SPINDLE_TEST_VSOCK_PORT:-10240}"
341 --memory-mib 4096
342 )
343
344 if [ "$no_cache" -eq 0 ]; then
345 args+=(
346 --cache-read-url "$CACHE_URL"
347 --cache-trusted-public-key "$CACHE_PUBKEY"
348 )
349 fi
350
351 if [ "$upload" -eq 1 ]; then
352 if [ -z "$upload_url" ]; then
353 upload_url="$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH"
354 fi
355 args+=(
356 --cache-upload-url "$upload_url"
357 )
358 fi
359
360 if [ -n "$activate" ]; then
361 args+=(
362 --activate-config "$activate"
363 )
364 fi
365
366 if [ -n "$db" ]; then
367 args+=(
368 --db "$db"
369 )
370 fi
371
372 local out
373 if ! out=$(spindle/spindle-microvm-run "${args[@]}" -- "$@" 2>&1); then
374 echo "$out" | strip_ansi >&2
375 strip_ansi "$work_dir/serial.log" >&2
376 strip_ansi "$work_dir/qemu.log" >&2
377 return 1
378 fi
379 echo "$out"
380}
381
382run_test_job() {
383 local name="$1"
384 local func="$2"
385 local port="$3"
386 export SPINDLE_TEST_VSOCK_PORT="$port"
387
388 local logfile="$TEMP_DIR/test-${name}.log"
389 local start
390 start=$(get_time_ms)
391 log "[$name] start (vsock port $port)"
392
393 local status="Passed"
394 if "$func" > "$logfile" 2>&1; then
395 status="Passed"
396 else
397 local rc=$?
398 if [ "$rc" -eq "$SKIP_TEST_RC" ]; then
399 status="Skipped"
400 else
401 status="Failed"
402 fi
403 fi
404
405 local duration_ms=$(($(get_time_ms) - start))
406 printf '%s\t%s\n' "$status" "$duration_ms" > "$TEMP_DIR/test-${name}.status"
407
408 local duration_str
409 duration_str=$(format_duration "$duration_ms")
410 if [ "$status" = "Failed" ]; then
411 printf "\n\033[0;31m>>> [%s] FAILED (%s)\033[0m\n" "$name" "$duration_str"
412 strip_ansi "$logfile" || true
413 elif [ "$status" = "Skipped" ]; then
414 printf "\n\033[0;33m>>> [%s] skipped (%s)\033[0m\n" "$name" "$duration_str"
415 strip_ansi "$logfile" || true
416 else
417 printf "\n\033[0;32m>>> [%s] passed (%s)\033[0m\n" "$name" "$duration_str"
418 fi
419}
420
421# schedules every selected test across at most $JOBS concurrent VMs (each on its
422# own vsock port), then aggregates the per-test status files into the summary
423# arrays. returns 1 if any test failed, 0 otherwise.
424run_tests() {
425 local base_port=10240
426 local idx=0
427 local running=0
428
429 for func in "${TESTS[@]}"; do
430 local name="${func#test_}"
431 name="${name//_/-}"
432 if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then
433 continue
434 fi
435
436 run_test_job "$name" "$func" "$((base_port + idx))" &
437 idx=$((idx + 1))
438 running=$((running + 1))
439
440 if [ "$running" -ge "$JOBS" ]; then
441 wait -n || true
442 running=$((running - 1))
443 fi
444 done
445 wait
446
447 local failed=0
448 for func in "${TESTS[@]}"; do
449 local name="${func#test_}"
450 name="${name//_/-}"
451 if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then
452 continue
453 fi
454
455 local statusfile="$TEMP_DIR/test-${name}.status"
456 if [ ! -f "$statusfile" ]; then
457 TEST_NAMES+=("$name")
458 TEST_STATUSES+=("Failed")
459 TEST_TIMES+=(0)
460 failed=1
461 continue
462 fi
463
464 local status duration_ms
465 IFS=$'\t' read -r status duration_ms < "$statusfile"
466 TEST_NAMES+=("$name")
467 TEST_STATUSES+=("$status")
468 TEST_TIMES+=("$duration_ms")
469 if [ "$status" = "Failed" ]; then
470 failed=1
471 fi
472 done
473
474 return "$failed"
475}
476
477test_realize() {
478 local test_store_path
479 test_store_path=$(nix-build -E 'with import <nixpkgs> {}; writeText "test-file" "hello from cache"' --no-out-link)
480 nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path"
481
482 local out
483 out=$(run_vm --name "realize" --timeout "60s" -- /run/current-system/sw/bin/bash -lc '
484set -euo pipefail
485store_path=$1
486cache_url=$(sed -n "s/^extra-substituters = //p" /run/spindle/nix.conf)
487cache_url=${cache_url%% *}
488if [ -z "$cache_url" ]; then
489 echo "error: cache URL not found in /run/spindle/nix.conf" >&2
490 exit 1
491fi
492
493http_version=$(/run/current-system/sw/bin/curl --http2-prior-knowledge -fsS -o /dev/null -w "%{http_version}" "$cache_url/nix-cache-info")
494echo "http_version=$http_version"
495case "$http_version" in
496 2|2.0) ;;
497 *)
498 echo "error: cache proxy did not negotiate HTTP/2 (got $http_version)" >&2
499 exit 1
500 ;;
501esac
502
503if [ "$(stat -c %d /tmp)" = "$(stat -c %d /workspace)" ]; then
504 echo "tmp_on_persist=yes"
505else
506 echo "tmp_on_persist=no"
507fi
508
509/run/current-system/sw/bin/nix-store --realise "$store_path" >/dev/null
510' bash "$test_store_path") || return 1
511
512 check_needles "$out" "^http_version=2(\\.0)?$" "^tmp_on_persist=yes$" || return 1
513 echo "success: store path realized from cache and cache proxy accepted cleartext HTTP/2"
514}
515
516test_build_upload() {
517 local nix_expr='with import <nixpkgs> {}; writeText "uploaded-test-file" "hello from vm upload"'
518 local out
519 out=$(run_vm --name "build-upload" --timeout "120s" --upload -- /run/current-system/sw/bin/bash -l -c "nix-build -E '$nix_expr' --no-out-link") || return 1
520
521 local built_path
522 built_path=$(echo "$out" | strip_ansi | grep -v '\.drv' | grep -o '/nix/store/[a-z0-9]*-uploaded-test-file' | head -n 1 || true)
523 if [ -z "$built_path" ]; then
524 echo "error: could not find built store path in vm output" >&2
525 return 1
526 fi
527 echo "extracted path: $built_path"
528
529 local hash
530 hash=$(basename "$built_path" | cut -d'-' -f1)
531 if ! curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then
532 echo "error: built store path was not uploaded to the binary cache" >&2
533 return 1
534 fi
535 echo "success: store path uploaded to cache"
536}
537
538test_ssh_store_upload() {
539 if ! host_is_nixos; then
540 skip_test "ssh store upload smoke only runs on nixos hosts"
541 fi
542
543 run_ssh_store_upload_case() {
544 local target="$1"
545 local label="$2"
546 local name="uploaded-test-file-${label}"
547 local content="hello from vm upload via ${label}"
548 local out
549 out=$(run_vm --name "$label" --timeout "180s" --upload-url "$target" -- /run/current-system/sw/bin/bash -l -c "nix-build -E 'with import <nixpkgs> {}; writeText \"$name\" \"$content\"' --no-out-link") || return 1
550
551 local built_path
552 built_path=$(echo "$out" | strip_ansi | grep -v '\.drv' | grep -o "/nix/store/[a-z0-9]*-${name}" | head -n 1 || true)
553 if [ -z "$built_path" ]; then
554 echo "error: could not find built store path in vm output for $label" >&2
555 echo "$out" | strip_ansi >&2
556 return 1
557 fi
558 if [ ! -e "$built_path" ]; then
559 echo "error: uploaded path missing from host store for $label: $built_path" >&2
560 return 1
561 fi
562 if [ "$(cat "$built_path")" != "$content" ]; then
563 echo "error: uploaded path content mismatch for $label" >&2
564 echo "path=$built_path" >&2
565 return 1
566 fi
567 echo "success: store path uploaded to $target via spindle"
568 }
569
570 run_ssh_store_upload_case "ssh-ng://localhost" "ssh-ng-upload"
571 run_ssh_store_upload_case "ssh://localhost" "ssh-upload"
572}
573
574test_networking() {
575 local hello_path
576 hello_path=$(nix-build -E 'with import <nixpkgs> {}; hello' --no-out-link)
577
578 local out
579 out=$(run_vm --name "networking" --timeout "120s" --no-cache -- /run/current-system/sw/bin/bash -c "/run/current-system/sw/bin/curl -I --connect-timeout 1 -m 1 http://10.0.2.2:$CACHE_PORT; /run/current-system/sw/bin/nix-store --realise $hello_path") || return 1
580
581 if echo "$out" | grep -qi -E "unreachable|timeout|failed to connect|timed out" || echo "$out" | grep -q "exited with code"; then
582 echo "success: host network access blocked"
583 else
584 echo "error: guest vm accessed host network or returned unexpected output" >&2
585 echo "$out" | strip_ansi >&2
586 return 1
587 fi
588 echo "success: guest vm reached the internet and substituted hello"
589}
590
591test_substitution_and_no_upload() {
592 local hello_path
593 hello_path=$(nix-build -E 'with import <nixpkgs> {}; hello' --no-out-link)
594
595 local out
596 out=$(run_vm --name "nixpkgs-hello" --timeout "180s" --upload -- /run/current-system/sw/bin/nix-store --realise "$hello_path") || return 1
597
598 # substituted from our proxy, and nothing uploaded back to the cache.
599 check_needles "$out" "copying path.*hello" "cache uploaded: 0" || return 1
600 echo "success: hello package substituted from upstream cache and was not uploaded"
601}
602
603# a pinned registry, reused by the dependency and registry-pin tests.
604ACTIVATION_REGISTRY='"registry": {
605 "nixpkgs": "github:nixos/nixpkgs/nixos-unstable",
606 "my-nixpkgs": "nixpkgs"
607 }'
608
609test_activation_services() {
610 local config='{
611 "services": {
612 "openssh": {
613 "enable": true,
614 "authorizedKeysFiles": ["/etc/ssh/authorized_keys"]
615 }
616 }
617 }'
618 local out
619 out=$(run_vm --name "activation-services" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1
620 check_needles "$out" "^active$" || return 1
621 echo "success: openssh service active after activation"
622}
623
624test_activation_dependencies() {
625 # pkg-config + openssl as bare dependencies (resolved via the pinned nixpkgs
626 # registry); their job is to prove the activation devshell exports build env
627 # vars like PKG_CONFIG_PATH, not just PATH. hello comes in via the github
628 # flakeref and (separately) the my-nixpkgs alias.
629 local config='{
630 '"$ACTIVATION_REGISTRY"',
631 "dependencies": [
632 "pkg-config",
633 "openssl",
634 "github:nixos/nixpkgs#hello",
635 "my-nixpkgs#hello"
636 ]
637 }'
638 # dependencies live in the activation devshell now, not systemPackages, so a
639 # step picks them up by sourcing the materialised env (this is what the
640 # engine's RunStep does automatically; the CLI runner does not, so we do it
641 # here). this script is reused across the build and cache-hit runs below.
642 local job='
643env_file=/run/spindle/devshell-env.sh
644[ -f "$env_file" ] && echo "env_file=present" || echo "env_file=missing"
645echo "shuttle_env=$(systemctl show shuttle -p Environment --value)"
646echo "step_xdg_cache_home=${XDG_CACHE_HOME:-unset}"
647mkdir -p "$XDG_CACHE_HOME/go-build"
648[ -w "$XDG_CACHE_HOME/go-build" ] && echo "go_build_cache_writable=yes"
649if [ -d /var/cache/shuttle ] && [ "$(stat -c %d /var/cache/shuttle)" = "$(stat -c %d /workspace)" ]; then
650 echo "shuttle_cache_on_persist=yes"
651else
652 echo "shuttle_cache_on_persist=no"
653fi
654if find /var/cache/shuttle/nix -mindepth 1 -print -quit 2>/dev/null | grep -q .; then
655 echo "shuttle_nix_cache_written=yes"
656else
657 echo "shuttle_nix_cache_written=no"
658fi
659
660
661
662# mirror RunStep
663. "$env_file"
664export PATH="$PATH:/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin"
665
666set -euo pipefail
667# bare deps: pkg-config must be on PATH, and it must locate openssl through the
668# devshell-provided PKG_CONFIG_PATH (openssls headers live in a separate `dev`
669# output, so this also proves that output got realised into the devshell).
670echo "pkgconfig=$(pkg-config --version)"
671pkg-config --exists openssl && echo "openssl_found=yes"
672echo "openssl_version=$(pkg-config --modversion openssl)"
673inc=$(pkg-config --variable=includedir openssl)
674echo "includedir=$inc"
675[ -f "$inc/openssl/ssl.h" ] && echo "dev_headers=found"
676
677# flakeref + aliased deps
678echo "hello=$(hello)"
679'
680
681 local db_path="$TEMP_DIR/activation-dependencies.db"
682
683 # first run: build the config, materialise the devshell env profile, upload
684 # its closure, and record both toplevel + env profile in the db.
685 local out
686 out=$(run_vm --name "activation-dependencies" --timeout "600s" --activate "$config" --db "$db_path" --upload -- /run/current-system/sw/bin/bash -l -c "$job") || return 1
687
688 check_needles "$out" \
689 "env_file=present" "shuttle_env=.*XDG_CACHE_HOME=/var/cache/shuttle" \
690 "step_xdg_cache_home=/workspace/.cache" "go_build_cache_writable=yes" \
691 "shuttle_cache_on_persist=yes" "shuttle_nix_cache_written=yes" \
692 "pkgconfig=[0-9]" "openssl_found=yes" "openssl_version=[0-9]" \
693 "dev_headers=found" "hello=Hello, world!" || return 1
694 echo "success: bare (pkg-config + openssl, dev headers found via PKG_CONFIG_PATH), flakeref, and aliased deps all resolved"
695
696 # second run: same config + db, no upload. the devshell .drv is absent (no
697 # eval happens on a cache hit), so the env must be re-read from the cached
698 # env profile. the deps must still resolve exactly as on the build run.
699 out=$(run_vm --name "activation-dependencies-cached" --timeout "300s" --activate "$config" --db "$db_path" -- /run/current-system/sw/bin/bash -l -c "$job") || return 1
700
701 check_needles "$out" \
702 "realizing cached NixOS config" \
703 "env_file=present" "shuttle_env=.*XDG_CACHE_HOME=/var/cache/shuttle" \
704 "step_xdg_cache_home=/workspace/.cache" "go_build_cache_writable=yes" \
705 "shuttle_cache_on_persist=yes" "pkgconfig=[0-9]" "openssl_found=yes" \
706 "openssl_version=[0-9]" "dev_headers=found" "hello=Hello, world!" || return 1
707 echo "success: cache-hit run re-read the devshell env from the cached profile (no drv) and all deps resolved"
708}
709
710test_activation_registry_pin() {
711 # the nixpkgs the image itself was built from; the registry override must NOT
712 # resolve to this. deterministic (locked in the repo flake), so safe to compare.
713 local base_nixpkgs
714 base_nixpkgs=$(nix eval --raw --impure --expr '(builtins.getFlake (toString ./.)).inputs.nixpkgs.outPath')
715
716 local config='{
717 '"$ACTIVATION_REGISTRY"'
718 }'
719 # the pinned nixpkgs must reach the system nix config: resolving it via the
720 # flakes CLI must not error "is not locked", and it must win the <nixpkgs> nixPath.
721 local out
722 out=$(run_vm --name "activation-registry-pin" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c '
723set -euo pipefail
724echo "lib_version=$(nix eval --raw nixpkgs#lib.version)"
725echo "nix_path=$(nix eval --raw --impure --expr "toString <nixpkgs>")"
726') || return 1
727
728 check_needles "$out" "lib_version=[0-9]" || return 1
729 local clean guest_nixpath
730 clean=$(echo "$out" | strip_ansi)
731 guest_nixpath=$(echo "$clean" | sed -n 's/^nix_path=//p' | head -n1)
732 if [ -z "$guest_nixpath" ] || [ "$guest_nixpath" = "$base_nixpkgs" ]; then
733 echo "error: guest <nixpkgs> nixPath did not resolve to the registry override (got '$guest_nixpath', base '$base_nixpkgs')" >&2
734 return 1
735 fi
736 echo "success: pinned nixpkgs registry reached the guest nix config (flakes CLI + <nixpkgs> nixPath)"
737}
738
739test_activation_cache_substitution() {
740 # a unique path that only exists in the configured (workflow) cache; the host
741 # seeds it so the guest can prove it substitutes through the read proxy.
742 local test_store_path
743 test_store_path=$(nix-build -E 'with import <nixpkgs> {}; writeText "activation-cache-test" "hello from the workflow cache"' --no-out-link)
744 nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path"
745
746 # a trivial config: this test only cares that the read proxy serves the
747 # workflow-cache path during an activated run.
748 local out
749 out=$(run_vm --name "activation-cache-substitution" --timeout "300s" --activate '{}' -- /run/current-system/sw/bin/bash -l -c '
750set -euo pipefail
751store_path=$1
752# the unique path only exists in the configured cache, so realising it proves it
753# was substituted through the read proxy and not built or found elsewhere.
754nix-store --realise "$store_path" >/dev/null
755echo "substituted=$(cat "$store_path")"
756' bash "$test_store_path") || return 1
757
758 check_needles "$out" "substituted=hello from the workflow cache" || return 1
759 echo "success: unique path substituted from the workflow cache through the read proxy"
760}
761
762test_activation_docker() {
763 local config='{
764 "virtualisation": {
765 "docker": { "enable": true }
766 }
767 }'
768 # docker.service is up, but the daemon socket can lag a beat behind activation;
769 # wait for it to answer, then pull+run a real image. this drives the slimmed
770 # kernel modules: overlay.ko storage plus bridge/iptables networking out of the
771 # pruned tree, with outbound DNS/network over the guest slirp link.
772 local out
773 out=$(run_vm --name "activation-docker" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c '
774set -euo pipefail
775echo "docker_unit=$(systemctl is-active docker)"
776for i in $(seq 1 60); do docker info >/dev/null 2>&1 && break; sleep 1; done
777docker info >/dev/null
778echo "storage_driver=$(docker info --format "{{.Driver}}")"
779docker run --rm alpine cat /etc/alpine-release | sed "s/^/alpine_release=/"
780docker run --rm alpine echo container-ran-ok
781') || return 1
782
783 check_needles "$out" \
784 "^docker_unit=active$" "storage_driver=overlay(2|fs)" \
785 "alpine_release=[0-9]+\." "container-ran-ok" || return 1
786 echo "success: docker service active, pulled and ran an alpine container on the overlay storage driver"
787}
788
789test_activation_cached_realize() {
790 local config='{
791 "services": {
792 "openssh": {
793 "enable": true,
794 "authorizedKeysFiles": ["/etc/ssh/authorized_keys"]
795 }
796 }
797 }'
798 local db_path="$TEMP_DIR/activation-cached.db"
799
800 # first run: build the config, upload its closure, and record the toplevel in
801 # the db. nothing cached yet, so this builds from scratch.
802 local out
803 out=$(run_vm --name "activation-cached-first" --timeout "600s" --activate "$config" --db "$db_path" --upload -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1
804 check_needles "$out" "^active$" || return 1
805
806 # second run: same config + db, no upload. must realize the recorded toplevel
807 # from the cache instead of rebuilding, and the cached system must come up.
808 out=$(run_vm --name "activation-cached-second" --timeout "300s" --activate "$config" --db "$db_path" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1
809
810 check_needles "$out" "realizing cached NixOS config" "^active$" || return 1
811 echo "success: second run realized the cached NixOS config from the cache and sshd came up"
812}
813
814test_alpine() {
815 local hello_path
816 hello_path=$(nix-build -E 'with import <nixpkgs> {}; hello' --no-out-link)
817
818 local out
819 out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine" --timeout "180s" --no-cache -- /bin/sh -lc '
820set -eu
821export HOME=/workspace
822hello_path=$1
823echo "release=$(cat /etc/alpine-release)"
824echo "user=$(id -un)"
825git version
826bash -c "echo bash=\$BASH_VERSION"
827touch /workspace/write-test
828echo "workspace writable"
829git ls-remote https://tangled.org/@tangled.org/core HEAD >/dev/null
830echo "git over https ok"
831apk add make
832echo "apk ok"
833# substitute a real package from cache.nixos.org over HTTPS and run it
834nix-store --realise "$hello_path" >/dev/null
835echo "ran=$("$hello_path/bin/hello")"
836' sh "$hello_path") || return 1
837
838 check_needles "$out" \
839 "release=" "user=spindle-workflow" "git version" "bash=" \
840 "workspace writable" "git over https ok" "apk ok" "ran=Hello, world!" || return 1
841 echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned + installed over the network, and substituted+ran a package from cache.nixos.org over HTTPS"
842}
843
844test_alpine_podman() {
845 # install podman via apk and run a real container as the workflow user.
846 # rootless podman lives entirely in the writable workspace (storage + runroot
847 # under XDG dirs there), uses podman's default storage driver, and pulls over
848 # the guest network like the other alpine tests.
849 local out
850 out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-podman" --timeout "300s" --no-cache -- /bin/sh -lc '
851set -eu
852# no env setup here on purpose: shuttle seeds USER/LOGNAME/HOME/SHELL from the
853# workflow users passwd entry and provisions XDG_RUNTIME_DIR, so rootless podman
854# works out of the box. asserting those below doubles as a check on that.
855
856# shadow-uidmap ships newuidmap/newgidmap which rootless podman uses to apply the
857# /etc/subuid + /etc/subgid ranges baked into the image.
858apk add podman shadow-uidmap
859echo "user=$(id -un) USER=$USER HOME=$HOME XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR"
860echo "newuidmap=$(command -v newuidmap)"
861echo "podman_version=$(podman --version)"
862
863podman info >/dev/null
864echo "storage_driver=$(podman info --format "{{.Store.GraphDriverName}}")"
865
866podman run --rm --network=host docker.io/library/alpine cat /etc/alpine-release | sed "s/^/container_release=/"
867podman run --rm --network=host docker.io/library/alpine echo container-ran-ok
868' sh) || return 1
869
870 check_needles "$out" \
871 "user=spindle-workflow USER=spindle-workflow HOME=/workspace XDG_RUNTIME_DIR=/run/user/970" \
872 "newuidmap=/" "podman_version=" \
873 "storage_driver=" "container_release=[0-9]+\." "container-ran-ok" || return 1
874 echo "success: alpine guest installed podman via apk and pulled + ran a rootless container"
875}
876
877# asserts a store path's narinfo shows up in the local cache, retrying briefly
878# since the post-build-hook enqueues uploads asynchronously.
879cache_has_path() {
880 local path="$1"
881 local hash
882 hash=$(basename "$path" | cut -d'-' -f1)
883 local i
884 for i in $(seq 1 20); do
885 if curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then
886 return 0
887 fi
888 sleep 0.5
889 done
890 return 1
891}
892
893test_alpine_nix() {
894 local test_store_path
895 test_store_path=$(nix-build -E 'with import <nixpkgs> {}; writeText "alpine-nix-test" "hello from cache to alpine"' --no-out-link)
896 nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path"
897
898 # exercise the full local-cache path: daemon connectivity, substitution,
899 # store-db queries, and a build via *both* the classic (nix-build) and the
900 # new flakes/nix-command (nix build) frontends. the two build derivations
901 # use distinct names so we can confirm each got uploaded back to the cache.
902 local out
903 out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-nix" --timeout "180s" --upload -- /bin/sh -lc '
904set -eu
905export HOME=/workspace
906store_path=$1
907
908echo "nix_version=$(nix --version | head -n1)"
909{ nix store info >/dev/null 2>&1 || nix store ping >/dev/null 2>&1; } && echo "daemon=ok"
910
911# substitute a path from the cache and query the store db about it
912nix-store --realise "$store_path" >/dev/null
913echo "substituted=$(cat "$store_path")"
914echo "requisites=$(nix-store --query --requisites "$store_path" | wc -l | tr -d " ")"
915nix path-info --json "$store_path" >/dev/null && echo "path_info=ok"
916
917# build via the new CLI; the substituted path is declared as a real input
918# (builtins.storePath) so nix must realise it into the build sandbox first.
919# heredoc is unquoted (the outer guest script is single-quoted, so a quoted
920# delimiter would close it), hence \$ escapes what nix/the builder must expand.
921export DEP="$store_path"
922cat > /workspace/new.nix <<NIXEOF
923let dep = builtins.storePath (builtins.getEnv "DEP"); in
924derivation {
925 name = "alpine-nix-build-new";
926 system = "x86_64-linux";
927 builder = "/bin/sh";
928 # the sandbox only provides the sh builtin shell (no coreutils in PATH), so
929 # stick to builtins: the build only succeeds if nix realised the declared
930 # storePath dependency into the sandbox, where [ -r ] can see it.
931 args = [ "-c" "[ -r \${dep} ] && echo via-nix-build-with-dep > \$out" ];
932}
933NIXEOF
934new_path=$(nix build --impure --file /workspace/new.nix --no-link --print-out-paths)
935echo "new_path=$new_path"
936echo "new_content=$(tr "\n" "|" < "$new_path")"
937
938# build via the classic CLI
939cat > /workspace/old.nix <<NIXEOF
940derivation {
941 name = "alpine-nix-build-old";
942 system = "x86_64-linux";
943 builder = "/bin/sh";
944 args = [ "-c" "echo built-on-alpine > \$out" ];
945}
946NIXEOF
947old_path=$(nix-build /workspace/old.nix --no-out-link)
948echo "old_path=$old_path"
949' sh "$test_store_path") || return 1
950
951 check_needles "$out" \
952 "daemon=ok" "substituted=hello from cache to alpine" "path_info=ok" \
953 "requisites=[1-9][0-9]*" "new_content=via-nix-build-with-dep" || return 1
954
955 local clean new_path old_path
956 clean=$(echo "$out" | strip_ansi)
957 new_path=$(echo "$clean" | grep -o 'new_path=/nix/store/[a-z0-9]*-alpine-nix-build-new' | cut -d= -f2)
958 old_path=$(echo "$clean" | grep -o 'old_path=/nix/store/[a-z0-9]*-alpine-nix-build-old' | cut -d= -f2)
959 if [ -z "$new_path" ] || [ -z "$old_path" ]; then
960 echo "error: could not extract both built store paths from alpine guest output" >&2
961 return 1
962 fi
963 if ! cache_has_path "$new_path"; then
964 echo "error: nix-build (new CLI) output was not uploaded to the cache" >&2
965 return 1
966 fi
967 if ! cache_has_path "$old_path"; then
968 echo "error: nix-build (classic CLI) output was not uploaded to the cache" >&2
969 return 1
970 fi
971 echo "success: alpine guest substituted, queried the store db, built via both CLIs, and uploaded both outputs"
972}
973
974TESTS=(
975 test_alpine
976 test_alpine_nix
977 test_alpine_podman
978 test_realize
979 test_build_upload
980 test_ssh_store_upload
981 test_networking
982 test_substitution_and_no_upload
983 test_activation_services
984 test_activation_dependencies
985 test_activation_registry_pin
986 test_activation_cache_substitution
987 test_activation_docker
988 test_activation_cached_realize
989)
990
991log "running ${#TESTS[@]} tests"
992if ! run_tests; then
993 exit 1
994fi
995
996SUCCESS=1
997log "passed!!"