gazagnaire.org
Solo5 hvt ABI and MFT1/ABI1 note format as wire codecs
10 kB
354 lines
1#!/bin/sh
2# End-to-end boot harness for solo5-kvm under *real* KVM on an Apple-silicon Mac.
3#
4# There is no /dev/kvm on macOS, but Apple's Virtualization.framework can expose
5# nested virtualisation to a Linux guest (M3+, macOS 15+). So this:
6#
7# 1. builds, in a Linux/arm64 container, the solo5-kvm tender and the small
8# aarch64 hvt guests, and packs them into an initramfs with a busybox init;
9# 2. boots a linuxkit kernel + that initramfs in a VM via `kvm-vz` (nested
10# virt on), where the guest kernel is itself a KVM host;
11# 3. the init runs each guest through solo5-kvm on the guest's own /dev/kvm and
12# prints a marker the host greps for.
13#
14# It is the solo5-hvf boot harness (test/boot/run.sh) closed over the KVM
15# backend: byte-identical guests, asserted the same way, but on real KVM.
16#
17# Needs Docker (for the Linux build) and `kvm-vz` (the ocaml-vz launcher). SKIPs
18# cleanly when either is missing or the host lacks nested virt.
19#
20# Usage: run_kvm_vz.sh [PATH/TO/kvm-vz] (defaults to dune's built kvm-vz)
21
22set -eu
23
24root=$(cd "$(dirname "$0")/../../.." && pwd) # monorepo root
25work=$(mktemp -d)
26trap 'rm -rf "$work"' EXIT
27skip() { echo "SKIP: $1"; exit 0; }
28
29command -v docker >/dev/null 2>&1 || skip "no docker"
30docker info >/dev/null 2>&1 || skip "no docker daemon"
31
32# A linuxkit kernel (6.6.13): an arm64 Image that boots from an external
33# initramfs and has arm64 KVM built in. (Docker Desktop's own kernel is built for
34# a disk root and silently ignores an initramfs, so it cannot be used here.)
35# Resolve the linuxkit OCI image from uniboot's cache if warm, else pull it; the
36# kernel ships gzipped, so decompress to a raw Image.
37kver=linuxkit/kernel:6.6.13
38kernel="$work/kernel"
39cached=$(find "$HOME/.cache/uniboot/checkout/linuxkit" -path '*rootfs/kernel' 2>/dev/null | head -1)
40if [ -n "$cached" ]; then
41 gunzip -c "$cached" >"$kernel" 2>/dev/null || cp "$cached" "$kernel"
42else
43 echo "solo5-kvm-vz: pulling $kver ..."
44 docker pull --platform linux/arm64 "$kver" >/dev/null 2>&1 || skip "cannot pull $kver"
45 cid=$(docker create --platform linux/arm64 "$kver" 2>/dev/null) || skip "cannot create $kver"
46 docker cp "$cid:/kernel" "$work/kernel.gz" >/dev/null 2>&1 \
47 || { docker rm "$cid" >/dev/null 2>&1; skip "no /kernel in $kver"; }
48 docker rm "$cid" >/dev/null 2>&1
49 gunzip -c "$work/kernel.gz" >"$kernel" 2>/dev/null || cp "$work/kernel.gz" "$kernel"
50fi
51[ -s "$kernel" ] || skip "kernel resolution failed"
52
53# kvm-vz: argument, or dune's codesigned build.
54kvmvz=${1:-}
55if [ -z "$kvmvz" ]; then
56 kvmvz="$root/_build/default/ocaml-vz/bin/kvm-vz-signed"
57 [ -x "$kvmvz" ] || (cd "$root" && dune build ocaml-vz/bin/kvm-vz-signed) \
58 || skip "cannot build kvm-vz"
59fi
60[ -x "$kvmvz" ] || skip "no kvm-vz at $kvmvz"
61
62image="mono-linux-test:ocaml-5.4"
63docker image inspect "$image" >/dev/null 2>&1 \
64 || skip "base image $image missing (run ./docker-test.sh once to build it)"
65
66# ---- The in-VM init: run each guest on the guest's real /dev/kvm. -------------
67# Built into the initramfs as /init. Markers: HAVE-DEVKVM, per-guest OK/FAIL, and
68# ALL-PASS, which the host asserts.
69cat >"$work/init" <<'INIT'
70#!/bin/busybox sh
71busybox mount -t devtmpfs dev /dev 2>/dev/null
72busybox mount -t proc proc /proc 2>/dev/null
73busybox mount -t sysfs sys /sys 2>/dev/null
74export PATH=/bin
75[ -e /dev/kvm ] || busybox mknod /dev/kvm c 10 232 2>/dev/null
76echo "KVMVZ-INIT-START"
77if [ -e /dev/kvm ]; then echo "KVMVZ-HAVE-DEVKVM"; else echo "KVMVZ-NO-DEVKVM"; fi
78pass=0; fail=0
79# A guest that hangs inside KVM_RUN (runaway, W^X fault loop) is stopped the way
80# upstream Solo5 stops one -- by a signal. A background killer sends SIGTERM after
81# a deadline (no timeout(1) applet in this initramfs), which the tender's handler
82# turns into exit 2; fast guests finish first and the killer is reaped.
83run() { # run NAME EXPECT
84 name=$1; want=$2
85 /solo5-kvm --mem=2 "/guests/$name.hvt" >/g.out 2>&1 &
86 gpid=$!
87 ( busybox sleep 2; busybox kill -TERM "$gpid" 2>/dev/null ) &
88 kpid=$!
89 wait "$gpid"; rc=$?
90 busybox kill "$kpid" 2>/dev/null
91 out=$(busybox cat /g.out)
92 if [ "$rc" = "$want" ]; then
93 echo " OK $name (exit $rc) ${out:+| $out}"; pass=$((pass+1))
94 else
95 echo " FAIL $name: exit $rc (want $want) | $out"; fail=$((fail+1))
96 fi
97}
98run halt 42
99run hello 0
100run oob 2
101run badhc 2
102run runaway 2
103run wx 2
104run data_noexec 2
105run virtio_magic 0
106run poll 0
107echo "=== seccomp audit (denied syscalls, if any) ==="
108dmesg 2>/dev/null | grep -iE "seccomp|syscall=" | tail -12
109echo "KVMVZ-RESULT pass=$pass fail=$fail"
110[ "$fail" = 0 ] && echo "KVMVZ-ALL-PASS"
111busybox poweroff -f
112INIT
113
114# ---- Build the Linux artifacts + initramfs in the container. -----------------
115# Native aarch64 in the container, so plain gcc/as assemble the guests (no cross
116# toolchain) and the solo5-kvm tender builds from source. The guest .S bodies are
117# the same as test/boot/run_kvm.sh; kept compact here (the hand-assembled set,
118# without the cross-cc real-unikernel and slirp cases).
119build='
120set -eu
121B=/tmp/dune-build
122dune build ocaml-solo5/bin/main_kvm.exe
123out=/tmp/kvmvz-out
124rm -rf $out && mkdir -p $out/root/bin $out/root/guests $out/root/dev \
125 $out/root/proc $out/root/sys
126
127# busybox + a shell and applets.
128cp /bin/busybox.static $out/root/bin/busybox 2>/dev/null || cp "$(command -v busybox)" $out/root/bin/busybox
129$out/root/bin/busybox --install -s $out/root/bin
130
131# the tender + its dynamic loader/libs (musl).
132cp $B/default/ocaml-solo5/bin/main_kvm.exe $out/root/solo5-kvm
133for lib in $(ldd $out/root/solo5-kvm 2>/dev/null | awk "/=>/{print \$3} /ld-musl/{print \$1}" | sort -u); do
134 [ -f "$lib" ] && { mkdir -p "$out/root$(dirname "$lib")"; cp "$lib" "$out/root$lib"; }
135done
136
137# the init.
138cp /src-out-init $out/root/init
139chmod +x $out/root/init $out/root/solo5-kvm
140
141# Solo5 ABI + manifest notes shared by every guest (.balign: .align is 2^N on ARM).
142notes() {
143cat <<EOF
144.section .note.solo5.abi, "a"
145.balign 4
146.long 6
147.long 16
148.long 0x31494241
149.asciz "Solo5"
150.balign 4
151.long 1
152.long 2
153.long 0
154.long 0
155.section .note.solo5.manifest, "a"
156.balign 4
157.long 6
158.long 12
159.long 0x3154464d
160.asciz "Solo5"
161.balign 4
162.long 0
163.long 1
164.long 0
165.text
166.balign 4
167.global _start
168_start:
169EOF
170}
171cat >/tmp/guest.ld <<EOF
172ENTRY(_start)
173PHDRS { text PT_LOAD FLAGS(5); data PT_LOAD FLAGS(6); note PT_NOTE FLAGS(4); }
174SECTIONS {
175 . = 0x100000;
176 .text : { *(.text) } :text
177 . = ALIGN(0x4000);
178 .data : { *(.data) } :data
179 .note : { *(.note.solo5.abi) *(.note.solo5.manifest) } :note
180}
181EOF
182build_guest() { { notes; cat; } >/tmp/$1.S; as -o /tmp/$1.o /tmp/$1.S; ld -T /tmp/guest.ld --build-id=none -o $out/root/guests/$1.hvt /tmp/$1.o; }
183
184build_guest halt <<EOF
185 movz x1, #0x40
186 movk x1, #0x1, lsl #32
187 adr x0, hc
188 str w0, [x1]
1891: b 1b
190.balign 8
191hc: .quad 0
192 .word 42
193 .word 0
194EOF
195build_guest hello <<EOF
196 adr x2, msg
197 adr x3, ps
198 str x2, [x3]
199 movz x1, #0x10
200 movk x1, #0x1, lsl #32
201 mov w0, w3
202 str w0, [x1]
203 movz x1, #0x40
204 movk x1, #0x1, lsl #32
205 adr x0, hs
206 str w0, [x1]
2071: b 1b
208msg: .ascii "hello\n"
209.set mlen, . - msg
210.data
211.balign 8
212ps: .quad 0
213 .quad mlen
214hs: .quad 0
215 .word 0
216 .word 0
217EOF
218build_guest oob <<EOF
219 movz x1, #0x10
220 movk x1, #0x1, lsl #32
221 movz w0, #0xffff
222 movk w0, #0xffff, lsl #16
223 str w0, [x1]
2241: b 1b
225EOF
226build_guest badhc <<EOF
227 movz x1, #0x100
228 movk x1, #0x1, lsl #32
229 movz w0, #0
230 str w0, [x1]
2311: b 1b
232EOF
233build_guest runaway <<EOF
234 udf #0
235EOF
236build_guest wx <<EOF
237 adr x2, _start
238 str w2, [x2]
239 movz x1, #0x40
240 movk x1, #0x1, lsl #32
241 adr x0, hs
242 str w0, [x1]
2431: b 1b
244.data
245.balign 8
246hs: .quad 0
247 .word 0
248 .word 0
249EOF
250build_guest data_noexec <<EOF
251 adr x0, data_exec
252 br x0
253.data
254.balign 4
255data_exec:
256 movz x1, #0x40
257 movk x1, #0x1, lsl #32
258 adr x0, data_hs
259 str w0, [x1]
2601: b 1b
261.balign 8
262data_hs: .quad 0
263 .word 0
264 .word 0
265EOF
266build_guest virtio_magic <<EOF
267 movz x1, #0x4000, lsl #16
268 movk x1, #0x1, lsl #32
269 ldr w0, [x1]
270 movz w2, #0x6976
271 movk w2, #0x7472, lsl #16
272 cmp w0, w2
273 b.ne fail
274 movz x1, #0x40
275 movk x1, #0x1, lsl #32
276 adr x0, hs_ok
277 str w0, [x1]
2781: b 1b
279fail:
280 movz x1, #0x40
281 movk x1, #0x1, lsl #32
282 adr x0, hs_fail
283 str w0, [x1]
2842: b 2b
285.data
286.balign 8
287hs_ok: .quad 0
288 .word 0
289 .word 0
290hs_fail: .quad 0
291 .word 1
292 .word 0
293EOF
294
295# A guest with no net device that issues POLL with a 1 ms deadline, then halts 0.
296# It exercises the no-device POLL wait, which on Linux is epoll_pwait under the
297# seccomp sandbox; a clean exit 0 proves the epoll path holds. POLL is hypercall
298# 3 (0x18); the struct is IN timeout_nsecs, OUT ready_set + ret, so it lives in
299# writable .data.
300build_guest poll <<EOF
301 adr x0, pp
302 movz x1, #0x18
303 movk x1, #0x1, lsl #32
304 str w0, [x1]
305 movz x1, #0x40
306 movk x1, #0x1, lsl #32
307 adr x0, hs
308 str w0, [x1]
3091: b 1b
310.data
311.balign 8
312pp: .quad 1000000
313 .quad 0
314 .word 0
315 .word 0
316hs: .quad 0
317 .word 0
318 .word 0
319EOF
320
321# Pack the initramfs.
322( cd $out/root && find . | busybox cpio -o -H newc 2>/dev/null | gzip -9 ) >$out/initramfs.cpio.gz
323echo "initramfs: $(wc -c <$out/initramfs.cpio.gz) bytes"
324cp $out/initramfs.cpio.gz /out-host/initramfs.cpio.gz
325'
326
327echo "solo5-kvm-vz: building Linux artifacts + initramfs in $image ..."
328docker run --rm --privileged \
329 -v "$root":/src \
330 -v /src/_opam \
331 -v "$work":/out-host \
332 -v "$work/init":/src-out-init:ro \
333 -w /src \
334 -e DUNE_BUILD_DIR=/tmp/dune-build \
335 "$image" \
336 bash -ceu "$build"
337
338[ -f "$work/initramfs.cpio.gz" ] || { echo "FAIL: initramfs not built"; exit 1; }
339
340# ---- Boot it under nested KVM and assert the markers. ------------------------
341echo "solo5-kvm-vz: booting under kvm-vz (nested virt) ..."
342"$kvmvz" --timeout 120 --cmdline "console=hvc0" \
343 --initrd "$work/initramfs.cpio.gz" "$kernel" 2>&1 | tee "$work/console.log" || true
344
345echo "---"
346if grep -q KVMVZ-ALL-PASS "$work/console.log"; then
347 echo "solo5-kvm-vz: ALL PASS (guests ran on real /dev/kvm)"
348 exit 0
349else
350 grep -q KVMVZ-NO-DEVKVM "$work/console.log" \
351 && echo "solo5-kvm-vz: guest kernel has no /dev/kvm (kernel lacks CONFIG_KVM)"
352 echo "solo5-kvm-vz: FAILED (no ALL-PASS marker)"
353 exit 1
354fi