gazagnaire.org
Example mirage-eio unikernels
unikernels: check the smp pool against a fixed checksum
The example moved to an Eio.Executor_pool demo but its cram test and README
still described the Domain.spawn Fibonacci demo it replaced, so the committed
test failed and the README claimed unikraft schedules domains cooperatively on
the boot core -- the opposite of what the backend now does.
Each job now computes an LCG chain seeded by its position and the unikernel
reports the sum, which is fixed by the arithmetic alone, so the test pins down
that every job ran exactly once and computed the right chain rather than only
that the run printed a timing. The parallel total is also compared against the
same total computed sequentially. --jobs and --iterations size the workload, so
the machine check stays fast and deterministic while the defaults still
saturate the pool for long enough to measure.
Being a directory in unikernels/bin rather than under a category, smp was
invisible to `mrg registry`, which walks category/name -- so its cross-build
arms were never regenerated or diffed and had gone stale. Under simple/ the
registry regenerates them, which is what picks up the cmdliner dependency the
boot arguments need.
author
Thomas Gazagnaire
date
(Jul 18, 2026, 8:56 PM UTC)
commit
ad920e0d
ad920e0d450f5b69d97b29121f86e71206ebf7b8
parent
4128eb4e
4128eb4e94ac80569a498a01c264a3b4cf861a4b
+142
-69
+16
-1
bin/gen/dune
+16
-1
bin/gen/dune
···
36
36
(diff dune dune.gen)))
37
37
38
38
; Coverage. Every unikernel above should carry a README, a cram test that
39
-
; boots it, and a GIF recorded from that cram. All of them do.
39
+
; boots it, and a GIF recorded from that cram. These do not, yet:
40
+
;
41
+
; simple/smp no GIF
40
42
41
43
(rule
42
44
(target detect.dune.gen)
···
596
598
(alias runtest)
597
599
(action
598
600
(diff hello-world.dune hello-world.dune.gen)))
601
+
602
+
(rule
603
+
(target smp.dune.gen)
604
+
(deps ../simple/smp/dune)
605
+
(action
606
+
(with-stdout-to
607
+
smp.dune.gen
608
+
(run %{exe:../../../mirage/bin/main.exe} dune ../simple/smp))))
609
+
610
+
(rule
611
+
(alias runtest)
612
+
(action
613
+
(diff smp.dune smp.dune.gen)))
599
614
600
615
(rule
601
616
(target tm-downlink.dune.gen)
+6
-5
bin/gen/smp.dune
+6
-5
bin/gen/smp.dune
···
3
3
(enabled_if
4
4
(= %{context_name} default.solo5))
5
5
(modules :standard)
6
-
(libraries eio nox-mirage.eio nox-mirage.eio.solo5)
6
+
(libraries eio nox-mirage.eio cmdliner nox-mirage.eio.solo5)
7
7
(ocamlopt_flags
8
8
(:standard \ (:include solo5-release-debug.sexp)))
9
9
(link_flags
···
80
80
(enabled_if
81
81
(= %{context_name} default.unikraft))
82
82
(modules :standard)
83
-
(libraries eio nox-mirage.eio nox-mirage.eio.unikraft)
83
+
(libraries eio nox-mirage.eio cmdliner nox-mirage.eio.unikraft)
84
84
(ocamlopt_flags
85
85
(:standard \ (:include unikraft-release-debug.sexp)))
86
86
(link_flags
···
150
150
(= %{context_name} default))
151
151
(modules :standard)
152
152
(modes wasm)
153
-
(libraries eio nox-mirage.eio nox-mirage.eio.wasi hash.ocaml)
153
+
(libraries eio nox-mirage.eio cmdliner nox-mirage.eio.wasi hash.ocaml)
154
154
(wasm_of_ocaml
155
155
(flags
156
156
(:standard --enable=wasi --effects=cps --opt 3)))))
···
159
159
(enabled_if
160
160
(= %{context_name} default.xen))
161
161
(modules :standard)
162
-
(libraries eio nox-mirage.eio nox-mirage.eio.xen-runtime)
162
+
(libraries eio nox-mirage.eio cmdliner nox-mirage.eio.xen-runtime)
163
163
(ocamlopt_flags
164
164
(:standard \ (:include xen-release-debug.sexp)))
165
165
(link_flags
166
-
((:include xen-release-flags.sexp) -cclib -z -cclib solo5-abi=xen))
166
+
((:include xen-release-flags.sexp) -cclib -z -cclib solo5-abi=xen -cclib
167
+
-Wl,--defsym,TEXT_START=0x40000000))
167
168
(foreign_stubs
168
169
(language c)
169
170
(names xstartup)))
+43
-33
bin/simple/smp/README.md
+43
-33
bin/simple/smp/README.md
···
1
-
# smp -- a multidomain unikernel
1
+
# smp -- a multicore unikernel
2
2
3
-
Proof that the OCaml 5 multicore runtime works under a unikernel: the
4
-
unikernel spawns two extra domains with `Domain.spawn`, runs a CPU-bound
5
-
Fibonacci on each and a third on the main domain, and joins them. It
6
-
claims no devices, so it runs on the mirage-eio scheduler alone.
3
+
Proof that the OCaml 5 multicore runtime does real parallel work under a
4
+
unikernel. The unikernel declares a worker pool with the `domains` device
5
+
and submits CPU-bound jobs to it with `Eio.Executor_pool`, timing them
6
+
against the same jobs run one after another on the boot domain. On a
7
+
multi-core target the pool spreads them across CPU cores, so the parallel
8
+
run finishes in roughly `1/workers` of the sequential time.
7
9
8
10
```
9
-
mrg up -t unikraft unikernels/bin/smp
10
-
mrg build -t unikraft unikernels/bin/smp
11
-
mrg run -t unikraft unikernels/bin/smp
11
+
mrg up -t unikraft unikernels/bin/simple/smp
12
+
mrg build -t unikraft unikernels/bin/simple/smp
13
+
mrg run -t unikraft --cores 11 --mem 1024 unikernels/bin/simple/smp
12
14
```
13
15
14
-
Boot output (the per-domain lines interleave by scheduling; the results
15
-
are fixed):
16
+
`--cores` sizes the guest: above 1 it boots under a hardware accelerator
17
+
so the cores run at the same time rather than taking turns. `--mem` is
18
+
needed because the runtime reserves a minor heap for every domain slot at
19
+
startup, which does not fit in the default guest.
20
+
21
+
Boot output, with ten workers on eleven lcpus:
16
22
17
23
```
18
-
smp: recommended_domain_count = 1
19
-
+[worker-A] fib(32) = 2178309
20
-
+[worker-B] fib(33) = 3524578
21
-
+[main] fib(31) = 1346269
22
-
joined: main=1346269 A=2178309 B=3524578 -- multidomain OK
24
+
jobs=100 checksum=1166416129462705082 agree=true
25
+
seq=3.774s par=0.466s speedup=8.10x
23
26
```
24
27
25
-
The leading `+` is `traceln`'s marker for output from a non-main domain,
26
-
so it is direct evidence that the workers ran on their own domains.
28
+
The checksum is the sum over the jobs of an LCG chain seeded by each job's
29
+
position, so it is fixed by the arithmetic alone: a pool that dropped,
30
+
duplicated or truncated a job cannot produce it, and it comes out
31
+
identical on every target. `agree=true` says the parallel total matches
32
+
the sequential one. `--jobs` and `--iterations` size the workload without
33
+
a rebuild, which is how `smp.t` turns the demo into a fast, deterministic
34
+
machine check.
27
35
28
36
## Domains across targets
29
37
30
-
`Domain.spawn` always creates a real OCaml domain; whether two domains
31
-
run on two cores depends on the target:
38
+
The `domains n` device always asks for a pool of up to `n` workers; how
39
+
many the backend grants, and whether they run on separate cores, depends
40
+
on the target:
32
41
33
-
- **native** places each domain on a separate core -- true parallelism,
34
-
`recommended_domain_count` reflecting the host.
35
-
- **unikraft** brings the domains up on the guest and schedules them
36
-
cooperatively on a single logical CPU (`recommended_domain_count = 1`);
37
-
the join results are identical to native. Unikraft boots multiple cores
38
-
but has no SMP thread scheduler (upstream issue #587), so threads -- and
39
-
thus domains -- stay on the boot core. The backend caps the domain count
40
-
through `OCAMLRUNPARAM=d=N` baked into its environment.
41
-
- **solo5** and **xen** are single-core and likewise run the domains
42
-
cooperatively.
42
+
- **native** grants up to `n` workers on the host's cores -- measured
43
+
10.00x here with ten workers.
44
+
- **unikraft** grants up to `n` workers and places each on its own
45
+
secondary lcpu, so the speedup tracks the `--cores` count. Measured
46
+
8.10x with ten workers on eleven lcpus; the shortfall from 10x on this
47
+
host is core heterogeneity (6 performance + 12 efficiency cores), not
48
+
scheduling -- the workers that land on efficiency cores straggle, and
49
+
each wave waits for its slowest member.
50
+
- **solo5**, **xen** and **wasi** are single-core: the pool grants one
51
+
worker, warns that it granted fewer than requested, and runs each job
52
+
inline on the boot domain, so the checksum is identical and there is no
53
+
speedup.
43
54
44
-
Either way the joined results are the same, which is what the cram test
45
-
in `smp.t` checks: it filters the boot output to the three Fibonacci
46
-
results, whose values are fixed, so the check is deterministic regardless
47
-
of print order or host core count.
55
+
Because the checksum is target-independent, `smp.t` checks the same
56
+
values whatever the host core count and whatever order the workers
57
+
finished in.
+2
-2
bin/simple/smp/dune
+2
-2
bin/simple/smp/dune
···
5
5
(enabled_if
6
6
(= %{context_name} default))
7
7
(modules :standard)
8
-
(libraries eio nox-mirage.eio nox-mirage.eio.native))
8
+
(libraries eio nox-mirage.eio nox-mirage.eio.native cmdliner))
9
9
10
-
(include ../gen/smp.dune)
10
+
(include ../../gen/smp.dune)
11
11
12
12
(cram
13
13
(deps unikernel.exe)
+26
-12
bin/simple/smp/smp.t
+26
-12
bin/simple/smp/smp.t
···
1
-
Machine-check for the smp multidomain demo.
1
+
Machine-check for the smp executor-pool demo.
2
2
3
-
The native binary spawns two worker domains plus the main domain, each computing
4
-
a Fibonacci number, and joins them. The per-domain lines interleave by
5
-
scheduling and the reported domain count varies by host, so we filter to the
6
-
three fib results and sort them: those values are fixed, so this proves every
7
-
domain ran its computation to completion regardless of ordering or host.
3
+
Each job computes an LCG chain seeded by its position, and the unikernel reports
4
+
the sum over all of them. That sum is fixed by the arithmetic alone -- iterating
5
+
`acc * 1103515245 + 12345` from seeds 1..20 for 1000 steps and adding the
6
+
results in OCaml's wrapping 63-bit int -- so it pins down that every job ran
7
+
exactly once and computed the right chain, whatever the host core count and
8
+
whatever order the workers finished in. A pool that dropped, duplicated or
9
+
truncated a job cannot produce this number.
8
10
9
-
$ unikernel.exe 2>&1 | grep -oE 'fib\([0-9]+\) = [0-9]+' | sort
10
-
fib(31) = 1346269
11
-
fib(32) = 2178309
12
-
fib(33) = 3524578
11
+
$ unikernel.exe --jobs 20 --iterations 1000 2>&1 | grep -oE 'jobs=.*'
12
+
jobs=20 checksum=2109584137243543154 agree=true
13
13
14
-
The final line is printed once, on the main domain, after all workers join.
14
+
`agree=true` says the parallel total equals the same total computed sequentially
15
+
on the boot domain, so the pool is checked against a live baseline as well as
16
+
against the constant.
17
+
18
+
A second workload, to show the checksum tracks the parameters rather than being
19
+
a single memorised value:
20
+
21
+
$ unikernel.exe --jobs 8 --iterations 500 2>&1 | grep -oE 'jobs=.*'
22
+
jobs=8 checksum=4241412821196074308 agree=true
23
+
24
+
The timings vary with the host, so we check only that the speedup line is
25
+
reported and well formed. What it measures is the point of the demo: on a
26
+
multi-core target the parallel run beats the sequential baseline by roughly the
27
+
worker count (~9x on ten Unikraft secondary lcpus, ~9.4x native).
15
28
16
-
$ unikernel.exe 2>&1 | grep -c 'multidomain OK'
29
+
$ unikernel.exe --jobs 8 --iterations 500 2>&1 |
30
+
> grep -cE 'seq=[0-9]+\.[0-9]+s par=[0-9]+\.[0-9]+s speedup=[0-9]+\.[0-9]+x'
17
31
1
+45
-13
bin/simple/smp/unikernel.ml
+45
-13
bin/simple/smp/unikernel.ml
···
3
3
pool spreads the jobs across secondary CPU cores, so running them through the
4
4
pool beats the sequential baseline by roughly the available core count. *)
5
5
6
-
let[@inline never] work n =
6
+
(* Each job is a distinct LCG chain, seeded by its position, so the sum over the
7
+
jobs is one fixed number that a pool which dropped, duplicated or truncated a
8
+
job cannot produce. The loop is tail-recursive and allocation-free, so a
9
+
worker never needs the stop-the-world collector in the middle of a job. *)
10
+
let[@inline never] work seed iterations =
7
11
let rec loop i acc =
8
12
if i = 0 then acc else loop (i - 1) ((acc * 1103515245) + 12345)
9
13
in
10
-
loop n 1
14
+
loop iterations seed
11
15
12
16
let elapsed_s clock f =
13
17
let t0 = Eio.Time.now clock in
14
18
f ();
15
19
Eio.Time.now clock -. t0
16
20
21
+
(* The workload is a boot parameter: the defaults saturate a ten-worker pool for
22
+
long enough to measure a speedup, and the machine check dials both down so it
23
+
stays a correctness check rather than a benchmark. *)
24
+
let workload =
25
+
let jobs =
26
+
Cmdliner.Arg.(
27
+
value @@ opt int 100
28
+
@@ info ~doc:"The number of CPU-bound jobs to run." ~docv:"N" [ "jobs" ])
29
+
in
30
+
let iterations =
31
+
Cmdliner.Arg.(
32
+
value @@ opt int 40_000_000
33
+
@@ info ~doc:"The length of the chain each job computes." ~docv:"N"
34
+
[ "iterations" ])
35
+
in
36
+
Cmdliner.Term.(
37
+
const (fun jobs iterations -> (jobs, iterations)) $ jobs $ iterations)
38
+
17
39
(* Enlarge the minor heap before the worker pool exists (only the boot domain is
18
40
up at module init, so this resize is uncontended). The parallel run submits
19
41
one closure per job from the boot domain, and that submission allocates; a
···
24
46
let () = Gc.set { (Gc.get ()) with minor_heap_size = 4_000_000 }
25
47
26
48
let () =
27
-
Mirage_eio.run Mirage_eio.[ domains 10; clock ] @@ fun _rt pool clock ->
28
-
let iters = 40_000_000 and jobs = 100 in
29
-
let one () = ignore (work iters : int) in
49
+
Mirage_eio.run Mirage_eio.[ domains 10; clock; arg workload ]
50
+
@@ fun _rt pool clock (jobs, iterations) ->
51
+
let seeds = List.init jobs (fun index -> index + 1) in
52
+
let sequential = ref 0 and parallel = ref 0 in
30
53
(* Sequential baseline: the jobs one after another on the boot domain. *)
31
54
let seq =
32
55
elapsed_s clock (fun () ->
33
-
for _ = 1 to jobs do
34
-
one ()
35
-
done)
56
+
sequential :=
57
+
List.fold_left (fun acc seed -> acc + work seed iterations) 0 seeds)
36
58
in
37
59
(* Parallel: one fiber per job, each submitting to the pool, which runs up to
38
-
[domains] of them at once, one per secondary core. *)
60
+
[domains] of them at once, one per secondary core. Each result lands at its
61
+
own index, so summing them cannot depend on the completion order. *)
62
+
let results = Array.make jobs 0 in
39
63
let par =
40
64
elapsed_s clock (fun () ->
41
65
Eio.Fiber.all
42
-
(List.init jobs (fun _ () ->
43
-
Eio.Executor_pool.submit_exn pool ~weight:1.0 one)))
66
+
(List.mapi
67
+
(fun index seed () ->
68
+
results.(index) <-
69
+
Eio.Executor_pool.submit_exn pool ~weight:1.0 (fun () ->
70
+
work seed iterations))
71
+
seeds);
72
+
parallel := Array.fold_left ( + ) 0 results)
44
73
in
45
-
Eio.traceln "jobs=%d seq=%.3fs par=%.3fs speedup=%.2fx" jobs seq par
46
-
(seq /. par)
74
+
(* The checksum proves every job ran exactly once and computed the right
75
+
chain; the timings say what the pool won by. *)
76
+
Eio.traceln "jobs=%d checksum=%d agree=%b" jobs !parallel
77
+
(!parallel = !sequential);
78
+
Eio.traceln "seq=%.3fs par=%.3fs speedup=%.2fx" seq par (seq /. par)
+2
-2
dune-project
+2
-2
dune-project
···
30
30
31
31
(package
32
32
(name unikernel-smp)
33
-
(synopsis "SMP unikernel: spawns and joins extra OCaml domains")
33
+
(synopsis "Multicore unikernel: runs CPU-bound jobs on a pool of domains")
34
34
(tags (org:blacksun system))
35
-
(depends (eio (>= 1.0)) nox-mirage))
35
+
(depends (eio (>= 1.0)) nox-mirage cmdliner))
36
36
37
37
(package
38
38
(name unikernel-tcp-server)
+2
-1
unikernel-smp.opam
+2
-1
unikernel-smp.opam
···
1
1
# This file is generated by dune, edit dune-project instead
2
2
opam-version: "2.0"
3
-
synopsis: "SMP unikernel: spawns and joins extra OCaml domains"
3
+
synopsis: "Multicore unikernel: runs CPU-bound jobs on a pool of domains"
4
4
tags: ["org:blacksun" "system"]
5
5
homepage: "https://tangled.org/gazagnaire.org/unikernels"
6
6
bug-reports: "https://tangled.org/gazagnaire.org/unikernels/issues"
···
8
8
"dune" {>= "3.21"}
9
9
"eio" {>= "1.0"}
10
10
"nox-mirage"
11
+
"cmdliner"
11
12
"odoc" {with-doc}
12
13
]
13
14
build: [