gazagnaire.org
Example mirage-eio unikernels
unikernels: turn tcp-server and tcp-client into a multicore probe
The echo pair proved the stack; it could not show whether the domains
pool buys anything over TCP. The server now offloads a CPU-bound LCG
chain per connection to the pool (accept and socket I/O stay on the
boot domain, where the from-scratch stack lives), and the client fires
the same requests sequentially then all at once: the ratio is the
multicore speedup as seen from outside, checksum-verified both ways.
author
Thomas Gazagnaire
date
(Jul 21, 2026, 12:35 AM UTC)
commit
fa12ed7e
fa12ed7e99b331e999d27e792f1e21f5cab2382e
parent
94637df2
94637df2350d30aa6213752ecec352317b577177
+87
-36
+51
-15
bin/network/tcp-client/unikernel.ml
+51
-15
bin/network/tcp-client/unikernel.ml
···
1
-
(* TCP client on the from-scratch stack. Net.v gives the guest's own
2
-
Ethernet/ARP/IPv4/TCP tower as a plain Eio.Net.t, so dialling is ordinary
3
-
Eio; the guest terminates the TCP and the slirp gateway NATs it to the host.
4
-
Pairs with network/tcp-server.
1
+
(* Load client for the multicore tcp-server: fire [--requests] compute requests
2
+
sequentially, then all at once. The concurrent/sequential ratio is the
3
+
server's multicore speedup seen from outside; the checksum agrees both ways.
5
4
6
-
unikernel-tcp-client --ipv4-gateway 10.0.0.1 --server 127.0.0.1:7 *)
5
+
unikernel-tcp-client --ipv4 10.0.0.1/24 --frame eth0=6010:6011 --server 10.0.0.2:80 *)
7
6
8
7
let server_arg =
9
8
Cmdliner.Arg.(
10
9
value
11
10
@@ opt
12
-
(Mirage_eio.Arg.host_port ~default:7 ())
13
-
(Ipaddr.of_string_exn "127.0.0.1", 7)
11
+
(Mirage_eio.Arg.host_port ~default:80 ())
12
+
(Ipaddr.of_string_exn "10.0.0.2", 80)
14
13
@@ info ~doc:"TCP server to dial, HOST:PORT." ~docv:"HOST:PORT" [ "server" ])
15
14
16
-
let () =
17
-
Mirage_eio.(run [ Net.v (netif "eth0"); arg server_arg ])
18
-
@@ fun _devices net server ->
15
+
let requests_arg =
16
+
Cmdliner.Arg.(
17
+
value @@ opt int 10
18
+
@@ info ~doc:"Number of compute requests to fire." ~docv:"N" [ "requests" ])
19
+
20
+
(* One request: dial, send the seed, read the checksum back. A fresh switch (and
21
+
so a fresh connection) per request, so the sequential and concurrent runs
22
+
differ only in whether the requests overlap. *)
23
+
let request net ip port seed =
19
24
Eio.Switch.run @@ fun sw ->
20
-
let ip, port = server in
21
-
Fmt.pr "tcp-client: dialling %a:%d@." Ipaddr.pp ip port;
22
25
let flow = Eio.Net.connect ~sw net (Eio_addr.tcp ip port) in
23
-
Eio.Flow.copy_string "hello from a unikernel\n" flow;
24
-
let buf = Eio.Buf_read.of_flow ~max_size:4096 flow in
25
-
Fmt.pr "tcp-client: server said %S@." (Eio.Buf_read.line buf)
26
+
Eio.Flow.copy_string (Fmt.str "%d\n" seed) flow;
27
+
let buf = Eio.Buf_read.of_flow ~max_size:64 flow in
28
+
int_of_string (String.trim (Eio.Buf_read.line buf))
29
+
30
+
let elapsed_s clock f =
31
+
let t0 = Eio.Time.now clock in
32
+
let v = f () in
33
+
(v, Eio.Time.now clock -. t0)
34
+
35
+
let () =
36
+
Mirage_eio.(
37
+
run [ Net.v (netif "eth0"); clock; arg server_arg; arg requests_arg ])
38
+
@@ fun _devices net clock server n ->
39
+
let ip, port = server in
40
+
let seeds = List.init n (fun i -> i + 1) in
41
+
Fmt.pr "tcp-client: %d compute requests to %a:%d@." n Ipaddr.pp ip port;
42
+
(* Sequential: one request at a time -- the server pool holds one chain. *)
43
+
let seq_sum, seq =
44
+
elapsed_s clock (fun () ->
45
+
List.fold_left (fun acc seed -> acc + request net ip port seed) 0 seeds)
46
+
in
47
+
(* Concurrent: every request in flight at once -- the server pool spreads the
48
+
chains across its worker cores. Each result lands at its own index, so the
49
+
sum cannot depend on completion order. *)
50
+
let results = Array.make n 0 in
51
+
let conc_sum, conc =
52
+
elapsed_s clock (fun () ->
53
+
Eio.Fiber.all
54
+
(List.mapi
55
+
(fun i seed () -> results.(i) <- request net ip port seed)
56
+
seeds);
57
+
Array.fold_left ( + ) 0 results)
58
+
in
59
+
Fmt.pr "tcp-client: checksum=%d agree=%b@." conc_sum (conc_sum = seq_sum);
60
+
Fmt.pr "tcp-client: seq=%.3fs conc=%.3fs speedup=%.2fx@." seq conc
61
+
(seq /. conc)
+1
-1
bin/network/tcp-server/dune
+1
-1
bin/network/tcp-server/dune
+34
-18
bin/network/tcp-server/unikernel.ml
+34
-18
bin/network/tcp-server/unikernel.ml
···
1
-
(* TCP server on the from-scratch stack: a TCP echo service over the guest's own
2
-
Ethernet/ARP/IPv4/TCP tower. Net.listener presents the stack as a
3
-
plain Eio listening socket, so accepting and echoing is ordinary Eio. The
4
-
partner of tcp-client: run both on one raw frame link and the client's
5
-
connection is served directly, no gateway between them.
1
+
(* A multicore TCP server: each connection sends a seed, the server offloads the
2
+
CPU-bound chain to the [domains] pool and writes back its checksum. Accept and
3
+
socket I/O stay on the boot domain (the from-scratch stack is domain-local);
4
+
only the compute crosses to a worker core, so concurrent clients are served in
5
+
parallel.
6
6
7
7
unikernel-tcp-server --ipv4 10.0.0.2/24 --frame eth0=6011:6010 *)
8
8
9
-
let echo flow =
10
-
let buf = Cstruct.create 4096 in
11
-
let rec loop () =
12
-
match Eio.Flow.single_read flow buf with
13
-
| n ->
14
-
Eio.Flow.write flow [ Cstruct.sub buf 0 n ];
15
-
loop ()
16
-
| exception End_of_file -> Eio.Flow.shutdown flow `Send
9
+
(* A tail-recursive, allocation-free LCG: a worker runs it without a GC pause. *)
10
+
let[@inline never] work seed iterations =
11
+
let rec loop i acc =
12
+
if i = 0 then acc else loop (i - 1) ((acc * 1103515245) + 12345)
17
13
in
18
-
loop ()
14
+
loop iterations seed
19
15
20
16
let port = 80
17
+
let iterations = 20_000_000
18
+
19
+
(* Read the seed, run the chain on the pool, write the checksum. *)
20
+
let serve pool flow =
21
+
let buf = Eio.Buf_read.of_flow ~max_size:64 flow in
22
+
match Eio.Buf_read.line buf with
23
+
| line ->
24
+
let seed = try int_of_string (String.trim line) with _ -> 0 in
25
+
let checksum =
26
+
Eio.Executor_pool.submit_exn pool ~weight:1.0 (fun () ->
27
+
work seed iterations)
28
+
in
29
+
Eio.Flow.copy_string (Fmt.str "%d\n" checksum) flow
30
+
| exception End_of_file -> ()
31
+
32
+
(* Enlarge the minor heap before the pool exists (uncontended: only the boot
33
+
domain runs at module init). Each submission allocates a closure on the boot
34
+
domain; a roomy minor heap keeps its collections -- stop-the-world pauses
35
+
that park every worker core -- rare while requests are in flight. *)
36
+
let () = Gc.set { (Gc.get ()) with minor_heap_size = 4_000_000 }
21
37
22
38
let () =
23
-
Mirage_eio.(run [ Net.listener ~backlog:8 ~port (netif "eth0") ])
24
-
@@ fun _devices listening ->
39
+
Mirage_eio.(run [ domains 10; Net.listener ~backlog:16 ~port (netif "eth0") ])
40
+
@@ fun _devices pool listening ->
25
41
Eio.Switch.run @@ fun sw ->
26
-
Fmt.pr "tcp-server: echoing on port %d@." port;
42
+
Fmt.pr "tcp-server: multicore compute service on port %d@." port;
27
43
while true do
28
44
let flow, _addr = Eio.Net.accept ~sw listening in
29
-
Eio.Fiber.fork ~sw (fun () -> echo flow)
45
+
Eio.Fiber.fork ~sw (fun () -> serve pool flow)
30
46
done
+1
-1
dune-project
+1
-1
dune-project