paulvall.dev
forked from
tangled.org/core
Monorepo for Tangled
shuttle,spindle/engines/microvm: have shuttle detect step ooms properly, parse serial for oom / kernel panic
Signed-off-by: dawn <dawn@tangled.org>
author
dawn
date
(Jul 24, 2026, 3:56 PM +0300)
commit
3c9fec1e
3c9fec1ef19cae68cbc71374ea1911f9c87abf0e
parent
9a53797e
9a53797ed94bd62f88da2870156bb8c200bf0b1f
change-id
zvmxymky
zvmxymkypyxquvszwprnmozznpkrlrls
+131
-17
+5
cmd/spindle-microvm-run/main_linux.go
+5
cmd/spindle-microvm-run/main_linux.go
+44
-10
shuttle/src/command.rs
+44
-10
shuttle/src/command.rs
···
153
153
})
154
154
}
155
155
156
+
fn read_oom_kill_count() -> u32 {
157
+
let content = match std::fs::read_to_string("/proc/vmstat") {
158
+
Ok(c) => c,
159
+
Err(_) => return 0,
160
+
};
161
+
for line in content.lines() {
162
+
let mut parts = line.split_whitespace();
163
+
if parts.next() == Some("oom_kill") {
164
+
if let Some(count_str) = parts.next() {
165
+
if let Ok(count) = count_str.parse::<u32>() {
166
+
return count;
167
+
}
168
+
}
169
+
}
170
+
}
171
+
0
172
+
}
173
+
156
174
pub fn spawn_streaming(mut spec: Spec) -> Result<StreamingCommand> {
175
+
let oom_kill_before = read_oom_kill_count();
157
176
let mut child = spawn(&mut spec)?;
158
177
let stdout = child.stdout.take().context("stdout pipe missing")?;
159
178
let stderr = child.stderr.take().context("stderr pipe missing")?;
···
164
183
drop(events_tx);
165
184
166
185
let exit = tokio::spawn(async move {
167
-
let exit = wait_child(&mut child, spec.timeout).await;
186
+
let exit = wait_child(&mut child, spec.timeout, oom_kill_before).await;
168
187
169
188
// ensure all output is observed before exiting
170
189
// this assumes children dont daemonize and hold onto the stdout/err
···
226
245
.with_context(|| format!("spawn {}", display_os(&spec.program)))
227
246
}
228
247
229
-
async fn wait_child(child: &mut Child, timeout: Option<Duration>) -> ExitResult {
248
+
async fn wait_child(
249
+
child: &mut Child,
250
+
timeout: Option<Duration>,
251
+
oom_kill_before: u32,
252
+
) -> ExitResult {
230
253
let wait = child.wait();
231
254
let status = match timeout {
232
255
Some(timeout) => match tokio::time::timeout(timeout, wait).await {
···
249
272
};
250
273
251
274
match status {
252
-
Ok(status) => ExitResult {
253
-
exit_code: status
254
-
.code()
255
-
.or_else(|| status.signal().map(|signal| 128 + signal))
256
-
.unwrap_or(1),
257
-
error: None,
258
-
timed_out: false,
259
-
},
275
+
Ok(status) => {
276
+
let code = status.code();
277
+
let signal = status.signal();
278
+
let exit_code = code.or_else(|| signal.map(|sig| 128 + sig)).unwrap_or(1);
279
+
280
+
let mut error = None;
281
+
if signal == Some(9) {
282
+
let oom_kill_after = read_oom_kill_count();
283
+
if oom_kill_after > oom_kill_before {
284
+
error = Some("guest process killed by guest kernel OOM".to_owned());
285
+
}
286
+
}
287
+
288
+
ExitResult {
289
+
exit_code,
290
+
error,
291
+
timed_out: false,
292
+
}
293
+
}
260
294
Err(error) => ExitResult {
261
295
exit_code: 1,
262
296
error: Some(error.to_string()),
+3
-1
spindle/engines/microvm/agent.go
+3
-1
spindle/engines/microvm/agent.go
···
186
186
} else if p := msg.BuiltPaths; p != nil {
187
187
// s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths))
188
188
} else if p := msg.ExecExit; p != nil {
189
+
var err error
189
190
if p.Error != "" {
190
191
s.l.Warn("guest exec error", "id", msg.Id, "error", p.Error)
192
+
err = fmt.Errorf("guest exec error: %s", p.Error)
191
193
}
192
194
if p.TimedOut {
193
195
return int(p.ExitCode), errGuestTimedOut
194
196
}
195
-
return int(p.ExitCode), nil
197
+
return int(p.ExitCode), err
196
198
}
197
199
}
198
200
}
+16
-4
spindle/engines/microvm/engine.go
+16
-4
spindle/engines/microvm/engine.go
···
11
11
"os"
12
12
"path/filepath"
13
13
"slices"
14
+
"strings"
14
15
"sync"
15
16
"sync/atomic"
16
17
"time"
···
248
249
if setupDone {
249
250
return
250
251
}
251
-
if detail := vmCrashLog(state.VM); detail != "" {
252
+
if detail := VMCrashLog(state.VM); detail != "" {
252
253
l.Error("microVM setup failed", "detail", detail)
253
254
}
254
255
if err := e.cleanupState(context.Background(), wid, state); err != nil {
···
416
417
if oom {
417
418
reason = "microVM killed by OOM (cgroup memory limit exceeded)"
418
419
}
419
-
if detail := vmCrashLog(state.VM); detail != "" {
420
+
if detail := VMCrashLog(state.VM); detail != "" {
420
421
fmt.Fprintf(stderr, "%s:\n%s\n", reason, detail)
421
422
l.Error(reason, "oom", oom, "detail", detail)
422
423
} else {
···
434
435
// the agent connection dropped while qemu stayed up (eg. the guest kernel
435
436
// OOM-killed the agent or a guest panic), so surface serial logs, those
436
437
// will be more helpful.
437
-
if detail := vmCrashLog(state.VM); detail != "" {
438
+
var crashErr error
439
+
if detail := VMCrashLog(state.VM); detail != "" {
438
440
fmt.Fprintf(stderr, "step failed (%v):\n%s\n", err, detail)
439
441
l.Error("step failed", "error", err, "detail", detail)
442
+
if parsedErr, ok := ParseCrashLog(detail); ok {
443
+
crashErr = parsedErr
444
+
} else {
445
+
if strings.Contains(err.Error(), "guest exec error:") {
446
+
crashErr = err
447
+
} else {
448
+
crashErr = fmt.Errorf("guest agent connection lost: %w", err)
449
+
}
450
+
}
440
451
} else {
441
452
l.Error("step failed", "error", err)
453
+
crashErr = err
442
454
}
443
-
return fmt.Errorf("%s:\n%w", category, err)
455
+
return fmt.Errorf("%s:\n%w", category, crashErr)
444
456
}
445
457
446
458
func (e *Engine) activateConfig(ctx context.Context, wid models.WorkflowId, state *workflowState, step Step, out io.Writer) error {
+1
-1
spindle/engines/microvm/qemu.go
+1
-1
spindle/engines/microvm/qemu.go
+42
spindle/engines/microvm/test-spindle-microvm.sh
+42
spindle/engines/microvm/test-spindle-microvm.sh
···
971
971
echo "success: alpine guest substituted, queried the store db, built via both CLIs, and uploaded both outputs"
972
972
}
973
973
974
+
test_oom_detection() {
975
+
local label spec
976
+
for label in "alpine" "nixos"; do
977
+
echo "testing oom on $label..."
978
+
local work_dir="$TEMP_DIR/work-oom-test-$label"
979
+
mkdir -p "$work_dir"
980
+
981
+
local -a cmd_args
982
+
if [ "$label" = "alpine" ]; then
983
+
spec="$ALPINE_IMAGE_SPEC_JSON"
984
+
cmd_args=(awk 'BEGIN { while(1) a[i++]=1 }')
985
+
else
986
+
spec="$IMAGE_SPEC_JSON"
987
+
cmd_args=(/run/current-system/sw/bin/jq -n '[repeat(1)]')
988
+
fi
989
+
990
+
local mem_mib=128
991
+
if [ "$label" = "nixos" ]; then
992
+
mem_mib=512
993
+
fi
994
+
995
+
local args=(
996
+
--image-spec "$spec"
997
+
--work-dir "$work_dir"
998
+
--exec-timeout "45s"
999
+
--port "$SPINDLE_TEST_VSOCK_PORT"
1000
+
--memory-mib "$mem_mib"
1001
+
)
1002
+
1003
+
local out
1004
+
if out=$(./spindle/spindle-microvm-run "${args[@]}" -- "${cmd_args[@]}" 2>&1); then
1005
+
echo "error: expected spindle-microvm-run to fail on $label, but it exited 0" >&2
1006
+
echo "output: $out" >&2
1007
+
return 1
1008
+
fi
1009
+
1010
+
check_needles "$out" "killed by guest kernel OOM" || return 1
1011
+
done
1012
+
echo "success: guest process OOM detected successfully on both alpine and nixos"
1013
+
}
1014
+
974
1015
TESTS=(
975
1016
test_alpine
976
1017
test_alpine_nix
···
986
1027
test_activation_cache_substitution
987
1028
test_activation_docker
988
1029
test_activation_cached_realize
1030
+
test_oom_detection
989
1031
)
990
1032
991
1033
log "running ${#TESTS[@]} tests"
+20
-1
spindle/engines/microvm/vm.go
+20
-1
spindle/engines/microvm/vm.go
···
14
14
"os"
15
15
"os/exec"
16
16
"path/filepath"
17
+
"regexp"
17
18
"slices"
18
19
"strings"
19
20
"sync/atomic"
···
285
286
return watchCtx, exited, cancel
286
287
}
287
288
288
-
func vmCrashLog(vm VMHandle) string {
289
+
func VMCrashLog(vm VMHandle) string {
289
290
if vm == nil {
290
291
return ""
291
292
}
···
373
374
374
375
return runner.Start(ctx, cfg, volumePaths, logger)
375
376
}
377
+
378
+
// checks serial log for ooms or kernel panic
379
+
// this is very linux specific! but these strings are stable in linux itself, see mm/oom_kill.c and kernel/panic.c
380
+
func ParseCrashLog(detail string) (error, bool) {
381
+
if strings.Contains(detail, "Out of memory:") {
382
+
// we can show process name where possible
383
+
re := regexp.MustCompile(`Out of memory: Killed process \d+ \(([^)]+)\)`)
384
+
matches := re.FindStringSubmatch(detail)
385
+
if len(matches) > 1 {
386
+
return fmt.Errorf("guest out of memory (process '%s' killed by guest kernel OOM)", matches[1]), true
387
+
}
388
+
return errors.New("guest out of memory (OOM killer invoked)"), true
389
+
}
390
+
if strings.Contains(detail, "Kernel panic") {
391
+
return errors.New("guest kernel panic"), true
392
+
}
393
+
return nil, false
394
+
}