spindle/microvm: cap each workflow vm's cpu and io in its cgroup #7

Signed-off-by: dawn dawn@tangled.org

+69 -6

5 changed files

Diff Round #0

Expand all

docs

DOCS.md

spindle

config

config.go

engines

microvm

cgroup.go

cid_test.go

engine.go

docs/DOCS.md

··· 1495 1495 per workflow cgroup (default: `4096`). 1496 1496 - `SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB`: Max swap 1497 1497 per workflow cgroup (default: `0`, no swap). 1498 + - `SPINDLE_MICROVM_PIPELINES_CGROUP_CPU_MAX_PERCENT`: Max CPU 1499 + quota per workflow cgroup, as a percentage of one core 1500 + (default: `0`, which caps each VM at its configured vCPU 1501 + count, a negative value disables the CPU limit). 1502 + - `SPINDLE_MICROVM_PIPELINES_CGROUP_IO_WEIGHT`: IO weight per 1503 + workflow cgroup, 1-10000 (default: `0`, IO unlimited). 1498 1504 - `SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB`: 1499 1505 Memory protected for spindle itself so it isn't OOM-killed 1500 1506 before the workflows (default: `512`).

spindle/config/config.go

··· 83 83 CgroupParent string `env:"CGROUP_PARENT, default=self"` 84 84 CgroupPidsMax int64 `env:"CGROUP_PIDS_MAX, default=4096"` 85 85 CgroupSwapMaxMiB *int64 `env:"CGROUP_SWAP_MAX_MIB"` 86 + // cpu.max quota as a percentage of one core. 0 caps each vm at its 87 + // configured vcpu count, negative disables the limit 88 + CgroupCPUMaxPercent int64 `env:"CGROUP_CPU_MAX_PERCENT, default=0"` 89 + // io.weight for workflow cgroups (1-10000, kernel default 100). 0 leaves 90 + // io unlimited 91 + CgroupIOWeight uint64 `env:"CGROUP_IO_WEIGHT, default=0"` 86 92 // memory.min that will get assigned to the supervisor (spindle itself) cgroup 87 93 CgroupSupervisorMemoryMinMiB int64 `env:"CGROUP_SUPERVISOR_MEMORY_MIN_MIB, default=512"` 88 94 }

+28

spindle/engines/microvm/cgroup.go

··· 7 7 "os" 8 8 "path/filepath" 9 9 "regexp" 10 + "strconv" 10 11 "strings" 11 12 "syscall" 12 13 ··· 32 33 MemoryMaxMiB int64 33 34 SwapMaxMiB *int64 34 35 PidsMax int64 36 + // cpu.max quota as a percentage of one core (100 = one core). <= 0 37 + // leaves cpu unlimited 38 + CPUQuotaPercent int64 39 + // io.weight (1-10000). 0 leaves io unlimited, written directly since 40 + // this cgroup2 lib only models io.bfq.weight/io.max 41 + IOWeight uint64 35 42 } 36 43 37 44 type CgroupParent struct { ··· 102 109 return nil, fmt.Errorf("create cgroup %q: %w", name, err) 103 110 } 104 111 112 + if limits.IOWeight > 0 { 113 + if err := manager.ToggleControllers([]string{"io"}, cgroup2.Enable); err != nil { 114 + _ = manager.Delete() 115 + return nil, fmt.Errorf("enable io controller for cgroup %q: %w", name, err) 116 + } 117 + ioWeightPath := filepath.Join(limits.Parent.mountpoint, strings.TrimPrefix(limits.Parent.group, "/"), name, "io.weight") 118 + if err := os.WriteFile(ioWeightPath, []byte(strconv.FormatUint(limits.IOWeight, 10)), 0); err != nil { 119 + _ = manager.Delete() 120 + return nil, fmt.Errorf("write io.weight for cgroup %q: %w", name, err) 121 + } 122 + } 123 + 105 124 if logger != nil { 106 125 logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group) 107 126 } ··· 127 146 if limits.PidsMax > 0 { 128 147 resources.Pids = &cgroup2.Pids{Max: limits.PidsMax} 129 148 } 149 + if limits.CPUQuotaPercent > 0 { 150 + quota := limits.CPUQuotaPercent * 1000 // 100% of one 100000us period 151 + resources.CPU = &cgroup2.CPU{Max: cgroup2.NewCPUMax(&quota, nil)} 152 + } 130 153 return resources 131 154 } 132 155 ··· 178 201 // in the "/" parent's subtree. this fails EBUSY at a populated cgroup 179 202 // namespace root (no-internal-process constraint); the real root is exempt. 180 203 func probeRootSubtreeControl(mountpoint string) error { 204 + // cpu/io may be unavailable (eg. no CONFIG_BLK_CGROUP), memory/pids 205 + // are the hard requirement 206 + if err := os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids +cpu +io"), 0); err == nil { 207 + return nil 208 + } 181 209 return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0) 182 210 } 183 211

+15

spindle/engines/microvm/cid_test.go

··· 28 28 t.Errorf("key not deterministic: %q vs %q", a, again) 29 29 } 30 30 } 31 + 32 + func TestCgroupResourcesCPUQuota(t *testing.T) { 33 + r := cgroupResources(CgroupLimits{CPUQuotaPercent: 200}) 34 + if r.CPU == nil { 35 + t.Fatal("cpu quota should produce a cpu controller config") 36 + } 37 + if got := string(r.CPU.Max); got != "200000 100000" { 38 + t.Errorf("cpu.max = %q, want %q", got, "200000 100000") 39 + } 40 + 41 + r = cgroupResources(CgroupLimits{}) 42 + if r.CPU != nil { 43 + t.Errorf("no quota should leave cpu unlimited, got %v", r.CPU) 44 + } 45 + }

+14 -6

spindle/engines/microvm/engine.go

··· 608 608 609 609 func (e *Engine) cgroupLimits(wid models.WorkflowId, spec ImageSpec) CgroupLimits { 610 610 cfg := e.cfg.MicroVMPipelines 611 + cpuQuotaPercent := cfg.CgroupCPUMaxPercent 612 + if cpuQuotaPercent == 0 { 613 + // a vm can only use its vcpus anyway, but without a cap those 614 + // threads still get full host cores and starve every other tenant 615 + cpuQuotaPercent = int64(spec.VCPUs) * 100 616 + } 611 617 return CgroupLimits{ 612 - Enabled: cfg.EnableCgroups, 613 - Parent: e.cgroupParent, 614 - Name: "workflow-" + wid.String(), 615 - MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, 616 - SwapMaxMiB: cfg.CgroupSwapMaxMiB, 617 - PidsMax: cfg.CgroupPidsMax, 618 + Enabled: cfg.EnableCgroups, 619 + Parent: e.cgroupParent, 620 + Name: "workflow-" + wid.String(), 621 + MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, 622 + SwapMaxMiB: cfg.CgroupSwapMaxMiB, 623 + PidsMax: cfg.CgroupPidsMax, 624 + CPUQuotaPercent: cpuQuotaPercent, 625 + IOWeight: cfg.CgroupIOWeight, 618 626 } 619 627 }