tangled.org
Monorepo for Tangled
tangled.org
1//go:build linux
2
3package microvm
4
5import (
6 "errors"
7 "fmt"
8 "log/slog"
9 "os"
10 "path/filepath"
11 "regexp"
12 "strings"
13 "syscall"
14
15 cgroups "github.com/containerd/cgroups/v3"
16 "github.com/containerd/cgroups/v3/cgroup2"
17 "github.com/prometheus/procfs"
18)
19
20var (
21 cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`)
22 cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`)
23)
24
25const (
26 cgroupParentSelf = "self"
27 supervisorCgroupName = "supervisor"
28)
29
30type CgroupLimits struct {
31 Enabled bool
32 Parent *CgroupParent
33 Name string
34 MemoryMaxMiB int64
35 SwapMaxMiB *int64
36 PidsMax int64
37}
38
39type CgroupParent struct {
40 root *cgroup2.Manager
41 mountpoint string
42 group string
43}
44
45type CgroupHandle struct {
46 manager *cgroup2.Manager
47}
48
49func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) {
50 if parent == "" {
51 parent = cgroupParentSelf
52 }
53 if cgroups.Mode() != cgroups.Unified {
54 return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode")
55 }
56
57 mountpoint, group, err := resolveCgroupParent(parent)
58 if err != nil {
59 return nil, err
60 }
61 if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil {
62 return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err)
63 }
64
65 root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint))
66 if err != nil {
67 return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err)
68 }
69
70 if group != "/" {
71 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
72 return nil, err
73 }
74 } else if err := probeRootSubtreeControl(mountpoint); err != nil {
75 if !errors.Is(err, syscall.EBUSY) {
76 return nil, fmt.Errorf("enable controllers in subtree_control of cgroup root %q: %w", mountpoint, err)
77 }
78 // populated namespace root, not the real root: vacate it too
79 if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil {
80 return nil, err
81 }
82 }
83
84 if logger != nil {
85 logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group)
86 }
87 return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil
88}
89
90func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) {
91 if !limits.Enabled {
92 return nil, nil
93 }
94 if limits.Parent == nil || limits.Parent.root == nil {
95 return nil, fmt.Errorf("cgroup parent is not initialized")
96 }
97 name := sanitizeCgroupName(limits.Name)
98 if name == "" {
99 return nil, fmt.Errorf("cgroup name is empty")
100 }
101
102 manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits))
103 if err != nil {
104 return nil, fmt.Errorf("create cgroup %q: %w", name, err)
105 }
106
107 if logger != nil {
108 logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group)
109 }
110 return &CgroupHandle{manager: manager}, nil
111}
112
113func cgroupResources(limits CgroupLimits) *cgroup2.Resources {
114 resources := &cgroup2.Resources{}
115 if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil {
116 memory := &cgroup2.Memory{}
117 if limits.MemoryMaxMiB > 0 {
118 maxBytes := limits.MemoryMaxMiB * 1024 * 1024
119 memory.Max = &maxBytes
120 }
121 if limits.SwapMaxMiB != nil {
122 swapBytes := *limits.SwapMaxMiB * 1024 * 1024
123 memory.Swap = &swapBytes
124 }
125 oomGroup := true
126 memory.OOMGroup = &oomGroup
127 resources.Memory = memory
128 }
129 if limits.PidsMax > 0 {
130 resources.Pids = &cgroup2.Pids{Max: limits.PidsMax}
131 }
132 return resources
133}
134
135func supervisorResources(memoryMinMiB int64) *cgroup2.Resources {
136 if memoryMinMiB <= 0 {
137 return nil
138 }
139 minBytes := memoryMinMiB * 1024 * 1024
140 return &cgroup2.Resources{
141 Memory: &cgroup2.Memory{Min: &minBytes},
142 }
143}
144
145func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error {
146 if h == nil || h.manager == nil {
147 return nil
148 }
149 if pid <= 0 {
150 return fmt.Errorf("invalid pid %d", pid)
151 }
152 if err := h.manager.AddProc(uint64(pid)); err != nil {
153 return fmt.Errorf("add pid %d to cgroup: %w", pid, err)
154 }
155 if logger != nil {
156 logger.Info("added process to microVM cgroup", "pid", pid)
157 }
158 return nil
159}
160
161func (h *CgroupHandle) Close() error {
162 if h == nil || h.manager == nil {
163 return nil
164 }
165 return h.manager.Delete()
166}
167
168func (h *CgroupHandle) OOMKilled() bool {
169 if h == nil || h.manager == nil {
170 return false
171 }
172 metrics, err := h.manager.Stat()
173 if err != nil || metrics == nil || metrics.MemoryEvents == nil {
174 return false
175 }
176 return metrics.MemoryEvents.OomKill > 0
177}
178
179// probeRootSubtreeControl enables the domain controllers the engine needs
180// in the "/" parent's subtree. this fails EBUSY at a populated cgroup
181// namespace root (no-internal-process constraint); the real root is exempt.
182func probeRootSubtreeControl(mountpoint string) error {
183 return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0)
184}
185
186func resolveCgroupParent(parent string) (string, string, error) {
187 mountpoint, err := cgroup2Mountpoint()
188 if err != nil {
189 return "", "", err
190 }
191
192 if parent == "" || parent == cgroupParentSelf {
193 group, err := selfCgroupV2Path()
194 if err != nil {
195 return "", "", err
196 }
197 return mountpoint, group, nil
198 }
199 if !filepath.IsAbs(parent) {
200 return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent)
201 }
202
203 cleanParent := filepath.Clean(parent)
204 rel, err := filepath.Rel(mountpoint, cleanParent)
205 if err != nil {
206 return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err)
207 }
208 if rel == ".." || strings.HasPrefix(rel, "../") {
209 return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint)
210 }
211 if rel == "." {
212 return mountpoint, "/", nil
213 }
214
215 group := "/" + filepath.ToSlash(rel)
216 if err := cgroup2.VerifyGroupPath(group); err != nil {
217 return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err)
218 }
219 return mountpoint, group, nil
220}
221
222func cgroup2Mountpoint() (string, error) {
223 mounts, err := procfs.GetMounts()
224 if err != nil {
225 return "", fmt.Errorf("read procfs mountinfo: %w", err)
226 }
227 for _, mount := range mounts {
228 if mount.FSType == "cgroup2" {
229 return mount.MountPoint, nil
230 }
231 }
232 return "", fmt.Errorf("cgroup v2 mountpoint not found")
233}
234
235func selfCgroupV2Path() (string, error) {
236 self, err := procfs.Self()
237 if err != nil {
238 return "", fmt.Errorf("open procfs self: %w", err)
239 }
240 groups, err := self.Cgroups()
241 if err != nil {
242 return "", fmt.Errorf("read procfs self cgroups: %w", err)
243 }
244 for _, group := range groups {
245 if group.HierarchyID != 0 {
246 continue
247 }
248 path := group.Path
249 if path == "" {
250 path = "/"
251 }
252 if err := cgroup2.VerifyGroupPath(path); err != nil {
253 return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err)
254 }
255 return path, nil
256 }
257 return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry")
258}
259
260func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error {
261 procs, err := parent.Procs(false)
262 if err != nil {
263 return fmt.Errorf("list parent cgroup processes: %w", err)
264 }
265
266 // first create with empty resources
267 supervisor, err := parent.NewChild(supervisorCgroupName, &cgroup2.Resources{})
268 if err != nil {
269 return fmt.Errorf("create supervisor cgroup: %w", err)
270 }
271
272 // move procs
273 for _, pid := range procs {
274 if err := supervisor.AddProc(pid); err != nil {
275 return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err)
276 }
277 }
278
279 // now apply resources. we can't do this while parent has procs still
280 if res := supervisorResources(supervisorMemoryMinMiB); res != nil {
281 // we use a "new" parent here, this is so we enable subtree_control.
282 // .Update() does not work here...
283 if _, err = parent.NewChild(supervisorCgroupName, res); err != nil {
284 return fmt.Errorf("apply supervisor cgroup resources: %w", err)
285 }
286 }
287
288 if logger != nil && len(procs) > 0 {
289 logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs))
290 }
291 return nil
292}
293
294func sanitizeCgroupName(name string) string {
295 name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-")
296 name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-")
297 return strings.Trim(name, "-_.")
298}