vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <sys/mman.h>
4#include <numaif.h>
5#include <errno.h>
6#include <string.h>
7
8#ifndef MAP_HUGE_2MB
9#define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT)
10#endif
11
12#ifndef MAP_HUGE_SHIFT
13#define MAP_HUGE_SHIFT 26
14#endif
15
16int region_alloc(size_t size, int numa_node, bool huge_pages, void **out_ptr)
17{
18 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
19 size_t alloc_size = size;
20
21 if (huge_pages) {
22 flags |= MAP_HUGETLB | MAP_HUGE_2MB;
23 size_t huge_page_size = 2 * 1024 * 1024;
24 alloc_size = (size + huge_page_size - 1) & ~(huge_page_size - 1);
25 }
26
27 void *ptr = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, flags, -1, 0);
28 if (ptr == MAP_FAILED) {
29 if (huge_pages && errno == ENOMEM) {
30 flags = MAP_PRIVATE | MAP_ANONYMOUS;
31 alloc_size = size;
32 ptr = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, flags, -1, 0);
33 if (ptr == MAP_FAILED)
34 return ICEPICK_E_ALLOC;
35 } else {
36 return huge_pages ? ICEPICK_E_HUGEPAGE : ICEPICK_E_ALLOC;
37 }
38 }
39
40 if (numa_node >= 0) {
41 unsigned long nodemask = 1UL << numa_node;
42 if (mbind(ptr, alloc_size, MPOL_BIND, &nodemask, sizeof(nodemask) * 8,
43 MPOL_MF_MOVE | MPOL_MF_STRICT) < 0) {
44 munmap(ptr, alloc_size);
45 return ICEPICK_E_NUMA;
46 }
47 }
48
49 if (mlock(ptr, alloc_size) < 0) {
50 munmap(ptr, alloc_size);
51 return ICEPICK_E_PERMISSION;
52 }
53
54 madvise(ptr, alloc_size, MADV_DONTFORK);
55
56 memset(ptr, 0, alloc_size);
57
58 *out_ptr = ptr;
59 return 0;
60}
61
62void region_free(void *ptr, size_t size)
63{
64 if (ptr) {
65 munlock(ptr, size);
66 munmap(ptr, size);
67 }
68}