vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <stdio.h>
6#include <unistd.h>
7#include <sched.h>
8#include <time.h>
9#include <sys/ioctl.h>
10#include <sys/syscall.h>
11#include <linux/perf_event.h>
12#include <linux/hw_breakpoint.h>
13#include <x86intrin.h>
14#include <cpuid.h>
15
16#define AMD_L3_MISS_EVENT 0x0104
17#define PERF_CACHE_CONFIG (PERF_COUNT_HW_CACHE_LL | \
18 (PERF_COUNT_HW_CACHE_OP_READ << 8) | \
19 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
20
21static bool is_amd(void)
22{
23 uint32_t eax, ebx, ecx, edx;
24 __cpuid(0, eax, ebx, ecx, edx);
25 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
26}
27
28static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
29 int group_fd, unsigned long flags)
30{
31 return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
32}
33
34int monitor_perf_open(int cpu, bool amd)
35{
36 struct perf_event_attr pe;
37 memset(&pe, 0, sizeof(pe));
38 pe.size = sizeof(pe);
39 pe.disabled = 1;
40 pe.exclude_kernel = 1;
41 pe.exclude_hv = 1;
42
43 if (amd) {
44 pe.type = PERF_TYPE_RAW;
45 pe.config = AMD_L3_MISS_EVENT;
46 } else {
47 pe.type = PERF_TYPE_HW_CACHE;
48 pe.config = PERF_CACHE_CONFIG;
49 }
50
51 int fd = perf_event_open(&pe, -1, cpu, -1, 0);
52 if (fd < 0)
53 return -1;
54
55 ioctl(fd, PERF_EVENT_IOC_RESET, 0);
56 ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
57
58 return fd;
59}
60
61static void nanosleep_relative(uint64_t ns)
62{
63 struct timespec ts;
64 ts.tv_sec = ns / 1000000000ULL;
65 ts.tv_nsec = ns % 1000000000ULL;
66 nanosleep(&ts, NULL);
67}
68
69static uint64_t probe_latency(volatile char *ptr, size_t size)
70{
71 size_t offset = ((size_t)rdtsc_start() % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE;
72 _mm_lfence();
73 uint64_t start = rdtsc_start();
74 (void)ptr[offset];
75 uint64_t end = rdtsc_end();
76 return cycles_to_ns(end - start);
77}
78
79static void reprime_region(icepick_monitor_t *mon)
80{
81 icepick_region_t *r = mon->region;
82
83 cpu_set_t old_affinity, new_affinity;
84 CPU_ZERO(&new_affinity);
85 CPU_SET(mon->cpu, &new_affinity);
86 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
87 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
88
89 clos_associate_thread(mon->msr_fd, r->clos_id);
90 prime_region(r->ptr, r->size, 3);
91 clos_associate_thread(mon->msr_fd, 0);
92
93 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
94
95 atomic_fetch_add(&mon->reprime_count, 1);
96}
97
98static void *monitor_thread(void *arg)
99{
100 icepick_monitor_t *mon = arg;
101 icepick_region_t *r = mon->region;
102
103 cpu_set_t affinity;
104 CPU_ZERO(&affinity);
105 CPU_SET(mon->cpu, &affinity);
106 sched_setaffinity(0, sizeof(affinity), &affinity);
107
108 uint64_t prev_count = 0;
109 if (mon->perf_fd >= 0) {
110 read(mon->perf_fd, &prev_count, sizeof(prev_count));
111 }
112
113 while (atomic_load(&mon->running)) {
114 if (mon->perf_fd >= 0) {
115 nanosleep_relative(mon->pmu_poll_interval_ns);
116
117 uint64_t curr_count;
118 if (read(mon->perf_fd, &curr_count, sizeof(curr_count)) != sizeof(curr_count))
119 continue;
120
121 uint64_t delta = curr_count - prev_count;
122 prev_count = curr_count;
123
124 if (delta < mon->miss_threshold)
125 continue;
126
127 atomic_fetch_add(&mon->eviction_count, delta);
128 }
129
130 unsigned fast_probes = 0;
131 while (fast_probes < MONITOR_PROBE_CONFIRM_COUNT && atomic_load(&mon->running)) {
132 nanosleep_relative(mon->probe_interval_ns);
133
134 uint64_t lat = probe_latency(r->ptr, r->size);
135 if (lat > MONITOR_L3_LATENCY_THRESH_NS) {
136 reprime_region(mon);
137 break;
138 }
139 fast_probes++;
140 }
141 }
142
143 return NULL;
144}
145
146static int find_ccx_cpu(icepick_region_t *region)
147{
148 int numa = region->numa_node;
149 if (numa < 0)
150 numa = 0;
151
152 char path[128];
153 snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa);
154
155 FILE *f = fopen(path, "r");
156 if (!f)
157 return 0;
158
159 int cpu = 0;
160 fscanf(f, "%d", &cpu);
161 fclose(f);
162
163 return cpu;
164}
165
166int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
167 uint64_t probe_ns, uint32_t miss_thresh,
168 icepick_monitor_t **out)
169{
170 if (!region || !out)
171 return ICEPICK_E_INVALID;
172
173 icepick_monitor_t *mon = calloc(1, sizeof(*mon));
174 if (!mon)
175 return ICEPICK_E_ALLOC;
176
177 mon->region = region;
178 mon->pmu_poll_interval_ns = pmu_poll_ns ? pmu_poll_ns : MONITOR_DEFAULT_PMU_POLL_NS;
179 mon->probe_interval_ns = probe_ns ? probe_ns : MONITOR_DEFAULT_PROBE_NS;
180 mon->miss_threshold = miss_thresh ? miss_thresh : MONITOR_DEFAULT_MISS_THRESH;
181
182 mon->cpu = find_ccx_cpu(region);
183 mon->msr_fd = msr_open(mon->cpu);
184 if (mon->msr_fd < 0) {
185 free(mon);
186 return ICEPICK_E_MSR;
187 }
188
189 mon->perf_fd = monitor_perf_open(mon->cpu, is_amd());
190 mon->degraded = (mon->perf_fd < 0);
191
192 atomic_store(&mon->running, true);
193 atomic_store(&mon->eviction_count, 0);
194 atomic_store(&mon->reprime_count, 0);
195
196 if (pthread_create(&mon->thread, NULL, monitor_thread, mon) != 0) {
197 if (mon->perf_fd >= 0)
198 close(mon->perf_fd);
199 msr_close(mon->msr_fd);
200 free(mon);
201 return ICEPICK_E_THREAD;
202 }
203
204 *out = mon;
205 return 0;
206}
207
208int icepick_monitor_start(icepick_region_t *region, icepick_monitor_t **out)
209{
210 return icepick_monitor_start_ex(region, 0, 0, 0, out);
211}
212
213int icepick_monitor_stop(icepick_monitor_t *mon)
214{
215 if (!mon)
216 return ICEPICK_E_INVALID;
217
218 atomic_store(&mon->running, false);
219 pthread_join(mon->thread, NULL);
220
221 if (mon->perf_fd >= 0)
222 close(mon->perf_fd);
223 msr_close(mon->msr_fd);
224 free(mon);
225
226 return 0;
227}
228
229uint64_t icepick_monitor_eviction_count(const icepick_monitor_t *mon)
230{
231 return mon ? atomic_load(&mon->eviction_count) : 0;
232}
233
234uint64_t icepick_monitor_reprime_count(const icepick_monitor_t *mon)
235{
236 return mon ? atomic_load(&mon->reprime_count) : 0;
237}
238
239bool icepick_monitor_is_degraded(const icepick_monitor_t *mon)
240{
241 return mon ? mon->degraded : true;
242}