vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <stdio.h>
6#include <unistd.h>
7#include <sched.h>
8#include <time.h>
9#include <sys/ioctl.h>
10#include <sys/syscall.h>
11#include <linux/perf_event.h>
12#include <linux/hw_breakpoint.h>
13#include <x86intrin.h>
14#include <cpuid.h>
15
16#define AMD_L3_MISS_EVENT 0x0104
17#define PERF_CACHE_CONFIG (PERF_COUNT_HW_CACHE_LL | \
18 (PERF_COUNT_HW_CACHE_OP_READ << 8) | \
19 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
20
21static bool is_amd(void)
22{
23 uint32_t eax, ebx, ecx, edx;
24 __cpuid(0, eax, ebx, ecx, edx);
25 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
26}
27
28static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
29 int group_fd, unsigned long flags)
30{
31 return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
32}
33
34int monitor_perf_open(int cpu, bool amd)
35{
36 struct perf_event_attr pe;
37 memset(&pe, 0, sizeof(pe));
38 pe.size = sizeof(pe);
39 pe.disabled = 1;
40 pe.exclude_kernel = 1;
41 pe.exclude_hv = 1;
42
43 if (amd) {
44 pe.type = PERF_TYPE_RAW;
45 pe.config = AMD_L3_MISS_EVENT;
46 } else {
47 pe.type = PERF_TYPE_HW_CACHE;
48 pe.config = PERF_CACHE_CONFIG;
49 }
50
51 int fd = perf_event_open(&pe, -1, cpu, -1, 0);
52 if (fd < 0)
53 return -1;
54
55 ioctl(fd, PERF_EVENT_IOC_RESET, 0);
56 ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
57
58 return fd;
59}
60
61static void nanosleep_relative(uint64_t ns)
62{
63 struct timespec ts;
64 ts.tv_sec = ns / 1000000000ULL;
65 ts.tv_nsec = ns % 1000000000ULL;
66 nanosleep(&ts, NULL);
67}
68
69static uint64_t probe_latency(volatile char *ptr, size_t size)
70{
71 size_t offset = ((size_t)rdtsc_start() % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE;
72 _mm_lfence();
73 uint64_t start = rdtsc_start();
74 (void)ptr[offset];
75 uint64_t end = rdtsc_end();
76 return cycles_to_ns(end - start);
77}
78
79static void reprime_region(icepick_monitor_t *mon)
80{
81 icepick_region_t *r = mon->region;
82
83 cpu_set_t old_affinity, new_affinity;
84 CPU_ZERO(&new_affinity);
85 CPU_SET(mon->cpu, &new_affinity);
86 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
87 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
88
89 clos_associate_thread(mon->msr_fd, r->clos_id);
90 prime_region(r->ptr, r->size, r->prime_iterations,
91 r->prime_strategy, r->access_pattern, r->stride_bytes);
92 clos_associate_thread(mon->msr_fd, 0);
93
94 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
95
96 atomic_fetch_add(&mon->reprime_count, 1);
97}
98
99static void *monitor_thread(void *arg)
100{
101 icepick_monitor_t *mon = arg;
102 icepick_region_t *r = mon->region;
103
104 cpu_set_t affinity;
105 CPU_ZERO(&affinity);
106 CPU_SET(mon->cpu, &affinity);
107 sched_setaffinity(0, sizeof(affinity), &affinity);
108
109 uint64_t prev_count = 0;
110 if (mon->perf_fd >= 0) {
111 read(mon->perf_fd, &prev_count, sizeof(prev_count));
112 }
113
114 while (atomic_load(&mon->running)) {
115 if (mon->perf_fd >= 0) {
116 nanosleep_relative(mon->pmu_poll_interval_ns);
117
118 uint64_t curr_count;
119 if (read(mon->perf_fd, &curr_count, sizeof(curr_count)) != sizeof(curr_count))
120 continue;
121
122 uint64_t delta = curr_count - prev_count;
123 prev_count = curr_count;
124
125 if (delta < mon->miss_threshold)
126 continue;
127
128 atomic_fetch_add(&mon->eviction_count, delta);
129 }
130
131 unsigned fast_probes = 0;
132 while (fast_probes < MONITOR_PROBE_CONFIRM_COUNT && atomic_load(&mon->running)) {
133 nanosleep_relative(mon->probe_interval_ns);
134
135 uint64_t lat = probe_latency(r->ptr, r->size);
136 if (lat > MONITOR_L3_LATENCY_THRESH_NS) {
137 reprime_region(mon);
138 break;
139 }
140 fast_probes++;
141 }
142 }
143
144 return NULL;
145}
146
147static int find_ccx_cpu(icepick_region_t *region)
148{
149 int numa = region->numa_node;
150 if (numa < 0)
151 numa = 0;
152
153 char path[128];
154 snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa);
155
156 FILE *f = fopen(path, "r");
157 if (!f)
158 return 0;
159
160 int cpu = 0;
161 fscanf(f, "%d", &cpu);
162 fclose(f);
163
164 return cpu;
165}
166
167int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
168 uint64_t probe_ns, uint32_t miss_thresh,
169 icepick_monitor_t **out)
170{
171 if (!region || !out)
172 return ICEPICK_E_INVALID;
173
174 icepick_monitor_t *mon = calloc(1, sizeof(*mon));
175 if (!mon)
176 return ICEPICK_E_ALLOC;
177
178 mon->region = region;
179 mon->pmu_poll_interval_ns = pmu_poll_ns ? pmu_poll_ns : MONITOR_DEFAULT_PMU_POLL_NS;
180 mon->probe_interval_ns = probe_ns ? probe_ns : MONITOR_DEFAULT_PROBE_NS;
181 mon->miss_threshold = miss_thresh ? miss_thresh : MONITOR_DEFAULT_MISS_THRESH;
182
183 mon->cpu = find_ccx_cpu(region);
184 mon->msr_fd = msr_open(mon->cpu);
185 if (mon->msr_fd < 0) {
186 free(mon);
187 return ICEPICK_E_MSR;
188 }
189
190 mon->perf_fd = monitor_perf_open(mon->cpu, is_amd());
191 mon->degraded = (mon->perf_fd < 0);
192
193 atomic_store(&mon->running, true);
194 atomic_store(&mon->eviction_count, 0);
195 atomic_store(&mon->reprime_count, 0);
196
197 if (pthread_create(&mon->thread, NULL, monitor_thread, mon) != 0) {
198 if (mon->perf_fd >= 0)
199 close(mon->perf_fd);
200 msr_close(mon->msr_fd);
201 free(mon);
202 return ICEPICK_E_THREAD;
203 }
204
205 *out = mon;
206 return 0;
207}
208
209int icepick_monitor_start(icepick_region_t *region, icepick_monitor_t **out)
210{
211 return icepick_monitor_start_ex(region, 0, 0, 0, out);
212}
213
214int icepick_monitor_stop(icepick_monitor_t *mon)
215{
216 if (!mon)
217 return ICEPICK_E_INVALID;
218
219 atomic_store(&mon->running, false);
220 pthread_join(mon->thread, NULL);
221
222 if (mon->perf_fd >= 0)
223 close(mon->perf_fd);
224 msr_close(mon->msr_fd);
225 free(mon);
226
227 return 0;
228}
229
230uint64_t icepick_monitor_eviction_count(const icepick_monitor_t *mon)
231{
232 return mon ? atomic_load(&mon->eviction_count) : 0;
233}
234
235uint64_t icepick_monitor_reprime_count(const icepick_monitor_t *mon)
236{
237 return mon ? atomic_load(&mon->reprime_count) : 0;
238}
239
240bool icepick_monitor_is_degraded(const icepick_monitor_t *mon)
241{
242 return mon ? mon->degraded : true;
243}