vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <stdio.h>
6#include <unistd.h>
7#include <sched.h>
8#include <time.h>
9#include <sys/ioctl.h>
10#include <sys/syscall.h>
11#include <linux/perf_event.h>
12#include <linux/hw_breakpoint.h>
13#include <x86intrin.h>
14#include <cpuid.h>
15
16#define AMD_L3_MISS_EVENT 0x0104
17#define PERF_CACHE_CONFIG (PERF_COUNT_HW_CACHE_LL | \
18 (PERF_COUNT_HW_CACHE_OP_READ << 8) | \
19 (PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
20
21static bool is_amd(void)
22{
23 uint32_t eax, ebx, ecx, edx;
24 __cpuid(0, eax, ebx, ecx, edx);
25 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
26}
27
28static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
29 int group_fd, unsigned long flags)
30{
31 return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
32}
33
34int monitor_perf_open(int cpu, bool amd)
35{
36 struct perf_event_attr pe;
37 memset(&pe, 0, sizeof(pe));
38 pe.size = sizeof(pe);
39 pe.disabled = 1;
40 pe.exclude_kernel = 1;
41 pe.exclude_hv = 1;
42
43 if (amd) {
44 pe.type = PERF_TYPE_RAW;
45 pe.config = AMD_L3_MISS_EVENT;
46 } else {
47 pe.type = PERF_TYPE_HW_CACHE;
48 pe.config = PERF_CACHE_CONFIG;
49 }
50
51 int fd = perf_event_open(&pe, -1, cpu, -1, 0);
52 if (fd < 0)
53 return -1;
54
55 ioctl(fd, PERF_EVENT_IOC_RESET, 0);
56 ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
57
58 return fd;
59}
60
61static void nanosleep_relative(uint64_t ns)
62{
63 struct timespec ts;
64 ts.tv_sec = ns / 1000000000ULL;
65 ts.tv_nsec = ns % 1000000000ULL;
66 nanosleep(&ts, NULL);
67}
68
69static uint64_t probe_latency(volatile char *ptr, size_t size)
70{
71 size_t offset = ((size_t)rdtsc_start() % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE;
72 _mm_lfence();
73 uint64_t start = rdtsc_start();
74 (void)ptr[offset];
75 uint64_t end = rdtsc_end();
76 return cycles_to_ns(end - start);
77}
78
79static void reprime_region(icepick_monitor_t *mon)
80{
81 icepick_region_t *r = mon->region;
82
83 cpu_set_t old_affinity, new_affinity;
84 CPU_ZERO(&new_affinity);
85 CPU_SET(mon->cpu, &new_affinity);
86 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
87 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
88
89 clos_associate_thread(mon->msr_fd, r->clos_id);
90 prime_region(r->ptr, r->size, r->prime_iterations,
91 r->prime_strategy, r->access_pattern, r->stride_bytes);
92 clos_associate_thread(mon->msr_fd, 0);
93
94 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
95
96 atomic_fetch_add(&mon->reprime_count, 1);
97}
98
99static void *monitor_thread(void *arg)
100{
101 icepick_monitor_t *mon = arg;
102 icepick_region_t *r = mon->region;
103
104 cpu_set_t affinity;
105 CPU_ZERO(&affinity);
106 CPU_SET(mon->cpu, &affinity);
107 sched_setaffinity(0, sizeof(affinity), &affinity);
108
109 uint64_t prev_count = 0;
110 if (mon->perf_fd >= 0) {
111 read(mon->perf_fd, &prev_count, sizeof(prev_count));
112 }
113
114 while (atomic_load(&mon->running)) {
115 if (mon->perf_fd >= 0) {
116 nanosleep_relative(mon->pmu_poll_interval_ns);
117
118 uint64_t curr_count;
119 if (read(mon->perf_fd, &curr_count, sizeof(curr_count)) != sizeof(curr_count))
120 continue;
121
122 uint64_t delta = curr_count - prev_count;
123 prev_count = curr_count;
124
125 if (delta < mon->miss_threshold)
126 continue;
127
128 atomic_fetch_add(&mon->eviction_count, delta);
129 }
130
131 unsigned fast_probes = 0;
132 while (fast_probes < MONITOR_PROBE_CONFIRM_COUNT && atomic_load(&mon->running)) {
133 nanosleep_relative(mon->probe_interval_ns);
134
135 uint64_t lat = probe_latency(r->ptr, r->size);
136 if (lat > MONITOR_L3_LATENCY_THRESH_NS) {
137 reprime_region(mon);
138 break;
139 }
140 fast_probes++;
141 }
142 }
143
144 return NULL;
145}
146
147static int find_ccx_cpu(icepick_region_t *region)
148{
149 return find_cpu_for_core_type(region->topo, region->core_type, region->numa_node);
150}
151
152int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
153 uint64_t probe_ns, uint32_t miss_thresh,
154 icepick_monitor_t **out)
155{
156 if (!region || !out)
157 return ICEPICK_E_INVALID;
158
159 icepick_monitor_t *mon = calloc(1, sizeof(*mon));
160 if (!mon)
161 return ICEPICK_E_ALLOC;
162
163 mon->region = region;
164 mon->pmu_poll_interval_ns = pmu_poll_ns ? pmu_poll_ns : MONITOR_DEFAULT_PMU_POLL_NS;
165 mon->probe_interval_ns = probe_ns ? probe_ns : MONITOR_DEFAULT_PROBE_NS;
166 mon->miss_threshold = miss_thresh ? miss_thresh : MONITOR_DEFAULT_MISS_THRESH;
167
168 mon->cpu = find_ccx_cpu(region);
169 mon->msr_fd = msr_open(mon->cpu);
170 if (mon->msr_fd < 0) {
171 free(mon);
172 return ICEPICK_E_MSR;
173 }
174
175 mon->perf_fd = monitor_perf_open(mon->cpu, is_amd());
176 mon->degraded = (mon->perf_fd < 0);
177
178 atomic_store(&mon->running, true);
179 atomic_store(&mon->eviction_count, 0);
180 atomic_store(&mon->reprime_count, 0);
181
182 if (pthread_create(&mon->thread, NULL, monitor_thread, mon) != 0) {
183 if (mon->perf_fd >= 0)
184 close(mon->perf_fd);
185 msr_close(mon->msr_fd);
186 free(mon);
187 return ICEPICK_E_THREAD;
188 }
189
190 *out = mon;
191 return 0;
192}
193
194int icepick_monitor_start(icepick_region_t *region, icepick_monitor_t **out)
195{
196 return icepick_monitor_start_ex(region, 0, 0, 0, out);
197}
198
199int icepick_monitor_stop(icepick_monitor_t *mon)
200{
201 if (!mon)
202 return ICEPICK_E_INVALID;
203
204 atomic_store(&mon->running, false);
205 pthread_join(mon->thread, NULL);
206
207 if (mon->perf_fd >= 0)
208 close(mon->perf_fd);
209 msr_close(mon->msr_fd);
210 free(mon);
211
212 return 0;
213}
214
215uint64_t icepick_monitor_eviction_count(const icepick_monitor_t *mon)
216{
217 return mon ? atomic_load(&mon->eviction_count) : 0;
218}
219
220uint64_t icepick_monitor_reprime_count(const icepick_monitor_t *mon)
221{
222 return mon ? atomic_load(&mon->reprime_count) : 0;
223}
224
225bool icepick_monitor_is_degraded(const icepick_monitor_t *mon)
226{
227 return mon ? mon->degraded : true;
228}