vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <sched.h>
5#include <unistd.h>
6#include <string.h>
7#include <x86intrin.h>
8
9static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way)
10{
11 uint32_t mask = 0;
12 for (unsigned i = 0; i < num_ways; i++)
13 mask |= (1U << (start_way + i));
14 return mask;
15}
16
17static unsigned find_start_way(uint32_t default_mask, unsigned num_ways)
18{
19 for (unsigned i = 0; i <= 32 - num_ways; i++) {
20 uint32_t candidate = compute_way_mask(num_ways, i);
21 if ((candidate & default_mask) == candidate)
22 return i;
23 }
24 return 0;
25}
26
27static void init_pointer_chase(volatile char *ptr, size_t size)
28{
29 size_t num_lines = size / CACHE_LINE_SIZE;
30 volatile size_t *indices = (volatile size_t *)ptr;
31
32 for (size_t i = 0; i < num_lines; i++)
33 indices[i * (CACHE_LINE_SIZE / sizeof(size_t))] = ((i + 1) % num_lines) * CACHE_LINE_SIZE;
34
35 for (size_t i = num_lines - 1; i > 0; i--) {
36 size_t j = (size_t)rdtsc_start() % (i + 1);
37 size_t idx_i = i * (CACHE_LINE_SIZE / sizeof(size_t));
38 size_t idx_j = j * (CACHE_LINE_SIZE / sizeof(size_t));
39 size_t tmp = indices[idx_i];
40 indices[idx_i] = indices[idx_j];
41 indices[idx_j] = tmp;
42 }
43}
44
45static void prime_temporal_sequential(volatile char *ptr, size_t size)
46{
47 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
48 (void)ptr[offset];
49}
50
51static void prime_temporal_reverse(volatile char *ptr, size_t size)
52{
53 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
54 (void)ptr[offset - CACHE_LINE_SIZE];
55}
56
57static void prime_temporal_strided(volatile char *ptr, size_t size, size_t stride)
58{
59 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
60 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
61 for (size_t offset = pass; offset < size; offset += effective_stride)
62 (void)ptr[offset];
63 }
64}
65
66static void prime_temporal_chase(volatile char *ptr, size_t size)
67{
68 volatile size_t *indices = (volatile size_t *)ptr;
69 size_t num_lines = size / CACHE_LINE_SIZE;
70 size_t offset = 0;
71
72 for (size_t i = 0; i < num_lines; i++) {
73 offset = indices[offset / sizeof(size_t)];
74 }
75
76 (void)offset;
77}
78
79static void prime_prefetcht2_sequential(volatile char *ptr, size_t size)
80{
81 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
82 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
83 _mm_mfence();
84 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
85 (void)ptr[offset];
86}
87
88static void prime_prefetcht2_reverse(volatile char *ptr, size_t size)
89{
90 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
91 _mm_prefetch((const char *)&ptr[offset - CACHE_LINE_SIZE], _MM_HINT_T2);
92 _mm_mfence();
93 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
94 (void)ptr[offset - CACHE_LINE_SIZE];
95}
96
97static void prime_prefetcht2_strided(volatile char *ptr, size_t size, size_t stride)
98{
99 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
100 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
101 for (size_t offset = pass; offset < size; offset += effective_stride)
102 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
103 }
104 _mm_mfence();
105 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
106 for (size_t offset = pass; offset < size; offset += effective_stride)
107 (void)ptr[offset];
108 }
109}
110
111static void prime_prefetchnta_sequential(volatile char *ptr, size_t size)
112{
113 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
114 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_NTA);
115 _mm_mfence();
116 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
117 (void)ptr[offset];
118}
119
120static void prime_nt_store_sequential(volatile char *ptr, size_t size)
121{
122 __m128i zero = _mm_setzero_si128();
123 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
124 _mm_stream_si128((__m128i *)&ptr[offset], zero);
125 _mm_stream_si128((__m128i *)&ptr[offset + 16], zero);
126 _mm_stream_si128((__m128i *)&ptr[offset + 32], zero);
127 _mm_stream_si128((__m128i *)&ptr[offset + 48], zero);
128 }
129 _mm_sfence();
130 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
131 (void)ptr[offset];
132}
133
134void prime_region(volatile char *ptr, size_t size, unsigned iterations,
135 icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
136 size_t stride)
137{
138 if (pattern == ICEPICK_PATTERN_POINTER_CHASE)
139 init_pointer_chase(ptr, size);
140
141 for (unsigned iter = 0; iter < iterations; iter++) {
142 switch (strategy) {
143 case ICEPICK_PRIME_TEMPORAL:
144 switch (pattern) {
145 case ICEPICK_PATTERN_SEQUENTIAL:
146 prime_temporal_sequential(ptr, size);
147 break;
148 case ICEPICK_PATTERN_REVERSE:
149 prime_temporal_reverse(ptr, size);
150 break;
151 case ICEPICK_PATTERN_STRIDED:
152 prime_temporal_strided(ptr, size, stride);
153 break;
154 case ICEPICK_PATTERN_POINTER_CHASE:
155 prime_temporal_chase(ptr, size);
156 break;
157 }
158 break;
159
160 case ICEPICK_PRIME_PREFETCHT2:
161 switch (pattern) {
162 case ICEPICK_PATTERN_SEQUENTIAL:
163 prime_prefetcht2_sequential(ptr, size);
164 break;
165 case ICEPICK_PATTERN_REVERSE:
166 prime_prefetcht2_reverse(ptr, size);
167 break;
168 case ICEPICK_PATTERN_STRIDED:
169 prime_prefetcht2_strided(ptr, size, stride);
170 break;
171 case ICEPICK_PATTERN_POINTER_CHASE:
172 prime_prefetcht2_sequential(ptr, size);
173 prime_temporal_chase(ptr, size);
174 break;
175 }
176 break;
177
178 case ICEPICK_PRIME_PREFETCHNTA:
179 prime_prefetchnta_sequential(ptr, size);
180 break;
181
182 case ICEPICK_PRIME_NT_STORE:
183 prime_nt_store_sequential(ptr, size);
184 break;
185 }
186 }
187}
188
189int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg,
190 icepick_region_t **region)
191{
192 if (!topo || !cfg || !region)
193 return ICEPICK_E_INVALID;
194
195 if (cfg->clos_id == 0 || cfg->clos_id >= topo->max_clos)
196 return ICEPICK_E_INVALID;
197
198 unsigned ways_needed = (cfg->size + topo->way_size - 1) / topo->way_size;
199 if (ways_needed == 0)
200 ways_needed = 1;
201
202 if (ways_needed >= topo->l3_ways)
203 return ICEPICK_E_TOO_LARGE;
204
205 int ret = clos_init(topo);
206 if (ret < 0)
207 return ret;
208
209 unsigned start_way = find_start_way(topo->default_way_mask, ways_needed);
210 uint32_t way_mask = compute_way_mask(ways_needed, start_way);
211
212 ret = clos_allocate(cfg->clos_id, way_mask);
213 if (ret < 0)
214 return ret;
215
216 icepick_region_t *r = calloc(1, sizeof(*r));
217 if (!r) {
218 clos_release(cfg->clos_id);
219 return ICEPICK_E_ALLOC;
220 }
221
222 r->clos_id = cfg->clos_id;
223 r->numa_node = cfg->numa_node;
224 r->way_mask = way_mask;
225 r->mba_throttle = cfg->mba_throttle;
226 r->core_type = cfg->core_type;
227 r->topo = topo;
228 r->size = ways_needed * topo->way_size;
229 r->monitor = NULL;
230 r->prime_strategy = cfg->prime_strategy;
231 r->access_pattern = cfg->access_pattern;
232 r->stride_bytes = cfg->stride_bytes;
233 r->prime_iterations = cfg->prime_iterations ? cfg->prime_iterations : 3;
234
235 ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
236 if (ret < 0) {
237 clos_release(cfg->clos_id);
238 free(r);
239 return ret;
240 }
241
242 int cpu = find_cpu_for_core_type(topo, cfg->core_type, cfg->numa_node);
243
244 cpu_set_t old_affinity, new_affinity;
245 CPU_ZERO(&new_affinity);
246 CPU_SET(cpu, &new_affinity);
247 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
248 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
249
250 if (topo->arch == ARCH_X86_64) {
251 int msr_fd = msr_open(cpu);
252 if (msr_fd < 0) {
253 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
254 region_free(r->ptr, r->size);
255 clos_release(cfg->clos_id);
256 free(r);
257 return msr_fd;
258 }
259
260 ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask);
261 if (ret < 0) {
262 msr_close(msr_fd);
263 goto cleanup_no_msr;
264 }
265
266 ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask);
267 if (ret < 0) {
268 msr_close(msr_fd);
269 goto cleanup_no_msr;
270 }
271
272 if (topo->mba_supported && cfg->mba_throttle > 0) {
273 ret = mba_configure(msr_fd, cfg->clos_id, cfg->mba_throttle);
274 if (ret < 0) {
275 msr_close(msr_fd);
276 goto cleanup_no_msr;
277 }
278 }
279
280 ret = clos_associate_thread(msr_fd, cfg->clos_id);
281 if (ret < 0) {
282 msr_close(msr_fd);
283 goto cleanup_no_msr;
284 }
285
286 prime_region(r->ptr, r->size, r->prime_iterations,
287 r->prime_strategy, r->access_pattern, r->stride_bytes);
288
289 clos_associate_thread(msr_fd, 0);
290 msr_close(msr_fd);
291 } else if (topo->arch == ARCH_ARM64) {
292 ret = mpam_configure_partition(cfg->clos_id, way_mask, cfg->mba_throttle);
293 if (ret < 0)
294 goto cleanup_no_msr;
295
296 ret = mpam_associate_thread(cfg->clos_id);
297 if (ret < 0)
298 goto cleanup_no_msr;
299
300 prime_region(r->ptr, r->size, r->prime_iterations,
301 r->prime_strategy, r->access_pattern, r->stride_bytes);
302
303 mpam_associate_thread(0);
304 }
305
306 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
307
308 if (cfg->verify) {
309 icepick_latency_stats_t stats;
310 ret = icepick_verify(r, &stats);
311 if (ret < 0) {
312 icepick_unlock(r);
313 return ret;
314 }
315 }
316
317 if (cfg->auto_monitor) {
318 icepick_monitor_t *mon;
319 ret = icepick_monitor_start_ex(r, cfg->pmu_poll_interval_ns,
320 cfg->probe_interval_ns,
321 cfg->miss_threshold, &mon);
322 if (ret < 0) {
323 icepick_unlock(r);
324 return ret;
325 }
326 r->monitor = mon;
327 }
328
329 *region = r;
330 return 0;
331
332cleanup_no_msr:
333 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
334 region_free(r->ptr, r->size);
335 clos_release(cfg->clos_id);
336 free(r);
337 return ret;
338}
339
340int icepick_unlock(icepick_region_t *region)
341{
342 if (!region)
343 return ICEPICK_E_INVALID;
344
345 if (region->monitor)
346 icepick_monitor_stop(region->monitor);
347
348 if (region->topo->arch == ARCH_X86_64) {
349 int cpu = sched_getcpu();
350 if (cpu < 0)
351 cpu = 0;
352
353 int msr_fd = msr_open(cpu);
354 if (msr_fd >= 0) {
355 clos_release(region->clos_id);
356 clos_configure_mask(msr_fd, region->clos_id, 0);
357 if (region->topo->mba_supported && region->mba_throttle > 0)
358 mba_configure(msr_fd, region->clos_id, 0);
359 clos_configure_mask(msr_fd, 0, clos_get_default_mask());
360 msr_close(msr_fd);
361 }
362 } else if (region->topo->arch == ARCH_ARM64) {
363 clos_release(region->clos_id);
364 mpam_configure_partition(region->clos_id, 0, 0);
365 }
366
367 region_free(region->ptr, region->size);
368 free(region);
369
370 return 0;
371}
372
373void *icepick_region_ptr(const icepick_region_t *region)
374{
375 return region ? region->ptr : NULL;
376}
377
378size_t icepick_region_size(const icepick_region_t *region)
379{
380 return region ? region->size : 0;
381}
382
383unsigned icepick_region_clos(const icepick_region_t *region)
384{
385 return region ? region->clos_id : 0;
386}
387
388const char *icepick_strerror(int err)
389{
390 switch (err) {
391 case 0: return "success";
392 case ICEPICK_E_NO_CAT: return "cat not supported on this cpu";
393 case ICEPICK_E_PERMISSION: return "permission denied (need root or cap_sys_rawio)";
394 case ICEPICK_E_NO_CLOS: return "no clos available or already allocated";
395 case ICEPICK_E_TOO_LARGE: return "requested size exceeds available cache ways";
396 case ICEPICK_E_NUMA: return "invalid numa node or binding failed";
397 case ICEPICK_E_HUGEPAGE: return "huge page allocation failed";
398 case ICEPICK_E_VERIFY: return "verification failed - data may not be cache-resident";
399 case ICEPICK_E_INVALID: return "invalid argument";
400 case ICEPICK_E_ALLOC: return "memory allocation failed";
401 case ICEPICK_E_MSR: return "msr operation failed";
402 case ICEPICK_E_THREAD: return "monitor thread creation failed";
403 default: return "unknown error";
404 }
405}