vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <sched.h>
5#include <unistd.h>
6#include <string.h>
7#include <x86intrin.h>
8
9static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way)
10{
11 uint32_t mask = 0;
12 for (unsigned i = 0; i < num_ways; i++)
13 mask |= (1U << (start_way + i));
14 return mask;
15}
16
17static unsigned find_start_way(uint32_t default_mask, unsigned num_ways)
18{
19 for (unsigned i = 0; i <= 32 - num_ways; i++) {
20 uint32_t candidate = compute_way_mask(num_ways, i);
21 if ((candidate & default_mask) == candidate)
22 return i;
23 }
24 return 0;
25}
26
27static void init_pointer_chase(volatile char *ptr, size_t size)
28{
29 size_t num_lines = size / CACHE_LINE_SIZE;
30 volatile size_t *indices = (volatile size_t *)ptr;
31
32 for (size_t i = 0; i < num_lines; i++)
33 indices[i * (CACHE_LINE_SIZE / sizeof(size_t))] = ((i + 1) % num_lines) * CACHE_LINE_SIZE;
34
35 for (size_t i = num_lines - 1; i > 0; i--) {
36 size_t j = (size_t)rdtsc_start() % (i + 1);
37 size_t idx_i = i * (CACHE_LINE_SIZE / sizeof(size_t));
38 size_t idx_j = j * (CACHE_LINE_SIZE / sizeof(size_t));
39 size_t tmp = indices[idx_i];
40 indices[idx_i] = indices[idx_j];
41 indices[idx_j] = tmp;
42 }
43}
44
45static void prime_temporal_sequential(volatile char *ptr, size_t size)
46{
47 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
48 (void)ptr[offset];
49}
50
51static void prime_temporal_reverse(volatile char *ptr, size_t size)
52{
53 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
54 (void)ptr[offset - CACHE_LINE_SIZE];
55}
56
57static void prime_temporal_strided(volatile char *ptr, size_t size, size_t stride)
58{
59 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
60 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
61 for (size_t offset = pass; offset < size; offset += effective_stride)
62 (void)ptr[offset];
63 }
64}
65
66static void prime_temporal_chase(volatile char *ptr, size_t size)
67{
68 volatile size_t *indices = (volatile size_t *)ptr;
69 size_t num_lines = size / CACHE_LINE_SIZE;
70 size_t offset = 0;
71
72 for (size_t i = 0; i < num_lines; i++) {
73 offset = indices[offset / sizeof(size_t)];
74 }
75
76 (void)offset;
77}
78
79static void prime_prefetcht2_sequential(volatile char *ptr, size_t size)
80{
81 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
82 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
83 _mm_mfence();
84 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
85 (void)ptr[offset];
86}
87
88static void prime_prefetcht2_reverse(volatile char *ptr, size_t size)
89{
90 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
91 _mm_prefetch((const char *)&ptr[offset - CACHE_LINE_SIZE], _MM_HINT_T2);
92 _mm_mfence();
93 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
94 (void)ptr[offset - CACHE_LINE_SIZE];
95}
96
97static void prime_prefetcht2_strided(volatile char *ptr, size_t size, size_t stride)
98{
99 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
100 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
101 for (size_t offset = pass; offset < size; offset += effective_stride)
102 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
103 }
104 _mm_mfence();
105 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
106 for (size_t offset = pass; offset < size; offset += effective_stride)
107 (void)ptr[offset];
108 }
109}
110
111static void prime_prefetchnta_sequential(volatile char *ptr, size_t size)
112{
113 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
114 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_NTA);
115 _mm_mfence();
116 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
117 (void)ptr[offset];
118}
119
120static void prime_nt_store_sequential(volatile char *ptr, size_t size)
121{
122 __m128i zero = _mm_setzero_si128();
123 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
124 _mm_stream_si128((__m128i *)&ptr[offset], zero);
125 _mm_stream_si128((__m128i *)&ptr[offset + 16], zero);
126 _mm_stream_si128((__m128i *)&ptr[offset + 32], zero);
127 _mm_stream_si128((__m128i *)&ptr[offset + 48], zero);
128 }
129 _mm_sfence();
130 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
131 (void)ptr[offset];
132}
133
134void prime_region(volatile char *ptr, size_t size, unsigned iterations,
135 icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
136 size_t stride)
137{
138 if (pattern == ICEPICK_PATTERN_POINTER_CHASE)
139 init_pointer_chase(ptr, size);
140
141 for (unsigned iter = 0; iter < iterations; iter++) {
142 switch (strategy) {
143 case ICEPICK_PRIME_TEMPORAL:
144 switch (pattern) {
145 case ICEPICK_PATTERN_SEQUENTIAL:
146 prime_temporal_sequential(ptr, size);
147 break;
148 case ICEPICK_PATTERN_REVERSE:
149 prime_temporal_reverse(ptr, size);
150 break;
151 case ICEPICK_PATTERN_STRIDED:
152 prime_temporal_strided(ptr, size, stride);
153 break;
154 case ICEPICK_PATTERN_POINTER_CHASE:
155 prime_temporal_chase(ptr, size);
156 break;
157 }
158 break;
159
160 case ICEPICK_PRIME_PREFETCHT2:
161 switch (pattern) {
162 case ICEPICK_PATTERN_SEQUENTIAL:
163 prime_prefetcht2_sequential(ptr, size);
164 break;
165 case ICEPICK_PATTERN_REVERSE:
166 prime_prefetcht2_reverse(ptr, size);
167 break;
168 case ICEPICK_PATTERN_STRIDED:
169 prime_prefetcht2_strided(ptr, size, stride);
170 break;
171 case ICEPICK_PATTERN_POINTER_CHASE:
172 prime_prefetcht2_sequential(ptr, size);
173 prime_temporal_chase(ptr, size);
174 break;
175 }
176 break;
177
178 case ICEPICK_PRIME_PREFETCHNTA:
179 prime_prefetchnta_sequential(ptr, size);
180 break;
181
182 case ICEPICK_PRIME_NT_STORE:
183 prime_nt_store_sequential(ptr, size);
184 break;
185 }
186 }
187}
188
189int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg,
190 icepick_region_t **region)
191{
192 if (!topo || !cfg || !region)
193 return ICEPICK_E_INVALID;
194
195 if (cfg->clos_id == 0 || cfg->clos_id >= topo->max_clos)
196 return ICEPICK_E_INVALID;
197
198 unsigned ways_needed = (cfg->size + topo->way_size - 1) / topo->way_size;
199 if (ways_needed == 0)
200 ways_needed = 1;
201
202 if (ways_needed >= topo->l3_ways)
203 return ICEPICK_E_TOO_LARGE;
204
205 int ret = clos_init(topo);
206 if (ret < 0)
207 return ret;
208
209 unsigned start_way = find_start_way(topo->default_way_mask, ways_needed);
210 uint32_t way_mask = compute_way_mask(ways_needed, start_way);
211
212 ret = clos_allocate(cfg->clos_id, way_mask);
213 if (ret < 0)
214 return ret;
215
216 icepick_region_t *r = calloc(1, sizeof(*r));
217 if (!r) {
218 clos_release(cfg->clos_id);
219 return ICEPICK_E_ALLOC;
220 }
221
222 r->clos_id = cfg->clos_id;
223 r->numa_node = cfg->numa_node;
224 r->way_mask = way_mask;
225 r->mba_throttle = cfg->mba_throttle;
226 r->core_type = cfg->core_type;
227 r->topo = topo;
228 r->size = ways_needed * topo->way_size;
229 r->monitor = NULL;
230 r->prime_strategy = cfg->prime_strategy;
231 r->access_pattern = cfg->access_pattern;
232 r->stride_bytes = cfg->stride_bytes;
233 r->prime_iterations = cfg->prime_iterations ? cfg->prime_iterations : 3;
234
235 ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
236 if (ret < 0) {
237 clos_release(cfg->clos_id);
238 free(r);
239 return ret;
240 }
241
242 int cpu = find_cpu_for_core_type(topo, cfg->core_type, cfg->numa_node);
243
244 cpu_set_t old_affinity, new_affinity;
245 CPU_ZERO(&new_affinity);
246 CPU_SET(cpu, &new_affinity);
247 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
248 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
249
250 int msr_fd = msr_open(cpu);
251 if (msr_fd < 0) {
252 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
253 region_free(r->ptr, r->size);
254 clos_release(cfg->clos_id);
255 free(r);
256 return msr_fd;
257 }
258
259 ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask);
260 if (ret < 0)
261 goto cleanup;
262
263 ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask);
264 if (ret < 0)
265 goto cleanup;
266
267 if (topo->mba_supported && cfg->mba_throttle > 0) {
268 ret = mba_configure(msr_fd, cfg->clos_id, cfg->mba_throttle);
269 if (ret < 0)
270 goto cleanup;
271 }
272
273 ret = clos_associate_thread(msr_fd, cfg->clos_id);
274 if (ret < 0)
275 goto cleanup;
276
277 prime_region(r->ptr, r->size, r->prime_iterations,
278 r->prime_strategy, r->access_pattern, r->stride_bytes);
279
280 clos_associate_thread(msr_fd, 0);
281
282 msr_close(msr_fd);
283 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
284
285 if (cfg->verify) {
286 icepick_latency_stats_t stats;
287 ret = icepick_verify(r, &stats);
288 if (ret < 0) {
289 icepick_unlock(r);
290 return ret;
291 }
292 }
293
294 if (cfg->auto_monitor) {
295 icepick_monitor_t *mon;
296 ret = icepick_monitor_start_ex(r, cfg->pmu_poll_interval_ns,
297 cfg->probe_interval_ns,
298 cfg->miss_threshold, &mon);
299 if (ret < 0) {
300 icepick_unlock(r);
301 return ret;
302 }
303 r->monitor = mon;
304 }
305
306 *region = r;
307 return 0;
308
309cleanup:
310 msr_close(msr_fd);
311 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
312 region_free(r->ptr, r->size);
313 clos_release(cfg->clos_id);
314 free(r);
315 return ret;
316}
317
318int icepick_unlock(icepick_region_t *region)
319{
320 if (!region)
321 return ICEPICK_E_INVALID;
322
323 if (region->monitor)
324 icepick_monitor_stop(region->monitor);
325
326 int cpu = sched_getcpu();
327 if (cpu < 0)
328 cpu = 0;
329
330 int msr_fd = msr_open(cpu);
331 if (msr_fd >= 0) {
332 clos_release(region->clos_id);
333 clos_configure_mask(msr_fd, region->clos_id, 0);
334 if (region->topo->mba_supported && region->mba_throttle > 0)
335 mba_configure(msr_fd, region->clos_id, 0);
336 clos_configure_mask(msr_fd, 0, clos_get_default_mask());
337 msr_close(msr_fd);
338 }
339
340 region_free(region->ptr, region->size);
341 free(region);
342
343 return 0;
344}
345
346void *icepick_region_ptr(const icepick_region_t *region)
347{
348 return region ? region->ptr : NULL;
349}
350
351size_t icepick_region_size(const icepick_region_t *region)
352{
353 return region ? region->size : 0;
354}
355
356unsigned icepick_region_clos(const icepick_region_t *region)
357{
358 return region ? region->clos_id : 0;
359}
360
361const char *icepick_strerror(int err)
362{
363 switch (err) {
364 case 0: return "success";
365 case ICEPICK_E_NO_CAT: return "cat not supported on this cpu";
366 case ICEPICK_E_PERMISSION: return "permission denied (need root or cap_sys_rawio)";
367 case ICEPICK_E_NO_CLOS: return "no clos available or already allocated";
368 case ICEPICK_E_TOO_LARGE: return "requested size exceeds available cache ways";
369 case ICEPICK_E_NUMA: return "invalid numa node or binding failed";
370 case ICEPICK_E_HUGEPAGE: return "huge page allocation failed";
371 case ICEPICK_E_VERIFY: return "verification failed - data may not be cache-resident";
372 case ICEPICK_E_INVALID: return "invalid argument";
373 case ICEPICK_E_ALLOC: return "memory allocation failed";
374 case ICEPICK_E_MSR: return "msr operation failed";
375 case ICEPICK_E_THREAD: return "monitor thread creation failed";
376 default: return "unknown error";
377 }
378}