vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <sched.h>
5#include <unistd.h>
6#include <string.h>
7#include <x86intrin.h>
8
9static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way)
10{
11 uint32_t mask = 0;
12 for (unsigned i = 0; i < num_ways; i++)
13 mask |= (1U << (start_way + i));
14 return mask;
15}
16
17static unsigned find_start_way(uint32_t default_mask, unsigned num_ways)
18{
19 for (unsigned i = 0; i <= 32 - num_ways; i++) {
20 uint32_t candidate = compute_way_mask(num_ways, i);
21 if ((candidate & default_mask) == candidate)
22 return i;
23 }
24 return 0;
25}
26
27static void init_pointer_chase(volatile char *ptr, size_t size)
28{
29 size_t num_lines = size / CACHE_LINE_SIZE;
30 volatile size_t *indices = (volatile size_t *)ptr;
31
32 for (size_t i = 0; i < num_lines; i++)
33 indices[i * (CACHE_LINE_SIZE / sizeof(size_t))] = ((i + 1) % num_lines) * CACHE_LINE_SIZE;
34
35 for (size_t i = num_lines - 1; i > 0; i--) {
36 size_t j = (size_t)rdtsc_start() % (i + 1);
37 size_t idx_i = i * (CACHE_LINE_SIZE / sizeof(size_t));
38 size_t idx_j = j * (CACHE_LINE_SIZE / sizeof(size_t));
39 size_t tmp = indices[idx_i];
40 indices[idx_i] = indices[idx_j];
41 indices[idx_j] = tmp;
42 }
43}
44
45static void prime_temporal_sequential(volatile char *ptr, size_t size)
46{
47 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
48 (void)ptr[offset];
49}
50
51static void prime_temporal_reverse(volatile char *ptr, size_t size)
52{
53 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
54 (void)ptr[offset - CACHE_LINE_SIZE];
55}
56
57static void prime_temporal_strided(volatile char *ptr, size_t size, size_t stride)
58{
59 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
60 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
61 for (size_t offset = pass; offset < size; offset += effective_stride)
62 (void)ptr[offset];
63 }
64}
65
66static void prime_temporal_chase(volatile char *ptr, size_t size)
67{
68 volatile size_t *indices = (volatile size_t *)ptr;
69 size_t num_lines = size / CACHE_LINE_SIZE;
70 size_t offset = 0;
71
72 for (size_t i = 0; i < num_lines; i++) {
73 offset = indices[offset / sizeof(size_t)];
74 }
75
76 (void)offset;
77}
78
79static void prime_prefetcht2_sequential(volatile char *ptr, size_t size)
80{
81 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
82 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
83 _mm_mfence();
84 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
85 (void)ptr[offset];
86}
87
88static void prime_prefetcht2_reverse(volatile char *ptr, size_t size)
89{
90 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
91 _mm_prefetch((const char *)&ptr[offset - CACHE_LINE_SIZE], _MM_HINT_T2);
92 _mm_mfence();
93 for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
94 (void)ptr[offset - CACHE_LINE_SIZE];
95}
96
97static void prime_prefetcht2_strided(volatile char *ptr, size_t size, size_t stride)
98{
99 size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
100 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
101 for (size_t offset = pass; offset < size; offset += effective_stride)
102 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
103 }
104 _mm_mfence();
105 for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
106 for (size_t offset = pass; offset < size; offset += effective_stride)
107 (void)ptr[offset];
108 }
109}
110
111static void prime_prefetchnta_sequential(volatile char *ptr, size_t size)
112{
113 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
114 _mm_prefetch((const char *)&ptr[offset], _MM_HINT_NTA);
115 _mm_mfence();
116 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
117 (void)ptr[offset];
118}
119
120static void prime_nt_store_sequential(volatile char *ptr, size_t size)
121{
122 __m128i zero = _mm_setzero_si128();
123 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
124 _mm_stream_si128((__m128i *)&ptr[offset], zero);
125 _mm_stream_si128((__m128i *)&ptr[offset + 16], zero);
126 _mm_stream_si128((__m128i *)&ptr[offset + 32], zero);
127 _mm_stream_si128((__m128i *)&ptr[offset + 48], zero);
128 }
129 _mm_sfence();
130 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
131 (void)ptr[offset];
132}
133
134void prime_region(volatile char *ptr, size_t size, unsigned iterations,
135 icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
136 size_t stride)
137{
138 if (pattern == ICEPICK_PATTERN_POINTER_CHASE)
139 init_pointer_chase(ptr, size);
140
141 for (unsigned iter = 0; iter < iterations; iter++) {
142 switch (strategy) {
143 case ICEPICK_PRIME_TEMPORAL:
144 switch (pattern) {
145 case ICEPICK_PATTERN_SEQUENTIAL:
146 prime_temporal_sequential(ptr, size);
147 break;
148 case ICEPICK_PATTERN_REVERSE:
149 prime_temporal_reverse(ptr, size);
150 break;
151 case ICEPICK_PATTERN_STRIDED:
152 prime_temporal_strided(ptr, size, stride);
153 break;
154 case ICEPICK_PATTERN_POINTER_CHASE:
155 prime_temporal_chase(ptr, size);
156 break;
157 }
158 break;
159
160 case ICEPICK_PRIME_PREFETCHT2:
161 switch (pattern) {
162 case ICEPICK_PATTERN_SEQUENTIAL:
163 prime_prefetcht2_sequential(ptr, size);
164 break;
165 case ICEPICK_PATTERN_REVERSE:
166 prime_prefetcht2_reverse(ptr, size);
167 break;
168 case ICEPICK_PATTERN_STRIDED:
169 prime_prefetcht2_strided(ptr, size, stride);
170 break;
171 case ICEPICK_PATTERN_POINTER_CHASE:
172 prime_prefetcht2_sequential(ptr, size);
173 prime_temporal_chase(ptr, size);
174 break;
175 }
176 break;
177
178 case ICEPICK_PRIME_PREFETCHNTA:
179 prime_prefetchnta_sequential(ptr, size);
180 break;
181
182 case ICEPICK_PRIME_NT_STORE:
183 prime_nt_store_sequential(ptr, size);
184 break;
185 }
186 }
187}
188
189int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg,
190 icepick_region_t **region)
191{
192 if (!topo || !cfg || !region)
193 return ICEPICK_E_INVALID;
194
195 if (cfg->clos_id == 0 || cfg->clos_id >= topo->max_clos)
196 return ICEPICK_E_INVALID;
197
198 unsigned ways_needed = (cfg->size + topo->way_size - 1) / topo->way_size;
199 if (ways_needed == 0)
200 ways_needed = 1;
201
202 if (ways_needed >= topo->l3_ways)
203 return ICEPICK_E_TOO_LARGE;
204
205 int ret = clos_init(topo);
206 if (ret < 0)
207 return ret;
208
209 unsigned start_way = find_start_way(topo->default_way_mask, ways_needed);
210 uint32_t way_mask = compute_way_mask(ways_needed, start_way);
211
212 ret = clos_allocate(cfg->clos_id, way_mask);
213 if (ret < 0)
214 return ret;
215
216 icepick_region_t *r = calloc(1, sizeof(*r));
217 if (!r) {
218 clos_release(cfg->clos_id);
219 return ICEPICK_E_ALLOC;
220 }
221
222 r->clos_id = cfg->clos_id;
223 r->numa_node = cfg->numa_node;
224 r->way_mask = way_mask;
225 r->mba_throttle = cfg->mba_throttle;
226 r->topo = topo;
227 r->size = ways_needed * topo->way_size;
228 r->monitor = NULL;
229 r->prime_strategy = cfg->prime_strategy;
230 r->access_pattern = cfg->access_pattern;
231 r->stride_bytes = cfg->stride_bytes;
232 r->prime_iterations = cfg->prime_iterations ? cfg->prime_iterations : 3;
233
234 ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
235 if (ret < 0) {
236 clos_release(cfg->clos_id);
237 free(r);
238 return ret;
239 }
240
241 int cpu = sched_getcpu();
242 if (cpu < 0)
243 cpu = 0;
244
245 cpu_set_t old_affinity, new_affinity;
246 CPU_ZERO(&new_affinity);
247 CPU_SET(cpu, &new_affinity);
248 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
249 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
250
251 int msr_fd = msr_open(cpu);
252 if (msr_fd < 0) {
253 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
254 region_free(r->ptr, r->size);
255 clos_release(cfg->clos_id);
256 free(r);
257 return msr_fd;
258 }
259
260 ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask);
261 if (ret < 0)
262 goto cleanup;
263
264 ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask);
265 if (ret < 0)
266 goto cleanup;
267
268 if (topo->mba_supported && cfg->mba_throttle > 0) {
269 ret = mba_configure(msr_fd, cfg->clos_id, cfg->mba_throttle);
270 if (ret < 0)
271 goto cleanup;
272 }
273
274 ret = clos_associate_thread(msr_fd, cfg->clos_id);
275 if (ret < 0)
276 goto cleanup;
277
278 prime_region(r->ptr, r->size, r->prime_iterations,
279 r->prime_strategy, r->access_pattern, r->stride_bytes);
280
281 clos_associate_thread(msr_fd, 0);
282
283 msr_close(msr_fd);
284 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
285
286 if (cfg->verify) {
287 icepick_latency_stats_t stats;
288 ret = icepick_verify(r, &stats);
289 if (ret < 0) {
290 icepick_unlock(r);
291 return ret;
292 }
293 }
294
295 if (cfg->auto_monitor) {
296 icepick_monitor_t *mon;
297 ret = icepick_monitor_start_ex(r, cfg->pmu_poll_interval_ns,
298 cfg->probe_interval_ns,
299 cfg->miss_threshold, &mon);
300 if (ret < 0) {
301 icepick_unlock(r);
302 return ret;
303 }
304 r->monitor = mon;
305 }
306
307 *region = r;
308 return 0;
309
310cleanup:
311 msr_close(msr_fd);
312 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
313 region_free(r->ptr, r->size);
314 clos_release(cfg->clos_id);
315 free(r);
316 return ret;
317}
318
319int icepick_unlock(icepick_region_t *region)
320{
321 if (!region)
322 return ICEPICK_E_INVALID;
323
324 if (region->monitor)
325 icepick_monitor_stop(region->monitor);
326
327 int cpu = sched_getcpu();
328 if (cpu < 0)
329 cpu = 0;
330
331 int msr_fd = msr_open(cpu);
332 if (msr_fd >= 0) {
333 clos_release(region->clos_id);
334 clos_configure_mask(msr_fd, region->clos_id, 0);
335 if (region->topo->mba_supported && region->mba_throttle > 0)
336 mba_configure(msr_fd, region->clos_id, 0);
337 clos_configure_mask(msr_fd, 0, clos_get_default_mask());
338 msr_close(msr_fd);
339 }
340
341 region_free(region->ptr, region->size);
342 free(region);
343
344 return 0;
345}
346
347void *icepick_region_ptr(const icepick_region_t *region)
348{
349 return region ? region->ptr : NULL;
350}
351
352size_t icepick_region_size(const icepick_region_t *region)
353{
354 return region ? region->size : 0;
355}
356
357unsigned icepick_region_clos(const icepick_region_t *region)
358{
359 return region ? region->clos_id : 0;
360}
361
362const char *icepick_strerror(int err)
363{
364 switch (err) {
365 case 0: return "success";
366 case ICEPICK_E_NO_CAT: return "cat not supported on this cpu";
367 case ICEPICK_E_PERMISSION: return "permission denied (need root or cap_sys_rawio)";
368 case ICEPICK_E_NO_CLOS: return "no clos available or already allocated";
369 case ICEPICK_E_TOO_LARGE: return "requested size exceeds available cache ways";
370 case ICEPICK_E_NUMA: return "invalid numa node or binding failed";
371 case ICEPICK_E_HUGEPAGE: return "huge page allocation failed";
372 case ICEPICK_E_VERIFY: return "verification failed - data may not be cache-resident";
373 case ICEPICK_E_INVALID: return "invalid argument";
374 case ICEPICK_E_ALLOC: return "memory allocation failed";
375 case ICEPICK_E_MSR: return "msr operation failed";
376 case ICEPICK_E_THREAD: return "monitor thread creation failed";
377 default: return "unknown error";
378 }
379}