vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <sched.h>
6#include <dirent.h>
7#include <stdio.h>
8#include <unistd.h>
9
10#ifdef __x86_64__
11#include <cpuid.h>
12
13static void cpuid(uint32_t leaf, uint32_t subleaf,
14 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)
15{
16 __cpuid_count(leaf, subleaf, *eax, *ebx, *ecx, *edx);
17}
18#endif
19
20arch_type_t detect_arch(void)
21{
22#ifdef __x86_64__
23 return ARCH_X86_64;
24#elif defined(__aarch64__)
25 return ARCH_ARM64;
26#else
27 return ARCH_UNKNOWN;
28#endif
29}
30
31#ifdef __x86_64__
32static bool detect_cat_support(icepick_topology_t *topo)
33{
34 uint32_t eax, ebx, ecx, edx;
35
36 cpuid(0x10, 0, &eax, &ebx, &ecx, &edx);
37 if (!(ebx & (1 << 1)))
38 return false;
39
40 cpuid(0x10, 1, &eax, &ebx, &ecx, &edx);
41
42 topo->l3_ways = (eax & 0x1F) + 1;
43 topo->max_clos = (edx & 0xFFFF) + 1;
44 topo->default_way_mask = ebx;
45
46 if (topo->max_clos > MAX_CLOS)
47 topo->max_clos = MAX_CLOS;
48
49 return true;
50}
51
52static bool detect_mba_support(icepick_topology_t *topo)
53{
54 uint32_t eax, ebx, ecx, edx;
55
56 cpuid(0x10, 0, &eax, &ebx, &ecx, &edx);
57 if (!(ebx & (1 << 3)))
58 return false;
59
60 cpuid(0x10, 3, &eax, &ebx, &ecx, &edx);
61
62 topo->max_mba_thrtl = (eax & 0xFFF) + 1;
63 topo->mba_is_linear = (ecx & (1 << 2)) != 0;
64
65 return true;
66}
67
68static void detect_amd_cache_topology(icepick_topology_t *topo)
69{
70 uint32_t eax, ebx, ecx, edx;
71
72 for (int i = 0; i < 16; i++) {
73 cpuid(0x8000001D, i, &eax, &ebx, &ecx, &edx);
74
75 uint32_t cache_type = eax & 0x1F;
76 if (cache_type == 0)
77 break;
78
79 uint32_t cache_level = (eax >> 5) & 0x7;
80 if (cache_level != 3)
81 continue;
82
83 uint32_t line_size = (ebx & 0xFFF) + 1;
84 uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1;
85 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
86 uint32_t sets = ecx + 1;
87
88 topo->l3_size = (size_t)line_size * partitions * ways * sets;
89 topo->l3_ways = ways;
90 topo->way_size = topo->l3_size / topo->l3_ways;
91 break;
92 }
93}
94
95static void detect_intel_cache_topology(icepick_topology_t *topo)
96{
97 uint32_t eax, ebx, ecx, edx;
98
99 for (int i = 0; i < 16; i++) {
100 cpuid(0x04, i, &eax, &ebx, &ecx, &edx);
101
102 uint32_t cache_type = eax & 0x1F;
103 if (cache_type == 0)
104 break;
105
106 uint32_t cache_level = (eax >> 5) & 0x7;
107 if (cache_level != 3)
108 continue;
109
110 uint32_t line_size = (ebx & 0xFFF) + 1;
111 uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1;
112 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
113 uint32_t sets = ecx + 1;
114
115 topo->l3_size = (size_t)line_size * partitions * ways * sets;
116 topo->l3_ways = ways;
117 topo->way_size = topo->l3_size / topo->l3_ways;
118 break;
119 }
120}
121
122static bool is_amd_cpu(void)
123{
124 uint32_t eax, ebx, ecx, edx;
125 cpuid(0, 0, &eax, &ebx, &ecx, &edx);
126 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
127}
128
129static bool detect_hybrid_architecture(icepick_topology_t *topo)
130{
131 (void)topo;
132 uint32_t eax, ebx, ecx, edx;
133
134 cpuid(0x00, 0, &eax, &ebx, &ecx, &edx);
135 if (eax < 0x1A)
136 return false;
137
138 cpuid(0x07, 0, &eax, &ebx, &ecx, &edx);
139 if (!(edx & (1 << 15)))
140 return false;
141
142 return true;
143}
144
145static void enumerate_hybrid_cores(icepick_topology_t *topo)
146{
147 int num_cpus = sysconf(_SC_NPROCESSORS_ONLN);
148 if (num_cpus <= 0)
149 return;
150
151 cpu_set_t original_affinity;
152 sched_getaffinity(0, sizeof(original_affinity), &original_affinity);
153
154 topo->pcore_count = 0;
155 topo->ecore_count = 0;
156
157 for (int cpu = 0; cpu < num_cpus && cpu < 320; cpu++) {
158 cpu_set_t cpuset;
159 CPU_ZERO(&cpuset);
160 CPU_SET(cpu, &cpuset);
161
162 if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0)
163 continue;
164
165 uint32_t eax, ebx, ecx, edx;
166 cpuid(0x1A, 0, &eax, &ebx, &ecx, &edx);
167
168 uint32_t core_type = (eax >> 24) & 0xFF;
169
170 if (core_type == 0x40 && topo->pcore_count < 64) {
171 topo->pcore_cpus[topo->pcore_count++] = cpu;
172 } else if (core_type == 0x20 && topo->ecore_count < 256) {
173 topo->ecore_cpus[topo->ecore_count++] = cpu;
174 }
175 }
176
177 sched_setaffinity(0, sizeof(original_affinity), &original_affinity);
178}
179#endif
180
181#ifdef __aarch64__
182static bool detect_mpam_support(icepick_topology_t *topo)
183{
184 FILE *f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/id", "r");
185 if (!f)
186 return false;
187 fclose(f);
188
189 topo->max_partid = 16;
190 return true;
191}
192
193static void detect_arm64_cache_topology(icepick_topology_t *topo)
194{
195 FILE *f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/size", "r");
196 if (f) {
197 unsigned size_kb = 0;
198 if (fscanf(f, "%uK", &size_kb) == 1)
199 topo->l3_size = (size_t)size_kb * 1024;
200 fclose(f);
201 }
202
203 f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/ways_of_associativity", "r");
204 if (f) {
205 fscanf(f, "%u", &topo->l3_ways);
206 fclose(f);
207 }
208
209 if (topo->l3_size > 0 && topo->l3_ways > 0)
210 topo->way_size = topo->l3_size / topo->l3_ways;
211
212 if (topo->l3_ways > 0)
213 topo->default_way_mask = (1U << topo->l3_ways) - 1;
214}
215#endif
216
217static int count_ccxs(void)
218{
219 DIR *dir = opendir("/sys/devices/system/node");
220 if (!dir)
221 return 1;
222
223 int count = 0;
224 struct dirent *entry;
225 while ((entry = readdir(dir))) {
226 if (strncmp(entry->d_name, "node", 4) == 0)
227 count++;
228 }
229 closedir(dir);
230
231 return count > 0 ? count : 1;
232}
233
234static void detect_numa_topology(icepick_topology_t *topo)
235{
236 topo->ccx_count = count_ccxs();
237 if (topo->ccx_count > MAX_CCXS)
238 topo->ccx_count = MAX_CCXS;
239
240 for (unsigned i = 0; i < topo->ccx_count; i++)
241 topo->ccx_numa_node[i] = i;
242}
243
244int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type,
245 int numa_node)
246{
247 (void)numa_node;
248
249 if (!topo->is_hybrid || core_type == ICEPICK_CORE_ANY) {
250 int cpu = sched_getcpu();
251 return cpu >= 0 ? cpu : 0;
252 }
253
254 if (core_type == ICEPICK_CORE_PCORE && topo->pcore_count > 0)
255 return topo->pcore_cpus[0];
256
257 if (core_type == ICEPICK_CORE_ECORE && topo->ecore_count > 0)
258 return topo->ecore_cpus[0];
259
260 return 0;
261}
262
263int icepick_discover_topology(icepick_topology_t **topo)
264{
265 icepick_topology_t *t = calloc(1, sizeof(*t));
266 if (!t)
267 return ICEPICK_E_ALLOC;
268
269 t->arch = detect_arch();
270
271#ifdef __x86_64__
272 if (t->arch == ARCH_X86_64) {
273 t->cat_supported = detect_cat_support(t);
274 if (!t->cat_supported) {
275 free(t);
276 return ICEPICK_E_NO_CAT;
277 }
278
279 t->mba_supported = detect_mba_support(t);
280
281 bool is_amd = is_amd_cpu();
282 if (is_amd)
283 detect_amd_cache_topology(t);
284 else
285 detect_intel_cache_topology(t);
286
287 if (!is_amd) {
288 t->is_hybrid = detect_hybrid_architecture(t);
289 if (t->is_hybrid)
290 enumerate_hybrid_cores(t);
291 }
292 }
293#endif
294
295#ifdef __aarch64__
296 if (t->arch == ARCH_ARM64) {
297 t->mpam_supported = detect_mpam_support(t);
298 if (!t->mpam_supported) {
299 free(t);
300 return ICEPICK_E_NO_CAT;
301 }
302
303 t->cat_supported = t->mpam_supported;
304 detect_arm64_cache_topology(t);
305 t->max_clos = t->max_partid;
306 }
307#endif
308
309 if (t->l3_size == 0 || t->l3_ways == 0) {
310 t->way_size = 2 * 1024 * 1024;
311 t->l3_size = t->way_size * t->l3_ways;
312 }
313
314 detect_numa_topology(t);
315
316 *topo = t;
317 return 0;
318}
319
320void icepick_free_topology(icepick_topology_t *topo)
321{
322 free(topo);
323}
324
325unsigned icepick_topology_l3_ways(const icepick_topology_t *topo)
326{
327 return topo->l3_ways;
328}
329
330size_t icepick_topology_l3_size(const icepick_topology_t *topo)
331{
332 return topo->l3_size;
333}
334
335size_t icepick_topology_way_size(const icepick_topology_t *topo)
336{
337 return topo->way_size;
338}
339
340unsigned icepick_topology_max_clos(const icepick_topology_t *topo)
341{
342 return topo->max_clos;
343}
344
345unsigned icepick_topology_ccx_count(const icepick_topology_t *topo)
346{
347 return topo->ccx_count;
348}
349
350bool icepick_topology_cat_supported(const icepick_topology_t *topo)
351{
352 return topo->cat_supported;
353}
354
355bool icepick_topology_mba_supported(const icepick_topology_t *topo)
356{
357 return topo->mba_supported;
358}
359
360bool icepick_topology_mba_is_linear(const icepick_topology_t *topo)
361{
362 return topo->mba_is_linear;
363}
364
365unsigned icepick_topology_max_mba_thrtl(const icepick_topology_t *topo)
366{
367 return topo->max_mba_thrtl;
368}
369
370bool icepick_topology_is_hybrid(const icepick_topology_t *topo)
371{
372 return topo->is_hybrid;
373}
374
375unsigned icepick_topology_pcore_count(const icepick_topology_t *topo)
376{
377 return topo->pcore_count;
378}
379
380unsigned icepick_topology_ecore_count(const icepick_topology_t *topo)
381{
382 return topo->ecore_count;
383}