vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <cpuid.h>
6#include <sched.h>
7#include <dirent.h>
8#include <stdio.h>
9#include <unistd.h>
10
11static void cpuid(uint32_t leaf, uint32_t subleaf,
12 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)
13{
14 __cpuid_count(leaf, subleaf, *eax, *ebx, *ecx, *edx);
15}
16
17static bool detect_cat_support(icepick_topology_t *topo)
18{
19 uint32_t eax, ebx, ecx, edx;
20
21 cpuid(0x10, 0, &eax, &ebx, &ecx, &edx);
22 if (!(ebx & (1 << 1)))
23 return false;
24
25 cpuid(0x10, 1, &eax, &ebx, &ecx, &edx);
26
27 topo->l3_ways = (eax & 0x1F) + 1;
28 topo->max_clos = (edx & 0xFFFF) + 1;
29 topo->default_way_mask = ebx;
30
31 if (topo->max_clos > MAX_CLOS)
32 topo->max_clos = MAX_CLOS;
33
34 return true;
35}
36
37static bool detect_mba_support(icepick_topology_t *topo)
38{
39 uint32_t eax, ebx, ecx, edx;
40
41 cpuid(0x10, 0, &eax, &ebx, &ecx, &edx);
42 if (!(ebx & (1 << 3)))
43 return false;
44
45 cpuid(0x10, 3, &eax, &ebx, &ecx, &edx);
46
47 topo->max_mba_thrtl = (eax & 0xFFF) + 1;
48 topo->mba_is_linear = (ecx & (1 << 2)) != 0;
49
50 return true;
51}
52
53static void detect_amd_cache_topology(icepick_topology_t *topo)
54{
55 uint32_t eax, ebx, ecx, edx;
56
57 for (int i = 0; i < 16; i++) {
58 cpuid(0x8000001D, i, &eax, &ebx, &ecx, &edx);
59
60 uint32_t cache_type = eax & 0x1F;
61 if (cache_type == 0)
62 break;
63
64 uint32_t cache_level = (eax >> 5) & 0x7;
65 if (cache_level != 3)
66 continue;
67
68 uint32_t line_size = (ebx & 0xFFF) + 1;
69 uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1;
70 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
71 uint32_t sets = ecx + 1;
72
73 topo->l3_size = (size_t)line_size * partitions * ways * sets;
74 topo->l3_ways = ways;
75 topo->way_size = topo->l3_size / topo->l3_ways;
76 break;
77 }
78}
79
80static void detect_intel_cache_topology(icepick_topology_t *topo)
81{
82 uint32_t eax, ebx, ecx, edx;
83
84 for (int i = 0; i < 16; i++) {
85 cpuid(0x04, i, &eax, &ebx, &ecx, &edx);
86
87 uint32_t cache_type = eax & 0x1F;
88 if (cache_type == 0)
89 break;
90
91 uint32_t cache_level = (eax >> 5) & 0x7;
92 if (cache_level != 3)
93 continue;
94
95 uint32_t line_size = (ebx & 0xFFF) + 1;
96 uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1;
97 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
98 uint32_t sets = ecx + 1;
99
100 topo->l3_size = (size_t)line_size * partitions * ways * sets;
101 topo->l3_ways = ways;
102 topo->way_size = topo->l3_size / topo->l3_ways;
103 break;
104 }
105}
106
107static bool is_amd_cpu(void)
108{
109 uint32_t eax, ebx, ecx, edx;
110 cpuid(0, 0, &eax, &ebx, &ecx, &edx);
111 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
112}
113
114static int count_ccxs(void)
115{
116 DIR *dir = opendir("/sys/devices/system/node");
117 if (!dir)
118 return 1;
119
120 int count = 0;
121 struct dirent *entry;
122 while ((entry = readdir(dir))) {
123 if (strncmp(entry->d_name, "node", 4) == 0)
124 count++;
125 }
126 closedir(dir);
127
128 return count > 0 ? count : 1;
129}
130
131static void detect_numa_topology(icepick_topology_t *topo)
132{
133 topo->ccx_count = count_ccxs();
134 if (topo->ccx_count > MAX_CCXS)
135 topo->ccx_count = MAX_CCXS;
136
137 for (unsigned i = 0; i < topo->ccx_count; i++)
138 topo->ccx_numa_node[i] = i;
139}
140
141static bool detect_hybrid_architecture(icepick_topology_t *topo)
142{
143 (void)topo;
144 uint32_t eax, ebx, ecx, edx;
145
146 cpuid(0x00, 0, &eax, &ebx, &ecx, &edx);
147 if (eax < 0x1A)
148 return false;
149
150 cpuid(0x07, 0, &eax, &ebx, &ecx, &edx);
151 if (!(edx & (1 << 15)))
152 return false;
153
154 return true;
155}
156
157static void enumerate_hybrid_cores(icepick_topology_t *topo)
158{
159 int num_cpus = sysconf(_SC_NPROCESSORS_ONLN);
160 if (num_cpus <= 0)
161 return;
162
163 cpu_set_t original_affinity;
164 sched_getaffinity(0, sizeof(original_affinity), &original_affinity);
165
166 topo->pcore_count = 0;
167 topo->ecore_count = 0;
168
169 for (int cpu = 0; cpu < num_cpus && cpu < 320; cpu++) {
170 cpu_set_t cpuset;
171 CPU_ZERO(&cpuset);
172 CPU_SET(cpu, &cpuset);
173
174 if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0)
175 continue;
176
177 uint32_t eax, ebx, ecx, edx;
178 cpuid(0x1A, 0, &eax, &ebx, &ecx, &edx);
179
180 uint32_t core_type = (eax >> 24) & 0xFF;
181
182 if (core_type == 0x40 && topo->pcore_count < 64) {
183 topo->pcore_cpus[topo->pcore_count++] = cpu;
184 } else if (core_type == 0x20 && topo->ecore_count < 256) {
185 topo->ecore_cpus[topo->ecore_count++] = cpu;
186 }
187 }
188
189 sched_setaffinity(0, sizeof(original_affinity), &original_affinity);
190}
191
192int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type,
193 int numa_node)
194{
195 (void)numa_node;
196
197 if (!topo->is_hybrid || core_type == ICEPICK_CORE_ANY) {
198 int cpu = sched_getcpu();
199 return cpu >= 0 ? cpu : 0;
200 }
201
202 if (core_type == ICEPICK_CORE_PCORE && topo->pcore_count > 0)
203 return topo->pcore_cpus[0];
204
205 if (core_type == ICEPICK_CORE_ECORE && topo->ecore_count > 0)
206 return topo->ecore_cpus[0];
207
208 return 0;
209}
210
211int icepick_discover_topology(icepick_topology_t **topo)
212{
213 icepick_topology_t *t = calloc(1, sizeof(*t));
214 if (!t)
215 return ICEPICK_E_ALLOC;
216
217 t->cat_supported = detect_cat_support(t);
218 if (!t->cat_supported) {
219 free(t);
220 return ICEPICK_E_NO_CAT;
221 }
222
223 t->mba_supported = detect_mba_support(t);
224
225 bool is_amd = is_amd_cpu();
226 if (is_amd)
227 detect_amd_cache_topology(t);
228 else
229 detect_intel_cache_topology(t);
230
231 if (t->l3_size == 0 || t->l3_ways == 0) {
232 t->way_size = 2 * 1024 * 1024;
233 t->l3_size = t->way_size * t->l3_ways;
234 }
235
236 detect_numa_topology(t);
237
238 if (!is_amd) {
239 t->is_hybrid = detect_hybrid_architecture(t);
240 if (t->is_hybrid)
241 enumerate_hybrid_cores(t);
242 }
243
244 *topo = t;
245 return 0;
246}
247
248void icepick_free_topology(icepick_topology_t *topo)
249{
250 free(topo);
251}
252
253unsigned icepick_topology_l3_ways(const icepick_topology_t *topo)
254{
255 return topo->l3_ways;
256}
257
258size_t icepick_topology_l3_size(const icepick_topology_t *topo)
259{
260 return topo->l3_size;
261}
262
263size_t icepick_topology_way_size(const icepick_topology_t *topo)
264{
265 return topo->way_size;
266}
267
268unsigned icepick_topology_max_clos(const icepick_topology_t *topo)
269{
270 return topo->max_clos;
271}
272
273unsigned icepick_topology_ccx_count(const icepick_topology_t *topo)
274{
275 return topo->ccx_count;
276}
277
278bool icepick_topology_cat_supported(const icepick_topology_t *topo)
279{
280 return topo->cat_supported;
281}
282
283bool icepick_topology_mba_supported(const icepick_topology_t *topo)
284{
285 return topo->mba_supported;
286}
287
288bool icepick_topology_mba_is_linear(const icepick_topology_t *topo)
289{
290 return topo->mba_is_linear;
291}
292
293unsigned icepick_topology_max_mba_thrtl(const icepick_topology_t *topo)
294{
295 return topo->max_mba_thrtl;
296}
297
298bool icepick_topology_is_hybrid(const icepick_topology_t *topo)
299{
300 return topo->is_hybrid;
301}
302
303unsigned icepick_topology_pcore_count(const icepick_topology_t *topo)
304{
305 return topo->pcore_count;
306}
307
308unsigned icepick_topology_ecore_count(const icepick_topology_t *topo)
309{
310 return topo->ecore_count;
311}