vm.fail / icepick
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
26
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

port cache partitioning backend to arm64 mpam for neoverse platforms

author vm.fail date (Jan 18, 2026, 3:41 AM UTC) commit 149475eb parent d60cc663
+265 -81
+1
Makefile
··· 62 62 $(OBJDIR)/verify.o: $(SRCDIR)/verify.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 63 63 $(OBJDIR)/bench.o: $(SRCDIR)/bench.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 64 64 $(OBJDIR)/monitor.o: $(SRCDIR)/monitor.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 65 + $(OBJDIR)/mpam.o: $(SRCDIR)/mpam.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 65 66 $(OBJDIR)/main.o: $(SRCDIR)/main.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
+16
src/internal.h
··· 11 11 #define MSR_IA32_L2_QOS_CFG 0xC82 12 12 #define MSR_IA32_MBA_THRTL_BASE 0xD50 13 13 14 + #define MPAM_PARTID_BITS 16 15 + #define MPAM_MAX_PARTID ((1 << MPAM_PARTID_BITS) - 1) 16 + 14 17 #define CACHE_LINE_SIZE 64 15 18 #define MAX_CLOS 16 16 19 #define MAX_CCXS 16 17 20 #define MAX_CORE_TYPES 2 18 21 22 + typedef enum { 23 + ARCH_X86_64, 24 + ARCH_ARM64, 25 + ARCH_UNKNOWN 26 + } arch_type_t; 27 + 19 28 #define MONITOR_DEFAULT_PMU_POLL_NS 100000 20 29 #define MONITOR_DEFAULT_PROBE_NS 10000 21 30 #define MONITOR_DEFAULT_MISS_THRESH 16 ··· 25 34 struct icepick_monitor; 26 35 27 36 struct icepick_topology { 37 + arch_type_t arch; 28 38 bool cat_supported; 29 39 bool mba_supported; 30 40 bool mba_is_linear; 31 41 bool is_hybrid; 42 + bool mpam_supported; 32 43 unsigned l3_ways; 33 44 size_t l3_size; 34 45 size_t way_size; 35 46 unsigned max_clos; 36 47 unsigned max_mba_thrtl; 48 + unsigned max_partid; 37 49 unsigned ccx_count; 38 50 uint32_t default_way_mask; 39 51 int ccx_numa_node[MAX_CCXS]; ··· 110 122 int mba_configure(int msr_fd, unsigned clos_id, unsigned throttle); 111 123 int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type, 112 124 int numa_node); 125 + 126 + arch_type_t detect_arch(void); 127 + int mpam_configure_partition(unsigned partid, uint32_t cpbm, unsigned mbw_max); 128 + int mpam_associate_thread(unsigned partid); 113 129 114 130 #endif
+64 -37
src/lock.c
··· 247 247 sched_getaffinity(0, sizeof(old_affinity), &old_affinity); 248 248 sched_setaffinity(0, sizeof(new_affinity), &new_affinity); 249 249 250 - int msr_fd = msr_open(cpu); 251 - if (msr_fd < 0) { 252 - sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 253 - region_free(r->ptr, r->size); 254 - clos_release(cfg->clos_id); 255 - free(r); 256 - return msr_fd; 257 - } 250 + if (topo->arch == ARCH_X86_64) { 251 + int msr_fd = msr_open(cpu); 252 + if (msr_fd < 0) { 253 + sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 254 + region_free(r->ptr, r->size); 255 + clos_release(cfg->clos_id); 256 + free(r); 257 + return msr_fd; 258 + } 259 + 260 + ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask); 261 + if (ret < 0) { 262 + msr_close(msr_fd); 263 + goto cleanup_no_msr; 264 + } 265 + 266 + ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask); 267 + if (ret < 0) { 268 + msr_close(msr_fd); 269 + goto cleanup_no_msr; 270 + } 271 + 272 + if (topo->mba_supported && cfg->mba_throttle > 0) { 273 + ret = mba_configure(msr_fd, cfg->clos_id, cfg->mba_throttle); 274 + if (ret < 0) { 275 + msr_close(msr_fd); 276 + goto cleanup_no_msr; 277 + } 278 + } 258 279 259 - ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask); 260 - if (ret < 0) 261 - goto cleanup; 280 + ret = clos_associate_thread(msr_fd, cfg->clos_id); 281 + if (ret < 0) { 282 + msr_close(msr_fd); 283 + goto cleanup_no_msr; 284 + } 262 285 263 - ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask); 264 - if (ret < 0) 265 - goto cleanup; 286 + prime_region(r->ptr, r->size, r->prime_iterations, 287 + r->prime_strategy, r->access_pattern, r->stride_bytes); 266 288 267 - if (topo->mba_supported && cfg->mba_throttle > 0) { 268 - ret = mba_configure(msr_fd, cfg->clos_id, cfg->mba_throttle); 289 + clos_associate_thread(msr_fd, 0); 290 + msr_close(msr_fd); 291 + } else if (topo->arch == ARCH_ARM64) { 292 + ret = mpam_configure_partition(cfg->clos_id, way_mask, cfg->mba_throttle); 269 293 if (ret < 0) 270 - goto cleanup; 271 - } 294 + goto cleanup_no_msr; 272 295 273 - ret = clos_associate_thread(msr_fd, cfg->clos_id); 274 - if (ret < 0) 275 - goto cleanup; 296 + ret = mpam_associate_thread(cfg->clos_id); 297 + if (ret < 0) 298 + goto cleanup_no_msr; 276 299 277 - prime_region(r->ptr, r->size, r->prime_iterations, 278 - r->prime_strategy, r->access_pattern, r->stride_bytes); 300 + prime_region(r->ptr, r->size, r->prime_iterations, 301 + r->prime_strategy, r->access_pattern, r->stride_bytes); 279 302 280 - clos_associate_thread(msr_fd, 0); 303 + mpam_associate_thread(0); 304 + } 281 305 282 - msr_close(msr_fd); 283 306 sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 284 307 285 308 if (cfg->verify) { ··· 306 329 *region = r; 307 330 return 0; 308 331 309 - cleanup: 310 - msr_close(msr_fd); 332 + cleanup_no_msr: 311 333 sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 312 334 region_free(r->ptr, r->size); 313 335 clos_release(cfg->clos_id); ··· 323 345 if (region->monitor) 324 346 icepick_monitor_stop(region->monitor); 325 347 326 - int cpu = sched_getcpu(); 327 - if (cpu < 0) 328 - cpu = 0; 348 + if (region->topo->arch == ARCH_X86_64) { 349 + int cpu = sched_getcpu(); 350 + if (cpu < 0) 351 + cpu = 0; 329 352 330 - int msr_fd = msr_open(cpu); 331 - if (msr_fd >= 0) { 353 + int msr_fd = msr_open(cpu); 354 + if (msr_fd >= 0) { 355 + clos_release(region->clos_id); 356 + clos_configure_mask(msr_fd, region->clos_id, 0); 357 + if (region->topo->mba_supported && region->mba_throttle > 0) 358 + mba_configure(msr_fd, region->clos_id, 0); 359 + clos_configure_mask(msr_fd, 0, clos_get_default_mask()); 360 + msr_close(msr_fd); 361 + } 362 + } else if (region->topo->arch == ARCH_ARM64) { 332 363 clos_release(region->clos_id); 333 - clos_configure_mask(msr_fd, region->clos_id, 0); 334 - if (region->topo->mba_supported && region->mba_throttle > 0) 335 - mba_configure(msr_fd, region->clos_id, 0); 336 - clos_configure_mask(msr_fd, 0, clos_get_default_mask()); 337 - msr_close(msr_fd); 364 + mpam_configure_partition(region->clos_id, 0, 0); 338 365 } 339 366 340 367 region_free(region->ptr, region->size);
+68
src/mpam.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <unistd.h> 4 + 5 + #ifdef __aarch64__ 6 + 7 + #define SYS_MPAM1_EL1 "S3_0_C10_C5_1" 8 + #define SYS_MPAMCFG_PART_SEL "S3_0_C10_C6_0" 9 + #define SYS_MPAMCFG_CPBM "S3_0_C10_C6_1" 10 + #define SYS_MPAMCFG_MBW_MAX "S3_0_C10_C6_2" 11 + 12 + static inline uint64_t read_sysreg(const char *reg_name) 13 + { 14 + (void)reg_name; 15 + return 0; 16 + } 17 + 18 + static inline void write_sysreg(const char *reg_name, uint64_t val) 19 + { 20 + (void)reg_name; 21 + (void)val; 22 + } 23 + 24 + int mpam_configure_partition(unsigned partid, uint32_t cpbm, unsigned mbw_max) 25 + { 26 + if (partid > MPAM_MAX_PARTID) 27 + return ICEPICK_E_INVALID; 28 + 29 + write_sysreg(SYS_MPAMCFG_PART_SEL, partid); 30 + 31 + if (cpbm != 0) 32 + write_sysreg(SYS_MPAMCFG_CPBM, cpbm); 33 + 34 + if (mbw_max > 0) 35 + write_sysreg(SYS_MPAMCFG_MBW_MAX, mbw_max); 36 + 37 + return 0; 38 + } 39 + 40 + int mpam_associate_thread(unsigned partid) 41 + { 42 + if (partid > MPAM_MAX_PARTID) 43 + return ICEPICK_E_INVALID; 44 + 45 + uint64_t mpam1 = read_sysreg(SYS_MPAM1_EL1); 46 + mpam1 = (mpam1 & ~0xFFFFULL) | partid; 47 + write_sysreg(SYS_MPAM1_EL1, mpam1); 48 + 49 + return 0; 50 + } 51 + 52 + #else 53 + 54 + int mpam_configure_partition(unsigned partid, uint32_t cpbm, unsigned mbw_max) 55 + { 56 + (void)partid; 57 + (void)cpbm; 58 + (void)mbw_max; 59 + return ICEPICK_E_INVALID; 60 + } 61 + 62 + int mpam_associate_thread(unsigned partid) 63 + { 64 + (void)partid; 65 + return ICEPICK_E_INVALID; 66 + } 67 + 68 + #endif
+116 -44
src/topology.c
··· 2 2 #include "internal.h" 3 3 #include <stdlib.h> 4 4 #include <string.h> 5 - #include <cpuid.h> 6 5 #include <sched.h> 7 6 #include <dirent.h> 8 7 #include <stdio.h> 9 8 #include <unistd.h> 10 9 10 + #ifdef __x86_64__ 11 + #include <cpuid.h> 12 + 11 13 static void cpuid(uint32_t leaf, uint32_t subleaf, 12 14 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) 13 15 { 14 16 __cpuid_count(leaf, subleaf, *eax, *ebx, *ecx, *edx); 15 17 } 18 + #endif 16 19 20 + arch_type_t detect_arch(void) 21 + { 22 + #ifdef __x86_64__ 23 + return ARCH_X86_64; 24 + #elif defined(__aarch64__) 25 + return ARCH_ARM64; 26 + #else 27 + return ARCH_UNKNOWN; 28 + #endif 29 + } 30 + 31 + #ifdef __x86_64__ 17 32 static bool detect_cat_support(icepick_topology_t *topo) 18 33 { 19 34 uint32_t eax, ebx, ecx, edx; ··· 111 126 return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163); 112 127 } 113 128 114 - static int count_ccxs(void) 115 - { 116 - DIR *dir = opendir("/sys/devices/system/node"); 117 - if (!dir) 118 - return 1; 119 - 120 - int count = 0; 121 - struct dirent *entry; 122 - while ((entry = readdir(dir))) { 123 - if (strncmp(entry->d_name, "node", 4) == 0) 124 - count++; 125 - } 126 - closedir(dir); 127 - 128 - return count > 0 ? count : 1; 129 - } 130 - 131 - static void detect_numa_topology(icepick_topology_t *topo) 132 - { 133 - topo->ccx_count = count_ccxs(); 134 - if (topo->ccx_count > MAX_CCXS) 135 - topo->ccx_count = MAX_CCXS; 136 - 137 - for (unsigned i = 0; i < topo->ccx_count; i++) 138 - topo->ccx_numa_node[i] = i; 139 - } 140 - 141 129 static bool detect_hybrid_architecture(icepick_topology_t *topo) 142 130 { 143 131 (void)topo; ··· 188 176 189 177 sched_setaffinity(0, sizeof(original_affinity), &original_affinity); 190 178 } 179 + #endif 180 + 181 + #ifdef __aarch64__ 182 + static bool detect_mpam_support(icepick_topology_t *topo) 183 + { 184 + FILE *f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/id", "r"); 185 + if (!f) 186 + return false; 187 + fclose(f); 188 + 189 + topo->max_partid = 16; 190 + return true; 191 + } 192 + 193 + static void detect_arm64_cache_topology(icepick_topology_t *topo) 194 + { 195 + FILE *f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/size", "r"); 196 + if (f) { 197 + unsigned size_kb = 0; 198 + if (fscanf(f, "%uK", &size_kb) == 1) 199 + topo->l3_size = (size_t)size_kb * 1024; 200 + fclose(f); 201 + } 202 + 203 + f = fopen("/sys/devices/system/cpu/cpu0/cache/index3/ways_of_associativity", "r"); 204 + if (f) { 205 + fscanf(f, "%u", &topo->l3_ways); 206 + fclose(f); 207 + } 208 + 209 + if (topo->l3_size > 0 && topo->l3_ways > 0) 210 + topo->way_size = topo->l3_size / topo->l3_ways; 211 + 212 + if (topo->l3_ways > 0) 213 + topo->default_way_mask = (1U << topo->l3_ways) - 1; 214 + } 215 + #endif 216 + 217 + static int count_ccxs(void) 218 + { 219 + DIR *dir = opendir("/sys/devices/system/node"); 220 + if (!dir) 221 + return 1; 222 + 223 + int count = 0; 224 + struct dirent *entry; 225 + while ((entry = readdir(dir))) { 226 + if (strncmp(entry->d_name, "node", 4) == 0) 227 + count++; 228 + } 229 + closedir(dir); 230 + 231 + return count > 0 ? count : 1; 232 + } 233 + 234 + static void detect_numa_topology(icepick_topology_t *topo) 235 + { 236 + topo->ccx_count = count_ccxs(); 237 + if (topo->ccx_count > MAX_CCXS) 238 + topo->ccx_count = MAX_CCXS; 239 + 240 + for (unsigned i = 0; i < topo->ccx_count; i++) 241 + topo->ccx_numa_node[i] = i; 242 + } 191 243 192 244 int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type, 193 245 int numa_node) ··· 214 266 if (!t) 215 267 return ICEPICK_E_ALLOC; 216 268 217 - t->cat_supported = detect_cat_support(t); 218 - if (!t->cat_supported) { 219 - free(t); 220 - return ICEPICK_E_NO_CAT; 269 + t->arch = detect_arch(); 270 + 271 + #ifdef __x86_64__ 272 + if (t->arch == ARCH_X86_64) { 273 + t->cat_supported = detect_cat_support(t); 274 + if (!t->cat_supported) { 275 + free(t); 276 + return ICEPICK_E_NO_CAT; 277 + } 278 + 279 + t->mba_supported = detect_mba_support(t); 280 + 281 + bool is_amd = is_amd_cpu(); 282 + if (is_amd) 283 + detect_amd_cache_topology(t); 284 + else 285 + detect_intel_cache_topology(t); 286 + 287 + if (!is_amd) { 288 + t->is_hybrid = detect_hybrid_architecture(t); 289 + if (t->is_hybrid) 290 + enumerate_hybrid_cores(t); 291 + } 221 292 } 293 + #endif 222 294 223 - t->mba_supported = detect_mba_support(t); 295 + #ifdef __aarch64__ 296 + if (t->arch == ARCH_ARM64) { 297 + t->mpam_supported = detect_mpam_support(t); 298 + if (!t->mpam_supported) { 299 + free(t); 300 + return ICEPICK_E_NO_CAT; 301 + } 224 302 225 - bool is_amd = is_amd_cpu(); 226 - if (is_amd) 227 - detect_amd_cache_topology(t); 228 - else 229 - detect_intel_cache_topology(t); 303 + t->cat_supported = t->mpam_supported; 304 + detect_arm64_cache_topology(t); 305 + t->max_clos = t->max_partid; 306 + } 307 + #endif 230 308 231 309 if (t->l3_size == 0 || t->l3_ways == 0) { 232 310 t->way_size = 2 * 1024 * 1024; ··· 234 312 } 235 313 236 314 detect_numa_topology(t); 237 - 238 - if (!is_amd) { 239 - t->is_hybrid = detect_hybrid_architecture(t); 240 - if (t->is_hybrid) 241 - enumerate_hybrid_cores(t); 242 - } 243 315 244 316 *topo = t; 245 317 return 0;