vm.fail / icepick
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
26
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

extend topology detection for intel hybrid architectures (p-core/e-core l3 partitioning)

author vm.fail date (Jan 18, 2026, 3:29 AM UTC) commit d60cc663 parent d93b499e
+146 -22
+20 -1
bindings/ocaml/icepick.ml
··· 15 15 | Pattern_strided 16 16 | Pattern_pointer_chase 17 17 18 + type core_type = 19 + | Core_any 20 + | Core_pcore 21 + | Core_ecore 22 + 18 23 let () = Callback.register_exception "Icepick_error" (Icepick_error "") 19 24 20 25 let () = ··· 43 48 let mba_supported = foreign "icepick_topology_mba_supported" (t @-> returning bool) 44 49 let mba_is_linear = foreign "icepick_topology_mba_is_linear" (t @-> returning bool) 45 50 let max_mba_thrtl = foreign "icepick_topology_max_mba_thrtl" (t @-> returning uint) 51 + let is_hybrid = foreign "icepick_topology_is_hybrid" (t @-> returning bool) 52 + let pcore_count = foreign "icepick_topology_pcore_count" (t @-> returning uint) 53 + let ecore_count = foreign "icepick_topology_ecore_count" (t @-> returning uint) 46 54 end 47 55 48 56 module Region = struct ··· 71 79 let stride_bytes = field t "stride_bytes" size_t 72 80 let prime_iterations = field t "prime_iterations" uint 73 81 let mba_throttle = field t "mba_throttle" uint 82 + let core_type = field t "core_type" int 74 83 let () = seal t 75 84 end 76 85 ··· 141 150 let mba_supported t = C.Topology.mba_supported t.ptr 142 151 let mba_is_linear t = C.Topology.mba_is_linear t.ptr 143 152 let max_mba_thrtl t = Unsigned.UInt.to_int (C.Topology.max_mba_thrtl t.ptr) 153 + let is_hybrid t = C.Topology.is_hybrid t.ptr 154 + let pcore_count t = Unsigned.UInt.to_int (C.Topology.pcore_count t.ptr) 155 + let ecore_count t = Unsigned.UInt.to_int (C.Topology.ecore_count t.ptr) 144 156 end 145 157 146 158 module Latency_stats = struct ··· 191 203 | Pattern_strided -> 2 192 204 | Pattern_pointer_chase -> 3 193 205 206 + let core_type_to_int = function 207 + | Core_any -> 0 208 + | Core_pcore -> 1 209 + | Core_ecore -> 2 210 + 194 211 let lock topo ~size ~clos ?(numa = -1) ?(huge_pages = true) ?(verify = false) 195 212 ?(prime_strategy = Prime_temporal) ?(access_pattern = Pattern_sequential) 196 - ?(stride = 0) ?(prime_iterations = 3) ?(mba_throttle = 0) () = 213 + ?(stride = 0) ?(prime_iterations = 3) ?(mba_throttle = 0) 214 + ?(core_type = Core_any) () = 197 215 let cfg = make C.Config.t in 198 216 setf cfg C.Config.size (Unsigned.Size_t.of_int size); 199 217 setf cfg C.Config.clos_id (Unsigned.UInt.of_int clos); ··· 209 227 setf cfg C.Config.stride_bytes (Unsigned.Size_t.of_int stride); 210 228 setf cfg C.Config.prime_iterations (Unsigned.UInt.of_int prime_iterations); 211 229 setf cfg C.Config.mba_throttle (Unsigned.UInt.of_int mba_throttle); 230 + setf cfg C.Config.core_type (core_type_to_int core_type); 212 231 let ptr_ref = allocate C.Region.t (from_voidp void null) in 213 232 let ret = C.lock topo.Topology.ptr (addr cfg) ptr_ref in 214 233 C.check_error ret;
+10 -1
bindings/ocaml/icepick.mli
··· 12 12 | Pattern_strided 13 13 | Pattern_pointer_chase 14 14 15 + type core_type = 16 + | Core_any 17 + | Core_pcore 18 + | Core_ecore 19 + 15 20 module Topology : sig 16 21 type t 17 22 ··· 25 30 val mba_supported : t -> bool 26 31 val mba_is_linear : t -> bool 27 32 val max_mba_thrtl : t -> int 33 + val is_hybrid : t -> bool 34 + val pcore_count : t -> int 35 + val ecore_count : t -> int 28 36 end 29 37 30 38 module Region : sig ··· 32 40 33 41 val lock : Topology.t -> size:int -> clos:int -> ?numa:int -> ?huge_pages:bool -> ?verify:bool -> 34 42 ?prime_strategy:prime_strategy -> ?access_pattern:access_pattern -> 35 - ?stride:int -> ?prime_iterations:int -> ?mba_throttle:int -> unit -> t 43 + ?stride:int -> ?prime_iterations:int -> ?mba_throttle:int -> 44 + ?core_type:core_type -> unit -> t 36 45 val unlock : t -> unit 37 46 val ptr : t -> nativeint 38 47 val size : t -> int
+10
include/icepick.h
··· 31 31 ICEPICK_PATTERN_POINTER_CHASE, 32 32 } icepick_access_pattern_t; 33 33 34 + typedef enum { 35 + ICEPICK_CORE_ANY, 36 + ICEPICK_CORE_PCORE, 37 + ICEPICK_CORE_ECORE, 38 + } icepick_core_type_t; 39 + 34 40 typedef struct icepick_topology icepick_topology_t; 35 41 typedef struct icepick_region icepick_region_t; 36 42 typedef struct icepick_monitor icepick_monitor_t; ··· 50 56 size_t stride_bytes; 51 57 unsigned prime_iterations; 52 58 unsigned mba_throttle; 59 + icepick_core_type_t core_type; 53 60 } icepick_config_t; 54 61 55 62 typedef struct { ··· 74 81 bool icepick_topology_mba_supported(const icepick_topology_t *topo); 75 82 bool icepick_topology_mba_is_linear(const icepick_topology_t *topo); 76 83 unsigned icepick_topology_max_mba_thrtl(const icepick_topology_t *topo); 84 + bool icepick_topology_is_hybrid(const icepick_topology_t *topo); 85 + unsigned icepick_topology_pcore_count(const icepick_topology_t *topo); 86 + unsigned icepick_topology_ecore_count(const icepick_topology_t *topo); 77 87 78 88 int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg, icepick_region_t **region); 79 89 int icepick_unlock(icepick_region_t *region);
+9
src/internal.h
··· 14 14 #define CACHE_LINE_SIZE 64 15 15 #define MAX_CLOS 16 16 16 #define MAX_CCXS 16 17 + #define MAX_CORE_TYPES 2 17 18 18 19 #define MONITOR_DEFAULT_PMU_POLL_NS 100000 19 20 #define MONITOR_DEFAULT_PROBE_NS 10000 ··· 27 28 bool cat_supported; 28 29 bool mba_supported; 29 30 bool mba_is_linear; 31 + bool is_hybrid; 30 32 unsigned l3_ways; 31 33 size_t l3_size; 32 34 size_t way_size; ··· 35 37 unsigned ccx_count; 36 38 uint32_t default_way_mask; 37 39 int ccx_numa_node[MAX_CCXS]; 40 + unsigned pcore_count; 41 + unsigned ecore_count; 42 + int pcore_cpus[64]; 43 + int ecore_cpus[256]; 38 44 }; 39 45 40 46 struct icepick_region { ··· 44 50 int numa_node; 45 51 uint32_t way_mask; 46 52 unsigned mba_throttle; 53 + icepick_core_type_t core_type; 47 54 icepick_topology_t *topo; 48 55 struct icepick_monitor *monitor; 49 56 icepick_prime_strategy_t prime_strategy; ··· 101 108 int monitor_perf_open(int cpu, bool is_amd); 102 109 103 110 int mba_configure(int msr_fd, unsigned clos_id, unsigned throttle); 111 + int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type, 112 + int numa_node); 104 113 105 114 #endif
+2 -3
src/lock.c
··· 223 223 r->numa_node = cfg->numa_node; 224 224 r->way_mask = way_mask; 225 225 r->mba_throttle = cfg->mba_throttle; 226 + r->core_type = cfg->core_type; 226 227 r->topo = topo; 227 228 r->size = ways_needed * topo->way_size; 228 229 r->monitor = NULL; ··· 238 239 return ret; 239 240 } 240 241 241 - int cpu = sched_getcpu(); 242 - if (cpu < 0) 243 - cpu = 0; 242 + int cpu = find_cpu_for_core_type(topo, cfg->core_type, cfg->numa_node); 244 243 245 244 cpu_set_t old_affinity, new_affinity; 246 245 CPU_ZERO(&new_affinity);
+1 -16
src/monitor.c
··· 146 146 147 147 static int find_ccx_cpu(icepick_region_t *region) 148 148 { 149 - int numa = region->numa_node; 150 - if (numa < 0) 151 - numa = 0; 152 - 153 - char path[128]; 154 - snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa); 155 - 156 - FILE *f = fopen(path, "r"); 157 - if (!f) 158 - return 0; 159 - 160 - int cpu = 0; 161 - fscanf(f, "%d", &cpu); 162 - fclose(f); 163 - 164 - return cpu; 149 + return find_cpu_for_core_type(region->topo, region->core_type, region->numa_node); 165 150 } 166 151 167 152 int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
+94 -1
src/topology.c
··· 6 6 #include <sched.h> 7 7 #include <dirent.h> 8 8 #include <stdio.h> 9 + #include <unistd.h> 9 10 10 11 static void cpuid(uint32_t leaf, uint32_t subleaf, 11 12 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) ··· 137 138 topo->ccx_numa_node[i] = i; 138 139 } 139 140 141 + static bool detect_hybrid_architecture(icepick_topology_t *topo) 142 + { 143 + (void)topo; 144 + uint32_t eax, ebx, ecx, edx; 145 + 146 + cpuid(0x00, 0, &eax, &ebx, &ecx, &edx); 147 + if (eax < 0x1A) 148 + return false; 149 + 150 + cpuid(0x07, 0, &eax, &ebx, &ecx, &edx); 151 + if (!(edx & (1 << 15))) 152 + return false; 153 + 154 + return true; 155 + } 156 + 157 + static void enumerate_hybrid_cores(icepick_topology_t *topo) 158 + { 159 + int num_cpus = sysconf(_SC_NPROCESSORS_ONLN); 160 + if (num_cpus <= 0) 161 + return; 162 + 163 + cpu_set_t original_affinity; 164 + sched_getaffinity(0, sizeof(original_affinity), &original_affinity); 165 + 166 + topo->pcore_count = 0; 167 + topo->ecore_count = 0; 168 + 169 + for (int cpu = 0; cpu < num_cpus && cpu < 320; cpu++) { 170 + cpu_set_t cpuset; 171 + CPU_ZERO(&cpuset); 172 + CPU_SET(cpu, &cpuset); 173 + 174 + if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) 175 + continue; 176 + 177 + uint32_t eax, ebx, ecx, edx; 178 + cpuid(0x1A, 0, &eax, &ebx, &ecx, &edx); 179 + 180 + uint32_t core_type = (eax >> 24) & 0xFF; 181 + 182 + if (core_type == 0x40 && topo->pcore_count < 64) { 183 + topo->pcore_cpus[topo->pcore_count++] = cpu; 184 + } else if (core_type == 0x20 && topo->ecore_count < 256) { 185 + topo->ecore_cpus[topo->ecore_count++] = cpu; 186 + } 187 + } 188 + 189 + sched_setaffinity(0, sizeof(original_affinity), &original_affinity); 190 + } 191 + 192 + int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type, 193 + int numa_node) 194 + { 195 + (void)numa_node; 196 + 197 + if (!topo->is_hybrid || core_type == ICEPICK_CORE_ANY) { 198 + int cpu = sched_getcpu(); 199 + return cpu >= 0 ? cpu : 0; 200 + } 201 + 202 + if (core_type == ICEPICK_CORE_PCORE && topo->pcore_count > 0) 203 + return topo->pcore_cpus[0]; 204 + 205 + if (core_type == ICEPICK_CORE_ECORE && topo->ecore_count > 0) 206 + return topo->ecore_cpus[0]; 207 + 208 + return 0; 209 + } 210 + 140 211 int icepick_discover_topology(icepick_topology_t **topo) 141 212 { 142 213 icepick_topology_t *t = calloc(1, sizeof(*t)); ··· 151 222 152 223 t->mba_supported = detect_mba_support(t); 153 224 154 - if (is_amd_cpu()) 225 + bool is_amd = is_amd_cpu(); 226 + if (is_amd) 155 227 detect_amd_cache_topology(t); 156 228 else 157 229 detect_intel_cache_topology(t); ··· 162 234 } 163 235 164 236 detect_numa_topology(t); 237 + 238 + if (!is_amd) { 239 + t->is_hybrid = detect_hybrid_architecture(t); 240 + if (t->is_hybrid) 241 + enumerate_hybrid_cores(t); 242 + } 165 243 166 244 *topo = t; 167 245 return 0; ··· 216 294 { 217 295 return topo->max_mba_thrtl; 218 296 } 297 + 298 + bool icepick_topology_is_hybrid(const icepick_topology_t *topo) 299 + { 300 + return topo->is_hybrid; 301 + } 302 + 303 + unsigned icepick_topology_pcore_count(const icepick_topology_t *topo) 304 + { 305 + return topo->pcore_count; 306 + } 307 + 308 + unsigned icepick_topology_ecore_count(const icepick_topology_t *topo) 309 + { 310 + return topo->ecore_count; 311 + }