vm.fail / icepick
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
26
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

initial commit

author vm.fail date (Jan 17, 2026, 11:34 PM UTC) commit 6fe2641d
+1739
+64
Makefile
··· 1 + CC = gcc 2 + CFLAGS = -O2 -Wall -Wextra -Werror -fPIC -march=native -I include 3 + LDFLAGS = -lnuma 4 + 5 + SRCDIR = src 6 + INCDIR = include 7 + OBJDIR = obj 8 + 9 + SRCS = $(wildcard $(SRCDIR)/*.c) 10 + OBJS = $(patsubst $(SRCDIR)/%.c,$(OBJDIR)/%.o,$(SRCS)) 11 + LIB_OBJS = $(filter-out $(OBJDIR)/main.o,$(OBJS)) 12 + 13 + TARGET = icepick 14 + LIBRARY = libicepick.a 15 + SHARED = libicepick.so 16 + 17 + PREFIX ?= /usr/local 18 + BINDIR = $(PREFIX)/bin 19 + LIBDIR = $(PREFIX)/lib 20 + INCLUDEDIR = $(PREFIX)/include 21 + 22 + .PHONY: all clean install uninstall 23 + 24 + all: $(TARGET) $(LIBRARY) $(SHARED) 25 + 26 + $(OBJDIR): 27 + mkdir -p $(OBJDIR) 28 + 29 + $(OBJDIR)/%.o: $(SRCDIR)/%.c | $(OBJDIR) 30 + $(CC) $(CFLAGS) -c $< -o $@ 31 + 32 + $(LIBRARY): $(LIB_OBJS) 33 + ar rcs $@ $^ 34 + 35 + $(SHARED): $(LIB_OBJS) 36 + $(CC) -shared -o $@ $^ $(LDFLAGS) 37 + 38 + $(TARGET): $(OBJDIR)/main.o $(LIBRARY) 39 + $(CC) $(CFLAGS) $< -L. -licepick $(LDFLAGS) -o $@ 40 + 41 + install: $(TARGET) $(LIBRARY) $(SHARED) 42 + install -d $(BINDIR) $(LIBDIR) $(INCLUDEDIR) 43 + install -m 755 $(TARGET) $(BINDIR)/ 44 + install -m 644 $(LIBRARY) $(LIBDIR)/ 45 + install -m 755 $(SHARED) $(LIBDIR)/ 46 + install -m 644 $(INCDIR)/icepick.h $(INCLUDEDIR)/ 47 + 48 + uninstall: 49 + rm -f $(BINDIR)/$(TARGET) 50 + rm -f $(LIBDIR)/$(LIBRARY) 51 + rm -f $(LIBDIR)/$(SHARED) 52 + rm -f $(INCLUDEDIR)/icepick.h 53 + 54 + clean: 55 + rm -rf $(OBJDIR) $(TARGET) $(LIBRARY) $(SHARED) 56 + 57 + $(OBJDIR)/msr.o: $(SRCDIR)/msr.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 58 + $(OBJDIR)/topology.o: $(SRCDIR)/topology.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 59 + $(OBJDIR)/clos.o: $(SRCDIR)/clos.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 60 + $(OBJDIR)/region.o: $(SRCDIR)/region.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 61 + $(OBJDIR)/lock.o: $(SRCDIR)/lock.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 62 + $(OBJDIR)/verify.o: $(SRCDIR)/verify.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 63 + $(OBJDIR)/bench.o: $(SRCDIR)/bench.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h 64 + $(OBJDIR)/main.o: $(SRCDIR)/main.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
+43
README.md
··· 1 + # icepick 2 + 3 + icepick use cache ways from the llc using cat/rdt msr configuration. target memory regions are primed into isolated clos partitions to prevent eviction by competing workloads. i als found it personally useful for pinning cryptographic state against side-channel probing + eliminating tail latency in hot paths 4 + 5 + ## build 6 + 7 + ``` 8 + make 9 + ``` 10 + 11 + ``` 12 + dune build 13 + ``` 14 + 15 + ## usage 16 + 17 + ``` 18 + icepick topology # enumerate l3 geometry, way count, clos availability 19 + icepick lock -s 4M -c 1 -v # allocate 4mb into clos 1, verify residency 20 + icepick bench -s 2M # compare locked vs unlocked access latency 21 + icepick status # dump current clos mask assignments 22 + ``` 23 + 24 + c api: 25 + 26 + ```c 27 + icepick_topology_t *topo; 28 + icepick_discover_topology(&topo); 29 + 30 + icepick_config_t cfg = { .size = 4*1024*1024, .clos_id = 1, .huge_pages = true }; 31 + icepick_region_t *region; 32 + icepick_lock(topo, &cfg, &region); 33 + 34 + void *ptr = icepick_region_ptr(region); 35 + ``` 36 + 37 + ocaml: 38 + 39 + ```ocaml 40 + let topo = Icepick.Topology.discover () in 41 + let region = Icepick.Region.lock topo ~size:0x400000 ~clos:1 () in 42 + let ptr = Icepick.Region.ptr region in 43 + ```
+4
bindings/ocaml/dune
··· 1 + (library 2 + (name icepick) 3 + (libraries ctypes ctypes.foreign bigarray) 4 + (c_library_flags -L/home/oni/Projects/icepick -licepick -lnuma))
+2
bindings/ocaml/dune-project
··· 1 + (lang dune 3.0) 2 + (name icepick)
+187
bindings/ocaml/icepick.ml
··· 1 + open Ctypes 2 + open Foreign 3 + 4 + exception Icepick_error of string 5 + 6 + let () = Callback.register_exception "Icepick_error" (Icepick_error "") 7 + 8 + let () = 9 + let _ = Dl.dlopen ~filename:"/home/oni/Projects/icepick/libicepick.so" 10 + ~flags:[Dl.RTLD_NOW; Dl.RTLD_GLOBAL] in 11 + () 12 + 13 + module C = struct 14 + let icepick_strerror = foreign "icepick_strerror" (int @-> returning string) 15 + 16 + let check_error ret = 17 + if ret < 0 then raise (Icepick_error (icepick_strerror ret)) 18 + 19 + module Topology = struct 20 + type t = unit ptr 21 + let t : t typ = ptr void 22 + 23 + let discover = foreign "icepick_discover_topology" (ptr t @-> returning int) 24 + let free = foreign "icepick_free_topology" (t @-> returning void) 25 + let l3_ways = foreign "icepick_topology_l3_ways" (t @-> returning uint) 26 + let l3_size = foreign "icepick_topology_l3_size" (t @-> returning size_t) 27 + let way_size = foreign "icepick_topology_way_size" (t @-> returning size_t) 28 + let max_clos = foreign "icepick_topology_max_clos" (t @-> returning uint) 29 + let ccx_count = foreign "icepick_topology_ccx_count" (t @-> returning uint) 30 + let cat_supported = foreign "icepick_topology_cat_supported" (t @-> returning bool) 31 + end 32 + 33 + module Region = struct 34 + type t = unit ptr 35 + let t : t typ = ptr void 36 + 37 + let ptr = foreign "icepick_region_ptr" (t @-> returning (ptr void)) 38 + let size = foreign "icepick_region_size" (t @-> returning size_t) 39 + let clos = foreign "icepick_region_clos" (t @-> returning uint) 40 + end 41 + 42 + module Config = struct 43 + type t 44 + let t : t structure typ = structure "icepick_config_t" 45 + let size = field t "size" size_t 46 + let clos_id = field t "clos_id" uint 47 + let numa_node = field t "numa_node" int 48 + let huge_pages = field t "huge_pages" bool 49 + let verify = field t "verify" bool 50 + let () = seal t 51 + end 52 + 53 + module Latency_stats = struct 54 + type t 55 + let t : t structure typ = structure "icepick_latency_stats_t" 56 + let mean_ns = field t "mean_ns" uint64_t 57 + let stddev_ns = field t "stddev_ns" uint64_t 58 + let p50_ns = field t "p50_ns" uint64_t 59 + let p99_ns = field t "p99_ns" uint64_t 60 + let p999_ns = field t "p999_ns" uint64_t 61 + let min_ns = field t "min_ns" uint64_t 62 + let max_ns = field t "max_ns" uint64_t 63 + let () = seal t 64 + end 65 + 66 + let lock = foreign "icepick_lock" 67 + (Topology.t @-> ptr Config.t @-> ptr Region.t @-> returning int) 68 + 69 + let unlock = foreign "icepick_unlock" (Region.t @-> returning int) 70 + 71 + let verify = foreign "icepick_verify" 72 + (Region.t @-> ptr Latency_stats.t @-> returning int) 73 + 74 + let bench = foreign "icepick_bench" 75 + (ptr void @-> size_t @-> size_t @-> ptr Latency_stats.t @-> returning int) 76 + end 77 + 78 + module Topology = struct 79 + type t = { 80 + ptr : C.Topology.t; 81 + mutable released : bool; 82 + } 83 + 84 + let release t = 85 + if not t.released then begin 86 + C.Topology.free t.ptr; 87 + t.released <- true 88 + end 89 + 90 + let discover () = 91 + let ptr_ref = allocate C.Topology.t (from_voidp void null) in 92 + let ret = C.Topology.discover ptr_ref in 93 + C.check_error ret; 94 + let t = { ptr = !@ ptr_ref; released = false } in 95 + Gc.finalise release t; 96 + t 97 + 98 + let l3_ways t = Unsigned.UInt.to_int (C.Topology.l3_ways t.ptr) 99 + let l3_size t = Unsigned.Size_t.to_int (C.Topology.l3_size t.ptr) 100 + let way_size t = Unsigned.Size_t.to_int (C.Topology.way_size t.ptr) 101 + let max_clos t = Unsigned.UInt.to_int (C.Topology.max_clos t.ptr) 102 + let ccx_count t = Unsigned.UInt.to_int (C.Topology.ccx_count t.ptr) 103 + let cat_supported t = C.Topology.cat_supported t.ptr 104 + end 105 + 106 + module Latency_stats = struct 107 + type t = { 108 + mean_ns : int; 109 + stddev_ns : int; 110 + p50_ns : int; 111 + p99_ns : int; 112 + p999_ns : int; 113 + min_ns : int; 114 + max_ns : int; 115 + } 116 + 117 + let of_c stats = 118 + { 119 + mean_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.mean_ns); 120 + stddev_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.stddev_ns); 121 + p50_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.p50_ns); 122 + p99_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.p99_ns); 123 + p999_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.p999_ns); 124 + min_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.min_ns); 125 + max_ns = Unsigned.UInt64.to_int (getf stats C.Latency_stats.max_ns); 126 + } 127 + end 128 + 129 + module Region = struct 130 + type t = { 131 + ptr : C.Region.t; 132 + topo : Topology.t; 133 + mutable released : bool; 134 + } 135 + 136 + let release t = 137 + if not t.released then begin 138 + let _ = C.unlock t.ptr in 139 + t.released <- true 140 + end 141 + 142 + let lock topo ~size ~clos ?(numa = -1) ?(huge_pages = true) ?(verify = false) () = 143 + let cfg = make C.Config.t in 144 + setf cfg C.Config.size (Unsigned.Size_t.of_int size); 145 + setf cfg C.Config.clos_id (Unsigned.UInt.of_int clos); 146 + setf cfg C.Config.numa_node numa; 147 + setf cfg C.Config.huge_pages huge_pages; 148 + setf cfg C.Config.verify verify; 149 + let ptr_ref = allocate C.Region.t (from_voidp void null) in 150 + let ret = C.lock topo.Topology.ptr (addr cfg) ptr_ref in 151 + C.check_error ret; 152 + let t = { ptr = !@ ptr_ref; topo; released = false } in 153 + Gc.finalise release t; 154 + t 155 + 156 + let unlock t = 157 + if not t.released then begin 158 + let ret = C.unlock t.ptr in 159 + t.released <- true; 160 + C.check_error ret 161 + end 162 + 163 + let ptr t = raw_address_of_ptr (C.Region.ptr t.ptr) 164 + let size t = Unsigned.Size_t.to_int (C.Region.size t.ptr) 165 + let clos t = Unsigned.UInt.to_int (C.Region.clos t.ptr) 166 + 167 + let to_bigarray_float64 t = 168 + let region_ptr = C.Region.ptr t.ptr in 169 + let region_size = Unsigned.Size_t.to_int (C.Region.size t.ptr) in 170 + let len = region_size / 8 in 171 + let typed_ptr = from_voidp double (to_voidp region_ptr) in 172 + CArray.from_ptr typed_ptr len |> CArray.to_list |> Array.of_list |> 173 + Bigarray.Array1.of_array Bigarray.Float64 Bigarray.c_layout 174 + end 175 + 176 + let verify region = 177 + let stats = make C.Latency_stats.t in 178 + let ret = C.verify region.Region.ptr (addr stats) in 179 + C.check_error ret; 180 + Latency_stats.of_c stats 181 + 182 + let bench ptr ~size ~iterations = 183 + let stats = make C.Latency_stats.t in 184 + let p = ptr_of_raw_address ptr in 185 + let ret = C.bench p (Unsigned.Size_t.of_int size) (Unsigned.Size_t.of_int iterations) (addr stats) in 186 + C.check_error ret; 187 + Latency_stats.of_c stats
+39
bindings/ocaml/icepick.mli
··· 1 + exception Icepick_error of string 2 + 3 + module Topology : sig 4 + type t 5 + 6 + val discover : unit -> t 7 + val l3_ways : t -> int 8 + val l3_size : t -> int 9 + val way_size : t -> int 10 + val max_clos : t -> int 11 + val ccx_count : t -> int 12 + val cat_supported : t -> bool 13 + end 14 + 15 + module Region : sig 16 + type t 17 + 18 + val lock : Topology.t -> size:int -> clos:int -> ?numa:int -> ?huge_pages:bool -> ?verify:bool -> unit -> t 19 + val unlock : t -> unit 20 + val ptr : t -> nativeint 21 + val size : t -> int 22 + val clos : t -> int 23 + val to_bigarray_float64 : t -> (float, Bigarray.float64_elt, Bigarray.c_layout) Bigarray.Array1.t 24 + end 25 + 26 + module Latency_stats : sig 27 + type t = { 28 + mean_ns : int; 29 + stddev_ns : int; 30 + p50_ns : int; 31 + p99_ns : int; 32 + p999_ns : int; 33 + min_ns : int; 34 + max_ns : int; 35 + } 36 + end 37 + 38 + val verify : Region.t -> Latency_stats.t 39 + val bench : nativeint -> size:int -> iterations:int -> Latency_stats.t
+4
bindings/ocaml/test/dune
··· 1 + (executable 2 + (name test) 3 + (libraries icepick) 4 + (link_flags (-cclib -L/home/oni/Projects/icepick) (-cclib -licepick) (-cclib -lnuma)))
+34
bindings/ocaml/test/test.ml
··· 1 + let () = 2 + print_endline "testing icepick ocaml bindings..."; 3 + print_endline ""; 4 + 5 + try 6 + let topo = Icepick.Topology.discover () in 7 + Printf.printf "l3 cache:\n"; 8 + Printf.printf " ways: %d\n" (Icepick.Topology.l3_ways topo); 9 + Printf.printf " size: %d mb\n" (Icepick.Topology.l3_size topo / 1024 / 1024); 10 + Printf.printf " way size: %d kb\n" (Icepick.Topology.way_size topo / 1024); 11 + Printf.printf " max clos: %d\n" (Icepick.Topology.max_clos topo); 12 + Printf.printf " ccx count: %d\n" (Icepick.Topology.ccx_count topo); 13 + Printf.printf " cat: %b\n" (Icepick.Topology.cat_supported topo); 14 + print_endline ""; 15 + 16 + print_endline "attempting to lock 2mb region..."; 17 + let region = Icepick.Region.lock topo ~size:(2 * 1024 * 1024) ~clos:1 () in 18 + Printf.printf " locked %d bytes at 0x%nx\n" 19 + (Icepick.Region.size region) 20 + (Icepick.Region.ptr region); 21 + Printf.printf " clos: %d\n" (Icepick.Region.clos region); 22 + 23 + print_endline "verifying..."; 24 + let stats = Icepick.verify region in 25 + Printf.printf " mean latency: %d ns\n" stats.mean_ns; 26 + Printf.printf " p99 latency: %d ns\n" stats.p99_ns; 27 + 28 + print_endline "unlocking..."; 29 + Icepick.Region.unlock region; 30 + print_endline "done." 31 + 32 + with Icepick.Icepick_error msg -> 33 + Printf.printf "icepick error: %s\n" msg; 34 + print_endline "(this is expected if cat is not supported on this cpu)"
+64
include/icepick.h
··· 1 + #ifndef ICEPICK_H 2 + #define ICEPICK_H 3 + 4 + #include <stddef.h> 5 + #include <stdint.h> 6 + #include <stdbool.h> 7 + 8 + #define ICEPICK_E_NO_CAT (-1) 9 + #define ICEPICK_E_PERMISSION (-2) 10 + #define ICEPICK_E_NO_CLOS (-3) 11 + #define ICEPICK_E_TOO_LARGE (-4) 12 + #define ICEPICK_E_NUMA (-5) 13 + #define ICEPICK_E_HUGEPAGE (-6) 14 + #define ICEPICK_E_VERIFY (-7) 15 + #define ICEPICK_E_INVALID (-8) 16 + #define ICEPICK_E_ALLOC (-9) 17 + #define ICEPICK_E_MSR (-10) 18 + 19 + typedef struct icepick_topology icepick_topology_t; 20 + typedef struct icepick_region icepick_region_t; 21 + 22 + typedef struct { 23 + size_t size; 24 + unsigned clos_id; 25 + int numa_node; 26 + bool huge_pages; 27 + bool verify; 28 + } icepick_config_t; 29 + 30 + typedef struct { 31 + uint64_t mean_ns; 32 + uint64_t stddev_ns; 33 + uint64_t p50_ns; 34 + uint64_t p99_ns; 35 + uint64_t p999_ns; 36 + uint64_t min_ns; 37 + uint64_t max_ns; 38 + } icepick_latency_stats_t; 39 + 40 + int icepick_discover_topology(icepick_topology_t **topo); 41 + void icepick_free_topology(icepick_topology_t *topo); 42 + 43 + unsigned icepick_topology_l3_ways(const icepick_topology_t *topo); 44 + size_t icepick_topology_l3_size(const icepick_topology_t *topo); 45 + size_t icepick_topology_way_size(const icepick_topology_t *topo); 46 + unsigned icepick_topology_max_clos(const icepick_topology_t *topo); 47 + unsigned icepick_topology_ccx_count(const icepick_topology_t *topo); 48 + bool icepick_topology_cat_supported(const icepick_topology_t *topo); 49 + 50 + int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg, icepick_region_t **region); 51 + int icepick_unlock(icepick_region_t *region); 52 + 53 + void *icepick_region_ptr(const icepick_region_t *region); 54 + size_t icepick_region_size(const icepick_region_t *region); 55 + unsigned icepick_region_clos(const icepick_region_t *region); 56 + 57 + int icepick_verify(const icepick_region_t *region, icepick_latency_stats_t *stats); 58 + 59 + int icepick_bench(void *ptr, size_t size, size_t iterations, 60 + icepick_latency_stats_t *stats); 61 + 62 + const char *icepick_strerror(int err); 63 + 64 + #endif
+85
src/bench.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <stdio.h> 4 + #include <stdlib.h> 5 + #include <string.h> 6 + #include <sched.h> 7 + 8 + int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations) 9 + { 10 + printf("running latency benchmark...\n\n"); 11 + printf("configuration:\n"); 12 + printf(" region size: %zu kb\n", size / 1024); 13 + printf(" iterations: %zu\n", iterations); 14 + printf(" access pattern: sequential, 64-byte stride\n\n"); 15 + 16 + void *unlocked_ptr; 17 + int ret = region_alloc(size, -1, true, &unlocked_ptr); 18 + if (ret < 0) { 19 + ret = region_alloc(size, -1, false, &unlocked_ptr); 20 + if (ret < 0) { 21 + fprintf(stderr, "failed to allocate unlocked region: %s\n", 22 + icepick_strerror(ret)); 23 + return ret; 24 + } 25 + } 26 + 27 + icepick_latency_stats_t unlocked_stats; 28 + ret = icepick_bench(unlocked_ptr, size, iterations, &unlocked_stats); 29 + region_free(unlocked_ptr, size); 30 + 31 + if (ret < 0) { 32 + fprintf(stderr, "unlocked benchmark failed: %s\n", icepick_strerror(ret)); 33 + return ret; 34 + } 35 + 36 + printf("results (unlocked, clos 0):\n"); 37 + printf(" mean latency: %lu ns\n", unlocked_stats.mean_ns); 38 + printf(" std deviation: %lu ns\n", unlocked_stats.stddev_ns); 39 + printf(" p50: %lu ns\n", unlocked_stats.p50_ns); 40 + printf(" p99: %lu ns\n", unlocked_stats.p99_ns); 41 + printf(" p99.9: %lu ns\n\n", unlocked_stats.p999_ns); 42 + 43 + icepick_config_t cfg = { 44 + .size = size, 45 + .clos_id = 1, 46 + .numa_node = -1, 47 + .huge_pages = true, 48 + .verify = false 49 + }; 50 + 51 + icepick_region_t *locked_region; 52 + ret = icepick_lock(topo, &cfg, &locked_region); 53 + if (ret < 0) { 54 + fprintf(stderr, "failed to lock region: %s\n", icepick_strerror(ret)); 55 + fprintf(stderr, "(locked benchmark skipped)\n"); 56 + return 0; 57 + } 58 + 59 + icepick_latency_stats_t locked_stats; 60 + ret = icepick_bench(icepick_region_ptr(locked_region), 61 + icepick_region_size(locked_region), 62 + iterations, &locked_stats); 63 + 64 + printf("results (locked, clos 1):\n"); 65 + printf(" mean latency: %lu ns\n", locked_stats.mean_ns); 66 + printf(" std deviation: %lu ns\n", locked_stats.stddev_ns); 67 + printf(" p50: %lu ns\n", locked_stats.p50_ns); 68 + printf(" p99: %lu ns\n", locked_stats.p99_ns); 69 + printf(" p99.9: %lu ns\n\n", locked_stats.p999_ns); 70 + 71 + icepick_unlock(locked_region); 72 + 73 + if (ret == 0 && unlocked_stats.mean_ns > 0 && locked_stats.mean_ns > 0) { 74 + printf("improvement:\n"); 75 + printf(" mean: %.1fx faster\n", 76 + (double)unlocked_stats.mean_ns / locked_stats.mean_ns); 77 + printf(" p99: %.1fx faster\n", 78 + (double)unlocked_stats.p99_ns / locked_stats.p99_ns); 79 + if (unlocked_stats.stddev_ns > 0 && locked_stats.stddev_ns > 0) 80 + printf(" jitter: %.1fx reduction\n", 81 + (double)unlocked_stats.stddev_ns / locked_stats.stddev_ns); 82 + } 83 + 84 + return 0; 85 + }
+85
src/clos.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <sched.h> 4 + #include <string.h> 5 + #include <unistd.h> 6 + 7 + clos_entry_t clos_table[MAX_CLOS]; 8 + static uint32_t default_mask; 9 + static bool initialized; 10 + 11 + int clos_init(icepick_topology_t *topo) 12 + { 13 + if (initialized) 14 + return 0; 15 + 16 + memset(clos_table, 0, sizeof(clos_table)); 17 + 18 + default_mask = topo->default_way_mask; 19 + 20 + clos_table[0].allocated = true; 21 + clos_table[0].way_mask = default_mask; 22 + 23 + initialized = true; 24 + return 0; 25 + } 26 + 27 + int clos_allocate(unsigned clos_id, uint32_t way_mask) 28 + { 29 + if (clos_id == 0 || clos_id >= MAX_CLOS) 30 + return ICEPICK_E_INVALID; 31 + 32 + if (clos_table[clos_id].allocated) 33 + return ICEPICK_E_NO_CLOS; 34 + 35 + clos_table[clos_id].allocated = true; 36 + clos_table[clos_id].way_mask = way_mask; 37 + 38 + clos_table[0].way_mask &= ~way_mask; 39 + 40 + return 0; 41 + } 42 + 43 + int clos_release(unsigned clos_id) 44 + { 45 + if (clos_id == 0 || clos_id >= MAX_CLOS) 46 + return ICEPICK_E_INVALID; 47 + 48 + if (!clos_table[clos_id].allocated) 49 + return 0; 50 + 51 + clos_table[0].way_mask |= clos_table[clos_id].way_mask; 52 + 53 + clos_table[clos_id].allocated = false; 54 + clos_table[clos_id].way_mask = 0; 55 + 56 + return 0; 57 + } 58 + 59 + int clos_configure_mask(int msr_fd, unsigned clos_id, uint32_t way_mask) 60 + { 61 + if (clos_id >= MAX_CLOS) 62 + return ICEPICK_E_INVALID; 63 + 64 + uint32_t msr = MSR_IA32_L3_QOS_MASK_0 + clos_id; 65 + return msr_write(msr_fd, msr, way_mask); 66 + } 67 + 68 + int clos_associate_thread(int msr_fd, unsigned clos_id) 69 + { 70 + if (clos_id >= MAX_CLOS) 71 + return ICEPICK_E_INVALID; 72 + 73 + uint64_t value; 74 + int ret = msr_read(msr_fd, MSR_IA32_PQR_ASSOC, &value); 75 + if (ret < 0) 76 + return ret; 77 + 78 + value = (value & 0xFFFFFFFF) | ((uint64_t)clos_id << 32); 79 + return msr_write(msr_fd, MSR_IA32_PQR_ASSOC, value); 80 + } 81 + 82 + uint32_t clos_get_default_mask(void) 83 + { 84 + return default_mask; 85 + }
+62
src/internal.h
··· 1 + #ifndef ICEPICK_INTERNAL_H 2 + #define ICEPICK_INTERNAL_H 3 + 4 + #include "../include/icepick.h" 5 + #include <stdint.h> 6 + 7 + #define MSR_IA32_PQR_ASSOC 0xC8F 8 + #define MSR_IA32_L3_QOS_MASK_0 0xC90 9 + 10 + #define CACHE_LINE_SIZE 64 11 + #define MAX_CLOS 16 12 + #define MAX_CCXS 16 13 + 14 + struct icepick_topology { 15 + bool cat_supported; 16 + unsigned l3_ways; 17 + size_t l3_size; 18 + size_t way_size; 19 + unsigned max_clos; 20 + unsigned ccx_count; 21 + uint32_t default_way_mask; 22 + int ccx_numa_node[MAX_CCXS]; 23 + }; 24 + 25 + struct icepick_region { 26 + void *ptr; 27 + size_t size; 28 + unsigned clos_id; 29 + int numa_node; 30 + uint32_t way_mask; 31 + icepick_topology_t *topo; 32 + }; 33 + 34 + typedef struct { 35 + bool allocated; 36 + uint32_t way_mask; 37 + } clos_entry_t; 38 + 39 + extern clos_entry_t clos_table[MAX_CLOS]; 40 + 41 + int msr_open(int cpu); 42 + void msr_close(int fd); 43 + int msr_read(int fd, uint32_t msr, uint64_t *value); 44 + int msr_write(int fd, uint32_t msr, uint64_t value); 45 + 46 + int clos_init(icepick_topology_t *topo); 47 + int clos_allocate(unsigned clos_id, uint32_t way_mask); 48 + int clos_release(unsigned clos_id); 49 + int clos_configure_mask(int msr_fd, unsigned clos_id, uint32_t way_mask); 50 + int clos_associate_thread(int msr_fd, unsigned clos_id); 51 + uint32_t clos_get_default_mask(void); 52 + 53 + int region_alloc(size_t size, int numa_node, bool huge_pages, void **out_ptr); 54 + void region_free(void *ptr, size_t size); 55 + 56 + uint64_t rdtsc_start(void); 57 + uint64_t rdtsc_end(void); 58 + uint64_t cycles_to_ns(uint64_t cycles); 59 + 60 + int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations); 61 + 62 + #endif
+193
src/lock.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <stdlib.h> 4 + #include <sched.h> 5 + #include <unistd.h> 6 + 7 + static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way) 8 + { 9 + uint32_t mask = 0; 10 + for (unsigned i = 0; i < num_ways; i++) 11 + mask |= (1U << (start_way + i)); 12 + return mask; 13 + } 14 + 15 + static unsigned find_start_way(uint32_t default_mask, unsigned num_ways) 16 + { 17 + for (unsigned i = 0; i <= 32 - num_ways; i++) { 18 + uint32_t candidate = compute_way_mask(num_ways, i); 19 + if ((candidate & default_mask) == candidate) 20 + return i; 21 + } 22 + return 0; 23 + } 24 + 25 + static void prime_region(volatile char *ptr, size_t size, unsigned iterations) 26 + { 27 + for (unsigned iter = 0; iter < iterations; iter++) { 28 + for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) { 29 + (void)ptr[offset]; 30 + } 31 + } 32 + } 33 + 34 + int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg, 35 + icepick_region_t **region) 36 + { 37 + if (!topo || !cfg || !region) 38 + return ICEPICK_E_INVALID; 39 + 40 + if (cfg->clos_id == 0 || cfg->clos_id >= topo->max_clos) 41 + return ICEPICK_E_INVALID; 42 + 43 + unsigned ways_needed = (cfg->size + topo->way_size - 1) / topo->way_size; 44 + if (ways_needed == 0) 45 + ways_needed = 1; 46 + 47 + if (ways_needed >= topo->l3_ways) 48 + return ICEPICK_E_TOO_LARGE; 49 + 50 + int ret = clos_init(topo); 51 + if (ret < 0) 52 + return ret; 53 + 54 + unsigned start_way = find_start_way(topo->default_way_mask, ways_needed); 55 + uint32_t way_mask = compute_way_mask(ways_needed, start_way); 56 + 57 + ret = clos_allocate(cfg->clos_id, way_mask); 58 + if (ret < 0) 59 + return ret; 60 + 61 + icepick_region_t *r = calloc(1, sizeof(*r)); 62 + if (!r) { 63 + clos_release(cfg->clos_id); 64 + return ICEPICK_E_ALLOC; 65 + } 66 + 67 + r->clos_id = cfg->clos_id; 68 + r->numa_node = cfg->numa_node; 69 + r->way_mask = way_mask; 70 + r->topo = topo; 71 + r->size = ways_needed * topo->way_size; 72 + 73 + ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr); 74 + if (ret < 0) { 75 + clos_release(cfg->clos_id); 76 + free(r); 77 + return ret; 78 + } 79 + 80 + int cpu = sched_getcpu(); 81 + if (cpu < 0) 82 + cpu = 0; 83 + 84 + cpu_set_t old_affinity, new_affinity; 85 + CPU_ZERO(&new_affinity); 86 + CPU_SET(cpu, &new_affinity); 87 + sched_getaffinity(0, sizeof(old_affinity), &old_affinity); 88 + sched_setaffinity(0, sizeof(new_affinity), &new_affinity); 89 + 90 + int msr_fd = msr_open(cpu); 91 + if (msr_fd < 0) { 92 + sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 93 + region_free(r->ptr, r->size); 94 + clos_release(cfg->clos_id); 95 + free(r); 96 + return msr_fd; 97 + } 98 + 99 + ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask); 100 + if (ret < 0) 101 + goto cleanup; 102 + 103 + ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask); 104 + if (ret < 0) 105 + goto cleanup; 106 + 107 + ret = clos_associate_thread(msr_fd, cfg->clos_id); 108 + if (ret < 0) 109 + goto cleanup; 110 + 111 + prime_region(r->ptr, r->size, 3); 112 + 113 + clos_associate_thread(msr_fd, 0); 114 + 115 + msr_close(msr_fd); 116 + sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 117 + 118 + if (cfg->verify) { 119 + icepick_latency_stats_t stats; 120 + ret = icepick_verify(r, &stats); 121 + if (ret < 0) { 122 + icepick_unlock(r); 123 + return ret; 124 + } 125 + } 126 + 127 + *region = r; 128 + return 0; 129 + 130 + cleanup: 131 + msr_close(msr_fd); 132 + sched_setaffinity(0, sizeof(old_affinity), &old_affinity); 133 + region_free(r->ptr, r->size); 134 + clos_release(cfg->clos_id); 135 + free(r); 136 + return ret; 137 + } 138 + 139 + int icepick_unlock(icepick_region_t *region) 140 + { 141 + if (!region) 142 + return ICEPICK_E_INVALID; 143 + 144 + int cpu = sched_getcpu(); 145 + if (cpu < 0) 146 + cpu = 0; 147 + 148 + int msr_fd = msr_open(cpu); 149 + if (msr_fd >= 0) { 150 + clos_release(region->clos_id); 151 + clos_configure_mask(msr_fd, region->clos_id, 0); 152 + clos_configure_mask(msr_fd, 0, clos_get_default_mask()); 153 + msr_close(msr_fd); 154 + } 155 + 156 + region_free(region->ptr, region->size); 157 + free(region); 158 + 159 + return 0; 160 + } 161 + 162 + void *icepick_region_ptr(const icepick_region_t *region) 163 + { 164 + return region ? region->ptr : NULL; 165 + } 166 + 167 + size_t icepick_region_size(const icepick_region_t *region) 168 + { 169 + return region ? region->size : 0; 170 + } 171 + 172 + unsigned icepick_region_clos(const icepick_region_t *region) 173 + { 174 + return region ? region->clos_id : 0; 175 + } 176 + 177 + const char *icepick_strerror(int err) 178 + { 179 + switch (err) { 180 + case 0: return "success"; 181 + case ICEPICK_E_NO_CAT: return "cat not supported on this cpu"; 182 + case ICEPICK_E_PERMISSION: return "permission denied (need root or cap_sys_rawio)"; 183 + case ICEPICK_E_NO_CLOS: return "no clos available or already allocated"; 184 + case ICEPICK_E_TOO_LARGE: return "requested size exceeds available cache ways"; 185 + case ICEPICK_E_NUMA: return "invalid numa node or binding failed"; 186 + case ICEPICK_E_HUGEPAGE: return "huge page allocation failed"; 187 + case ICEPICK_E_VERIFY: return "verification failed - data may not be cache-resident"; 188 + case ICEPICK_E_INVALID: return "invalid argument"; 189 + case ICEPICK_E_ALLOC: return "memory allocation failed"; 190 + case ICEPICK_E_MSR: return "msr operation failed"; 191 + default: return "unknown error"; 192 + } 193 + }
+259
src/main.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <stdio.h> 4 + #include <stdlib.h> 5 + #include <string.h> 6 + #include <getopt.h> 7 + 8 + static void usage(const char *prog) 9 + { 10 + fprintf(stderr, 11 + "usage: %s <command> [options]\n\n" 12 + "commands:\n" 13 + " topology show cache topology and cat capabilities\n" 14 + " lock lock a memory region into l3 cache\n" 15 + " unlock release a locked region\n" 16 + " status show current clos configuration\n" 17 + " bench run latency benchmark\n\n" 18 + "use '%s <command> --help' for command-specific options.\n", 19 + prog, prog); 20 + } 21 + 22 + static size_t parse_size(const char *str) 23 + { 24 + char *end; 25 + size_t val = strtoul(str, &end, 10); 26 + if (*end == 'K' || *end == 'k') 27 + val *= 1024; 28 + else if (*end == 'M' || *end == 'm') 29 + val *= 1024 * 1024; 30 + else if (*end == 'G' || *end == 'g') 31 + val *= 1024 * 1024 * 1024; 32 + return val; 33 + } 34 + 35 + static int cmd_topology(int argc, char **argv) 36 + { 37 + (void)argc; 38 + (void)argv; 39 + 40 + icepick_topology_t *topo; 41 + int ret = icepick_discover_topology(&topo); 42 + if (ret < 0) { 43 + fprintf(stderr, "failed to discover topology: %s\n", 44 + icepick_strerror(ret)); 45 + return 1; 46 + } 47 + 48 + printf("l3 cache:\n"); 49 + printf(" total size: %zu mb\n", 50 + icepick_topology_l3_size(topo) / (1024 * 1024)); 51 + printf(" ways: %u\n", icepick_topology_l3_ways(topo)); 52 + printf(" per-way size: %zu kb\n", 53 + icepick_topology_way_size(topo) / 1024); 54 + printf(" ccx count: %u\n\n", icepick_topology_ccx_count(topo)); 55 + 56 + printf("cat configuration:\n"); 57 + printf(" supported: %s\n", 58 + icepick_topology_cat_supported(topo) ? "yes" : "no"); 59 + printf(" max clos: %u\n\n", icepick_topology_max_clos(topo)); 60 + 61 + size_t max_lockable = (icepick_topology_l3_ways(topo) - 1) * 62 + icepick_topology_way_size(topo); 63 + printf("maximum lockable: %zu mb (%u ways, reserving 1 for system)\n", 64 + max_lockable / (1024 * 1024), icepick_topology_l3_ways(topo) - 1); 65 + 66 + icepick_free_topology(topo); 67 + return 0; 68 + } 69 + 70 + static int cmd_lock(int argc, char **argv) 71 + { 72 + size_t size = 0; 73 + unsigned clos = 1; 74 + int numa = -1; 75 + bool verify = false; 76 + bool huge = true; 77 + 78 + static struct option long_options[] = { 79 + {"size", required_argument, 0, 's'}, 80 + {"clos", required_argument, 0, 'c'}, 81 + {"numa", required_argument, 0, 'n'}, 82 + {"verify", no_argument, 0, 'v'}, 83 + {"no-huge", no_argument, 0, 'H'}, 84 + {"help", no_argument, 0, 'h'}, 85 + {0, 0, 0, 0} 86 + }; 87 + 88 + int opt; 89 + while ((opt = getopt_long(argc, argv, "s:c:n:vHh", long_options, NULL)) != -1) { 90 + switch (opt) { 91 + case 's': size = parse_size(optarg); break; 92 + case 'c': clos = atoi(optarg); break; 93 + case 'n': numa = atoi(optarg); break; 94 + case 'v': verify = true; break; 95 + case 'H': huge = false; break; 96 + case 'h': 97 + printf("usage: icepick lock [options]\n\n" 98 + "options:\n" 99 + " -s, --size SIZE region size (e.g., 4M, 2048K)\n" 100 + " -c, --clos ID clos id to use (1-15, default: 1)\n" 101 + " -n, --numa NODE numa node (default: any)\n" 102 + " -v, --verify verify lock after priming\n" 103 + " -H, --no-huge don't use huge pages\n"); 104 + return 0; 105 + default: 106 + return 1; 107 + } 108 + } 109 + 110 + if (size == 0) { 111 + fprintf(stderr, "error: --size is required\n"); 112 + return 1; 113 + } 114 + 115 + icepick_topology_t *topo; 116 + int ret = icepick_discover_topology(&topo); 117 + if (ret < 0) { 118 + fprintf(stderr, "failed to discover topology: %s\n", 119 + icepick_strerror(ret)); 120 + return 1; 121 + } 122 + 123 + printf("allocating region:\n"); 124 + printf(" requested: %zu kb\n", size / 1024); 125 + 126 + icepick_config_t cfg = { 127 + .size = size, 128 + .clos_id = clos, 129 + .numa_node = numa, 130 + .huge_pages = huge, 131 + .verify = verify 132 + }; 133 + 134 + icepick_region_t *region; 135 + ret = icepick_lock(topo, &cfg, &region); 136 + if (ret < 0) { 137 + fprintf(stderr, "lock failed: %s\n", icepick_strerror(ret)); 138 + icepick_free_topology(topo); 139 + return 1; 140 + } 141 + 142 + printf(" allocated: %zu kb\n", icepick_region_size(region) / 1024); 143 + printf(" address: %p\n", icepick_region_ptr(region)); 144 + printf(" clos: %u\n\n", icepick_region_clos(region)); 145 + 146 + if (verify) { 147 + icepick_latency_stats_t stats; 148 + icepick_verify(region, &stats); 149 + printf("verification:\n"); 150 + printf(" mean latency: %lu ns\n", stats.mean_ns); 151 + printf(" p99 latency: %lu ns\n", stats.p99_ns); 152 + printf(" status: %s\n\n", 153 + stats.mean_ns <= 50 ? "locked" : "warning - latency high"); 154 + } 155 + 156 + printf("region locked. press enter to unlock and exit...\n"); 157 + getchar(); 158 + 159 + icepick_unlock(region); 160 + icepick_free_topology(topo); 161 + printf("region unlocked.\n"); 162 + 163 + return 0; 164 + } 165 + 166 + static int cmd_status(int argc, char **argv) 167 + { 168 + (void)argc; 169 + (void)argv; 170 + 171 + icepick_topology_t *topo; 172 + int ret = icepick_discover_topology(&topo); 173 + if (ret < 0) { 174 + fprintf(stderr, "failed to discover topology: %s\n", 175 + icepick_strerror(ret)); 176 + return 1; 177 + } 178 + 179 + printf("clos configuration:\n"); 180 + printf(" clos 0: 0x%04x (system default)\n", clos_get_default_mask()); 181 + 182 + for (unsigned i = 1; i < icepick_topology_max_clos(topo); i++) { 183 + if (clos_table[i].allocated) 184 + printf(" clos %u: 0x%04x (allocated)\n", i, clos_table[i].way_mask); 185 + } 186 + 187 + icepick_free_topology(topo); 188 + return 0; 189 + } 190 + 191 + static int cmd_bench(int argc, char **argv) 192 + { 193 + size_t size = 1024 * 1024; 194 + size_t iterations = 1000000; 195 + 196 + static struct option long_options[] = { 197 + {"size", required_argument, 0, 's'}, 198 + {"iterations", required_argument, 0, 'i'}, 199 + {"help", no_argument, 0, 'h'}, 200 + {0, 0, 0, 0} 201 + }; 202 + 203 + int opt; 204 + while ((opt = getopt_long(argc, argv, "s:i:h", long_options, NULL)) != -1) { 205 + switch (opt) { 206 + case 's': size = parse_size(optarg); break; 207 + case 'i': iterations = strtoul(optarg, NULL, 10); break; 208 + case 'h': 209 + printf("usage: icepick bench [options]\n\n" 210 + "options:\n" 211 + " -s, --size SIZE region size (default: 1M)\n" 212 + " -i, --iterations NUM iteration count (default: 1000000)\n"); 213 + return 0; 214 + default: 215 + return 1; 216 + } 217 + } 218 + 219 + icepick_topology_t *topo; 220 + int ret = icepick_discover_topology(&topo); 221 + if (ret < 0) { 222 + fprintf(stderr, "failed to discover topology: %s\n", 223 + icepick_strerror(ret)); 224 + return 1; 225 + } 226 + 227 + ret = bench_compare(topo, size, iterations); 228 + icepick_free_topology(topo); 229 + return ret < 0 ? 1 : 0; 230 + } 231 + 232 + int main(int argc, char **argv) 233 + { 234 + if (argc < 2) { 235 + usage(argv[0]); 236 + return 1; 237 + } 238 + 239 + const char *cmd = argv[1]; 240 + argc--; 241 + argv++; 242 + 243 + if (strcmp(cmd, "topology") == 0) 244 + return cmd_topology(argc, argv); 245 + if (strcmp(cmd, "lock") == 0) 246 + return cmd_lock(argc, argv); 247 + if (strcmp(cmd, "status") == 0) 248 + return cmd_status(argc, argv); 249 + if (strcmp(cmd, "bench") == 0) 250 + return cmd_bench(argc, argv); 251 + if (strcmp(cmd, "--help") == 0 || strcmp(cmd, "-h") == 0) { 252 + usage(argv[0]); 253 + return 0; 254 + } 255 + 256 + fprintf(stderr, "unknown command: %s\n", cmd); 257 + usage(argv[0]); 258 + return 1; 259 + }
+41
src/msr.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <fcntl.h> 4 + #include <unistd.h> 5 + #include <stdio.h> 6 + #include <errno.h> 7 + 8 + int msr_open(int cpu) 9 + { 10 + char path[64]; 11 + snprintf(path, sizeof(path), "/dev/cpu/%d/msr", cpu); 12 + int fd = open(path, O_RDWR); 13 + if (fd < 0) { 14 + if (errno == ENOENT) 15 + return ICEPICK_E_NO_CAT; 16 + if (errno == EACCES || errno == EPERM) 17 + return ICEPICK_E_PERMISSION; 18 + return ICEPICK_E_MSR; 19 + } 20 + return fd; 21 + } 22 + 23 + void msr_close(int fd) 24 + { 25 + if (fd >= 0) 26 + close(fd); 27 + } 28 + 29 + int msr_read(int fd, uint32_t msr, uint64_t *value) 30 + { 31 + if (pread(fd, value, sizeof(*value), msr) != sizeof(*value)) 32 + return ICEPICK_E_MSR; 33 + return 0; 34 + } 35 + 36 + int msr_write(int fd, uint32_t msr, uint64_t value) 37 + { 38 + if (pwrite(fd, &value, sizeof(value), msr) != sizeof(value)) 39 + return ICEPICK_E_MSR; 40 + return 0; 41 + }
+68
src/region.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <sys/mman.h> 4 + #include <numaif.h> 5 + #include <errno.h> 6 + #include <string.h> 7 + 8 + #ifndef MAP_HUGE_2MB 9 + #define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT) 10 + #endif 11 + 12 + #ifndef MAP_HUGE_SHIFT 13 + #define MAP_HUGE_SHIFT 26 14 + #endif 15 + 16 + int region_alloc(size_t size, int numa_node, bool huge_pages, void **out_ptr) 17 + { 18 + int flags = MAP_PRIVATE | MAP_ANONYMOUS; 19 + size_t alloc_size = size; 20 + 21 + if (huge_pages) { 22 + flags |= MAP_HUGETLB | MAP_HUGE_2MB; 23 + size_t huge_page_size = 2 * 1024 * 1024; 24 + alloc_size = (size + huge_page_size - 1) & ~(huge_page_size - 1); 25 + } 26 + 27 + void *ptr = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, flags, -1, 0); 28 + if (ptr == MAP_FAILED) { 29 + if (huge_pages && errno == ENOMEM) { 30 + flags = MAP_PRIVATE | MAP_ANONYMOUS; 31 + alloc_size = size; 32 + ptr = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, flags, -1, 0); 33 + if (ptr == MAP_FAILED) 34 + return ICEPICK_E_ALLOC; 35 + } else { 36 + return huge_pages ? ICEPICK_E_HUGEPAGE : ICEPICK_E_ALLOC; 37 + } 38 + } 39 + 40 + if (numa_node >= 0) { 41 + unsigned long nodemask = 1UL << numa_node; 42 + if (mbind(ptr, alloc_size, MPOL_BIND, &nodemask, sizeof(nodemask) * 8, 43 + MPOL_MF_MOVE | MPOL_MF_STRICT) < 0) { 44 + munmap(ptr, alloc_size); 45 + return ICEPICK_E_NUMA; 46 + } 47 + } 48 + 49 + if (mlock(ptr, alloc_size) < 0) { 50 + munmap(ptr, alloc_size); 51 + return ICEPICK_E_PERMISSION; 52 + } 53 + 54 + madvise(ptr, alloc_size, MADV_DONTFORK); 55 + 56 + memset(ptr, 0, alloc_size); 57 + 58 + *out_ptr = ptr; 59 + return 0; 60 + } 61 + 62 + void region_free(void *ptr, size_t size) 63 + { 64 + if (ptr) { 65 + munlock(ptr, size); 66 + munmap(ptr, size); 67 + } 68 + }
+185
src/topology.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <stdlib.h> 4 + #include <string.h> 5 + #include <cpuid.h> 6 + #include <sched.h> 7 + #include <dirent.h> 8 + #include <stdio.h> 9 + 10 + static void cpuid(uint32_t leaf, uint32_t subleaf, 11 + uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) 12 + { 13 + __cpuid_count(leaf, subleaf, *eax, *ebx, *ecx, *edx); 14 + } 15 + 16 + static bool detect_cat_support(icepick_topology_t *topo) 17 + { 18 + uint32_t eax, ebx, ecx, edx; 19 + 20 + cpuid(0x10, 0, &eax, &ebx, &ecx, &edx); 21 + if (!(ebx & (1 << 1))) 22 + return false; 23 + 24 + cpuid(0x10, 1, &eax, &ebx, &ecx, &edx); 25 + 26 + topo->l3_ways = (eax & 0x1F) + 1; 27 + topo->max_clos = (edx & 0xFFFF) + 1; 28 + topo->default_way_mask = ebx; 29 + 30 + if (topo->max_clos > MAX_CLOS) 31 + topo->max_clos = MAX_CLOS; 32 + 33 + return true; 34 + } 35 + 36 + static void detect_amd_cache_topology(icepick_topology_t *topo) 37 + { 38 + uint32_t eax, ebx, ecx, edx; 39 + 40 + for (int i = 0; i < 16; i++) { 41 + cpuid(0x8000001D, i, &eax, &ebx, &ecx, &edx); 42 + 43 + uint32_t cache_type = eax & 0x1F; 44 + if (cache_type == 0) 45 + break; 46 + 47 + uint32_t cache_level = (eax >> 5) & 0x7; 48 + if (cache_level != 3) 49 + continue; 50 + 51 + uint32_t line_size = (ebx & 0xFFF) + 1; 52 + uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1; 53 + uint32_t ways = ((ebx >> 22) & 0x3FF) + 1; 54 + uint32_t sets = ecx + 1; 55 + 56 + topo->l3_size = (size_t)line_size * partitions * ways * sets; 57 + topo->l3_ways = ways; 58 + topo->way_size = topo->l3_size / topo->l3_ways; 59 + break; 60 + } 61 + } 62 + 63 + static void detect_intel_cache_topology(icepick_topology_t *topo) 64 + { 65 + uint32_t eax, ebx, ecx, edx; 66 + 67 + for (int i = 0; i < 16; i++) { 68 + cpuid(0x04, i, &eax, &ebx, &ecx, &edx); 69 + 70 + uint32_t cache_type = eax & 0x1F; 71 + if (cache_type == 0) 72 + break; 73 + 74 + uint32_t cache_level = (eax >> 5) & 0x7; 75 + if (cache_level != 3) 76 + continue; 77 + 78 + uint32_t line_size = (ebx & 0xFFF) + 1; 79 + uint32_t partitions = ((ebx >> 12) & 0x3FF) + 1; 80 + uint32_t ways = ((ebx >> 22) & 0x3FF) + 1; 81 + uint32_t sets = ecx + 1; 82 + 83 + topo->l3_size = (size_t)line_size * partitions * ways * sets; 84 + topo->l3_ways = ways; 85 + topo->way_size = topo->l3_size / topo->l3_ways; 86 + break; 87 + } 88 + } 89 + 90 + static bool is_amd_cpu(void) 91 + { 92 + uint32_t eax, ebx, ecx, edx; 93 + cpuid(0, 0, &eax, &ebx, &ecx, &edx); 94 + return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163); 95 + } 96 + 97 + static int count_ccxs(void) 98 + { 99 + DIR *dir = opendir("/sys/devices/system/node"); 100 + if (!dir) 101 + return 1; 102 + 103 + int count = 0; 104 + struct dirent *entry; 105 + while ((entry = readdir(dir))) { 106 + if (strncmp(entry->d_name, "node", 4) == 0) 107 + count++; 108 + } 109 + closedir(dir); 110 + 111 + return count > 0 ? count : 1; 112 + } 113 + 114 + static void detect_numa_topology(icepick_topology_t *topo) 115 + { 116 + topo->ccx_count = count_ccxs(); 117 + if (topo->ccx_count > MAX_CCXS) 118 + topo->ccx_count = MAX_CCXS; 119 + 120 + for (unsigned i = 0; i < topo->ccx_count; i++) 121 + topo->ccx_numa_node[i] = i; 122 + } 123 + 124 + int icepick_discover_topology(icepick_topology_t **topo) 125 + { 126 + icepick_topology_t *t = calloc(1, sizeof(*t)); 127 + if (!t) 128 + return ICEPICK_E_ALLOC; 129 + 130 + t->cat_supported = detect_cat_support(t); 131 + if (!t->cat_supported) { 132 + free(t); 133 + return ICEPICK_E_NO_CAT; 134 + } 135 + 136 + if (is_amd_cpu()) 137 + detect_amd_cache_topology(t); 138 + else 139 + detect_intel_cache_topology(t); 140 + 141 + if (t->l3_size == 0 || t->l3_ways == 0) { 142 + t->way_size = 2 * 1024 * 1024; 143 + t->l3_size = t->way_size * t->l3_ways; 144 + } 145 + 146 + detect_numa_topology(t); 147 + 148 + *topo = t; 149 + return 0; 150 + } 151 + 152 + void icepick_free_topology(icepick_topology_t *topo) 153 + { 154 + free(topo); 155 + } 156 + 157 + unsigned icepick_topology_l3_ways(const icepick_topology_t *topo) 158 + { 159 + return topo->l3_ways; 160 + } 161 + 162 + size_t icepick_topology_l3_size(const icepick_topology_t *topo) 163 + { 164 + return topo->l3_size; 165 + } 166 + 167 + size_t icepick_topology_way_size(const icepick_topology_t *topo) 168 + { 169 + return topo->way_size; 170 + } 171 + 172 + unsigned icepick_topology_max_clos(const icepick_topology_t *topo) 173 + { 174 + return topo->max_clos; 175 + } 176 + 177 + unsigned icepick_topology_ccx_count(const icepick_topology_t *topo) 178 + { 179 + return topo->ccx_count; 180 + } 181 + 182 + bool icepick_topology_cat_supported(const icepick_topology_t *topo) 183 + { 184 + return topo->cat_supported; 185 + }
+172
src/verify.c
··· 1 + #define _GNU_SOURCE 2 + #include "internal.h" 3 + #include <stdlib.h> 4 + #include <string.h> 5 + #include <time.h> 6 + #include <x86intrin.h> 7 + 8 + #define VERIFY_SAMPLES 1000 9 + #define L3_THRESHOLD_NS 50 10 + 11 + static uint64_t tsc_freq_khz; 12 + 13 + uint64_t rdtsc_start(void) 14 + { 15 + unsigned int aux; 16 + _mm_lfence(); 17 + return __rdtscp(&aux); 18 + } 19 + 20 + uint64_t rdtsc_end(void) 21 + { 22 + unsigned int aux; 23 + uint64_t tsc = __rdtscp(&aux); 24 + _mm_lfence(); 25 + return tsc; 26 + } 27 + 28 + static void calibrate_tsc(void) 29 + { 30 + if (tsc_freq_khz != 0) 31 + return; 32 + 33 + struct timespec start, end; 34 + clock_gettime(CLOCK_MONOTONIC, &start); 35 + uint64_t tsc_start = rdtsc_start(); 36 + 37 + volatile unsigned long dummy = 0; 38 + for (int i = 0; i < 10000000; i++) 39 + dummy += i; 40 + (void)dummy; 41 + 42 + uint64_t tsc_end = rdtsc_end(); 43 + clock_gettime(CLOCK_MONOTONIC, &end); 44 + 45 + uint64_t ns = (end.tv_sec - start.tv_sec) * 1000000000ULL + 46 + (end.tv_nsec - start.tv_nsec); 47 + uint64_t cycles = tsc_end - tsc_start; 48 + 49 + tsc_freq_khz = (cycles * 1000000ULL) / ns; 50 + } 51 + 52 + uint64_t cycles_to_ns(uint64_t cycles) 53 + { 54 + if (tsc_freq_khz == 0) 55 + calibrate_tsc(); 56 + return (cycles * 1000000ULL) / tsc_freq_khz; 57 + } 58 + 59 + static int compare_u64(const void *a, const void *b) 60 + { 61 + uint64_t va = *(const uint64_t *)a; 62 + uint64_t vb = *(const uint64_t *)b; 63 + return (va > vb) - (va < vb); 64 + } 65 + 66 + static uint64_t isqrt(uint64_t n) 67 + { 68 + if (n == 0) 69 + return 0; 70 + uint64_t x = n; 71 + uint64_t y = (x + 1) / 2; 72 + while (y < x) { 73 + x = y; 74 + y = (x + n / x) / 2; 75 + } 76 + return x; 77 + } 78 + 79 + static void compute_stats(uint64_t *samples, size_t count, 80 + icepick_latency_stats_t *stats) 81 + { 82 + qsort(samples, count, sizeof(uint64_t), compare_u64); 83 + 84 + uint64_t sum = 0; 85 + for (size_t i = 0; i < count; i++) 86 + sum += samples[i]; 87 + 88 + stats->mean_ns = sum / count; 89 + stats->min_ns = samples[0]; 90 + stats->max_ns = samples[count - 1]; 91 + stats->p50_ns = samples[count / 2]; 92 + stats->p99_ns = samples[(count * 99) / 100]; 93 + stats->p999_ns = samples[(count * 999) / 1000]; 94 + 95 + uint64_t var_sum = 0; 96 + for (size_t i = 0; i < count; i++) { 97 + int64_t diff = samples[i] - stats->mean_ns; 98 + var_sum += diff * diff; 99 + } 100 + stats->stddev_ns = isqrt(var_sum / count); 101 + } 102 + 103 + int icepick_verify(const icepick_region_t *region, icepick_latency_stats_t *stats) 104 + { 105 + if (!region || !stats) 106 + return ICEPICK_E_INVALID; 107 + 108 + calibrate_tsc(); 109 + 110 + uint64_t *samples = malloc(VERIFY_SAMPLES * sizeof(uint64_t)); 111 + if (!samples) 112 + return ICEPICK_E_ALLOC; 113 + 114 + volatile char *ptr = region->ptr; 115 + size_t size = region->size; 116 + 117 + unsigned int seed = (unsigned int)time(NULL); 118 + for (int i = 0; i < VERIFY_SAMPLES; i++) { 119 + size_t offset = (rand_r(&seed) % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE; 120 + 121 + uint64_t start = rdtsc_start(); 122 + (void)ptr[offset]; 123 + uint64_t end = rdtsc_end(); 124 + 125 + samples[i] = cycles_to_ns(end - start); 126 + } 127 + 128 + compute_stats(samples, VERIFY_SAMPLES, stats); 129 + free(samples); 130 + 131 + if (stats->mean_ns > L3_THRESHOLD_NS) 132 + return ICEPICK_E_VERIFY; 133 + 134 + return 0; 135 + } 136 + 137 + int icepick_bench(void *ptr, size_t size, size_t iterations, 138 + icepick_latency_stats_t *stats) 139 + { 140 + if (!ptr || !stats || size == 0 || iterations == 0) 141 + return ICEPICK_E_INVALID; 142 + 143 + calibrate_tsc(); 144 + 145 + size_t sample_count = iterations > 100000 ? 100000 : iterations; 146 + uint64_t *samples = malloc(sample_count * sizeof(uint64_t)); 147 + if (!samples) 148 + return ICEPICK_E_ALLOC; 149 + 150 + volatile char *p = ptr; 151 + size_t stride = CACHE_LINE_SIZE; 152 + size_t sample_idx = 0; 153 + size_t sample_interval = iterations / sample_count; 154 + if (sample_interval == 0) 155 + sample_interval = 1; 156 + 157 + for (size_t i = 0; i < iterations; i++) { 158 + size_t offset = (i * stride) % size; 159 + 160 + uint64_t start = rdtsc_start(); 161 + (void)p[offset]; 162 + uint64_t end = rdtsc_end(); 163 + 164 + if (i % sample_interval == 0 && sample_idx < sample_count) 165 + samples[sample_idx++] = cycles_to_ns(end - start); 166 + } 167 + 168 + compute_stats(samples, sample_idx, stats); 169 + free(samples); 170 + 171 + return 0; 172 + }
+110
tools/check_support.sh
··· 1 + #!/bin/bash 2 + 3 + set -e 4 + 5 + echo "icepick hardware support check" 6 + echo 7 + 8 + check_cpuid() { 9 + if ! command -v cpuid &> /dev/null; then 10 + echo "- cpuid command not found. install cpuid package for detailed info." 11 + return 1 12 + fi 13 + 14 + echo "checking cat support via cpuid..." 15 + if cpuid -1 -l 0x10 -s 0 2>/dev/null | grep -q "L3 CAT"; then 16 + echo "+ l3 cat supported" 17 + cpuid -1 -l 0x10 -s 1 2>/dev/null | grep -E "(ways|CLOS)" || true 18 + else 19 + echo "- l3 cat not supported on this cpu" 20 + fi 21 + echo 22 + } 23 + 24 + check_msr_module() { 25 + echo "checking msr kernel module..." 26 + if lsmod | grep -q "^msr "; then 27 + echo "+ msr module loaded" 28 + else 29 + echo "- msr module not loaded" 30 + echo " run: sudo modprobe msr" 31 + fi 32 + 33 + if [ -e /dev/cpu/0/msr ]; then 34 + echo "+ /dev/cpu/0/msr exists" 35 + if [ -r /dev/cpu/0/msr ]; then 36 + echo "+ msr device readable" 37 + else 38 + echo "- msr device not readable (need root or cap_sys_rawio)" 39 + fi 40 + else 41 + echo "- /dev/cpu/0/msr does not exist" 42 + fi 43 + echo 44 + } 45 + 46 + check_resctrl() { 47 + echo "checking resctrl filesystem..." 48 + if mount | grep -q "resctrl"; then 49 + echo "+ resctrl mounted" 50 + mount | grep resctrl 51 + else 52 + echo "- resctrl not mounted" 53 + echo " run: sudo mount -t resctrl resctrl /sys/fs/resctrl" 54 + fi 55 + 56 + if [ -d /sys/fs/resctrl/info/L3 ]; then 57 + echo 58 + echo "l3 cat info from resctrl:" 59 + echo " cbm mask: $(cat /sys/fs/resctrl/info/L3/cbm_mask 2>/dev/null || echo 'n/a')" 60 + echo " min cbm: $(cat /sys/fs/resctrl/info/L3/min_cbm_bits 2>/dev/null || echo 'n/a')" 61 + echo " num clos: $(cat /sys/fs/resctrl/info/L3/num_closids 2>/dev/null || echo 'n/a')" 62 + fi 63 + echo 64 + } 65 + 66 + check_hugepages() { 67 + echo "checking huge pages..." 68 + local nr_hugepages=$(cat /proc/sys/vm/nr_hugepages 2>/dev/null || echo "0") 69 + local free_hugepages=$(grep HugePages_Free /proc/meminfo 2>/dev/null | awk '{print $2}' || echo "0") 70 + local hugepage_size=$(grep Hugepagesize /proc/meminfo 2>/dev/null | awk '{print $2, $3}' || echo "n/a") 71 + 72 + echo " configured: $nr_hugepages" 73 + echo " free: $free_hugepages" 74 + echo " size: $hugepage_size" 75 + 76 + if [ "$free_hugepages" -gt 0 ]; then 77 + echo "+ huge pages available" 78 + else 79 + echo "- no free huge pages" 80 + echo " run: sudo sh -c 'echo 64 > /proc/sys/vm/nr_hugepages'" 81 + fi 82 + echo 83 + } 84 + 85 + cpu_info() { 86 + echo "cpu information:" 87 + grep -m1 "model name" /proc/cpuinfo | cut -d: -f2 | xargs echo " model:" 88 + 89 + local vendor=$(grep -m1 "vendor_id" /proc/cpuinfo | cut -d: -f2 | xargs) 90 + echo " vendor: $vendor" 91 + 92 + if [ "$vendor" = "AuthenticAMD" ]; then 93 + local family=$(grep -m1 "cpu family" /proc/cpuinfo | cut -d: -f2 | xargs) 94 + local model=$(grep -m1 "^model" /proc/cpuinfo | head -1 | cut -d: -f2 | xargs) 95 + echo " family: $family" 96 + echo " model: $model" 97 + if [ "$family" -ge 23 ]; then 98 + echo "+ zen architecture (cat likely supported)" 99 + fi 100 + fi 101 + echo 102 + } 103 + 104 + cpu_info 105 + check_cpuid 106 + check_msr_module 107 + check_resctrl 108 + check_hugepages 109 + 110 + echo "check complete"
+38
tools/setup_resctrl.sh
··· 1 + #!/bin/bash 2 + 3 + set -e 4 + 5 + RESCTRL_MOUNT="/sys/fs/resctrl" 6 + 7 + if [ "$(id -u)" -ne 0 ]; then 8 + echo "error: this script must be run as root" 9 + exit 1 10 + fi 11 + 12 + if ! grep -q "resctrl" /proc/filesystems 2>/dev/null; then 13 + echo "error: resctrl filesystem not supported by this kernel" 14 + echo "ensure config_x86_cpu_resctrl is enabled" 15 + exit 1 16 + fi 17 + 18 + if mount | grep -q "resctrl"; then 19 + echo "resctrl already mounted at:" 20 + mount | grep resctrl 21 + exit 0 22 + fi 23 + 24 + mkdir -p "$RESCTRL_MOUNT" 25 + 26 + echo "mounting resctrl filesystem..." 27 + mount -t resctrl resctrl "$RESCTRL_MOUNT" 28 + 29 + echo "resctrl mounted successfully at $RESCTRL_MOUNT" 30 + echo 31 + echo "l3 cat info:" 32 + if [ -d "$RESCTRL_MOUNT/info/L3" ]; then 33 + echo " cbm mask: $(cat $RESCTRL_MOUNT/info/L3/cbm_mask)" 34 + echo " min cbm: $(cat $RESCTRL_MOUNT/info/L3/min_cbm_bits)" 35 + echo " num clos: $(cat $RESCTRL_MOUNT/info/L3/num_closids)" 36 + else 37 + echo " l3 cat info not available" 38 + fi