cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
add hybrid pmu/probe eviction monitor with auto-reprime on l3 miss detection
author
vm.fail
date
(Jan 18, 2026, 2:57 AM UTC)
commit
a93f1852
a93f1852d96cd6e97d036d955c228dada9d0f0e2
parent
6fe2641d
6fe2641d515c2ed1340ada1080eef9ee3135428e
vm.fail
date
(Jan 18, 2026, 2:57 AM UTC)
commit
a93f1852
a93f1852d96cd6e97d036d955c228dada9d0f0e2
parent
6fe2641d
6fe2641d515c2ed1340ada1080eef9ee3135428e
+375
-2
+2
-1
Makefile
+2
-1
Makefile
···
1
1
CC = gcc
2
2
CFLAGS = -O2 -Wall -Wextra -Werror -fPIC -march=native -I include
3
-
LDFLAGS = -lnuma
3
+
LDFLAGS = -lnuma -lpthread
4
4
5
5
SRCDIR = src
6
6
INCDIR = include
···
61
61
$(OBJDIR)/lock.o: $(SRCDIR)/lock.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
62
62
$(OBJDIR)/verify.o: $(SRCDIR)/verify.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
63
63
$(OBJDIR)/bench.o: $(SRCDIR)/bench.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
64
+
$(OBJDIR)/monitor.o: $(SRCDIR)/monitor.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
64
65
$(OBJDIR)/main.o: $(SRCDIR)/main.c $(SRCDIR)/internal.h $(INCDIR)/icepick.h
+58
bindings/ocaml/icepick.ml
+58
bindings/ocaml/icepick.ml
···
73
73
74
74
let bench = foreign "icepick_bench"
75
75
(ptr void @-> size_t @-> size_t @-> ptr Latency_stats.t @-> returning int)
76
+
77
+
module Monitor = struct
78
+
type t = unit ptr
79
+
let t : t typ = ptr void
80
+
81
+
let start = foreign "icepick_monitor_start" (Region.t @-> ptr t @-> returning int)
82
+
let start_ex = foreign "icepick_monitor_start_ex"
83
+
(Region.t @-> uint64_t @-> uint64_t @-> uint32_t @-> ptr t @-> returning int)
84
+
let stop = foreign "icepick_monitor_stop" (t @-> returning int)
85
+
let eviction_count = foreign "icepick_monitor_eviction_count" (t @-> returning uint64_t)
86
+
let reprime_count = foreign "icepick_monitor_reprime_count" (t @-> returning uint64_t)
87
+
let is_degraded = foreign "icepick_monitor_is_degraded" (t @-> returning bool)
88
+
end
76
89
end
77
90
78
91
module Topology = struct
···
171
184
let typed_ptr = from_voidp double (to_voidp region_ptr) in
172
185
CArray.from_ptr typed_ptr len |> CArray.to_list |> Array.of_list |>
173
186
Bigarray.Array1.of_array Bigarray.Float64 Bigarray.c_layout
187
+
end
188
+
189
+
module Monitor = struct
190
+
type t = {
191
+
ptr : C.Monitor.t;
192
+
region : Region.t;
193
+
mutable stopped : bool;
194
+
}
195
+
196
+
let release t =
197
+
if not t.stopped then begin
198
+
let _ = C.Monitor.stop t.ptr in
199
+
t.stopped <- true
200
+
end
201
+
202
+
let start region =
203
+
let ptr_ref = allocate C.Monitor.t (from_voidp void null) in
204
+
let ret = C.Monitor.start region.Region.ptr ptr_ref in
205
+
C.check_error ret;
206
+
let t = { ptr = !@ ptr_ref; region; stopped = false } in
207
+
Gc.finalise release t;
208
+
t
209
+
210
+
let start_ex region ~pmu_poll_ns ~probe_ns ~miss_thresh =
211
+
let ptr_ref = allocate C.Monitor.t (from_voidp void null) in
212
+
let ret = C.Monitor.start_ex region.Region.ptr
213
+
(Unsigned.UInt64.of_int pmu_poll_ns)
214
+
(Unsigned.UInt64.of_int probe_ns)
215
+
(Unsigned.UInt32.of_int miss_thresh)
216
+
ptr_ref in
217
+
C.check_error ret;
218
+
let t = { ptr = !@ ptr_ref; region; stopped = false } in
219
+
Gc.finalise release t;
220
+
t
221
+
222
+
let stop t =
223
+
if not t.stopped then begin
224
+
let ret = C.Monitor.stop t.ptr in
225
+
t.stopped <- true;
226
+
C.check_error ret
227
+
end
228
+
229
+
let eviction_count t = Unsigned.UInt64.to_int (C.Monitor.eviction_count t.ptr)
230
+
let reprime_count t = Unsigned.UInt64.to_int (C.Monitor.reprime_count t.ptr)
231
+
let is_degraded t = C.Monitor.is_degraded t.ptr
174
232
end
175
233
176
234
let verify region =
+11
bindings/ocaml/icepick.mli
+11
bindings/ocaml/icepick.mli
···
35
35
}
36
36
end
37
37
38
+
module Monitor : sig
39
+
type t
40
+
41
+
val start : Region.t -> t
42
+
val start_ex : Region.t -> pmu_poll_ns:int -> probe_ns:int -> miss_thresh:int -> t
43
+
val stop : t -> unit
44
+
val eviction_count : t -> int
45
+
val reprime_count : t -> int
46
+
val is_degraded : t -> bool
47
+
end
48
+
38
49
val verify : Region.t -> Latency_stats.t
39
50
val bench : nativeint -> size:int -> iterations:int -> Latency_stats.t
+15
include/icepick.h
+15
include/icepick.h
···
15
15
#define ICEPICK_E_INVALID (-8)
16
16
#define ICEPICK_E_ALLOC (-9)
17
17
#define ICEPICK_E_MSR (-10)
18
+
#define ICEPICK_E_THREAD (-11)
18
19
19
20
typedef struct icepick_topology icepick_topology_t;
20
21
typedef struct icepick_region icepick_region_t;
22
+
typedef struct icepick_monitor icepick_monitor_t;
21
23
22
24
typedef struct {
23
25
size_t size;
···
25
27
int numa_node;
26
28
bool huge_pages;
27
29
bool verify;
30
+
bool auto_monitor;
31
+
uint64_t pmu_poll_interval_ns;
32
+
uint64_t probe_interval_ns;
33
+
uint32_t miss_threshold;
28
34
} icepick_config_t;
29
35
30
36
typedef struct {
···
58
64
59
65
int icepick_bench(void *ptr, size_t size, size_t iterations,
60
66
icepick_latency_stats_t *stats);
67
+
68
+
int icepick_monitor_start(icepick_region_t *region, icepick_monitor_t **mon);
69
+
int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
70
+
uint64_t probe_ns, uint32_t miss_thresh,
71
+
icepick_monitor_t **mon);
72
+
int icepick_monitor_stop(icepick_monitor_t *mon);
73
+
uint64_t icepick_monitor_eviction_count(const icepick_monitor_t *mon);
74
+
uint64_t icepick_monitor_reprime_count(const icepick_monitor_t *mon);
75
+
bool icepick_monitor_is_degraded(const icepick_monitor_t *mon);
61
76
62
77
const char *icepick_strerror(int err);
63
78
+29
src/internal.h
+29
src/internal.h
···
3
3
4
4
#include "../include/icepick.h"
5
5
#include <stdint.h>
6
+
#include <stdatomic.h>
7
+
#include <pthread.h>
6
8
7
9
#define MSR_IA32_PQR_ASSOC 0xC8F
8
10
#define MSR_IA32_L3_QOS_MASK_0 0xC90
···
11
13
#define MAX_CLOS 16
12
14
#define MAX_CCXS 16
13
15
16
+
#define MONITOR_DEFAULT_PMU_POLL_NS 100000
17
+
#define MONITOR_DEFAULT_PROBE_NS 10000
18
+
#define MONITOR_DEFAULT_MISS_THRESH 16
19
+
#define MONITOR_PROBE_CONFIRM_COUNT 8
20
+
#define MONITOR_L3_LATENCY_THRESH_NS 50
21
+
22
+
struct icepick_monitor;
23
+
14
24
struct icepick_topology {
15
25
bool cat_supported;
16
26
unsigned l3_ways;
···
29
39
int numa_node;
30
40
uint32_t way_mask;
31
41
icepick_topology_t *topo;
42
+
struct icepick_monitor *monitor;
43
+
};
44
+
45
+
struct icepick_monitor {
46
+
icepick_region_t *region;
47
+
pthread_t thread;
48
+
int perf_fd;
49
+
int msr_fd;
50
+
int cpu;
51
+
atomic_bool running;
52
+
atomic_uint_fast64_t eviction_count;
53
+
atomic_uint_fast64_t reprime_count;
54
+
uint64_t pmu_poll_interval_ns;
55
+
uint64_t probe_interval_ns;
56
+
uint32_t miss_threshold;
57
+
bool degraded;
32
58
};
33
59
34
60
typedef struct {
···
58
84
uint64_t cycles_to_ns(uint64_t cycles);
59
85
60
86
int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations);
87
+
88
+
void prime_region(volatile char *ptr, size_t size, unsigned iterations);
89
+
int monitor_perf_open(int cpu, bool is_amd);
61
90
62
91
#endif
+18
-1
src/lock.c
+18
-1
src/lock.c
···
22
22
return 0;
23
23
}
24
24
25
-
static void prime_region(volatile char *ptr, size_t size, unsigned iterations)
25
+
void prime_region(volatile char *ptr, size_t size, unsigned iterations)
26
26
{
27
27
for (unsigned iter = 0; iter < iterations; iter++) {
28
28
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
···
69
69
r->way_mask = way_mask;
70
70
r->topo = topo;
71
71
r->size = ways_needed * topo->way_size;
72
+
r->monitor = NULL;
72
73
73
74
ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
74
75
if (ret < 0) {
···
124
125
}
125
126
}
126
127
128
+
if (cfg->auto_monitor) {
129
+
icepick_monitor_t *mon;
130
+
ret = icepick_monitor_start_ex(r, cfg->pmu_poll_interval_ns,
131
+
cfg->probe_interval_ns,
132
+
cfg->miss_threshold, &mon);
133
+
if (ret < 0) {
134
+
icepick_unlock(r);
135
+
return ret;
136
+
}
137
+
r->monitor = mon;
138
+
}
139
+
127
140
*region = r;
128
141
return 0;
129
142
···
140
153
{
141
154
if (!region)
142
155
return ICEPICK_E_INVALID;
156
+
157
+
if (region->monitor)
158
+
icepick_monitor_stop(region->monitor);
143
159
144
160
int cpu = sched_getcpu();
145
161
if (cpu < 0)
···
188
204
case ICEPICK_E_INVALID: return "invalid argument";
189
205
case ICEPICK_E_ALLOC: return "memory allocation failed";
190
206
case ICEPICK_E_MSR: return "msr operation failed";
207
+
case ICEPICK_E_THREAD: return "monitor thread creation failed";
191
208
default: return "unknown error";
192
209
}
193
210
}
+242
src/monitor.c
+242
src/monitor.c
···
1
+
#define _GNU_SOURCE
2
+
#include "internal.h"
3
+
#include <stdlib.h>
4
+
#include <string.h>
5
+
#include <stdio.h>
6
+
#include <unistd.h>
7
+
#include <sched.h>
8
+
#include <time.h>
9
+
#include <sys/ioctl.h>
10
+
#include <sys/syscall.h>
11
+
#include <linux/perf_event.h>
12
+
#include <linux/hw_breakpoint.h>
13
+
#include <x86intrin.h>
14
+
#include <cpuid.h>
15
+
16
+
#define AMD_L3_MISS_EVENT 0x0104
17
+
#define PERF_CACHE_CONFIG (PERF_COUNT_HW_CACHE_LL | \
18
+
(PERF_COUNT_HW_CACHE_OP_READ << 8) | \
19
+
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
20
+
21
+
static bool is_amd(void)
22
+
{
23
+
uint32_t eax, ebx, ecx, edx;
24
+
__cpuid(0, eax, ebx, ecx, edx);
25
+
return (ebx == 0x68747541 && edx == 0x69746E65 && ecx == 0x444D4163);
26
+
}
27
+
28
+
static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu,
29
+
int group_fd, unsigned long flags)
30
+
{
31
+
return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
32
+
}
33
+
34
+
int monitor_perf_open(int cpu, bool amd)
35
+
{
36
+
struct perf_event_attr pe;
37
+
memset(&pe, 0, sizeof(pe));
38
+
pe.size = sizeof(pe);
39
+
pe.disabled = 1;
40
+
pe.exclude_kernel = 1;
41
+
pe.exclude_hv = 1;
42
+
43
+
if (amd) {
44
+
pe.type = PERF_TYPE_RAW;
45
+
pe.config = AMD_L3_MISS_EVENT;
46
+
} else {
47
+
pe.type = PERF_TYPE_HW_CACHE;
48
+
pe.config = PERF_CACHE_CONFIG;
49
+
}
50
+
51
+
int fd = perf_event_open(&pe, -1, cpu, -1, 0);
52
+
if (fd < 0)
53
+
return -1;
54
+
55
+
ioctl(fd, PERF_EVENT_IOC_RESET, 0);
56
+
ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
57
+
58
+
return fd;
59
+
}
60
+
61
+
static void nanosleep_relative(uint64_t ns)
62
+
{
63
+
struct timespec ts;
64
+
ts.tv_sec = ns / 1000000000ULL;
65
+
ts.tv_nsec = ns % 1000000000ULL;
66
+
nanosleep(&ts, NULL);
67
+
}
68
+
69
+
static uint64_t probe_latency(volatile char *ptr, size_t size)
70
+
{
71
+
size_t offset = ((size_t)rdtsc_start() % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE;
72
+
_mm_lfence();
73
+
uint64_t start = rdtsc_start();
74
+
(void)ptr[offset];
75
+
uint64_t end = rdtsc_end();
76
+
return cycles_to_ns(end - start);
77
+
}
78
+
79
+
static void reprime_region(icepick_monitor_t *mon)
80
+
{
81
+
icepick_region_t *r = mon->region;
82
+
83
+
cpu_set_t old_affinity, new_affinity;
84
+
CPU_ZERO(&new_affinity);
85
+
CPU_SET(mon->cpu, &new_affinity);
86
+
sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
87
+
sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
88
+
89
+
clos_associate_thread(mon->msr_fd, r->clos_id);
90
+
prime_region(r->ptr, r->size, 3);
91
+
clos_associate_thread(mon->msr_fd, 0);
92
+
93
+
sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
94
+
95
+
atomic_fetch_add(&mon->reprime_count, 1);
96
+
}
97
+
98
+
static void *monitor_thread(void *arg)
99
+
{
100
+
icepick_monitor_t *mon = arg;
101
+
icepick_region_t *r = mon->region;
102
+
103
+
cpu_set_t affinity;
104
+
CPU_ZERO(&affinity);
105
+
CPU_SET(mon->cpu, &affinity);
106
+
sched_setaffinity(0, sizeof(affinity), &affinity);
107
+
108
+
uint64_t prev_count = 0;
109
+
if (mon->perf_fd >= 0) {
110
+
read(mon->perf_fd, &prev_count, sizeof(prev_count));
111
+
}
112
+
113
+
while (atomic_load(&mon->running)) {
114
+
if (mon->perf_fd >= 0) {
115
+
nanosleep_relative(mon->pmu_poll_interval_ns);
116
+
117
+
uint64_t curr_count;
118
+
if (read(mon->perf_fd, &curr_count, sizeof(curr_count)) != sizeof(curr_count))
119
+
continue;
120
+
121
+
uint64_t delta = curr_count - prev_count;
122
+
prev_count = curr_count;
123
+
124
+
if (delta < mon->miss_threshold)
125
+
continue;
126
+
127
+
atomic_fetch_add(&mon->eviction_count, delta);
128
+
}
129
+
130
+
unsigned fast_probes = 0;
131
+
while (fast_probes < MONITOR_PROBE_CONFIRM_COUNT && atomic_load(&mon->running)) {
132
+
nanosleep_relative(mon->probe_interval_ns);
133
+
134
+
uint64_t lat = probe_latency(r->ptr, r->size);
135
+
if (lat > MONITOR_L3_LATENCY_THRESH_NS) {
136
+
reprime_region(mon);
137
+
break;
138
+
}
139
+
fast_probes++;
140
+
}
141
+
}
142
+
143
+
return NULL;
144
+
}
145
+
146
+
static int find_ccx_cpu(icepick_region_t *region)
147
+
{
148
+
int numa = region->numa_node;
149
+
if (numa < 0)
150
+
numa = 0;
151
+
152
+
char path[128];
153
+
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa);
154
+
155
+
FILE *f = fopen(path, "r");
156
+
if (!f)
157
+
return 0;
158
+
159
+
int cpu = 0;
160
+
fscanf(f, "%d", &cpu);
161
+
fclose(f);
162
+
163
+
return cpu;
164
+
}
165
+
166
+
int icepick_monitor_start_ex(icepick_region_t *region, uint64_t pmu_poll_ns,
167
+
uint64_t probe_ns, uint32_t miss_thresh,
168
+
icepick_monitor_t **out)
169
+
{
170
+
if (!region || !out)
171
+
return ICEPICK_E_INVALID;
172
+
173
+
icepick_monitor_t *mon = calloc(1, sizeof(*mon));
174
+
if (!mon)
175
+
return ICEPICK_E_ALLOC;
176
+
177
+
mon->region = region;
178
+
mon->pmu_poll_interval_ns = pmu_poll_ns ? pmu_poll_ns : MONITOR_DEFAULT_PMU_POLL_NS;
179
+
mon->probe_interval_ns = probe_ns ? probe_ns : MONITOR_DEFAULT_PROBE_NS;
180
+
mon->miss_threshold = miss_thresh ? miss_thresh : MONITOR_DEFAULT_MISS_THRESH;
181
+
182
+
mon->cpu = find_ccx_cpu(region);
183
+
mon->msr_fd = msr_open(mon->cpu);
184
+
if (mon->msr_fd < 0) {
185
+
free(mon);
186
+
return ICEPICK_E_MSR;
187
+
}
188
+
189
+
mon->perf_fd = monitor_perf_open(mon->cpu, is_amd());
190
+
mon->degraded = (mon->perf_fd < 0);
191
+
192
+
atomic_store(&mon->running, true);
193
+
atomic_store(&mon->eviction_count, 0);
194
+
atomic_store(&mon->reprime_count, 0);
195
+
196
+
if (pthread_create(&mon->thread, NULL, monitor_thread, mon) != 0) {
197
+
if (mon->perf_fd >= 0)
198
+
close(mon->perf_fd);
199
+
msr_close(mon->msr_fd);
200
+
free(mon);
201
+
return ICEPICK_E_THREAD;
202
+
}
203
+
204
+
*out = mon;
205
+
return 0;
206
+
}
207
+
208
+
int icepick_monitor_start(icepick_region_t *region, icepick_monitor_t **out)
209
+
{
210
+
return icepick_monitor_start_ex(region, 0, 0, 0, out);
211
+
}
212
+
213
+
int icepick_monitor_stop(icepick_monitor_t *mon)
214
+
{
215
+
if (!mon)
216
+
return ICEPICK_E_INVALID;
217
+
218
+
atomic_store(&mon->running, false);
219
+
pthread_join(mon->thread, NULL);
220
+
221
+
if (mon->perf_fd >= 0)
222
+
close(mon->perf_fd);
223
+
msr_close(mon->msr_fd);
224
+
free(mon);
225
+
226
+
return 0;
227
+
}
228
+
229
+
uint64_t icepick_monitor_eviction_count(const icepick_monitor_t *mon)
230
+
{
231
+
return mon ? atomic_load(&mon->eviction_count) : 0;
232
+
}
233
+
234
+
uint64_t icepick_monitor_reprime_count(const icepick_monitor_t *mon)
235
+
{
236
+
return mon ? atomic_load(&mon->reprime_count) : 0;
237
+
}
238
+
239
+
bool icepick_monitor_is_degraded(const icepick_monitor_t *mon)
240
+
{
241
+
return mon ? mon->degraded : true;
242
+
}