cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
implement prefetcht2/nt priming strategies with configurable access patterns for improved l3 residency
author
vm.fail
date
(Jan 18, 2026, 3:04 AM UTC)
commit
4163caec
4163caec75d82a77b0c94d849f104967ac2a047e
parent
a93f1852
a93f1852d96cd6e97d036d955c228dada9d0f0e2
vm.fail
date
(Jan 18, 2026, 3:04 AM UTC)
commit
4163caec
4163caec75d82a77b0c94d849f104967ac2a047e
parent
a93f1852
a93f1852d96cd6e97d036d955c228dada9d0f0e2
+249
-8
+43
-1
bindings/ocaml/icepick.ml
+43
-1
bindings/ocaml/icepick.ml
···
3
3
4
4
exception Icepick_error of string
5
5
6
+
type prime_strategy =
7
+
| Prime_temporal
8
+
| Prime_prefetcht2
9
+
| Prime_prefetchnta
10
+
| Prime_nt_store
11
+
12
+
type access_pattern =
13
+
| Pattern_sequential
14
+
| Pattern_reverse
15
+
| Pattern_strided
16
+
| Pattern_pointer_chase
17
+
6
18
let () = Callback.register_exception "Icepick_error" (Icepick_error "")
7
19
8
20
let () =
···
47
59
let numa_node = field t "numa_node" int
48
60
let huge_pages = field t "huge_pages" bool
49
61
let verify = field t "verify" bool
62
+
let auto_monitor = field t "auto_monitor" bool
63
+
let pmu_poll_interval_ns = field t "pmu_poll_interval_ns" uint64_t
64
+
let probe_interval_ns = field t "probe_interval_ns" uint64_t
65
+
let miss_threshold = field t "miss_threshold" uint32_t
66
+
let prime_strategy = field t "prime_strategy" int
67
+
let access_pattern = field t "access_pattern" int
68
+
let stride_bytes = field t "stride_bytes" size_t
69
+
let prime_iterations = field t "prime_iterations" uint
50
70
let () = seal t
51
71
end
52
72
···
152
172
t.released <- true
153
173
end
154
174
155
-
let lock topo ~size ~clos ?(numa = -1) ?(huge_pages = true) ?(verify = false) () =
175
+
let prime_strategy_to_int = function
176
+
| Prime_temporal -> 0
177
+
| Prime_prefetcht2 -> 1
178
+
| Prime_prefetchnta -> 2
179
+
| Prime_nt_store -> 3
180
+
181
+
let access_pattern_to_int = function
182
+
| Pattern_sequential -> 0
183
+
| Pattern_reverse -> 1
184
+
| Pattern_strided -> 2
185
+
| Pattern_pointer_chase -> 3
186
+
187
+
let lock topo ~size ~clos ?(numa = -1) ?(huge_pages = true) ?(verify = false)
188
+
?(prime_strategy = Prime_temporal) ?(access_pattern = Pattern_sequential)
189
+
?(stride = 0) ?(prime_iterations = 3) () =
156
190
let cfg = make C.Config.t in
157
191
setf cfg C.Config.size (Unsigned.Size_t.of_int size);
158
192
setf cfg C.Config.clos_id (Unsigned.UInt.of_int clos);
159
193
setf cfg C.Config.numa_node numa;
160
194
setf cfg C.Config.huge_pages huge_pages;
161
195
setf cfg C.Config.verify verify;
196
+
setf cfg C.Config.auto_monitor false;
197
+
setf cfg C.Config.pmu_poll_interval_ns (Unsigned.UInt64.of_int 0);
198
+
setf cfg C.Config.probe_interval_ns (Unsigned.UInt64.of_int 0);
199
+
setf cfg C.Config.miss_threshold (Unsigned.UInt32.of_int 0);
200
+
setf cfg C.Config.prime_strategy (prime_strategy_to_int prime_strategy);
201
+
setf cfg C.Config.access_pattern (access_pattern_to_int access_pattern);
202
+
setf cfg C.Config.stride_bytes (Unsigned.Size_t.of_int stride);
203
+
setf cfg C.Config.prime_iterations (Unsigned.UInt.of_int prime_iterations);
162
204
let ptr_ref = allocate C.Region.t (from_voidp void null) in
163
205
let ret = C.lock topo.Topology.ptr (addr cfg) ptr_ref in
164
206
C.check_error ret;
+15
-1
bindings/ocaml/icepick.mli
+15
-1
bindings/ocaml/icepick.mli
···
1
1
exception Icepick_error of string
2
2
3
+
type prime_strategy =
4
+
| Prime_temporal
5
+
| Prime_prefetcht2
6
+
| Prime_prefetchnta
7
+
| Prime_nt_store
8
+
9
+
type access_pattern =
10
+
| Pattern_sequential
11
+
| Pattern_reverse
12
+
| Pattern_strided
13
+
| Pattern_pointer_chase
14
+
3
15
module Topology : sig
4
16
type t
5
17
···
15
27
module Region : sig
16
28
type t
17
29
18
-
val lock : Topology.t -> size:int -> clos:int -> ?numa:int -> ?huge_pages:bool -> ?verify:bool -> unit -> t
30
+
val lock : Topology.t -> size:int -> clos:int -> ?numa:int -> ?huge_pages:bool -> ?verify:bool ->
31
+
?prime_strategy:prime_strategy -> ?access_pattern:access_pattern ->
32
+
?stride:int -> ?prime_iterations:int -> unit -> t
19
33
val unlock : t -> unit
20
34
val ptr : t -> nativeint
21
35
val size : t -> int
+18
include/icepick.h
+18
include/icepick.h
···
17
17
#define ICEPICK_E_MSR (-10)
18
18
#define ICEPICK_E_THREAD (-11)
19
19
20
+
typedef enum {
21
+
ICEPICK_PRIME_TEMPORAL,
22
+
ICEPICK_PRIME_PREFETCHT2,
23
+
ICEPICK_PRIME_PREFETCHNTA,
24
+
ICEPICK_PRIME_NT_STORE,
25
+
} icepick_prime_strategy_t;
26
+
27
+
typedef enum {
28
+
ICEPICK_PATTERN_SEQUENTIAL,
29
+
ICEPICK_PATTERN_REVERSE,
30
+
ICEPICK_PATTERN_STRIDED,
31
+
ICEPICK_PATTERN_POINTER_CHASE,
32
+
} icepick_access_pattern_t;
33
+
20
34
typedef struct icepick_topology icepick_topology_t;
21
35
typedef struct icepick_region icepick_region_t;
22
36
typedef struct icepick_monitor icepick_monitor_t;
···
31
45
uint64_t pmu_poll_interval_ns;
32
46
uint64_t probe_interval_ns;
33
47
uint32_t miss_threshold;
48
+
icepick_prime_strategy_t prime_strategy;
49
+
icepick_access_pattern_t access_pattern;
50
+
size_t stride_bytes;
51
+
unsigned prime_iterations;
34
52
} icepick_config_t;
35
53
36
54
typedef struct {
+7
-1
src/internal.h
+7
-1
src/internal.h
···
40
40
uint32_t way_mask;
41
41
icepick_topology_t *topo;
42
42
struct icepick_monitor *monitor;
43
+
icepick_prime_strategy_t prime_strategy;
44
+
icepick_access_pattern_t access_pattern;
45
+
size_t stride_bytes;
46
+
unsigned prime_iterations;
43
47
};
44
48
45
49
struct icepick_monitor {
···
85
89
86
90
int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations);
87
91
88
-
void prime_region(volatile char *ptr, size_t size, unsigned iterations);
92
+
void prime_region(volatile char *ptr, size_t size, unsigned iterations,
93
+
icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
94
+
size_t stride);
89
95
int monitor_perf_open(int cpu, bool is_amd);
90
96
91
97
#endif
+164
-4
src/lock.c
+164
-4
src/lock.c
···
3
3
#include <stdlib.h>
4
4
#include <sched.h>
5
5
#include <unistd.h>
6
+
#include <string.h>
7
+
#include <x86intrin.h>
6
8
7
9
static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way)
8
10
{
···
22
24
return 0;
23
25
}
24
26
25
-
void prime_region(volatile char *ptr, size_t size, unsigned iterations)
27
+
static void init_pointer_chase(volatile char *ptr, size_t size)
26
28
{
27
-
for (unsigned iter = 0; iter < iterations; iter++) {
28
-
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
29
+
size_t num_lines = size / CACHE_LINE_SIZE;
30
+
volatile size_t *indices = (volatile size_t *)ptr;
31
+
32
+
for (size_t i = 0; i < num_lines; i++)
33
+
indices[i * (CACHE_LINE_SIZE / sizeof(size_t))] = ((i + 1) % num_lines) * CACHE_LINE_SIZE;
34
+
35
+
for (size_t i = num_lines - 1; i > 0; i--) {
36
+
size_t j = (size_t)rdtsc_start() % (i + 1);
37
+
size_t idx_i = i * (CACHE_LINE_SIZE / sizeof(size_t));
38
+
size_t idx_j = j * (CACHE_LINE_SIZE / sizeof(size_t));
39
+
size_t tmp = indices[idx_i];
40
+
indices[idx_i] = indices[idx_j];
41
+
indices[idx_j] = tmp;
42
+
}
43
+
}
44
+
45
+
static void prime_temporal_sequential(volatile char *ptr, size_t size)
46
+
{
47
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
48
+
(void)ptr[offset];
49
+
}
50
+
51
+
static void prime_temporal_reverse(volatile char *ptr, size_t size)
52
+
{
53
+
for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
54
+
(void)ptr[offset - CACHE_LINE_SIZE];
55
+
}
56
+
57
+
static void prime_temporal_strided(volatile char *ptr, size_t size, size_t stride)
58
+
{
59
+
size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
60
+
for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
61
+
for (size_t offset = pass; offset < size; offset += effective_stride)
62
+
(void)ptr[offset];
63
+
}
64
+
}
65
+
66
+
static void prime_temporal_chase(volatile char *ptr, size_t size)
67
+
{
68
+
volatile size_t *indices = (volatile size_t *)ptr;
69
+
size_t num_lines = size / CACHE_LINE_SIZE;
70
+
size_t offset = 0;
71
+
72
+
for (size_t i = 0; i < num_lines; i++) {
73
+
offset = indices[offset / sizeof(size_t)];
74
+
}
75
+
76
+
(void)offset;
77
+
}
78
+
79
+
static void prime_prefetcht2_sequential(volatile char *ptr, size_t size)
80
+
{
81
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
82
+
_mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
83
+
_mm_mfence();
84
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
85
+
(void)ptr[offset];
86
+
}
87
+
88
+
static void prime_prefetcht2_reverse(volatile char *ptr, size_t size)
89
+
{
90
+
for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
91
+
_mm_prefetch((const char *)&ptr[offset - CACHE_LINE_SIZE], _MM_HINT_T2);
92
+
_mm_mfence();
93
+
for (size_t offset = size; offset > 0; offset -= CACHE_LINE_SIZE)
94
+
(void)ptr[offset - CACHE_LINE_SIZE];
95
+
}
96
+
97
+
static void prime_prefetcht2_strided(volatile char *ptr, size_t size, size_t stride)
98
+
{
99
+
size_t effective_stride = stride ? stride : (CACHE_LINE_SIZE * 8);
100
+
for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
101
+
for (size_t offset = pass; offset < size; offset += effective_stride)
102
+
_mm_prefetch((const char *)&ptr[offset], _MM_HINT_T2);
103
+
}
104
+
_mm_mfence();
105
+
for (size_t pass = 0; pass < effective_stride; pass += CACHE_LINE_SIZE) {
106
+
for (size_t offset = pass; offset < size; offset += effective_stride)
29
107
(void)ptr[offset];
108
+
}
109
+
}
110
+
111
+
static void prime_prefetchnta_sequential(volatile char *ptr, size_t size)
112
+
{
113
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
114
+
_mm_prefetch((const char *)&ptr[offset], _MM_HINT_NTA);
115
+
_mm_mfence();
116
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
117
+
(void)ptr[offset];
118
+
}
119
+
120
+
static void prime_nt_store_sequential(volatile char *ptr, size_t size)
121
+
{
122
+
__m128i zero = _mm_setzero_si128();
123
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
124
+
_mm_stream_si128((__m128i *)&ptr[offset], zero);
125
+
_mm_stream_si128((__m128i *)&ptr[offset + 16], zero);
126
+
_mm_stream_si128((__m128i *)&ptr[offset + 32], zero);
127
+
_mm_stream_si128((__m128i *)&ptr[offset + 48], zero);
128
+
}
129
+
_mm_sfence();
130
+
for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE)
131
+
(void)ptr[offset];
132
+
}
133
+
134
+
void prime_region(volatile char *ptr, size_t size, unsigned iterations,
135
+
icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
136
+
size_t stride)
137
+
{
138
+
if (pattern == ICEPICK_PATTERN_POINTER_CHASE)
139
+
init_pointer_chase(ptr, size);
140
+
141
+
for (unsigned iter = 0; iter < iterations; iter++) {
142
+
switch (strategy) {
143
+
case ICEPICK_PRIME_TEMPORAL:
144
+
switch (pattern) {
145
+
case ICEPICK_PATTERN_SEQUENTIAL:
146
+
prime_temporal_sequential(ptr, size);
147
+
break;
148
+
case ICEPICK_PATTERN_REVERSE:
149
+
prime_temporal_reverse(ptr, size);
150
+
break;
151
+
case ICEPICK_PATTERN_STRIDED:
152
+
prime_temporal_strided(ptr, size, stride);
153
+
break;
154
+
case ICEPICK_PATTERN_POINTER_CHASE:
155
+
prime_temporal_chase(ptr, size);
156
+
break;
157
+
}
158
+
break;
159
+
160
+
case ICEPICK_PRIME_PREFETCHT2:
161
+
switch (pattern) {
162
+
case ICEPICK_PATTERN_SEQUENTIAL:
163
+
prime_prefetcht2_sequential(ptr, size);
164
+
break;
165
+
case ICEPICK_PATTERN_REVERSE:
166
+
prime_prefetcht2_reverse(ptr, size);
167
+
break;
168
+
case ICEPICK_PATTERN_STRIDED:
169
+
prime_prefetcht2_strided(ptr, size, stride);
170
+
break;
171
+
case ICEPICK_PATTERN_POINTER_CHASE:
172
+
prime_prefetcht2_sequential(ptr, size);
173
+
prime_temporal_chase(ptr, size);
174
+
break;
175
+
}
176
+
break;
177
+
178
+
case ICEPICK_PRIME_PREFETCHNTA:
179
+
prime_prefetchnta_sequential(ptr, size);
180
+
break;
181
+
182
+
case ICEPICK_PRIME_NT_STORE:
183
+
prime_nt_store_sequential(ptr, size);
184
+
break;
30
185
}
31
186
}
32
187
}
···
70
225
r->topo = topo;
71
226
r->size = ways_needed * topo->way_size;
72
227
r->monitor = NULL;
228
+
r->prime_strategy = cfg->prime_strategy;
229
+
r->access_pattern = cfg->access_pattern;
230
+
r->stride_bytes = cfg->stride_bytes;
231
+
r->prime_iterations = cfg->prime_iterations ? cfg->prime_iterations : 3;
73
232
74
233
ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
75
234
if (ret < 0) {
···
109
268
if (ret < 0)
110
269
goto cleanup;
111
270
112
-
prime_region(r->ptr, r->size, 3);
271
+
prime_region(r->ptr, r->size, r->prime_iterations,
272
+
r->prime_strategy, r->access_pattern, r->stride_bytes);
113
273
114
274
clos_associate_thread(msr_fd, 0);
115
275
+2
-1
src/monitor.c
+2
-1
src/monitor.c
···
87
87
sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
88
88
89
89
clos_associate_thread(mon->msr_fd, r->clos_id);
90
-
prime_region(r->ptr, r->size, 3);
90
+
prime_region(r->ptr, r->size, r->prime_iterations,
91
+
r->prime_strategy, r->access_pattern, r->stride_bytes);
91
92
clos_associate_thread(mon->msr_fd, 0);
92
93
93
94
sched_setaffinity(0, sizeof(old_affinity), &old_affinity);