vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <sched.h>
5#include <unistd.h>
6
7static uint32_t compute_way_mask(unsigned num_ways, unsigned start_way)
8{
9 uint32_t mask = 0;
10 for (unsigned i = 0; i < num_ways; i++)
11 mask |= (1U << (start_way + i));
12 return mask;
13}
14
15static unsigned find_start_way(uint32_t default_mask, unsigned num_ways)
16{
17 for (unsigned i = 0; i <= 32 - num_ways; i++) {
18 uint32_t candidate = compute_way_mask(num_ways, i);
19 if ((candidate & default_mask) == candidate)
20 return i;
21 }
22 return 0;
23}
24
25void prime_region(volatile char *ptr, size_t size, unsigned iterations)
26{
27 for (unsigned iter = 0; iter < iterations; iter++) {
28 for (size_t offset = 0; offset < size; offset += CACHE_LINE_SIZE) {
29 (void)ptr[offset];
30 }
31 }
32}
33
34int icepick_lock(icepick_topology_t *topo, const icepick_config_t *cfg,
35 icepick_region_t **region)
36{
37 if (!topo || !cfg || !region)
38 return ICEPICK_E_INVALID;
39
40 if (cfg->clos_id == 0 || cfg->clos_id >= topo->max_clos)
41 return ICEPICK_E_INVALID;
42
43 unsigned ways_needed = (cfg->size + topo->way_size - 1) / topo->way_size;
44 if (ways_needed == 0)
45 ways_needed = 1;
46
47 if (ways_needed >= topo->l3_ways)
48 return ICEPICK_E_TOO_LARGE;
49
50 int ret = clos_init(topo);
51 if (ret < 0)
52 return ret;
53
54 unsigned start_way = find_start_way(topo->default_way_mask, ways_needed);
55 uint32_t way_mask = compute_way_mask(ways_needed, start_way);
56
57 ret = clos_allocate(cfg->clos_id, way_mask);
58 if (ret < 0)
59 return ret;
60
61 icepick_region_t *r = calloc(1, sizeof(*r));
62 if (!r) {
63 clos_release(cfg->clos_id);
64 return ICEPICK_E_ALLOC;
65 }
66
67 r->clos_id = cfg->clos_id;
68 r->numa_node = cfg->numa_node;
69 r->way_mask = way_mask;
70 r->topo = topo;
71 r->size = ways_needed * topo->way_size;
72 r->monitor = NULL;
73
74 ret = region_alloc(r->size, cfg->numa_node, cfg->huge_pages, &r->ptr);
75 if (ret < 0) {
76 clos_release(cfg->clos_id);
77 free(r);
78 return ret;
79 }
80
81 int cpu = sched_getcpu();
82 if (cpu < 0)
83 cpu = 0;
84
85 cpu_set_t old_affinity, new_affinity;
86 CPU_ZERO(&new_affinity);
87 CPU_SET(cpu, &new_affinity);
88 sched_getaffinity(0, sizeof(old_affinity), &old_affinity);
89 sched_setaffinity(0, sizeof(new_affinity), &new_affinity);
90
91 int msr_fd = msr_open(cpu);
92 if (msr_fd < 0) {
93 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
94 region_free(r->ptr, r->size);
95 clos_release(cfg->clos_id);
96 free(r);
97 return msr_fd;
98 }
99
100 ret = clos_configure_mask(msr_fd, 0, clos_table[0].way_mask);
101 if (ret < 0)
102 goto cleanup;
103
104 ret = clos_configure_mask(msr_fd, cfg->clos_id, way_mask);
105 if (ret < 0)
106 goto cleanup;
107
108 ret = clos_associate_thread(msr_fd, cfg->clos_id);
109 if (ret < 0)
110 goto cleanup;
111
112 prime_region(r->ptr, r->size, 3);
113
114 clos_associate_thread(msr_fd, 0);
115
116 msr_close(msr_fd);
117 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
118
119 if (cfg->verify) {
120 icepick_latency_stats_t stats;
121 ret = icepick_verify(r, &stats);
122 if (ret < 0) {
123 icepick_unlock(r);
124 return ret;
125 }
126 }
127
128 if (cfg->auto_monitor) {
129 icepick_monitor_t *mon;
130 ret = icepick_monitor_start_ex(r, cfg->pmu_poll_interval_ns,
131 cfg->probe_interval_ns,
132 cfg->miss_threshold, &mon);
133 if (ret < 0) {
134 icepick_unlock(r);
135 return ret;
136 }
137 r->monitor = mon;
138 }
139
140 *region = r;
141 return 0;
142
143cleanup:
144 msr_close(msr_fd);
145 sched_setaffinity(0, sizeof(old_affinity), &old_affinity);
146 region_free(r->ptr, r->size);
147 clos_release(cfg->clos_id);
148 free(r);
149 return ret;
150}
151
152int icepick_unlock(icepick_region_t *region)
153{
154 if (!region)
155 return ICEPICK_E_INVALID;
156
157 if (region->monitor)
158 icepick_monitor_stop(region->monitor);
159
160 int cpu = sched_getcpu();
161 if (cpu < 0)
162 cpu = 0;
163
164 int msr_fd = msr_open(cpu);
165 if (msr_fd >= 0) {
166 clos_release(region->clos_id);
167 clos_configure_mask(msr_fd, region->clos_id, 0);
168 clos_configure_mask(msr_fd, 0, clos_get_default_mask());
169 msr_close(msr_fd);
170 }
171
172 region_free(region->ptr, region->size);
173 free(region);
174
175 return 0;
176}
177
178void *icepick_region_ptr(const icepick_region_t *region)
179{
180 return region ? region->ptr : NULL;
181}
182
183size_t icepick_region_size(const icepick_region_t *region)
184{
185 return region ? region->size : 0;
186}
187
188unsigned icepick_region_clos(const icepick_region_t *region)
189{
190 return region ? region->clos_id : 0;
191}
192
193const char *icepick_strerror(int err)
194{
195 switch (err) {
196 case 0: return "success";
197 case ICEPICK_E_NO_CAT: return "cat not supported on this cpu";
198 case ICEPICK_E_PERMISSION: return "permission denied (need root or cap_sys_rawio)";
199 case ICEPICK_E_NO_CLOS: return "no clos available or already allocated";
200 case ICEPICK_E_TOO_LARGE: return "requested size exceeds available cache ways";
201 case ICEPICK_E_NUMA: return "invalid numa node or binding failed";
202 case ICEPICK_E_HUGEPAGE: return "huge page allocation failed";
203 case ICEPICK_E_VERIFY: return "verification failed - data may not be cache-resident";
204 case ICEPICK_E_INVALID: return "invalid argument";
205 case ICEPICK_E_ALLOC: return "memory allocation failed";
206 case ICEPICK_E_MSR: return "msr operation failed";
207 case ICEPICK_E_THREAD: return "monitor thread creation failed";
208 default: return "unknown error";
209 }
210}