vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#define _GNU_SOURCE
2#include "internal.h"
3#include <stdlib.h>
4#include <string.h>
5#include <time.h>
6#include <x86intrin.h>
7
8#define VERIFY_SAMPLES 1000
9#define L3_THRESHOLD_NS 50
10
11static uint64_t tsc_freq_khz;
12
13uint64_t rdtsc_start(void)
14{
15 unsigned int aux;
16 _mm_lfence();
17 return __rdtscp(&aux);
18}
19
20uint64_t rdtsc_end(void)
21{
22 unsigned int aux;
23 uint64_t tsc = __rdtscp(&aux);
24 _mm_lfence();
25 return tsc;
26}
27
28static void calibrate_tsc(void)
29{
30 if (tsc_freq_khz != 0)
31 return;
32
33 struct timespec start, end;
34 clock_gettime(CLOCK_MONOTONIC, &start);
35 uint64_t tsc_start = rdtsc_start();
36
37 volatile unsigned long dummy = 0;
38 for (int i = 0; i < 10000000; i++)
39 dummy += i;
40 (void)dummy;
41
42 uint64_t tsc_end = rdtsc_end();
43 clock_gettime(CLOCK_MONOTONIC, &end);
44
45 uint64_t ns = (end.tv_sec - start.tv_sec) * 1000000000ULL +
46 (end.tv_nsec - start.tv_nsec);
47 uint64_t cycles = tsc_end - tsc_start;
48
49 tsc_freq_khz = (cycles * 1000000ULL) / ns;
50}
51
52uint64_t cycles_to_ns(uint64_t cycles)
53{
54 if (tsc_freq_khz == 0)
55 calibrate_tsc();
56 return (cycles * 1000000ULL) / tsc_freq_khz;
57}
58
59static int compare_u64(const void *a, const void *b)
60{
61 uint64_t va = *(const uint64_t *)a;
62 uint64_t vb = *(const uint64_t *)b;
63 return (va > vb) - (va < vb);
64}
65
66static uint64_t isqrt(uint64_t n)
67{
68 if (n == 0)
69 return 0;
70 uint64_t x = n;
71 uint64_t y = (x + 1) / 2;
72 while (y < x) {
73 x = y;
74 y = (x + n / x) / 2;
75 }
76 return x;
77}
78
79static void compute_stats(uint64_t *samples, size_t count,
80 icepick_latency_stats_t *stats)
81{
82 qsort(samples, count, sizeof(uint64_t), compare_u64);
83
84 uint64_t sum = 0;
85 for (size_t i = 0; i < count; i++)
86 sum += samples[i];
87
88 stats->mean_ns = sum / count;
89 stats->min_ns = samples[0];
90 stats->max_ns = samples[count - 1];
91 stats->p50_ns = samples[count / 2];
92 stats->p99_ns = samples[(count * 99) / 100];
93 stats->p999_ns = samples[(count * 999) / 1000];
94
95 uint64_t var_sum = 0;
96 for (size_t i = 0; i < count; i++) {
97 int64_t diff = samples[i] - stats->mean_ns;
98 var_sum += diff * diff;
99 }
100 stats->stddev_ns = isqrt(var_sum / count);
101}
102
103int icepick_verify(const icepick_region_t *region, icepick_latency_stats_t *stats)
104{
105 if (!region || !stats)
106 return ICEPICK_E_INVALID;
107
108 calibrate_tsc();
109
110 uint64_t *samples = malloc(VERIFY_SAMPLES * sizeof(uint64_t));
111 if (!samples)
112 return ICEPICK_E_ALLOC;
113
114 volatile char *ptr = region->ptr;
115 size_t size = region->size;
116
117 unsigned int seed = (unsigned int)time(NULL);
118 for (int i = 0; i < VERIFY_SAMPLES; i++) {
119 size_t offset = (rand_r(&seed) % (size / CACHE_LINE_SIZE)) * CACHE_LINE_SIZE;
120
121 uint64_t start = rdtsc_start();
122 (void)ptr[offset];
123 uint64_t end = rdtsc_end();
124
125 samples[i] = cycles_to_ns(end - start);
126 }
127
128 compute_stats(samples, VERIFY_SAMPLES, stats);
129 free(samples);
130
131 if (stats->mean_ns > L3_THRESHOLD_NS)
132 return ICEPICK_E_VERIFY;
133
134 return 0;
135}
136
137int icepick_bench(void *ptr, size_t size, size_t iterations,
138 icepick_latency_stats_t *stats)
139{
140 if (!ptr || !stats || size == 0 || iterations == 0)
141 return ICEPICK_E_INVALID;
142
143 calibrate_tsc();
144
145 size_t sample_count = iterations > 100000 ? 100000 : iterations;
146 uint64_t *samples = malloc(sample_count * sizeof(uint64_t));
147 if (!samples)
148 return ICEPICK_E_ALLOC;
149
150 volatile char *p = ptr;
151 size_t stride = CACHE_LINE_SIZE;
152 size_t sample_idx = 0;
153 size_t sample_interval = iterations / sample_count;
154 if (sample_interval == 0)
155 sample_interval = 1;
156
157 for (size_t i = 0; i < iterations; i++) {
158 size_t offset = (i * stride) % size;
159
160 uint64_t start = rdtsc_start();
161 (void)p[offset];
162 uint64_t end = rdtsc_end();
163
164 if (i % sample_interval == 0 && sample_idx < sample_count)
165 samples[sample_idx++] = cycles_to_ns(end - start);
166 }
167
168 compute_stats(samples, sample_idx, stats);
169 free(samples);
170
171 return 0;
172}