vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#ifndef ICEPICK_INTERNAL_H
2#define ICEPICK_INTERNAL_H
3
4#include "../include/icepick.h"
5#include <stdint.h>
6#include <stdatomic.h>
7#include <pthread.h>
8
9#define MSR_IA32_PQR_ASSOC 0xC8F
10#define MSR_IA32_L3_QOS_MASK_0 0xC90
11#define MSR_IA32_L2_QOS_CFG 0xC82
12#define MSR_IA32_MBA_THRTL_BASE 0xD50
13
14#define CACHE_LINE_SIZE 64
15#define MAX_CLOS 16
16#define MAX_CCXS 16
17#define MAX_CORE_TYPES 2
18
19#define MONITOR_DEFAULT_PMU_POLL_NS 100000
20#define MONITOR_DEFAULT_PROBE_NS 10000
21#define MONITOR_DEFAULT_MISS_THRESH 16
22#define MONITOR_PROBE_CONFIRM_COUNT 8
23#define MONITOR_L3_LATENCY_THRESH_NS 50
24
25struct icepick_monitor;
26
27struct icepick_topology {
28 bool cat_supported;
29 bool mba_supported;
30 bool mba_is_linear;
31 bool is_hybrid;
32 unsigned l3_ways;
33 size_t l3_size;
34 size_t way_size;
35 unsigned max_clos;
36 unsigned max_mba_thrtl;
37 unsigned ccx_count;
38 uint32_t default_way_mask;
39 int ccx_numa_node[MAX_CCXS];
40 unsigned pcore_count;
41 unsigned ecore_count;
42 int pcore_cpus[64];
43 int ecore_cpus[256];
44};
45
46struct icepick_region {
47 void *ptr;
48 size_t size;
49 unsigned clos_id;
50 int numa_node;
51 uint32_t way_mask;
52 unsigned mba_throttle;
53 icepick_core_type_t core_type;
54 icepick_topology_t *topo;
55 struct icepick_monitor *monitor;
56 icepick_prime_strategy_t prime_strategy;
57 icepick_access_pattern_t access_pattern;
58 size_t stride_bytes;
59 unsigned prime_iterations;
60};
61
62struct icepick_monitor {
63 icepick_region_t *region;
64 pthread_t thread;
65 int perf_fd;
66 int msr_fd;
67 int cpu;
68 atomic_bool running;
69 atomic_uint_fast64_t eviction_count;
70 atomic_uint_fast64_t reprime_count;
71 uint64_t pmu_poll_interval_ns;
72 uint64_t probe_interval_ns;
73 uint32_t miss_threshold;
74 bool degraded;
75};
76
77typedef struct {
78 bool allocated;
79 uint32_t way_mask;
80} clos_entry_t;
81
82extern clos_entry_t clos_table[MAX_CLOS];
83
84int msr_open(int cpu);
85void msr_close(int fd);
86int msr_read(int fd, uint32_t msr, uint64_t *value);
87int msr_write(int fd, uint32_t msr, uint64_t value);
88
89int clos_init(icepick_topology_t *topo);
90int clos_allocate(unsigned clos_id, uint32_t way_mask);
91int clos_release(unsigned clos_id);
92int clos_configure_mask(int msr_fd, unsigned clos_id, uint32_t way_mask);
93int clos_associate_thread(int msr_fd, unsigned clos_id);
94uint32_t clos_get_default_mask(void);
95
96int region_alloc(size_t size, int numa_node, bool huge_pages, void **out_ptr);
97void region_free(void *ptr, size_t size);
98
99uint64_t rdtsc_start(void);
100uint64_t rdtsc_end(void);
101uint64_t cycles_to_ns(uint64_t cycles);
102
103int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations);
104
105void prime_region(volatile char *ptr, size_t size, unsigned iterations,
106 icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
107 size_t stride);
108int monitor_perf_open(int cpu, bool is_amd);
109
110int mba_configure(int msr_fd, unsigned clos_id, unsigned throttle);
111int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type,
112 int numa_node);
113
114#endif