vm.fail
cache line locking on AMD x86_64 utilising L3 CAT pseudo-locking
1#ifndef ICEPICK_INTERNAL_H
2#define ICEPICK_INTERNAL_H
3
4#include "../include/icepick.h"
5#include <stdint.h>
6#include <stdatomic.h>
7#include <pthread.h>
8
9#define MSR_IA32_PQR_ASSOC 0xC8F
10#define MSR_IA32_L3_QOS_MASK_0 0xC90
11#define MSR_IA32_L2_QOS_CFG 0xC82
12#define MSR_IA32_MBA_THRTL_BASE 0xD50
13
14#define MPAM_PARTID_BITS 16
15#define MPAM_MAX_PARTID ((1 << MPAM_PARTID_BITS) - 1)
16
17#define CACHE_LINE_SIZE 64
18#define MAX_CLOS 16
19#define MAX_CCXS 16
20#define MAX_CORE_TYPES 2
21
22typedef enum {
23 ARCH_X86_64,
24 ARCH_ARM64,
25 ARCH_UNKNOWN
26} arch_type_t;
27
28#define MONITOR_DEFAULT_PMU_POLL_NS 100000
29#define MONITOR_DEFAULT_PROBE_NS 10000
30#define MONITOR_DEFAULT_MISS_THRESH 16
31#define MONITOR_PROBE_CONFIRM_COUNT 8
32#define MONITOR_L3_LATENCY_THRESH_NS 50
33
34struct icepick_monitor;
35
36struct icepick_topology {
37 arch_type_t arch;
38 bool cat_supported;
39 bool mba_supported;
40 bool mba_is_linear;
41 bool is_hybrid;
42 bool mpam_supported;
43 unsigned l3_ways;
44 size_t l3_size;
45 size_t way_size;
46 unsigned max_clos;
47 unsigned max_mba_thrtl;
48 unsigned max_partid;
49 unsigned ccx_count;
50 uint32_t default_way_mask;
51 int ccx_numa_node[MAX_CCXS];
52 unsigned pcore_count;
53 unsigned ecore_count;
54 int pcore_cpus[64];
55 int ecore_cpus[256];
56};
57
58struct icepick_region {
59 void *ptr;
60 size_t size;
61 unsigned clos_id;
62 int numa_node;
63 uint32_t way_mask;
64 unsigned mba_throttle;
65 icepick_core_type_t core_type;
66 icepick_topology_t *topo;
67 struct icepick_monitor *monitor;
68 icepick_prime_strategy_t prime_strategy;
69 icepick_access_pattern_t access_pattern;
70 size_t stride_bytes;
71 unsigned prime_iterations;
72};
73
74struct icepick_monitor {
75 icepick_region_t *region;
76 pthread_t thread;
77 int perf_fd;
78 int msr_fd;
79 int cpu;
80 atomic_bool running;
81 atomic_uint_fast64_t eviction_count;
82 atomic_uint_fast64_t reprime_count;
83 uint64_t pmu_poll_interval_ns;
84 uint64_t probe_interval_ns;
85 uint32_t miss_threshold;
86 bool degraded;
87};
88
89typedef struct {
90 bool allocated;
91 uint32_t way_mask;
92} clos_entry_t;
93
94extern clos_entry_t clos_table[MAX_CLOS];
95
96int msr_open(int cpu);
97void msr_close(int fd);
98int msr_read(int fd, uint32_t msr, uint64_t *value);
99int msr_write(int fd, uint32_t msr, uint64_t value);
100
101int clos_init(icepick_topology_t *topo);
102int clos_allocate(unsigned clos_id, uint32_t way_mask);
103int clos_release(unsigned clos_id);
104int clos_configure_mask(int msr_fd, unsigned clos_id, uint32_t way_mask);
105int clos_associate_thread(int msr_fd, unsigned clos_id);
106uint32_t clos_get_default_mask(void);
107
108int region_alloc(size_t size, int numa_node, bool huge_pages, void **out_ptr);
109void region_free(void *ptr, size_t size);
110
111uint64_t rdtsc_start(void);
112uint64_t rdtsc_end(void);
113uint64_t cycles_to_ns(uint64_t cycles);
114
115int bench_compare(icepick_topology_t *topo, size_t size, size_t iterations);
116
117void prime_region(volatile char *ptr, size_t size, unsigned iterations,
118 icepick_prime_strategy_t strategy, icepick_access_pattern_t pattern,
119 size_t stride);
120int monitor_perf_open(int cpu, bool is_amd);
121
122int mba_configure(int msr_fd, unsigned clos_id, unsigned throttle);
123int find_cpu_for_core_type(const icepick_topology_t *topo, icepick_core_type_t core_type,
124 int numa_node);
125
126arch_type_t detect_arch(void);
127int mpam_configure_partition(unsigned partid, uint32_t cpbm, unsigned mbw_max);
128int mpam_associate_thread(unsigned partid);
129
130#endif