vm.fail
hypervisor kernel integrity enforcement via EPT/NPT write interception for Xen/HVM guests
1#include <xen/sched.h>
2#include <xen/domain.h>
3#include <xen/guest_access.h>
4#include <xen/hypercall.h>
5#include <xen/mm.h>
6#include <xen/errno.h>
7#include <xen/lib.h>
8#include <asm/p2m.h>
9#include <asm/turnstile.h>
10
11static struct turnstile_domain_state *get_turnstile_state(struct domain *d)
12{
13 return d->arch.turnstile;
14}
15
16int turnstile_domain_init(struct domain *d)
17{
18 struct turnstile_domain_state *state;
19
20 state = xzalloc(struct turnstile_domain_state);
21 if ( !state )
22 return -ENOMEM;
23
24 rwlock_init(&state->lock);
25 state->policy = TURNSTILE_POLICY_DISABLED;
26 state->ring_head = 0;
27 state->ring_tail = 0;
28 state->ring_seq = 0;
29 state->overflow_count = 0;
30 state->num_ranges = 0;
31 state->func_entries = NULL;
32 state->num_func_entries = 0;
33 state->jump_entries = NULL;
34 state->num_jump_entries = 0;
35 state->write_grant.active = false;
36
37 d->arch.turnstile = state;
38
39 return 0;
40}
41
42void turnstile_domain_destroy(struct domain *d)
43{
44 struct turnstile_domain_state *state = get_turnstile_state(d);
45
46 if ( !state )
47 return;
48
49 xfree(state->func_entries);
50 xfree(state->jump_entries);
51 xfree(state);
52 d->arch.turnstile = NULL;
53}
54
55static bool gpa_in_range(struct turnstile_domain_state *state, uint64_t gpa)
56{
57 unsigned int i;
58
59 for ( i = 0; i < state->num_ranges; i++ )
60 {
61 if ( state->ranges[i].active &&
62 gpa >= state->ranges[i].gpa_start &&
63 gpa < state->ranges[i].gpa_end )
64 return true;
65 }
66 return false;
67}
68
69static bool check_write_grant(struct turnstile_domain_state *state, uint64_t gpa,
70 bool *grant_expired)
71{
72 *grant_expired = false;
73
74 if ( !state->write_grant.active )
75 return false;
76
77 if ( NOW() > state->write_grant.expiry )
78 {
79 *grant_expired = true;
80 return false;
81 }
82
83 return gpa >= state->write_grant.gpa_start &&
84 gpa < state->write_grant.gpa_end;
85}
86
87static int check_implicit_ftrace(struct turnstile_domain_state *state,
88 uint64_t gpa, const uint8_t *insn_bytes)
89{
90 unsigned int i;
91 uint64_t vaddr = gpa + 0xffff800000000000ULL;
92
93 for ( i = 0; i < state->num_func_entries; i++ )
94 {
95 if ( state->func_entries[i] == vaddr )
96 {
97 if ( insn_bytes[0] == 0xe8 )
98 return TURNSTILE_IMPLICIT_FTRACE;
99
100 if ( insn_bytes[0] == 0x0f && insn_bytes[1] == 0x1f &&
101 insn_bytes[2] == 0x44 && insn_bytes[3] == 0x00 &&
102 insn_bytes[4] == 0x00 )
103 return TURNSTILE_IMPLICIT_FTRACE;
104
105 if ( insn_bytes[0] == 0x90 ||
106 (insn_bytes[0] == 0x66 && insn_bytes[1] == 0x90) )
107 return TURNSTILE_IMPLICIT_FTRACE;
108 }
109 }
110 return TURNSTILE_IMPLICIT_DENIED;
111}
112
113static int check_implicit_static_key(struct turnstile_domain_state *state,
114 uint64_t gpa, const uint8_t *insn_bytes)
115{
116 unsigned int i;
117 uint64_t vaddr = gpa + 0xffff800000000000ULL;
118
119 for ( i = 0; i < state->num_jump_entries; i++ )
120 {
121 uint64_t code_addr = state->jump_entries[i * 3];
122 if ( code_addr == vaddr )
123 {
124 if ( insn_bytes[0] == 0x0f && insn_bytes[1] == 0x1f )
125 return TURNSTILE_IMPLICIT_STATIC;
126
127 if ( insn_bytes[0] == 0xe9 )
128 return TURNSTILE_IMPLICIT_STATIC;
129
130 if ( insn_bytes[0] == 0xeb )
131 return TURNSTILE_IMPLICIT_STATIC;
132
133 if ( insn_bytes[0] == 0x90 )
134 return TURNSTILE_IMPLICIT_STATIC;
135
136 if ( insn_bytes[0] == 0x66 && insn_bytes[1] == 0x90 )
137 return TURNSTILE_IMPLICIT_STATIC;
138 }
139 }
140 return TURNSTILE_IMPLICIT_DENIED;
141}
142
143static void log_violation(struct turnstile_domain_state *state,
144 uint64_t gpa, uint64_t rip, uint64_t cr3,
145 uint32_t access_flags, uint32_t response,
146 const uint8_t *insn_bytes)
147{
148 struct turnstile_violation *v;
149 uint32_t head;
150 uint32_t next_head;
151
152 head = state->ring_head;
153 next_head = (head + 1) % TURNSTILE_RING_SIZE;
154
155 if ( next_head == state->ring_tail )
156 {
157 state->overflow_count++;
158 state->stats.ring_overflows++;
159 state->ring_tail = (state->ring_tail + 1) % TURNSTILE_RING_SIZE;
160 }
161
162 v = &state->ring[head];
163 v->timestamp = NOW();
164 v->gpa = gpa;
165 v->rip = rip;
166 v->cr3 = cr3;
167 v->access_flags = access_flags;
168 v->response = response;
169 v->seq = state->ring_seq++;
170 memcpy(v->insn_bytes, insn_bytes, 16);
171
172 state->ring_head = next_head;
173 state->stats.violations_total++;
174
175 if ( response )
176 state->stats.violations_blocked++;
177 else
178 state->stats.violations_allowed++;
179}
180
181int turnstile_check_violation(struct domain *d, uint64_t gpa, uint64_t rip,
182 uint64_t cr3, uint32_t access_flags,
183 const uint8_t *insn_bytes)
184{
185 struct turnstile_domain_state *state = get_turnstile_state(d);
186 int implicit;
187 int block = 0;
188 bool in_range;
189 bool grant_valid;
190 bool grant_expired;
191 uint32_t policy;
192
193 if ( !state || state->policy == TURNSTILE_POLICY_DISABLED )
194 return 0;
195
196 read_lock(&state->lock);
197
198 in_range = gpa_in_range(state, gpa);
199 if ( !in_range )
200 {
201 read_unlock(&state->lock);
202 return 0;
203 }
204
205 grant_valid = check_write_grant(state, gpa, &grant_expired);
206 if ( grant_valid )
207 {
208 read_unlock(&state->lock);
209 write_lock(&state->lock);
210 state->stats.explicit_write_grants++;
211 write_unlock(&state->lock);
212 return 0;
213 }
214
215 implicit = check_implicit_ftrace(state, gpa, insn_bytes);
216 if ( implicit == TURNSTILE_IMPLICIT_FTRACE )
217 {
218 read_unlock(&state->lock);
219 write_lock(&state->lock);
220 state->stats.implicit_ftrace++;
221 write_unlock(&state->lock);
222 return 0;
223 }
224
225 implicit = check_implicit_static_key(state, gpa, insn_bytes);
226 if ( implicit == TURNSTILE_IMPLICIT_STATIC )
227 {
228 read_unlock(&state->lock);
229 write_lock(&state->lock);
230 state->stats.implicit_static_key++;
231 write_unlock(&state->lock);
232 return 0;
233 }
234
235 policy = state->policy;
236 read_unlock(&state->lock);
237
238 if ( policy == TURNSTILE_POLICY_ENFORCE )
239 block = 1;
240
241 write_lock(&state->lock);
242 if ( grant_expired )
243 state->write_grant.active = false;
244 log_violation(state, gpa, rip, cr3, access_flags, block, insn_bytes);
245 write_unlock(&state->lock);
246
247 return block;
248}
249
250static long turnstile_set_policy(struct domain *d,
251 XEN_GUEST_HANDLE_PARAM(void) arg)
252{
253 struct turnstile_domain_state *state = get_turnstile_state(d);
254 struct turnstile_op_set_policy op;
255
256 if ( !state )
257 return -EINVAL;
258
259 if ( copy_from_guest(&op, arg, 1) )
260 return -EFAULT;
261
262 if ( op.policy > TURNSTILE_POLICY_ENFORCE )
263 return -EINVAL;
264
265 write_lock(&state->lock);
266 state->policy = op.policy;
267 write_unlock(&state->lock);
268
269 return 0;
270}
271
272static long turnstile_protect_range(struct domain *d,
273 XEN_GUEST_HANDLE_PARAM(void) arg)
274{
275 struct turnstile_domain_state *state = get_turnstile_state(d);
276 struct turnstile_op_protect_range op;
277 unsigned int i;
278 int slot = -1;
279
280 if ( !state )
281 return -EINVAL;
282
283 if ( copy_from_guest(&op, arg, 1) )
284 return -EFAULT;
285
286 if ( op.length == 0 )
287 return -EINVAL;
288
289 write_lock(&state->lock);
290
291 for ( i = 0; i < state->num_ranges; i++ )
292 {
293 if ( state->ranges[i].active &&
294 state->ranges[i].gpa_start == op.gpa_start &&
295 state->ranges[i].gpa_end == op.gpa_start + op.length )
296 {
297 write_unlock(&state->lock);
298 return 0;
299 }
300 }
301
302 for ( i = 0; i < TURNSTILE_MAX_RANGES; i++ )
303 {
304 if ( !state->ranges[i].active )
305 {
306 slot = i;
307 break;
308 }
309 }
310
311 if ( slot < 0 )
312 {
313 write_unlock(&state->lock);
314 return -ENOSPC;
315 }
316
317 state->ranges[slot].gpa_start = op.gpa_start;
318 state->ranges[slot].gpa_end = op.gpa_start + op.length;
319 state->ranges[slot].active = true;
320 state->ranges[slot].original_type = p2m_ram_rw;
321
322 if ( (unsigned int)slot >= state->num_ranges )
323 state->num_ranges = slot + 1;
324
325 write_unlock(&state->lock);
326
327 return 0;
328}
329
330static long turnstile_unprotect_range(struct domain *d,
331 XEN_GUEST_HANDLE_PARAM(void) arg)
332{
333 struct turnstile_domain_state *state = get_turnstile_state(d);
334 struct turnstile_op_protect_range op;
335 unsigned int i;
336 bool found = false;
337
338 if ( !state )
339 return -EINVAL;
340
341 if ( copy_from_guest(&op, arg, 1) )
342 return -EFAULT;
343
344 write_lock(&state->lock);
345
346 for ( i = 0; i < state->num_ranges; i++ )
347 {
348 if ( state->ranges[i].active &&
349 state->ranges[i].gpa_start == op.gpa_start )
350 {
351 state->ranges[i].active = false;
352 found = true;
353 break;
354 }
355 }
356
357 write_unlock(&state->lock);
358
359 return found ? 0 : -ENOENT;
360}
361
362static long turnstile_request_write(struct domain *d,
363 XEN_GUEST_HANDLE_PARAM(void) arg)
364{
365 struct turnstile_domain_state *state = get_turnstile_state(d);
366 struct turnstile_op_request_write op;
367
368 if ( !state )
369 return -EINVAL;
370
371 if ( copy_from_guest(&op, arg, 1) )
372 return -EFAULT;
373
374 if ( op.timeout_ms > 1000 )
375 return -EINVAL;
376
377 if ( op.length == 0 )
378 return -EINVAL;
379
380 write_lock(&state->lock);
381
382 if ( !gpa_in_range(state, op.gpa_start) )
383 {
384 write_unlock(&state->lock);
385 return -ENOENT;
386 }
387
388 state->write_grant.gpa_start = op.gpa_start;
389 state->write_grant.gpa_end = op.gpa_start + op.length;
390 state->write_grant.expiry = NOW() + MILLISECS(op.timeout_ms);
391 state->write_grant.active = true;
392
393 write_unlock(&state->lock);
394
395 return 0;
396}
397
398static long turnstile_get_violations(struct domain *d,
399 XEN_GUEST_HANDLE_PARAM(void) arg)
400{
401 struct turnstile_domain_state *state = get_turnstile_state(d);
402 struct turnstile_op_get_violations op;
403 void *buf_ptr;
404 uint32_t count = 0;
405
406 if ( !state )
407 return -EINVAL;
408
409 if ( copy_from_guest(&op, arg, 1) )
410 return -EFAULT;
411
412 if ( op.max_entries == 0 )
413 return -EINVAL;
414
415 buf_ptr = (void *)(unsigned long)op.buffer_ptr;
416
417 write_lock(&state->lock);
418
419 while ( state->ring_tail != state->ring_head && count < op.max_entries )
420 {
421 struct turnstile_violation *v = &state->ring[state->ring_tail];
422 void *dest = buf_ptr + (count * sizeof(struct turnstile_violation));
423
424 if ( raw_copy_to_guest(dest, v, sizeof(struct turnstile_violation)) )
425 {
426 write_unlock(&state->lock);
427 return -EFAULT;
428 }
429
430 state->ring_tail = (state->ring_tail + 1) % TURNSTILE_RING_SIZE;
431 count++;
432 }
433
434 op.entries_returned = count;
435 op.overflow_count = state->overflow_count;
436 state->overflow_count = 0;
437
438 write_unlock(&state->lock);
439
440 if ( copy_to_guest(arg, &op, 1) )
441 return -EFAULT;
442
443 return 0;
444}
445
446static long turnstile_get_stats(struct domain *d,
447 XEN_GUEST_HANDLE_PARAM(void) arg)
448{
449 struct turnstile_domain_state *state = get_turnstile_state(d);
450 struct turnstile_stats stats;
451
452 if ( !state )
453 return -EINVAL;
454
455 read_lock(&state->lock);
456 stats = state->stats;
457 read_unlock(&state->lock);
458
459 if ( copy_to_guest(arg, &stats, 1) )
460 return -EFAULT;
461
462 return 0;
463}
464
465static long turnstile_upload_metadata(struct domain *d,
466 XEN_GUEST_HANDLE_PARAM(void) arg)
467{
468 struct turnstile_domain_state *state = get_turnstile_state(d);
469 struct turnstile_op_upload_metadata op;
470 void *buf_ptr;
471 uint64_t *new_entries;
472 unsigned int alloc_count;
473
474 if ( !state )
475 return -EINVAL;
476
477 if ( copy_from_guest(&op, arg, 1) )
478 return -EFAULT;
479
480 if ( op.count == 0 )
481 return -EINVAL;
482
483 if ( op.metadata_type == TURNSTILE_META_FUNC_ENTRIES )
484 {
485 if ( op.count > TURNSTILE_MAX_FUNC_ENTRIES )
486 return -EINVAL;
487
488 alloc_count = op.count;
489 }
490 else if ( op.metadata_type == TURNSTILE_META_JUMP_ENTRIES )
491 {
492 if ( op.count > TURNSTILE_MAX_JUMP_ENTRIES )
493 return -EINVAL;
494
495 alloc_count = op.count * 3;
496 }
497 else
498 {
499 return -EINVAL;
500 }
501
502 new_entries = xmalloc_array(uint64_t, alloc_count);
503 if ( !new_entries )
504 return -ENOMEM;
505
506 buf_ptr = (void *)(unsigned long)op.buffer_ptr;
507
508 if ( raw_copy_from_guest(new_entries, buf_ptr, alloc_count * sizeof(uint64_t)) )
509 {
510 xfree(new_entries);
511 return -EFAULT;
512 }
513
514 write_lock(&state->lock);
515
516 if ( op.metadata_type == TURNSTILE_META_FUNC_ENTRIES )
517 {
518 xfree(state->func_entries);
519 state->func_entries = new_entries;
520 state->num_func_entries = op.count;
521 }
522 else
523 {
524 xfree(state->jump_entries);
525 state->jump_entries = new_entries;
526 state->num_jump_entries = op.count;
527 }
528
529 write_unlock(&state->lock);
530
531 return 0;
532}
533
534long do_turnstile_op(unsigned int op, domid_t domid, XEN_GUEST_HANDLE_PARAM(void) arg)
535{
536 struct domain *d;
537 long ret;
538 bool self = false;
539
540 if ( domid == DOMID_SELF )
541 {
542 if ( op != TURNSTILE_OP_REQUEST_WRITE )
543 return -EPERM;
544
545 d = current->domain;
546 get_knownalive_domain(d);
547 self = true;
548 }
549 else
550 {
551 if ( !is_control_domain(current->domain) )
552 return -EPERM;
553
554 d = get_domain_by_id(domid);
555 if ( !d )
556 return -ESRCH;
557 }
558
559 if ( !is_hvm_domain(d) )
560 {
561 put_domain(d);
562 return -EINVAL;
563 }
564
565 if ( !get_turnstile_state(d) )
566 {
567 if ( self )
568 {
569 put_domain(d);
570 return -EINVAL;
571 }
572
573 ret = turnstile_domain_init(d);
574 if ( ret )
575 {
576 put_domain(d);
577 return ret;
578 }
579 }
580
581 switch ( op )
582 {
583 case TURNSTILE_OP_SET_POLICY:
584 ret = turnstile_set_policy(d, arg);
585 break;
586
587 case TURNSTILE_OP_PROTECT_RANGE:
588 ret = turnstile_protect_range(d, arg);
589 break;
590
591 case TURNSTILE_OP_UNPROTECT_RANGE:
592 ret = turnstile_unprotect_range(d, arg);
593 break;
594
595 case TURNSTILE_OP_REQUEST_WRITE:
596 ret = turnstile_request_write(d, arg);
597 break;
598
599 case TURNSTILE_OP_GET_VIOLATIONS:
600 ret = turnstile_get_violations(d, arg);
601 break;
602
603 case TURNSTILE_OP_GET_STATS:
604 ret = turnstile_get_stats(d, arg);
605 break;
606
607 case TURNSTILE_OP_UPLOAD_METADATA:
608 ret = turnstile_upload_metadata(d, arg);
609 break;
610
611 default:
612 ret = -ENOSYS;
613 }
614
615 put_domain(d);
616 return ret;
617}