tools/testing/selftests/kvm/lib/x86_64/processor.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * tools/testing/selftests/kvm/lib/x86_64/processor.c
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7
   8 #include "linux/bitmap.h"
   9 #include "test_util.h"
  10 #include "kvm_util.h"
  11 #include "processor.h"
  12
  13 #ifndef NUM_INTERRUPTS
  14 #define NUM_INTERRUPTS 256
  15 #endif
  16
  17 #define DEFAULT_CODE_SELECTOR 0x8
  18 #define DEFAULT_DATA_SELECTOR 0x10
  19
  20 #define MAX_NR_CPUID_ENTRIES 100
  21
  22 vm_vaddr_t exception_handlers;
  23 bool host_cpu_is_amd;
  24 bool host_cpu_is_intel;
  25
  26 static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent)
  27 {
  28         fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
  29                 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
  30                 indent, "",
  31                 regs->rax, regs->rbx, regs->rcx, regs->rdx);
  32         fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
  33                 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
  34                 indent, "",
  35                 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
  36         fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
  37                 "r10: 0x%.16llx r11: 0x%.16llx\n",
  38                 indent, "",
  39                 regs->r8, regs->r9, regs->r10, regs->r11);
  40         fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
  41                 "r14: 0x%.16llx r15: 0x%.16llx\n",
  42                 indent, "",
  43                 regs->r12, regs->r13, regs->r14, regs->r15);
  44         fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
  45                 indent, "",
  46                 regs->rip, regs->rflags);
  47 }
  48
  49 static void segment_dump(FILE *stream, struct kvm_segment *segment,
  50                          uint8_t indent)
  51 {
  52         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
  53                 "selector: 0x%.4x type: 0x%.2x\n",
  54                 indent, "", segment->base, segment->limit,
  55                 segment->selector, segment->type);
  56         fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
  57                 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
  58                 indent, "", segment->present, segment->dpl,
  59                 segment->db, segment->s, segment->l);
  60         fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
  61                 "unusable: 0x%.2x padding: 0x%.2x\n",
  62                 indent, "", segment->g, segment->avl,
  63                 segment->unusable, segment->padding);
  64 }
  65
  66 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
  67                         uint8_t indent)
  68 {
  69         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
  70                 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
  71                 indent, "", dtable->base, dtable->limit,
  72                 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
  73 }
  74
  75 static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent)
  76 {
  77         unsigned int i;
  78
  79         fprintf(stream, "%*scs:\n", indent, "");
  80         segment_dump(stream, &sregs->cs, indent + 2);
  81         fprintf(stream, "%*sds:\n", indent, "");
  82         segment_dump(stream, &sregs->ds, indent + 2);
  83         fprintf(stream, "%*ses:\n", indent, "");
  84         segment_dump(stream, &sregs->es, indent + 2);
  85         fprintf(stream, "%*sfs:\n", indent, "");
  86         segment_dump(stream, &sregs->fs, indent + 2);
  87         fprintf(stream, "%*sgs:\n", indent, "");
  88         segment_dump(stream, &sregs->gs, indent + 2);
  89         fprintf(stream, "%*sss:\n", indent, "");
  90         segment_dump(stream, &sregs->ss, indent + 2);
  91         fprintf(stream, "%*str:\n", indent, "");
  92         segment_dump(stream, &sregs->tr, indent + 2);
  93         fprintf(stream, "%*sldt:\n", indent, "");
  94         segment_dump(stream, &sregs->ldt, indent + 2);
  95
  96         fprintf(stream, "%*sgdt:\n", indent, "");
  97         dtable_dump(stream, &sregs->gdt, indent + 2);
  98         fprintf(stream, "%*sidt:\n", indent, "");
  99         dtable_dump(stream, &sregs->idt, indent + 2);
 100
 101         fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
 102                 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
 103                 indent, "",
 104                 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
 105         fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
 106                 "apic_base: 0x%.16llx\n",
 107                 indent, "",
 108                 sregs->cr8, sregs->efer, sregs->apic_base);
 109
 110         fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
 111         for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
 112                 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
 113                         sregs->interrupt_bitmap[i]);
 114         }
 115 }
 116
 117 bool kvm_is_tdp_enabled(void)
 118 {
 119         if (host_cpu_is_intel)
 120                 return get_kvm_intel_param_bool("ept");
 121         else
 122                 return get_kvm_amd_param_bool("npt");
 123 }
 124
 125 void virt_arch_pgd_alloc(struct kvm_vm *vm)
 126 {
 127         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
 128                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 129
 130         /* If needed, create page map l4 table. */
 131         if (!vm->pgd_created) {
 132                 vm->pgd = vm_alloc_page_table(vm);
 133                 vm->pgd_created = true;
 134         }
 135 }
 136
 137 static void *virt_get_pte(struct kvm_vm *vm, uint64_t *parent_pte,
 138                           uint64_t vaddr, int level)
 139 {
 140         uint64_t pt_gpa = PTE_GET_PA(*parent_pte);
 141         uint64_t *page_table = addr_gpa2hva(vm, pt_gpa);
 142         int index = (vaddr >> PG_LEVEL_SHIFT(level)) & 0x1ffu;
 143
 144         TEST_ASSERT((*parent_pte & PTE_PRESENT_MASK) || parent_pte == &vm->pgd,
 145                     "Parent PTE (level %d) not PRESENT for gva: 0x%08lx",
 146                     level + 1, vaddr);
 147
 148         return &page_table[index];
 149 }
 150
 151 static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
 152                                        uint64_t *parent_pte,
 153                                        uint64_t vaddr,
 154                                        uint64_t paddr,
 155                                        int current_level,
 156                                        int target_level)
 157 {
 158         uint64_t *pte = virt_get_pte(vm, parent_pte, vaddr, current_level);
 159
 160         if (!(*pte & PTE_PRESENT_MASK)) {
 161                 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK;
 162                 if (current_level == target_level)
 163                         *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK);
 164                 else
 165                         *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
 166         } else {
 167                 /*
 168                  * Entry already present.  Assert that the caller doesn't want
 169                  * a hugepage at this level, and that there isn't a hugepage at
 170                  * this level.
 171                  */
 172                 TEST_ASSERT(current_level != target_level,
 173                             "Cannot create hugepage at level: %u, vaddr: 0x%lx",
 174                             current_level, vaddr);
 175                 TEST_ASSERT(!(*pte & PTE_LARGE_MASK),
 176                             "Cannot create page table at level: %u, vaddr: 0x%lx",
 177                             current_level, vaddr);
 178         }
 179         return pte;
 180 }
 181
 182 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level)
 183 {
 184         const uint64_t pg_size = PG_LEVEL_SIZE(level);
 185         uint64_t *pml4e, *pdpe, *pde;
 186         uint64_t *pte;
 187
 188         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,
 189                     "Unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 190
 191         TEST_ASSERT((vaddr % pg_size) == 0,
 192                     "Virtual address not aligned,\n"
 193                     "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size);
 194         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)),
 195                     "Invalid virtual address, vaddr: 0x%lx", vaddr);
 196         TEST_ASSERT((paddr % pg_size) == 0,
 197                     "Physical address not aligned,\n"
 198                     "  paddr: 0x%lx page size: 0x%lx", paddr, pg_size);
 199         TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
 200                     "Physical address beyond maximum supported,\n"
 201                     "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
 202                     paddr, vm->max_gfn, vm->page_size);
 203
 204         /*
 205          * Allocate upper level page tables, if not already present.  Return
 206          * early if a hugepage was created.
 207          */
 208         pml4e = virt_create_upper_pte(vm, &vm->pgd, vaddr, paddr, PG_LEVEL_512G, level);
 209         if (*pml4e & PTE_LARGE_MASK)
 210                 return;
 211
 212         pdpe = virt_create_upper_pte(vm, pml4e, vaddr, paddr, PG_LEVEL_1G, level);
 213         if (*pdpe & PTE_LARGE_MASK)
 214                 return;
 215
 216         pde = virt_create_upper_pte(vm, pdpe, vaddr, paddr, PG_LEVEL_2M, level);
 217         if (*pde & PTE_LARGE_MASK)
 218                 return;
 219
 220         /* Fill in page table entry. */
 221         pte = virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);
 222         TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),
 223                     "PTE already present for 4k page at vaddr: 0x%lx", vaddr);
 224         *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK);
 225 }
 226
 227 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
 228 {
 229         __virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K);
 230 }
 231
 232 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 233                     uint64_t nr_bytes, int level)
 234 {
 235         uint64_t pg_size = PG_LEVEL_SIZE(level);
 236         uint64_t nr_pages = nr_bytes / pg_size;
 237         int i;
 238
 239         TEST_ASSERT(nr_bytes % pg_size == 0,
 240                     "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx",
 241                     nr_bytes, pg_size);
 242
 243         for (i = 0; i < nr_pages; i++) {
 244                 __virt_pg_map(vm, vaddr, paddr, level);
 245
 246                 vaddr += pg_size;
 247                 paddr += pg_size;
 248         }
 249 }
 250
 251 static bool vm_is_target_pte(uint64_t *pte, int *level, int current_level)
 252 {
 253         if (*pte & PTE_LARGE_MASK) {
 254                 TEST_ASSERT(*level == PG_LEVEL_NONE ||
 255                             *level == current_level,
 256                             "Unexpected hugepage at level %d", current_level);
 257                 *level = current_level;
 258         }
 259
 260         return *level == current_level;
 261 }
 262
 263 uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr,
 264                                     int *level)
 265 {
 266         uint64_t *pml4e, *pdpe, *pde;
 267
 268         TEST_ASSERT(*level >= PG_LEVEL_NONE && *level < PG_LEVEL_NUM,
 269                     "Invalid PG_LEVEL_* '%d'", *level);
 270
 271         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
 272                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 273         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
 274                 (vaddr >> vm->page_shift)),
 275                 "Invalid virtual address, vaddr: 0x%lx",
 276                 vaddr);
 277         /*
 278          * Based on the mode check above there are 48 bits in the vaddr, so
 279          * shift 16 to sign extend the last bit (bit-47),
 280          */
 281         TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16),
 282                 "Canonical check failed.  The virtual address is invalid.");
 283
 284         pml4e = virt_get_pte(vm, &vm->pgd, vaddr, PG_LEVEL_512G);
 285         if (vm_is_target_pte(pml4e, level, PG_LEVEL_512G))
 286                 return pml4e;
 287
 288         pdpe = virt_get_pte(vm, pml4e, vaddr, PG_LEVEL_1G);
 289         if (vm_is_target_pte(pdpe, level, PG_LEVEL_1G))
 290                 return pdpe;
 291
 292         pde = virt_get_pte(vm, pdpe, vaddr, PG_LEVEL_2M);
 293         if (vm_is_target_pte(pde, level, PG_LEVEL_2M))
 294                 return pde;
 295
 296         return virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);
 297 }
 298
 299 uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr)
 300 {
 301         int level = PG_LEVEL_4K;
 302
 303         return __vm_get_page_table_entry(vm, vaddr, &level);
 304 }
 305
 306 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 307 {
 308         uint64_t *pml4e, *pml4e_start;
 309         uint64_t *pdpe, *pdpe_start;
 310         uint64_t *pde, *pde_start;
 311         uint64_t *pte, *pte_start;
 312
 313         if (!vm->pgd_created)
 314                 return;
 315
 316         fprintf(stream, "%*s                                          "
 317                 "                no\n", indent, "");
 318         fprintf(stream, "%*s      index hvaddr         gpaddr         "
 319                 "addr         w exec dirty\n",
 320                 indent, "");
 321         pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd);
 322         for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
 323                 pml4e = &pml4e_start[n1];
 324                 if (!(*pml4e & PTE_PRESENT_MASK))
 325                         continue;
 326                 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "
 327                         " %u\n",
 328                         indent, "",
 329                         pml4e - pml4e_start, pml4e,
 330                         addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),
 331                         !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK));
 332
 333                 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
 334                 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
 335                         pdpe = &pdpe_start[n2];
 336                         if (!(*pdpe & PTE_PRESENT_MASK))
 337                                 continue;
 338                         fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10llx "
 339                                 "%u  %u\n",
 340                                 indent, "",
 341                                 pdpe - pdpe_start, pdpe,
 342                                 addr_hva2gpa(vm, pdpe),
 343                                 PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK),
 344                                 !!(*pdpe & PTE_NX_MASK));
 345
 346                         pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
 347                         for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
 348                                 pde = &pde_start[n3];
 349                                 if (!(*pde & PTE_PRESENT_MASK))
 350                                         continue;
 351                                 fprintf(stream, "%*spde   0x%-3zx %p "
 352                                         "0x%-12lx 0x%-10llx %u  %u\n",
 353                                         indent, "", pde - pde_start, pde,
 354                                         addr_hva2gpa(vm, pde),
 355                                         PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK),
 356                                         !!(*pde & PTE_NX_MASK));
 357
 358                                 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
 359                                 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
 360                                         pte = &pte_start[n4];
 361                                         if (!(*pte & PTE_PRESENT_MASK))
 362                                                 continue;
 363                                         fprintf(stream, "%*spte   0x%-3zx %p "
 364                                                 "0x%-12lx 0x%-10llx %u  %u "
 365                                                 "    %u    0x%-10lx\n",
 366                                                 indent, "",
 367                                                 pte - pte_start, pte,
 368                                                 addr_hva2gpa(vm, pte),
 369                                                 PTE_GET_PFN(*pte),
 370                                                 !!(*pte & PTE_WRITABLE_MASK),
 371                                                 !!(*pte & PTE_NX_MASK),
 372                                                 !!(*pte & PTE_DIRTY_MASK),
 373                                                 ((uint64_t) n1 << 27)
 374                                                         | ((uint64_t) n2 << 18)
 375                                                         | ((uint64_t) n3 << 9)
 376                                                         | ((uint64_t) n4));
 377                                 }
 378                         }
 379                 }
 380         }
 381 }
 382
 383 /*
 384  * Set Unusable Segment
 385  *
 386  * Input Args: None
 387  *
 388  * Output Args:
 389  *   segp - Pointer to segment register
 390  *
 391  * Return: None
 392  *
 393  * Sets the segment register pointed to by @segp to an unusable state.
 394  */
 395 static void kvm_seg_set_unusable(struct kvm_segment *segp)
 396 {
 397         memset(segp, 0, sizeof(*segp));
 398         segp->unusable = true;
 399 }
 400
 401 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
 402 {
 403         void *gdt = addr_gva2hva(vm, vm->gdt);
 404         struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
 405
 406         desc->limit0 = segp->limit & 0xFFFF;
 407         desc->base0 = segp->base & 0xFFFF;
 408         desc->base1 = segp->base >> 16;
 409         desc->type = segp->type;
 410         desc->s = segp->s;
 411         desc->dpl = segp->dpl;
 412         desc->p = segp->present;
 413         desc->limit1 = segp->limit >> 16;
 414         desc->avl = segp->avl;
 415         desc->l = segp->l;
 416         desc->db = segp->db;
 417         desc->g = segp->g;
 418         desc->base2 = segp->base >> 24;
 419         if (!segp->s)
 420                 desc->base3 = segp->base >> 32;
 421 }
 422
 423
 424 /*
 425  * Set Long Mode Flat Kernel Code Segment
 426  *
 427  * Input Args:
 428  *   vm - VM whose GDT is being filled, or NULL to only write segp
 429  *   selector - selector value
 430  *
 431  * Output Args:
 432  *   segp - Pointer to KVM segment
 433  *
 434  * Return: None
 435  *
 436  * Sets up the KVM segment pointed to by @segp, to be a code segment
 437  * with the selector value given by @selector.
 438  */
 439 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
 440         struct kvm_segment *segp)
 441 {
 442         memset(segp, 0, sizeof(*segp));
 443         segp->selector = selector;
 444         segp->limit = 0xFFFFFFFFu;
 445         segp->s = 0x1; /* kTypeCodeData */
 446         segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
 447                                           * | kFlagCodeReadable
 448                                           */
 449         segp->g = true;
 450         segp->l = true;
 451         segp->present = 1;
 452         if (vm)
 453                 kvm_seg_fill_gdt_64bit(vm, segp);
 454 }
 455
 456 /*
 457  * Set Long Mode Flat Kernel Data Segment
 458  *
 459  * Input Args:
 460  *   vm - VM whose GDT is being filled, or NULL to only write segp
 461  *   selector - selector value
 462  *
 463  * Output Args:
 464  *   segp - Pointer to KVM segment
 465  *
 466  * Return: None
 467  *
 468  * Sets up the KVM segment pointed to by @segp, to be a data segment
 469  * with the selector value given by @selector.
 470  */
 471 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
 472         struct kvm_segment *segp)
 473 {
 474         memset(segp, 0, sizeof(*segp));
 475         segp->selector = selector;
 476         segp->limit = 0xFFFFFFFFu;
 477         segp->s = 0x1; /* kTypeCodeData */
 478         segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
 479                                           * | kFlagDataWritable
 480                                           */
 481         segp->g = true;
 482         segp->present = true;
 483         if (vm)
 484                 kvm_seg_fill_gdt_64bit(vm, segp);
 485 }
 486
 487 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 488 {
 489         int level = PG_LEVEL_NONE;
 490         uint64_t *pte = __vm_get_page_table_entry(vm, gva, &level);
 491
 492         TEST_ASSERT(*pte & PTE_PRESENT_MASK,
 493                     "Leaf PTE not PRESENT for gva: 0x%08lx", gva);
 494
 495         /*
 496          * No need for a hugepage mask on the PTE, x86-64 requires the "unused"
 497          * address bits to be zero.
 498          */
 499         return PTE_GET_PA(*pte) | (gva & ~HUGEPAGE_MASK(level));
 500 }
 501
 502 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
 503 {
 504         if (!vm->gdt)
 505                 vm->gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
 506
 507         dt->base = vm->gdt;
 508         dt->limit = getpagesize();
 509 }
 510
 511 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
 512                                 int selector)
 513 {
 514         if (!vm->tss)
 515                 vm->tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
 516
 517         memset(segp, 0, sizeof(*segp));
 518         segp->base = vm->tss;
 519         segp->limit = 0x67;
 520         segp->selector = selector;
 521         segp->type = 0xb;
 522         segp->present = 1;
 523         kvm_seg_fill_gdt_64bit(vm, segp);
 524 }
 525
 526 static void vcpu_setup(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
 527 {
 528         struct kvm_sregs sregs;
 529
 530         /* Set mode specific system register values. */
 531         vcpu_sregs_get(vcpu, &sregs);
 532
 533         sregs.idt.limit = 0;
 534
 535         kvm_setup_gdt(vm, &sregs.gdt);
 536
 537         switch (vm->mode) {
 538         case VM_MODE_PXXV48_4K:
 539                 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
 540                 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
 541                 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
 542
 543                 kvm_seg_set_unusable(&sregs.ldt);
 544                 kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs);
 545                 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds);
 546                 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es);
 547                 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18);
 548                 break;
 549
 550         default:
 551                 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
 552         }
 553
 554         sregs.cr3 = vm->pgd;
 555         vcpu_sregs_set(vcpu, &sregs);
 556 }
 557
 558 void kvm_arch_vm_post_create(struct kvm_vm *vm)
 559 {
 560         vm_create_irqchip(vm);
 561         sync_global_to_guest(vm, host_cpu_is_intel);
 562         sync_global_to_guest(vm, host_cpu_is_amd);
 563 }
 564
 565 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 566                                   void *guest_code)
 567 {
 568         struct kvm_mp_state mp_state;
 569         struct kvm_regs regs;
 570         vm_vaddr_t stack_vaddr;
 571         struct kvm_vcpu *vcpu;
 572
 573         stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
 574                                        DEFAULT_GUEST_STACK_VADDR_MIN,
 575                                        MEM_REGION_DATA);
 576
 577         stack_vaddr += DEFAULT_STACK_PGS * getpagesize();
 578
 579         /*
 580          * Align stack to match calling sequence requirements in section "The
 581          * Stack Frame" of the System V ABI AMD64 Architecture Processor
 582          * Supplement, which requires the value (%rsp + 8) to be a multiple of
 583          * 16 when control is transferred to the function entry point.
 584          *
 585          * If this code is ever used to launch a vCPU with 32-bit entry point it
 586          * may need to subtract 4 bytes instead of 8 bytes.
 587          */
 588         TEST_ASSERT(IS_ALIGNED(stack_vaddr, PAGE_SIZE),
 589                     "__vm_vaddr_alloc() did not provide a page-aligned address");
 590         stack_vaddr -= 8;
 591
 592         vcpu = __vm_vcpu_add(vm, vcpu_id);
 593         vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
 594         vcpu_setup(vm, vcpu);
 595
 596         /* Setup guest general purpose registers */
 597         vcpu_regs_get(vcpu, &regs);
 598         regs.rflags = regs.rflags | 0x2;
 599         regs.rsp = stack_vaddr;
 600         regs.rip = (unsigned long) guest_code;
 601         vcpu_regs_set(vcpu, &regs);
 602
 603         /* Setup the MP state */
 604         mp_state.mp_state = 0;
 605         vcpu_mp_state_set(vcpu, &mp_state);
 606
 607         return vcpu;
 608 }
 609
 610 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id)
 611 {
 612         struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
 613
 614         vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
 615
 616         return vcpu;
 617 }
 618
 619 void vcpu_arch_free(struct kvm_vcpu *vcpu)
 620 {
 621         if (vcpu->cpuid)
 622                 free(vcpu->cpuid);
 623 }
 624
 625 /* Do not use kvm_supported_cpuid directly except for validity checks. */
 626 static void *kvm_supported_cpuid;
 627
 628 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
 629 {
 630         int kvm_fd;
 631
 632         if (kvm_supported_cpuid)
 633                 return kvm_supported_cpuid;
 634
 635         kvm_supported_cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
 636         kvm_fd = open_kvm_dev_path_or_exit();
 637
 638         kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID,
 639                   (struct kvm_cpuid2 *)kvm_supported_cpuid);
 640
 641         close(kvm_fd);
 642         return kvm_supported_cpuid;
 643 }
 644
 645 static uint32_t __kvm_cpu_has(const struct kvm_cpuid2 *cpuid,
 646                               uint32_t function, uint32_t index,
 647                               uint8_t reg, uint8_t lo, uint8_t hi)
 648 {
 649         const struct kvm_cpuid_entry2 *entry;
 650         int i;
 651
 652         for (i = 0; i < cpuid->nent; i++) {
 653                 entry = &cpuid->entries[i];
 654
 655                 /*
 656                  * The output registers in kvm_cpuid_entry2 are in alphabetical
 657                  * order, but kvm_x86_cpu_feature matches that mess, so yay
 658                  * pointer shenanigans!
 659                  */
 660                 if (entry->function == function && entry->index == index)
 661                         return ((&entry->eax)[reg] & GENMASK(hi, lo)) >> lo;
 662         }
 663
 664         return 0;
 665 }
 666
 667 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
 668                    struct kvm_x86_cpu_feature feature)
 669 {
 670         return __kvm_cpu_has(cpuid, feature.function, feature.index,
 671                              feature.reg, feature.bit, feature.bit);
 672 }
 673
 674 uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
 675                             struct kvm_x86_cpu_property property)
 676 {
 677         return __kvm_cpu_has(cpuid, property.function, property.index,
 678                              property.reg, property.lo_bit, property.hi_bit);
 679 }
 680
 681 uint64_t kvm_get_feature_msr(uint64_t msr_index)
 682 {
 683         struct {
 684                 struct kvm_msrs header;
 685                 struct kvm_msr_entry entry;
 686         } buffer = {};
 687         int r, kvm_fd;
 688
 689         buffer.header.nmsrs = 1;
 690         buffer.entry.index = msr_index;
 691         kvm_fd = open_kvm_dev_path_or_exit();
 692
 693         r = __kvm_ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
 694         TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_GET_MSRS, r));
 695
 696         close(kvm_fd);
 697         return buffer.entry.data;
 698 }
 699
 700 void __vm_xsave_require_permission(uint64_t xfeature, const char *name)
 701 {
 702         int kvm_fd;
 703         u64 bitmask;
 704         long rc;
 705         struct kvm_device_attr attr = {
 706                 .group = 0,
 707                 .attr = KVM_X86_XCOMP_GUEST_SUPP,
 708                 .addr = (unsigned long) &bitmask,
 709         };
 710
 711         TEST_ASSERT(!kvm_supported_cpuid,
 712                     "kvm_get_supported_cpuid() cannot be used before ARCH_REQ_XCOMP_GUEST_PERM");
 713
 714         TEST_ASSERT(is_power_of_2(xfeature),
 715                     "Dynamic XFeatures must be enabled one at a time");
 716
 717         kvm_fd = open_kvm_dev_path_or_exit();
 718         rc = __kvm_ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr);
 719         close(kvm_fd);
 720
 721         if (rc == -1 && (errno == ENXIO || errno == EINVAL))
 722                 __TEST_REQUIRE(0, "KVM_X86_XCOMP_GUEST_SUPP not supported");
 723
 724         TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc);
 725
 726         __TEST_REQUIRE(bitmask & xfeature,
 727                        "Required XSAVE feature '%s' not supported", name);
 728
 729         TEST_REQUIRE(!syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, ilog2(xfeature)));
 730
 731         rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask);
 732         TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc);
 733         TEST_ASSERT(bitmask & xfeature,
 734                     "'%s' (0x%lx) not permitted after prctl(ARCH_REQ_XCOMP_GUEST_PERM) permitted=0x%lx",
 735                     name, xfeature, bitmask);
 736 }
 737
 738 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid)
 739 {
 740         TEST_ASSERT(cpuid != vcpu->cpuid, "@cpuid can't be the vCPU's CPUID");
 741
 742         /* Allow overriding the default CPUID. */
 743         if (vcpu->cpuid && vcpu->cpuid->nent < cpuid->nent) {
 744                 free(vcpu->cpuid);
 745                 vcpu->cpuid = NULL;
 746         }
 747
 748         if (!vcpu->cpuid)
 749                 vcpu->cpuid = allocate_kvm_cpuid2(cpuid->nent);
 750
 751         memcpy(vcpu->cpuid, cpuid, kvm_cpuid2_size(cpuid->nent));
 752         vcpu_set_cpuid(vcpu);
 753 }
 754
 755 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr)
 756 {
 757         struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, 0x80000008);
 758
 759         entry->eax = (entry->eax & ~0xff) | maxphyaddr;
 760         vcpu_set_cpuid(vcpu);
 761 }
 762
 763 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function)
 764 {
 765         struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function);
 766
 767         entry->eax = 0;
 768         entry->ebx = 0;
 769         entry->ecx = 0;
 770         entry->edx = 0;
 771         vcpu_set_cpuid(vcpu);
 772 }
 773
 774 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
 775                                      struct kvm_x86_cpu_feature feature,
 776                                      bool set)
 777 {
 778         struct kvm_cpuid_entry2 *entry;
 779         u32 *reg;
 780
 781         entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index);
 782         reg = (&entry->eax) + feature.reg;
 783
 784         if (set)
 785                 *reg |= BIT(feature.bit);
 786         else
 787                 *reg &= ~BIT(feature.bit);
 788
 789         vcpu_set_cpuid(vcpu);
 790 }
 791
 792 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index)
 793 {
 794         struct {
 795                 struct kvm_msrs header;
 796                 struct kvm_msr_entry entry;
 797         } buffer = {};
 798
 799         buffer.header.nmsrs = 1;
 800         buffer.entry.index = msr_index;
 801
 802         vcpu_msrs_get(vcpu, &buffer.header);
 803
 804         return buffer.entry.data;
 805 }
 806
 807 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value)
 808 {
 809         struct {
 810                 struct kvm_msrs header;
 811                 struct kvm_msr_entry entry;
 812         } buffer = {};
 813
 814         memset(&buffer, 0, sizeof(buffer));
 815         buffer.header.nmsrs = 1;
 816         buffer.entry.index = msr_index;
 817         buffer.entry.data = msr_value;
 818
 819         return __vcpu_ioctl(vcpu, KVM_SET_MSRS, &buffer.header);
 820 }
 821
 822 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
 823 {
 824         va_list ap;
 825         struct kvm_regs regs;
 826
 827         TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
 828                     "  num: %u",
 829                     num);
 830
 831         va_start(ap, num);
 832         vcpu_regs_get(vcpu, &regs);
 833
 834         if (num >= 1)
 835                 regs.rdi = va_arg(ap, uint64_t);
 836
 837         if (num >= 2)
 838                 regs.rsi = va_arg(ap, uint64_t);
 839
 840         if (num >= 3)
 841                 regs.rdx = va_arg(ap, uint64_t);
 842
 843         if (num >= 4)
 844                 regs.rcx = va_arg(ap, uint64_t);
 845
 846         if (num >= 5)
 847                 regs.r8 = va_arg(ap, uint64_t);
 848
 849         if (num >= 6)
 850                 regs.r9 = va_arg(ap, uint64_t);
 851
 852         vcpu_regs_set(vcpu, &regs);
 853         va_end(ap);
 854 }
 855
 856 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
 857 {
 858         struct kvm_regs regs;
 859         struct kvm_sregs sregs;
 860
 861         fprintf(stream, "%*svCPU ID: %u\n", indent, "", vcpu->id);
 862
 863         fprintf(stream, "%*sregs:\n", indent + 2, "");
 864         vcpu_regs_get(vcpu, &regs);
 865         regs_dump(stream, &regs, indent + 4);
 866
 867         fprintf(stream, "%*ssregs:\n", indent + 2, "");
 868         vcpu_sregs_get(vcpu, &sregs);
 869         sregs_dump(stream, &sregs, indent + 4);
 870 }
 871
 872 static struct kvm_msr_list *__kvm_get_msr_index_list(bool feature_msrs)
 873 {
 874         struct kvm_msr_list *list;
 875         struct kvm_msr_list nmsrs;
 876         int kvm_fd, r;
 877
 878         kvm_fd = open_kvm_dev_path_or_exit();
 879
 880         nmsrs.nmsrs = 0;
 881         if (!feature_msrs)
 882                 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
 883         else
 884                 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, &nmsrs);
 885
 886         TEST_ASSERT(r == -1 && errno == E2BIG,
 887                     "Expected -E2BIG, got rc: %i errno: %i (%s)",
 888                     r, errno, strerror(errno));
 889
 890         list = malloc(sizeof(*list) + nmsrs.nmsrs * sizeof(list->indices[0]));
 891         TEST_ASSERT(list, "-ENOMEM when allocating MSR index list");
 892         list->nmsrs = nmsrs.nmsrs;
 893
 894         if (!feature_msrs)
 895                 kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
 896         else
 897                 kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, list);
 898         close(kvm_fd);
 899
 900         TEST_ASSERT(list->nmsrs == nmsrs.nmsrs,
 901                     "Number of MSRs in list changed, was %d, now %d",
 902                     nmsrs.nmsrs, list->nmsrs);
 903         return list;
 904 }
 905
 906 const struct kvm_msr_list *kvm_get_msr_index_list(void)
 907 {
 908         static const struct kvm_msr_list *list;
 909
 910         if (!list)
 911                 list = __kvm_get_msr_index_list(false);
 912         return list;
 913 }
 914
 915
 916 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void)
 917 {
 918         static const struct kvm_msr_list *list;
 919
 920         if (!list)
 921                 list = __kvm_get_msr_index_list(true);
 922         return list;
 923 }
 924
 925 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index)
 926 {
 927         const struct kvm_msr_list *list = kvm_get_msr_index_list();
 928         int i;
 929
 930         for (i = 0; i < list->nmsrs; ++i) {
 931                 if (list->indices[i] == msr_index)
 932                         return true;
 933         }
 934
 935         return false;
 936 }
 937
 938 static void vcpu_save_xsave_state(struct kvm_vcpu *vcpu,
 939                                   struct kvm_x86_state *state)
 940 {
 941         int size = vm_check_cap(vcpu->vm, KVM_CAP_XSAVE2);
 942
 943         if (size) {
 944                 state->xsave = malloc(size);
 945                 vcpu_xsave2_get(vcpu, state->xsave);
 946         } else {
 947                 state->xsave = malloc(sizeof(struct kvm_xsave));
 948                 vcpu_xsave_get(vcpu, state->xsave);
 949         }
 950 }
 951
 952 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu)
 953 {
 954         const struct kvm_msr_list *msr_list = kvm_get_msr_index_list();
 955         struct kvm_x86_state *state;
 956         int i;
 957
 958         static int nested_size = -1;
 959
 960         if (nested_size == -1) {
 961                 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
 962                 TEST_ASSERT(nested_size <= sizeof(state->nested_),
 963                             "Nested state size too big, %i > %zi",
 964                             nested_size, sizeof(state->nested_));
 965         }
 966
 967         /*
 968          * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
 969          * guest state is consistent only after userspace re-enters the
 970          * kernel with KVM_RUN.  Complete IO prior to migrating state
 971          * to a new VM.
 972          */
 973         vcpu_run_complete_io(vcpu);
 974
 975         state = malloc(sizeof(*state) + msr_list->nmsrs * sizeof(state->msrs.entries[0]));
 976         TEST_ASSERT(state, "-ENOMEM when allocating kvm state");
 977
 978         vcpu_events_get(vcpu, &state->events);
 979         vcpu_mp_state_get(vcpu, &state->mp_state);
 980         vcpu_regs_get(vcpu, &state->regs);
 981         vcpu_save_xsave_state(vcpu, state);
 982
 983         if (kvm_has_cap(KVM_CAP_XCRS))
 984                 vcpu_xcrs_get(vcpu, &state->xcrs);
 985
 986         vcpu_sregs_get(vcpu, &state->sregs);
 987
 988         if (nested_size) {
 989                 state->nested.size = sizeof(state->nested_);
 990
 991                 vcpu_nested_state_get(vcpu, &state->nested);
 992                 TEST_ASSERT(state->nested.size <= nested_size,
 993                             "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
 994                             state->nested.size, nested_size);
 995         } else {
 996                 state->nested.size = 0;
 997         }
 998
 999         state->msrs.nmsrs = msr_list->nmsrs;
1000         for (i = 0; i < msr_list->nmsrs; i++)
1001                 state->msrs.entries[i].index = msr_list->indices[i];
1002         vcpu_msrs_get(vcpu, &state->msrs);
1003
1004         vcpu_debugregs_get(vcpu, &state->debugregs);
1005
1006         return state;
1007 }
1008
1009 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state)
1010 {
1011         vcpu_sregs_set(vcpu, &state->sregs);
1012         vcpu_msrs_set(vcpu, &state->msrs);
1013
1014         if (kvm_has_cap(KVM_CAP_XCRS))
1015                 vcpu_xcrs_set(vcpu, &state->xcrs);
1016
1017         vcpu_xsave_set(vcpu,  state->xsave);
1018         vcpu_events_set(vcpu, &state->events);
1019         vcpu_mp_state_set(vcpu, &state->mp_state);
1020         vcpu_debugregs_set(vcpu, &state->debugregs);
1021         vcpu_regs_set(vcpu, &state->regs);
1022
1023         if (state->nested.size)
1024                 vcpu_nested_state_set(vcpu, &state->nested);
1025 }
1026
1027 void kvm_x86_state_cleanup(struct kvm_x86_state *state)
1028 {
1029         free(state->xsave);
1030         free(state);
1031 }
1032
1033 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits)
1034 {
1035         if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) {
1036                 *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32;
1037                 *va_bits = 32;
1038         } else {
1039                 *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1040                 *va_bits = kvm_cpu_property(X86_PROPERTY_MAX_VIRT_ADDR);
1041         }
1042 }
1043
1044 static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr,
1045                           int dpl, unsigned short selector)
1046 {
1047         struct idt_entry *base =
1048                 (struct idt_entry *)addr_gva2hva(vm, vm->idt);
1049         struct idt_entry *e = &base[vector];
1050
1051         memset(e, 0, sizeof(*e));
1052         e->offset0 = addr;
1053         e->selector = selector;
1054         e->ist = 0;
1055         e->type = 14;
1056         e->dpl = dpl;
1057         e->p = 1;
1058         e->offset1 = addr >> 16;
1059         e->offset2 = addr >> 32;
1060 }
1061
1062
1063 static bool kvm_fixup_exception(struct ex_regs *regs)
1064 {
1065         if (regs->r9 != KVM_EXCEPTION_MAGIC || regs->rip != regs->r10)
1066                 return false;
1067
1068         if (regs->vector == DE_VECTOR)
1069                 return false;
1070
1071         regs->rip = regs->r11;
1072         regs->r9 = regs->vector;
1073         regs->r10 = regs->error_code;
1074         return true;
1075 }
1076
1077 void route_exception(struct ex_regs *regs)
1078 {
1079         typedef void(*handler)(struct ex_regs *);
1080         handler *handlers = (handler *)exception_handlers;
1081
1082         if (handlers && handlers[regs->vector]) {
1083                 handlers[regs->vector](regs);
1084                 return;
1085         }
1086
1087         if (kvm_fixup_exception(regs))
1088                 return;
1089
1090         ucall_assert(UCALL_UNHANDLED,
1091                      "Unhandled exception in guest", __FILE__, __LINE__,
1092                      "Unhandled exception '0x%lx' at guest RIP '0x%lx'",
1093                      regs->vector, regs->rip);
1094 }
1095
1096 void vm_init_descriptor_tables(struct kvm_vm *vm)
1097 {
1098         extern void *idt_handlers;
1099         int i;
1100
1101         vm->idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1102         vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1103         /* Handlers have the same address in both address spaces.*/
1104         for (i = 0; i < NUM_INTERRUPTS; i++)
1105                 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
1106                         DEFAULT_CODE_SELECTOR);
1107 }
1108
1109 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu)
1110 {
1111         struct kvm_vm *vm = vcpu->vm;
1112         struct kvm_sregs sregs;
1113
1114         vcpu_sregs_get(vcpu, &sregs);
1115         sregs.idt.base = vm->idt;
1116         sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1;
1117         sregs.gdt.base = vm->gdt;
1118         sregs.gdt.limit = getpagesize() - 1;
1119         kvm_seg_set_kernel_data_64bit(NULL, DEFAULT_DATA_SELECTOR, &sregs.gs);
1120         vcpu_sregs_set(vcpu, &sregs);
1121         *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
1122 }
1123
1124 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
1125                                void (*handler)(struct ex_regs *))
1126 {
1127         vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers);
1128
1129         handlers[vector] = (vm_vaddr_t)handler;
1130 }
1131
1132 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
1133 {
1134         struct ucall uc;
1135
1136         if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED)
1137                 REPORT_GUEST_ASSERT(uc);
1138 }
1139
1140 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
1141                                                uint32_t function, uint32_t index)
1142 {
1143         int i;
1144
1145         for (i = 0; i < cpuid->nent; i++) {
1146                 if (cpuid->entries[i].function == function &&
1147                     cpuid->entries[i].index == index)
1148                         return &cpuid->entries[i];
1149         }
1150
1151         TEST_FAIL("CPUID function 0x%x index 0x%x not found ", function, index);
1152
1153         return NULL;
1154 }
1155
1156 #define X86_HYPERCALL(inputs...)                                        \
1157 ({                                                                      \
1158         uint64_t r;                                                     \
1159                                                                         \
1160         asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t"          \
1161                      "jnz 1f\n\t"                                       \
1162                      "vmcall\n\t"                                       \
1163                      "jmp 2f\n\t"                                       \
1164                      "1: vmmcall\n\t"                                   \
1165                      "2:"                                               \
1166                      : "=a"(r)                                          \
1167                      : [use_vmmcall] "r" (host_cpu_is_amd), inputs);    \
1168                                                                         \
1169         r;                                                              \
1170 })
1171
1172 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
1173                        uint64_t a3)
1174 {
1175         return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1176 }
1177
1178 uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
1179 {
1180         return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1));
1181 }
1182
1183 void xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
1184 {
1185         GUEST_ASSERT(!__xen_hypercall(nr, a0, a1));
1186 }
1187
1188 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
1189 {
1190         static struct kvm_cpuid2 *cpuid;
1191         int kvm_fd;
1192
1193         if (cpuid)
1194                 return cpuid;
1195
1196         cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
1197         kvm_fd = open_kvm_dev_path_or_exit();
1198
1199         kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1200
1201         close(kvm_fd);
1202         return cpuid;
1203 }
1204
1205 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu)
1206 {
1207         static struct kvm_cpuid2 *cpuid_full;
1208         const struct kvm_cpuid2 *cpuid_sys, *cpuid_hv;
1209         int i, nent = 0;
1210
1211         if (!cpuid_full) {
1212                 cpuid_sys = kvm_get_supported_cpuid();
1213                 cpuid_hv = kvm_get_supported_hv_cpuid();
1214
1215                 cpuid_full = allocate_kvm_cpuid2(cpuid_sys->nent + cpuid_hv->nent);
1216                 if (!cpuid_full) {
1217                         perror("malloc");
1218                         abort();
1219                 }
1220
1221                 /* Need to skip KVM CPUID leaves 0x400000xx */
1222                 for (i = 0; i < cpuid_sys->nent; i++) {
1223                         if (cpuid_sys->entries[i].function >= 0x40000000 &&
1224                             cpuid_sys->entries[i].function < 0x40000100)
1225                                 continue;
1226                         cpuid_full->entries[nent] = cpuid_sys->entries[i];
1227                         nent++;
1228                 }
1229
1230                 memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
1231                        cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
1232                 cpuid_full->nent = nent + cpuid_hv->nent;
1233         }
1234
1235         vcpu_init_cpuid(vcpu, cpuid_full);
1236 }
1237
1238 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu)
1239 {
1240         struct kvm_cpuid2 *cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
1241
1242         vcpu_ioctl(vcpu, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1243
1244         return cpuid;
1245 }
1246
1247 unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
1248 {
1249         const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
1250         unsigned long ht_gfn, max_gfn, max_pfn;
1251         uint8_t maxphyaddr;
1252
1253         max_gfn = (1ULL << (vm->pa_bits - vm->page_shift)) - 1;
1254
1255         /* Avoid reserved HyperTransport region on AMD processors.  */
1256         if (!host_cpu_is_amd)
1257                 return max_gfn;
1258
1259         /* On parts with <40 physical address bits, the area is fully hidden */
1260         if (vm->pa_bits < 40)
1261                 return max_gfn;
1262
1263         /* Before family 17h, the HyperTransport area is just below 1T.  */
1264         ht_gfn = (1 << 28) - num_ht_pages;
1265         if (this_cpu_family() < 0x17)
1266                 goto done;
1267
1268         /*
1269          * Otherwise it's at the top of the physical address space, possibly
1270          * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX.  Use
1271          * the old conservative value if MAXPHYADDR is not enumerated.
1272          */
1273         if (!this_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR))
1274                 goto done;
1275
1276         maxphyaddr = this_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1277         max_pfn = (1ULL << (maxphyaddr - vm->page_shift)) - 1;
1278
1279         if (this_cpu_has_p(X86_PROPERTY_PHYS_ADDR_REDUCTION))
1280                 max_pfn >>= this_cpu_property(X86_PROPERTY_PHYS_ADDR_REDUCTION);
1281
1282         ht_gfn = max_pfn - num_ht_pages;
1283 done:
1284         return min(max_gfn, ht_gfn - 1);
1285 }
1286
1287 /* Returns true if kvm_intel was loaded with unrestricted_guest=1. */
1288 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
1289 {
1290         /* Ensure that a KVM vendor-specific module is loaded. */
1291         if (vm == NULL)
1292                 close(open_kvm_dev_path_or_exit());
1293
1294         return get_kvm_intel_param_bool("unrestricted_guest");
1295 }
1296
1297 void kvm_selftest_arch_init(void)
1298 {
1299         host_cpu_is_intel = this_cpu_is_intel();
1300         host_cpu_is_amd = this_cpu_is_amd();
1301 }
1302
1303 bool sys_clocksource_is_based_on_tsc(void)
1304 {
1305         char *clk_name = sys_get_cur_clocksource();
1306         bool ret = !strcmp(clk_name, "tsc\n") ||
1307                    !strcmp(clk_name, "hyperv_clocksource_tsc_page\n");
1308
1309         free(clk_name);
1310
1311         return ret;
1312 }