tools/testing/selftests/kvm/lib/x86_64/processor.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * tools/testing/selftests/kvm/lib/x86_64/processor.c
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7
   8 #include "linux/bitmap.h"
   9 #include "test_util.h"
  10 #include "kvm_util.h"
  11 #include "processor.h"
  12
  13 #ifndef NUM_INTERRUPTS
  14 #define NUM_INTERRUPTS 256
  15 #endif
  16
  17 #define DEFAULT_CODE_SELECTOR 0x8
  18 #define DEFAULT_DATA_SELECTOR 0x10
  19
  20 #define MAX_NR_CPUID_ENTRIES 100
  21
  22 vm_vaddr_t exception_handlers;
  23 bool host_cpu_is_amd;
  24 bool host_cpu_is_intel;
  25
  26 static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent)
  27 {
  28         fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
  29                 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
  30                 indent, "",
  31                 regs->rax, regs->rbx, regs->rcx, regs->rdx);
  32         fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
  33                 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
  34                 indent, "",
  35                 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
  36         fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
  37                 "r10: 0x%.16llx r11: 0x%.16llx\n",
  38                 indent, "",
  39                 regs->r8, regs->r9, regs->r10, regs->r11);
  40         fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
  41                 "r14: 0x%.16llx r15: 0x%.16llx\n",
  42                 indent, "",
  43                 regs->r12, regs->r13, regs->r14, regs->r15);
  44         fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
  45                 indent, "",
  46                 regs->rip, regs->rflags);
  47 }
  48
  49 static void segment_dump(FILE *stream, struct kvm_segment *segment,
  50                          uint8_t indent)
  51 {
  52         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
  53                 "selector: 0x%.4x type: 0x%.2x\n",
  54                 indent, "", segment->base, segment->limit,
  55                 segment->selector, segment->type);
  56         fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
  57                 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
  58                 indent, "", segment->present, segment->dpl,
  59                 segment->db, segment->s, segment->l);
  60         fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
  61                 "unusable: 0x%.2x padding: 0x%.2x\n",
  62                 indent, "", segment->g, segment->avl,
  63                 segment->unusable, segment->padding);
  64 }
  65
  66 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
  67                         uint8_t indent)
  68 {
  69         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
  70                 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
  71                 indent, "", dtable->base, dtable->limit,
  72                 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
  73 }
  74
  75 static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent)
  76 {
  77         unsigned int i;
  78
  79         fprintf(stream, "%*scs:\n", indent, "");
  80         segment_dump(stream, &sregs->cs, indent + 2);
  81         fprintf(stream, "%*sds:\n", indent, "");
  82         segment_dump(stream, &sregs->ds, indent + 2);
  83         fprintf(stream, "%*ses:\n", indent, "");
  84         segment_dump(stream, &sregs->es, indent + 2);
  85         fprintf(stream, "%*sfs:\n", indent, "");
  86         segment_dump(stream, &sregs->fs, indent + 2);
  87         fprintf(stream, "%*sgs:\n", indent, "");
  88         segment_dump(stream, &sregs->gs, indent + 2);
  89         fprintf(stream, "%*sss:\n", indent, "");
  90         segment_dump(stream, &sregs->ss, indent + 2);
  91         fprintf(stream, "%*str:\n", indent, "");
  92         segment_dump(stream, &sregs->tr, indent + 2);
  93         fprintf(stream, "%*sldt:\n", indent, "");
  94         segment_dump(stream, &sregs->ldt, indent + 2);
  95
  96         fprintf(stream, "%*sgdt:\n", indent, "");
  97         dtable_dump(stream, &sregs->gdt, indent + 2);
  98         fprintf(stream, "%*sidt:\n", indent, "");
  99         dtable_dump(stream, &sregs->idt, indent + 2);
 100
 101         fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
 102                 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
 103                 indent, "",
 104                 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
 105         fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
 106                 "apic_base: 0x%.16llx\n",
 107                 indent, "",
 108                 sregs->cr8, sregs->efer, sregs->apic_base);
 109
 110         fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
 111         for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
 112                 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
 113                         sregs->interrupt_bitmap[i]);
 114         }
 115 }
 116
 117 bool kvm_is_tdp_enabled(void)
 118 {
 119         if (host_cpu_is_intel)
 120                 return get_kvm_intel_param_bool("ept");
 121         else
 122                 return get_kvm_amd_param_bool("npt");
 123 }
 124
 125 void virt_arch_pgd_alloc(struct kvm_vm *vm)
 126 {
 127         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
 128                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 129
 130         /* If needed, create page map l4 table. */
 131         if (!vm->pgd_created) {
 132                 vm->pgd = vm_alloc_page_table(vm);
 133                 vm->pgd_created = true;
 134         }
 135 }
 136
 137 static void *virt_get_pte(struct kvm_vm *vm, uint64_t *parent_pte,
 138                           uint64_t vaddr, int level)
 139 {
 140         uint64_t pt_gpa = PTE_GET_PA(*parent_pte);
 141         uint64_t *page_table = addr_gpa2hva(vm, pt_gpa);
 142         int index = (vaddr >> PG_LEVEL_SHIFT(level)) & 0x1ffu;
 143
 144         TEST_ASSERT((*parent_pte & PTE_PRESENT_MASK) || parent_pte == &vm->pgd,
 145                     "Parent PTE (level %d) not PRESENT for gva: 0x%08lx",
 146                     level + 1, vaddr);
 147
 148         return &page_table[index];
 149 }
 150
 151 static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
 152                                        uint64_t *parent_pte,
 153                                        uint64_t vaddr,
 154                                        uint64_t paddr,
 155                                        int current_level,
 156                                        int target_level)
 157 {
 158         uint64_t *pte = virt_get_pte(vm, parent_pte, vaddr, current_level);
 159
 160         if (!(*pte & PTE_PRESENT_MASK)) {
 161                 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK;
 162                 if (current_level == target_level)
 163                         *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK);
 164                 else
 165                         *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
 166         } else {
 167                 /*
 168                  * Entry already present.  Assert that the caller doesn't want
 169                  * a hugepage at this level, and that there isn't a hugepage at
 170                  * this level.
 171                  */
 172                 TEST_ASSERT(current_level != target_level,
 173                             "Cannot create hugepage at level: %u, vaddr: 0x%lx\n",
 174                             current_level, vaddr);
 175                 TEST_ASSERT(!(*pte & PTE_LARGE_MASK),
 176                             "Cannot create page table at level: %u, vaddr: 0x%lx\n",
 177                             current_level, vaddr);
 178         }
 179         return pte;
 180 }
 181
 182 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level)
 183 {
 184         const uint64_t pg_size = PG_LEVEL_SIZE(level);
 185         uint64_t *pml4e, *pdpe, *pde;
 186         uint64_t *pte;
 187
 188         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,
 189                     "Unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 190
 191         TEST_ASSERT((vaddr % pg_size) == 0,
 192                     "Virtual address not aligned,\n"
 193                     "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size);
 194         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)),
 195                     "Invalid virtual address, vaddr: 0x%lx", vaddr);
 196         TEST_ASSERT((paddr % pg_size) == 0,
 197                     "Physical address not aligned,\n"
 198                     "  paddr: 0x%lx page size: 0x%lx", paddr, pg_size);
 199         TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
 200                     "Physical address beyond maximum supported,\n"
 201                     "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
 202                     paddr, vm->max_gfn, vm->page_size);
 203
 204         /*
 205          * Allocate upper level page tables, if not already present.  Return
 206          * early if a hugepage was created.
 207          */
 208         pml4e = virt_create_upper_pte(vm, &vm->pgd, vaddr, paddr, PG_LEVEL_512G, level);
 209         if (*pml4e & PTE_LARGE_MASK)
 210                 return;
 211
 212         pdpe = virt_create_upper_pte(vm, pml4e, vaddr, paddr, PG_LEVEL_1G, level);
 213         if (*pdpe & PTE_LARGE_MASK)
 214                 return;
 215
 216         pde = virt_create_upper_pte(vm, pdpe, vaddr, paddr, PG_LEVEL_2M, level);
 217         if (*pde & PTE_LARGE_MASK)
 218                 return;
 219
 220         /* Fill in page table entry. */
 221         pte = virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);
 222         TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),
 223                     "PTE already present for 4k page at vaddr: 0x%lx\n", vaddr);
 224         *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK);
 225 }
 226
 227 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
 228 {
 229         __virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K);
 230 }
 231
 232 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 233                     uint64_t nr_bytes, int level)
 234 {
 235         uint64_t pg_size = PG_LEVEL_SIZE(level);
 236         uint64_t nr_pages = nr_bytes / pg_size;
 237         int i;
 238
 239         TEST_ASSERT(nr_bytes % pg_size == 0,
 240                     "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx",
 241                     nr_bytes, pg_size);
 242
 243         for (i = 0; i < nr_pages; i++) {
 244                 __virt_pg_map(vm, vaddr, paddr, level);
 245
 246                 vaddr += pg_size;
 247                 paddr += pg_size;
 248         }
 249 }
 250
 251 static bool vm_is_target_pte(uint64_t *pte, int *level, int current_level)
 252 {
 253         if (*pte & PTE_LARGE_MASK) {
 254                 TEST_ASSERT(*level == PG_LEVEL_NONE ||
 255                             *level == current_level,
 256                             "Unexpected hugepage at level %d\n", current_level);
 257                 *level = current_level;
 258         }
 259
 260         return *level == current_level;
 261 }
 262
 263 uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr,
 264                                     int *level)
 265 {
 266         uint64_t *pml4e, *pdpe, *pde;
 267
 268         TEST_ASSERT(*level >= PG_LEVEL_NONE && *level < PG_LEVEL_NUM,
 269                     "Invalid PG_LEVEL_* '%d'", *level);
 270
 271         TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
 272                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 273         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
 274                 (vaddr >> vm->page_shift)),
 275                 "Invalid virtual address, vaddr: 0x%lx",
 276                 vaddr);
 277         /*
 278          * Based on the mode check above there are 48 bits in the vaddr, so
 279          * shift 16 to sign extend the last bit (bit-47),
 280          */
 281         TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16),
 282                 "Canonical check failed.  The virtual address is invalid.");
 283
 284         pml4e = virt_get_pte(vm, &vm->pgd, vaddr, PG_LEVEL_512G);
 285         if (vm_is_target_pte(pml4e, level, PG_LEVEL_512G))
 286                 return pml4e;
 287
 288         pdpe = virt_get_pte(vm, pml4e, vaddr, PG_LEVEL_1G);
 289         if (vm_is_target_pte(pdpe, level, PG_LEVEL_1G))
 290                 return pdpe;
 291
 292         pde = virt_get_pte(vm, pdpe, vaddr, PG_LEVEL_2M);
 293         if (vm_is_target_pte(pde, level, PG_LEVEL_2M))
 294                 return pde;
 295
 296         return virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);
 297 }
 298
 299 uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr)
 300 {
 301         int level = PG_LEVEL_4K;
 302
 303         return __vm_get_page_table_entry(vm, vaddr, &level);
 304 }
 305
 306 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 307 {
 308         uint64_t *pml4e, *pml4e_start;
 309         uint64_t *pdpe, *pdpe_start;
 310         uint64_t *pde, *pde_start;
 311         uint64_t *pte, *pte_start;
 312
 313         if (!vm->pgd_created)
 314                 return;
 315
 316         fprintf(stream, "%*s                                          "
 317                 "                no\n", indent, "");
 318         fprintf(stream, "%*s      index hvaddr         gpaddr         "
 319                 "addr         w exec dirty\n",
 320                 indent, "");
 321         pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd);
 322         for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
 323                 pml4e = &pml4e_start[n1];
 324                 if (!(*pml4e & PTE_PRESENT_MASK))
 325                         continue;
 326                 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "
 327                         " %u\n",
 328                         indent, "",
 329                         pml4e - pml4e_start, pml4e,
 330                         addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),
 331                         !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK));
 332
 333                 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
 334                 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
 335                         pdpe = &pdpe_start[n2];
 336                         if (!(*pdpe & PTE_PRESENT_MASK))
 337                                 continue;
 338                         fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10llx "
 339                                 "%u  %u\n",
 340                                 indent, "",
 341                                 pdpe - pdpe_start, pdpe,
 342                                 addr_hva2gpa(vm, pdpe),
 343                                 PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK),
 344                                 !!(*pdpe & PTE_NX_MASK));
 345
 346                         pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
 347                         for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
 348                                 pde = &pde_start[n3];
 349                                 if (!(*pde & PTE_PRESENT_MASK))
 350                                         continue;
 351                                 fprintf(stream, "%*spde   0x%-3zx %p "
 352                                         "0x%-12lx 0x%-10llx %u  %u\n",
 353                                         indent, "", pde - pde_start, pde,
 354                                         addr_hva2gpa(vm, pde),
 355                                         PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK),
 356                                         !!(*pde & PTE_NX_MASK));
 357
 358                                 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
 359                                 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
 360                                         pte = &pte_start[n4];
 361                                         if (!(*pte & PTE_PRESENT_MASK))
 362                                                 continue;
 363                                         fprintf(stream, "%*spte   0x%-3zx %p "
 364                                                 "0x%-12lx 0x%-10llx %u  %u "
 365                                                 "    %u    0x%-10lx\n",
 366                                                 indent, "",
 367                                                 pte - pte_start, pte,
 368                                                 addr_hva2gpa(vm, pte),
 369                                                 PTE_GET_PFN(*pte),
 370                                                 !!(*pte & PTE_WRITABLE_MASK),
 371                                                 !!(*pte & PTE_NX_MASK),
 372                                                 !!(*pte & PTE_DIRTY_MASK),
 373                                                 ((uint64_t) n1 << 27)
 374                                                         | ((uint64_t) n2 << 18)
 375                                                         | ((uint64_t) n3 << 9)
 376                                                         | ((uint64_t) n4));
 377                                 }
 378                         }
 379                 }
 380         }
 381 }
 382
 383 /*
 384  * Set Unusable Segment
 385  *
 386  * Input Args: None
 387  *
 388  * Output Args:
 389  *   segp - Pointer to segment register
 390  *
 391  * Return: None
 392  *
 393  * Sets the segment register pointed to by @segp to an unusable state.
 394  */
 395 static void kvm_seg_set_unusable(struct kvm_segment *segp)
 396 {
 397         memset(segp, 0, sizeof(*segp));
 398         segp->unusable = true;
 399 }
 400
 401 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
 402 {
 403         void *gdt = addr_gva2hva(vm, vm->gdt);
 404         struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
 405
 406         desc->limit0 = segp->limit & 0xFFFF;
 407         desc->base0 = segp->base & 0xFFFF;
 408         desc->base1 = segp->base >> 16;
 409         desc->type = segp->type;
 410         desc->s = segp->s;
 411         desc->dpl = segp->dpl;
 412         desc->p = segp->present;
 413         desc->limit1 = segp->limit >> 16;
 414         desc->avl = segp->avl;
 415         desc->l = segp->l;
 416         desc->db = segp->db;
 417         desc->g = segp->g;
 418         desc->base2 = segp->base >> 24;
 419         if (!segp->s)
 420                 desc->base3 = segp->base >> 32;
 421 }
 422
 423
 424 /*
 425  * Set Long Mode Flat Kernel Code Segment
 426  *
 427  * Input Args:
 428  *   vm - VM whose GDT is being filled, or NULL to only write segp
 429  *   selector - selector value
 430  *
 431  * Output Args:
 432  *   segp - Pointer to KVM segment
 433  *
 434  * Return: None
 435  *
 436  * Sets up the KVM segment pointed to by @segp, to be a code segment
 437  * with the selector value given by @selector.
 438  */
 439 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
 440         struct kvm_segment *segp)
 441 {
 442         memset(segp, 0, sizeof(*segp));
 443         segp->selector = selector;
 444         segp->limit = 0xFFFFFFFFu;
 445         segp->s = 0x1; /* kTypeCodeData */
 446         segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
 447                                           * | kFlagCodeReadable
 448                                           */
 449         segp->g = true;
 450         segp->l = true;
 451         segp->present = 1;
 452         if (vm)
 453                 kvm_seg_fill_gdt_64bit(vm, segp);
 454 }
 455
 456 /*
 457  * Set Long Mode Flat Kernel Data Segment
 458  *
 459  * Input Args:
 460  *   vm - VM whose GDT is being filled, or NULL to only write segp
 461  *   selector - selector value
 462  *
 463  * Output Args:
 464  *   segp - Pointer to KVM segment
 465  *
 466  * Return: None
 467  *
 468  * Sets up the KVM segment pointed to by @segp, to be a data segment
 469  * with the selector value given by @selector.
 470  */
 471 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
 472         struct kvm_segment *segp)
 473 {
 474         memset(segp, 0, sizeof(*segp));
 475         segp->selector = selector;
 476         segp->limit = 0xFFFFFFFFu;
 477         segp->s = 0x1; /* kTypeCodeData */
 478         segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
 479                                           * | kFlagDataWritable
 480                                           */
 481         segp->g = true;
 482         segp->present = true;
 483         if (vm)
 484                 kvm_seg_fill_gdt_64bit(vm, segp);
 485 }
 486
 487 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 488 {
 489         int level = PG_LEVEL_NONE;
 490         uint64_t *pte = __vm_get_page_table_entry(vm, gva, &level);
 491
 492         TEST_ASSERT(*pte & PTE_PRESENT_MASK,
 493                     "Leaf PTE not PRESENT for gva: 0x%08lx", gva);
 494
 495         /*
 496          * No need for a hugepage mask on the PTE, x86-64 requires the "unused"
 497          * address bits to be zero.
 498          */
 499         return PTE_GET_PA(*pte) | (gva & ~HUGEPAGE_MASK(level));
 500 }
 501
 502 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
 503 {
 504         if (!vm->gdt)
 505                 vm->gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
 506
 507         dt->base = vm->gdt;
 508         dt->limit = getpagesize();
 509 }
 510
 511 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
 512                                 int selector)
 513 {
 514         if (!vm->tss)
 515                 vm->tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
 516
 517         memset(segp, 0, sizeof(*segp));
 518         segp->base = vm->tss;
 519         segp->limit = 0x67;
 520         segp->selector = selector;
 521         segp->type = 0xb;
 522         segp->present = 1;
 523         kvm_seg_fill_gdt_64bit(vm, segp);
 524 }
 525
 526 static void vcpu_setup(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
 527 {
 528         struct kvm_sregs sregs;
 529
 530         /* Set mode specific system register values. */
 531         vcpu_sregs_get(vcpu, &sregs);
 532
 533         sregs.idt.limit = 0;
 534
 535         kvm_setup_gdt(vm, &sregs.gdt);
 536
 537         switch (vm->mode) {
 538         case VM_MODE_PXXV48_4K:
 539                 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
 540                 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
 541                 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
 542
 543                 kvm_seg_set_unusable(&sregs.ldt);
 544                 kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs);
 545                 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds);
 546                 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es);
 547                 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18);
 548                 break;
 549
 550         default:
 551                 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
 552         }
 553
 554         sregs.cr3 = vm->pgd;
 555         vcpu_sregs_set(vcpu, &sregs);
 556 }
 557
 558 void kvm_arch_vm_post_create(struct kvm_vm *vm)
 559 {
 560         vm_create_irqchip(vm);
 561         sync_global_to_guest(vm, host_cpu_is_intel);
 562         sync_global_to_guest(vm, host_cpu_is_amd);
 563 }
 564
 565 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
 566 {
 567         struct kvm_regs regs;
 568
 569         vcpu_regs_get(vcpu, &regs);
 570         regs.rip = (unsigned long) guest_code;
 571         vcpu_regs_set(vcpu, &regs);
 572 }
 573
 574 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
 575 {
 576         struct kvm_mp_state mp_state;
 577         struct kvm_regs regs;
 578         vm_vaddr_t stack_vaddr;
 579         struct kvm_vcpu *vcpu;
 580
 581         stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
 582                                        DEFAULT_GUEST_STACK_VADDR_MIN,
 583                                        MEM_REGION_DATA);
 584
 585         stack_vaddr += DEFAULT_STACK_PGS * getpagesize();
 586
 587         /*
 588          * Align stack to match calling sequence requirements in section "The
 589          * Stack Frame" of the System V ABI AMD64 Architecture Processor
 590          * Supplement, which requires the value (%rsp + 8) to be a multiple of
 591          * 16 when control is transferred to the function entry point.
 592          *
 593          * If this code is ever used to launch a vCPU with 32-bit entry point it
 594          * may need to subtract 4 bytes instead of 8 bytes.
 595          */
 596         TEST_ASSERT(IS_ALIGNED(stack_vaddr, PAGE_SIZE),
 597                     "__vm_vaddr_alloc() did not provide a page-aligned address");
 598         stack_vaddr -= 8;
 599
 600         vcpu = __vm_vcpu_add(vm, vcpu_id);
 601         vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
 602         vcpu_setup(vm, vcpu);
 603
 604         /* Setup guest general purpose registers */
 605         vcpu_regs_get(vcpu, &regs);
 606         regs.rflags = regs.rflags | 0x2;
 607         regs.rsp = stack_vaddr;
 608         vcpu_regs_set(vcpu, &regs);
 609
 610         /* Setup the MP state */
 611         mp_state.mp_state = 0;
 612         vcpu_mp_state_set(vcpu, &mp_state);
 613
 614         return vcpu;
 615 }
 616
 617 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id)
 618 {
 619         struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
 620
 621         vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
 622
 623         return vcpu;
 624 }
 625
 626 void vcpu_arch_free(struct kvm_vcpu *vcpu)
 627 {
 628         if (vcpu->cpuid)
 629                 free(vcpu->cpuid);
 630 }
 631
 632 /* Do not use kvm_supported_cpuid directly except for validity checks. */
 633 static void *kvm_supported_cpuid;
 634
 635 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
 636 {
 637         int kvm_fd;
 638
 639         if (kvm_supported_cpuid)
 640                 return kvm_supported_cpuid;
 641
 642         kvm_supported_cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
 643         kvm_fd = open_kvm_dev_path_or_exit();
 644
 645         kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID,
 646                   (struct kvm_cpuid2 *)kvm_supported_cpuid);
 647
 648         close(kvm_fd);
 649         return kvm_supported_cpuid;
 650 }
 651
 652 static uint32_t __kvm_cpu_has(const struct kvm_cpuid2 *cpuid,
 653                               uint32_t function, uint32_t index,
 654                               uint8_t reg, uint8_t lo, uint8_t hi)
 655 {
 656         const struct kvm_cpuid_entry2 *entry;
 657         int i;
 658
 659         for (i = 0; i < cpuid->nent; i++) {
 660                 entry = &cpuid->entries[i];
 661
 662                 /*
 663                  * The output registers in kvm_cpuid_entry2 are in alphabetical
 664                  * order, but kvm_x86_cpu_feature matches that mess, so yay
 665                  * pointer shenanigans!
 666                  */
 667                 if (entry->function == function && entry->index == index)
 668                         return ((&entry->eax)[reg] & GENMASK(hi, lo)) >> lo;
 669         }
 670
 671         return 0;
 672 }
 673
 674 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
 675                    struct kvm_x86_cpu_feature feature)
 676 {
 677         return __kvm_cpu_has(cpuid, feature.function, feature.index,
 678                              feature.reg, feature.bit, feature.bit);
 679 }
 680
 681 uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
 682                             struct kvm_x86_cpu_property property)
 683 {
 684         return __kvm_cpu_has(cpuid, property.function, property.index,
 685                              property.reg, property.lo_bit, property.hi_bit);
 686 }
 687
 688 uint64_t kvm_get_feature_msr(uint64_t msr_index)
 689 {
 690         struct {
 691                 struct kvm_msrs header;
 692                 struct kvm_msr_entry entry;
 693         } buffer = {};
 694         int r, kvm_fd;
 695
 696         buffer.header.nmsrs = 1;
 697         buffer.entry.index = msr_index;
 698         kvm_fd = open_kvm_dev_path_or_exit();
 699
 700         r = __kvm_ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
 701         TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_GET_MSRS, r));
 702
 703         close(kvm_fd);
 704         return buffer.entry.data;
 705 }
 706
 707 void __vm_xsave_require_permission(uint64_t xfeature, const char *name)
 708 {
 709         int kvm_fd;
 710         u64 bitmask;
 711         long rc;
 712         struct kvm_device_attr attr = {
 713                 .group = 0,
 714                 .attr = KVM_X86_XCOMP_GUEST_SUPP,
 715                 .addr = (unsigned long) &bitmask,
 716         };
 717
 718         TEST_ASSERT(!kvm_supported_cpuid,
 719                     "kvm_get_supported_cpuid() cannot be used before ARCH_REQ_XCOMP_GUEST_PERM");
 720
 721         TEST_ASSERT(is_power_of_2(xfeature),
 722                     "Dynamic XFeatures must be enabled one at a time");
 723
 724         kvm_fd = open_kvm_dev_path_or_exit();
 725         rc = __kvm_ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr);
 726         close(kvm_fd);
 727
 728         if (rc == -1 && (errno == ENXIO || errno == EINVAL))
 729                 __TEST_REQUIRE(0, "KVM_X86_XCOMP_GUEST_SUPP not supported");
 730
 731         TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc);
 732
 733         __TEST_REQUIRE(bitmask & xfeature,
 734                        "Required XSAVE feature '%s' not supported", name);
 735
 736         TEST_REQUIRE(!syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, ilog2(xfeature)));
 737
 738         rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask);
 739         TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc);
 740         TEST_ASSERT(bitmask & xfeature,
 741                     "'%s' (0x%lx) not permitted after prctl(ARCH_REQ_XCOMP_GUEST_PERM) permitted=0x%lx",
 742                     name, xfeature, bitmask);
 743 }
 744
 745 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid)
 746 {
 747         TEST_ASSERT(cpuid != vcpu->cpuid, "@cpuid can't be the vCPU's CPUID");
 748
 749         /* Allow overriding the default CPUID. */
 750         if (vcpu->cpuid && vcpu->cpuid->nent < cpuid->nent) {
 751                 free(vcpu->cpuid);
 752                 vcpu->cpuid = NULL;
 753         }
 754
 755         if (!vcpu->cpuid)
 756                 vcpu->cpuid = allocate_kvm_cpuid2(cpuid->nent);
 757
 758         memcpy(vcpu->cpuid, cpuid, kvm_cpuid2_size(cpuid->nent));
 759         vcpu_set_cpuid(vcpu);
 760 }
 761
 762 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr)
 763 {
 764         struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, 0x80000008);
 765
 766         entry->eax = (entry->eax & ~0xff) | maxphyaddr;
 767         vcpu_set_cpuid(vcpu);
 768 }
 769
 770 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function)
 771 {
 772         struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function);
 773
 774         entry->eax = 0;
 775         entry->ebx = 0;
 776         entry->ecx = 0;
 777         entry->edx = 0;
 778         vcpu_set_cpuid(vcpu);
 779 }
 780
 781 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
 782                                      struct kvm_x86_cpu_feature feature,
 783                                      bool set)
 784 {
 785         struct kvm_cpuid_entry2 *entry;
 786         u32 *reg;
 787
 788         entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index);
 789         reg = (&entry->eax) + feature.reg;
 790
 791         if (set)
 792                 *reg |= BIT(feature.bit);
 793         else
 794                 *reg &= ~BIT(feature.bit);
 795
 796         vcpu_set_cpuid(vcpu);
 797 }
 798
 799 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index)
 800 {
 801         struct {
 802                 struct kvm_msrs header;
 803                 struct kvm_msr_entry entry;
 804         } buffer = {};
 805
 806         buffer.header.nmsrs = 1;
 807         buffer.entry.index = msr_index;
 808
 809         vcpu_msrs_get(vcpu, &buffer.header);
 810
 811         return buffer.entry.data;
 812 }
 813
 814 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value)
 815 {
 816         struct {
 817                 struct kvm_msrs header;
 818                 struct kvm_msr_entry entry;
 819         } buffer = {};
 820
 821         memset(&buffer, 0, sizeof(buffer));
 822         buffer.header.nmsrs = 1;
 823         buffer.entry.index = msr_index;
 824         buffer.entry.data = msr_value;
 825
 826         return __vcpu_ioctl(vcpu, KVM_SET_MSRS, &buffer.header);
 827 }
 828
 829 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
 830 {
 831         va_list ap;
 832         struct kvm_regs regs;
 833
 834         TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
 835                     "  num: %u\n",
 836                     num);
 837
 838         va_start(ap, num);
 839         vcpu_regs_get(vcpu, &regs);
 840
 841         if (num >= 1)
 842                 regs.rdi = va_arg(ap, uint64_t);
 843
 844         if (num >= 2)
 845                 regs.rsi = va_arg(ap, uint64_t);
 846
 847         if (num >= 3)
 848                 regs.rdx = va_arg(ap, uint64_t);
 849
 850         if (num >= 4)
 851                 regs.rcx = va_arg(ap, uint64_t);
 852
 853         if (num >= 5)
 854                 regs.r8 = va_arg(ap, uint64_t);
 855
 856         if (num >= 6)
 857                 regs.r9 = va_arg(ap, uint64_t);
 858
 859         vcpu_regs_set(vcpu, &regs);
 860         va_end(ap);
 861 }
 862
 863 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent)
 864 {
 865         struct kvm_regs regs;
 866         struct kvm_sregs sregs;
 867
 868         fprintf(stream, "%*svCPU ID: %u\n", indent, "", vcpu->id);
 869
 870         fprintf(stream, "%*sregs:\n", indent + 2, "");
 871         vcpu_regs_get(vcpu, &regs);
 872         regs_dump(stream, &regs, indent + 4);
 873
 874         fprintf(stream, "%*ssregs:\n", indent + 2, "");
 875         vcpu_sregs_get(vcpu, &sregs);
 876         sregs_dump(stream, &sregs, indent + 4);
 877 }
 878
 879 static struct kvm_msr_list *__kvm_get_msr_index_list(bool feature_msrs)
 880 {
 881         struct kvm_msr_list *list;
 882         struct kvm_msr_list nmsrs;
 883         int kvm_fd, r;
 884
 885         kvm_fd = open_kvm_dev_path_or_exit();
 886
 887         nmsrs.nmsrs = 0;
 888         if (!feature_msrs)
 889                 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
 890         else
 891                 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, &nmsrs);
 892
 893         TEST_ASSERT(r == -1 && errno == E2BIG,
 894                     "Expected -E2BIG, got rc: %i errno: %i (%s)",
 895                     r, errno, strerror(errno));
 896
 897         list = malloc(sizeof(*list) + nmsrs.nmsrs * sizeof(list->indices[0]));
 898         TEST_ASSERT(list, "-ENOMEM when allocating MSR index list");
 899         list->nmsrs = nmsrs.nmsrs;
 900
 901         if (!feature_msrs)
 902                 kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
 903         else
 904                 kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, list);
 905         close(kvm_fd);
 906
 907         TEST_ASSERT(list->nmsrs == nmsrs.nmsrs,
 908                     "Number of MSRs in list changed, was %d, now %d",
 909                     nmsrs.nmsrs, list->nmsrs);
 910         return list;
 911 }
 912
 913 const struct kvm_msr_list *kvm_get_msr_index_list(void)
 914 {
 915         static const struct kvm_msr_list *list;
 916
 917         if (!list)
 918                 list = __kvm_get_msr_index_list(false);
 919         return list;
 920 }
 921
 922
 923 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void)
 924 {
 925         static const struct kvm_msr_list *list;
 926
 927         if (!list)
 928                 list = __kvm_get_msr_index_list(true);
 929         return list;
 930 }
 931
 932 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index)
 933 {
 934         const struct kvm_msr_list *list = kvm_get_msr_index_list();
 935         int i;
 936
 937         for (i = 0; i < list->nmsrs; ++i) {
 938                 if (list->indices[i] == msr_index)
 939                         return true;
 940         }
 941
 942         return false;
 943 }
 944
 945 static void vcpu_save_xsave_state(struct kvm_vcpu *vcpu,
 946                                   struct kvm_x86_state *state)
 947 {
 948         int size = vm_check_cap(vcpu->vm, KVM_CAP_XSAVE2);
 949
 950         if (size) {
 951                 state->xsave = malloc(size);
 952                 vcpu_xsave2_get(vcpu, state->xsave);
 953         } else {
 954                 state->xsave = malloc(sizeof(struct kvm_xsave));
 955                 vcpu_xsave_get(vcpu, state->xsave);
 956         }
 957 }
 958
 959 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu)
 960 {
 961         const struct kvm_msr_list *msr_list = kvm_get_msr_index_list();
 962         struct kvm_x86_state *state;
 963         int i;
 964
 965         static int nested_size = -1;
 966
 967         if (nested_size == -1) {
 968                 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
 969                 TEST_ASSERT(nested_size <= sizeof(state->nested_),
 970                             "Nested state size too big, %i > %zi",
 971                             nested_size, sizeof(state->nested_));
 972         }
 973
 974         /*
 975          * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
 976          * guest state is consistent only after userspace re-enters the
 977          * kernel with KVM_RUN.  Complete IO prior to migrating state
 978          * to a new VM.
 979          */
 980         vcpu_run_complete_io(vcpu);
 981
 982         state = malloc(sizeof(*state) + msr_list->nmsrs * sizeof(state->msrs.entries[0]));
 983         TEST_ASSERT(state, "-ENOMEM when allocating kvm state");
 984
 985         vcpu_events_get(vcpu, &state->events);
 986         vcpu_mp_state_get(vcpu, &state->mp_state);
 987         vcpu_regs_get(vcpu, &state->regs);
 988         vcpu_save_xsave_state(vcpu, state);
 989
 990         if (kvm_has_cap(KVM_CAP_XCRS))
 991                 vcpu_xcrs_get(vcpu, &state->xcrs);
 992
 993         vcpu_sregs_get(vcpu, &state->sregs);
 994
 995         if (nested_size) {
 996                 state->nested.size = sizeof(state->nested_);
 997
 998                 vcpu_nested_state_get(vcpu, &state->nested);
 999                 TEST_ASSERT(state->nested.size <= nested_size,
1000                             "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1001                             state->nested.size, nested_size);
1002         } else {
1003                 state->nested.size = 0;
1004         }
1005
1006         state->msrs.nmsrs = msr_list->nmsrs;
1007         for (i = 0; i < msr_list->nmsrs; i++)
1008                 state->msrs.entries[i].index = msr_list->indices[i];
1009         vcpu_msrs_get(vcpu, &state->msrs);
1010
1011         vcpu_debugregs_get(vcpu, &state->debugregs);
1012
1013         return state;
1014 }
1015
1016 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state)
1017 {
1018         vcpu_sregs_set(vcpu, &state->sregs);
1019         vcpu_msrs_set(vcpu, &state->msrs);
1020
1021         if (kvm_has_cap(KVM_CAP_XCRS))
1022                 vcpu_xcrs_set(vcpu, &state->xcrs);
1023
1024         vcpu_xsave_set(vcpu,  state->xsave);
1025         vcpu_events_set(vcpu, &state->events);
1026         vcpu_mp_state_set(vcpu, &state->mp_state);
1027         vcpu_debugregs_set(vcpu, &state->debugregs);
1028         vcpu_regs_set(vcpu, &state->regs);
1029
1030         if (state->nested.size)
1031                 vcpu_nested_state_set(vcpu, &state->nested);
1032 }
1033
1034 void kvm_x86_state_cleanup(struct kvm_x86_state *state)
1035 {
1036         free(state->xsave);
1037         free(state);
1038 }
1039
1040 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits)
1041 {
1042         if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) {
1043                 *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32;
1044                 *va_bits = 32;
1045         } else {
1046                 *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1047                 *va_bits = kvm_cpu_property(X86_PROPERTY_MAX_VIRT_ADDR);
1048         }
1049 }
1050
1051 static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr,
1052                           int dpl, unsigned short selector)
1053 {
1054         struct idt_entry *base =
1055                 (struct idt_entry *)addr_gva2hva(vm, vm->idt);
1056         struct idt_entry *e = &base[vector];
1057
1058         memset(e, 0, sizeof(*e));
1059         e->offset0 = addr;
1060         e->selector = selector;
1061         e->ist = 0;
1062         e->type = 14;
1063         e->dpl = dpl;
1064         e->p = 1;
1065         e->offset1 = addr >> 16;
1066         e->offset2 = addr >> 32;
1067 }
1068
1069
1070 static bool kvm_fixup_exception(struct ex_regs *regs)
1071 {
1072         if (regs->r9 != KVM_EXCEPTION_MAGIC || regs->rip != regs->r10)
1073                 return false;
1074
1075         if (regs->vector == DE_VECTOR)
1076                 return false;
1077
1078         regs->rip = regs->r11;
1079         regs->r9 = regs->vector;
1080         regs->r10 = regs->error_code;
1081         return true;
1082 }
1083
1084 void route_exception(struct ex_regs *regs)
1085 {
1086         typedef void(*handler)(struct ex_regs *);
1087         handler *handlers = (handler *)exception_handlers;
1088
1089         if (handlers && handlers[regs->vector]) {
1090                 handlers[regs->vector](regs);
1091                 return;
1092         }
1093
1094         if (kvm_fixup_exception(regs))
1095                 return;
1096
1097         ucall_assert(UCALL_UNHANDLED,
1098                      "Unhandled exception in guest", __FILE__, __LINE__,
1099                      "Unhandled exception '0x%lx' at guest RIP '0x%lx'",
1100                      regs->vector, regs->rip);
1101 }
1102
1103 void vm_init_descriptor_tables(struct kvm_vm *vm)
1104 {
1105         extern void *idt_handlers;
1106         int i;
1107
1108         vm->idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1109         vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1110         /* Handlers have the same address in both address spaces.*/
1111         for (i = 0; i < NUM_INTERRUPTS; i++)
1112                 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
1113                         DEFAULT_CODE_SELECTOR);
1114 }
1115
1116 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu)
1117 {
1118         struct kvm_vm *vm = vcpu->vm;
1119         struct kvm_sregs sregs;
1120
1121         vcpu_sregs_get(vcpu, &sregs);
1122         sregs.idt.base = vm->idt;
1123         sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1;
1124         sregs.gdt.base = vm->gdt;
1125         sregs.gdt.limit = getpagesize() - 1;
1126         kvm_seg_set_kernel_data_64bit(NULL, DEFAULT_DATA_SELECTOR, &sregs.gs);
1127         vcpu_sregs_set(vcpu, &sregs);
1128         *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
1129 }
1130
1131 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
1132                                void (*handler)(struct ex_regs *))
1133 {
1134         vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers);
1135
1136         handlers[vector] = (vm_vaddr_t)handler;
1137 }
1138
1139 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
1140 {
1141         struct ucall uc;
1142
1143         if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED)
1144                 REPORT_GUEST_ASSERT(uc);
1145 }
1146
1147 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
1148                                                uint32_t function, uint32_t index)
1149 {
1150         int i;
1151
1152         for (i = 0; i < cpuid->nent; i++) {
1153                 if (cpuid->entries[i].function == function &&
1154                     cpuid->entries[i].index == index)
1155                         return &cpuid->entries[i];
1156         }
1157
1158         TEST_FAIL("CPUID function 0x%x index 0x%x not found ", function, index);
1159
1160         return NULL;
1161 }
1162
1163 #define X86_HYPERCALL(inputs...)                                        \
1164 ({                                                                      \
1165         uint64_t r;                                                     \
1166                                                                         \
1167         asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t"          \
1168                      "jnz 1f\n\t"                                       \
1169                      "vmcall\n\t"                                       \
1170                      "jmp 2f\n\t"                                       \
1171                      "1: vmmcall\n\t"                                   \
1172                      "2:"                                               \
1173                      : "=a"(r)                                          \
1174                      : [use_vmmcall] "r" (host_cpu_is_amd), inputs);    \
1175                                                                         \
1176         r;                                                              \
1177 })
1178
1179 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
1180                        uint64_t a3)
1181 {
1182         return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1183 }
1184
1185 uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
1186 {
1187         return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1));
1188 }
1189
1190 void xen_hypercall(uint64_t nr, uint64_t a0, void *a1)
1191 {
1192         GUEST_ASSERT(!__xen_hypercall(nr, a0, a1));
1193 }
1194
1195 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
1196 {
1197         static struct kvm_cpuid2 *cpuid;
1198         int kvm_fd;
1199
1200         if (cpuid)
1201                 return cpuid;
1202
1203         cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
1204         kvm_fd = open_kvm_dev_path_or_exit();
1205
1206         kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1207
1208         close(kvm_fd);
1209         return cpuid;
1210 }
1211
1212 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu)
1213 {
1214         static struct kvm_cpuid2 *cpuid_full;
1215         const struct kvm_cpuid2 *cpuid_sys, *cpuid_hv;
1216         int i, nent = 0;
1217
1218         if (!cpuid_full) {
1219                 cpuid_sys = kvm_get_supported_cpuid();
1220                 cpuid_hv = kvm_get_supported_hv_cpuid();
1221
1222                 cpuid_full = allocate_kvm_cpuid2(cpuid_sys->nent + cpuid_hv->nent);
1223                 if (!cpuid_full) {
1224                         perror("malloc");
1225                         abort();
1226                 }
1227
1228                 /* Need to skip KVM CPUID leaves 0x400000xx */
1229                 for (i = 0; i < cpuid_sys->nent; i++) {
1230                         if (cpuid_sys->entries[i].function >= 0x40000000 &&
1231                             cpuid_sys->entries[i].function < 0x40000100)
1232                                 continue;
1233                         cpuid_full->entries[nent] = cpuid_sys->entries[i];
1234                         nent++;
1235                 }
1236
1237                 memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
1238                        cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
1239                 cpuid_full->nent = nent + cpuid_hv->nent;
1240         }
1241
1242         vcpu_init_cpuid(vcpu, cpuid_full);
1243 }
1244
1245 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu)
1246 {
1247         struct kvm_cpuid2 *cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
1248
1249         vcpu_ioctl(vcpu, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1250
1251         return cpuid;
1252 }
1253
1254 unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
1255 {
1256         const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
1257         unsigned long ht_gfn, max_gfn, max_pfn;
1258         uint8_t maxphyaddr;
1259
1260         max_gfn = (1ULL << (vm->pa_bits - vm->page_shift)) - 1;
1261
1262         /* Avoid reserved HyperTransport region on AMD processors.  */
1263         if (!host_cpu_is_amd)
1264                 return max_gfn;
1265
1266         /* On parts with <40 physical address bits, the area is fully hidden */
1267         if (vm->pa_bits < 40)
1268                 return max_gfn;
1269
1270         /* Before family 17h, the HyperTransport area is just below 1T.  */
1271         ht_gfn = (1 << 28) - num_ht_pages;
1272         if (this_cpu_family() < 0x17)
1273                 goto done;
1274
1275         /*
1276          * Otherwise it's at the top of the physical address space, possibly
1277          * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX.  Use
1278          * the old conservative value if MAXPHYADDR is not enumerated.
1279          */
1280         if (!this_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR))
1281                 goto done;
1282
1283         maxphyaddr = this_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1284         max_pfn = (1ULL << (maxphyaddr - vm->page_shift)) - 1;
1285
1286         if (this_cpu_has_p(X86_PROPERTY_PHYS_ADDR_REDUCTION))
1287                 max_pfn >>= this_cpu_property(X86_PROPERTY_PHYS_ADDR_REDUCTION);
1288
1289         ht_gfn = max_pfn - num_ht_pages;
1290 done:
1291         return min(max_gfn, ht_gfn - 1);
1292 }
1293
1294 /* Returns true if kvm_intel was loaded with unrestricted_guest=1. */
1295 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
1296 {
1297         /* Ensure that a KVM vendor-specific module is loaded. */
1298         if (vm == NULL)
1299                 close(open_kvm_dev_path_or_exit());
1300
1301         return get_kvm_intel_param_bool("unrestricted_guest");
1302 }
1303
1304 void kvm_selftest_arch_init(void)
1305 {
1306         host_cpu_is_intel = this_cpu_is_intel();
1307         host_cpu_is_amd = this_cpu_is_amd();
1308 }