arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/perf_event.h>
  36 #include <asm/tlbflush.h>
  37 #include <asm/desc.h>
  38 #include <asm/debugreg.h>
  39 #include <asm/kvm_para.h>
  40
  41 #include <asm/virtext.h>
  42 #include "trace.h"
  43
  44 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static const struct x86_cpu_id svm_cpu_id[] = {
  50         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  51         {}
  52 };
  53 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  54
  55 #define IOPM_ALLOC_ORDER 2
  56 #define MSRPM_ALLOC_ORDER 1
  57
  58 #define SEG_TYPE_LDT 2
  59 #define SEG_TYPE_BUSY_TSS16 3
  60
  61 #define SVM_FEATURE_NPT            (1 <<  0)
  62 #define SVM_FEATURE_LBRV           (1 <<  1)
  63 #define SVM_FEATURE_SVML           (1 <<  2)
  64 #define SVM_FEATURE_NRIP           (1 <<  3)
  65 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  66 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  67 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  68 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  69 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  70
  71 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  72 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  73 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  74
  75 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  76
  77 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  78 #define TSC_RATIO_MIN           0x0000000000000001ULL
  79 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  80
  81 static bool erratum_383_found __read_mostly;
  82
  83 static const u32 host_save_user_msrs[] = {
  84 #ifdef CONFIG_X86_64
  85         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  86         MSR_FS_BASE,
  87 #endif
  88         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  89         MSR_TSC_AUX,
  90 };
  91
  92 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  93
  94 struct kvm_vcpu;
  95
  96 struct nested_state {
  97         struct vmcb *hsave;
  98         u64 hsave_msr;
  99         u64 vm_cr_msr;
 100         u64 vmcb;
 101
 102         /* These are the merged vectors */
 103         u32 *msrpm;
 104
 105         /* gpa pointers to the real vectors */
 106         u64 vmcb_msrpm;
 107         u64 vmcb_iopm;
 108
 109         /* A VMEXIT is required but not yet emulated */
 110         bool exit_required;
 111
 112         /* cache for intercepts of the guest */
 113         u32 intercept_cr;
 114         u32 intercept_dr;
 115         u32 intercept_exceptions;
 116         u64 intercept;
 117
 118         /* Nested Paging related state */
 119         u64 nested_cr3;
 120 };
 121
 122 #define MSRPM_OFFSETS   16
 123 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 124
 125 /*
 126  * Set osvw_len to higher value when updated Revision Guides
 127  * are published and we know what the new status bits are
 128  */
 129 static uint64_t osvw_len = 4, osvw_status;
 130
 131 struct vcpu_svm {
 132         struct kvm_vcpu vcpu;
 133         struct vmcb *vmcb;
 134         unsigned long vmcb_pa;
 135         struct svm_cpu_data *svm_data;
 136         uint64_t asid_generation;
 137         uint64_t sysenter_esp;
 138         uint64_t sysenter_eip;
 139         uint64_t tsc_aux;
 140
 141         u64 next_rip;
 142
 143         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 144         struct {
 145                 u16 fs;
 146                 u16 gs;
 147                 u16 ldt;
 148                 u64 gs_base;
 149         } host;
 150
 151         u32 *msrpm;
 152
 153         ulong nmi_iret_rip;
 154
 155         struct nested_state nested;
 156
 157         bool nmi_singlestep;
 158
 159         unsigned int3_injected;
 160         unsigned long int3_rip;
 161         u32 apf_reason;
 162
 163         /* cached guest cpuid flags for faster access */
 164         bool nrips_enabled      : 1;
 165 };
 166
 167 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 168 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 169
 170 #define MSR_INVALID                     0xffffffffU
 171
 172 static const struct svm_direct_access_msrs {
 173         u32 index;   /* Index of the MSR */
 174         bool always; /* True if intercept is always on */
 175 } direct_access_msrs[] = {
 176         { .index = MSR_STAR,                            .always = true  },
 177         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 178 #ifdef CONFIG_X86_64
 179         { .index = MSR_GS_BASE,                         .always = true  },
 180         { .index = MSR_FS_BASE,                         .always = true  },
 181         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 182         { .index = MSR_LSTAR,                           .always = true  },
 183         { .index = MSR_CSTAR,                           .always = true  },
 184         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 185 #endif
 186         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 187         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 188         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 189         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 190         { .index = MSR_INVALID,                         .always = false },
 191 };
 192
 193 /* enable NPT for AMD64 and X86 with PAE */
 194 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 195 static bool npt_enabled = true;
 196 #else
 197 static bool npt_enabled;
 198 #endif
 199
 200 /* allow nested paging (virtualized MMU) for all guests */
 201 static int npt = true;
 202 module_param(npt, int, S_IRUGO);
 203
 204 /* allow nested virtualization in KVM/SVM */
 205 static int nested = true;
 206 module_param(nested, int, S_IRUGO);
 207
 208 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 209 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 210 static void svm_complete_interrupts(struct vcpu_svm *svm);
 211
 212 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 213 static int nested_svm_intercept(struct vcpu_svm *svm);
 214 static int nested_svm_vmexit(struct vcpu_svm *svm);
 215 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 216                                       bool has_error_code, u32 error_code);
 217
 218 enum {
 219         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 220                             pause filter count */
 221         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 222         VMCB_ASID,       /* ASID */
 223         VMCB_INTR,       /* int_ctl, int_vector */
 224         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 225         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 226         VMCB_DR,         /* DR6, DR7 */
 227         VMCB_DT,         /* GDT, IDT */
 228         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 229         VMCB_CR2,        /* CR2 only */
 230         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 231         VMCB_DIRTY_MAX,
 232 };
 233
 234 /* TPR and CR2 are always written before VMRUN */
 235 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 236
 237 static inline void mark_all_dirty(struct vmcb *vmcb)
 238 {
 239         vmcb->control.clean = 0;
 240 }
 241
 242 static inline void mark_all_clean(struct vmcb *vmcb)
 243 {
 244         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 245                                & ~VMCB_ALWAYS_DIRTY_MASK;
 246 }
 247
 248 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 249 {
 250         vmcb->control.clean &= ~(1 << bit);
 251 }
 252
 253 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 254 {
 255         return container_of(vcpu, struct vcpu_svm, vcpu);
 256 }
 257
 258 static void recalc_intercepts(struct vcpu_svm *svm)
 259 {
 260         struct vmcb_control_area *c, *h;
 261         struct nested_state *g;
 262
 263         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 264
 265         if (!is_guest_mode(&svm->vcpu))
 266                 return;
 267
 268         c = &svm->vmcb->control;
 269         h = &svm->nested.hsave->control;
 270         g = &svm->nested;
 271
 272         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 273         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 274         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 275         c->intercept = h->intercept | g->intercept;
 276 }
 277
 278 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 279 {
 280         if (is_guest_mode(&svm->vcpu))
 281                 return svm->nested.hsave;
 282         else
 283                 return svm->vmcb;
 284 }
 285
 286 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 287 {
 288         struct vmcb *vmcb = get_host_vmcb(svm);
 289
 290         vmcb->control.intercept_cr |= (1U << bit);
 291
 292         recalc_intercepts(svm);
 293 }
 294
 295 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 296 {
 297         struct vmcb *vmcb = get_host_vmcb(svm);
 298
 299         vmcb->control.intercept_cr &= ~(1U << bit);
 300
 301         recalc_intercepts(svm);
 302 }
 303
 304 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 305 {
 306         struct vmcb *vmcb = get_host_vmcb(svm);
 307
 308         return vmcb->control.intercept_cr & (1U << bit);
 309 }
 310
 311 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 312 {
 313         struct vmcb *vmcb = get_host_vmcb(svm);
 314
 315         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 316                 | (1 << INTERCEPT_DR1_READ)
 317                 | (1 << INTERCEPT_DR2_READ)
 318                 | (1 << INTERCEPT_DR3_READ)
 319                 | (1 << INTERCEPT_DR4_READ)
 320                 | (1 << INTERCEPT_DR5_READ)
 321                 | (1 << INTERCEPT_DR6_READ)
 322                 | (1 << INTERCEPT_DR7_READ)
 323                 | (1 << INTERCEPT_DR0_WRITE)
 324                 | (1 << INTERCEPT_DR1_WRITE)
 325                 | (1 << INTERCEPT_DR2_WRITE)
 326                 | (1 << INTERCEPT_DR3_WRITE)
 327                 | (1 << INTERCEPT_DR4_WRITE)
 328                 | (1 << INTERCEPT_DR5_WRITE)
 329                 | (1 << INTERCEPT_DR6_WRITE)
 330                 | (1 << INTERCEPT_DR7_WRITE);
 331
 332         recalc_intercepts(svm);
 333 }
 334
 335 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 336 {
 337         struct vmcb *vmcb = get_host_vmcb(svm);
 338
 339         vmcb->control.intercept_dr = 0;
 340
 341         recalc_intercepts(svm);
 342 }
 343
 344 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 345 {
 346         struct vmcb *vmcb = get_host_vmcb(svm);
 347
 348         vmcb->control.intercept_exceptions |= (1U << bit);
 349
 350         recalc_intercepts(svm);
 351 }
 352
 353 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 354 {
 355         struct vmcb *vmcb = get_host_vmcb(svm);
 356
 357         vmcb->control.intercept_exceptions &= ~(1U << bit);
 358
 359         recalc_intercepts(svm);
 360 }
 361
 362 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 363 {
 364         struct vmcb *vmcb = get_host_vmcb(svm);
 365
 366         vmcb->control.intercept |= (1ULL << bit);
 367
 368         recalc_intercepts(svm);
 369 }
 370
 371 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 372 {
 373         struct vmcb *vmcb = get_host_vmcb(svm);
 374
 375         vmcb->control.intercept &= ~(1ULL << bit);
 376
 377         recalc_intercepts(svm);
 378 }
 379
 380 static inline void enable_gif(struct vcpu_svm *svm)
 381 {
 382         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 383 }
 384
 385 static inline void disable_gif(struct vcpu_svm *svm)
 386 {
 387         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 388 }
 389
 390 static inline bool gif_set(struct vcpu_svm *svm)
 391 {
 392         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 393 }
 394
 395 static unsigned long iopm_base;
 396
 397 struct kvm_ldttss_desc {
 398         u16 limit0;
 399         u16 base0;
 400         unsigned base1:8, type:5, dpl:2, p:1;
 401         unsigned limit1:4, zero0:3, g:1, base2:8;
 402         u32 base3;
 403         u32 zero1;
 404 } __attribute__((packed));
 405
 406 struct svm_cpu_data {
 407         int cpu;
 408
 409         u64 asid_generation;
 410         u32 max_asid;
 411         u32 next_asid;
 412         struct kvm_ldttss_desc *tss_desc;
 413
 414         struct page *save_area;
 415 };
 416
 417 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 418
 419 struct svm_init_data {
 420         int cpu;
 421         int r;
 422 };
 423
 424 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 425
 426 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 427 #define MSRS_RANGE_SIZE 2048
 428 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 429
 430 static u32 svm_msrpm_offset(u32 msr)
 431 {
 432         u32 offset;
 433         int i;
 434
 435         for (i = 0; i < NUM_MSR_MAPS; i++) {
 436                 if (msr < msrpm_ranges[i] ||
 437                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 438                         continue;
 439
 440                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 441                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 442
 443                 /* Now we have the u8 offset - but need the u32 offset */
 444                 return offset / 4;
 445         }
 446
 447         /* MSR not in any range */
 448         return MSR_INVALID;
 449 }
 450
 451 #define MAX_INST_SIZE 15
 452
 453 static inline void clgi(void)
 454 {
 455         asm volatile (__ex(SVM_CLGI));
 456 }
 457
 458 static inline void stgi(void)
 459 {
 460         asm volatile (__ex(SVM_STGI));
 461 }
 462
 463 static inline void invlpga(unsigned long addr, u32 asid)
 464 {
 465         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 466 }
 467
 468 static int get_npt_level(void)
 469 {
 470 #ifdef CONFIG_X86_64
 471         return PT64_ROOT_LEVEL;
 472 #else
 473         return PT32E_ROOT_LEVEL;
 474 #endif
 475 }
 476
 477 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 478 {
 479         vcpu->arch.efer = efer;
 480         if (!npt_enabled && !(efer & EFER_LMA))
 481                 efer &= ~EFER_LME;
 482
 483         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 484         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 485 }
 486
 487 static int is_external_interrupt(u32 info)
 488 {
 489         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 490         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 491 }
 492
 493 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 494 {
 495         struct vcpu_svm *svm = to_svm(vcpu);
 496         u32 ret = 0;
 497
 498         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 499                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 500         return ret;
 501 }
 502
 503 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 504 {
 505         struct vcpu_svm *svm = to_svm(vcpu);
 506
 507         if (mask == 0)
 508                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 509         else
 510                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 511
 512 }
 513
 514 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 515 {
 516         struct vcpu_svm *svm = to_svm(vcpu);
 517
 518         if (svm->vmcb->control.next_rip != 0) {
 519                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 520                 svm->next_rip = svm->vmcb->control.next_rip;
 521         }
 522
 523         if (!svm->next_rip) {
 524                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 525                                 EMULATE_DONE)
 526                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 527                 return;
 528         }
 529         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 530                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 531                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 532
 533         kvm_rip_write(vcpu, svm->next_rip);
 534         svm_set_interrupt_shadow(vcpu, 0);
 535 }
 536
 537 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 538                                 bool has_error_code, u32 error_code,
 539                                 bool reinject)
 540 {
 541         struct vcpu_svm *svm = to_svm(vcpu);
 542
 543         /*
 544          * If we are within a nested VM we'd better #VMEXIT and let the guest
 545          * handle the exception
 546          */
 547         if (!reinject &&
 548             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 549                 return;
 550
 551         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 552                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 553
 554                 /*
 555                  * For guest debugging where we have to reinject #BP if some
 556                  * INT3 is guest-owned:
 557                  * Emulate nRIP by moving RIP forward. Will fail if injection
 558                  * raises a fault that is not intercepted. Still better than
 559                  * failing in all cases.
 560                  */
 561                 skip_emulated_instruction(&svm->vcpu);
 562                 rip = kvm_rip_read(&svm->vcpu);
 563                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 564                 svm->int3_injected = rip - old_rip;
 565         }
 566
 567         svm->vmcb->control.event_inj = nr
 568                 | SVM_EVTINJ_VALID
 569                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 570                 | SVM_EVTINJ_TYPE_EXEPT;
 571         svm->vmcb->control.event_inj_err = error_code;
 572 }
 573
 574 static void svm_init_erratum_383(void)
 575 {
 576         u32 low, high;
 577         int err;
 578         u64 val;
 579
 580         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 581                 return;
 582
 583         /* Use _safe variants to not break nested virtualization */
 584         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 585         if (err)
 586                 return;
 587
 588         val |= (1ULL << 47);
 589
 590         low  = lower_32_bits(val);
 591         high = upper_32_bits(val);
 592
 593         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 594
 595         erratum_383_found = true;
 596 }
 597
 598 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 599 {
 600         /*
 601          * Guests should see errata 400 and 415 as fixed (assuming that
 602          * HLT and IO instructions are intercepted).
 603          */
 604         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 605         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 606
 607         /*
 608          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 609          * all osvw.status bits inside that length, including bit 0 (which is
 610          * reserved for erratum 298), are valid. However, if host processor's
 611          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 612          * be conservative here and therefore we tell the guest that erratum 298
 613          * is present (because we really don't know).
 614          */
 615         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 616                 vcpu->arch.osvw.status |= 1;
 617 }
 618
 619 static int has_svm(void)
 620 {
 621         const char *msg;
 622
 623         if (!cpu_has_svm(&msg)) {
 624                 printk(KERN_INFO "has_svm: %s\n", msg);
 625                 return 0;
 626         }
 627
 628         return 1;
 629 }
 630
 631 static void svm_hardware_disable(void)
 632 {
 633         /* Make sure we clean up behind us */
 634         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 635                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 636
 637         cpu_svm_disable();
 638
 639         amd_pmu_disable_virt();
 640 }
 641
 642 static int svm_hardware_enable(void)
 643 {
 644
 645         struct svm_cpu_data *sd;
 646         uint64_t efer;
 647         struct desc_ptr gdt_descr;
 648         struct desc_struct *gdt;
 649         int me = raw_smp_processor_id();
 650
 651         rdmsrl(MSR_EFER, efer);
 652         if (efer & EFER_SVME)
 653                 return -EBUSY;
 654
 655         if (!has_svm()) {
 656                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 657                 return -EINVAL;
 658         }
 659         sd = per_cpu(svm_data, me);
 660         if (!sd) {
 661                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 662                 return -EINVAL;
 663         }
 664
 665         sd->asid_generation = 1;
 666         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 667         sd->next_asid = sd->max_asid + 1;
 668
 669         native_store_gdt(&gdt_descr);
 670         gdt = (struct desc_struct *)gdt_descr.address;
 671         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 672
 673         wrmsrl(MSR_EFER, efer | EFER_SVME);
 674
 675         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 676
 677         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 678                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 679                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 680         }
 681
 682
 683         /*
 684          * Get OSVW bits.
 685          *
 686          * Note that it is possible to have a system with mixed processor
 687          * revisions and therefore different OSVW bits. If bits are not the same
 688          * on different processors then choose the worst case (i.e. if erratum
 689          * is present on one processor and not on another then assume that the
 690          * erratum is present everywhere).
 691          */
 692         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 693                 uint64_t len, status = 0;
 694                 int err;
 695
 696                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 697                 if (!err)
 698                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 699                                                       &err);
 700
 701                 if (err)
 702                         osvw_status = osvw_len = 0;
 703                 else {
 704                         if (len < osvw_len)
 705                                 osvw_len = len;
 706                         osvw_status |= status;
 707                         osvw_status &= (1ULL << osvw_len) - 1;
 708                 }
 709         } else
 710                 osvw_status = osvw_len = 0;
 711
 712         svm_init_erratum_383();
 713
 714         amd_pmu_enable_virt();
 715
 716         return 0;
 717 }
 718
 719 static void svm_cpu_uninit(int cpu)
 720 {
 721         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 722
 723         if (!sd)
 724                 return;
 725
 726         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 727         __free_page(sd->save_area);
 728         kfree(sd);
 729 }
 730
 731 static int svm_cpu_init(int cpu)
 732 {
 733         struct svm_cpu_data *sd;
 734         int r;
 735
 736         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 737         if (!sd)
 738                 return -ENOMEM;
 739         sd->cpu = cpu;
 740         sd->save_area = alloc_page(GFP_KERNEL);
 741         r = -ENOMEM;
 742         if (!sd->save_area)
 743                 goto err_1;
 744
 745         per_cpu(svm_data, cpu) = sd;
 746
 747         return 0;
 748
 749 err_1:
 750         kfree(sd);
 751         return r;
 752
 753 }
 754
 755 static bool valid_msr_intercept(u32 index)
 756 {
 757         int i;
 758
 759         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 760                 if (direct_access_msrs[i].index == index)
 761                         return true;
 762
 763         return false;
 764 }
 765
 766 static void set_msr_interception(u32 *msrpm, unsigned msr,
 767                                  int read, int write)
 768 {
 769         u8 bit_read, bit_write;
 770         unsigned long tmp;
 771         u32 offset;
 772
 773         /*
 774          * If this warning triggers extend the direct_access_msrs list at the
 775          * beginning of the file
 776          */
 777         WARN_ON(!valid_msr_intercept(msr));
 778
 779         offset    = svm_msrpm_offset(msr);
 780         bit_read  = 2 * (msr & 0x0f);
 781         bit_write = 2 * (msr & 0x0f) + 1;
 782         tmp       = msrpm[offset];
 783
 784         BUG_ON(offset == MSR_INVALID);
 785
 786         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 787         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 788
 789         msrpm[offset] = tmp;
 790 }
 791
 792 static void svm_vcpu_init_msrpm(u32 *msrpm)
 793 {
 794         int i;
 795
 796         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 797
 798         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 799                 if (!direct_access_msrs[i].always)
 800                         continue;
 801
 802                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 803         }
 804 }
 805
 806 static void add_msr_offset(u32 offset)
 807 {
 808         int i;
 809
 810         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 811
 812                 /* Offset already in list? */
 813                 if (msrpm_offsets[i] == offset)
 814                         return;
 815
 816                 /* Slot used by another offset? */
 817                 if (msrpm_offsets[i] != MSR_INVALID)
 818                         continue;
 819
 820                 /* Add offset to list */
 821                 msrpm_offsets[i] = offset;
 822
 823                 return;
 824         }
 825
 826         /*
 827          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 828          * increase MSRPM_OFFSETS in this case.
 829          */
 830         BUG();
 831 }
 832
 833 static void init_msrpm_offsets(void)
 834 {
 835         int i;
 836
 837         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 838
 839         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 840                 u32 offset;
 841
 842                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 843                 BUG_ON(offset == MSR_INVALID);
 844
 845                 add_msr_offset(offset);
 846         }
 847 }
 848
 849 static void svm_enable_lbrv(struct vcpu_svm *svm)
 850 {
 851         u32 *msrpm = svm->msrpm;
 852
 853         svm->vmcb->control.lbr_ctl = 1;
 854         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 855         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 856         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 857         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 858 }
 859
 860 static void svm_disable_lbrv(struct vcpu_svm *svm)
 861 {
 862         u32 *msrpm = svm->msrpm;
 863
 864         svm->vmcb->control.lbr_ctl = 0;
 865         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 866         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 867         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 868         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 869 }
 870
 871 static __init int svm_hardware_setup(void)
 872 {
 873         int cpu;
 874         struct page *iopm_pages;
 875         void *iopm_va;
 876         int r;
 877
 878         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 879
 880         if (!iopm_pages)
 881                 return -ENOMEM;
 882
 883         iopm_va = page_address(iopm_pages);
 884         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 885         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 886
 887         init_msrpm_offsets();
 888
 889         if (boot_cpu_has(X86_FEATURE_NX))
 890                 kvm_enable_efer_bits(EFER_NX);
 891
 892         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 893                 kvm_enable_efer_bits(EFER_FFXSR);
 894
 895         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 896                 kvm_has_tsc_control = true;
 897                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
 898                 kvm_tsc_scaling_ratio_frac_bits = 32;
 899         }
 900
 901         if (nested) {
 902                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 903                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 904         }
 905
 906         for_each_possible_cpu(cpu) {
 907                 r = svm_cpu_init(cpu);
 908                 if (r)
 909                         goto err;
 910         }
 911
 912         if (!boot_cpu_has(X86_FEATURE_NPT))
 913                 npt_enabled = false;
 914
 915         if (npt_enabled && !npt) {
 916                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 917                 npt_enabled = false;
 918         }
 919
 920         if (npt_enabled) {
 921                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 922                 kvm_enable_tdp();
 923         } else
 924                 kvm_disable_tdp();
 925
 926         return 0;
 927
 928 err:
 929         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 930         iopm_base = 0;
 931         return r;
 932 }
 933
 934 static __exit void svm_hardware_unsetup(void)
 935 {
 936         int cpu;
 937
 938         for_each_possible_cpu(cpu)
 939                 svm_cpu_uninit(cpu);
 940
 941         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
 942         iopm_base = 0;
 943 }
 944
 945 static void init_seg(struct vmcb_seg *seg)
 946 {
 947         seg->selector = 0;
 948         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
 949                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
 950         seg->limit = 0xffff;
 951         seg->base = 0;
 952 }
 953
 954 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 955 {
 956         seg->selector = 0;
 957         seg->attrib = SVM_SELECTOR_P_MASK | type;
 958         seg->limit = 0xffff;
 959         seg->base = 0;
 960 }
 961
 962 static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
 963 {
 964         struct vcpu_svm *svm = to_svm(vcpu);
 965
 966         return svm->vmcb->control.tsc_offset;
 967 }
 968
 969 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 970 {
 971         struct vcpu_svm *svm = to_svm(vcpu);
 972         u64 g_tsc_offset = 0;
 973
 974         if (is_guest_mode(vcpu)) {
 975                 g_tsc_offset = svm->vmcb->control.tsc_offset -
 976                                svm->nested.hsave->control.tsc_offset;
 977                 svm->nested.hsave->control.tsc_offset = offset;
 978         } else
 979                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 980                                            svm->vmcb->control.tsc_offset,
 981                                            offset);
 982
 983         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
 984
 985         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 986 }
 987
 988 static void svm_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment)
 989 {
 990         struct vcpu_svm *svm = to_svm(vcpu);
 991
 992         svm->vmcb->control.tsc_offset += adjustment;
 993         if (is_guest_mode(vcpu))
 994                 svm->nested.hsave->control.tsc_offset += adjustment;
 995         else
 996                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 997                                      svm->vmcb->control.tsc_offset - adjustment,
 998                                      svm->vmcb->control.tsc_offset);
 999
1000         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1001 }
1002
1003 static void init_vmcb(struct vcpu_svm *svm)
1004 {
1005         struct vmcb_control_area *control = &svm->vmcb->control;
1006         struct vmcb_save_area *save = &svm->vmcb->save;
1007
1008         svm->vcpu.fpu_active = 1;
1009         svm->vcpu.arch.hflags = 0;
1010
1011         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1012         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1013         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1014         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1015         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1016         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1017         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1018
1019         set_dr_intercepts(svm);
1020
1021         set_exception_intercept(svm, PF_VECTOR);
1022         set_exception_intercept(svm, UD_VECTOR);
1023         set_exception_intercept(svm, MC_VECTOR);
1024         set_exception_intercept(svm, AC_VECTOR);
1025         set_exception_intercept(svm, DB_VECTOR);
1026
1027         set_intercept(svm, INTERCEPT_INTR);
1028         set_intercept(svm, INTERCEPT_NMI);
1029         set_intercept(svm, INTERCEPT_SMI);
1030         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1031         set_intercept(svm, INTERCEPT_RDPMC);
1032         set_intercept(svm, INTERCEPT_CPUID);
1033         set_intercept(svm, INTERCEPT_INVD);
1034         set_intercept(svm, INTERCEPT_HLT);
1035         set_intercept(svm, INTERCEPT_INVLPG);
1036         set_intercept(svm, INTERCEPT_INVLPGA);
1037         set_intercept(svm, INTERCEPT_IOIO_PROT);
1038         set_intercept(svm, INTERCEPT_MSR_PROT);
1039         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1040         set_intercept(svm, INTERCEPT_SHUTDOWN);
1041         set_intercept(svm, INTERCEPT_VMRUN);
1042         set_intercept(svm, INTERCEPT_VMMCALL);
1043         set_intercept(svm, INTERCEPT_VMLOAD);
1044         set_intercept(svm, INTERCEPT_VMSAVE);
1045         set_intercept(svm, INTERCEPT_STGI);
1046         set_intercept(svm, INTERCEPT_CLGI);
1047         set_intercept(svm, INTERCEPT_SKINIT);
1048         set_intercept(svm, INTERCEPT_WBINVD);
1049         set_intercept(svm, INTERCEPT_MONITOR);
1050         set_intercept(svm, INTERCEPT_MWAIT);
1051         set_intercept(svm, INTERCEPT_XSETBV);
1052
1053         control->iopm_base_pa = iopm_base;
1054         control->msrpm_base_pa = __pa(svm->msrpm);
1055         control->int_ctl = V_INTR_MASKING_MASK;
1056
1057         init_seg(&save->es);
1058         init_seg(&save->ss);
1059         init_seg(&save->ds);
1060         init_seg(&save->fs);
1061         init_seg(&save->gs);
1062
1063         save->cs.selector = 0xf000;
1064         save->cs.base = 0xffff0000;
1065         /* Executable/Readable Code Segment */
1066         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1067                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1068         save->cs.limit = 0xffff;
1069
1070         save->gdtr.limit = 0xffff;
1071         save->idtr.limit = 0xffff;
1072
1073         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1074         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1075
1076         svm_set_efer(&svm->vcpu, 0);
1077         save->dr6 = 0xffff0ff0;
1078         kvm_set_rflags(&svm->vcpu, 2);
1079         save->rip = 0x0000fff0;
1080         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1081
1082         /*
1083          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1084          * It also updates the guest-visible cr0 value.
1085          */
1086         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1087         kvm_mmu_reset_context(&svm->vcpu);
1088
1089         save->cr4 = X86_CR4_PAE;
1090         /* rdx = ?? */
1091
1092         if (npt_enabled) {
1093                 /* Setup VMCB for Nested Paging */
1094                 control->nested_ctl = 1;
1095                 clr_intercept(svm, INTERCEPT_INVLPG);
1096                 clr_exception_intercept(svm, PF_VECTOR);
1097                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1098                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1099                 save->g_pat = svm->vcpu.arch.pat;
1100                 save->cr3 = 0;
1101                 save->cr4 = 0;
1102         }
1103         svm->asid_generation = 0;
1104
1105         svm->nested.vmcb = 0;
1106         svm->vcpu.arch.hflags = 0;
1107
1108         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1109                 control->pause_filter_count = 3000;
1110                 set_intercept(svm, INTERCEPT_PAUSE);
1111         }
1112
1113         mark_all_dirty(svm->vmcb);
1114
1115         enable_gif(svm);
1116 }
1117
1118 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1119 {
1120         struct vcpu_svm *svm = to_svm(vcpu);
1121         u32 dummy;
1122         u32 eax = 1;
1123
1124         if (!init_event) {
1125                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1126                                            MSR_IA32_APICBASE_ENABLE;
1127                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1128                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1129         }
1130         init_vmcb(svm);
1131
1132         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1133         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1134 }
1135
1136 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1137 {
1138         struct vcpu_svm *svm;
1139         struct page *page;
1140         struct page *msrpm_pages;
1141         struct page *hsave_page;
1142         struct page *nested_msrpm_pages;
1143         int err;
1144
1145         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1146         if (!svm) {
1147                 err = -ENOMEM;
1148                 goto out;
1149         }
1150
1151         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1152         if (err)
1153                 goto free_svm;
1154
1155         err = -ENOMEM;
1156         page = alloc_page(GFP_KERNEL);
1157         if (!page)
1158                 goto uninit;
1159
1160         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1161         if (!msrpm_pages)
1162                 goto free_page1;
1163
1164         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1165         if (!nested_msrpm_pages)
1166                 goto free_page2;
1167
1168         hsave_page = alloc_page(GFP_KERNEL);
1169         if (!hsave_page)
1170                 goto free_page3;
1171
1172         svm->nested.hsave = page_address(hsave_page);
1173
1174         svm->msrpm = page_address(msrpm_pages);
1175         svm_vcpu_init_msrpm(svm->msrpm);
1176
1177         svm->nested.msrpm = page_address(nested_msrpm_pages);
1178         svm_vcpu_init_msrpm(svm->nested.msrpm);
1179
1180         svm->vmcb = page_address(page);
1181         clear_page(svm->vmcb);
1182         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1183         svm->asid_generation = 0;
1184         init_vmcb(svm);
1185
1186         svm_init_osvw(&svm->vcpu);
1187
1188         return &svm->vcpu;
1189
1190 free_page3:
1191         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1192 free_page2:
1193         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1194 free_page1:
1195         __free_page(page);
1196 uninit:
1197         kvm_vcpu_uninit(&svm->vcpu);
1198 free_svm:
1199         kmem_cache_free(kvm_vcpu_cache, svm);
1200 out:
1201         return ERR_PTR(err);
1202 }
1203
1204 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1205 {
1206         struct vcpu_svm *svm = to_svm(vcpu);
1207
1208         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1209         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1210         __free_page(virt_to_page(svm->nested.hsave));
1211         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1212         kvm_vcpu_uninit(vcpu);
1213         kmem_cache_free(kvm_vcpu_cache, svm);
1214 }
1215
1216 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1217 {
1218         struct vcpu_svm *svm = to_svm(vcpu);
1219         int i;
1220
1221         if (unlikely(cpu != vcpu->cpu)) {
1222                 svm->asid_generation = 0;
1223                 mark_all_dirty(svm->vmcb);
1224         }
1225
1226 #ifdef CONFIG_X86_64
1227         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1228 #endif
1229         savesegment(fs, svm->host.fs);
1230         savesegment(gs, svm->host.gs);
1231         svm->host.ldt = kvm_read_ldt();
1232
1233         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1234                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1235
1236         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1237                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1238                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1239                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
1240                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1241                 }
1242         }
1243         /* This assumes that the kernel never uses MSR_TSC_AUX */
1244         if (static_cpu_has(X86_FEATURE_RDTSCP))
1245                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1246 }
1247
1248 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1249 {
1250         struct vcpu_svm *svm = to_svm(vcpu);
1251         int i;
1252
1253         ++vcpu->stat.host_state_reload;
1254         kvm_load_ldt(svm->host.ldt);
1255 #ifdef CONFIG_X86_64
1256         loadsegment(fs, svm->host.fs);
1257         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1258         load_gs_index(svm->host.gs);
1259 #else
1260 #ifdef CONFIG_X86_32_LAZY_GS
1261         loadsegment(gs, svm->host.gs);
1262 #endif
1263 #endif
1264         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1265                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1266 }
1267
1268 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1269 {
1270         return to_svm(vcpu)->vmcb->save.rflags;
1271 }
1272
1273 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1274 {
1275        /*
1276         * Any change of EFLAGS.VM is accompained by a reload of SS
1277         * (caused by either a task switch or an inter-privilege IRET),
1278         * so we do not need to update the CPL here.
1279         */
1280         to_svm(vcpu)->vmcb->save.rflags = rflags;
1281 }
1282
1283 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1284 {
1285         switch (reg) {
1286         case VCPU_EXREG_PDPTR:
1287                 BUG_ON(!npt_enabled);
1288                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1289                 break;
1290         default:
1291                 BUG();
1292         }
1293 }
1294
1295 static void svm_set_vintr(struct vcpu_svm *svm)
1296 {
1297         set_intercept(svm, INTERCEPT_VINTR);
1298 }
1299
1300 static void svm_clear_vintr(struct vcpu_svm *svm)
1301 {
1302         clr_intercept(svm, INTERCEPT_VINTR);
1303 }
1304
1305 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1306 {
1307         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1308
1309         switch (seg) {
1310         case VCPU_SREG_CS: return &save->cs;
1311         case VCPU_SREG_DS: return &save->ds;
1312         case VCPU_SREG_ES: return &save->es;
1313         case VCPU_SREG_FS: return &save->fs;
1314         case VCPU_SREG_GS: return &save->gs;
1315         case VCPU_SREG_SS: return &save->ss;
1316         case VCPU_SREG_TR: return &save->tr;
1317         case VCPU_SREG_LDTR: return &save->ldtr;
1318         }
1319         BUG();
1320         return NULL;
1321 }
1322
1323 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1324 {
1325         struct vmcb_seg *s = svm_seg(vcpu, seg);
1326
1327         return s->base;
1328 }
1329
1330 static void svm_get_segment(struct kvm_vcpu *vcpu,
1331                             struct kvm_segment *var, int seg)
1332 {
1333         struct vmcb_seg *s = svm_seg(vcpu, seg);
1334
1335         var->base = s->base;
1336         var->limit = s->limit;
1337         var->selector = s->selector;
1338         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1339         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1340         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1341         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1342         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1343         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1344         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1345
1346         /*
1347          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1348          * However, the SVM spec states that the G bit is not observed by the
1349          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1350          * So let's synthesize a legal G bit for all segments, this helps
1351          * running KVM nested. It also helps cross-vendor migration, because
1352          * Intel's vmentry has a check on the 'G' bit.
1353          */
1354         var->g = s->limit > 0xfffff;
1355
1356         /*
1357          * AMD's VMCB does not have an explicit unusable field, so emulate it
1358          * for cross vendor migration purposes by "not present"
1359          */
1360         var->unusable = !var->present || (var->type == 0);
1361
1362         switch (seg) {
1363         case VCPU_SREG_TR:
1364                 /*
1365                  * Work around a bug where the busy flag in the tr selector
1366                  * isn't exposed
1367                  */
1368                 var->type |= 0x2;
1369                 break;
1370         case VCPU_SREG_DS:
1371         case VCPU_SREG_ES:
1372         case VCPU_SREG_FS:
1373         case VCPU_SREG_GS:
1374                 /*
1375                  * The accessed bit must always be set in the segment
1376                  * descriptor cache, although it can be cleared in the
1377                  * descriptor, the cached bit always remains at 1. Since
1378                  * Intel has a check on this, set it here to support
1379                  * cross-vendor migration.
1380                  */
1381                 if (!var->unusable)
1382                         var->type |= 0x1;
1383                 break;
1384         case VCPU_SREG_SS:
1385                 /*
1386                  * On AMD CPUs sometimes the DB bit in the segment
1387                  * descriptor is left as 1, although the whole segment has
1388                  * been made unusable. Clear it here to pass an Intel VMX
1389                  * entry check when cross vendor migrating.
1390                  */
1391                 if (var->unusable)
1392                         var->db = 0;
1393                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1394                 break;
1395         }
1396 }
1397
1398 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1399 {
1400         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1401
1402         return save->cpl;
1403 }
1404
1405 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1406 {
1407         struct vcpu_svm *svm = to_svm(vcpu);
1408
1409         dt->size = svm->vmcb->save.idtr.limit;
1410         dt->address = svm->vmcb->save.idtr.base;
1411 }
1412
1413 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1414 {
1415         struct vcpu_svm *svm = to_svm(vcpu);
1416
1417         svm->vmcb->save.idtr.limit = dt->size;
1418         svm->vmcb->save.idtr.base = dt->address ;
1419         mark_dirty(svm->vmcb, VMCB_DT);
1420 }
1421
1422 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1423 {
1424         struct vcpu_svm *svm = to_svm(vcpu);
1425
1426         dt->size = svm->vmcb->save.gdtr.limit;
1427         dt->address = svm->vmcb->save.gdtr.base;
1428 }
1429
1430 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1431 {
1432         struct vcpu_svm *svm = to_svm(vcpu);
1433
1434         svm->vmcb->save.gdtr.limit = dt->size;
1435         svm->vmcb->save.gdtr.base = dt->address ;
1436         mark_dirty(svm->vmcb, VMCB_DT);
1437 }
1438
1439 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1440 {
1441 }
1442
1443 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1444 {
1445 }
1446
1447 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1448 {
1449 }
1450
1451 static void update_cr0_intercept(struct vcpu_svm *svm)
1452 {
1453         ulong gcr0 = svm->vcpu.arch.cr0;
1454         u64 *hcr0 = &svm->vmcb->save.cr0;
1455
1456         if (!svm->vcpu.fpu_active)
1457                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1458         else
1459                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1460                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1461
1462         mark_dirty(svm->vmcb, VMCB_CR);
1463
1464         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1465                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1466                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1467         } else {
1468                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1469                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1470         }
1471 }
1472
1473 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1474 {
1475         struct vcpu_svm *svm = to_svm(vcpu);
1476
1477 #ifdef CONFIG_X86_64
1478         if (vcpu->arch.efer & EFER_LME) {
1479                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1480                         vcpu->arch.efer |= EFER_LMA;
1481                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1482                 }
1483
1484                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1485                         vcpu->arch.efer &= ~EFER_LMA;
1486                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1487                 }
1488         }
1489 #endif
1490         vcpu->arch.cr0 = cr0;
1491
1492         if (!npt_enabled)
1493                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1494
1495         if (!vcpu->fpu_active)
1496                 cr0 |= X86_CR0_TS;
1497         /*
1498          * re-enable caching here because the QEMU bios
1499          * does not do it - this results in some delay at
1500          * reboot
1501          */
1502         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1503                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1504         svm->vmcb->save.cr0 = cr0;
1505         mark_dirty(svm->vmcb, VMCB_CR);
1506         update_cr0_intercept(svm);
1507 }
1508
1509 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1510 {
1511         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1512         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1513
1514         if (cr4 & X86_CR4_VMXE)
1515                 return 1;
1516
1517         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1518                 svm_flush_tlb(vcpu);
1519
1520         vcpu->arch.cr4 = cr4;
1521         if (!npt_enabled)
1522                 cr4 |= X86_CR4_PAE;
1523         cr4 |= host_cr4_mce;
1524         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1525         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1526         return 0;
1527 }
1528
1529 static void svm_set_segment(struct kvm_vcpu *vcpu,
1530                             struct kvm_segment *var, int seg)
1531 {
1532         struct vcpu_svm *svm = to_svm(vcpu);
1533         struct vmcb_seg *s = svm_seg(vcpu, seg);
1534
1535         s->base = var->base;
1536         s->limit = var->limit;
1537         s->selector = var->selector;
1538         if (var->unusable)
1539                 s->attrib = 0;
1540         else {
1541                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1542                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1543                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1544                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1545                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1546                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1547                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1548                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1549         }
1550
1551         /*
1552          * This is always accurate, except if SYSRET returned to a segment
1553          * with SS.DPL != 3.  Intel does not have this quirk, and always
1554          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1555          * would entail passing the CPL to userspace and back.
1556          */
1557         if (seg == VCPU_SREG_SS)
1558                 svm->vmcb->save.cpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1559
1560         mark_dirty(svm->vmcb, VMCB_SEG);
1561 }
1562
1563 static void update_bp_intercept(struct kvm_vcpu *vcpu)
1564 {
1565         struct vcpu_svm *svm = to_svm(vcpu);
1566
1567         clr_exception_intercept(svm, BP_VECTOR);
1568
1569         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1570                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1571                         set_exception_intercept(svm, BP_VECTOR);
1572         } else
1573                 vcpu->guest_debug = 0;
1574 }
1575
1576 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1577 {
1578         if (sd->next_asid > sd->max_asid) {
1579                 ++sd->asid_generation;
1580                 sd->next_asid = 1;
1581                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1582         }
1583
1584         svm->asid_generation = sd->asid_generation;
1585         svm->vmcb->control.asid = sd->next_asid++;
1586
1587         mark_dirty(svm->vmcb, VMCB_ASID);
1588 }
1589
1590 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
1591 {
1592         return to_svm(vcpu)->vmcb->save.dr6;
1593 }
1594
1595 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1596 {
1597         struct vcpu_svm *svm = to_svm(vcpu);
1598
1599         svm->vmcb->save.dr6 = value;
1600         mark_dirty(svm->vmcb, VMCB_DR);
1601 }
1602
1603 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1604 {
1605         struct vcpu_svm *svm = to_svm(vcpu);
1606
1607         get_debugreg(vcpu->arch.db[0], 0);
1608         get_debugreg(vcpu->arch.db[1], 1);
1609         get_debugreg(vcpu->arch.db[2], 2);
1610         get_debugreg(vcpu->arch.db[3], 3);
1611         vcpu->arch.dr6 = svm_get_dr6(vcpu);
1612         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1613
1614         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1615         set_dr_intercepts(svm);
1616 }
1617
1618 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1619 {
1620         struct vcpu_svm *svm = to_svm(vcpu);
1621
1622         svm->vmcb->save.dr7 = value;
1623         mark_dirty(svm->vmcb, VMCB_DR);
1624 }
1625
1626 static int pf_interception(struct vcpu_svm *svm)
1627 {
1628         u64 fault_address = svm->vmcb->control.exit_info_2;
1629         u32 error_code;
1630         int r = 1;
1631
1632         switch (svm->apf_reason) {
1633         default:
1634                 error_code = svm->vmcb->control.exit_info_1;
1635
1636                 trace_kvm_page_fault(fault_address, error_code);
1637                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1638                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1639                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1640                         svm->vmcb->control.insn_bytes,
1641                         svm->vmcb->control.insn_len);
1642                 break;
1643         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1644                 svm->apf_reason = 0;
1645                 local_irq_disable();
1646                 kvm_async_pf_task_wait(fault_address);
1647                 local_irq_enable();
1648                 break;
1649         case KVM_PV_REASON_PAGE_READY:
1650                 svm->apf_reason = 0;
1651                 local_irq_disable();
1652                 kvm_async_pf_task_wake(fault_address);
1653                 local_irq_enable();
1654                 break;
1655         }
1656         return r;
1657 }
1658
1659 static int db_interception(struct vcpu_svm *svm)
1660 {
1661         struct kvm_run *kvm_run = svm->vcpu.run;
1662
1663         if (!(svm->vcpu.guest_debug &
1664               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1665                 !svm->nmi_singlestep) {
1666                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1667                 return 1;
1668         }
1669
1670         if (svm->nmi_singlestep) {
1671                 svm->nmi_singlestep = false;
1672                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1673                         svm->vmcb->save.rflags &=
1674                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1675         }
1676
1677         if (svm->vcpu.guest_debug &
1678             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1679                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1680                 kvm_run->debug.arch.pc =
1681                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1682                 kvm_run->debug.arch.exception = DB_VECTOR;
1683                 return 0;
1684         }
1685
1686         return 1;
1687 }
1688
1689 static int bp_interception(struct vcpu_svm *svm)
1690 {
1691         struct kvm_run *kvm_run = svm->vcpu.run;
1692
1693         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1694         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1695         kvm_run->debug.arch.exception = BP_VECTOR;
1696         return 0;
1697 }
1698
1699 static int ud_interception(struct vcpu_svm *svm)
1700 {
1701         int er;
1702
1703         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
1704         if (er != EMULATE_DONE)
1705                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1706         return 1;
1707 }
1708
1709 static int ac_interception(struct vcpu_svm *svm)
1710 {
1711         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1712         return 1;
1713 }
1714
1715 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1716 {
1717         struct vcpu_svm *svm = to_svm(vcpu);
1718
1719         clr_exception_intercept(svm, NM_VECTOR);
1720
1721         svm->vcpu.fpu_active = 1;
1722         update_cr0_intercept(svm);
1723 }
1724
1725 static int nm_interception(struct vcpu_svm *svm)
1726 {
1727         svm_fpu_activate(&svm->vcpu);
1728         return 1;
1729 }
1730
1731 static bool is_erratum_383(void)
1732 {
1733         int err, i;
1734         u64 value;
1735
1736         if (!erratum_383_found)
1737                 return false;
1738
1739         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1740         if (err)
1741                 return false;
1742
1743         /* Bit 62 may or may not be set for this mce */
1744         value &= ~(1ULL << 62);
1745
1746         if (value != 0xb600000000010015ULL)
1747                 return false;
1748
1749         /* Clear MCi_STATUS registers */
1750         for (i = 0; i < 6; ++i)
1751                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1752
1753         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1754         if (!err) {
1755                 u32 low, high;
1756
1757                 value &= ~(1ULL << 2);
1758                 low    = lower_32_bits(value);
1759                 high   = upper_32_bits(value);
1760
1761                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1762         }
1763
1764         /* Flush tlb to evict multi-match entries */
1765         __flush_tlb_all();
1766
1767         return true;
1768 }
1769
1770 static void svm_handle_mce(struct vcpu_svm *svm)
1771 {
1772         if (is_erratum_383()) {
1773                 /*
1774                  * Erratum 383 triggered. Guest state is corrupt so kill the
1775                  * guest.
1776                  */
1777                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1778
1779                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1780
1781                 return;
1782         }
1783
1784         /*
1785          * On an #MC intercept the MCE handler is not called automatically in
1786          * the host. So do it by hand here.
1787          */
1788         asm volatile (
1789                 "int $0x12\n");
1790         /* not sure if we ever come back to this point */
1791
1792         return;
1793 }
1794
1795 static int mc_interception(struct vcpu_svm *svm)
1796 {
1797         return 1;
1798 }
1799
1800 static int shutdown_interception(struct vcpu_svm *svm)
1801 {
1802         struct kvm_run *kvm_run = svm->vcpu.run;
1803
1804         /*
1805          * VMCB is undefined after a SHUTDOWN intercept
1806          * so reinitialize it.
1807          */
1808         clear_page(svm->vmcb);
1809         init_vmcb(svm);
1810
1811         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1812         return 0;
1813 }
1814
1815 static int io_interception(struct vcpu_svm *svm)
1816 {
1817         struct kvm_vcpu *vcpu = &svm->vcpu;
1818         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1819         int size, in, string;
1820         unsigned port;
1821
1822         ++svm->vcpu.stat.io_exits;
1823         string = (io_info & SVM_IOIO_STR_MASK) != 0;
1824         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1825         if (string || in)
1826                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
1827
1828         port = io_info >> 16;
1829         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1830         svm->next_rip = svm->vmcb->control.exit_info_2;
1831         skip_emulated_instruction(&svm->vcpu);
1832
1833         return kvm_fast_pio_out(vcpu, size, port);
1834 }
1835
1836 static int nmi_interception(struct vcpu_svm *svm)
1837 {
1838         return 1;
1839 }
1840
1841 static int intr_interception(struct vcpu_svm *svm)
1842 {
1843         ++svm->vcpu.stat.irq_exits;
1844         return 1;
1845 }
1846
1847 static int nop_on_interception(struct vcpu_svm *svm)
1848 {
1849         return 1;
1850 }
1851
1852 static int halt_interception(struct vcpu_svm *svm)
1853 {
1854         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1855         return kvm_emulate_halt(&svm->vcpu);
1856 }
1857
1858 static int vmmcall_interception(struct vcpu_svm *svm)
1859 {
1860         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1861         return kvm_emulate_hypercall(&svm->vcpu);
1862 }
1863
1864 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
1865 {
1866         struct vcpu_svm *svm = to_svm(vcpu);
1867
1868         return svm->nested.nested_cr3;
1869 }
1870
1871 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
1872 {
1873         struct vcpu_svm *svm = to_svm(vcpu);
1874         u64 cr3 = svm->nested.nested_cr3;
1875         u64 pdpte;
1876         int ret;
1877
1878         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
1879                                        offset_in_page(cr3) + index * 8, 8);
1880         if (ret)
1881                 return 0;
1882         return pdpte;
1883 }
1884
1885 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
1886                                    unsigned long root)
1887 {
1888         struct vcpu_svm *svm = to_svm(vcpu);
1889
1890         svm->vmcb->control.nested_cr3 = root;
1891         mark_dirty(svm->vmcb, VMCB_NPT);
1892         svm_flush_tlb(vcpu);
1893 }
1894
1895 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
1896                                        struct x86_exception *fault)
1897 {
1898         struct vcpu_svm *svm = to_svm(vcpu);
1899
1900         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
1901                 /*
1902                  * TODO: track the cause of the nested page fault, and
1903                  * correctly fill in the high bits of exit_info_1.
1904                  */
1905                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
1906                 svm->vmcb->control.exit_code_hi = 0;
1907                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
1908                 svm->vmcb->control.exit_info_2 = fault->address;
1909         }
1910
1911         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
1912         svm->vmcb->control.exit_info_1 |= fault->error_code;
1913
1914         /*
1915          * The present bit is always zero for page structure faults on real
1916          * hardware.
1917          */
1918         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
1919                 svm->vmcb->control.exit_info_1 &= ~1;
1920
1921         nested_svm_vmexit(svm);
1922 }
1923
1924 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
1925 {
1926         WARN_ON(mmu_is_nested(vcpu));
1927         kvm_init_shadow_mmu(vcpu);
1928         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
1929         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
1930         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
1931         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
1932         vcpu->arch.mmu.shadow_root_level = get_npt_level();
1933         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
1934         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
1935 }
1936
1937 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
1938 {
1939         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
1940 }
1941
1942 static int nested_svm_check_permissions(struct vcpu_svm *svm)
1943 {
1944         if (!(svm->vcpu.arch.efer & EFER_SVME)
1945             || !is_paging(&svm->vcpu)) {
1946                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1947                 return 1;
1948         }
1949
1950         if (svm->vmcb->save.cpl) {
1951                 kvm_inject_gp(&svm->vcpu, 0);
1952                 return 1;
1953         }
1954
1955        return 0;
1956 }
1957
1958 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
1959                                       bool has_error_code, u32 error_code)
1960 {
1961         int vmexit;
1962
1963         if (!is_guest_mode(&svm->vcpu))
1964                 return 0;
1965
1966         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
1967         svm->vmcb->control.exit_code_hi = 0;
1968         svm->vmcb->control.exit_info_1 = error_code;
1969         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
1970
1971         vmexit = nested_svm_intercept(svm);
1972         if (vmexit == NESTED_EXIT_DONE)
1973                 svm->nested.exit_required = true;
1974
1975         return vmexit;
1976 }
1977
1978 /* This function returns true if it is save to enable the irq window */
1979 static inline bool nested_svm_intr(struct vcpu_svm *svm)
1980 {
1981         if (!is_guest_mode(&svm->vcpu))
1982                 return true;
1983
1984         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1985                 return true;
1986
1987         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1988                 return false;
1989
1990         /*
1991          * if vmexit was already requested (by intercepted exception
1992          * for instance) do not overwrite it with "external interrupt"
1993          * vmexit.
1994          */
1995         if (svm->nested.exit_required)
1996                 return false;
1997
1998         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
1999         svm->vmcb->control.exit_info_1 = 0;
2000         svm->vmcb->control.exit_info_2 = 0;
2001
2002         if (svm->nested.intercept & 1ULL) {
2003                 /*
2004                  * The #vmexit can't be emulated here directly because this
2005                  * code path runs with irqs and preemption disabled. A
2006                  * #vmexit emulation might sleep. Only signal request for
2007                  * the #vmexit here.
2008                  */
2009                 svm->nested.exit_required = true;
2010                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2011                 return false;
2012         }
2013
2014         return true;
2015 }
2016
2017 /* This function returns true if it is save to enable the nmi window */
2018 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2019 {
2020         if (!is_guest_mode(&svm->vcpu))
2021                 return true;
2022
2023         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2024                 return true;
2025
2026         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2027         svm->nested.exit_required = true;
2028
2029         return false;
2030 }
2031
2032 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2033 {
2034         struct page *page;
2035
2036         might_sleep();
2037
2038         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2039         if (is_error_page(page))
2040                 goto error;
2041
2042         *_page = page;
2043
2044         return kmap(page);
2045
2046 error:
2047         kvm_inject_gp(&svm->vcpu, 0);
2048
2049         return NULL;
2050 }
2051
2052 static void nested_svm_unmap(struct page *page)
2053 {
2054         kunmap(page);
2055         kvm_release_page_dirty(page);
2056 }
2057
2058 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2059 {
2060         unsigned port, size, iopm_len;
2061         u16 val, mask;
2062         u8 start_bit;
2063         u64 gpa;
2064
2065         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2066                 return NESTED_EXIT_HOST;
2067
2068         port = svm->vmcb->control.exit_info_1 >> 16;
2069         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2070                 SVM_IOIO_SIZE_SHIFT;
2071         gpa  = svm->nested.vmcb_iopm + (port / 8);
2072         start_bit = port % 8;
2073         iopm_len = (start_bit + size > 8) ? 2 : 1;
2074         mask = (0xf >> (4 - size)) << start_bit;
2075         val = 0;
2076
2077         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2078                 return NESTED_EXIT_DONE;
2079
2080         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2081 }
2082
2083 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2084 {
2085         u32 offset, msr, value;
2086         int write, mask;
2087
2088         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2089                 return NESTED_EXIT_HOST;
2090
2091         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2092         offset = svm_msrpm_offset(msr);
2093         write  = svm->vmcb->control.exit_info_1 & 1;
2094         mask   = 1 << ((2 * (msr & 0xf)) + write);
2095
2096         if (offset == MSR_INVALID)
2097                 return NESTED_EXIT_DONE;
2098
2099         /* Offset is in 32 bit units but need in 8 bit units */
2100         offset *= 4;
2101
2102         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2103                 return NESTED_EXIT_DONE;
2104
2105         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2106 }
2107
2108 static int nested_svm_exit_special(struct vcpu_svm *svm)
2109 {
2110         u32 exit_code = svm->vmcb->control.exit_code;
2111
2112         switch (exit_code) {
2113         case SVM_EXIT_INTR:
2114         case SVM_EXIT_NMI:
2115         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2116                 return NESTED_EXIT_HOST;
2117         case SVM_EXIT_NPF:
2118                 /* For now we are always handling NPFs when using them */
2119                 if (npt_enabled)
2120                         return NESTED_EXIT_HOST;
2121                 break;
2122         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2123                 /* When we're shadowing, trap PFs, but not async PF */
2124                 if (!npt_enabled && svm->apf_reason == 0)
2125                         return NESTED_EXIT_HOST;
2126                 break;
2127         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
2128                 nm_interception(svm);
2129                 break;
2130         default:
2131                 break;
2132         }
2133
2134         return NESTED_EXIT_CONTINUE;
2135 }
2136
2137 /*
2138  * If this function returns true, this #vmexit was already handled
2139  */
2140 static int nested_svm_intercept(struct vcpu_svm *svm)
2141 {
2142         u32 exit_code = svm->vmcb->control.exit_code;
2143         int vmexit = NESTED_EXIT_HOST;
2144
2145         switch (exit_code) {
2146         case SVM_EXIT_MSR:
2147                 vmexit = nested_svm_exit_handled_msr(svm);
2148                 break;
2149         case SVM_EXIT_IOIO:
2150                 vmexit = nested_svm_intercept_ioio(svm);
2151                 break;
2152         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2153                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2154                 if (svm->nested.intercept_cr & bit)
2155                         vmexit = NESTED_EXIT_DONE;
2156                 break;
2157         }
2158         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2159                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2160                 if (svm->nested.intercept_dr & bit)
2161                         vmexit = NESTED_EXIT_DONE;
2162                 break;
2163         }
2164         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2165                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2166                 if (svm->nested.intercept_exceptions & excp_bits)
2167                         vmexit = NESTED_EXIT_DONE;
2168                 /* async page fault always cause vmexit */
2169                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2170                          svm->apf_reason != 0)
2171                         vmexit = NESTED_EXIT_DONE;
2172                 break;
2173         }
2174         case SVM_EXIT_ERR: {
2175                 vmexit = NESTED_EXIT_DONE;
2176                 break;
2177         }
2178         default: {
2179                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2180                 if (svm->nested.intercept & exit_bits)
2181                         vmexit = NESTED_EXIT_DONE;
2182         }
2183         }
2184
2185         return vmexit;
2186 }
2187
2188 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2189 {
2190         int vmexit;
2191
2192         vmexit = nested_svm_intercept(svm);
2193
2194         if (vmexit == NESTED_EXIT_DONE)
2195                 nested_svm_vmexit(svm);
2196
2197         return vmexit;
2198 }
2199
2200 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2201 {
2202         struct vmcb_control_area *dst  = &dst_vmcb->control;
2203         struct vmcb_control_area *from = &from_vmcb->control;
2204
2205         dst->intercept_cr         = from->intercept_cr;
2206         dst->intercept_dr         = from->intercept_dr;
2207         dst->intercept_exceptions = from->intercept_exceptions;
2208         dst->intercept            = from->intercept;
2209         dst->iopm_base_pa         = from->iopm_base_pa;
2210         dst->msrpm_base_pa        = from->msrpm_base_pa;
2211         dst->tsc_offset           = from->tsc_offset;
2212         dst->asid                 = from->asid;
2213         dst->tlb_ctl              = from->tlb_ctl;
2214         dst->int_ctl              = from->int_ctl;
2215         dst->int_vector           = from->int_vector;
2216         dst->int_state            = from->int_state;
2217         dst->exit_code            = from->exit_code;
2218         dst->exit_code_hi         = from->exit_code_hi;
2219         dst->exit_info_1          = from->exit_info_1;
2220         dst->exit_info_2          = from->exit_info_2;
2221         dst->exit_int_info        = from->exit_int_info;
2222         dst->exit_int_info_err    = from->exit_int_info_err;
2223         dst->nested_ctl           = from->nested_ctl;
2224         dst->event_inj            = from->event_inj;
2225         dst->event_inj_err        = from->event_inj_err;
2226         dst->nested_cr3           = from->nested_cr3;
2227         dst->lbr_ctl              = from->lbr_ctl;
2228 }
2229
2230 static int nested_svm_vmexit(struct vcpu_svm *svm)
2231 {
2232         struct vmcb *nested_vmcb;
2233         struct vmcb *hsave = svm->nested.hsave;
2234         struct vmcb *vmcb = svm->vmcb;
2235         struct page *page;
2236
2237         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2238                                        vmcb->control.exit_info_1,
2239                                        vmcb->control.exit_info_2,
2240                                        vmcb->control.exit_int_info,
2241                                        vmcb->control.exit_int_info_err,
2242                                        KVM_ISA_SVM);
2243
2244         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2245         if (!nested_vmcb)
2246                 return 1;
2247
2248         /* Exit Guest-Mode */
2249         leave_guest_mode(&svm->vcpu);
2250         svm->nested.vmcb = 0;
2251
2252         /* Give the current vmcb to the guest */
2253         disable_gif(svm);
2254
2255         nested_vmcb->save.es     = vmcb->save.es;
2256         nested_vmcb->save.cs     = vmcb->save.cs;
2257         nested_vmcb->save.ss     = vmcb->save.ss;
2258         nested_vmcb->save.ds     = vmcb->save.ds;
2259         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2260         nested_vmcb->save.idtr   = vmcb->save.idtr;
2261         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2262         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2263         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
2264         nested_vmcb->save.cr2    = vmcb->save.cr2;
2265         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2266         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2267         nested_vmcb->save.rip    = vmcb->save.rip;
2268         nested_vmcb->save.rsp    = vmcb->save.rsp;
2269         nested_vmcb->save.rax    = vmcb->save.rax;
2270         nested_vmcb->save.dr7    = vmcb->save.dr7;
2271         nested_vmcb->save.dr6    = vmcb->save.dr6;
2272         nested_vmcb->save.cpl    = vmcb->save.cpl;
2273
2274         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2275         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2276         nested_vmcb->control.int_state         = vmcb->control.int_state;
2277         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2278         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2279         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2280         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2281         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2282         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2283
2284         if (svm->nrips_enabled)
2285                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
2286
2287         /*
2288          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2289          * to make sure that we do not lose injected events. So check event_inj
2290          * here and copy it to exit_int_info if it is valid.
2291          * Exit_int_info and event_inj can't be both valid because the case
2292          * below only happens on a VMRUN instruction intercept which has
2293          * no valid exit_int_info set.
2294          */
2295         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2296                 struct vmcb_control_area *nc = &nested_vmcb->control;
2297
2298                 nc->exit_int_info     = vmcb->control.event_inj;
2299                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2300         }
2301
2302         nested_vmcb->control.tlb_ctl           = 0;
2303         nested_vmcb->control.event_inj         = 0;
2304         nested_vmcb->control.event_inj_err     = 0;
2305
2306         /* We always set V_INTR_MASKING and remember the old value in hflags */
2307         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2308                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2309
2310         /* Restore the original control entries */
2311         copy_vmcb_control_area(vmcb, hsave);
2312
2313         kvm_clear_exception_queue(&svm->vcpu);
2314         kvm_clear_interrupt_queue(&svm->vcpu);
2315
2316         svm->nested.nested_cr3 = 0;
2317
2318         /* Restore selected save entries */
2319         svm->vmcb->save.es = hsave->save.es;
2320         svm->vmcb->save.cs = hsave->save.cs;
2321         svm->vmcb->save.ss = hsave->save.ss;
2322         svm->vmcb->save.ds = hsave->save.ds;
2323         svm->vmcb->save.gdtr = hsave->save.gdtr;
2324         svm->vmcb->save.idtr = hsave->save.idtr;
2325         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2326         svm_set_efer(&svm->vcpu, hsave->save.efer);
2327         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2328         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2329         if (npt_enabled) {
2330                 svm->vmcb->save.cr3 = hsave->save.cr3;
2331                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2332         } else {
2333                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2334         }
2335         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2336         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2337         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2338         svm->vmcb->save.dr7 = 0;
2339         svm->vmcb->save.cpl = 0;
2340         svm->vmcb->control.exit_int_info = 0;
2341
2342         mark_all_dirty(svm->vmcb);
2343
2344         nested_svm_unmap(page);
2345
2346         nested_svm_uninit_mmu_context(&svm->vcpu);
2347         kvm_mmu_reset_context(&svm->vcpu);
2348         kvm_mmu_load(&svm->vcpu);
2349
2350         return 0;
2351 }
2352
2353 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2354 {
2355         /*
2356          * This function merges the msr permission bitmaps of kvm and the
2357          * nested vmcb. It is optimized in that it only merges the parts where
2358          * the kvm msr permission bitmap may contain zero bits
2359          */
2360         int i;
2361
2362         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2363                 return true;
2364
2365         for (i = 0; i < MSRPM_OFFSETS; i++) {
2366                 u32 value, p;
2367                 u64 offset;
2368
2369                 if (msrpm_offsets[i] == 0xffffffff)
2370                         break;
2371
2372                 p      = msrpm_offsets[i];
2373                 offset = svm->nested.vmcb_msrpm + (p * 4);
2374
2375                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2376                         return false;
2377
2378                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2379         }
2380
2381         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2382
2383         return true;
2384 }
2385
2386 static bool nested_vmcb_checks(struct vmcb *vmcb)
2387 {
2388         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2389                 return false;
2390
2391         if (vmcb->control.asid == 0)
2392                 return false;
2393
2394         if (vmcb->control.nested_ctl && !npt_enabled)
2395                 return false;
2396
2397         return true;
2398 }
2399
2400 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2401 {
2402         struct vmcb *nested_vmcb;
2403         struct vmcb *hsave = svm->nested.hsave;
2404         struct vmcb *vmcb = svm->vmcb;
2405         struct page *page;
2406         u64 vmcb_gpa;
2407
2408         vmcb_gpa = svm->vmcb->save.rax;
2409
2410         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2411         if (!nested_vmcb)
2412                 return false;
2413
2414         if (!nested_vmcb_checks(nested_vmcb)) {
2415                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2416                 nested_vmcb->control.exit_code_hi = 0;
2417                 nested_vmcb->control.exit_info_1  = 0;
2418                 nested_vmcb->control.exit_info_2  = 0;
2419
2420                 nested_svm_unmap(page);
2421
2422                 return false;
2423         }
2424
2425         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2426                                nested_vmcb->save.rip,
2427                                nested_vmcb->control.int_ctl,
2428                                nested_vmcb->control.event_inj,
2429                                nested_vmcb->control.nested_ctl);
2430
2431         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2432                                     nested_vmcb->control.intercept_cr >> 16,
2433                                     nested_vmcb->control.intercept_exceptions,
2434                                     nested_vmcb->control.intercept);
2435
2436         /* Clear internal status */
2437         kvm_clear_exception_queue(&svm->vcpu);
2438         kvm_clear_interrupt_queue(&svm->vcpu);
2439
2440         /*
2441          * Save the old vmcb, so we don't need to pick what we save, but can
2442          * restore everything when a VMEXIT occurs
2443          */
2444         hsave->save.es     = vmcb->save.es;
2445         hsave->save.cs     = vmcb->save.cs;
2446         hsave->save.ss     = vmcb->save.ss;
2447         hsave->save.ds     = vmcb->save.ds;
2448         hsave->save.gdtr   = vmcb->save.gdtr;
2449         hsave->save.idtr   = vmcb->save.idtr;
2450         hsave->save.efer   = svm->vcpu.arch.efer;
2451         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2452         hsave->save.cr4    = svm->vcpu.arch.cr4;
2453         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2454         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2455         hsave->save.rsp    = vmcb->save.rsp;
2456         hsave->save.rax    = vmcb->save.rax;
2457         if (npt_enabled)
2458                 hsave->save.cr3    = vmcb->save.cr3;
2459         else
2460                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
2461
2462         copy_vmcb_control_area(hsave, vmcb);
2463
2464         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2465                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2466         else
2467                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2468
2469         if (nested_vmcb->control.nested_ctl) {
2470                 kvm_mmu_unload(&svm->vcpu);
2471                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2472                 nested_svm_init_mmu_context(&svm->vcpu);
2473         }
2474
2475         /* Load the nested guest state */
2476         svm->vmcb->save.es = nested_vmcb->save.es;
2477         svm->vmcb->save.cs = nested_vmcb->save.cs;
2478         svm->vmcb->save.ss = nested_vmcb->save.ss;
2479         svm->vmcb->save.ds = nested_vmcb->save.ds;
2480         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2481         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2482         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2483         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2484         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2485         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2486         if (npt_enabled) {
2487                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2488                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2489         } else
2490                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2491
2492         /* Guest paging mode is active - reset mmu */
2493         kvm_mmu_reset_context(&svm->vcpu);
2494
2495         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2496         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2497         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2498         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2499
2500         /* In case we don't even reach vcpu_run, the fields are not updated */
2501         svm->vmcb->save.rax = nested_vmcb->save.rax;
2502         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2503         svm->vmcb->save.rip = nested_vmcb->save.rip;
2504         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2505         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2506         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2507
2508         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2509         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2510
2511         /* cache intercepts */
2512         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2513         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
2514         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2515         svm->nested.intercept            = nested_vmcb->control.intercept;
2516
2517         svm_flush_tlb(&svm->vcpu);
2518         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2519         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2520                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2521         else
2522                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2523
2524         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2525                 /* We only want the cr8 intercept bits of the guest */
2526                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2527                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2528         }
2529
2530         /* We don't want to see VMMCALLs from a nested guest */
2531         clr_intercept(svm, INTERCEPT_VMMCALL);
2532
2533         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2534         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2535         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2536         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2537         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2538         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2539
2540         nested_svm_unmap(page);
2541
2542         /* Enter Guest-Mode */
2543         enter_guest_mode(&svm->vcpu);
2544
2545         /*
2546          * Merge guest and host intercepts - must be called  with vcpu in
2547          * guest-mode to take affect here
2548          */
2549         recalc_intercepts(svm);
2550
2551         svm->nested.vmcb = vmcb_gpa;
2552
2553         enable_gif(svm);
2554
2555         mark_all_dirty(svm->vmcb);
2556
2557         return true;
2558 }
2559
2560 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2561 {
2562         to_vmcb->save.fs = from_vmcb->save.fs;
2563         to_vmcb->save.gs = from_vmcb->save.gs;
2564         to_vmcb->save.tr = from_vmcb->save.tr;
2565         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2566         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2567         to_vmcb->save.star = from_vmcb->save.star;
2568         to_vmcb->save.lstar = from_vmcb->save.lstar;
2569         to_vmcb->save.cstar = from_vmcb->save.cstar;
2570         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2571         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2572         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2573         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2574 }
2575
2576 static int vmload_interception(struct vcpu_svm *svm)
2577 {
2578         struct vmcb *nested_vmcb;
2579         struct page *page;
2580
2581         if (nested_svm_check_permissions(svm))
2582                 return 1;
2583
2584         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2585         if (!nested_vmcb)
2586                 return 1;
2587
2588         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2589         skip_emulated_instruction(&svm->vcpu);
2590
2591         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2592         nested_svm_unmap(page);
2593
2594         return 1;
2595 }
2596
2597 static int vmsave_interception(struct vcpu_svm *svm)
2598 {
2599         struct vmcb *nested_vmcb;
2600         struct page *page;
2601
2602         if (nested_svm_check_permissions(svm))
2603                 return 1;
2604
2605         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2606         if (!nested_vmcb)
2607                 return 1;
2608
2609         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2610         skip_emulated_instruction(&svm->vcpu);
2611
2612         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2613         nested_svm_unmap(page);
2614
2615         return 1;
2616 }
2617
2618 static int vmrun_interception(struct vcpu_svm *svm)
2619 {
2620         if (nested_svm_check_permissions(svm))
2621                 return 1;
2622
2623         /* Save rip after vmrun instruction */
2624         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2625
2626         if (!nested_svm_vmrun(svm))
2627                 return 1;
2628
2629         if (!nested_svm_vmrun_msrpm(svm))
2630                 goto failed;
2631
2632         return 1;
2633
2634 failed:
2635
2636         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2637         svm->vmcb->control.exit_code_hi = 0;
2638         svm->vmcb->control.exit_info_1  = 0;
2639         svm->vmcb->control.exit_info_2  = 0;
2640
2641         nested_svm_vmexit(svm);
2642
2643         return 1;
2644 }
2645
2646 static int stgi_interception(struct vcpu_svm *svm)
2647 {
2648         if (nested_svm_check_permissions(svm))
2649                 return 1;
2650
2651         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2652         skip_emulated_instruction(&svm->vcpu);
2653         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2654
2655         enable_gif(svm);
2656
2657         return 1;
2658 }
2659
2660 static int clgi_interception(struct vcpu_svm *svm)
2661 {
2662         if (nested_svm_check_permissions(svm))
2663                 return 1;
2664
2665         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2666         skip_emulated_instruction(&svm->vcpu);
2667
2668         disable_gif(svm);
2669
2670         /* After a CLGI no interrupts should come */
2671         svm_clear_vintr(svm);
2672         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2673
2674         mark_dirty(svm->vmcb, VMCB_INTR);
2675
2676         return 1;
2677 }
2678
2679 static int invlpga_interception(struct vcpu_svm *svm)
2680 {
2681         struct kvm_vcpu *vcpu = &svm->vcpu;
2682
2683         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2684                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2685
2686         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2687         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2688
2689         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2690         skip_emulated_instruction(&svm->vcpu);
2691         return 1;
2692 }
2693
2694 static int skinit_interception(struct vcpu_svm *svm)
2695 {
2696         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2697
2698         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2699         return 1;
2700 }
2701
2702 static int wbinvd_interception(struct vcpu_svm *svm)
2703 {
2704         kvm_emulate_wbinvd(&svm->vcpu);
2705         return 1;
2706 }
2707
2708 static int xsetbv_interception(struct vcpu_svm *svm)
2709 {
2710         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2711         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
2712
2713         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2714                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2715                 skip_emulated_instruction(&svm->vcpu);
2716         }
2717
2718         return 1;
2719 }
2720
2721 static int task_switch_interception(struct vcpu_svm *svm)
2722 {
2723         u16 tss_selector;
2724         int reason;
2725         int int_type = svm->vmcb->control.exit_int_info &
2726                 SVM_EXITINTINFO_TYPE_MASK;
2727         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2728         uint32_t type =
2729                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2730         uint32_t idt_v =
2731                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2732         bool has_error_code = false;
2733         u32 error_code = 0;
2734
2735         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2736
2737         if (svm->vmcb->control.exit_info_2 &
2738             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2739                 reason = TASK_SWITCH_IRET;
2740         else if (svm->vmcb->control.exit_info_2 &
2741                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2742                 reason = TASK_SWITCH_JMP;
2743         else if (idt_v)
2744                 reason = TASK_SWITCH_GATE;
2745         else
2746                 reason = TASK_SWITCH_CALL;
2747
2748         if (reason == TASK_SWITCH_GATE) {
2749                 switch (type) {
2750                 case SVM_EXITINTINFO_TYPE_NMI:
2751                         svm->vcpu.arch.nmi_injected = false;
2752                         break;
2753                 case SVM_EXITINTINFO_TYPE_EXEPT:
2754                         if (svm->vmcb->control.exit_info_2 &
2755                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2756                                 has_error_code = true;
2757                                 error_code =
2758                                         (u32)svm->vmcb->control.exit_info_2;
2759                         }
2760                         kvm_clear_exception_queue(&svm->vcpu);
2761                         break;
2762                 case SVM_EXITINTINFO_TYPE_INTR:
2763                         kvm_clear_interrupt_queue(&svm->vcpu);
2764                         break;
2765                 default:
2766                         break;
2767                 }
2768         }
2769
2770         if (reason != TASK_SWITCH_GATE ||
2771             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2772             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2773              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2774                 skip_emulated_instruction(&svm->vcpu);
2775
2776         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2777                 int_vec = -1;
2778
2779         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2780                                 has_error_code, error_code) == EMULATE_FAIL) {
2781                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2782                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2783                 svm->vcpu.run->internal.ndata = 0;
2784                 return 0;
2785         }
2786         return 1;
2787 }
2788
2789 static int cpuid_interception(struct vcpu_svm *svm)
2790 {
2791         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2792         kvm_emulate_cpuid(&svm->vcpu);
2793         return 1;
2794 }
2795
2796 static int iret_interception(struct vcpu_svm *svm)
2797 {
2798         ++svm->vcpu.stat.nmi_window_exits;
2799         clr_intercept(svm, INTERCEPT_IRET);
2800         svm->vcpu.arch.hflags |= HF_IRET_MASK;
2801         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2802         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2803         return 1;
2804 }
2805
2806 static int invlpg_interception(struct vcpu_svm *svm)
2807 {
2808         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2809                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2810
2811         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2812         skip_emulated_instruction(&svm->vcpu);
2813         return 1;
2814 }
2815
2816 static int emulate_on_interception(struct vcpu_svm *svm)
2817 {
2818         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2819 }
2820
2821 static int rdpmc_interception(struct vcpu_svm *svm)
2822 {
2823         int err;
2824
2825         if (!static_cpu_has(X86_FEATURE_NRIPS))
2826                 return emulate_on_interception(svm);
2827
2828         err = kvm_rdpmc(&svm->vcpu);
2829         kvm_complete_insn_gp(&svm->vcpu, err);
2830
2831         return 1;
2832 }
2833
2834 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2835                                             unsigned long val)
2836 {
2837         unsigned long cr0 = svm->vcpu.arch.cr0;
2838         bool ret = false;
2839         u64 intercept;
2840
2841         intercept = svm->nested.intercept;
2842
2843         if (!is_guest_mode(&svm->vcpu) ||
2844             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
2845                 return false;
2846
2847         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2848         val &= ~SVM_CR0_SELECTIVE_MASK;
2849
2850         if (cr0 ^ val) {
2851                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2852                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2853         }
2854
2855         return ret;
2856 }
2857
2858 #define CR_VALID (1ULL << 63)
2859
2860 static int cr_interception(struct vcpu_svm *svm)
2861 {
2862         int reg, cr;
2863         unsigned long val;
2864         int err;
2865
2866         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2867                 return emulate_on_interception(svm);
2868
2869         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2870                 return emulate_on_interception(svm);
2871
2872         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2873         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2874                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2875         else
2876                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2877
2878         err = 0;
2879         if (cr >= 16) { /* mov to cr */
2880                 cr -= 16;
2881                 val = kvm_register_read(&svm->vcpu, reg);
2882                 switch (cr) {
2883                 case 0:
2884                         if (!check_selective_cr0_intercepted(svm, val))
2885                                 err = kvm_set_cr0(&svm->vcpu, val);
2886                         else
2887                                 return 1;
2888
2889                         break;
2890                 case 3:
2891                         err = kvm_set_cr3(&svm->vcpu, val);
2892                         break;
2893                 case 4:
2894                         err = kvm_set_cr4(&svm->vcpu, val);
2895                         break;
2896                 case 8:
2897                         err = kvm_set_cr8(&svm->vcpu, val);
2898                         break;
2899                 default:
2900                         WARN(1, "unhandled write to CR%d", cr);
2901                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2902                         return 1;
2903                 }
2904         } else { /* mov from cr */
2905                 switch (cr) {
2906                 case 0:
2907                         val = kvm_read_cr0(&svm->vcpu);
2908                         break;
2909                 case 2:
2910                         val = svm->vcpu.arch.cr2;
2911                         break;
2912                 case 3:
2913                         val = kvm_read_cr3(&svm->vcpu);
2914                         break;
2915                 case 4:
2916                         val = kvm_read_cr4(&svm->vcpu);
2917                         break;
2918                 case 8:
2919                         val = kvm_get_cr8(&svm->vcpu);
2920                         break;
2921                 default:
2922                         WARN(1, "unhandled read from CR%d", cr);
2923                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2924                         return 1;
2925                 }
2926                 kvm_register_write(&svm->vcpu, reg, val);
2927         }
2928         kvm_complete_insn_gp(&svm->vcpu, err);
2929
2930         return 1;
2931 }
2932
2933 static int dr_interception(struct vcpu_svm *svm)
2934 {
2935         int reg, dr;
2936         unsigned long val;
2937
2938         if (svm->vcpu.guest_debug == 0) {
2939                 /*
2940                  * No more DR vmexits; force a reload of the debug registers
2941                  * and reenter on this instruction.  The next vmexit will
2942                  * retrieve the full state of the debug registers.
2943                  */
2944                 clr_dr_intercepts(svm);
2945                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2946                 return 1;
2947         }
2948
2949         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2950                 return emulate_on_interception(svm);
2951
2952         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2953         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2954
2955         if (dr >= 16) { /* mov to DRn */
2956                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
2957                         return 1;
2958                 val = kvm_register_read(&svm->vcpu, reg);
2959                 kvm_set_dr(&svm->vcpu, dr - 16, val);
2960         } else {
2961                 if (!kvm_require_dr(&svm->vcpu, dr))
2962                         return 1;
2963                 kvm_get_dr(&svm->vcpu, dr, &val);
2964                 kvm_register_write(&svm->vcpu, reg, val);
2965         }
2966
2967         skip_emulated_instruction(&svm->vcpu);
2968
2969         return 1;
2970 }
2971
2972 static int cr8_write_interception(struct vcpu_svm *svm)
2973 {
2974         struct kvm_run *kvm_run = svm->vcpu.run;
2975         int r;
2976
2977         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2978         /* instruction emulation calls kvm_set_cr8() */
2979         r = cr_interception(svm);
2980         if (lapic_in_kernel(&svm->vcpu))
2981                 return r;
2982         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2983                 return r;
2984         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2985         return 0;
2986 }
2987
2988 static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2989 {
2990         struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu));
2991         return vmcb->control.tsc_offset + host_tsc;
2992 }
2993
2994 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2995 {
2996         struct vcpu_svm *svm = to_svm(vcpu);
2997
2998         switch (msr_info->index) {
2999         case MSR_IA32_TSC: {
3000                 msr_info->data = svm->vmcb->control.tsc_offset +
3001                         kvm_scale_tsc(vcpu, rdtsc());
3002
3003                 break;
3004         }
3005         case MSR_STAR:
3006                 msr_info->data = svm->vmcb->save.star;
3007                 break;
3008 #ifdef CONFIG_X86_64
3009         case MSR_LSTAR:
3010                 msr_info->data = svm->vmcb->save.lstar;
3011                 break;
3012         case MSR_CSTAR:
3013                 msr_info->data = svm->vmcb->save.cstar;
3014                 break;
3015         case MSR_KERNEL_GS_BASE:
3016                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3017                 break;
3018         case MSR_SYSCALL_MASK:
3019                 msr_info->data = svm->vmcb->save.sfmask;
3020                 break;
3021 #endif
3022         case MSR_IA32_SYSENTER_CS:
3023                 msr_info->data = svm->vmcb->save.sysenter_cs;
3024                 break;
3025         case MSR_IA32_SYSENTER_EIP:
3026                 msr_info->data = svm->sysenter_eip;
3027                 break;
3028         case MSR_IA32_SYSENTER_ESP:
3029                 msr_info->data = svm->sysenter_esp;
3030                 break;
3031         case MSR_TSC_AUX:
3032                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3033                         return 1;
3034                 msr_info->data = svm->tsc_aux;
3035                 break;
3036         /*
3037          * Nobody will change the following 5 values in the VMCB so we can
3038          * safely return them on rdmsr. They will always be 0 until LBRV is
3039          * implemented.
3040          */
3041         case MSR_IA32_DEBUGCTLMSR:
3042                 msr_info->data = svm->vmcb->save.dbgctl;
3043                 break;
3044         case MSR_IA32_LASTBRANCHFROMIP:
3045                 msr_info->data = svm->vmcb->save.br_from;
3046                 break;
3047         case MSR_IA32_LASTBRANCHTOIP:
3048                 msr_info->data = svm->vmcb->save.br_to;
3049                 break;
3050         case MSR_IA32_LASTINTFROMIP:
3051                 msr_info->data = svm->vmcb->save.last_excp_from;
3052                 break;
3053         case MSR_IA32_LASTINTTOIP:
3054                 msr_info->data = svm->vmcb->save.last_excp_to;
3055                 break;
3056         case MSR_VM_HSAVE_PA:
3057                 msr_info->data = svm->nested.hsave_msr;
3058                 break;
3059         case MSR_VM_CR:
3060                 msr_info->data = svm->nested.vm_cr_msr;
3061                 break;
3062         case MSR_IA32_UCODE_REV:
3063                 msr_info->data = 0x01000065;
3064                 break;
3065         case MSR_F15H_IC_CFG: {
3066
3067                 int family, model;
3068
3069                 family = guest_cpuid_family(vcpu);
3070                 model  = guest_cpuid_model(vcpu);
3071
3072                 if (family < 0 || model < 0)
3073                         return kvm_get_msr_common(vcpu, msr_info);
3074
3075                 msr_info->data = 0;
3076
3077                 if (family == 0x15 &&
3078                     (model >= 0x2 && model < 0x20))
3079                         msr_info->data = 0x1E;
3080                 }
3081                 break;
3082         default:
3083                 return kvm_get_msr_common(vcpu, msr_info);
3084         }
3085         return 0;
3086 }
3087
3088 static int rdmsr_interception(struct vcpu_svm *svm)
3089 {
3090         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3091         struct msr_data msr_info;
3092
3093         msr_info.index = ecx;
3094         msr_info.host_initiated = false;
3095         if (svm_get_msr(&svm->vcpu, &msr_info)) {
3096                 trace_kvm_msr_read_ex(ecx);
3097                 kvm_inject_gp(&svm->vcpu, 0);
3098         } else {
3099                 trace_kvm_msr_read(ecx, msr_info.data);
3100
3101                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3102                                    msr_info.data & 0xffffffff);
3103                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3104                                    msr_info.data >> 32);
3105                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3106                 skip_emulated_instruction(&svm->vcpu);
3107         }
3108         return 1;
3109 }
3110
3111 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3112 {
3113         struct vcpu_svm *svm = to_svm(vcpu);
3114         int svm_dis, chg_mask;
3115
3116         if (data & ~SVM_VM_CR_VALID_MASK)
3117                 return 1;
3118
3119         chg_mask = SVM_VM_CR_VALID_MASK;
3120
3121         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3122                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3123
3124         svm->nested.vm_cr_msr &= ~chg_mask;
3125         svm->nested.vm_cr_msr |= (data & chg_mask);
3126
3127         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3128
3129         /* check for svm_disable while efer.svme is set */
3130         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3131                 return 1;
3132
3133         return 0;
3134 }
3135
3136 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3137 {
3138         struct vcpu_svm *svm = to_svm(vcpu);
3139
3140         u32 ecx = msr->index;
3141         u64 data = msr->data;
3142         switch (ecx) {
3143         case MSR_IA32_TSC:
3144                 kvm_write_tsc(vcpu, msr);
3145                 break;
3146         case MSR_STAR:
3147                 svm->vmcb->save.star = data;
3148                 break;
3149 #ifdef CONFIG_X86_64
3150         case MSR_LSTAR:
3151                 svm->vmcb->save.lstar = data;
3152                 break;
3153         case MSR_CSTAR:
3154                 svm->vmcb->save.cstar = data;
3155                 break;
3156         case MSR_KERNEL_GS_BASE:
3157                 svm->vmcb->save.kernel_gs_base = data;
3158                 break;
3159         case MSR_SYSCALL_MASK:
3160                 svm->vmcb->save.sfmask = data;
3161                 break;
3162 #endif
3163         case MSR_IA32_SYSENTER_CS:
3164                 svm->vmcb->save.sysenter_cs = data;
3165                 break;
3166         case MSR_IA32_SYSENTER_EIP:
3167                 svm->sysenter_eip = data;
3168                 svm->vmcb->save.sysenter_eip = data;
3169                 break;
3170         case MSR_IA32_SYSENTER_ESP:
3171                 svm->sysenter_esp = data;
3172                 svm->vmcb->save.sysenter_esp = data;
3173                 break;
3174         case MSR_TSC_AUX:
3175                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3176                         return 1;
3177
3178                 /*
3179                  * This is rare, so we update the MSR here instead of using
3180                  * direct_access_msrs.  Doing that would require a rdmsr in
3181                  * svm_vcpu_put.
3182                  */
3183                 svm->tsc_aux = data;
3184                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
3185                 break;
3186         case MSR_IA32_DEBUGCTLMSR:
3187                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3188                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3189                                     __func__, data);
3190                         break;
3191                 }
3192                 if (data & DEBUGCTL_RESERVED_BITS)
3193                         return 1;
3194
3195                 svm->vmcb->save.dbgctl = data;
3196                 mark_dirty(svm->vmcb, VMCB_LBR);
3197                 if (data & (1ULL<<0))
3198                         svm_enable_lbrv(svm);
3199                 else
3200                         svm_disable_lbrv(svm);
3201                 break;
3202         case MSR_VM_HSAVE_PA:
3203                 svm->nested.hsave_msr = data;
3204                 break;
3205         case MSR_VM_CR:
3206                 return svm_set_vm_cr(vcpu, data);
3207         case MSR_VM_IGNNE:
3208                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3209                 break;
3210         default:
3211                 return kvm_set_msr_common(vcpu, msr);
3212         }
3213         return 0;
3214 }
3215
3216 static int wrmsr_interception(struct vcpu_svm *svm)
3217 {
3218         struct msr_data msr;
3219         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3220         u64 data = kvm_read_edx_eax(&svm->vcpu);
3221
3222         msr.data = data;
3223         msr.index = ecx;
3224         msr.host_initiated = false;
3225
3226         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3227         if (kvm_set_msr(&svm->vcpu, &msr)) {
3228                 trace_kvm_msr_write_ex(ecx, data);
3229                 kvm_inject_gp(&svm->vcpu, 0);
3230         } else {
3231                 trace_kvm_msr_write(ecx, data);
3232                 skip_emulated_instruction(&svm->vcpu);
3233         }
3234         return 1;
3235 }
3236
3237 static int msr_interception(struct vcpu_svm *svm)
3238 {
3239         if (svm->vmcb->control.exit_info_1)
3240                 return wrmsr_interception(svm);
3241         else
3242                 return rdmsr_interception(svm);
3243 }
3244
3245 static int interrupt_window_interception(struct vcpu_svm *svm)
3246 {
3247         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3248         svm_clear_vintr(svm);
3249         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3250         mark_dirty(svm->vmcb, VMCB_INTR);
3251         ++svm->vcpu.stat.irq_window_exits;
3252         return 1;
3253 }
3254
3255 static int pause_interception(struct vcpu_svm *svm)
3256 {
3257         kvm_vcpu_on_spin(&(svm->vcpu));
3258         return 1;
3259 }
3260
3261 static int nop_interception(struct vcpu_svm *svm)
3262 {
3263         skip_emulated_instruction(&(svm->vcpu));
3264         return 1;
3265 }
3266
3267 static int monitor_interception(struct vcpu_svm *svm)
3268 {
3269         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3270         return nop_interception(svm);
3271 }
3272
3273 static int mwait_interception(struct vcpu_svm *svm)
3274 {
3275         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3276         return nop_interception(svm);
3277 }
3278
3279 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3280         [SVM_EXIT_READ_CR0]                     = cr_interception,
3281         [SVM_EXIT_READ_CR3]                     = cr_interception,
3282         [SVM_EXIT_READ_CR4]                     = cr_interception,
3283         [SVM_EXIT_READ_CR8]                     = cr_interception,
3284         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3285         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3286         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3287         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3288         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3289         [SVM_EXIT_READ_DR0]                     = dr_interception,
3290         [SVM_EXIT_READ_DR1]                     = dr_interception,
3291         [SVM_EXIT_READ_DR2]                     = dr_interception,
3292         [SVM_EXIT_READ_DR3]                     = dr_interception,
3293         [SVM_EXIT_READ_DR4]                     = dr_interception,
3294         [SVM_EXIT_READ_DR5]                     = dr_interception,
3295         [SVM_EXIT_READ_DR6]                     = dr_interception,
3296         [SVM_EXIT_READ_DR7]                     = dr_interception,
3297         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3298         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3299         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3300         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3301         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3302         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3303         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3304         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3305         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3306         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3307         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3308         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3309         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
3310         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3311         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
3312         [SVM_EXIT_INTR]                         = intr_interception,
3313         [SVM_EXIT_NMI]                          = nmi_interception,
3314         [SVM_EXIT_SMI]                          = nop_on_interception,
3315         [SVM_EXIT_INIT]                         = nop_on_interception,
3316         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3317         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
3318         [SVM_EXIT_CPUID]                        = cpuid_interception,
3319         [SVM_EXIT_IRET]                         = iret_interception,
3320         [SVM_EXIT_INVD]                         = emulate_on_interception,
3321         [SVM_EXIT_PAUSE]                        = pause_interception,
3322         [SVM_EXIT_HLT]                          = halt_interception,
3323         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3324         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3325         [SVM_EXIT_IOIO]                         = io_interception,
3326         [SVM_EXIT_MSR]                          = msr_interception,
3327         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3328         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3329         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3330         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
3331         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3332         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3333         [SVM_EXIT_STGI]                         = stgi_interception,
3334         [SVM_EXIT_CLGI]                         = clgi_interception,
3335         [SVM_EXIT_SKINIT]                       = skinit_interception,
3336         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
3337         [SVM_EXIT_MONITOR]                      = monitor_interception,
3338         [SVM_EXIT_MWAIT]                        = mwait_interception,
3339         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
3340         [SVM_EXIT_NPF]                          = pf_interception,
3341         [SVM_EXIT_RSM]                          = emulate_on_interception,
3342 };
3343
3344 static void dump_vmcb(struct kvm_vcpu *vcpu)
3345 {
3346         struct vcpu_svm *svm = to_svm(vcpu);
3347         struct vmcb_control_area *control = &svm->vmcb->control;
3348         struct vmcb_save_area *save = &svm->vmcb->save;
3349
3350         pr_err("VMCB Control Area:\n");
3351         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
3352         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
3353         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
3354         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
3355         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
3356         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
3357         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3358         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3359         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3360         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3361         pr_err("%-20s%d\n", "asid:", control->asid);
3362         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3363         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3364         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3365         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3366         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3367         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3368         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3369         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3370         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3371         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3372         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3373         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3374         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3375         pr_err("%-20s%lld\n", "lbr_ctl:", control->lbr_ctl);
3376         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3377         pr_err("VMCB State Save Area:\n");
3378         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3379                "es:",
3380                save->es.selector, save->es.attrib,
3381                save->es.limit, save->es.base);
3382         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3383                "cs:",
3384                save->cs.selector, save->cs.attrib,
3385                save->cs.limit, save->cs.base);
3386         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3387                "ss:",
3388                save->ss.selector, save->ss.attrib,
3389                save->ss.limit, save->ss.base);
3390         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3391                "ds:",
3392                save->ds.selector, save->ds.attrib,
3393                save->ds.limit, save->ds.base);
3394         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3395                "fs:",
3396                save->fs.selector, save->fs.attrib,
3397                save->fs.limit, save->fs.base);
3398         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3399                "gs:",
3400                save->gs.selector, save->gs.attrib,
3401                save->gs.limit, save->gs.base);
3402         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3403                "gdtr:",
3404                save->gdtr.selector, save->gdtr.attrib,
3405                save->gdtr.limit, save->gdtr.base);
3406         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3407                "ldtr:",
3408                save->ldtr.selector, save->ldtr.attrib,
3409                save->ldtr.limit, save->ldtr.base);
3410         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3411                "idtr:",
3412                save->idtr.selector, save->idtr.attrib,
3413                save->idtr.limit, save->idtr.base);
3414         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3415                "tr:",
3416                save->tr.selector, save->tr.attrib,
3417                save->tr.limit, save->tr.base);
3418         pr_err("cpl:            %d                efer:         %016llx\n",
3419                 save->cpl, save->efer);
3420         pr_err("%-15s %016llx %-13s %016llx\n",
3421                "cr0:", save->cr0, "cr2:", save->cr2);
3422         pr_err("%-15s %016llx %-13s %016llx\n",
3423                "cr3:", save->cr3, "cr4:", save->cr4);
3424         pr_err("%-15s %016llx %-13s %016llx\n",
3425                "dr6:", save->dr6, "dr7:", save->dr7);
3426         pr_err("%-15s %016llx %-13s %016llx\n",
3427                "rip:", save->rip, "rflags:", save->rflags);
3428         pr_err("%-15s %016llx %-13s %016llx\n",
3429                "rsp:", save->rsp, "rax:", save->rax);
3430         pr_err("%-15s %016llx %-13s %016llx\n",
3431                "star:", save->star, "lstar:", save->lstar);
3432         pr_err("%-15s %016llx %-13s %016llx\n",
3433                "cstar:", save->cstar, "sfmask:", save->sfmask);
3434         pr_err("%-15s %016llx %-13s %016llx\n",
3435                "kernel_gs_base:", save->kernel_gs_base,
3436                "sysenter_cs:", save->sysenter_cs);
3437         pr_err("%-15s %016llx %-13s %016llx\n",
3438                "sysenter_esp:", save->sysenter_esp,
3439                "sysenter_eip:", save->sysenter_eip);
3440         pr_err("%-15s %016llx %-13s %016llx\n",
3441                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3442         pr_err("%-15s %016llx %-13s %016llx\n",
3443                "br_from:", save->br_from, "br_to:", save->br_to);
3444         pr_err("%-15s %016llx %-13s %016llx\n",
3445                "excp_from:", save->last_excp_from,
3446                "excp_to:", save->last_excp_to);
3447 }
3448
3449 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
3450 {
3451         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3452
3453         *info1 = control->exit_info_1;
3454         *info2 = control->exit_info_2;
3455 }
3456
3457 static int handle_exit(struct kvm_vcpu *vcpu)
3458 {
3459         struct vcpu_svm *svm = to_svm(vcpu);
3460         struct kvm_run *kvm_run = vcpu->run;
3461         u32 exit_code = svm->vmcb->control.exit_code;
3462
3463         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3464
3465         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3466                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3467         if (npt_enabled)
3468                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3469
3470         if (unlikely(svm->nested.exit_required)) {
3471                 nested_svm_vmexit(svm);
3472                 svm->nested.exit_required = false;
3473
3474                 return 1;
3475         }
3476
3477         if (is_guest_mode(vcpu)) {
3478                 int vmexit;
3479
3480                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3481                                         svm->vmcb->control.exit_info_1,
3482                                         svm->vmcb->control.exit_info_2,
3483                                         svm->vmcb->control.exit_int_info,
3484                                         svm->vmcb->control.exit_int_info_err,
3485                                         KVM_ISA_SVM);
3486
3487                 vmexit = nested_svm_exit_special(svm);
3488
3489                 if (vmexit == NESTED_EXIT_CONTINUE)
3490                         vmexit = nested_svm_exit_handled(svm);
3491
3492                 if (vmexit == NESTED_EXIT_DONE)
3493                         return 1;
3494         }
3495
3496         svm_complete_interrupts(svm);
3497
3498         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3499                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3500                 kvm_run->fail_entry.hardware_entry_failure_reason
3501                         = svm->vmcb->control.exit_code;
3502                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3503                 dump_vmcb(vcpu);
3504                 return 0;
3505         }
3506
3507         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3508             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3509             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3510             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3511                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3512                        "exit_code 0x%x\n",
3513                        __func__, svm->vmcb->control.exit_int_info,
3514                        exit_code);
3515
3516         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3517             || !svm_exit_handlers[exit_code]) {
3518                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3519                 kvm_queue_exception(vcpu, UD_VECTOR);
3520                 return 1;
3521         }
3522
3523         return svm_exit_handlers[exit_code](svm);
3524 }
3525
3526 static void reload_tss(struct kvm_vcpu *vcpu)
3527 {
3528         int cpu = raw_smp_processor_id();
3529
3530         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3531         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3532         load_TR_desc();
3533 }
3534
3535 static void pre_svm_run(struct vcpu_svm *svm)
3536 {
3537         int cpu = raw_smp_processor_id();
3538
3539         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3540
3541         /* FIXME: handle wraparound of asid_generation */
3542         if (svm->asid_generation != sd->asid_generation)
3543                 new_asid(svm, sd);
3544 }
3545
3546 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3547 {
3548         struct vcpu_svm *svm = to_svm(vcpu);
3549
3550         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3551         vcpu->arch.hflags |= HF_NMI_MASK;
3552         set_intercept(svm, INTERCEPT_IRET);
3553         ++vcpu->stat.nmi_injections;
3554 }
3555
3556 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3557 {
3558         struct vmcb_control_area *control;
3559
3560         control = &svm->vmcb->control;
3561         control->int_vector = irq;
3562         control->int_ctl &= ~V_INTR_PRIO_MASK;
3563         control->int_ctl |= V_IRQ_MASK |
3564                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3565         mark_dirty(svm->vmcb, VMCB_INTR);
3566 }
3567
3568 static void svm_set_irq(struct kvm_vcpu *vcpu)
3569 {
3570         struct vcpu_svm *svm = to_svm(vcpu);
3571
3572         BUG_ON(!(gif_set(svm)));
3573
3574         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3575         ++vcpu->stat.irq_injections;
3576
3577         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3578                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3579 }
3580
3581 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3582 {
3583         struct vcpu_svm *svm = to_svm(vcpu);
3584
3585         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3586                 return;
3587
3588         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3589
3590         if (irr == -1)
3591                 return;
3592
3593         if (tpr >= irr)
3594                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3595 }
3596
3597 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
3598 {
3599         return;
3600 }
3601
3602 static bool svm_get_enable_apicv(void)
3603 {
3604         return false;
3605 }
3606
3607 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
3608 {
3609 }
3610
3611 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
3612 {
3613         return;
3614 }
3615
3616 static void svm_sync_pir_to_irr(struct kvm_vcpu *vcpu)
3617 {
3618         return;
3619 }
3620
3621 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3622 {
3623         struct vcpu_svm *svm = to_svm(vcpu);
3624         struct vmcb *vmcb = svm->vmcb;
3625         int ret;
3626         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3627               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3628         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3629
3630         return ret;
3631 }
3632
3633 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3634 {
3635         struct vcpu_svm *svm = to_svm(vcpu);
3636
3637         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3638 }
3639
3640 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3641 {
3642         struct vcpu_svm *svm = to_svm(vcpu);
3643
3644         if (masked) {
3645                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3646                 set_intercept(svm, INTERCEPT_IRET);
3647         } else {
3648                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3649                 clr_intercept(svm, INTERCEPT_IRET);
3650         }
3651 }
3652
3653 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3654 {
3655         struct vcpu_svm *svm = to_svm(vcpu);
3656         struct vmcb *vmcb = svm->vmcb;
3657         int ret;
3658
3659         if (!gif_set(svm) ||
3660              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3661                 return 0;
3662
3663         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3664
3665         if (is_guest_mode(vcpu))
3666                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3667
3668         return ret;
3669 }
3670
3671 static void enable_irq_window(struct kvm_vcpu *vcpu)
3672 {
3673         struct vcpu_svm *svm = to_svm(vcpu);
3674
3675         /*
3676          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3677          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3678          * get that intercept, this function will be called again though and
3679          * we'll get the vintr intercept.
3680          */
3681         if (gif_set(svm) && nested_svm_intr(svm)) {
3682                 svm_set_vintr(svm);
3683                 svm_inject_irq(svm, 0x0);
3684         }
3685 }
3686
3687 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3688 {
3689         struct vcpu_svm *svm = to_svm(vcpu);
3690
3691         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3692             == HF_NMI_MASK)
3693                 return; /* IRET will cause a vm exit */
3694
3695         /*
3696          * Something prevents NMI from been injected. Single step over possible
3697          * problem (IRET or exception injection or interrupt shadow)
3698          */
3699         svm->nmi_singlestep = true;
3700         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3701 }
3702
3703 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3704 {
3705         return 0;
3706 }
3707
3708 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3709 {
3710         struct vcpu_svm *svm = to_svm(vcpu);
3711
3712         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3713                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3714         else
3715                 svm->asid_generation--;
3716 }
3717
3718 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3719 {
3720 }
3721
3722 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3723 {
3724         struct vcpu_svm *svm = to_svm(vcpu);
3725
3726         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3727                 return;
3728
3729         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3730                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3731                 kvm_set_cr8(vcpu, cr8);
3732         }
3733 }
3734
3735 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3736 {
3737         struct vcpu_svm *svm = to_svm(vcpu);
3738         u64 cr8;
3739
3740         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3741                 return;
3742
3743         cr8 = kvm_get_cr8(vcpu);
3744         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3745         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3746 }
3747
3748 static void svm_complete_interrupts(struct vcpu_svm *svm)
3749 {
3750         u8 vector;
3751         int type;
3752         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3753         unsigned int3_injected = svm->int3_injected;
3754
3755         svm->int3_injected = 0;
3756
3757         /*
3758          * If we've made progress since setting HF_IRET_MASK, we've
3759          * executed an IRET and can allow NMI injection.
3760          */
3761         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3762             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3763                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3764                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3765         }
3766
3767         svm->vcpu.arch.nmi_injected = false;
3768         kvm_clear_exception_queue(&svm->vcpu);
3769         kvm_clear_interrupt_queue(&svm->vcpu);
3770
3771         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3772                 return;
3773
3774         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3775
3776         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3777         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3778
3779         switch (type) {
3780         case SVM_EXITINTINFO_TYPE_NMI:
3781                 svm->vcpu.arch.nmi_injected = true;
3782                 break;
3783         case SVM_EXITINTINFO_TYPE_EXEPT:
3784                 /*
3785                  * In case of software exceptions, do not reinject the vector,
3786                  * but re-execute the instruction instead. Rewind RIP first
3787                  * if we emulated INT3 before.
3788                  */
3789                 if (kvm_exception_is_soft(vector)) {
3790                         if (vector == BP_VECTOR && int3_injected &&
3791                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3792                                 kvm_rip_write(&svm->vcpu,
3793                                               kvm_rip_read(&svm->vcpu) -
3794                                               int3_injected);
3795                         break;
3796                 }
3797                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3798                         u32 err = svm->vmcb->control.exit_int_info_err;
3799                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
3800
3801                 } else
3802                         kvm_requeue_exception(&svm->vcpu, vector);
3803                 break;
3804         case SVM_EXITINTINFO_TYPE_INTR:
3805                 kvm_queue_interrupt(&svm->vcpu, vector, false);
3806                 break;
3807         default:
3808                 break;
3809         }
3810 }
3811
3812 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3813 {
3814         struct vcpu_svm *svm = to_svm(vcpu);
3815         struct vmcb_control_area *control = &svm->vmcb->control;
3816
3817         control->exit_int_info = control->event_inj;
3818         control->exit_int_info_err = control->event_inj_err;
3819         control->event_inj = 0;
3820         svm_complete_interrupts(svm);
3821 }
3822
3823 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3824 {
3825         struct vcpu_svm *svm = to_svm(vcpu);
3826
3827         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3828         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3829         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3830
3831         /*
3832          * A vmexit emulation is required before the vcpu can be executed
3833          * again.
3834          */
3835         if (unlikely(svm->nested.exit_required))
3836                 return;
3837
3838         pre_svm_run(svm);
3839
3840         sync_lapic_to_cr8(vcpu);
3841
3842         svm->vmcb->save.cr2 = vcpu->arch.cr2;
3843
3844         clgi();
3845
3846         local_irq_enable();
3847
3848         asm volatile (
3849                 "push %%" _ASM_BP "; \n\t"
3850                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
3851                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
3852                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
3853                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
3854                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
3855                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
3856 #ifdef CONFIG_X86_64
3857                 "mov %c[r8](%[svm]),  %%r8  \n\t"
3858                 "mov %c[r9](%[svm]),  %%r9  \n\t"
3859                 "mov %c[r10](%[svm]), %%r10 \n\t"
3860                 "mov %c[r11](%[svm]), %%r11 \n\t"
3861                 "mov %c[r12](%[svm]), %%r12 \n\t"
3862                 "mov %c[r13](%[svm]), %%r13 \n\t"
3863                 "mov %c[r14](%[svm]), %%r14 \n\t"
3864                 "mov %c[r15](%[svm]), %%r15 \n\t"
3865 #endif
3866
3867                 /* Enter guest mode */
3868                 "push %%" _ASM_AX " \n\t"
3869                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
3870                 __ex(SVM_VMLOAD) "\n\t"
3871                 __ex(SVM_VMRUN) "\n\t"
3872                 __ex(SVM_VMSAVE) "\n\t"
3873                 "pop %%" _ASM_AX " \n\t"
3874
3875                 /* Save guest registers, load host registers */
3876                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
3877                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
3878                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
3879                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
3880                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
3881                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
3882 #ifdef CONFIG_X86_64
3883                 "mov %%r8,  %c[r8](%[svm]) \n\t"
3884                 "mov %%r9,  %c[r9](%[svm]) \n\t"
3885                 "mov %%r10, %c[r10](%[svm]) \n\t"
3886                 "mov %%r11, %c[r11](%[svm]) \n\t"
3887                 "mov %%r12, %c[r12](%[svm]) \n\t"
3888                 "mov %%r13, %c[r13](%[svm]) \n\t"
3889                 "mov %%r14, %c[r14](%[svm]) \n\t"
3890                 "mov %%r15, %c[r15](%[svm]) \n\t"
3891 #endif
3892                 "pop %%" _ASM_BP
3893                 :
3894                 : [svm]"a"(svm),
3895                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
3896                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
3897                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
3898                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
3899                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
3900                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
3901                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
3902 #ifdef CONFIG_X86_64
3903                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
3904                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
3905                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
3906                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
3907                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
3908                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
3909                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
3910                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
3911 #endif
3912                 : "cc", "memory"
3913 #ifdef CONFIG_X86_64
3914                 , "rbx", "rcx", "rdx", "rsi", "rdi"
3915                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
3916 #else
3917                 , "ebx", "ecx", "edx", "esi", "edi"
3918 #endif
3919                 );
3920
3921 #ifdef CONFIG_X86_64
3922         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3923 #else
3924         loadsegment(fs, svm->host.fs);
3925 #ifndef CONFIG_X86_32_LAZY_GS
3926         loadsegment(gs, svm->host.gs);
3927 #endif
3928 #endif
3929
3930         reload_tss(vcpu);
3931
3932         local_irq_disable();
3933
3934         vcpu->arch.cr2 = svm->vmcb->save.cr2;
3935         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3936         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3937         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3938
3939         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3940                 kvm_before_handle_nmi(&svm->vcpu);
3941
3942         stgi();
3943
3944         /* Any pending NMI will happen here */
3945
3946         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3947                 kvm_after_handle_nmi(&svm->vcpu);
3948
3949         sync_cr8_to_lapic(vcpu);
3950
3951         svm->next_rip = 0;
3952
3953         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3954
3955         /* if exit due to PF check for async PF */
3956         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3957                 svm->apf_reason = kvm_read_and_reset_pf_reason();
3958
3959         if (npt_enabled) {
3960                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3961                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3962         }
3963
3964         /*
3965          * We need to handle MC intercepts here before the vcpu has a chance to
3966          * change the physical cpu
3967          */
3968         if (unlikely(svm->vmcb->control.exit_code ==
3969                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
3970                 svm_handle_mce(svm);
3971
3972         mark_all_clean(svm->vmcb);
3973 }
3974
3975 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3976 {
3977         struct vcpu_svm *svm = to_svm(vcpu);
3978
3979         svm->vmcb->save.cr3 = root;
3980         mark_dirty(svm->vmcb, VMCB_CR);
3981         svm_flush_tlb(vcpu);
3982 }
3983
3984 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3985 {
3986         struct vcpu_svm *svm = to_svm(vcpu);
3987
3988         svm->vmcb->control.nested_cr3 = root;
3989         mark_dirty(svm->vmcb, VMCB_NPT);
3990
3991         /* Also sync guest cr3 here in case we live migrate */
3992         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
3993         mark_dirty(svm->vmcb, VMCB_CR);
3994
3995         svm_flush_tlb(vcpu);
3996 }
3997
3998 static int is_disabled(void)
3999 {
4000         u64 vm_cr;
4001
4002         rdmsrl(MSR_VM_CR, vm_cr);
4003         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4004                 return 1;
4005
4006         return 0;
4007 }
4008
4009 static void
4010 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4011 {
4012         /*
4013          * Patch in the VMMCALL instruction:
4014          */
4015         hypercall[0] = 0x0f;
4016         hypercall[1] = 0x01;
4017         hypercall[2] = 0xd9;
4018 }
4019
4020 static void svm_check_processor_compat(void *rtn)
4021 {
4022         *(int *)rtn = 0;
4023 }
4024
4025 static bool svm_cpu_has_accelerated_tpr(void)
4026 {
4027         return false;
4028 }
4029
4030 static bool svm_has_high_real_mode_segbase(void)
4031 {
4032         return true;
4033 }
4034
4035 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4036 {
4037         return 0;
4038 }
4039
4040 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
4041 {
4042         struct vcpu_svm *svm = to_svm(vcpu);
4043
4044         /* Update nrips enabled cache */
4045         svm->nrips_enabled = !!guest_cpuid_has_nrips(&svm->vcpu);
4046 }
4047
4048 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4049 {
4050         switch (func) {
4051         case 0x80000001:
4052                 if (nested)
4053                         entry->ecx |= (1 << 2); /* Set SVM bit */
4054                 break;
4055         case 0x8000000A:
4056                 entry->eax = 1; /* SVM revision 1 */
4057                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
4058                                    ASID emulation to nested SVM */
4059                 entry->ecx = 0; /* Reserved */
4060                 entry->edx = 0; /* Per default do not support any
4061                                    additional features */
4062
4063                 /* Support next_rip if host supports it */
4064                 if (boot_cpu_has(X86_FEATURE_NRIPS))
4065                         entry->edx |= SVM_FEATURE_NRIP;
4066
4067                 /* Support NPT for the guest if enabled */
4068                 if (npt_enabled)
4069                         entry->edx |= SVM_FEATURE_NPT;
4070
4071                 break;
4072         }
4073 }
4074
4075 static int svm_get_lpage_level(void)
4076 {
4077         return PT_PDPE_LEVEL;
4078 }
4079
4080 static bool svm_rdtscp_supported(void)
4081 {
4082         return boot_cpu_has(X86_FEATURE_RDTSCP);
4083 }
4084
4085 static bool svm_invpcid_supported(void)
4086 {
4087         return false;
4088 }
4089
4090 static bool svm_mpx_supported(void)
4091 {
4092         return false;
4093 }
4094
4095 static bool svm_xsaves_supported(void)
4096 {
4097         return false;
4098 }
4099
4100 static bool svm_has_wbinvd_exit(void)
4101 {
4102         return true;
4103 }
4104
4105 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4106 {
4107         struct vcpu_svm *svm = to_svm(vcpu);
4108
4109         set_exception_intercept(svm, NM_VECTOR);
4110         update_cr0_intercept(svm);
4111 }
4112
4113 #define PRE_EX(exit)  { .exit_code = (exit), \
4114                         .stage = X86_ICPT_PRE_EXCEPT, }
4115 #define POST_EX(exit) { .exit_code = (exit), \
4116                         .stage = X86_ICPT_POST_EXCEPT, }
4117 #define POST_MEM(exit) { .exit_code = (exit), \
4118                         .stage = X86_ICPT_POST_MEMACCESS, }
4119
4120 static const struct __x86_intercept {
4121         u32 exit_code;
4122         enum x86_intercept_stage stage;
4123 } x86_intercept_map[] = {
4124         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4125         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4126         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4127         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4128         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4129         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4130         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4131         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4132         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4133         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4134         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4135         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4136         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4137         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4138         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4139         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4140         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4141         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4142         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4143         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4144         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4145         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4146         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4147         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4148         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4149         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4150         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4151         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4152         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4153         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4154         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4155         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4156         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4157         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4158         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4159         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4160         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4161         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4162         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4163         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4164         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4165         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4166         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4167         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4168         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4169         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4170 };
4171
4172 #undef PRE_EX
4173 #undef POST_EX
4174 #undef POST_MEM
4175
4176 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4177                                struct x86_instruction_info *info,
4178                                enum x86_intercept_stage stage)
4179 {
4180         struct vcpu_svm *svm = to_svm(vcpu);
4181         int vmexit, ret = X86EMUL_CONTINUE;
4182         struct __x86_intercept icpt_info;
4183         struct vmcb *vmcb = svm->vmcb;
4184
4185         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4186                 goto out;
4187
4188         icpt_info = x86_intercept_map[info->intercept];
4189
4190         if (stage != icpt_info.stage)
4191                 goto out;
4192
4193         switch (icpt_info.exit_code) {
4194         case SVM_EXIT_READ_CR0:
4195                 if (info->intercept == x86_intercept_cr_read)
4196                         icpt_info.exit_code += info->modrm_reg;
4197                 break;
4198         case SVM_EXIT_WRITE_CR0: {
4199                 unsigned long cr0, val;
4200                 u64 intercept;
4201
4202                 if (info->intercept == x86_intercept_cr_write)
4203                         icpt_info.exit_code += info->modrm_reg;
4204
4205                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4206                     info->intercept == x86_intercept_clts)
4207                         break;
4208
4209                 intercept = svm->nested.intercept;
4210
4211                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
4212                         break;
4213
4214                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4215                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4216
4217                 if (info->intercept == x86_intercept_lmsw) {
4218                         cr0 &= 0xfUL;
4219                         val &= 0xfUL;
4220                         /* lmsw can't clear PE - catch this here */
4221                         if (cr0 & X86_CR0_PE)
4222                                 val |= X86_CR0_PE;
4223                 }
4224
4225                 if (cr0 ^ val)
4226                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4227
4228                 break;
4229         }
4230         case SVM_EXIT_READ_DR0:
4231         case SVM_EXIT_WRITE_DR0:
4232                 icpt_info.exit_code += info->modrm_reg;
4233                 break;
4234         case SVM_EXIT_MSR:
4235                 if (info->intercept == x86_intercept_wrmsr)
4236                         vmcb->control.exit_info_1 = 1;
4237                 else
4238                         vmcb->control.exit_info_1 = 0;
4239                 break;
4240         case SVM_EXIT_PAUSE:
4241                 /*
4242                  * We get this for NOP only, but pause
4243                  * is rep not, check this here
4244                  */
4245                 if (info->rep_prefix != REPE_PREFIX)
4246                         goto out;
4247         case SVM_EXIT_IOIO: {
4248                 u64 exit_info;
4249                 u32 bytes;
4250
4251                 if (info->intercept == x86_intercept_in ||
4252                     info->intercept == x86_intercept_ins) {
4253                         exit_info = ((info->src_val & 0xffff) << 16) |
4254                                 SVM_IOIO_TYPE_MASK;
4255                         bytes = info->dst_bytes;
4256                 } else {
4257                         exit_info = (info->dst_val & 0xffff) << 16;
4258                         bytes = info->src_bytes;
4259                 }
4260
4261                 if (info->intercept == x86_intercept_outs ||
4262                     info->intercept == x86_intercept_ins)
4263                         exit_info |= SVM_IOIO_STR_MASK;
4264
4265                 if (info->rep_prefix)
4266                         exit_info |= SVM_IOIO_REP_MASK;
4267
4268                 bytes = min(bytes, 4u);
4269
4270                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4271
4272                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4273
4274                 vmcb->control.exit_info_1 = exit_info;
4275                 vmcb->control.exit_info_2 = info->next_rip;
4276
4277                 break;
4278         }
4279         default:
4280                 break;
4281         }
4282
4283         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4284         if (static_cpu_has(X86_FEATURE_NRIPS))
4285                 vmcb->control.next_rip  = info->next_rip;
4286         vmcb->control.exit_code = icpt_info.exit_code;
4287         vmexit = nested_svm_exit_handled(svm);
4288
4289         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4290                                            : X86EMUL_CONTINUE;
4291
4292 out:
4293         return ret;
4294 }
4295
4296 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
4297 {
4298         local_irq_enable();
4299 }
4300
4301 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4302 {
4303 }
4304
4305 static struct kvm_x86_ops svm_x86_ops = {
4306         .cpu_has_kvm_support = has_svm,
4307         .disabled_by_bios = is_disabled,
4308         .hardware_setup = svm_hardware_setup,
4309         .hardware_unsetup = svm_hardware_unsetup,
4310         .check_processor_compatibility = svm_check_processor_compat,
4311         .hardware_enable = svm_hardware_enable,
4312         .hardware_disable = svm_hardware_disable,
4313         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4314         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
4315
4316         .vcpu_create = svm_create_vcpu,
4317         .vcpu_free = svm_free_vcpu,
4318         .vcpu_reset = svm_vcpu_reset,
4319
4320         .prepare_guest_switch = svm_prepare_guest_switch,
4321         .vcpu_load = svm_vcpu_load,
4322         .vcpu_put = svm_vcpu_put,
4323
4324         .update_bp_intercept = update_bp_intercept,
4325         .get_msr = svm_get_msr,
4326         .set_msr = svm_set_msr,
4327         .get_segment_base = svm_get_segment_base,
4328         .get_segment = svm_get_segment,
4329         .set_segment = svm_set_segment,
4330         .get_cpl = svm_get_cpl,
4331         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4332         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
4333         .decache_cr3 = svm_decache_cr3,
4334         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
4335         .set_cr0 = svm_set_cr0,
4336         .set_cr3 = svm_set_cr3,
4337         .set_cr4 = svm_set_cr4,
4338         .set_efer = svm_set_efer,
4339         .get_idt = svm_get_idt,
4340         .set_idt = svm_set_idt,
4341         .get_gdt = svm_get_gdt,
4342         .set_gdt = svm_set_gdt,
4343         .get_dr6 = svm_get_dr6,
4344         .set_dr6 = svm_set_dr6,
4345         .set_dr7 = svm_set_dr7,
4346         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4347         .cache_reg = svm_cache_reg,
4348         .get_rflags = svm_get_rflags,
4349         .set_rflags = svm_set_rflags,
4350         .fpu_activate = svm_fpu_activate,
4351         .fpu_deactivate = svm_fpu_deactivate,
4352
4353         .tlb_flush = svm_flush_tlb,
4354
4355         .run = svm_vcpu_run,
4356         .handle_exit = handle_exit,
4357         .skip_emulated_instruction = skip_emulated_instruction,
4358         .set_interrupt_shadow = svm_set_interrupt_shadow,
4359         .get_interrupt_shadow = svm_get_interrupt_shadow,
4360         .patch_hypercall = svm_patch_hypercall,
4361         .set_irq = svm_set_irq,
4362         .set_nmi = svm_inject_nmi,
4363         .queue_exception = svm_queue_exception,
4364         .cancel_injection = svm_cancel_injection,
4365         .interrupt_allowed = svm_interrupt_allowed,
4366         .nmi_allowed = svm_nmi_allowed,
4367         .get_nmi_mask = svm_get_nmi_mask,
4368         .set_nmi_mask = svm_set_nmi_mask,
4369         .enable_nmi_window = enable_nmi_window,
4370         .enable_irq_window = enable_irq_window,
4371         .update_cr8_intercept = update_cr8_intercept,
4372         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
4373         .get_enable_apicv = svm_get_enable_apicv,
4374         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4375         .load_eoi_exitmap = svm_load_eoi_exitmap,
4376         .sync_pir_to_irr = svm_sync_pir_to_irr,
4377
4378         .set_tss_addr = svm_set_tss_addr,
4379         .get_tdp_level = get_npt_level,
4380         .get_mt_mask = svm_get_mt_mask,
4381
4382         .get_exit_info = svm_get_exit_info,
4383
4384         .get_lpage_level = svm_get_lpage_level,
4385
4386         .cpuid_update = svm_cpuid_update,
4387
4388         .rdtscp_supported = svm_rdtscp_supported,
4389         .invpcid_supported = svm_invpcid_supported,
4390         .mpx_supported = svm_mpx_supported,
4391         .xsaves_supported = svm_xsaves_supported,
4392
4393         .set_supported_cpuid = svm_set_supported_cpuid,
4394
4395         .has_wbinvd_exit = svm_has_wbinvd_exit,
4396
4397         .read_tsc_offset = svm_read_tsc_offset,
4398         .write_tsc_offset = svm_write_tsc_offset,
4399         .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
4400         .read_l1_tsc = svm_read_l1_tsc,
4401
4402         .set_tdp_cr3 = set_tdp_cr3,
4403
4404         .check_intercept = svm_check_intercept,
4405         .handle_external_intr = svm_handle_external_intr,
4406
4407         .sched_in = svm_sched_in,
4408
4409         .pmu_ops = &amd_pmu_ops,
4410 };
4411
4412 static int __init svm_init(void)
4413 {
4414         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
4415                         __alignof__(struct vcpu_svm), THIS_MODULE);
4416 }
4417
4418 static void __exit svm_exit(void)
4419 {
4420         kvm_exit();
4421 }
4422
4423 module_init(svm_init)
4424 module_exit(svm_exit)