1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
166 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
168 #define X2(x...) x, x
169 #define X3(x...) X2(x), x
170 #define X4(x...) X2(x), X2(x)
171 #define X5(x...) X4(x), x
172 #define X6(x...) X4(x), X2(x)
173 #define X7(x...) X4(x), X3(x)
174 #define X8(x...) X4(x), X4(x)
175 #define X16(x...) X8(x), X8(x)
177 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
178 #define FASTOP_SIZE 8
181 * fastop functions have a special calling convention:
186 * flags: rflags (in/out)
187 * ex: rsi (in:fastop pointer, out:zero if exception)
189 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
190 * different operand sizes can be reached by calculation, rather than a jump
191 * table (which would be bigger than the code).
193 * fastop functions are declared as taking a never-defined fastop parameter,
194 * so they can't be called from C directly.
203 int (*execute)(struct x86_emulate_ctxt *ctxt);
204 const struct opcode *group;
205 const struct group_dual *gdual;
206 const struct gprefix *gprefix;
207 const struct escape *esc;
208 void (*fastop)(struct fastop *fake);
210 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
214 struct opcode mod012[8];
215 struct opcode mod3[8];
219 struct opcode pfx_no;
220 struct opcode pfx_66;
221 struct opcode pfx_f2;
222 struct opcode pfx_f3;
227 struct opcode high[64];
230 /* EFLAGS bit definitions. */
231 #define EFLG_ID (1<<21)
232 #define EFLG_VIP (1<<20)
233 #define EFLG_VIF (1<<19)
234 #define EFLG_AC (1<<18)
235 #define EFLG_VM (1<<17)
236 #define EFLG_RF (1<<16)
237 #define EFLG_IOPL (3<<12)
238 #define EFLG_NT (1<<14)
239 #define EFLG_OF (1<<11)
240 #define EFLG_DF (1<<10)
241 #define EFLG_IF (1<<9)
242 #define EFLG_TF (1<<8)
243 #define EFLG_SF (1<<7)
244 #define EFLG_ZF (1<<6)
245 #define EFLG_AF (1<<4)
246 #define EFLG_PF (1<<2)
247 #define EFLG_CF (1<<0)
249 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
250 #define EFLG_RESERVED_ONE_MASK 2
252 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
254 if (!(ctxt->regs_valid & (1 << nr))) {
255 ctxt->regs_valid |= 1 << nr;
256 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
258 return ctxt->_regs[nr];
261 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 ctxt->regs_valid |= 1 << nr;
264 ctxt->regs_dirty |= 1 << nr;
265 return &ctxt->_regs[nr];
268 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
271 return reg_write(ctxt, nr);
274 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
278 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
279 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
282 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
284 ctxt->regs_dirty = 0;
285 ctxt->regs_valid = 0;
289 * These EFLAGS bits are restored from saved value during emulation, and
290 * any changes are written back to the saved value after emulation.
292 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
300 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
302 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
303 #define FOP_RET "ret \n\t"
305 #define FOP_START(op) \
306 extern void em_##op(struct fastop *fake); \
307 asm(".pushsection .text, \"ax\" \n\t" \
308 ".global em_" #op " \n\t" \
315 #define FOPNOP() FOP_ALIGN FOP_RET
317 #define FOP1E(op, dst) \
318 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
320 #define FOP1EEX(op, dst) \
321 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
323 #define FASTOP1(op) \
328 ON64(FOP1E(op##q, rax)) \
331 /* 1-operand, using src2 (for MUL/DIV r/m) */
332 #define FASTOP1SRC2(op, name) \
337 ON64(FOP1E(op, rcx)) \
340 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
341 #define FASTOP1SRC2EX(op, name) \
346 ON64(FOP1EEX(op, rcx)) \
349 #define FOP2E(op, dst, src) \
350 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
352 #define FASTOP2(op) \
354 FOP2E(op##b, al, dl) \
355 FOP2E(op##w, ax, dx) \
356 FOP2E(op##l, eax, edx) \
357 ON64(FOP2E(op##q, rax, rdx)) \
360 /* 2 operand, word only */
361 #define FASTOP2W(op) \
364 FOP2E(op##w, ax, dx) \
365 FOP2E(op##l, eax, edx) \
366 ON64(FOP2E(op##q, rax, rdx)) \
369 /* 2 operand, src is CL */
370 #define FASTOP2CL(op) \
372 FOP2E(op##b, al, cl) \
373 FOP2E(op##w, ax, cl) \
374 FOP2E(op##l, eax, cl) \
375 ON64(FOP2E(op##q, rax, cl)) \
378 #define FOP3E(op, dst, src, src2) \
379 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
381 /* 3-operand, word-only, src2=cl */
382 #define FASTOP3WCL(op) \
385 FOP3E(op##w, ax, dx, cl) \
386 FOP3E(op##l, eax, edx, cl) \
387 ON64(FOP3E(op##q, rax, rdx, cl)) \
390 /* Special case for SETcc - 1 instruction per cc */
391 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
393 asm(".global kvm_fastop_exception \n"
394 "kvm_fastop_exception: xor %esi, %esi; ret");
415 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
418 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
419 enum x86_intercept intercept,
420 enum x86_intercept_stage stage)
422 struct x86_instruction_info info = {
423 .intercept = intercept,
424 .rep_prefix = ctxt->rep_prefix,
425 .modrm_mod = ctxt->modrm_mod,
426 .modrm_reg = ctxt->modrm_reg,
427 .modrm_rm = ctxt->modrm_rm,
428 .src_val = ctxt->src.val64,
429 .dst_val = ctxt->dst.val64,
430 .src_bytes = ctxt->src.bytes,
431 .dst_bytes = ctxt->dst.bytes,
432 .ad_bytes = ctxt->ad_bytes,
433 .next_rip = ctxt->eip,
436 return ctxt->ops->intercept(ctxt, &info, stage);
439 static void assign_masked(ulong *dest, ulong src, ulong mask)
441 *dest = (*dest & ~mask) | (src & mask);
444 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
446 return (1UL << (ctxt->ad_bytes << 3)) - 1;
449 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
452 struct desc_struct ss;
454 if (ctxt->mode == X86EMUL_MODE_PROT64)
456 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
457 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
460 static int stack_size(struct x86_emulate_ctxt *ctxt)
462 return (__fls(stack_mask(ctxt)) + 1) >> 3;
465 /* Access/update address held in a register, based on addressing mode. */
466 static inline unsigned long
467 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
469 if (ctxt->ad_bytes == sizeof(unsigned long))
472 return reg & ad_mask(ctxt);
475 static inline unsigned long
476 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
478 return address_mask(ctxt, reg);
481 static void masked_increment(ulong *reg, ulong mask, int inc)
483 assign_masked(reg, *reg + inc, mask);
487 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
491 if (ctxt->ad_bytes == sizeof(unsigned long))
494 mask = ad_mask(ctxt);
495 masked_increment(reg, mask, inc);
498 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
500 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
503 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
505 register_address_increment(ctxt, &ctxt->_eip, rel);
508 static u32 desc_limit_scaled(struct desc_struct *desc)
510 u32 limit = get_desc_limit(desc);
512 return desc->g ? (limit << 12) | 0xfff : limit;
515 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
517 ctxt->has_seg_override = true;
518 ctxt->seg_override = seg;
521 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
523 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
526 return ctxt->ops->get_cached_segment_base(ctxt, seg);
529 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
531 if (!ctxt->has_seg_override)
534 return ctxt->seg_override;
537 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
538 u32 error, bool valid)
540 ctxt->exception.vector = vec;
541 ctxt->exception.error_code = error;
542 ctxt->exception.error_code_valid = valid;
543 return X86EMUL_PROPAGATE_FAULT;
546 static int emulate_db(struct x86_emulate_ctxt *ctxt)
548 return emulate_exception(ctxt, DB_VECTOR, 0, false);
551 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
553 return emulate_exception(ctxt, GP_VECTOR, err, true);
556 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
558 return emulate_exception(ctxt, SS_VECTOR, err, true);
561 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
563 return emulate_exception(ctxt, UD_VECTOR, 0, false);
566 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
568 return emulate_exception(ctxt, TS_VECTOR, err, true);
571 static int emulate_de(struct x86_emulate_ctxt *ctxt)
573 return emulate_exception(ctxt, DE_VECTOR, 0, false);
576 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
578 return emulate_exception(ctxt, NM_VECTOR, 0, false);
581 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
584 struct desc_struct desc;
586 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
590 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
595 struct desc_struct desc;
597 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
598 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
602 * x86 defines three classes of vector instructions: explicitly
603 * aligned, explicitly unaligned, and the rest, which change behaviour
604 * depending on whether they're AVX encoded or not.
606 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
607 * subject to the same check.
609 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
611 if (likely(size < 16))
614 if (ctxt->d & Aligned)
616 else if (ctxt->d & Unaligned)
618 else if (ctxt->d & Avx)
624 static int __linearize(struct x86_emulate_ctxt *ctxt,
625 struct segmented_address addr,
626 unsigned size, bool write, bool fetch,
629 struct desc_struct desc;
636 la = seg_base(ctxt, addr.seg) + addr.ea;
637 switch (ctxt->mode) {
638 case X86EMUL_MODE_PROT64:
639 if (((signed long)la << 16) >> 16 != la)
640 return emulate_gp(ctxt, 0);
643 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
647 /* code segment in protected mode or read-only data segment */
648 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
649 || !(desc.type & 2)) && write)
651 /* unreadable code segment */
652 if (!fetch && (desc.type & 8) && !(desc.type & 2))
654 lim = desc_limit_scaled(&desc);
655 if ((desc.type & 8) || !(desc.type & 4)) {
656 /* expand-up segment */
657 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
660 /* expand-down segment */
661 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
663 lim = desc.d ? 0xffffffff : 0xffff;
664 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
667 cpl = ctxt->ops->cpl(ctxt);
668 if (!(desc.type & 8)) {
672 } else if ((desc.type & 8) && !(desc.type & 4)) {
673 /* nonconforming code segment */
676 } else if ((desc.type & 8) && (desc.type & 4)) {
677 /* conforming code segment */
683 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
685 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
686 return emulate_gp(ctxt, 0);
688 return X86EMUL_CONTINUE;
690 if (addr.seg == VCPU_SREG_SS)
691 return emulate_ss(ctxt, sel);
693 return emulate_gp(ctxt, sel);
696 static int linearize(struct x86_emulate_ctxt *ctxt,
697 struct segmented_address addr,
698 unsigned size, bool write,
701 return __linearize(ctxt, addr, size, write, false, linear);
705 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
706 struct segmented_address addr,
713 rc = linearize(ctxt, addr, size, false, &linear);
714 if (rc != X86EMUL_CONTINUE)
716 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
720 * Fetch the next byte of the instruction being emulated which is pointed to
721 * by ctxt->_eip, then increment ctxt->_eip.
723 * Also prefetch the remaining bytes of the instruction without crossing page
724 * boundary if they are not in fetch_cache yet.
726 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
728 struct fetch_cache *fc = &ctxt->fetch;
732 if (ctxt->_eip == fc->end) {
733 unsigned long linear;
734 struct segmented_address addr = { .seg = VCPU_SREG_CS,
736 cur_size = fc->end - fc->start;
737 size = min(15UL - cur_size,
738 PAGE_SIZE - offset_in_page(ctxt->_eip));
739 rc = __linearize(ctxt, addr, size, false, true, &linear);
740 if (unlikely(rc != X86EMUL_CONTINUE))
742 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
743 size, &ctxt->exception);
744 if (unlikely(rc != X86EMUL_CONTINUE))
748 *dest = fc->data[ctxt->_eip - fc->start];
750 return X86EMUL_CONTINUE;
753 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
754 void *dest, unsigned size)
758 /* x86 instructions are limited to 15 bytes. */
759 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
760 return X86EMUL_UNHANDLEABLE;
762 rc = do_insn_fetch_byte(ctxt, dest++);
763 if (rc != X86EMUL_CONTINUE)
766 return X86EMUL_CONTINUE;
769 /* Fetch next part of the instruction being emulated. */
770 #define insn_fetch(_type, _ctxt) \
771 ({ unsigned long _x; \
772 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
773 if (rc != X86EMUL_CONTINUE) \
778 #define insn_fetch_arr(_arr, _size, _ctxt) \
779 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
780 if (rc != X86EMUL_CONTINUE) \
785 * Given the 'reg' portion of a ModRM byte, and a register block, return a
786 * pointer into the block that addresses the relevant register.
787 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
789 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
793 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
795 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
796 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
798 p = reg_rmw(ctxt, modrm_reg);
802 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
803 struct segmented_address addr,
804 u16 *size, unsigned long *address, int op_bytes)
811 rc = segmented_read_std(ctxt, addr, size, 2);
812 if (rc != X86EMUL_CONTINUE)
815 rc = segmented_read_std(ctxt, addr, address, op_bytes);
829 FASTOP1SRC2(mul, mul_ex);
830 FASTOP1SRC2(imul, imul_ex);
831 FASTOP1SRC2EX(div, div_ex);
832 FASTOP1SRC2EX(idiv, idiv_ex);
861 static u8 test_cc(unsigned int condition, unsigned long flags)
864 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
866 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
867 asm("push %[flags]; popf; call *%[fastop]"
868 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
872 static void fetch_register_operand(struct operand *op)
876 op->val = *(u8 *)op->addr.reg;
879 op->val = *(u16 *)op->addr.reg;
882 op->val = *(u32 *)op->addr.reg;
885 op->val = *(u64 *)op->addr.reg;
890 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
892 ctxt->ops->get_fpu(ctxt);
894 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
895 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
896 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
897 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
898 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
899 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
900 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
901 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
903 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
904 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
905 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
906 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
907 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
908 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
909 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
910 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
914 ctxt->ops->put_fpu(ctxt);
917 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
920 ctxt->ops->get_fpu(ctxt);
922 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
923 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
924 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
925 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
926 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
927 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
928 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
929 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
931 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
932 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
933 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
934 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
935 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
936 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
937 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
938 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
942 ctxt->ops->put_fpu(ctxt);
945 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
947 ctxt->ops->get_fpu(ctxt);
949 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
950 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
951 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
952 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
953 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
954 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
955 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
956 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
959 ctxt->ops->put_fpu(ctxt);
962 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
964 ctxt->ops->get_fpu(ctxt);
966 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
967 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
968 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
969 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
970 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
971 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
972 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
973 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
976 ctxt->ops->put_fpu(ctxt);
979 static int em_fninit(struct x86_emulate_ctxt *ctxt)
981 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
982 return emulate_nm(ctxt);
984 ctxt->ops->get_fpu(ctxt);
985 asm volatile("fninit");
986 ctxt->ops->put_fpu(ctxt);
987 return X86EMUL_CONTINUE;
990 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
994 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
995 return emulate_nm(ctxt);
997 ctxt->ops->get_fpu(ctxt);
998 asm volatile("fnstcw %0": "+m"(fcw));
999 ctxt->ops->put_fpu(ctxt);
1001 /* force 2 byte destination */
1002 ctxt->dst.bytes = 2;
1003 ctxt->dst.val = fcw;
1005 return X86EMUL_CONTINUE;
1008 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1012 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1013 return emulate_nm(ctxt);
1015 ctxt->ops->get_fpu(ctxt);
1016 asm volatile("fnstsw %0": "+m"(fsw));
1017 ctxt->ops->put_fpu(ctxt);
1019 /* force 2 byte destination */
1020 ctxt->dst.bytes = 2;
1021 ctxt->dst.val = fsw;
1023 return X86EMUL_CONTINUE;
1026 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1029 unsigned reg = ctxt->modrm_reg;
1031 if (!(ctxt->d & ModRM))
1032 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1034 if (ctxt->d & Sse) {
1038 read_sse_reg(ctxt, &op->vec_val, reg);
1041 if (ctxt->d & Mmx) {
1050 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1051 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1053 fetch_register_operand(op);
1054 op->orig_val = op->val;
1057 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1059 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1060 ctxt->modrm_seg = VCPU_SREG_SS;
1063 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1067 int index_reg = 0, base_reg = 0, scale;
1068 int rc = X86EMUL_CONTINUE;
1071 if (ctxt->rex_prefix) {
1072 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1073 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1074 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1077 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1078 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1079 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1080 ctxt->modrm_seg = VCPU_SREG_DS;
1082 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1084 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1085 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1087 if (ctxt->d & Sse) {
1090 op->addr.xmm = ctxt->modrm_rm;
1091 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1094 if (ctxt->d & Mmx) {
1097 op->addr.mm = ctxt->modrm_rm & 7;
1100 fetch_register_operand(op);
1106 if (ctxt->ad_bytes == 2) {
1107 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1108 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1109 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1110 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1112 /* 16-bit ModR/M decode. */
1113 switch (ctxt->modrm_mod) {
1115 if (ctxt->modrm_rm == 6)
1116 modrm_ea += insn_fetch(u16, ctxt);
1119 modrm_ea += insn_fetch(s8, ctxt);
1122 modrm_ea += insn_fetch(u16, ctxt);
1125 switch (ctxt->modrm_rm) {
1127 modrm_ea += bx + si;
1130 modrm_ea += bx + di;
1133 modrm_ea += bp + si;
1136 modrm_ea += bp + di;
1145 if (ctxt->modrm_mod != 0)
1152 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1153 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1154 ctxt->modrm_seg = VCPU_SREG_SS;
1155 modrm_ea = (u16)modrm_ea;
1157 /* 32/64-bit ModR/M decode. */
1158 if ((ctxt->modrm_rm & 7) == 4) {
1159 sib = insn_fetch(u8, ctxt);
1160 index_reg |= (sib >> 3) & 7;
1161 base_reg |= sib & 7;
1164 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1165 modrm_ea += insn_fetch(s32, ctxt);
1167 modrm_ea += reg_read(ctxt, base_reg);
1168 adjust_modrm_seg(ctxt, base_reg);
1171 modrm_ea += reg_read(ctxt, index_reg) << scale;
1172 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1173 if (ctxt->mode == X86EMUL_MODE_PROT64)
1174 ctxt->rip_relative = 1;
1176 base_reg = ctxt->modrm_rm;
1177 modrm_ea += reg_read(ctxt, base_reg);
1178 adjust_modrm_seg(ctxt, base_reg);
1180 switch (ctxt->modrm_mod) {
1182 if (ctxt->modrm_rm == 5)
1183 modrm_ea += insn_fetch(s32, ctxt);
1186 modrm_ea += insn_fetch(s8, ctxt);
1189 modrm_ea += insn_fetch(s32, ctxt);
1193 op->addr.mem.ea = modrm_ea;
1198 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1201 int rc = X86EMUL_CONTINUE;
1204 switch (ctxt->ad_bytes) {
1206 op->addr.mem.ea = insn_fetch(u16, ctxt);
1209 op->addr.mem.ea = insn_fetch(u32, ctxt);
1212 op->addr.mem.ea = insn_fetch(u64, ctxt);
1219 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1223 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1224 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1226 if (ctxt->src.bytes == 2)
1227 sv = (s16)ctxt->src.val & (s16)mask;
1228 else if (ctxt->src.bytes == 4)
1229 sv = (s32)ctxt->src.val & (s32)mask;
1231 sv = (s64)ctxt->src.val & (s64)mask;
1233 ctxt->dst.addr.mem.ea += (sv >> 3);
1236 /* only subword offset */
1237 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1240 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1241 unsigned long addr, void *dest, unsigned size)
1244 struct read_cache *mc = &ctxt->mem_read;
1246 if (mc->pos < mc->end)
1249 WARN_ON((mc->end + size) >= sizeof(mc->data));
1251 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1253 if (rc != X86EMUL_CONTINUE)
1259 memcpy(dest, mc->data + mc->pos, size);
1261 return X86EMUL_CONTINUE;
1264 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1265 struct segmented_address addr,
1272 rc = linearize(ctxt, addr, size, false, &linear);
1273 if (rc != X86EMUL_CONTINUE)
1275 return read_emulated(ctxt, linear, data, size);
1278 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1279 struct segmented_address addr,
1286 rc = linearize(ctxt, addr, size, true, &linear);
1287 if (rc != X86EMUL_CONTINUE)
1289 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1293 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1294 struct segmented_address addr,
1295 const void *orig_data, const void *data,
1301 rc = linearize(ctxt, addr, size, true, &linear);
1302 if (rc != X86EMUL_CONTINUE)
1304 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1305 size, &ctxt->exception);
1308 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1309 unsigned int size, unsigned short port,
1312 struct read_cache *rc = &ctxt->io_read;
1314 if (rc->pos == rc->end) { /* refill pio read ahead */
1315 unsigned int in_page, n;
1316 unsigned int count = ctxt->rep_prefix ?
1317 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1318 in_page = (ctxt->eflags & EFLG_DF) ?
1319 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1320 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1321 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1325 rc->pos = rc->end = 0;
1326 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1331 if (ctxt->rep_prefix && (ctxt->d & String) &&
1332 !(ctxt->eflags & EFLG_DF)) {
1333 ctxt->dst.data = rc->data + rc->pos;
1334 ctxt->dst.type = OP_MEM_STR;
1335 ctxt->dst.count = (rc->end - rc->pos) / size;
1338 memcpy(dest, rc->data + rc->pos, size);
1344 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1345 u16 index, struct desc_struct *desc)
1350 ctxt->ops->get_idt(ctxt, &dt);
1352 if (dt.size < index * 8 + 7)
1353 return emulate_gp(ctxt, index << 3 | 0x2);
1355 addr = dt.address + index * 8;
1356 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1360 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1361 u16 selector, struct desc_ptr *dt)
1363 const struct x86_emulate_ops *ops = ctxt->ops;
1366 if (selector & 1 << 2) {
1367 struct desc_struct desc;
1370 memset (dt, 0, sizeof *dt);
1371 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1375 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1376 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1378 ops->get_gdt(ctxt, dt);
1381 /* allowed just for 8 bytes segments */
1382 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1383 u16 selector, struct desc_struct *desc,
1387 u16 index = selector >> 3;
1390 get_descriptor_table_ptr(ctxt, selector, &dt);
1392 if (dt.size < index * 8 + 7)
1393 return emulate_gp(ctxt, selector & 0xfffc);
1395 *desc_addr_p = addr = dt.address + index * 8;
1396 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1400 /* allowed just for 8 bytes segments */
1401 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1402 u16 selector, struct desc_struct *desc)
1405 u16 index = selector >> 3;
1408 get_descriptor_table_ptr(ctxt, selector, &dt);
1410 if (dt.size < index * 8 + 7)
1411 return emulate_gp(ctxt, selector & 0xfffc);
1413 addr = dt.address + index * 8;
1414 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1418 /* Does not support long mode */
1419 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1420 u16 selector, int seg, u8 cpl, bool in_task_switch)
1422 struct desc_struct seg_desc, old_desc;
1424 unsigned err_vec = GP_VECTOR;
1426 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1432 memset(&seg_desc, 0, sizeof seg_desc);
1434 if (ctxt->mode == X86EMUL_MODE_REAL) {
1435 /* set real mode segment descriptor (keep limit etc. for
1437 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1438 set_desc_base(&seg_desc, selector << 4);
1440 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1441 /* VM86 needs a clean new segment descriptor */
1442 set_desc_base(&seg_desc, selector << 4);
1443 set_desc_limit(&seg_desc, 0xffff);
1453 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1454 if ((seg == VCPU_SREG_CS
1455 || (seg == VCPU_SREG_SS
1456 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1457 || seg == VCPU_SREG_TR)
1461 /* TR should be in GDT only */
1462 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1465 if (null_selector) /* for NULL selector skip all following checks */
1468 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1469 if (ret != X86EMUL_CONTINUE)
1472 err_code = selector & 0xfffc;
1473 err_vec = GP_VECTOR;
1475 /* can't load system descriptor into segment selector */
1476 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1480 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1489 * segment is not a writable data segment or segment
1490 * selector's RPL != CPL or segment selector's RPL != CPL
1492 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1496 if (in_task_switch && rpl != dpl)
1499 if (!(seg_desc.type & 8))
1502 if (seg_desc.type & 4) {
1508 if (rpl > cpl || dpl != cpl)
1511 /* CS(RPL) <- CPL */
1512 selector = (selector & 0xfffc) | cpl;
1515 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1517 old_desc = seg_desc;
1518 seg_desc.type |= 2; /* busy */
1519 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1520 sizeof(seg_desc), &ctxt->exception);
1521 if (ret != X86EMUL_CONTINUE)
1524 case VCPU_SREG_LDTR:
1525 if (seg_desc.s || seg_desc.type != 2)
1528 default: /* DS, ES, FS, or GS */
1530 * segment is not a data or readable code segment or
1531 * ((segment is a data or nonconforming code segment)
1532 * and (both RPL and CPL > DPL))
1534 if ((seg_desc.type & 0xa) == 0x8 ||
1535 (((seg_desc.type & 0xc) != 0xc) &&
1536 (rpl > dpl && cpl > dpl)))
1542 /* mark segment as accessed */
1544 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1545 if (ret != X86EMUL_CONTINUE)
1547 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1548 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1549 sizeof(base3), &ctxt->exception);
1550 if (ret != X86EMUL_CONTINUE)
1554 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1555 return X86EMUL_CONTINUE;
1557 emulate_exception(ctxt, err_vec, err_code, true);
1558 return X86EMUL_PROPAGATE_FAULT;
1561 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1562 u16 selector, int seg)
1564 u8 cpl = ctxt->ops->cpl(ctxt);
1565 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1568 static void write_register_operand(struct operand *op)
1570 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1571 switch (op->bytes) {
1573 *(u8 *)op->addr.reg = (u8)op->val;
1576 *(u16 *)op->addr.reg = (u16)op->val;
1579 *op->addr.reg = (u32)op->val;
1580 break; /* 64b: zero-extend */
1582 *op->addr.reg = op->val;
1587 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1593 write_register_operand(op);
1596 if (ctxt->lock_prefix)
1597 rc = segmented_cmpxchg(ctxt,
1603 rc = segmented_write(ctxt,
1607 if (rc != X86EMUL_CONTINUE)
1611 rc = segmented_write(ctxt,
1614 op->bytes * op->count);
1615 if (rc != X86EMUL_CONTINUE)
1619 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1622 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1630 return X86EMUL_CONTINUE;
1633 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1635 struct segmented_address addr;
1637 rsp_increment(ctxt, -bytes);
1638 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1639 addr.seg = VCPU_SREG_SS;
1641 return segmented_write(ctxt, addr, data, bytes);
1644 static int em_push(struct x86_emulate_ctxt *ctxt)
1646 /* Disable writeback. */
1647 ctxt->dst.type = OP_NONE;
1648 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1651 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1652 void *dest, int len)
1655 struct segmented_address addr;
1657 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1658 addr.seg = VCPU_SREG_SS;
1659 rc = segmented_read(ctxt, addr, dest, len);
1660 if (rc != X86EMUL_CONTINUE)
1663 rsp_increment(ctxt, len);
1667 static int em_pop(struct x86_emulate_ctxt *ctxt)
1669 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1672 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1673 void *dest, int len)
1676 unsigned long val, change_mask;
1677 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1678 int cpl = ctxt->ops->cpl(ctxt);
1680 rc = emulate_pop(ctxt, &val, len);
1681 if (rc != X86EMUL_CONTINUE)
1684 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1685 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1687 switch(ctxt->mode) {
1688 case X86EMUL_MODE_PROT64:
1689 case X86EMUL_MODE_PROT32:
1690 case X86EMUL_MODE_PROT16:
1692 change_mask |= EFLG_IOPL;
1694 change_mask |= EFLG_IF;
1696 case X86EMUL_MODE_VM86:
1698 return emulate_gp(ctxt, 0);
1699 change_mask |= EFLG_IF;
1701 default: /* real mode */
1702 change_mask |= (EFLG_IOPL | EFLG_IF);
1706 *(unsigned long *)dest =
1707 (ctxt->eflags & ~change_mask) | (val & change_mask);
1712 static int em_popf(struct x86_emulate_ctxt *ctxt)
1714 ctxt->dst.type = OP_REG;
1715 ctxt->dst.addr.reg = &ctxt->eflags;
1716 ctxt->dst.bytes = ctxt->op_bytes;
1717 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1720 static int em_enter(struct x86_emulate_ctxt *ctxt)
1723 unsigned frame_size = ctxt->src.val;
1724 unsigned nesting_level = ctxt->src2.val & 31;
1728 return X86EMUL_UNHANDLEABLE;
1730 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1731 rc = push(ctxt, &rbp, stack_size(ctxt));
1732 if (rc != X86EMUL_CONTINUE)
1734 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1736 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1737 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1739 return X86EMUL_CONTINUE;
1742 static int em_leave(struct x86_emulate_ctxt *ctxt)
1744 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1746 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1749 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1751 int seg = ctxt->src2.val;
1753 ctxt->src.val = get_segment_selector(ctxt, seg);
1755 return em_push(ctxt);
1758 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1760 int seg = ctxt->src2.val;
1761 unsigned long selector;
1764 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1765 if (rc != X86EMUL_CONTINUE)
1768 if (ctxt->modrm_reg == VCPU_SREG_SS)
1769 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1771 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1775 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1777 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1778 int rc = X86EMUL_CONTINUE;
1779 int reg = VCPU_REGS_RAX;
1781 while (reg <= VCPU_REGS_RDI) {
1782 (reg == VCPU_REGS_RSP) ?
1783 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1786 if (rc != X86EMUL_CONTINUE)
1795 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1797 ctxt->src.val = (unsigned long)ctxt->eflags;
1798 return em_push(ctxt);
1801 static int em_popa(struct x86_emulate_ctxt *ctxt)
1803 int rc = X86EMUL_CONTINUE;
1804 int reg = VCPU_REGS_RDI;
1806 while (reg >= VCPU_REGS_RAX) {
1807 if (reg == VCPU_REGS_RSP) {
1808 rsp_increment(ctxt, ctxt->op_bytes);
1812 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1813 if (rc != X86EMUL_CONTINUE)
1820 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1822 const struct x86_emulate_ops *ops = ctxt->ops;
1829 /* TODO: Add limit checks */
1830 ctxt->src.val = ctxt->eflags;
1832 if (rc != X86EMUL_CONTINUE)
1835 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1837 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1839 if (rc != X86EMUL_CONTINUE)
1842 ctxt->src.val = ctxt->_eip;
1844 if (rc != X86EMUL_CONTINUE)
1847 ops->get_idt(ctxt, &dt);
1849 eip_addr = dt.address + (irq << 2);
1850 cs_addr = dt.address + (irq << 2) + 2;
1852 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1853 if (rc != X86EMUL_CONTINUE)
1856 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1857 if (rc != X86EMUL_CONTINUE)
1860 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1861 if (rc != X86EMUL_CONTINUE)
1869 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1873 invalidate_registers(ctxt);
1874 rc = __emulate_int_real(ctxt, irq);
1875 if (rc == X86EMUL_CONTINUE)
1876 writeback_registers(ctxt);
1880 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1882 switch(ctxt->mode) {
1883 case X86EMUL_MODE_REAL:
1884 return __emulate_int_real(ctxt, irq);
1885 case X86EMUL_MODE_VM86:
1886 case X86EMUL_MODE_PROT16:
1887 case X86EMUL_MODE_PROT32:
1888 case X86EMUL_MODE_PROT64:
1890 /* Protected mode interrupts unimplemented yet */
1891 return X86EMUL_UNHANDLEABLE;
1895 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1897 int rc = X86EMUL_CONTINUE;
1898 unsigned long temp_eip = 0;
1899 unsigned long temp_eflags = 0;
1900 unsigned long cs = 0;
1901 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1902 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1903 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1904 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1906 /* TODO: Add stack limit check */
1908 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1910 if (rc != X86EMUL_CONTINUE)
1913 if (temp_eip & ~0xffff)
1914 return emulate_gp(ctxt, 0);
1916 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1918 if (rc != X86EMUL_CONTINUE)
1921 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1923 if (rc != X86EMUL_CONTINUE)
1926 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1928 if (rc != X86EMUL_CONTINUE)
1931 ctxt->_eip = temp_eip;
1934 if (ctxt->op_bytes == 4)
1935 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1936 else if (ctxt->op_bytes == 2) {
1937 ctxt->eflags &= ~0xffff;
1938 ctxt->eflags |= temp_eflags;
1941 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1942 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1947 static int em_iret(struct x86_emulate_ctxt *ctxt)
1949 switch(ctxt->mode) {
1950 case X86EMUL_MODE_REAL:
1951 return emulate_iret_real(ctxt);
1952 case X86EMUL_MODE_VM86:
1953 case X86EMUL_MODE_PROT16:
1954 case X86EMUL_MODE_PROT32:
1955 case X86EMUL_MODE_PROT64:
1957 /* iret from protected mode unimplemented yet */
1958 return X86EMUL_UNHANDLEABLE;
1962 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1967 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1969 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1970 if (rc != X86EMUL_CONTINUE)
1974 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1975 return X86EMUL_CONTINUE;
1978 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1980 int rc = X86EMUL_CONTINUE;
1982 switch (ctxt->modrm_reg) {
1983 case 2: /* call near abs */ {
1985 old_eip = ctxt->_eip;
1986 ctxt->_eip = ctxt->src.val;
1987 ctxt->src.val = old_eip;
1991 case 4: /* jmp abs */
1992 ctxt->_eip = ctxt->src.val;
1994 case 5: /* jmp far */
1995 rc = em_jmp_far(ctxt);
2004 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2006 u64 old = ctxt->dst.orig_val64;
2008 if (ctxt->dst.bytes == 16)
2009 return X86EMUL_UNHANDLEABLE;
2011 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2012 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2013 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2014 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2015 ctxt->eflags &= ~EFLG_ZF;
2017 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2018 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2020 ctxt->eflags |= EFLG_ZF;
2022 return X86EMUL_CONTINUE;
2025 static int em_ret(struct x86_emulate_ctxt *ctxt)
2027 ctxt->dst.type = OP_REG;
2028 ctxt->dst.addr.reg = &ctxt->_eip;
2029 ctxt->dst.bytes = ctxt->op_bytes;
2030 return em_pop(ctxt);
2033 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2037 int cpl = ctxt->ops->cpl(ctxt);
2039 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2040 if (rc != X86EMUL_CONTINUE)
2042 if (ctxt->op_bytes == 4)
2043 ctxt->_eip = (u32)ctxt->_eip;
2044 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2045 if (rc != X86EMUL_CONTINUE)
2047 /* Outer-privilege level return is not implemented */
2048 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2049 return X86EMUL_UNHANDLEABLE;
2050 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2054 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2058 rc = em_ret_far(ctxt);
2059 if (rc != X86EMUL_CONTINUE)
2061 rsp_increment(ctxt, ctxt->src.val);
2062 return X86EMUL_CONTINUE;
2065 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2067 /* Save real source value, then compare EAX against destination. */
2068 ctxt->dst.orig_val = ctxt->dst.val;
2069 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2070 ctxt->src.orig_val = ctxt->src.val;
2071 ctxt->src.val = ctxt->dst.orig_val;
2072 fastop(ctxt, em_cmp);
2074 if (ctxt->eflags & EFLG_ZF) {
2075 /* Success: write back to memory. */
2076 ctxt->dst.val = ctxt->src.orig_val;
2078 /* Failure: write the value we saw to EAX. */
2079 ctxt->dst.type = OP_REG;
2080 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2081 ctxt->dst.val = ctxt->dst.orig_val;
2083 return X86EMUL_CONTINUE;
2086 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2088 int seg = ctxt->src2.val;
2092 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2094 rc = load_segment_descriptor(ctxt, sel, seg);
2095 if (rc != X86EMUL_CONTINUE)
2098 ctxt->dst.val = ctxt->src.val;
2103 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2104 struct desc_struct *cs, struct desc_struct *ss)
2106 cs->l = 0; /* will be adjusted later */
2107 set_desc_base(cs, 0); /* flat segment */
2108 cs->g = 1; /* 4kb granularity */
2109 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2110 cs->type = 0x0b; /* Read, Execute, Accessed */
2112 cs->dpl = 0; /* will be adjusted later */
2117 set_desc_base(ss, 0); /* flat segment */
2118 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2119 ss->g = 1; /* 4kb granularity */
2121 ss->type = 0x03; /* Read/Write, Accessed */
2122 ss->d = 1; /* 32bit stack segment */
2129 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2131 u32 eax, ebx, ecx, edx;
2134 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2135 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2136 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2137 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2140 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2142 const struct x86_emulate_ops *ops = ctxt->ops;
2143 u32 eax, ebx, ecx, edx;
2146 * syscall should always be enabled in longmode - so only become
2147 * vendor specific (cpuid) if other modes are active...
2149 if (ctxt->mode == X86EMUL_MODE_PROT64)
2154 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2156 * Intel ("GenuineIntel")
2157 * remark: Intel CPUs only support "syscall" in 64bit
2158 * longmode. Also an 64bit guest with a
2159 * 32bit compat-app running will #UD !! While this
2160 * behaviour can be fixed (by emulating) into AMD
2161 * response - CPUs of AMD can't behave like Intel.
2163 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2164 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2165 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2168 /* AMD ("AuthenticAMD") */
2169 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2170 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2171 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2174 /* AMD ("AMDisbetter!") */
2175 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2176 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2177 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2180 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2184 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2186 const struct x86_emulate_ops *ops = ctxt->ops;
2187 struct desc_struct cs, ss;
2192 /* syscall is not available in real mode */
2193 if (ctxt->mode == X86EMUL_MODE_REAL ||
2194 ctxt->mode == X86EMUL_MODE_VM86)
2195 return emulate_ud(ctxt);
2197 if (!(em_syscall_is_enabled(ctxt)))
2198 return emulate_ud(ctxt);
2200 ops->get_msr(ctxt, MSR_EFER, &efer);
2201 setup_syscalls_segments(ctxt, &cs, &ss);
2203 if (!(efer & EFER_SCE))
2204 return emulate_ud(ctxt);
2206 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2208 cs_sel = (u16)(msr_data & 0xfffc);
2209 ss_sel = (u16)(msr_data + 8);
2211 if (efer & EFER_LMA) {
2215 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2216 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2218 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2219 if (efer & EFER_LMA) {
2220 #ifdef CONFIG_X86_64
2221 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2224 ctxt->mode == X86EMUL_MODE_PROT64 ?
2225 MSR_LSTAR : MSR_CSTAR, &msr_data);
2226 ctxt->_eip = msr_data;
2228 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2229 ctxt->eflags &= ~(msr_data | EFLG_RF);
2233 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2234 ctxt->_eip = (u32)msr_data;
2236 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2239 return X86EMUL_CONTINUE;
2242 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2244 const struct x86_emulate_ops *ops = ctxt->ops;
2245 struct desc_struct cs, ss;
2250 ops->get_msr(ctxt, MSR_EFER, &efer);
2251 /* inject #GP if in real mode */
2252 if (ctxt->mode == X86EMUL_MODE_REAL)
2253 return emulate_gp(ctxt, 0);
2256 * Not recognized on AMD in compat mode (but is recognized in legacy
2259 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2260 && !vendor_intel(ctxt))
2261 return emulate_ud(ctxt);
2263 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2264 * Therefore, we inject an #UD.
2266 if (ctxt->mode == X86EMUL_MODE_PROT64)
2267 return emulate_ud(ctxt);
2269 setup_syscalls_segments(ctxt, &cs, &ss);
2271 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2272 switch (ctxt->mode) {
2273 case X86EMUL_MODE_PROT32:
2274 if ((msr_data & 0xfffc) == 0x0)
2275 return emulate_gp(ctxt, 0);
2277 case X86EMUL_MODE_PROT64:
2278 if (msr_data == 0x0)
2279 return emulate_gp(ctxt, 0);
2285 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2286 cs_sel = (u16)msr_data;
2287 cs_sel &= ~SELECTOR_RPL_MASK;
2288 ss_sel = cs_sel + 8;
2289 ss_sel &= ~SELECTOR_RPL_MASK;
2290 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2295 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2296 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2298 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2299 ctxt->_eip = msr_data;
2301 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2302 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2304 return X86EMUL_CONTINUE;
2307 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2309 const struct x86_emulate_ops *ops = ctxt->ops;
2310 struct desc_struct cs, ss;
2313 u16 cs_sel = 0, ss_sel = 0;
2315 /* inject #GP if in real mode or Virtual 8086 mode */
2316 if (ctxt->mode == X86EMUL_MODE_REAL ||
2317 ctxt->mode == X86EMUL_MODE_VM86)
2318 return emulate_gp(ctxt, 0);
2320 setup_syscalls_segments(ctxt, &cs, &ss);
2322 if ((ctxt->rex_prefix & 0x8) != 0x0)
2323 usermode = X86EMUL_MODE_PROT64;
2325 usermode = X86EMUL_MODE_PROT32;
2329 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2331 case X86EMUL_MODE_PROT32:
2332 cs_sel = (u16)(msr_data + 16);
2333 if ((msr_data & 0xfffc) == 0x0)
2334 return emulate_gp(ctxt, 0);
2335 ss_sel = (u16)(msr_data + 24);
2337 case X86EMUL_MODE_PROT64:
2338 cs_sel = (u16)(msr_data + 32);
2339 if (msr_data == 0x0)
2340 return emulate_gp(ctxt, 0);
2341 ss_sel = cs_sel + 8;
2346 cs_sel |= SELECTOR_RPL_MASK;
2347 ss_sel |= SELECTOR_RPL_MASK;
2349 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2350 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2352 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2353 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2355 return X86EMUL_CONTINUE;
2358 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2361 if (ctxt->mode == X86EMUL_MODE_REAL)
2363 if (ctxt->mode == X86EMUL_MODE_VM86)
2365 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2366 return ctxt->ops->cpl(ctxt) > iopl;
2369 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2372 const struct x86_emulate_ops *ops = ctxt->ops;
2373 struct desc_struct tr_seg;
2376 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2377 unsigned mask = (1 << len) - 1;
2380 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2383 if (desc_limit_scaled(&tr_seg) < 103)
2385 base = get_desc_base(&tr_seg);
2386 #ifdef CONFIG_X86_64
2387 base |= ((u64)base3) << 32;
2389 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2390 if (r != X86EMUL_CONTINUE)
2392 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2394 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2395 if (r != X86EMUL_CONTINUE)
2397 if ((perm >> bit_idx) & mask)
2402 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2408 if (emulator_bad_iopl(ctxt))
2409 if (!emulator_io_port_access_allowed(ctxt, port, len))
2412 ctxt->perm_ok = true;
2417 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2418 struct tss_segment_16 *tss)
2420 tss->ip = ctxt->_eip;
2421 tss->flag = ctxt->eflags;
2422 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2423 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2424 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2425 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2426 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2427 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2428 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2429 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2431 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2432 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2433 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2434 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2435 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2438 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2439 struct tss_segment_16 *tss)
2444 ctxt->_eip = tss->ip;
2445 ctxt->eflags = tss->flag | 2;
2446 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2447 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2448 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2449 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2450 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2451 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2452 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2453 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2456 * SDM says that segment selectors are loaded before segment
2459 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2460 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2461 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2462 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2463 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2468 * Now load segment descriptors. If fault happens at this stage
2469 * it is handled in a context of new task
2471 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2472 if (ret != X86EMUL_CONTINUE)
2474 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2475 if (ret != X86EMUL_CONTINUE)
2477 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2478 if (ret != X86EMUL_CONTINUE)
2480 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2481 if (ret != X86EMUL_CONTINUE)
2483 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2484 if (ret != X86EMUL_CONTINUE)
2487 return X86EMUL_CONTINUE;
2490 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2491 u16 tss_selector, u16 old_tss_sel,
2492 ulong old_tss_base, struct desc_struct *new_desc)
2494 const struct x86_emulate_ops *ops = ctxt->ops;
2495 struct tss_segment_16 tss_seg;
2497 u32 new_tss_base = get_desc_base(new_desc);
2499 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2501 if (ret != X86EMUL_CONTINUE)
2502 /* FIXME: need to provide precise fault address */
2505 save_state_to_tss16(ctxt, &tss_seg);
2507 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2509 if (ret != X86EMUL_CONTINUE)
2510 /* FIXME: need to provide precise fault address */
2513 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2515 if (ret != X86EMUL_CONTINUE)
2516 /* FIXME: need to provide precise fault address */
2519 if (old_tss_sel != 0xffff) {
2520 tss_seg.prev_task_link = old_tss_sel;
2522 ret = ops->write_std(ctxt, new_tss_base,
2523 &tss_seg.prev_task_link,
2524 sizeof tss_seg.prev_task_link,
2526 if (ret != X86EMUL_CONTINUE)
2527 /* FIXME: need to provide precise fault address */
2531 return load_state_from_tss16(ctxt, &tss_seg);
2534 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2535 struct tss_segment_32 *tss)
2537 /* CR3 and ldt selector are not saved intentionally */
2538 tss->eip = ctxt->_eip;
2539 tss->eflags = ctxt->eflags;
2540 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2541 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2542 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2543 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2544 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2545 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2546 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2547 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2549 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2550 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2551 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2552 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2553 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2554 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2557 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2558 struct tss_segment_32 *tss)
2563 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2564 return emulate_gp(ctxt, 0);
2565 ctxt->_eip = tss->eip;
2566 ctxt->eflags = tss->eflags | 2;
2568 /* General purpose registers */
2569 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2570 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2571 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2572 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2573 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2574 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2575 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2576 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2579 * SDM says that segment selectors are loaded before segment
2580 * descriptors. This is important because CPL checks will
2583 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2584 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2585 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2586 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2587 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2588 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2589 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2592 * If we're switching between Protected Mode and VM86, we need to make
2593 * sure to update the mode before loading the segment descriptors so
2594 * that the selectors are interpreted correctly.
2596 if (ctxt->eflags & X86_EFLAGS_VM) {
2597 ctxt->mode = X86EMUL_MODE_VM86;
2600 ctxt->mode = X86EMUL_MODE_PROT32;
2605 * Now load segment descriptors. If fault happenes at this stage
2606 * it is handled in a context of new task
2608 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2609 if (ret != X86EMUL_CONTINUE)
2611 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2612 if (ret != X86EMUL_CONTINUE)
2614 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2615 if (ret != X86EMUL_CONTINUE)
2617 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2618 if (ret != X86EMUL_CONTINUE)
2620 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2621 if (ret != X86EMUL_CONTINUE)
2623 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2627 if (ret != X86EMUL_CONTINUE)
2630 return X86EMUL_CONTINUE;
2633 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2634 u16 tss_selector, u16 old_tss_sel,
2635 ulong old_tss_base, struct desc_struct *new_desc)
2637 const struct x86_emulate_ops *ops = ctxt->ops;
2638 struct tss_segment_32 tss_seg;
2640 u32 new_tss_base = get_desc_base(new_desc);
2641 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2642 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2644 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2646 if (ret != X86EMUL_CONTINUE)
2647 /* FIXME: need to provide precise fault address */
2650 save_state_to_tss32(ctxt, &tss_seg);
2652 /* Only GP registers and segment selectors are saved */
2653 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2654 ldt_sel_offset - eip_offset, &ctxt->exception);
2655 if (ret != X86EMUL_CONTINUE)
2656 /* FIXME: need to provide precise fault address */
2659 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2661 if (ret != X86EMUL_CONTINUE)
2662 /* FIXME: need to provide precise fault address */
2665 if (old_tss_sel != 0xffff) {
2666 tss_seg.prev_task_link = old_tss_sel;
2668 ret = ops->write_std(ctxt, new_tss_base,
2669 &tss_seg.prev_task_link,
2670 sizeof tss_seg.prev_task_link,
2672 if (ret != X86EMUL_CONTINUE)
2673 /* FIXME: need to provide precise fault address */
2677 return load_state_from_tss32(ctxt, &tss_seg);
2680 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2681 u16 tss_selector, int idt_index, int reason,
2682 bool has_error_code, u32 error_code)
2684 const struct x86_emulate_ops *ops = ctxt->ops;
2685 struct desc_struct curr_tss_desc, next_tss_desc;
2687 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2688 ulong old_tss_base =
2689 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2693 /* FIXME: old_tss_base == ~0 ? */
2695 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2696 if (ret != X86EMUL_CONTINUE)
2698 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2699 if (ret != X86EMUL_CONTINUE)
2702 /* FIXME: check that next_tss_desc is tss */
2705 * Check privileges. The three cases are task switch caused by...
2707 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2708 * 2. Exception/IRQ/iret: No check is performed
2709 * 3. jmp/call to TSS: Check against DPL of the TSS
2711 if (reason == TASK_SWITCH_GATE) {
2712 if (idt_index != -1) {
2713 /* Software interrupts */
2714 struct desc_struct task_gate_desc;
2717 ret = read_interrupt_descriptor(ctxt, idt_index,
2719 if (ret != X86EMUL_CONTINUE)
2722 dpl = task_gate_desc.dpl;
2723 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2724 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2726 } else if (reason != TASK_SWITCH_IRET) {
2727 int dpl = next_tss_desc.dpl;
2728 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2729 return emulate_gp(ctxt, tss_selector);
2733 desc_limit = desc_limit_scaled(&next_tss_desc);
2734 if (!next_tss_desc.p ||
2735 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2736 desc_limit < 0x2b)) {
2737 emulate_ts(ctxt, tss_selector & 0xfffc);
2738 return X86EMUL_PROPAGATE_FAULT;
2741 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2742 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2743 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2746 if (reason == TASK_SWITCH_IRET)
2747 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2749 /* set back link to prev task only if NT bit is set in eflags
2750 note that old_tss_sel is not used after this point */
2751 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2752 old_tss_sel = 0xffff;
2754 if (next_tss_desc.type & 8)
2755 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2756 old_tss_base, &next_tss_desc);
2758 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2759 old_tss_base, &next_tss_desc);
2760 if (ret != X86EMUL_CONTINUE)
2763 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2764 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2766 if (reason != TASK_SWITCH_IRET) {
2767 next_tss_desc.type |= (1 << 1); /* set busy flag */
2768 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2771 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2772 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2774 if (has_error_code) {
2775 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2776 ctxt->lock_prefix = 0;
2777 ctxt->src.val = (unsigned long) error_code;
2778 ret = em_push(ctxt);
2784 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2785 u16 tss_selector, int idt_index, int reason,
2786 bool has_error_code, u32 error_code)
2790 invalidate_registers(ctxt);
2791 ctxt->_eip = ctxt->eip;
2792 ctxt->dst.type = OP_NONE;
2794 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2795 has_error_code, error_code);
2797 if (rc == X86EMUL_CONTINUE) {
2798 ctxt->eip = ctxt->_eip;
2799 writeback_registers(ctxt);
2802 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2805 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2808 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2810 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2811 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2814 static int em_das(struct x86_emulate_ctxt *ctxt)
2817 bool af, cf, old_cf;
2819 cf = ctxt->eflags & X86_EFLAGS_CF;
2825 af = ctxt->eflags & X86_EFLAGS_AF;
2826 if ((al & 0x0f) > 9 || af) {
2828 cf = old_cf | (al >= 250);
2833 if (old_al > 0x99 || old_cf) {
2839 /* Set PF, ZF, SF */
2840 ctxt->src.type = OP_IMM;
2842 ctxt->src.bytes = 1;
2843 fastop(ctxt, em_or);
2844 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2846 ctxt->eflags |= X86_EFLAGS_CF;
2848 ctxt->eflags |= X86_EFLAGS_AF;
2849 return X86EMUL_CONTINUE;
2852 static int em_aam(struct x86_emulate_ctxt *ctxt)
2856 if (ctxt->src.val == 0)
2857 return emulate_de(ctxt);
2859 al = ctxt->dst.val & 0xff;
2860 ah = al / ctxt->src.val;
2861 al %= ctxt->src.val;
2863 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2865 /* Set PF, ZF, SF */
2866 ctxt->src.type = OP_IMM;
2868 ctxt->src.bytes = 1;
2869 fastop(ctxt, em_or);
2871 return X86EMUL_CONTINUE;
2874 static int em_aad(struct x86_emulate_ctxt *ctxt)
2876 u8 al = ctxt->dst.val & 0xff;
2877 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2879 al = (al + (ah * ctxt->src.val)) & 0xff;
2881 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2883 /* Set PF, ZF, SF */
2884 ctxt->src.type = OP_IMM;
2886 ctxt->src.bytes = 1;
2887 fastop(ctxt, em_or);
2889 return X86EMUL_CONTINUE;
2892 static int em_call(struct x86_emulate_ctxt *ctxt)
2894 long rel = ctxt->src.val;
2896 ctxt->src.val = (unsigned long)ctxt->_eip;
2898 return em_push(ctxt);
2901 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2907 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2908 old_eip = ctxt->_eip;
2910 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2911 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2912 return X86EMUL_CONTINUE;
2915 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2917 ctxt->src.val = old_cs;
2919 if (rc != X86EMUL_CONTINUE)
2922 ctxt->src.val = old_eip;
2923 return em_push(ctxt);
2926 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2930 ctxt->dst.type = OP_REG;
2931 ctxt->dst.addr.reg = &ctxt->_eip;
2932 ctxt->dst.bytes = ctxt->op_bytes;
2933 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2934 if (rc != X86EMUL_CONTINUE)
2936 rsp_increment(ctxt, ctxt->src.val);
2937 return X86EMUL_CONTINUE;
2940 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2942 /* Write back the register source. */
2943 ctxt->src.val = ctxt->dst.val;
2944 write_register_operand(&ctxt->src);
2946 /* Write back the memory destination with implicit LOCK prefix. */
2947 ctxt->dst.val = ctxt->src.orig_val;
2948 ctxt->lock_prefix = 1;
2949 return X86EMUL_CONTINUE;
2952 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2954 ctxt->dst.val = ctxt->src2.val;
2955 return fastop(ctxt, em_imul);
2958 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2960 ctxt->dst.type = OP_REG;
2961 ctxt->dst.bytes = ctxt->src.bytes;
2962 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2963 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2965 return X86EMUL_CONTINUE;
2968 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2972 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2973 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2974 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2975 return X86EMUL_CONTINUE;
2978 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2982 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2983 return emulate_gp(ctxt, 0);
2984 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2985 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2986 return X86EMUL_CONTINUE;
2989 static int em_mov(struct x86_emulate_ctxt *ctxt)
2991 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2992 return X86EMUL_CONTINUE;
2995 #define FFL(x) bit(X86_FEATURE_##x)
2997 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2999 u32 ebx, ecx, edx, eax = 1;
3003 * Check MOVBE is set in the guest-visible CPUID leaf.
3005 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3006 if (!(ecx & FFL(MOVBE)))
3007 return emulate_ud(ctxt);
3009 switch (ctxt->op_bytes) {
3012 * From MOVBE definition: "...When the operand size is 16 bits,
3013 * the upper word of the destination register remains unchanged
3016 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3017 * rules so we have to do the operation almost per hand.
3019 tmp = (u16)ctxt->src.val;
3020 ctxt->dst.val &= ~0xffffUL;
3021 ctxt->dst.val |= (unsigned long)swab16(tmp);
3024 ctxt->dst.val = swab32((u32)ctxt->src.val);
3027 ctxt->dst.val = swab64(ctxt->src.val);
3030 return X86EMUL_PROPAGATE_FAULT;
3032 return X86EMUL_CONTINUE;
3035 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3037 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3038 return emulate_gp(ctxt, 0);
3040 /* Disable writeback. */
3041 ctxt->dst.type = OP_NONE;
3042 return X86EMUL_CONTINUE;
3045 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3049 if (ctxt->mode == X86EMUL_MODE_PROT64)
3050 val = ctxt->src.val & ~0ULL;
3052 val = ctxt->src.val & ~0U;
3054 /* #UD condition is already handled. */
3055 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3056 return emulate_gp(ctxt, 0);
3058 /* Disable writeback. */
3059 ctxt->dst.type = OP_NONE;
3060 return X86EMUL_CONTINUE;
3063 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3067 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3068 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3069 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3070 return emulate_gp(ctxt, 0);
3072 return X86EMUL_CONTINUE;
3075 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3079 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3080 return emulate_gp(ctxt, 0);
3082 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3083 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3084 return X86EMUL_CONTINUE;
3087 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3089 if (ctxt->modrm_reg > VCPU_SREG_GS)
3090 return emulate_ud(ctxt);
3092 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3093 return X86EMUL_CONTINUE;
3096 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3098 u16 sel = ctxt->src.val;
3100 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3101 return emulate_ud(ctxt);
3103 if (ctxt->modrm_reg == VCPU_SREG_SS)
3104 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3106 /* Disable writeback. */
3107 ctxt->dst.type = OP_NONE;
3108 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3111 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3113 u16 sel = ctxt->src.val;
3115 /* Disable writeback. */
3116 ctxt->dst.type = OP_NONE;
3117 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3120 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3122 u16 sel = ctxt->src.val;
3124 /* Disable writeback. */
3125 ctxt->dst.type = OP_NONE;
3126 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3129 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3134 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3135 if (rc == X86EMUL_CONTINUE)
3136 ctxt->ops->invlpg(ctxt, linear);
3137 /* Disable writeback. */
3138 ctxt->dst.type = OP_NONE;
3139 return X86EMUL_CONTINUE;
3142 static int em_clts(struct x86_emulate_ctxt *ctxt)
3146 cr0 = ctxt->ops->get_cr(ctxt, 0);
3148 ctxt->ops->set_cr(ctxt, 0, cr0);
3149 return X86EMUL_CONTINUE;
3152 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3156 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3157 return X86EMUL_UNHANDLEABLE;
3159 rc = ctxt->ops->fix_hypercall(ctxt);
3160 if (rc != X86EMUL_CONTINUE)
3163 /* Let the processor re-execute the fixed hypercall */
3164 ctxt->_eip = ctxt->eip;
3165 /* Disable writeback. */
3166 ctxt->dst.type = OP_NONE;
3167 return X86EMUL_CONTINUE;
3170 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3171 void (*get)(struct x86_emulate_ctxt *ctxt,
3172 struct desc_ptr *ptr))
3174 struct desc_ptr desc_ptr;
3176 if (ctxt->mode == X86EMUL_MODE_PROT64)
3178 get(ctxt, &desc_ptr);
3179 if (ctxt->op_bytes == 2) {
3181 desc_ptr.address &= 0x00ffffff;
3183 /* Disable writeback. */
3184 ctxt->dst.type = OP_NONE;
3185 return segmented_write(ctxt, ctxt->dst.addr.mem,
3186 &desc_ptr, 2 + ctxt->op_bytes);
3189 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3191 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3194 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3196 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3199 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3201 struct desc_ptr desc_ptr;
3204 if (ctxt->mode == X86EMUL_MODE_PROT64)
3206 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3207 &desc_ptr.size, &desc_ptr.address,
3209 if (rc != X86EMUL_CONTINUE)
3211 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3212 /* Disable writeback. */
3213 ctxt->dst.type = OP_NONE;
3214 return X86EMUL_CONTINUE;
3217 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3221 rc = ctxt->ops->fix_hypercall(ctxt);
3223 /* Disable writeback. */
3224 ctxt->dst.type = OP_NONE;
3228 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3230 struct desc_ptr desc_ptr;
3233 if (ctxt->mode == X86EMUL_MODE_PROT64)
3235 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3236 &desc_ptr.size, &desc_ptr.address,
3238 if (rc != X86EMUL_CONTINUE)
3240 ctxt->ops->set_idt(ctxt, &desc_ptr);
3241 /* Disable writeback. */
3242 ctxt->dst.type = OP_NONE;
3243 return X86EMUL_CONTINUE;
3246 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3248 if (ctxt->dst.type == OP_MEM)
3249 ctxt->dst.bytes = 2;
3250 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3251 return X86EMUL_CONTINUE;
3254 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3256 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3257 | (ctxt->src.val & 0x0f));
3258 ctxt->dst.type = OP_NONE;
3259 return X86EMUL_CONTINUE;
3262 static int em_loop(struct x86_emulate_ctxt *ctxt)
3264 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3265 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3266 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3267 jmp_rel(ctxt, ctxt->src.val);
3269 return X86EMUL_CONTINUE;
3272 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3274 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3275 jmp_rel(ctxt, ctxt->src.val);
3277 return X86EMUL_CONTINUE;
3280 static int em_in(struct x86_emulate_ctxt *ctxt)
3282 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3284 return X86EMUL_IO_NEEDED;
3286 return X86EMUL_CONTINUE;
3289 static int em_out(struct x86_emulate_ctxt *ctxt)
3291 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3293 /* Disable writeback. */
3294 ctxt->dst.type = OP_NONE;
3295 return X86EMUL_CONTINUE;
3298 static int em_cli(struct x86_emulate_ctxt *ctxt)
3300 if (emulator_bad_iopl(ctxt))
3301 return emulate_gp(ctxt, 0);
3303 ctxt->eflags &= ~X86_EFLAGS_IF;
3304 return X86EMUL_CONTINUE;
3307 static int em_sti(struct x86_emulate_ctxt *ctxt)
3309 if (emulator_bad_iopl(ctxt))
3310 return emulate_gp(ctxt, 0);
3312 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3313 ctxt->eflags |= X86_EFLAGS_IF;
3314 return X86EMUL_CONTINUE;
3317 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3319 u32 eax, ebx, ecx, edx;
3321 eax = reg_read(ctxt, VCPU_REGS_RAX);
3322 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3323 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3324 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3325 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3326 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3327 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3328 return X86EMUL_CONTINUE;
3331 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3335 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3336 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3338 ctxt->eflags &= ~0xffUL;
3339 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3340 return X86EMUL_CONTINUE;
3343 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3345 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3346 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3347 return X86EMUL_CONTINUE;
3350 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3352 switch (ctxt->op_bytes) {
3353 #ifdef CONFIG_X86_64
3355 asm("bswap %0" : "+r"(ctxt->dst.val));
3359 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3362 return X86EMUL_CONTINUE;
3365 static bool valid_cr(int nr)
3377 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3379 if (!valid_cr(ctxt->modrm_reg))
3380 return emulate_ud(ctxt);
3382 return X86EMUL_CONTINUE;
3385 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3387 u64 new_val = ctxt->src.val64;
3388 int cr = ctxt->modrm_reg;
3391 static u64 cr_reserved_bits[] = {
3392 0xffffffff00000000ULL,
3393 0, 0, 0, /* CR3 checked later */
3400 return emulate_ud(ctxt);
3402 if (new_val & cr_reserved_bits[cr])
3403 return emulate_gp(ctxt, 0);
3408 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3409 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3410 return emulate_gp(ctxt, 0);
3412 cr4 = ctxt->ops->get_cr(ctxt, 4);
3413 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3415 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3416 !(cr4 & X86_CR4_PAE))
3417 return emulate_gp(ctxt, 0);
3424 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3425 if (efer & EFER_LMA)
3426 rsvd = CR3_L_MODE_RESERVED_BITS;
3429 return emulate_gp(ctxt, 0);
3434 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3436 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3437 return emulate_gp(ctxt, 0);
3443 return X86EMUL_CONTINUE;
3446 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3450 ctxt->ops->get_dr(ctxt, 7, &dr7);
3452 /* Check if DR7.Global_Enable is set */
3453 return dr7 & (1 << 13);
3456 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3458 int dr = ctxt->modrm_reg;
3462 return emulate_ud(ctxt);
3464 cr4 = ctxt->ops->get_cr(ctxt, 4);
3465 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3466 return emulate_ud(ctxt);
3468 if (check_dr7_gd(ctxt))
3469 return emulate_db(ctxt);
3471 return X86EMUL_CONTINUE;
3474 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3476 u64 new_val = ctxt->src.val64;
3477 int dr = ctxt->modrm_reg;
3479 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3480 return emulate_gp(ctxt, 0);
3482 return check_dr_read(ctxt);
3485 static int check_svme(struct x86_emulate_ctxt *ctxt)
3489 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3491 if (!(efer & EFER_SVME))
3492 return emulate_ud(ctxt);
3494 return X86EMUL_CONTINUE;
3497 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3499 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3501 /* Valid physical address? */
3502 if (rax & 0xffff000000000000ULL)
3503 return emulate_gp(ctxt, 0);
3505 return check_svme(ctxt);
3508 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3510 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3512 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3513 return emulate_ud(ctxt);
3515 return X86EMUL_CONTINUE;
3518 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3520 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3521 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3523 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3524 ctxt->ops->check_pmc(ctxt, rcx))
3525 return emulate_gp(ctxt, 0);
3527 return X86EMUL_CONTINUE;
3530 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3532 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3533 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3534 return emulate_gp(ctxt, 0);
3536 return X86EMUL_CONTINUE;
3539 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3541 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3542 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3543 return emulate_gp(ctxt, 0);
3545 return X86EMUL_CONTINUE;
3548 #define D(_y) { .flags = (_y) }
3549 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3550 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3551 .check_perm = (_p) }
3552 #define N D(NotImpl)
3553 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3554 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3555 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3556 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3557 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3558 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3559 #define II(_f, _e, _i) \
3560 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3561 #define IIP(_f, _e, _i, _p) \
3562 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3563 .check_perm = (_p) }
3564 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3566 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3567 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3568 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3569 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3570 #define I2bvIP(_f, _e, _i, _p) \
3571 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3573 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3574 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3575 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3577 static const struct opcode group7_rm1[] = {
3578 DI(SrcNone | Priv, monitor),
3579 DI(SrcNone | Priv, mwait),
3583 static const struct opcode group7_rm3[] = {
3584 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3585 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3586 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3587 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3588 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3589 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3590 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3591 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3594 static const struct opcode group7_rm7[] = {
3596 DIP(SrcNone, rdtscp, check_rdtsc),
3600 static const struct opcode group1[] = {
3602 F(Lock | PageTable, em_or),
3605 F(Lock | PageTable, em_and),
3611 static const struct opcode group1A[] = {
3612 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3615 static const struct opcode group2[] = {
3616 F(DstMem | ModRM, em_rol),
3617 F(DstMem | ModRM, em_ror),
3618 F(DstMem | ModRM, em_rcl),
3619 F(DstMem | ModRM, em_rcr),
3620 F(DstMem | ModRM, em_shl),
3621 F(DstMem | ModRM, em_shr),
3622 F(DstMem | ModRM, em_shl),
3623 F(DstMem | ModRM, em_sar),
3626 static const struct opcode group3[] = {
3627 F(DstMem | SrcImm | NoWrite, em_test),
3628 F(DstMem | SrcImm | NoWrite, em_test),
3629 F(DstMem | SrcNone | Lock, em_not),
3630 F(DstMem | SrcNone | Lock, em_neg),
3631 F(DstXacc | Src2Mem, em_mul_ex),
3632 F(DstXacc | Src2Mem, em_imul_ex),
3633 F(DstXacc | Src2Mem, em_div_ex),
3634 F(DstXacc | Src2Mem, em_idiv_ex),
3637 static const struct opcode group4[] = {
3638 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3639 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3643 static const struct opcode group5[] = {
3644 F(DstMem | SrcNone | Lock, em_inc),
3645 F(DstMem | SrcNone | Lock, em_dec),
3646 I(SrcMem | Stack, em_grp45),
3647 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3648 I(SrcMem | Stack, em_grp45),
3649 I(SrcMemFAddr | ImplicitOps, em_grp45),
3650 I(SrcMem | Stack, em_grp45), D(Undefined),
3653 static const struct opcode group6[] = {
3656 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3657 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3661 static const struct group_dual group7 = { {
3662 II(Mov | DstMem, em_sgdt, sgdt),
3663 II(Mov | DstMem, em_sidt, sidt),
3664 II(SrcMem | Priv, em_lgdt, lgdt),
3665 II(SrcMem | Priv, em_lidt, lidt),
3666 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3667 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3668 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3670 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3672 N, EXT(0, group7_rm3),
3673 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3674 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3678 static const struct opcode group8[] = {
3680 F(DstMem | SrcImmByte | NoWrite, em_bt),
3681 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3682 F(DstMem | SrcImmByte | Lock, em_btr),
3683 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3686 static const struct group_dual group9 = { {
3687 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3689 N, N, N, N, N, N, N, N,
3692 static const struct opcode group11[] = {
3693 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3697 static const struct gprefix pfx_0f_6f_0f_7f = {
3698 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3701 static const struct gprefix pfx_vmovntpx = {
3702 I(0, em_mov), N, N, N,
3705 static const struct gprefix pfx_0f_28_0f_29 = {
3706 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3709 static const struct escape escape_d9 = { {
3710 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3713 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3721 N, N, N, N, N, N, N, N,
3723 N, N, N, N, N, N, N, N,
3725 N, N, N, N, N, N, N, N,
3727 N, N, N, N, N, N, N, N,
3730 static const struct escape escape_db = { {
3731 N, N, N, N, N, N, N, N,
3734 N, N, N, N, N, N, N, N,
3736 N, N, N, N, N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3742 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3744 N, N, N, N, N, N, N, N,
3746 N, N, N, N, N, N, N, N,
3748 N, N, N, N, N, N, N, N,
3751 static const struct escape escape_dd = { {
3752 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3755 N, N, N, N, N, N, N, N,
3757 N, N, N, N, N, N, N, N,
3759 N, N, N, N, N, N, N, N,
3761 N, N, N, N, N, N, N, N,
3763 N, N, N, N, N, N, N, N,
3765 N, N, N, N, N, N, N, N,
3767 N, N, N, N, N, N, N, N,
3769 N, N, N, N, N, N, N, N,
3772 static const struct opcode opcode_table[256] = {
3774 F6ALU(Lock, em_add),
3775 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3776 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3778 F6ALU(Lock | PageTable, em_or),
3779 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3782 F6ALU(Lock, em_adc),
3783 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3784 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3786 F6ALU(Lock, em_sbb),
3787 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3788 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3790 F6ALU(Lock | PageTable, em_and), N, N,
3792 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3794 F6ALU(Lock, em_xor), N, N,
3796 F6ALU(NoWrite, em_cmp), N, N,
3798 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3800 X8(I(SrcReg | Stack, em_push)),
3802 X8(I(DstReg | Stack, em_pop)),
3804 I(ImplicitOps | Stack | No64, em_pusha),
3805 I(ImplicitOps | Stack | No64, em_popa),
3806 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3809 I(SrcImm | Mov | Stack, em_push),
3810 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3811 I(SrcImmByte | Mov | Stack, em_push),
3812 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3813 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3814 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3818 G(ByteOp | DstMem | SrcImm, group1),
3819 G(DstMem | SrcImm, group1),
3820 G(ByteOp | DstMem | SrcImm | No64, group1),
3821 G(DstMem | SrcImmByte, group1),
3822 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3823 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3825 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3826 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3827 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3828 D(ModRM | SrcMem | NoAccess | DstReg),
3829 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3832 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3834 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3835 I(SrcImmFAddr | No64, em_call_far), N,
3836 II(ImplicitOps | Stack, em_pushf, pushf),
3837 II(ImplicitOps | Stack, em_popf, popf),
3838 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3840 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3841 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3842 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3843 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3845 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3846 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3847 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3848 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3850 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3852 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3854 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3855 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3856 I(ImplicitOps | Stack, em_ret),
3857 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3858 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3859 G(ByteOp, group11), G(0, group11),
3861 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3862 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3863 I(ImplicitOps | Stack, em_ret_far),
3864 D(ImplicitOps), DI(SrcImmByte, intn),
3865 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3867 G(Src2One | ByteOp, group2), G(Src2One, group2),
3868 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3869 I(DstAcc | SrcImmUByte | No64, em_aam),
3870 I(DstAcc | SrcImmUByte | No64, em_aad),
3871 F(DstAcc | ByteOp | No64, em_salc),
3872 I(DstAcc | SrcXLat | ByteOp, em_mov),
3874 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3876 X3(I(SrcImmByte, em_loop)),
3877 I(SrcImmByte, em_jcxz),
3878 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3879 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3881 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3882 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3883 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3884 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3886 N, DI(ImplicitOps, icebp), N, N,
3887 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3888 G(ByteOp, group3), G(0, group3),
3890 D(ImplicitOps), D(ImplicitOps),
3891 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3892 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3895 static const struct opcode twobyte_table[256] = {
3897 G(0, group6), GD(0, &group7), N, N,
3898 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3899 II(ImplicitOps | Priv, em_clts, clts), N,
3900 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3901 N, D(ImplicitOps | ModRM), N, N,
3903 N, N, N, N, N, N, N, N,
3904 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3906 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3907 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3908 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3910 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3913 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3914 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3915 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3918 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3919 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3920 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3921 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3922 I(ImplicitOps | EmulateOnUD, em_sysenter),
3923 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3925 N, N, N, N, N, N, N, N,
3927 X16(D(DstReg | SrcMem | ModRM)),
3929 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3934 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3939 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3943 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3945 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3946 II(ImplicitOps, em_cpuid, cpuid),
3947 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3948 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3949 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3951 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3952 DI(ImplicitOps, rsm),
3953 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3954 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3955 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3956 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3958 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3959 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3960 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3961 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3962 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3963 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3967 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3968 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3969 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3971 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3972 N, D(DstMem | SrcReg | ModRM | Mov),
3973 N, N, N, GD(0, &group9),
3975 X8(I(DstReg, em_bswap)),
3977 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3979 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3981 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3984 static const struct gprefix three_byte_0f_38_f0 = {
3985 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3988 static const struct gprefix three_byte_0f_38_f1 = {
3989 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3993 * Insns below are selected by the prefix which indexed by the third opcode
3996 static const struct opcode opcode_map_0f_38[256] = {
3998 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4000 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4002 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
4003 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4022 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4026 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4032 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4033 unsigned size, bool sign_extension)
4035 int rc = X86EMUL_CONTINUE;
4039 op->addr.mem.ea = ctxt->_eip;
4040 /* NB. Immediates are sign-extended as necessary. */
4041 switch (op->bytes) {
4043 op->val = insn_fetch(s8, ctxt);
4046 op->val = insn_fetch(s16, ctxt);
4049 op->val = insn_fetch(s32, ctxt);
4052 op->val = insn_fetch(s64, ctxt);
4055 if (!sign_extension) {
4056 switch (op->bytes) {
4064 op->val &= 0xffffffff;
4072 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4075 int rc = X86EMUL_CONTINUE;
4079 decode_register_operand(ctxt, op);
4082 rc = decode_imm(ctxt, op, 1, false);
4085 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4089 if (ctxt->d & BitOp)
4090 fetch_bit_operand(ctxt);
4091 op->orig_val = op->val;
4094 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4098 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4099 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4100 fetch_register_operand(op);
4101 op->orig_val = op->val;
4105 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4106 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4107 fetch_register_operand(op);
4108 op->orig_val = op->val;
4111 if (ctxt->d & ByteOp) {
4116 op->bytes = ctxt->op_bytes;
4117 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4118 fetch_register_operand(op);
4119 op->orig_val = op->val;
4123 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4125 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4126 op->addr.mem.seg = VCPU_SREG_ES;
4133 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4134 fetch_register_operand(op);
4138 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4141 rc = decode_imm(ctxt, op, 1, true);
4148 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4151 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4154 ctxt->memop.bytes = 1;
4155 if (ctxt->memop.type == OP_REG) {
4156 ctxt->memop.addr.reg = decode_register(ctxt,
4157 ctxt->modrm_rm, true);
4158 fetch_register_operand(&ctxt->memop);
4162 ctxt->memop.bytes = 2;
4165 ctxt->memop.bytes = 4;
4168 rc = decode_imm(ctxt, op, 2, false);
4171 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4175 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4177 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4178 op->addr.mem.seg = seg_override(ctxt);
4184 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4186 register_address(ctxt,
4187 reg_read(ctxt, VCPU_REGS_RBX) +
4188 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4189 op->addr.mem.seg = seg_override(ctxt);
4194 op->addr.mem.ea = ctxt->_eip;
4195 op->bytes = ctxt->op_bytes + 2;
4196 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4199 ctxt->memop.bytes = ctxt->op_bytes + 2;
4202 op->val = VCPU_SREG_ES;
4205 op->val = VCPU_SREG_CS;
4208 op->val = VCPU_SREG_SS;
4211 op->val = VCPU_SREG_DS;
4214 op->val = VCPU_SREG_FS;
4217 op->val = VCPU_SREG_GS;
4220 /* Special instructions do their own operand decoding. */
4222 op->type = OP_NONE; /* Disable writeback. */
4230 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4232 int rc = X86EMUL_CONTINUE;
4233 int mode = ctxt->mode;
4234 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4235 bool op_prefix = false;
4236 struct opcode opcode;
4238 ctxt->memop.type = OP_NONE;
4239 ctxt->memopp = NULL;
4240 ctxt->_eip = ctxt->eip;
4241 ctxt->fetch.start = ctxt->_eip;
4242 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4243 ctxt->opcode_len = 1;
4245 memcpy(ctxt->fetch.data, insn, insn_len);
4248 case X86EMUL_MODE_REAL:
4249 case X86EMUL_MODE_VM86:
4250 case X86EMUL_MODE_PROT16:
4251 def_op_bytes = def_ad_bytes = 2;
4253 case X86EMUL_MODE_PROT32:
4254 def_op_bytes = def_ad_bytes = 4;
4256 #ifdef CONFIG_X86_64
4257 case X86EMUL_MODE_PROT64:
4263 return EMULATION_FAILED;
4266 ctxt->op_bytes = def_op_bytes;
4267 ctxt->ad_bytes = def_ad_bytes;
4269 /* Legacy prefixes. */
4271 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4272 case 0x66: /* operand-size override */
4274 /* switch between 2/4 bytes */
4275 ctxt->op_bytes = def_op_bytes ^ 6;
4277 case 0x67: /* address-size override */
4278 if (mode == X86EMUL_MODE_PROT64)
4279 /* switch between 4/8 bytes */
4280 ctxt->ad_bytes = def_ad_bytes ^ 12;
4282 /* switch between 2/4 bytes */
4283 ctxt->ad_bytes = def_ad_bytes ^ 6;
4285 case 0x26: /* ES override */
4286 case 0x2e: /* CS override */
4287 case 0x36: /* SS override */
4288 case 0x3e: /* DS override */
4289 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4291 case 0x64: /* FS override */
4292 case 0x65: /* GS override */
4293 set_seg_override(ctxt, ctxt->b & 7);
4295 case 0x40 ... 0x4f: /* REX */
4296 if (mode != X86EMUL_MODE_PROT64)
4298 ctxt->rex_prefix = ctxt->b;
4300 case 0xf0: /* LOCK */
4301 ctxt->lock_prefix = 1;
4303 case 0xf2: /* REPNE/REPNZ */
4304 case 0xf3: /* REP/REPE/REPZ */
4305 ctxt->rep_prefix = ctxt->b;
4311 /* Any legacy prefix after a REX prefix nullifies its effect. */
4313 ctxt->rex_prefix = 0;
4319 if (ctxt->rex_prefix & 8)
4320 ctxt->op_bytes = 8; /* REX.W */
4322 /* Opcode byte(s). */
4323 opcode = opcode_table[ctxt->b];
4324 /* Two-byte opcode? */
4325 if (ctxt->b == 0x0f) {
4326 ctxt->opcode_len = 2;
4327 ctxt->b = insn_fetch(u8, ctxt);
4328 opcode = twobyte_table[ctxt->b];
4330 /* 0F_38 opcode map */
4331 if (ctxt->b == 0x38) {
4332 ctxt->opcode_len = 3;
4333 ctxt->b = insn_fetch(u8, ctxt);
4334 opcode = opcode_map_0f_38[ctxt->b];
4337 ctxt->d = opcode.flags;
4339 if (ctxt->d & ModRM)
4340 ctxt->modrm = insn_fetch(u8, ctxt);
4342 /* vex-prefix instructions are not implemented */
4343 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4344 (mode == X86EMUL_MODE_PROT64 ||
4345 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4349 while (ctxt->d & GroupMask) {
4350 switch (ctxt->d & GroupMask) {
4352 goffset = (ctxt->modrm >> 3) & 7;
4353 opcode = opcode.u.group[goffset];
4356 goffset = (ctxt->modrm >> 3) & 7;
4357 if ((ctxt->modrm >> 6) == 3)
4358 opcode = opcode.u.gdual->mod3[goffset];
4360 opcode = opcode.u.gdual->mod012[goffset];
4363 goffset = ctxt->modrm & 7;
4364 opcode = opcode.u.group[goffset];
4367 if (ctxt->rep_prefix && op_prefix)
4368 return EMULATION_FAILED;
4369 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4370 switch (simd_prefix) {
4371 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4372 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4373 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4374 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4378 if (ctxt->modrm > 0xbf)
4379 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4381 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4384 return EMULATION_FAILED;
4387 ctxt->d &= ~(u64)GroupMask;
4388 ctxt->d |= opcode.flags;
4393 return EMULATION_FAILED;
4395 ctxt->execute = opcode.u.execute;
4396 ctxt->check_perm = opcode.check_perm;
4397 ctxt->intercept = opcode.intercept;
4399 if (ctxt->d & NotImpl)
4400 return EMULATION_FAILED;
4402 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4403 return EMULATION_FAILED;
4405 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4408 if (ctxt->d & Op3264) {
4409 if (mode == X86EMUL_MODE_PROT64)
4416 ctxt->op_bytes = 16;
4417 else if (ctxt->d & Mmx)
4420 /* ModRM and SIB bytes. */
4421 if (ctxt->d & ModRM) {
4422 rc = decode_modrm(ctxt, &ctxt->memop);
4423 if (!ctxt->has_seg_override)
4424 set_seg_override(ctxt, ctxt->modrm_seg);
4425 } else if (ctxt->d & MemAbs)
4426 rc = decode_abs(ctxt, &ctxt->memop);
4427 if (rc != X86EMUL_CONTINUE)
4430 if (!ctxt->has_seg_override)
4431 set_seg_override(ctxt, VCPU_SREG_DS);
4433 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4435 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4436 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4439 * Decode and fetch the source operand: register, memory
4442 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4443 if (rc != X86EMUL_CONTINUE)
4447 * Decode and fetch the second source operand: register, memory
4450 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4451 if (rc != X86EMUL_CONTINUE)
4454 /* Decode and fetch the destination operand: register or memory. */
4455 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4458 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4459 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4461 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4464 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4466 return ctxt->d & PageTable;
4469 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4471 /* The second termination condition only applies for REPE
4472 * and REPNE. Test if the repeat string operation prefix is
4473 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4474 * corresponding termination condition according to:
4475 * - if REPE/REPZ and ZF = 0 then done
4476 * - if REPNE/REPNZ and ZF = 1 then done
4478 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4479 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4480 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4481 ((ctxt->eflags & EFLG_ZF) == 0))
4482 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4483 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4489 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4493 ctxt->ops->get_fpu(ctxt);
4494 asm volatile("1: fwait \n\t"
4496 ".pushsection .fixup,\"ax\" \n\t"
4498 "movb $1, %[fault] \n\t"
4501 _ASM_EXTABLE(1b, 3b)
4502 : [fault]"+qm"(fault));
4503 ctxt->ops->put_fpu(ctxt);
4505 if (unlikely(fault))
4506 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4508 return X86EMUL_CONTINUE;
4511 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4514 if (op->type == OP_MM)
4515 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4518 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4520 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4521 if (!(ctxt->d & ByteOp))
4522 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4523 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4524 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4526 : "c"(ctxt->src2.val));
4527 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4528 if (!fop) /* exception is returned in fop variable */
4529 return emulate_de(ctxt);
4530 return X86EMUL_CONTINUE;
4533 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4535 const struct x86_emulate_ops *ops = ctxt->ops;
4536 int rc = X86EMUL_CONTINUE;
4537 int saved_dst_type = ctxt->dst.type;
4539 ctxt->mem_read.pos = 0;
4541 /* LOCK prefix is allowed only with some instructions */
4542 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4543 rc = emulate_ud(ctxt);
4547 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4548 rc = emulate_ud(ctxt);
4552 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4553 (ctxt->d & Undefined)) {
4554 rc = emulate_ud(ctxt);
4558 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4559 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4560 rc = emulate_ud(ctxt);
4564 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4565 rc = emulate_nm(ctxt);
4569 if (ctxt->d & Mmx) {
4570 rc = flush_pending_x87_faults(ctxt);
4571 if (rc != X86EMUL_CONTINUE)
4574 * Now that we know the fpu is exception safe, we can fetch
4577 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4578 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4579 if (!(ctxt->d & Mov))
4580 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4583 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4584 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4585 X86_ICPT_PRE_EXCEPT);
4586 if (rc != X86EMUL_CONTINUE)
4590 /* Privileged instruction can be executed only in CPL=0 */
4591 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4592 rc = emulate_gp(ctxt, 0);
4596 /* Instruction can only be executed in protected mode */
4597 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4598 rc = emulate_ud(ctxt);
4602 /* Do instruction specific permission checks */
4603 if (ctxt->check_perm) {
4604 rc = ctxt->check_perm(ctxt);
4605 if (rc != X86EMUL_CONTINUE)
4609 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4610 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4611 X86_ICPT_POST_EXCEPT);
4612 if (rc != X86EMUL_CONTINUE)
4616 if (ctxt->rep_prefix && (ctxt->d & String)) {
4617 /* All REP prefixes have the same first termination condition */
4618 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4619 ctxt->eip = ctxt->_eip;
4624 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4625 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4626 ctxt->src.valptr, ctxt->src.bytes);
4627 if (rc != X86EMUL_CONTINUE)
4629 ctxt->src.orig_val64 = ctxt->src.val64;
4632 if (ctxt->src2.type == OP_MEM) {
4633 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4634 &ctxt->src2.val, ctxt->src2.bytes);
4635 if (rc != X86EMUL_CONTINUE)
4639 if ((ctxt->d & DstMask) == ImplicitOps)
4643 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4644 /* optimisation - avoid slow emulated read if Mov */
4645 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4646 &ctxt->dst.val, ctxt->dst.bytes);
4647 if (rc != X86EMUL_CONTINUE)
4650 ctxt->dst.orig_val = ctxt->dst.val;
4654 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4655 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4656 X86_ICPT_POST_MEMACCESS);
4657 if (rc != X86EMUL_CONTINUE)
4661 if (ctxt->execute) {
4662 if (ctxt->d & Fastop) {
4663 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4664 rc = fastop(ctxt, fop);
4665 if (rc != X86EMUL_CONTINUE)
4669 rc = ctxt->execute(ctxt);
4670 if (rc != X86EMUL_CONTINUE)
4675 if (ctxt->opcode_len == 2)
4677 else if (ctxt->opcode_len == 3)
4678 goto threebyte_insn;
4681 case 0x63: /* movsxd */
4682 if (ctxt->mode != X86EMUL_MODE_PROT64)
4683 goto cannot_emulate;
4684 ctxt->dst.val = (s32) ctxt->src.val;
4686 case 0x70 ... 0x7f: /* jcc (short) */
4687 if (test_cc(ctxt->b, ctxt->eflags))
4688 jmp_rel(ctxt, ctxt->src.val);
4690 case 0x8d: /* lea r16/r32, m */
4691 ctxt->dst.val = ctxt->src.addr.mem.ea;
4693 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4694 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4695 ctxt->dst.type = OP_NONE;
4699 case 0x98: /* cbw/cwde/cdqe */
4700 switch (ctxt->op_bytes) {
4701 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4702 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4703 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4706 case 0xcc: /* int3 */
4707 rc = emulate_int(ctxt, 3);
4709 case 0xcd: /* int n */
4710 rc = emulate_int(ctxt, ctxt->src.val);
4712 case 0xce: /* into */
4713 if (ctxt->eflags & EFLG_OF)
4714 rc = emulate_int(ctxt, 4);
4716 case 0xe9: /* jmp rel */
4717 case 0xeb: /* jmp rel short */
4718 jmp_rel(ctxt, ctxt->src.val);
4719 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4721 case 0xf4: /* hlt */
4722 ctxt->ops->halt(ctxt);
4724 case 0xf5: /* cmc */
4725 /* complement carry flag from eflags reg */
4726 ctxt->eflags ^= EFLG_CF;
4728 case 0xf8: /* clc */
4729 ctxt->eflags &= ~EFLG_CF;
4731 case 0xf9: /* stc */
4732 ctxt->eflags |= EFLG_CF;
4734 case 0xfc: /* cld */
4735 ctxt->eflags &= ~EFLG_DF;
4737 case 0xfd: /* std */
4738 ctxt->eflags |= EFLG_DF;
4741 goto cannot_emulate;
4744 if (rc != X86EMUL_CONTINUE)
4748 if (ctxt->d & SrcWrite) {
4749 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4750 rc = writeback(ctxt, &ctxt->src);
4751 if (rc != X86EMUL_CONTINUE)
4754 if (!(ctxt->d & NoWrite)) {
4755 rc = writeback(ctxt, &ctxt->dst);
4756 if (rc != X86EMUL_CONTINUE)
4761 * restore dst type in case the decoding will be reused
4762 * (happens for string instruction )
4764 ctxt->dst.type = saved_dst_type;
4766 if ((ctxt->d & SrcMask) == SrcSI)
4767 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4769 if ((ctxt->d & DstMask) == DstDI)
4770 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4772 if (ctxt->rep_prefix && (ctxt->d & String)) {
4774 struct read_cache *r = &ctxt->io_read;
4775 if ((ctxt->d & SrcMask) == SrcSI)
4776 count = ctxt->src.count;
4778 count = ctxt->dst.count;
4779 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4782 if (!string_insn_completed(ctxt)) {
4784 * Re-enter guest when pio read ahead buffer is empty
4785 * or, if it is not used, after each 1024 iteration.
4787 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4788 (r->end == 0 || r->end != r->pos)) {
4790 * Reset read cache. Usually happens before
4791 * decode, but since instruction is restarted
4792 * we have to do it here.
4794 ctxt->mem_read.end = 0;
4795 writeback_registers(ctxt);
4796 return EMULATION_RESTART;
4798 goto done; /* skip rip writeback */
4802 ctxt->eip = ctxt->_eip;
4805 if (rc == X86EMUL_PROPAGATE_FAULT)
4806 ctxt->have_exception = true;
4807 if (rc == X86EMUL_INTERCEPTED)
4808 return EMULATION_INTERCEPTED;
4810 if (rc == X86EMUL_CONTINUE)
4811 writeback_registers(ctxt);
4813 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4817 case 0x09: /* wbinvd */
4818 (ctxt->ops->wbinvd)(ctxt);
4820 case 0x08: /* invd */
4821 case 0x0d: /* GrpP (prefetch) */
4822 case 0x18: /* Grp16 (prefetch/nop) */
4823 case 0x1f: /* nop */
4825 case 0x20: /* mov cr, reg */
4826 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4828 case 0x21: /* mov from dr to reg */
4829 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4831 case 0x40 ... 0x4f: /* cmov */
4832 if (test_cc(ctxt->b, ctxt->eflags))
4833 ctxt->dst.val = ctxt->src.val;
4834 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4835 ctxt->op_bytes != 4)
4836 ctxt->dst.type = OP_NONE; /* no writeback */
4838 case 0x80 ... 0x8f: /* jnz rel, etc*/
4839 if (test_cc(ctxt->b, ctxt->eflags))
4840 jmp_rel(ctxt, ctxt->src.val);
4842 case 0x90 ... 0x9f: /* setcc r/m8 */
4843 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4845 case 0xae: /* clflush */
4847 case 0xb6 ... 0xb7: /* movzx */
4848 ctxt->dst.bytes = ctxt->op_bytes;
4849 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4850 : (u16) ctxt->src.val;
4852 case 0xbe ... 0xbf: /* movsx */
4853 ctxt->dst.bytes = ctxt->op_bytes;
4854 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4855 (s16) ctxt->src.val;
4857 case 0xc3: /* movnti */
4858 ctxt->dst.bytes = ctxt->op_bytes;
4859 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4860 (u32) ctxt->src.val;
4863 goto cannot_emulate;
4868 if (rc != X86EMUL_CONTINUE)
4874 return EMULATION_FAILED;
4877 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4879 invalidate_registers(ctxt);
4882 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4884 writeback_registers(ctxt);