1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include "kvm_emulate.h"
24 #include <linux/stringify.h>
25 #include <asm/fpu/api.h>
26 #include <asm/debugreg.h>
27 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
198 int (*execute)(struct x86_emulate_ctxt *ctxt);
199 const struct opcode *group;
200 const struct group_dual *gdual;
201 const struct gprefix *gprefix;
202 const struct escape *esc;
203 const struct instr_dual *idual;
204 const struct mode_dual *mdual;
205 void (*fastop)(struct fastop *fake);
207 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
211 struct opcode mod012[8];
212 struct opcode mod3[8];
216 struct opcode pfx_no;
217 struct opcode pfx_66;
218 struct opcode pfx_f2;
219 struct opcode pfx_f3;
224 struct opcode high[64];
228 struct opcode mod012;
233 struct opcode mode32;
234 struct opcode mode64;
237 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
239 enum x86_transfer_type {
241 X86_TRANSFER_CALL_JMP,
243 X86_TRANSFER_TASK_SWITCH,
246 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
248 if (!(ctxt->regs_valid & (1 << nr))) {
249 ctxt->regs_valid |= 1 << nr;
250 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
252 return ctxt->_regs[nr];
255 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
257 ctxt->regs_valid |= 1 << nr;
258 ctxt->regs_dirty |= 1 << nr;
259 return &ctxt->_regs[nr];
262 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
265 return reg_write(ctxt, nr);
268 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
272 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
273 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
276 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
278 ctxt->regs_dirty = 0;
279 ctxt->regs_valid = 0;
283 * These EFLAGS bits are restored from saved value during emulation, and
284 * any changes are written back to the saved value after emulation.
286 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
287 X86_EFLAGS_PF|X86_EFLAGS_CF)
296 * fastop functions have a special calling convention:
301 * flags: rflags (in/out)
302 * ex: rsi (in:fastop pointer, out:zero if exception)
304 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
305 * different operand sizes can be reached by calculation, rather than a jump
306 * table (which would be bigger than the code).
308 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
310 #define __FOP_FUNC(name) \
311 ".align " __stringify(FASTOP_SIZE) " \n\t" \
312 ".type " name ", @function \n\t" \
315 #define FOP_FUNC(name) \
318 #define __FOP_RET(name) \
320 ".size " name ", .-" name "\n\t"
322 #define FOP_RET(name) \
325 #define FOP_START(op) \
326 extern void em_##op(struct fastop *fake); \
327 asm(".pushsection .text, \"ax\" \n\t" \
328 ".global em_" #op " \n\t" \
329 ".align " __stringify(FASTOP_SIZE) " \n\t" \
335 #define __FOPNOP(name) \
340 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
342 #define FOP1E(op, dst) \
343 __FOP_FUNC(#op "_" #dst) \
344 "10: " #op " %" #dst " \n\t" \
345 __FOP_RET(#op "_" #dst)
347 #define FOP1EEX(op, dst) \
348 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
350 #define FASTOP1(op) \
355 ON64(FOP1E(op##q, rax)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m) */
359 #define FASTOP1SRC2(op, name) \
364 ON64(FOP1E(op, rcx)) \
367 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
368 #define FASTOP1SRC2EX(op, name) \
373 ON64(FOP1EEX(op, rcx)) \
376 #define FOP2E(op, dst, src) \
377 __FOP_FUNC(#op "_" #dst "_" #src) \
378 #op " %" #src ", %" #dst " \n\t" \
379 __FOP_RET(#op "_" #dst "_" #src)
381 #define FASTOP2(op) \
383 FOP2E(op##b, al, dl) \
384 FOP2E(op##w, ax, dx) \
385 FOP2E(op##l, eax, edx) \
386 ON64(FOP2E(op##q, rax, rdx)) \
389 /* 2 operand, word only */
390 #define FASTOP2W(op) \
393 FOP2E(op##w, ax, dx) \
394 FOP2E(op##l, eax, edx) \
395 ON64(FOP2E(op##q, rax, rdx)) \
398 /* 2 operand, src is CL */
399 #define FASTOP2CL(op) \
401 FOP2E(op##b, al, cl) \
402 FOP2E(op##w, ax, cl) \
403 FOP2E(op##l, eax, cl) \
404 ON64(FOP2E(op##q, rax, cl)) \
407 /* 2 operand, src and dest are reversed */
408 #define FASTOP2R(op, name) \
410 FOP2E(op##b, dl, al) \
411 FOP2E(op##w, dx, ax) \
412 FOP2E(op##l, edx, eax) \
413 ON64(FOP2E(op##q, rdx, rax)) \
416 #define FOP3E(op, dst, src, src2) \
417 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
418 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
419 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
421 /* 3-operand, word-only, src2=cl */
422 #define FASTOP3WCL(op) \
425 FOP3E(op##w, ax, dx, cl) \
426 FOP3E(op##l, eax, edx, cl) \
427 ON64(FOP3E(op##q, rax, rdx, cl)) \
430 /* Special case for SETcc - 1 instruction per cc */
431 #define FOP_SETCC(op) \
433 ".type " #op ", @function \n\t" \
438 asm(".pushsection .fixup, \"ax\"\n"
439 ".global kvm_fastop_exception \n"
440 "kvm_fastop_exception: xor %esi, %esi; ret\n"
464 "pushf; sbb %al, %al; popf \n\t"
469 * XXX: inoutclob user must know where the argument is being expanded.
470 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
472 #define asm_safe(insn, inoutclob...) \
476 asm volatile("1:" insn "\n" \
478 ".pushsection .fixup, \"ax\"\n" \
479 "3: movl $1, %[_fault]\n" \
482 _ASM_EXTABLE(1b, 3b) \
483 : [_fault] "+qm"(_fault) inoutclob ); \
485 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
488 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
489 enum x86_intercept intercept,
490 enum x86_intercept_stage stage)
492 struct x86_instruction_info info = {
493 .intercept = intercept,
494 .rep_prefix = ctxt->rep_prefix,
495 .modrm_mod = ctxt->modrm_mod,
496 .modrm_reg = ctxt->modrm_reg,
497 .modrm_rm = ctxt->modrm_rm,
498 .src_val = ctxt->src.val64,
499 .dst_val = ctxt->dst.val64,
500 .src_bytes = ctxt->src.bytes,
501 .dst_bytes = ctxt->dst.bytes,
502 .ad_bytes = ctxt->ad_bytes,
503 .next_rip = ctxt->eip,
506 return ctxt->ops->intercept(ctxt, &info, stage);
509 static void assign_masked(ulong *dest, ulong src, ulong mask)
511 *dest = (*dest & ~mask) | (src & mask);
514 static void assign_register(unsigned long *reg, u64 val, int bytes)
516 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
519 *(u8 *)reg = (u8)val;
522 *(u16 *)reg = (u16)val;
526 break; /* 64b: zero-extend */
533 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
535 return (1UL << (ctxt->ad_bytes << 3)) - 1;
538 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
541 struct desc_struct ss;
543 if (ctxt->mode == X86EMUL_MODE_PROT64)
545 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
546 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
549 static int stack_size(struct x86_emulate_ctxt *ctxt)
551 return (__fls(stack_mask(ctxt)) + 1) >> 3;
554 /* Access/update address held in a register, based on addressing mode. */
555 static inline unsigned long
556 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
558 if (ctxt->ad_bytes == sizeof(unsigned long))
561 return reg & ad_mask(ctxt);
564 static inline unsigned long
565 register_address(struct x86_emulate_ctxt *ctxt, int reg)
567 return address_mask(ctxt, reg_read(ctxt, reg));
570 static void masked_increment(ulong *reg, ulong mask, int inc)
572 assign_masked(reg, *reg + inc, mask);
576 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
578 ulong *preg = reg_rmw(ctxt, reg);
580 assign_register(preg, *preg + inc, ctxt->ad_bytes);
583 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
585 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
588 static u32 desc_limit_scaled(struct desc_struct *desc)
590 u32 limit = get_desc_limit(desc);
592 return desc->g ? (limit << 12) | 0xfff : limit;
595 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
597 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
600 return ctxt->ops->get_cached_segment_base(ctxt, seg);
603 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
604 u32 error, bool valid)
607 ctxt->exception.vector = vec;
608 ctxt->exception.error_code = error;
609 ctxt->exception.error_code_valid = valid;
610 return X86EMUL_PROPAGATE_FAULT;
613 static int emulate_db(struct x86_emulate_ctxt *ctxt)
615 return emulate_exception(ctxt, DB_VECTOR, 0, false);
618 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
620 return emulate_exception(ctxt, GP_VECTOR, err, true);
623 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
625 return emulate_exception(ctxt, SS_VECTOR, err, true);
628 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
630 return emulate_exception(ctxt, UD_VECTOR, 0, false);
633 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
635 return emulate_exception(ctxt, TS_VECTOR, err, true);
638 static int emulate_de(struct x86_emulate_ctxt *ctxt)
640 return emulate_exception(ctxt, DE_VECTOR, 0, false);
643 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
645 return emulate_exception(ctxt, NM_VECTOR, 0, false);
648 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
651 struct desc_struct desc;
653 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
657 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
662 struct desc_struct desc;
664 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
665 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
668 static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
670 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
673 static inline bool emul_is_noncanonical_address(u64 la,
674 struct x86_emulate_ctxt *ctxt)
676 return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
680 * x86 defines three classes of vector instructions: explicitly
681 * aligned, explicitly unaligned, and the rest, which change behaviour
682 * depending on whether they're AVX encoded or not.
684 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
685 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
686 * 512 bytes of data must be aligned to a 16 byte boundary.
688 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
690 u64 alignment = ctxt->d & AlignMask;
692 if (likely(size < 16))
707 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
708 struct segmented_address addr,
709 unsigned *max_size, unsigned size,
710 bool write, bool fetch,
711 enum x86emul_mode mode, ulong *linear)
713 struct desc_struct desc;
720 la = seg_base(ctxt, addr.seg) + addr.ea;
723 case X86EMUL_MODE_PROT64:
725 va_bits = ctxt_virt_addr_bits(ctxt);
726 if (get_canonical(la, va_bits) != la)
729 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
730 if (size > *max_size)
734 *linear = la = (u32)la;
735 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
739 /* code segment in protected mode or read-only data segment */
740 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
741 || !(desc.type & 2)) && write)
743 /* unreadable code segment */
744 if (!fetch && (desc.type & 8) && !(desc.type & 2))
746 lim = desc_limit_scaled(&desc);
747 if (!(desc.type & 8) && (desc.type & 4)) {
748 /* expand-down segment */
751 lim = desc.d ? 0xffffffff : 0xffff;
755 if (lim == 0xffffffff)
758 *max_size = (u64)lim + 1 - addr.ea;
759 if (size > *max_size)
764 if (la & (insn_alignment(ctxt, size) - 1))
765 return emulate_gp(ctxt, 0);
766 return X86EMUL_CONTINUE;
768 if (addr.seg == VCPU_SREG_SS)
769 return emulate_ss(ctxt, 0);
771 return emulate_gp(ctxt, 0);
774 static int linearize(struct x86_emulate_ctxt *ctxt,
775 struct segmented_address addr,
776 unsigned size, bool write,
780 return __linearize(ctxt, addr, &max_size, size, write, false,
784 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
785 enum x86emul_mode mode)
790 struct segmented_address addr = { .seg = VCPU_SREG_CS,
793 if (ctxt->op_bytes != sizeof(unsigned long))
794 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
795 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
796 if (rc == X86EMUL_CONTINUE)
797 ctxt->_eip = addr.ea;
801 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
803 return assign_eip(ctxt, dst, ctxt->mode);
806 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
807 const struct desc_struct *cs_desc)
809 enum x86emul_mode mode = ctxt->mode;
813 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
817 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
819 mode = X86EMUL_MODE_PROT64;
821 mode = X86EMUL_MODE_PROT32; /* temporary value */
824 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
825 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
826 rc = assign_eip(ctxt, dst, mode);
827 if (rc == X86EMUL_CONTINUE)
832 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
834 return assign_eip_near(ctxt, ctxt->_eip + rel);
837 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
838 void *data, unsigned size)
840 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
843 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
844 ulong linear, void *data,
847 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
850 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
851 struct segmented_address addr,
858 rc = linearize(ctxt, addr, size, false, &linear);
859 if (rc != X86EMUL_CONTINUE)
861 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
864 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
865 struct segmented_address addr,
872 rc = linearize(ctxt, addr, size, true, &linear);
873 if (rc != X86EMUL_CONTINUE)
875 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
879 * Prefetch the remaining bytes of the instruction without crossing page
880 * boundary if they are not in fetch_cache yet.
882 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
885 unsigned size, max_size;
886 unsigned long linear;
887 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
888 struct segmented_address addr = { .seg = VCPU_SREG_CS,
889 .ea = ctxt->eip + cur_size };
892 * We do not know exactly how many bytes will be needed, and
893 * __linearize is expensive, so fetch as much as possible. We
894 * just have to avoid going beyond the 15 byte limit, the end
895 * of the segment, or the end of the page.
897 * __linearize is called with size 0 so that it does not do any
898 * boundary check itself. Instead, we use max_size to check
901 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
903 if (unlikely(rc != X86EMUL_CONTINUE))
906 size = min_t(unsigned, 15UL ^ cur_size, max_size);
907 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
910 * One instruction can only straddle two pages,
911 * and one has been loaded at the beginning of
912 * x86_decode_insn. So, if not enough bytes
913 * still, we must have hit the 15-byte boundary.
915 if (unlikely(size < op_size))
916 return emulate_gp(ctxt, 0);
918 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
919 size, &ctxt->exception);
920 if (unlikely(rc != X86EMUL_CONTINUE))
922 ctxt->fetch.end += size;
923 return X86EMUL_CONTINUE;
926 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
929 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
931 if (unlikely(done_size < size))
932 return __do_insn_fetch_bytes(ctxt, size - done_size);
934 return X86EMUL_CONTINUE;
937 /* Fetch next part of the instruction being emulated. */
938 #define insn_fetch(_type, _ctxt) \
941 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
942 if (rc != X86EMUL_CONTINUE) \
944 ctxt->_eip += sizeof(_type); \
945 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
946 ctxt->fetch.ptr += sizeof(_type); \
950 #define insn_fetch_arr(_arr, _size, _ctxt) \
952 rc = do_insn_fetch_bytes(_ctxt, _size); \
953 if (rc != X86EMUL_CONTINUE) \
955 ctxt->_eip += (_size); \
956 memcpy(_arr, ctxt->fetch.ptr, _size); \
957 ctxt->fetch.ptr += (_size); \
961 * Given the 'reg' portion of a ModRM byte, and a register block, return a
962 * pointer into the block that addresses the relevant register.
963 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
965 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
969 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
971 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
972 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
974 p = reg_rmw(ctxt, modrm_reg);
978 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
979 struct segmented_address addr,
980 u16 *size, unsigned long *address, int op_bytes)
987 rc = segmented_read_std(ctxt, addr, size, 2);
988 if (rc != X86EMUL_CONTINUE)
991 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1005 FASTOP1SRC2(mul, mul_ex);
1006 FASTOP1SRC2(imul, imul_ex);
1007 FASTOP1SRC2EX(div, div_ex);
1008 FASTOP1SRC2EX(idiv, idiv_ex);
1037 FASTOP2R(cmp, cmp_r);
1039 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1041 /* If src is zero, do not writeback, but update flags */
1042 if (ctxt->src.val == 0)
1043 ctxt->dst.type = OP_NONE;
1044 return fastop(ctxt, em_bsf);
1047 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1049 /* If src is zero, do not writeback, but update flags */
1050 if (ctxt->src.val == 0)
1051 ctxt->dst.type = OP_NONE;
1052 return fastop(ctxt, em_bsr);
1055 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1058 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1060 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1061 asm("push %[flags]; popf; " CALL_NOSPEC
1062 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1066 static void fetch_register_operand(struct operand *op)
1068 switch (op->bytes) {
1070 op->val = *(u8 *)op->addr.reg;
1073 op->val = *(u16 *)op->addr.reg;
1076 op->val = *(u32 *)op->addr.reg;
1079 op->val = *(u64 *)op->addr.reg;
1084 static void emulator_get_fpu(void)
1088 fpregs_assert_state_consistent();
1089 if (test_thread_flag(TIF_NEED_FPU_LOAD))
1090 switch_fpu_return();
1093 static void emulator_put_fpu(void)
1098 static void read_sse_reg(sse128_t *data, int reg)
1102 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1103 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1104 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1105 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1106 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1107 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1108 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1109 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1110 #ifdef CONFIG_X86_64
1111 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1112 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1113 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1114 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1115 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1116 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1117 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1118 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1125 static void write_sse_reg(sse128_t *data, int reg)
1129 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1130 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1131 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1132 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1133 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1134 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1135 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1136 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1137 #ifdef CONFIG_X86_64
1138 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1139 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1140 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1141 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1142 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1143 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1144 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1145 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1152 static void read_mmx_reg(u64 *data, int reg)
1156 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1157 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1158 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1159 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1160 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1161 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1162 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1163 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1169 static void write_mmx_reg(u64 *data, int reg)
1173 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1174 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1175 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1176 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1177 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1178 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1179 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1180 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1186 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1188 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1189 return emulate_nm(ctxt);
1192 asm volatile("fninit");
1194 return X86EMUL_CONTINUE;
1197 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1201 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1202 return emulate_nm(ctxt);
1205 asm volatile("fnstcw %0": "+m"(fcw));
1208 ctxt->dst.val = fcw;
1210 return X86EMUL_CONTINUE;
1213 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1217 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1218 return emulate_nm(ctxt);
1221 asm volatile("fnstsw %0": "+m"(fsw));
1224 ctxt->dst.val = fsw;
1226 return X86EMUL_CONTINUE;
1229 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1232 unsigned reg = ctxt->modrm_reg;
1234 if (!(ctxt->d & ModRM))
1235 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1237 if (ctxt->d & Sse) {
1241 read_sse_reg(&op->vec_val, reg);
1244 if (ctxt->d & Mmx) {
1253 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1254 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1256 fetch_register_operand(op);
1257 op->orig_val = op->val;
1260 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1262 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1263 ctxt->modrm_seg = VCPU_SREG_SS;
1266 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1270 int index_reg, base_reg, scale;
1271 int rc = X86EMUL_CONTINUE;
1274 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1275 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1276 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1278 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1279 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1280 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1281 ctxt->modrm_seg = VCPU_SREG_DS;
1283 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1285 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1286 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1288 if (ctxt->d & Sse) {
1291 op->addr.xmm = ctxt->modrm_rm;
1292 read_sse_reg(&op->vec_val, ctxt->modrm_rm);
1295 if (ctxt->d & Mmx) {
1298 op->addr.mm = ctxt->modrm_rm & 7;
1301 fetch_register_operand(op);
1307 if (ctxt->ad_bytes == 2) {
1308 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1309 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1310 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1311 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1313 /* 16-bit ModR/M decode. */
1314 switch (ctxt->modrm_mod) {
1316 if (ctxt->modrm_rm == 6)
1317 modrm_ea += insn_fetch(u16, ctxt);
1320 modrm_ea += insn_fetch(s8, ctxt);
1323 modrm_ea += insn_fetch(u16, ctxt);
1326 switch (ctxt->modrm_rm) {
1328 modrm_ea += bx + si;
1331 modrm_ea += bx + di;
1334 modrm_ea += bp + si;
1337 modrm_ea += bp + di;
1346 if (ctxt->modrm_mod != 0)
1353 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1354 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1355 ctxt->modrm_seg = VCPU_SREG_SS;
1356 modrm_ea = (u16)modrm_ea;
1358 /* 32/64-bit ModR/M decode. */
1359 if ((ctxt->modrm_rm & 7) == 4) {
1360 sib = insn_fetch(u8, ctxt);
1361 index_reg |= (sib >> 3) & 7;
1362 base_reg |= sib & 7;
1365 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1366 modrm_ea += insn_fetch(s32, ctxt);
1368 modrm_ea += reg_read(ctxt, base_reg);
1369 adjust_modrm_seg(ctxt, base_reg);
1370 /* Increment ESP on POP [ESP] */
1371 if ((ctxt->d & IncSP) &&
1372 base_reg == VCPU_REGS_RSP)
1373 modrm_ea += ctxt->op_bytes;
1376 modrm_ea += reg_read(ctxt, index_reg) << scale;
1377 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1378 modrm_ea += insn_fetch(s32, ctxt);
1379 if (ctxt->mode == X86EMUL_MODE_PROT64)
1380 ctxt->rip_relative = 1;
1382 base_reg = ctxt->modrm_rm;
1383 modrm_ea += reg_read(ctxt, base_reg);
1384 adjust_modrm_seg(ctxt, base_reg);
1386 switch (ctxt->modrm_mod) {
1388 modrm_ea += insn_fetch(s8, ctxt);
1391 modrm_ea += insn_fetch(s32, ctxt);
1395 op->addr.mem.ea = modrm_ea;
1396 if (ctxt->ad_bytes != 8)
1397 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1403 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1406 int rc = X86EMUL_CONTINUE;
1409 switch (ctxt->ad_bytes) {
1411 op->addr.mem.ea = insn_fetch(u16, ctxt);
1414 op->addr.mem.ea = insn_fetch(u32, ctxt);
1417 op->addr.mem.ea = insn_fetch(u64, ctxt);
1424 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1428 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1429 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1431 if (ctxt->src.bytes == 2)
1432 sv = (s16)ctxt->src.val & (s16)mask;
1433 else if (ctxt->src.bytes == 4)
1434 sv = (s32)ctxt->src.val & (s32)mask;
1436 sv = (s64)ctxt->src.val & (s64)mask;
1438 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1439 ctxt->dst.addr.mem.ea + (sv >> 3));
1442 /* only subword offset */
1443 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1446 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1447 unsigned long addr, void *dest, unsigned size)
1450 struct read_cache *mc = &ctxt->mem_read;
1452 if (mc->pos < mc->end)
1455 WARN_ON((mc->end + size) >= sizeof(mc->data));
1457 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1459 if (rc != X86EMUL_CONTINUE)
1465 memcpy(dest, mc->data + mc->pos, size);
1467 return X86EMUL_CONTINUE;
1470 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1471 struct segmented_address addr,
1478 rc = linearize(ctxt, addr, size, false, &linear);
1479 if (rc != X86EMUL_CONTINUE)
1481 return read_emulated(ctxt, linear, data, size);
1484 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1485 struct segmented_address addr,
1492 rc = linearize(ctxt, addr, size, true, &linear);
1493 if (rc != X86EMUL_CONTINUE)
1495 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1499 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1500 struct segmented_address addr,
1501 const void *orig_data, const void *data,
1507 rc = linearize(ctxt, addr, size, true, &linear);
1508 if (rc != X86EMUL_CONTINUE)
1510 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1511 size, &ctxt->exception);
1514 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1515 unsigned int size, unsigned short port,
1518 struct read_cache *rc = &ctxt->io_read;
1520 if (rc->pos == rc->end) { /* refill pio read ahead */
1521 unsigned int in_page, n;
1522 unsigned int count = ctxt->rep_prefix ?
1523 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1524 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1525 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1526 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1527 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1530 rc->pos = rc->end = 0;
1531 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1536 if (ctxt->rep_prefix && (ctxt->d & String) &&
1537 !(ctxt->eflags & X86_EFLAGS_DF)) {
1538 ctxt->dst.data = rc->data + rc->pos;
1539 ctxt->dst.type = OP_MEM_STR;
1540 ctxt->dst.count = (rc->end - rc->pos) / size;
1543 memcpy(dest, rc->data + rc->pos, size);
1549 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1550 u16 index, struct desc_struct *desc)
1555 ctxt->ops->get_idt(ctxt, &dt);
1557 if (dt.size < index * 8 + 7)
1558 return emulate_gp(ctxt, index << 3 | 0x2);
1560 addr = dt.address + index * 8;
1561 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1564 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1565 u16 selector, struct desc_ptr *dt)
1567 const struct x86_emulate_ops *ops = ctxt->ops;
1570 if (selector & 1 << 2) {
1571 struct desc_struct desc;
1574 memset(dt, 0, sizeof(*dt));
1575 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1579 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1580 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1582 ops->get_gdt(ctxt, dt);
1585 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1586 u16 selector, ulong *desc_addr_p)
1589 u16 index = selector >> 3;
1592 get_descriptor_table_ptr(ctxt, selector, &dt);
1594 if (dt.size < index * 8 + 7)
1595 return emulate_gp(ctxt, selector & 0xfffc);
1597 addr = dt.address + index * 8;
1599 #ifdef CONFIG_X86_64
1600 if (addr >> 32 != 0) {
1603 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1604 if (!(efer & EFER_LMA))
1609 *desc_addr_p = addr;
1610 return X86EMUL_CONTINUE;
1613 /* allowed just for 8 bytes segments */
1614 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1615 u16 selector, struct desc_struct *desc,
1620 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1621 if (rc != X86EMUL_CONTINUE)
1624 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1627 /* allowed just for 8 bytes segments */
1628 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1629 u16 selector, struct desc_struct *desc)
1634 rc = get_descriptor_ptr(ctxt, selector, &addr);
1635 if (rc != X86EMUL_CONTINUE)
1638 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1641 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1642 u16 selector, int seg, u8 cpl,
1643 enum x86_transfer_type transfer,
1644 struct desc_struct *desc)
1646 struct desc_struct seg_desc, old_desc;
1648 unsigned err_vec = GP_VECTOR;
1650 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1656 memset(&seg_desc, 0, sizeof(seg_desc));
1658 if (ctxt->mode == X86EMUL_MODE_REAL) {
1659 /* set real mode segment descriptor (keep limit etc. for
1661 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1662 set_desc_base(&seg_desc, selector << 4);
1664 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1665 /* VM86 needs a clean new segment descriptor */
1666 set_desc_base(&seg_desc, selector << 4);
1667 set_desc_limit(&seg_desc, 0xffff);
1677 /* TR should be in GDT only */
1678 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1681 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1682 if (null_selector) {
1683 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1686 if (seg == VCPU_SREG_SS) {
1687 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1691 * ctxt->ops->set_segment expects the CPL to be in
1692 * SS.DPL, so fake an expand-up 32-bit data segment.
1702 /* Skip all following checks */
1706 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1707 if (ret != X86EMUL_CONTINUE)
1710 err_code = selector & 0xfffc;
1711 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1714 /* can't load system descriptor into segment selector */
1715 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1716 if (transfer == X86_TRANSFER_CALL_JMP)
1717 return X86EMUL_UNHANDLEABLE;
1722 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1731 * segment is not a writable data segment or segment
1732 * selector's RPL != CPL or segment selector's RPL != CPL
1734 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1738 if (!(seg_desc.type & 8))
1741 if (seg_desc.type & 4) {
1747 if (rpl > cpl || dpl != cpl)
1750 /* in long-mode d/b must be clear if l is set */
1751 if (seg_desc.d && seg_desc.l) {
1754 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1755 if (efer & EFER_LMA)
1759 /* CS(RPL) <- CPL */
1760 selector = (selector & 0xfffc) | cpl;
1763 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1765 old_desc = seg_desc;
1766 seg_desc.type |= 2; /* busy */
1767 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1768 sizeof(seg_desc), &ctxt->exception);
1769 if (ret != X86EMUL_CONTINUE)
1772 case VCPU_SREG_LDTR:
1773 if (seg_desc.s || seg_desc.type != 2)
1776 default: /* DS, ES, FS, or GS */
1778 * segment is not a data or readable code segment or
1779 * ((segment is a data or nonconforming code segment)
1780 * and (both RPL and CPL > DPL))
1782 if ((seg_desc.type & 0xa) == 0x8 ||
1783 (((seg_desc.type & 0xc) != 0xc) &&
1784 (rpl > dpl && cpl > dpl)))
1790 /* mark segment as accessed */
1791 if (!(seg_desc.type & 1)) {
1793 ret = write_segment_descriptor(ctxt, selector,
1795 if (ret != X86EMUL_CONTINUE)
1798 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1799 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1800 if (ret != X86EMUL_CONTINUE)
1802 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1803 ((u64)base3 << 32), ctxt))
1804 return emulate_gp(ctxt, 0);
1807 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1810 return X86EMUL_CONTINUE;
1812 return emulate_exception(ctxt, err_vec, err_code, true);
1815 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1816 u16 selector, int seg)
1818 u8 cpl = ctxt->ops->cpl(ctxt);
1821 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1822 * they can load it at CPL<3 (Intel's manual says only LSS can,
1825 * However, the Intel manual says that putting IST=1/DPL=3 in
1826 * an interrupt gate will result in SS=3 (the AMD manual instead
1827 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1828 * and only forbid it here.
1830 if (seg == VCPU_SREG_SS && selector == 3 &&
1831 ctxt->mode == X86EMUL_MODE_PROT64)
1832 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1834 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1835 X86_TRANSFER_NONE, NULL);
1838 static void write_register_operand(struct operand *op)
1840 return assign_register(op->addr.reg, op->val, op->bytes);
1843 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1847 write_register_operand(op);
1850 if (ctxt->lock_prefix)
1851 return segmented_cmpxchg(ctxt,
1857 return segmented_write(ctxt,
1863 return segmented_write(ctxt,
1866 op->bytes * op->count);
1869 write_sse_reg(&op->vec_val, op->addr.xmm);
1872 write_mmx_reg(&op->mm_val, op->addr.mm);
1880 return X86EMUL_CONTINUE;
1883 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1885 struct segmented_address addr;
1887 rsp_increment(ctxt, -bytes);
1888 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1889 addr.seg = VCPU_SREG_SS;
1891 return segmented_write(ctxt, addr, data, bytes);
1894 static int em_push(struct x86_emulate_ctxt *ctxt)
1896 /* Disable writeback. */
1897 ctxt->dst.type = OP_NONE;
1898 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1901 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1902 void *dest, int len)
1905 struct segmented_address addr;
1907 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1908 addr.seg = VCPU_SREG_SS;
1909 rc = segmented_read(ctxt, addr, dest, len);
1910 if (rc != X86EMUL_CONTINUE)
1913 rsp_increment(ctxt, len);
1917 static int em_pop(struct x86_emulate_ctxt *ctxt)
1919 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1922 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1923 void *dest, int len)
1926 unsigned long val, change_mask;
1927 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1928 int cpl = ctxt->ops->cpl(ctxt);
1930 rc = emulate_pop(ctxt, &val, len);
1931 if (rc != X86EMUL_CONTINUE)
1934 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1935 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1936 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1937 X86_EFLAGS_AC | X86_EFLAGS_ID;
1939 switch(ctxt->mode) {
1940 case X86EMUL_MODE_PROT64:
1941 case X86EMUL_MODE_PROT32:
1942 case X86EMUL_MODE_PROT16:
1944 change_mask |= X86_EFLAGS_IOPL;
1946 change_mask |= X86_EFLAGS_IF;
1948 case X86EMUL_MODE_VM86:
1950 return emulate_gp(ctxt, 0);
1951 change_mask |= X86_EFLAGS_IF;
1953 default: /* real mode */
1954 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1958 *(unsigned long *)dest =
1959 (ctxt->eflags & ~change_mask) | (val & change_mask);
1964 static int em_popf(struct x86_emulate_ctxt *ctxt)
1966 ctxt->dst.type = OP_REG;
1967 ctxt->dst.addr.reg = &ctxt->eflags;
1968 ctxt->dst.bytes = ctxt->op_bytes;
1969 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1972 static int em_enter(struct x86_emulate_ctxt *ctxt)
1975 unsigned frame_size = ctxt->src.val;
1976 unsigned nesting_level = ctxt->src2.val & 31;
1980 return X86EMUL_UNHANDLEABLE;
1982 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1983 rc = push(ctxt, &rbp, stack_size(ctxt));
1984 if (rc != X86EMUL_CONTINUE)
1986 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1988 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1989 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1991 return X86EMUL_CONTINUE;
1994 static int em_leave(struct x86_emulate_ctxt *ctxt)
1996 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1998 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
2001 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
2003 int seg = ctxt->src2.val;
2005 ctxt->src.val = get_segment_selector(ctxt, seg);
2006 if (ctxt->op_bytes == 4) {
2007 rsp_increment(ctxt, -2);
2011 return em_push(ctxt);
2014 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
2016 int seg = ctxt->src2.val;
2017 unsigned long selector;
2020 rc = emulate_pop(ctxt, &selector, 2);
2021 if (rc != X86EMUL_CONTINUE)
2024 if (ctxt->modrm_reg == VCPU_SREG_SS)
2025 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2026 if (ctxt->op_bytes > 2)
2027 rsp_increment(ctxt, ctxt->op_bytes - 2);
2029 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2033 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2035 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2036 int rc = X86EMUL_CONTINUE;
2037 int reg = VCPU_REGS_RAX;
2039 while (reg <= VCPU_REGS_RDI) {
2040 (reg == VCPU_REGS_RSP) ?
2041 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2044 if (rc != X86EMUL_CONTINUE)
2053 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2055 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2056 return em_push(ctxt);
2059 static int em_popa(struct x86_emulate_ctxt *ctxt)
2061 int rc = X86EMUL_CONTINUE;
2062 int reg = VCPU_REGS_RDI;
2065 while (reg >= VCPU_REGS_RAX) {
2066 if (reg == VCPU_REGS_RSP) {
2067 rsp_increment(ctxt, ctxt->op_bytes);
2071 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2072 if (rc != X86EMUL_CONTINUE)
2074 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2080 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2082 const struct x86_emulate_ops *ops = ctxt->ops;
2089 /* TODO: Add limit checks */
2090 ctxt->src.val = ctxt->eflags;
2092 if (rc != X86EMUL_CONTINUE)
2095 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2097 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2099 if (rc != X86EMUL_CONTINUE)
2102 ctxt->src.val = ctxt->_eip;
2104 if (rc != X86EMUL_CONTINUE)
2107 ops->get_idt(ctxt, &dt);
2109 eip_addr = dt.address + (irq << 2);
2110 cs_addr = dt.address + (irq << 2) + 2;
2112 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2113 if (rc != X86EMUL_CONTINUE)
2116 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2117 if (rc != X86EMUL_CONTINUE)
2120 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2121 if (rc != X86EMUL_CONTINUE)
2129 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2133 invalidate_registers(ctxt);
2134 rc = __emulate_int_real(ctxt, irq);
2135 if (rc == X86EMUL_CONTINUE)
2136 writeback_registers(ctxt);
2140 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2142 switch(ctxt->mode) {
2143 case X86EMUL_MODE_REAL:
2144 return __emulate_int_real(ctxt, irq);
2145 case X86EMUL_MODE_VM86:
2146 case X86EMUL_MODE_PROT16:
2147 case X86EMUL_MODE_PROT32:
2148 case X86EMUL_MODE_PROT64:
2150 /* Protected mode interrupts unimplemented yet */
2151 return X86EMUL_UNHANDLEABLE;
2155 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2157 int rc = X86EMUL_CONTINUE;
2158 unsigned long temp_eip = 0;
2159 unsigned long temp_eflags = 0;
2160 unsigned long cs = 0;
2161 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2162 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2163 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2164 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2165 X86_EFLAGS_AC | X86_EFLAGS_ID |
2167 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2170 /* TODO: Add stack limit check */
2172 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2174 if (rc != X86EMUL_CONTINUE)
2177 if (temp_eip & ~0xffff)
2178 return emulate_gp(ctxt, 0);
2180 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2182 if (rc != X86EMUL_CONTINUE)
2185 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2187 if (rc != X86EMUL_CONTINUE)
2190 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2192 if (rc != X86EMUL_CONTINUE)
2195 ctxt->_eip = temp_eip;
2197 if (ctxt->op_bytes == 4)
2198 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2199 else if (ctxt->op_bytes == 2) {
2200 ctxt->eflags &= ~0xffff;
2201 ctxt->eflags |= temp_eflags;
2204 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2205 ctxt->eflags |= X86_EFLAGS_FIXED;
2206 ctxt->ops->set_nmi_mask(ctxt, false);
2211 static int em_iret(struct x86_emulate_ctxt *ctxt)
2213 switch(ctxt->mode) {
2214 case X86EMUL_MODE_REAL:
2215 return emulate_iret_real(ctxt);
2216 case X86EMUL_MODE_VM86:
2217 case X86EMUL_MODE_PROT16:
2218 case X86EMUL_MODE_PROT32:
2219 case X86EMUL_MODE_PROT64:
2221 /* iret from protected mode unimplemented yet */
2222 return X86EMUL_UNHANDLEABLE;
2226 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2230 struct desc_struct new_desc;
2231 u8 cpl = ctxt->ops->cpl(ctxt);
2233 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2235 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2236 X86_TRANSFER_CALL_JMP,
2238 if (rc != X86EMUL_CONTINUE)
2241 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2242 /* Error handling is not implemented. */
2243 if (rc != X86EMUL_CONTINUE)
2244 return X86EMUL_UNHANDLEABLE;
2249 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2251 return assign_eip_near(ctxt, ctxt->src.val);
2254 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2259 old_eip = ctxt->_eip;
2260 rc = assign_eip_near(ctxt, ctxt->src.val);
2261 if (rc != X86EMUL_CONTINUE)
2263 ctxt->src.val = old_eip;
2268 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2270 u64 old = ctxt->dst.orig_val64;
2272 if (ctxt->dst.bytes == 16)
2273 return X86EMUL_UNHANDLEABLE;
2275 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2276 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2277 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2278 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2279 ctxt->eflags &= ~X86_EFLAGS_ZF;
2281 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2282 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2284 ctxt->eflags |= X86_EFLAGS_ZF;
2286 return X86EMUL_CONTINUE;
2289 static int em_ret(struct x86_emulate_ctxt *ctxt)
2294 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2295 if (rc != X86EMUL_CONTINUE)
2298 return assign_eip_near(ctxt, eip);
2301 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2304 unsigned long eip, cs;
2305 int cpl = ctxt->ops->cpl(ctxt);
2306 struct desc_struct new_desc;
2308 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2309 if (rc != X86EMUL_CONTINUE)
2311 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2312 if (rc != X86EMUL_CONTINUE)
2314 /* Outer-privilege level return is not implemented */
2315 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2316 return X86EMUL_UNHANDLEABLE;
2317 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2320 if (rc != X86EMUL_CONTINUE)
2322 rc = assign_eip_far(ctxt, eip, &new_desc);
2323 /* Error handling is not implemented. */
2324 if (rc != X86EMUL_CONTINUE)
2325 return X86EMUL_UNHANDLEABLE;
2330 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2334 rc = em_ret_far(ctxt);
2335 if (rc != X86EMUL_CONTINUE)
2337 rsp_increment(ctxt, ctxt->src.val);
2338 return X86EMUL_CONTINUE;
2341 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2343 /* Save real source value, then compare EAX against destination. */
2344 ctxt->dst.orig_val = ctxt->dst.val;
2345 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2346 ctxt->src.orig_val = ctxt->src.val;
2347 ctxt->src.val = ctxt->dst.orig_val;
2348 fastop(ctxt, em_cmp);
2350 if (ctxt->eflags & X86_EFLAGS_ZF) {
2351 /* Success: write back to memory; no update of EAX */
2352 ctxt->src.type = OP_NONE;
2353 ctxt->dst.val = ctxt->src.orig_val;
2355 /* Failure: write the value we saw to EAX. */
2356 ctxt->src.type = OP_REG;
2357 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2358 ctxt->src.val = ctxt->dst.orig_val;
2359 /* Create write-cycle to dest by writing the same value */
2360 ctxt->dst.val = ctxt->dst.orig_val;
2362 return X86EMUL_CONTINUE;
2365 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2367 int seg = ctxt->src2.val;
2371 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2373 rc = load_segment_descriptor(ctxt, sel, seg);
2374 if (rc != X86EMUL_CONTINUE)
2377 ctxt->dst.val = ctxt->src.val;
2381 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2383 #ifdef CONFIG_X86_64
2384 return ctxt->ops->guest_has_long_mode(ctxt);
2390 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2392 desc->g = (flags >> 23) & 1;
2393 desc->d = (flags >> 22) & 1;
2394 desc->l = (flags >> 21) & 1;
2395 desc->avl = (flags >> 20) & 1;
2396 desc->p = (flags >> 15) & 1;
2397 desc->dpl = (flags >> 13) & 3;
2398 desc->s = (flags >> 12) & 1;
2399 desc->type = (flags >> 8) & 15;
2402 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2405 struct desc_struct desc;
2409 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2412 offset = 0x7f84 + n * 12;
2414 offset = 0x7f2c + (n - 3) * 12;
2416 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2417 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2418 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2419 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2420 return X86EMUL_CONTINUE;
2423 #ifdef CONFIG_X86_64
2424 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2427 struct desc_struct desc;
2432 offset = 0x7e00 + n * 16;
2434 selector = GET_SMSTATE(u16, smstate, offset);
2435 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2436 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2437 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2438 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2440 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2441 return X86EMUL_CONTINUE;
2445 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2446 u64 cr0, u64 cr3, u64 cr4)
2451 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2453 if (cr4 & X86_CR4_PCIDE) {
2458 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2460 return X86EMUL_UNHANDLEABLE;
2463 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2464 * Then enable protected mode. However, PCID cannot be enabled
2465 * if EFER.LMA=0, so set it separately.
2467 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2469 return X86EMUL_UNHANDLEABLE;
2471 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2473 return X86EMUL_UNHANDLEABLE;
2475 if (cr4 & X86_CR4_PCIDE) {
2476 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2478 return X86EMUL_UNHANDLEABLE;
2480 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2482 return X86EMUL_UNHANDLEABLE;
2487 return X86EMUL_CONTINUE;
2490 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2491 const char *smstate)
2493 struct desc_struct desc;
2496 u32 val, cr0, cr3, cr4;
2499 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2500 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2501 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2502 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2504 for (i = 0; i < 8; i++)
2505 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2507 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2508 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2509 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2510 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2512 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2513 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2514 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2515 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2516 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2518 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2519 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2520 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2521 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2522 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2524 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2525 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2526 ctxt->ops->set_gdt(ctxt, &dt);
2528 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2529 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2530 ctxt->ops->set_idt(ctxt, &dt);
2532 for (i = 0; i < 6; i++) {
2533 int r = rsm_load_seg_32(ctxt, smstate, i);
2534 if (r != X86EMUL_CONTINUE)
2538 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2540 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2542 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2545 #ifdef CONFIG_X86_64
2546 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2547 const char *smstate)
2549 struct desc_struct desc;
2551 u64 val, cr0, cr3, cr4;
2556 for (i = 0; i < 16; i++)
2557 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2559 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2560 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2562 val = GET_SMSTATE(u32, smstate, 0x7f68);
2563 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2564 val = GET_SMSTATE(u32, smstate, 0x7f60);
2565 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2567 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2568 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2569 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2570 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2571 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2572 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2574 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2575 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2576 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2577 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2578 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2579 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2581 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2582 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2583 ctxt->ops->set_idt(ctxt, &dt);
2585 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2586 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2587 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2588 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2589 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2590 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2592 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2593 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2594 ctxt->ops->set_gdt(ctxt, &dt);
2596 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2597 if (r != X86EMUL_CONTINUE)
2600 for (i = 0; i < 6; i++) {
2601 r = rsm_load_seg_64(ctxt, smstate, i);
2602 if (r != X86EMUL_CONTINUE)
2606 return X86EMUL_CONTINUE;
2610 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2612 unsigned long cr0, cr4, efer;
2617 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2618 return emulate_ud(ctxt);
2620 smbase = ctxt->ops->get_smbase(ctxt);
2622 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2623 if (ret != X86EMUL_CONTINUE)
2624 return X86EMUL_UNHANDLEABLE;
2626 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2627 ctxt->ops->set_nmi_mask(ctxt, false);
2629 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2630 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2633 * Get back to real mode, to prepare a safe state in which to load
2634 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2635 * supports long mode.
2637 if (emulator_has_longmode(ctxt)) {
2638 struct desc_struct cs_desc;
2640 /* Zero CR4.PCIDE before CR0.PG. */
2641 cr4 = ctxt->ops->get_cr(ctxt, 4);
2642 if (cr4 & X86_CR4_PCIDE)
2643 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2645 /* A 32-bit code segment is required to clear EFER.LMA. */
2646 memset(&cs_desc, 0, sizeof(cs_desc));
2648 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2649 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2652 /* For the 64-bit case, this will clear EFER.LMA. */
2653 cr0 = ctxt->ops->get_cr(ctxt, 0);
2654 if (cr0 & X86_CR0_PE)
2655 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2657 if (emulator_has_longmode(ctxt)) {
2658 /* Clear CR4.PAE before clearing EFER.LME. */
2659 cr4 = ctxt->ops->get_cr(ctxt, 4);
2660 if (cr4 & X86_CR4_PAE)
2661 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2663 /* And finally go back to 32-bit mode. */
2665 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2669 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2670 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2673 if (ctxt->ops->pre_leave_smm(ctxt, buf))
2674 return X86EMUL_UNHANDLEABLE;
2676 #ifdef CONFIG_X86_64
2677 if (emulator_has_longmode(ctxt))
2678 ret = rsm_load_state_64(ctxt, buf);
2681 ret = rsm_load_state_32(ctxt, buf);
2683 if (ret != X86EMUL_CONTINUE) {
2684 /* FIXME: should triple fault */
2685 return X86EMUL_UNHANDLEABLE;
2688 ctxt->ops->post_leave_smm(ctxt);
2690 return X86EMUL_CONTINUE;
2694 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2695 struct desc_struct *cs, struct desc_struct *ss)
2697 cs->l = 0; /* will be adjusted later */
2698 set_desc_base(cs, 0); /* flat segment */
2699 cs->g = 1; /* 4kb granularity */
2700 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2701 cs->type = 0x0b; /* Read, Execute, Accessed */
2703 cs->dpl = 0; /* will be adjusted later */
2708 set_desc_base(ss, 0); /* flat segment */
2709 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2710 ss->g = 1; /* 4kb granularity */
2712 ss->type = 0x03; /* Read/Write, Accessed */
2713 ss->d = 1; /* 32bit stack segment */
2720 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2722 u32 eax, ebx, ecx, edx;
2725 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2726 return is_guest_vendor_intel(ebx, ecx, edx);
2729 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2731 const struct x86_emulate_ops *ops = ctxt->ops;
2732 u32 eax, ebx, ecx, edx;
2735 * syscall should always be enabled in longmode - so only become
2736 * vendor specific (cpuid) if other modes are active...
2738 if (ctxt->mode == X86EMUL_MODE_PROT64)
2743 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
2745 * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a
2746 * 64bit guest with a 32bit compat-app running will #UD !! While this
2747 * behaviour can be fixed (by emulating) into AMD response - CPUs of
2748 * AMD can't behave like Intel.
2750 if (is_guest_vendor_intel(ebx, ecx, edx))
2753 if (is_guest_vendor_amd(ebx, ecx, edx) ||
2754 is_guest_vendor_hygon(ebx, ecx, edx))
2758 * default: (not Intel, not AMD, not Hygon), apply Intel's
2764 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2766 const struct x86_emulate_ops *ops = ctxt->ops;
2767 struct desc_struct cs, ss;
2772 /* syscall is not available in real mode */
2773 if (ctxt->mode == X86EMUL_MODE_REAL ||
2774 ctxt->mode == X86EMUL_MODE_VM86)
2775 return emulate_ud(ctxt);
2777 if (!(em_syscall_is_enabled(ctxt)))
2778 return emulate_ud(ctxt);
2780 ops->get_msr(ctxt, MSR_EFER, &efer);
2781 if (!(efer & EFER_SCE))
2782 return emulate_ud(ctxt);
2784 setup_syscalls_segments(ctxt, &cs, &ss);
2785 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2787 cs_sel = (u16)(msr_data & 0xfffc);
2788 ss_sel = (u16)(msr_data + 8);
2790 if (efer & EFER_LMA) {
2794 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2795 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2797 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2798 if (efer & EFER_LMA) {
2799 #ifdef CONFIG_X86_64
2800 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2803 ctxt->mode == X86EMUL_MODE_PROT64 ?
2804 MSR_LSTAR : MSR_CSTAR, &msr_data);
2805 ctxt->_eip = msr_data;
2807 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2808 ctxt->eflags &= ~msr_data;
2809 ctxt->eflags |= X86_EFLAGS_FIXED;
2813 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2814 ctxt->_eip = (u32)msr_data;
2816 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2819 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2820 return X86EMUL_CONTINUE;
2823 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2825 const struct x86_emulate_ops *ops = ctxt->ops;
2826 struct desc_struct cs, ss;
2831 ops->get_msr(ctxt, MSR_EFER, &efer);
2832 /* inject #GP if in real mode */
2833 if (ctxt->mode == X86EMUL_MODE_REAL)
2834 return emulate_gp(ctxt, 0);
2837 * Not recognized on AMD in compat mode (but is recognized in legacy
2840 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2841 && !vendor_intel(ctxt))
2842 return emulate_ud(ctxt);
2844 /* sysenter/sysexit have not been tested in 64bit mode. */
2845 if (ctxt->mode == X86EMUL_MODE_PROT64)
2846 return X86EMUL_UNHANDLEABLE;
2848 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2849 if ((msr_data & 0xfffc) == 0x0)
2850 return emulate_gp(ctxt, 0);
2852 setup_syscalls_segments(ctxt, &cs, &ss);
2853 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2854 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2855 ss_sel = cs_sel + 8;
2856 if (efer & EFER_LMA) {
2861 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2862 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2864 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2865 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2867 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2868 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2871 return X86EMUL_CONTINUE;
2874 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2876 const struct x86_emulate_ops *ops = ctxt->ops;
2877 struct desc_struct cs, ss;
2878 u64 msr_data, rcx, rdx;
2880 u16 cs_sel = 0, ss_sel = 0;
2882 /* inject #GP if in real mode or Virtual 8086 mode */
2883 if (ctxt->mode == X86EMUL_MODE_REAL ||
2884 ctxt->mode == X86EMUL_MODE_VM86)
2885 return emulate_gp(ctxt, 0);
2887 setup_syscalls_segments(ctxt, &cs, &ss);
2889 if ((ctxt->rex_prefix & 0x8) != 0x0)
2890 usermode = X86EMUL_MODE_PROT64;
2892 usermode = X86EMUL_MODE_PROT32;
2894 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2895 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2899 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2901 case X86EMUL_MODE_PROT32:
2902 cs_sel = (u16)(msr_data + 16);
2903 if ((msr_data & 0xfffc) == 0x0)
2904 return emulate_gp(ctxt, 0);
2905 ss_sel = (u16)(msr_data + 24);
2909 case X86EMUL_MODE_PROT64:
2910 cs_sel = (u16)(msr_data + 32);
2911 if (msr_data == 0x0)
2912 return emulate_gp(ctxt, 0);
2913 ss_sel = cs_sel + 8;
2916 if (emul_is_noncanonical_address(rcx, ctxt) ||
2917 emul_is_noncanonical_address(rdx, ctxt))
2918 return emulate_gp(ctxt, 0);
2921 cs_sel |= SEGMENT_RPL_MASK;
2922 ss_sel |= SEGMENT_RPL_MASK;
2924 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2925 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2928 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2930 return X86EMUL_CONTINUE;
2933 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2936 if (ctxt->mode == X86EMUL_MODE_REAL)
2938 if (ctxt->mode == X86EMUL_MODE_VM86)
2940 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2941 return ctxt->ops->cpl(ctxt) > iopl;
2944 #define VMWARE_PORT_VMPORT (0x5658)
2945 #define VMWARE_PORT_VMRPC (0x5659)
2947 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2950 const struct x86_emulate_ops *ops = ctxt->ops;
2951 struct desc_struct tr_seg;
2954 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2955 unsigned mask = (1 << len) - 1;
2959 * VMware allows access to these ports even if denied
2960 * by TSS I/O permission bitmap. Mimic behavior.
2962 if (enable_vmware_backdoor &&
2963 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2966 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2969 if (desc_limit_scaled(&tr_seg) < 103)
2971 base = get_desc_base(&tr_seg);
2972 #ifdef CONFIG_X86_64
2973 base |= ((u64)base3) << 32;
2975 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2976 if (r != X86EMUL_CONTINUE)
2978 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2980 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2981 if (r != X86EMUL_CONTINUE)
2983 if ((perm >> bit_idx) & mask)
2988 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2994 if (emulator_bad_iopl(ctxt))
2995 if (!emulator_io_port_access_allowed(ctxt, port, len))
2998 ctxt->perm_ok = true;
3003 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
3006 * Intel CPUs mask the counter and pointers in quite strange
3007 * manner when ECX is zero due to REP-string optimizations.
3009 #ifdef CONFIG_X86_64
3010 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3013 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3016 case 0xa4: /* movsb */
3017 case 0xa5: /* movsd/w */
3018 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3020 case 0xaa: /* stosb */
3021 case 0xab: /* stosd/w */
3022 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3027 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3028 struct tss_segment_16 *tss)
3030 tss->ip = ctxt->_eip;
3031 tss->flag = ctxt->eflags;
3032 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3033 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3034 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3035 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3036 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3037 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3038 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3039 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3041 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3042 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3043 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3044 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3045 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3048 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3049 struct tss_segment_16 *tss)
3054 ctxt->_eip = tss->ip;
3055 ctxt->eflags = tss->flag | 2;
3056 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3057 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3058 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3059 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3060 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3061 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3062 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3063 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3066 * SDM says that segment selectors are loaded before segment
3069 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3070 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3071 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3072 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3073 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3078 * Now load segment descriptors. If fault happens at this stage
3079 * it is handled in a context of new task
3081 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3082 X86_TRANSFER_TASK_SWITCH, NULL);
3083 if (ret != X86EMUL_CONTINUE)
3085 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3086 X86_TRANSFER_TASK_SWITCH, NULL);
3087 if (ret != X86EMUL_CONTINUE)
3089 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3090 X86_TRANSFER_TASK_SWITCH, NULL);
3091 if (ret != X86EMUL_CONTINUE)
3093 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3094 X86_TRANSFER_TASK_SWITCH, NULL);
3095 if (ret != X86EMUL_CONTINUE)
3097 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3098 X86_TRANSFER_TASK_SWITCH, NULL);
3099 if (ret != X86EMUL_CONTINUE)
3102 return X86EMUL_CONTINUE;
3105 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3106 u16 tss_selector, u16 old_tss_sel,
3107 ulong old_tss_base, struct desc_struct *new_desc)
3109 struct tss_segment_16 tss_seg;
3111 u32 new_tss_base = get_desc_base(new_desc);
3113 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3114 if (ret != X86EMUL_CONTINUE)
3117 save_state_to_tss16(ctxt, &tss_seg);
3119 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3120 if (ret != X86EMUL_CONTINUE)
3123 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3124 if (ret != X86EMUL_CONTINUE)
3127 if (old_tss_sel != 0xffff) {
3128 tss_seg.prev_task_link = old_tss_sel;
3130 ret = linear_write_system(ctxt, new_tss_base,
3131 &tss_seg.prev_task_link,
3132 sizeof(tss_seg.prev_task_link));
3133 if (ret != X86EMUL_CONTINUE)
3137 return load_state_from_tss16(ctxt, &tss_seg);
3140 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3141 struct tss_segment_32 *tss)
3143 /* CR3 and ldt selector are not saved intentionally */
3144 tss->eip = ctxt->_eip;
3145 tss->eflags = ctxt->eflags;
3146 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3147 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3148 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3149 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3150 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3151 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3152 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3153 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3155 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3156 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3157 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3158 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3159 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3160 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3163 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3164 struct tss_segment_32 *tss)
3169 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3170 return emulate_gp(ctxt, 0);
3171 ctxt->_eip = tss->eip;
3172 ctxt->eflags = tss->eflags | 2;
3174 /* General purpose registers */
3175 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3176 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3177 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3178 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3179 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3180 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3181 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3182 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3185 * SDM says that segment selectors are loaded before segment
3186 * descriptors. This is important because CPL checks will
3189 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3190 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3191 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3192 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3193 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3194 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3195 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3198 * If we're switching between Protected Mode and VM86, we need to make
3199 * sure to update the mode before loading the segment descriptors so
3200 * that the selectors are interpreted correctly.
3202 if (ctxt->eflags & X86_EFLAGS_VM) {
3203 ctxt->mode = X86EMUL_MODE_VM86;
3206 ctxt->mode = X86EMUL_MODE_PROT32;
3211 * Now load segment descriptors. If fault happenes at this stage
3212 * it is handled in a context of new task
3214 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3215 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3216 if (ret != X86EMUL_CONTINUE)
3218 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3219 X86_TRANSFER_TASK_SWITCH, NULL);
3220 if (ret != X86EMUL_CONTINUE)
3222 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3223 X86_TRANSFER_TASK_SWITCH, NULL);
3224 if (ret != X86EMUL_CONTINUE)
3226 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3227 X86_TRANSFER_TASK_SWITCH, NULL);
3228 if (ret != X86EMUL_CONTINUE)
3230 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3231 X86_TRANSFER_TASK_SWITCH, NULL);
3232 if (ret != X86EMUL_CONTINUE)
3234 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3235 X86_TRANSFER_TASK_SWITCH, NULL);
3236 if (ret != X86EMUL_CONTINUE)
3238 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3239 X86_TRANSFER_TASK_SWITCH, NULL);
3244 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3245 u16 tss_selector, u16 old_tss_sel,
3246 ulong old_tss_base, struct desc_struct *new_desc)
3248 struct tss_segment_32 tss_seg;
3250 u32 new_tss_base = get_desc_base(new_desc);
3251 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3252 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3254 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3255 if (ret != X86EMUL_CONTINUE)
3258 save_state_to_tss32(ctxt, &tss_seg);
3260 /* Only GP registers and segment selectors are saved */
3261 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3262 ldt_sel_offset - eip_offset);
3263 if (ret != X86EMUL_CONTINUE)
3266 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3267 if (ret != X86EMUL_CONTINUE)
3270 if (old_tss_sel != 0xffff) {
3271 tss_seg.prev_task_link = old_tss_sel;
3273 ret = linear_write_system(ctxt, new_tss_base,
3274 &tss_seg.prev_task_link,
3275 sizeof(tss_seg.prev_task_link));
3276 if (ret != X86EMUL_CONTINUE)
3280 return load_state_from_tss32(ctxt, &tss_seg);
3283 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3284 u16 tss_selector, int idt_index, int reason,
3285 bool has_error_code, u32 error_code)
3287 const struct x86_emulate_ops *ops = ctxt->ops;
3288 struct desc_struct curr_tss_desc, next_tss_desc;
3290 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3291 ulong old_tss_base =
3292 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3294 ulong desc_addr, dr7;
3296 /* FIXME: old_tss_base == ~0 ? */
3298 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3299 if (ret != X86EMUL_CONTINUE)
3301 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3302 if (ret != X86EMUL_CONTINUE)
3305 /* FIXME: check that next_tss_desc is tss */
3308 * Check privileges. The three cases are task switch caused by...
3310 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3311 * 2. Exception/IRQ/iret: No check is performed
3312 * 3. jmp/call to TSS/task-gate: No check is performed since the
3313 * hardware checks it before exiting.
3315 if (reason == TASK_SWITCH_GATE) {
3316 if (idt_index != -1) {
3317 /* Software interrupts */
3318 struct desc_struct task_gate_desc;
3321 ret = read_interrupt_descriptor(ctxt, idt_index,
3323 if (ret != X86EMUL_CONTINUE)
3326 dpl = task_gate_desc.dpl;
3327 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3328 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3332 desc_limit = desc_limit_scaled(&next_tss_desc);
3333 if (!next_tss_desc.p ||
3334 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3335 desc_limit < 0x2b)) {
3336 return emulate_ts(ctxt, tss_selector & 0xfffc);
3339 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3340 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3341 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3344 if (reason == TASK_SWITCH_IRET)
3345 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3347 /* set back link to prev task only if NT bit is set in eflags
3348 note that old_tss_sel is not used after this point */
3349 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3350 old_tss_sel = 0xffff;
3352 if (next_tss_desc.type & 8)
3353 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3354 old_tss_base, &next_tss_desc);
3356 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3357 old_tss_base, &next_tss_desc);
3358 if (ret != X86EMUL_CONTINUE)
3361 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3362 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3364 if (reason != TASK_SWITCH_IRET) {
3365 next_tss_desc.type |= (1 << 1); /* set busy flag */
3366 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3369 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3370 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3372 if (has_error_code) {
3373 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3374 ctxt->lock_prefix = 0;
3375 ctxt->src.val = (unsigned long) error_code;
3376 ret = em_push(ctxt);
3379 ops->get_dr(ctxt, 7, &dr7);
3380 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3385 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3386 u16 tss_selector, int idt_index, int reason,
3387 bool has_error_code, u32 error_code)
3391 invalidate_registers(ctxt);
3392 ctxt->_eip = ctxt->eip;
3393 ctxt->dst.type = OP_NONE;
3395 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3396 has_error_code, error_code);
3398 if (rc == X86EMUL_CONTINUE) {
3399 ctxt->eip = ctxt->_eip;
3400 writeback_registers(ctxt);
3403 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3406 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3409 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3411 register_address_increment(ctxt, reg, df * op->bytes);
3412 op->addr.mem.ea = register_address(ctxt, reg);
3415 static int em_das(struct x86_emulate_ctxt *ctxt)
3418 bool af, cf, old_cf;
3420 cf = ctxt->eflags & X86_EFLAGS_CF;
3426 af = ctxt->eflags & X86_EFLAGS_AF;
3427 if ((al & 0x0f) > 9 || af) {
3429 cf = old_cf | (al >= 250);
3434 if (old_al > 0x99 || old_cf) {
3440 /* Set PF, ZF, SF */
3441 ctxt->src.type = OP_IMM;
3443 ctxt->src.bytes = 1;
3444 fastop(ctxt, em_or);
3445 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3447 ctxt->eflags |= X86_EFLAGS_CF;
3449 ctxt->eflags |= X86_EFLAGS_AF;
3450 return X86EMUL_CONTINUE;
3453 static int em_aam(struct x86_emulate_ctxt *ctxt)
3457 if (ctxt->src.val == 0)
3458 return emulate_de(ctxt);
3460 al = ctxt->dst.val & 0xff;
3461 ah = al / ctxt->src.val;
3462 al %= ctxt->src.val;
3464 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3466 /* Set PF, ZF, SF */
3467 ctxt->src.type = OP_IMM;
3469 ctxt->src.bytes = 1;
3470 fastop(ctxt, em_or);
3472 return X86EMUL_CONTINUE;
3475 static int em_aad(struct x86_emulate_ctxt *ctxt)
3477 u8 al = ctxt->dst.val & 0xff;
3478 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3480 al = (al + (ah * ctxt->src.val)) & 0xff;
3482 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3484 /* Set PF, ZF, SF */
3485 ctxt->src.type = OP_IMM;
3487 ctxt->src.bytes = 1;
3488 fastop(ctxt, em_or);
3490 return X86EMUL_CONTINUE;
3493 static int em_call(struct x86_emulate_ctxt *ctxt)
3496 long rel = ctxt->src.val;
3498 ctxt->src.val = (unsigned long)ctxt->_eip;
3499 rc = jmp_rel(ctxt, rel);
3500 if (rc != X86EMUL_CONTINUE)
3502 return em_push(ctxt);
3505 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3510 struct desc_struct old_desc, new_desc;
3511 const struct x86_emulate_ops *ops = ctxt->ops;
3512 int cpl = ctxt->ops->cpl(ctxt);
3513 enum x86emul_mode prev_mode = ctxt->mode;
3515 old_eip = ctxt->_eip;
3516 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3518 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3519 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3520 X86_TRANSFER_CALL_JMP, &new_desc);
3521 if (rc != X86EMUL_CONTINUE)
3524 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3525 if (rc != X86EMUL_CONTINUE)
3528 ctxt->src.val = old_cs;
3530 if (rc != X86EMUL_CONTINUE)
3533 ctxt->src.val = old_eip;
3535 /* If we failed, we tainted the memory, but the very least we should
3537 if (rc != X86EMUL_CONTINUE) {
3538 pr_warn_once("faulting far call emulation tainted memory\n");
3543 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3544 ctxt->mode = prev_mode;
3549 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3554 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3555 if (rc != X86EMUL_CONTINUE)
3557 rc = assign_eip_near(ctxt, eip);
3558 if (rc != X86EMUL_CONTINUE)
3560 rsp_increment(ctxt, ctxt->src.val);
3561 return X86EMUL_CONTINUE;
3564 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3566 /* Write back the register source. */
3567 ctxt->src.val = ctxt->dst.val;
3568 write_register_operand(&ctxt->src);
3570 /* Write back the memory destination with implicit LOCK prefix. */
3571 ctxt->dst.val = ctxt->src.orig_val;
3572 ctxt->lock_prefix = 1;
3573 return X86EMUL_CONTINUE;
3576 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3578 ctxt->dst.val = ctxt->src2.val;
3579 return fastop(ctxt, em_imul);
3582 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3584 ctxt->dst.type = OP_REG;
3585 ctxt->dst.bytes = ctxt->src.bytes;
3586 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3587 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3589 return X86EMUL_CONTINUE;
3592 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3596 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3597 return emulate_gp(ctxt, 0);
3598 ctxt->dst.val = tsc_aux;
3599 return X86EMUL_CONTINUE;
3602 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3606 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3607 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3608 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3609 return X86EMUL_CONTINUE;
3612 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3616 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3617 return emulate_gp(ctxt, 0);
3618 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3619 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3620 return X86EMUL_CONTINUE;
3623 static int em_mov(struct x86_emulate_ctxt *ctxt)
3625 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3626 return X86EMUL_CONTINUE;
3629 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3633 if (!ctxt->ops->guest_has_movbe(ctxt))
3634 return emulate_ud(ctxt);
3636 switch (ctxt->op_bytes) {
3639 * From MOVBE definition: "...When the operand size is 16 bits,
3640 * the upper word of the destination register remains unchanged
3643 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3644 * rules so we have to do the operation almost per hand.
3646 tmp = (u16)ctxt->src.val;
3647 ctxt->dst.val &= ~0xffffUL;
3648 ctxt->dst.val |= (unsigned long)swab16(tmp);
3651 ctxt->dst.val = swab32((u32)ctxt->src.val);
3654 ctxt->dst.val = swab64(ctxt->src.val);
3659 return X86EMUL_CONTINUE;
3662 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3664 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3665 return emulate_gp(ctxt, 0);
3667 /* Disable writeback. */
3668 ctxt->dst.type = OP_NONE;
3669 return X86EMUL_CONTINUE;
3672 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3676 if (ctxt->mode == X86EMUL_MODE_PROT64)
3677 val = ctxt->src.val & ~0ULL;
3679 val = ctxt->src.val & ~0U;
3681 /* #UD condition is already handled. */
3682 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3683 return emulate_gp(ctxt, 0);
3685 /* Disable writeback. */
3686 ctxt->dst.type = OP_NONE;
3687 return X86EMUL_CONTINUE;
3690 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3694 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3695 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3696 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3697 return emulate_gp(ctxt, 0);
3699 return X86EMUL_CONTINUE;
3702 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3706 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3707 return emulate_gp(ctxt, 0);
3709 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3710 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3711 return X86EMUL_CONTINUE;
3714 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3716 if (segment > VCPU_SREG_GS &&
3717 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3718 ctxt->ops->cpl(ctxt) > 0)
3719 return emulate_gp(ctxt, 0);
3721 ctxt->dst.val = get_segment_selector(ctxt, segment);
3722 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3723 ctxt->dst.bytes = 2;
3724 return X86EMUL_CONTINUE;
3727 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3729 if (ctxt->modrm_reg > VCPU_SREG_GS)
3730 return emulate_ud(ctxt);
3732 return em_store_sreg(ctxt, ctxt->modrm_reg);
3735 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3737 u16 sel = ctxt->src.val;
3739 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3740 return emulate_ud(ctxt);
3742 if (ctxt->modrm_reg == VCPU_SREG_SS)
3743 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3745 /* Disable writeback. */
3746 ctxt->dst.type = OP_NONE;
3747 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3750 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3752 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3755 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3757 u16 sel = ctxt->src.val;
3759 /* Disable writeback. */
3760 ctxt->dst.type = OP_NONE;
3761 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3764 static int em_str(struct x86_emulate_ctxt *ctxt)
3766 return em_store_sreg(ctxt, VCPU_SREG_TR);
3769 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3771 u16 sel = ctxt->src.val;
3773 /* Disable writeback. */
3774 ctxt->dst.type = OP_NONE;
3775 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3778 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3783 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3784 if (rc == X86EMUL_CONTINUE)
3785 ctxt->ops->invlpg(ctxt, linear);
3786 /* Disable writeback. */
3787 ctxt->dst.type = OP_NONE;
3788 return X86EMUL_CONTINUE;
3791 static int em_clts(struct x86_emulate_ctxt *ctxt)
3795 cr0 = ctxt->ops->get_cr(ctxt, 0);
3797 ctxt->ops->set_cr(ctxt, 0, cr0);
3798 return X86EMUL_CONTINUE;
3801 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3803 int rc = ctxt->ops->fix_hypercall(ctxt);
3805 if (rc != X86EMUL_CONTINUE)
3808 /* Let the processor re-execute the fixed hypercall */
3809 ctxt->_eip = ctxt->eip;
3810 /* Disable writeback. */
3811 ctxt->dst.type = OP_NONE;
3812 return X86EMUL_CONTINUE;
3815 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3816 void (*get)(struct x86_emulate_ctxt *ctxt,
3817 struct desc_ptr *ptr))
3819 struct desc_ptr desc_ptr;
3821 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3822 ctxt->ops->cpl(ctxt) > 0)
3823 return emulate_gp(ctxt, 0);
3825 if (ctxt->mode == X86EMUL_MODE_PROT64)
3827 get(ctxt, &desc_ptr);
3828 if (ctxt->op_bytes == 2) {
3830 desc_ptr.address &= 0x00ffffff;
3832 /* Disable writeback. */
3833 ctxt->dst.type = OP_NONE;
3834 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3835 &desc_ptr, 2 + ctxt->op_bytes);
3838 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3840 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3843 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3845 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3848 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3850 struct desc_ptr desc_ptr;
3853 if (ctxt->mode == X86EMUL_MODE_PROT64)
3855 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3856 &desc_ptr.size, &desc_ptr.address,
3858 if (rc != X86EMUL_CONTINUE)
3860 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3861 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3862 return emulate_gp(ctxt, 0);
3864 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3866 ctxt->ops->set_idt(ctxt, &desc_ptr);
3867 /* Disable writeback. */
3868 ctxt->dst.type = OP_NONE;
3869 return X86EMUL_CONTINUE;
3872 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3874 return em_lgdt_lidt(ctxt, true);
3877 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3879 return em_lgdt_lidt(ctxt, false);
3882 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3884 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3885 ctxt->ops->cpl(ctxt) > 0)
3886 return emulate_gp(ctxt, 0);
3888 if (ctxt->dst.type == OP_MEM)
3889 ctxt->dst.bytes = 2;
3890 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3891 return X86EMUL_CONTINUE;
3894 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3896 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3897 | (ctxt->src.val & 0x0f));
3898 ctxt->dst.type = OP_NONE;
3899 return X86EMUL_CONTINUE;
3902 static int em_loop(struct x86_emulate_ctxt *ctxt)
3904 int rc = X86EMUL_CONTINUE;
3906 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3907 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3908 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3909 rc = jmp_rel(ctxt, ctxt->src.val);
3914 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3916 int rc = X86EMUL_CONTINUE;
3918 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3919 rc = jmp_rel(ctxt, ctxt->src.val);
3924 static int em_in(struct x86_emulate_ctxt *ctxt)
3926 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3928 return X86EMUL_IO_NEEDED;
3930 return X86EMUL_CONTINUE;
3933 static int em_out(struct x86_emulate_ctxt *ctxt)
3935 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3937 /* Disable writeback. */
3938 ctxt->dst.type = OP_NONE;
3939 return X86EMUL_CONTINUE;
3942 static int em_cli(struct x86_emulate_ctxt *ctxt)
3944 if (emulator_bad_iopl(ctxt))
3945 return emulate_gp(ctxt, 0);
3947 ctxt->eflags &= ~X86_EFLAGS_IF;
3948 return X86EMUL_CONTINUE;
3951 static int em_sti(struct x86_emulate_ctxt *ctxt)
3953 if (emulator_bad_iopl(ctxt))
3954 return emulate_gp(ctxt, 0);
3956 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3957 ctxt->eflags |= X86_EFLAGS_IF;
3958 return X86EMUL_CONTINUE;
3961 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3963 u32 eax, ebx, ecx, edx;
3966 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3967 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3968 ctxt->ops->cpl(ctxt)) {
3969 return emulate_gp(ctxt, 0);
3972 eax = reg_read(ctxt, VCPU_REGS_RAX);
3973 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3974 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3975 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3976 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3977 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3978 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3979 return X86EMUL_CONTINUE;
3982 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3986 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3988 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3990 ctxt->eflags &= ~0xffUL;
3991 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3992 return X86EMUL_CONTINUE;
3995 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3997 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3998 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3999 return X86EMUL_CONTINUE;
4002 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4004 switch (ctxt->op_bytes) {
4005 #ifdef CONFIG_X86_64
4007 asm("bswap %0" : "+r"(ctxt->dst.val));
4011 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4014 return X86EMUL_CONTINUE;
4017 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4019 /* emulating clflush regardless of cpuid */
4020 return X86EMUL_CONTINUE;
4023 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4025 ctxt->dst.val = (s32) ctxt->src.val;
4026 return X86EMUL_CONTINUE;
4029 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4031 if (!ctxt->ops->guest_has_fxsr(ctxt))
4032 return emulate_ud(ctxt);
4034 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4035 return emulate_nm(ctxt);
4038 * Don't emulate a case that should never be hit, instead of working
4039 * around a lack of fxsave64/fxrstor64 on old compilers.
4041 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4042 return X86EMUL_UNHANDLEABLE;
4044 return X86EMUL_CONTINUE;
4048 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4049 * and restore MXCSR.
4051 static size_t __fxstate_size(int nregs)
4053 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4056 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4059 if (ctxt->mode == X86EMUL_MODE_PROT64)
4060 return __fxstate_size(16);
4062 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4063 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4067 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4070 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4071 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4073 * 3) 64-bit mode with REX.W prefix
4074 * - like (2), but XMM 8-15 are being saved and restored
4075 * 4) 64-bit mode without REX.W prefix
4076 * - like (3), but FIP and FDP are 64 bit
4078 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4079 * desired result. (4) is not emulated.
4081 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4082 * and FPU DS) should match.
4084 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4086 struct fxregs_state fx_state;
4089 rc = check_fxsr(ctxt);
4090 if (rc != X86EMUL_CONTINUE)
4095 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4099 if (rc != X86EMUL_CONTINUE)
4102 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4103 fxstate_size(ctxt));
4107 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4108 * in the host registers (via FXSAVE) instead, so they won't be modified.
4109 * (preemption has to stay disabled until FXRSTOR).
4111 * Use noinline to keep the stack for other functions called by callers small.
4113 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4114 const size_t used_size)
4116 struct fxregs_state fx_tmp;
4119 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4120 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4121 __fxstate_size(16) - used_size);
4126 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4128 struct fxregs_state fx_state;
4132 rc = check_fxsr(ctxt);
4133 if (rc != X86EMUL_CONTINUE)
4136 size = fxstate_size(ctxt);
4137 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4138 if (rc != X86EMUL_CONTINUE)
4143 if (size < __fxstate_size(16)) {
4144 rc = fxregs_fixup(&fx_state, size);
4145 if (rc != X86EMUL_CONTINUE)
4149 if (fx_state.mxcsr >> 16) {
4150 rc = emulate_gp(ctxt, 0);
4154 if (rc == X86EMUL_CONTINUE)
4155 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4163 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4167 eax = reg_read(ctxt, VCPU_REGS_RAX);
4168 edx = reg_read(ctxt, VCPU_REGS_RDX);
4169 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4171 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4172 return emulate_gp(ctxt, 0);
4174 return X86EMUL_CONTINUE;
4177 static bool valid_cr(int nr)
4189 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4191 if (!valid_cr(ctxt->modrm_reg))
4192 return emulate_ud(ctxt);
4194 return X86EMUL_CONTINUE;
4197 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4199 u64 new_val = ctxt->src.val64;
4200 int cr = ctxt->modrm_reg;
4203 static u64 cr_reserved_bits[] = {
4204 0xffffffff00000000ULL,
4205 0, 0, 0, /* CR3 checked later */
4212 return emulate_ud(ctxt);
4214 if (new_val & cr_reserved_bits[cr])
4215 return emulate_gp(ctxt, 0);
4220 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4221 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4222 return emulate_gp(ctxt, 0);
4224 cr4 = ctxt->ops->get_cr(ctxt, 4);
4225 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4227 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4228 !(cr4 & X86_CR4_PAE))
4229 return emulate_gp(ctxt, 0);
4236 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4237 if (efer & EFER_LMA) {
4239 u32 eax, ebx, ecx, edx;
4243 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4245 maxphyaddr = eax & 0xff;
4248 rsvd = rsvd_bits(maxphyaddr, 63);
4249 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4250 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4254 return emulate_gp(ctxt, 0);
4259 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4261 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4262 return emulate_gp(ctxt, 0);
4268 return X86EMUL_CONTINUE;
4271 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4275 ctxt->ops->get_dr(ctxt, 7, &dr7);
4277 /* Check if DR7.Global_Enable is set */
4278 return dr7 & (1 << 13);
4281 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4283 int dr = ctxt->modrm_reg;
4287 return emulate_ud(ctxt);
4289 cr4 = ctxt->ops->get_cr(ctxt, 4);
4290 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4291 return emulate_ud(ctxt);
4293 if (check_dr7_gd(ctxt)) {
4296 ctxt->ops->get_dr(ctxt, 6, &dr6);
4297 dr6 &= ~DR_TRAP_BITS;
4298 dr6 |= DR6_BD | DR6_RTM;
4299 ctxt->ops->set_dr(ctxt, 6, dr6);
4300 return emulate_db(ctxt);
4303 return X86EMUL_CONTINUE;
4306 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4308 u64 new_val = ctxt->src.val64;
4309 int dr = ctxt->modrm_reg;
4311 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4312 return emulate_gp(ctxt, 0);
4314 return check_dr_read(ctxt);
4317 static int check_svme(struct x86_emulate_ctxt *ctxt)
4321 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4323 if (!(efer & EFER_SVME))
4324 return emulate_ud(ctxt);
4326 return X86EMUL_CONTINUE;
4329 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4331 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4333 /* Valid physical address? */
4334 if (rax & 0xffff000000000000ULL)
4335 return emulate_gp(ctxt, 0);
4337 return check_svme(ctxt);
4340 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4342 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4344 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4345 return emulate_ud(ctxt);
4347 return X86EMUL_CONTINUE;
4350 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4352 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4353 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4356 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4357 * in Ring3 when CR4.PCE=0.
4359 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4360 return X86EMUL_CONTINUE;
4362 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4363 ctxt->ops->check_pmc(ctxt, rcx))
4364 return emulate_gp(ctxt, 0);
4366 return X86EMUL_CONTINUE;
4369 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4371 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4372 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4373 return emulate_gp(ctxt, 0);
4375 return X86EMUL_CONTINUE;
4378 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4380 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4381 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4382 return emulate_gp(ctxt, 0);
4384 return X86EMUL_CONTINUE;
4387 #define D(_y) { .flags = (_y) }
4388 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4389 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4390 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4391 #define N D(NotImpl)
4392 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4393 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4394 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4395 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4396 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4397 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4398 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4399 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4400 #define II(_f, _e, _i) \
4401 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4402 #define IIP(_f, _e, _i, _p) \
4403 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4404 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4405 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4407 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4408 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4409 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4410 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4411 #define I2bvIP(_f, _e, _i, _p) \
4412 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4414 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4415 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4416 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4418 static const struct opcode group7_rm0[] = {
4420 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4424 static const struct opcode group7_rm1[] = {
4425 DI(SrcNone | Priv, monitor),
4426 DI(SrcNone | Priv, mwait),
4430 static const struct opcode group7_rm2[] = {
4432 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4436 static const struct opcode group7_rm3[] = {
4437 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4438 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4439 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4440 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4441 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4442 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4443 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4444 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4447 static const struct opcode group7_rm7[] = {
4449 DIP(SrcNone, rdtscp, check_rdtsc),
4453 static const struct opcode group1[] = {
4455 F(Lock | PageTable, em_or),
4458 F(Lock | PageTable, em_and),
4464 static const struct opcode group1A[] = {
4465 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4468 static const struct opcode group2[] = {
4469 F(DstMem | ModRM, em_rol),
4470 F(DstMem | ModRM, em_ror),
4471 F(DstMem | ModRM, em_rcl),
4472 F(DstMem | ModRM, em_rcr),
4473 F(DstMem | ModRM, em_shl),
4474 F(DstMem | ModRM, em_shr),
4475 F(DstMem | ModRM, em_shl),
4476 F(DstMem | ModRM, em_sar),
4479 static const struct opcode group3[] = {
4480 F(DstMem | SrcImm | NoWrite, em_test),
4481 F(DstMem | SrcImm | NoWrite, em_test),
4482 F(DstMem | SrcNone | Lock, em_not),
4483 F(DstMem | SrcNone | Lock, em_neg),
4484 F(DstXacc | Src2Mem, em_mul_ex),
4485 F(DstXacc | Src2Mem, em_imul_ex),
4486 F(DstXacc | Src2Mem, em_div_ex),
4487 F(DstXacc | Src2Mem, em_idiv_ex),
4490 static const struct opcode group4[] = {
4491 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4492 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4496 static const struct opcode group5[] = {
4497 F(DstMem | SrcNone | Lock, em_inc),
4498 F(DstMem | SrcNone | Lock, em_dec),
4499 I(SrcMem | NearBranch, em_call_near_abs),
4500 I(SrcMemFAddr | ImplicitOps, em_call_far),
4501 I(SrcMem | NearBranch, em_jmp_abs),
4502 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4503 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4506 static const struct opcode group6[] = {
4507 II(Prot | DstMem, em_sldt, sldt),
4508 II(Prot | DstMem, em_str, str),
4509 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4510 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4514 static const struct group_dual group7 = { {
4515 II(Mov | DstMem, em_sgdt, sgdt),
4516 II(Mov | DstMem, em_sidt, sidt),
4517 II(SrcMem | Priv, em_lgdt, lgdt),
4518 II(SrcMem | Priv, em_lidt, lidt),
4519 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4520 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4521 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4527 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4528 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4532 static const struct opcode group8[] = {
4534 F(DstMem | SrcImmByte | NoWrite, em_bt),
4535 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4536 F(DstMem | SrcImmByte | Lock, em_btr),
4537 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4541 * The "memory" destination is actually always a register, since we come
4542 * from the register case of group9.
4544 static const struct gprefix pfx_0f_c7_7 = {
4545 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4549 static const struct group_dual group9 = { {
4550 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4552 N, N, N, N, N, N, N,
4553 GP(0, &pfx_0f_c7_7),
4556 static const struct opcode group11[] = {
4557 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4561 static const struct gprefix pfx_0f_ae_7 = {
4562 I(SrcMem | ByteOp, em_clflush), N, N, N,
4565 static const struct group_dual group15 = { {
4566 I(ModRM | Aligned16, em_fxsave),
4567 I(ModRM | Aligned16, em_fxrstor),
4568 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4570 N, N, N, N, N, N, N, N,
4573 static const struct gprefix pfx_0f_6f_0f_7f = {
4574 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4577 static const struct instr_dual instr_dual_0f_2b = {
4581 static const struct gprefix pfx_0f_2b = {
4582 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4585 static const struct gprefix pfx_0f_10_0f_11 = {
4586 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4589 static const struct gprefix pfx_0f_28_0f_29 = {
4590 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4593 static const struct gprefix pfx_0f_e7 = {
4594 N, I(Sse, em_mov), N, N,
4597 static const struct escape escape_d9 = { {
4598 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4601 N, N, N, N, N, N, N, N,
4603 N, N, N, N, N, N, N, N,
4605 N, N, N, N, N, N, N, N,
4607 N, N, N, N, N, N, N, N,
4609 N, N, N, N, N, N, N, N,
4611 N, N, N, N, N, N, N, N,
4613 N, N, N, N, N, N, N, N,
4615 N, N, N, N, N, N, N, N,
4618 static const struct escape escape_db = { {
4619 N, N, N, N, N, N, N, N,
4622 N, N, N, N, N, N, N, N,
4624 N, N, N, N, N, N, N, N,
4626 N, N, N, N, N, N, N, N,
4628 N, N, N, N, N, N, N, N,
4630 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4632 N, N, N, N, N, N, N, N,
4634 N, N, N, N, N, N, N, N,
4636 N, N, N, N, N, N, N, N,
4639 static const struct escape escape_dd = { {
4640 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4643 N, N, N, N, N, N, N, N,
4645 N, N, N, N, N, N, N, N,
4647 N, N, N, N, N, N, N, N,
4649 N, N, N, N, N, N, N, N,
4651 N, N, N, N, N, N, N, N,
4653 N, N, N, N, N, N, N, N,
4655 N, N, N, N, N, N, N, N,
4657 N, N, N, N, N, N, N, N,
4660 static const struct instr_dual instr_dual_0f_c3 = {
4661 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4664 static const struct mode_dual mode_dual_63 = {
4665 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4668 static const struct opcode opcode_table[256] = {
4670 F6ALU(Lock, em_add),
4671 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4672 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4674 F6ALU(Lock | PageTable, em_or),
4675 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4678 F6ALU(Lock, em_adc),
4679 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4680 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4682 F6ALU(Lock, em_sbb),
4683 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4684 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4686 F6ALU(Lock | PageTable, em_and), N, N,
4688 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4690 F6ALU(Lock, em_xor), N, N,
4692 F6ALU(NoWrite, em_cmp), N, N,
4694 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4696 X8(I(SrcReg | Stack, em_push)),
4698 X8(I(DstReg | Stack, em_pop)),
4700 I(ImplicitOps | Stack | No64, em_pusha),
4701 I(ImplicitOps | Stack | No64, em_popa),
4702 N, MD(ModRM, &mode_dual_63),
4705 I(SrcImm | Mov | Stack, em_push),
4706 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4707 I(SrcImmByte | Mov | Stack, em_push),
4708 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4709 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4710 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4712 X16(D(SrcImmByte | NearBranch)),
4714 G(ByteOp | DstMem | SrcImm, group1),
4715 G(DstMem | SrcImm, group1),
4716 G(ByteOp | DstMem | SrcImm | No64, group1),
4717 G(DstMem | SrcImmByte, group1),
4718 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4719 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4721 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4722 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4723 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4724 D(ModRM | SrcMem | NoAccess | DstReg),
4725 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4728 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4730 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4731 I(SrcImmFAddr | No64, em_call_far), N,
4732 II(ImplicitOps | Stack, em_pushf, pushf),
4733 II(ImplicitOps | Stack, em_popf, popf),
4734 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4736 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4737 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4738 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4739 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4741 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4742 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4743 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4744 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4746 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4748 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4750 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4751 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4752 I(ImplicitOps | NearBranch, em_ret),
4753 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4754 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4755 G(ByteOp, group11), G(0, group11),
4757 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4758 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4759 I(ImplicitOps, em_ret_far),
4760 D(ImplicitOps), DI(SrcImmByte, intn),
4761 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4763 G(Src2One | ByteOp, group2), G(Src2One, group2),
4764 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4765 I(DstAcc | SrcImmUByte | No64, em_aam),
4766 I(DstAcc | SrcImmUByte | No64, em_aad),
4767 F(DstAcc | ByteOp | No64, em_salc),
4768 I(DstAcc | SrcXLat | ByteOp, em_mov),
4770 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4772 X3(I(SrcImmByte | NearBranch, em_loop)),
4773 I(SrcImmByte | NearBranch, em_jcxz),
4774 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4775 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4777 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4778 I(SrcImmFAddr | No64, em_jmp_far),
4779 D(SrcImmByte | ImplicitOps | NearBranch),
4780 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4781 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4783 N, DI(ImplicitOps, icebp), N, N,
4784 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4785 G(ByteOp, group3), G(0, group3),
4787 D(ImplicitOps), D(ImplicitOps),
4788 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4789 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4792 static const struct opcode twobyte_table[256] = {
4794 G(0, group6), GD(0, &group7), N, N,
4795 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4796 II(ImplicitOps | Priv, em_clts, clts), N,
4797 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4798 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4800 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4801 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4803 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4804 D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4805 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4806 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4807 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */
4808 D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
4810 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4811 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4812 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4814 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4817 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4818 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4819 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4822 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4823 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4824 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4825 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4826 I(ImplicitOps | EmulateOnUD, em_sysenter),
4827 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4829 N, N, N, N, N, N, N, N,
4831 X16(D(DstReg | SrcMem | ModRM)),
4833 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4838 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4843 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4845 X16(D(SrcImm | NearBranch)),
4847 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4849 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4850 II(ImplicitOps, em_cpuid, cpuid),
4851 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4852 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4853 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4855 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4856 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4857 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4858 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4859 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4860 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4862 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4863 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4864 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4865 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4866 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4867 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4871 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4872 I(DstReg | SrcMem | ModRM, em_bsf_c),
4873 I(DstReg | SrcMem | ModRM, em_bsr_c),
4874 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4876 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4877 N, ID(0, &instr_dual_0f_c3),
4878 N, N, N, GD(0, &group9),
4880 X8(I(DstReg, em_bswap)),
4882 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4884 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4885 N, N, N, N, N, N, N, N,
4887 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4890 static const struct instr_dual instr_dual_0f_38_f0 = {
4891 I(DstReg | SrcMem | Mov, em_movbe), N
4894 static const struct instr_dual instr_dual_0f_38_f1 = {
4895 I(DstMem | SrcReg | Mov, em_movbe), N
4898 static const struct gprefix three_byte_0f_38_f0 = {
4899 ID(0, &instr_dual_0f_38_f0), N, N, N
4902 static const struct gprefix three_byte_0f_38_f1 = {
4903 ID(0, &instr_dual_0f_38_f1), N, N, N
4907 * Insns below are selected by the prefix which indexed by the third opcode
4910 static const struct opcode opcode_map_0f_38[256] = {
4912 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4914 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4916 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4917 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4938 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4942 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4948 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4949 unsigned size, bool sign_extension)
4951 int rc = X86EMUL_CONTINUE;
4955 op->addr.mem.ea = ctxt->_eip;
4956 /* NB. Immediates are sign-extended as necessary. */
4957 switch (op->bytes) {
4959 op->val = insn_fetch(s8, ctxt);
4962 op->val = insn_fetch(s16, ctxt);
4965 op->val = insn_fetch(s32, ctxt);
4968 op->val = insn_fetch(s64, ctxt);
4971 if (!sign_extension) {
4972 switch (op->bytes) {
4980 op->val &= 0xffffffff;
4988 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4991 int rc = X86EMUL_CONTINUE;
4995 decode_register_operand(ctxt, op);
4998 rc = decode_imm(ctxt, op, 1, false);
5001 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5005 if (ctxt->d & BitOp)
5006 fetch_bit_operand(ctxt);
5007 op->orig_val = op->val;
5010 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5014 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5015 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5016 fetch_register_operand(op);
5017 op->orig_val = op->val;
5021 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
5022 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5023 fetch_register_operand(op);
5024 op->orig_val = op->val;
5027 if (ctxt->d & ByteOp) {
5032 op->bytes = ctxt->op_bytes;
5033 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5034 fetch_register_operand(op);
5035 op->orig_val = op->val;
5039 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5041 register_address(ctxt, VCPU_REGS_RDI);
5042 op->addr.mem.seg = VCPU_SREG_ES;
5049 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5050 fetch_register_operand(op);
5055 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5058 rc = decode_imm(ctxt, op, 1, true);
5066 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5069 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5072 ctxt->memop.bytes = 1;
5073 if (ctxt->memop.type == OP_REG) {
5074 ctxt->memop.addr.reg = decode_register(ctxt,
5075 ctxt->modrm_rm, true);
5076 fetch_register_operand(&ctxt->memop);
5080 ctxt->memop.bytes = 2;
5083 ctxt->memop.bytes = 4;
5086 rc = decode_imm(ctxt, op, 2, false);
5089 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5093 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5095 register_address(ctxt, VCPU_REGS_RSI);
5096 op->addr.mem.seg = ctxt->seg_override;
5102 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5105 reg_read(ctxt, VCPU_REGS_RBX) +
5106 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5107 op->addr.mem.seg = ctxt->seg_override;
5112 op->addr.mem.ea = ctxt->_eip;
5113 op->bytes = ctxt->op_bytes + 2;
5114 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5117 ctxt->memop.bytes = ctxt->op_bytes + 2;
5121 op->val = VCPU_SREG_ES;
5125 op->val = VCPU_SREG_CS;
5129 op->val = VCPU_SREG_SS;
5133 op->val = VCPU_SREG_DS;
5137 op->val = VCPU_SREG_FS;
5141 op->val = VCPU_SREG_GS;
5144 /* Special instructions do their own operand decoding. */
5146 op->type = OP_NONE; /* Disable writeback. */
5154 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5156 int rc = X86EMUL_CONTINUE;
5157 int mode = ctxt->mode;
5158 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5159 bool op_prefix = false;
5160 bool has_seg_override = false;
5161 struct opcode opcode;
5163 struct desc_struct desc;
5165 ctxt->memop.type = OP_NONE;
5166 ctxt->memopp = NULL;
5167 ctxt->_eip = ctxt->eip;
5168 ctxt->fetch.ptr = ctxt->fetch.data;
5169 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5170 ctxt->opcode_len = 1;
5171 ctxt->intercept = x86_intercept_none;
5173 memcpy(ctxt->fetch.data, insn, insn_len);
5175 rc = __do_insn_fetch_bytes(ctxt, 1);
5176 if (rc != X86EMUL_CONTINUE)
5181 case X86EMUL_MODE_REAL:
5182 case X86EMUL_MODE_VM86:
5183 def_op_bytes = def_ad_bytes = 2;
5184 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5186 def_op_bytes = def_ad_bytes = 4;
5188 case X86EMUL_MODE_PROT16:
5189 def_op_bytes = def_ad_bytes = 2;
5191 case X86EMUL_MODE_PROT32:
5192 def_op_bytes = def_ad_bytes = 4;
5194 #ifdef CONFIG_X86_64
5195 case X86EMUL_MODE_PROT64:
5201 return EMULATION_FAILED;
5204 ctxt->op_bytes = def_op_bytes;
5205 ctxt->ad_bytes = def_ad_bytes;
5207 /* Legacy prefixes. */
5209 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5210 case 0x66: /* operand-size override */
5212 /* switch between 2/4 bytes */
5213 ctxt->op_bytes = def_op_bytes ^ 6;
5215 case 0x67: /* address-size override */
5216 if (mode == X86EMUL_MODE_PROT64)
5217 /* switch between 4/8 bytes */
5218 ctxt->ad_bytes = def_ad_bytes ^ 12;
5220 /* switch between 2/4 bytes */
5221 ctxt->ad_bytes = def_ad_bytes ^ 6;
5223 case 0x26: /* ES override */
5224 has_seg_override = true;
5225 ctxt->seg_override = VCPU_SREG_ES;
5227 case 0x2e: /* CS override */
5228 has_seg_override = true;
5229 ctxt->seg_override = VCPU_SREG_CS;
5231 case 0x36: /* SS override */
5232 has_seg_override = true;
5233 ctxt->seg_override = VCPU_SREG_SS;
5235 case 0x3e: /* DS override */
5236 has_seg_override = true;
5237 ctxt->seg_override = VCPU_SREG_DS;
5239 case 0x64: /* FS override */
5240 has_seg_override = true;
5241 ctxt->seg_override = VCPU_SREG_FS;
5243 case 0x65: /* GS override */
5244 has_seg_override = true;
5245 ctxt->seg_override = VCPU_SREG_GS;
5247 case 0x40 ... 0x4f: /* REX */
5248 if (mode != X86EMUL_MODE_PROT64)
5250 ctxt->rex_prefix = ctxt->b;
5252 case 0xf0: /* LOCK */
5253 ctxt->lock_prefix = 1;
5255 case 0xf2: /* REPNE/REPNZ */
5256 case 0xf3: /* REP/REPE/REPZ */
5257 ctxt->rep_prefix = ctxt->b;
5263 /* Any legacy prefix after a REX prefix nullifies its effect. */
5265 ctxt->rex_prefix = 0;
5271 if (ctxt->rex_prefix & 8)
5272 ctxt->op_bytes = 8; /* REX.W */
5274 /* Opcode byte(s). */
5275 opcode = opcode_table[ctxt->b];
5276 /* Two-byte opcode? */
5277 if (ctxt->b == 0x0f) {
5278 ctxt->opcode_len = 2;
5279 ctxt->b = insn_fetch(u8, ctxt);
5280 opcode = twobyte_table[ctxt->b];
5282 /* 0F_38 opcode map */
5283 if (ctxt->b == 0x38) {
5284 ctxt->opcode_len = 3;
5285 ctxt->b = insn_fetch(u8, ctxt);
5286 opcode = opcode_map_0f_38[ctxt->b];
5289 ctxt->d = opcode.flags;
5291 if (ctxt->d & ModRM)
5292 ctxt->modrm = insn_fetch(u8, ctxt);
5294 /* vex-prefix instructions are not implemented */
5295 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5296 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5300 while (ctxt->d & GroupMask) {
5301 switch (ctxt->d & GroupMask) {
5303 goffset = (ctxt->modrm >> 3) & 7;
5304 opcode = opcode.u.group[goffset];
5307 goffset = (ctxt->modrm >> 3) & 7;
5308 if ((ctxt->modrm >> 6) == 3)
5309 opcode = opcode.u.gdual->mod3[goffset];
5311 opcode = opcode.u.gdual->mod012[goffset];
5314 goffset = ctxt->modrm & 7;
5315 opcode = opcode.u.group[goffset];
5318 if (ctxt->rep_prefix && op_prefix)
5319 return EMULATION_FAILED;
5320 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5321 switch (simd_prefix) {
5322 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5323 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5324 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5325 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5329 if (ctxt->modrm > 0xbf) {
5330 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5331 u32 index = array_index_nospec(
5332 ctxt->modrm - 0xc0, size);
5334 opcode = opcode.u.esc->high[index];
5336 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5340 if ((ctxt->modrm >> 6) == 3)
5341 opcode = opcode.u.idual->mod3;
5343 opcode = opcode.u.idual->mod012;
5346 if (ctxt->mode == X86EMUL_MODE_PROT64)
5347 opcode = opcode.u.mdual->mode64;
5349 opcode = opcode.u.mdual->mode32;
5352 return EMULATION_FAILED;
5355 ctxt->d &= ~(u64)GroupMask;
5356 ctxt->d |= opcode.flags;
5361 return EMULATION_FAILED;
5363 ctxt->execute = opcode.u.execute;
5365 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5366 return EMULATION_FAILED;
5368 if (unlikely(ctxt->d &
5369 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5372 * These are copied unconditionally here, and checked unconditionally
5373 * in x86_emulate_insn.
5375 ctxt->check_perm = opcode.check_perm;
5376 ctxt->intercept = opcode.intercept;
5378 if (ctxt->d & NotImpl)
5379 return EMULATION_FAILED;
5381 if (mode == X86EMUL_MODE_PROT64) {
5382 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5384 else if (ctxt->d & NearBranch)
5388 if (ctxt->d & Op3264) {
5389 if (mode == X86EMUL_MODE_PROT64)
5395 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5399 ctxt->op_bytes = 16;
5400 else if (ctxt->d & Mmx)
5404 /* ModRM and SIB bytes. */
5405 if (ctxt->d & ModRM) {
5406 rc = decode_modrm(ctxt, &ctxt->memop);
5407 if (!has_seg_override) {
5408 has_seg_override = true;
5409 ctxt->seg_override = ctxt->modrm_seg;
5411 } else if (ctxt->d & MemAbs)
5412 rc = decode_abs(ctxt, &ctxt->memop);
5413 if (rc != X86EMUL_CONTINUE)
5416 if (!has_seg_override)
5417 ctxt->seg_override = VCPU_SREG_DS;
5419 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5422 * Decode and fetch the source operand: register, memory
5425 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5426 if (rc != X86EMUL_CONTINUE)
5430 * Decode and fetch the second source operand: register, memory
5433 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5434 if (rc != X86EMUL_CONTINUE)
5437 /* Decode and fetch the destination operand: register or memory. */
5438 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5440 if (ctxt->rip_relative && likely(ctxt->memopp))
5441 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5442 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5445 if (rc == X86EMUL_PROPAGATE_FAULT)
5446 ctxt->have_exception = true;
5447 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5450 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5452 return ctxt->d & PageTable;
5455 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5457 /* The second termination condition only applies for REPE
5458 * and REPNE. Test if the repeat string operation prefix is
5459 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5460 * corresponding termination condition according to:
5461 * - if REPE/REPZ and ZF = 0 then done
5462 * - if REPNE/REPNZ and ZF = 1 then done
5464 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5465 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5466 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5467 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5468 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5469 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5475 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5480 rc = asm_safe("fwait");
5483 if (unlikely(rc != X86EMUL_CONTINUE))
5484 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5486 return X86EMUL_CONTINUE;
5489 static void fetch_possible_mmx_operand(struct operand *op)
5491 if (op->type == OP_MM)
5492 read_mmx_reg(&op->mm_val, op->addr.mm);
5495 static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5497 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5499 if (!(ctxt->d & ByteOp))
5500 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5502 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5503 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5504 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5505 : "c"(ctxt->src2.val));
5507 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5508 if (!fop) /* exception is returned in fop variable */
5509 return emulate_de(ctxt);
5510 return X86EMUL_CONTINUE;
5513 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5515 memset(&ctxt->rip_relative, 0,
5516 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5518 ctxt->io_read.pos = 0;
5519 ctxt->io_read.end = 0;
5520 ctxt->mem_read.end = 0;
5523 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5525 const struct x86_emulate_ops *ops = ctxt->ops;
5526 int rc = X86EMUL_CONTINUE;
5527 int saved_dst_type = ctxt->dst.type;
5528 unsigned emul_flags;
5530 ctxt->mem_read.pos = 0;
5532 /* LOCK prefix is allowed only with some instructions */
5533 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5534 rc = emulate_ud(ctxt);
5538 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5539 rc = emulate_ud(ctxt);
5543 emul_flags = ctxt->ops->get_hflags(ctxt);
5544 if (unlikely(ctxt->d &
5545 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5546 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5547 (ctxt->d & Undefined)) {
5548 rc = emulate_ud(ctxt);
5552 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5553 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5554 rc = emulate_ud(ctxt);
5558 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5559 rc = emulate_nm(ctxt);
5563 if (ctxt->d & Mmx) {
5564 rc = flush_pending_x87_faults(ctxt);
5565 if (rc != X86EMUL_CONTINUE)
5568 * Now that we know the fpu is exception safe, we can fetch
5571 fetch_possible_mmx_operand(&ctxt->src);
5572 fetch_possible_mmx_operand(&ctxt->src2);
5573 if (!(ctxt->d & Mov))
5574 fetch_possible_mmx_operand(&ctxt->dst);
5577 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5578 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5579 X86_ICPT_PRE_EXCEPT);
5580 if (rc != X86EMUL_CONTINUE)
5584 /* Instruction can only be executed in protected mode */
5585 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5586 rc = emulate_ud(ctxt);
5590 /* Privileged instruction can be executed only in CPL=0 */
5591 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5592 if (ctxt->d & PrivUD)
5593 rc = emulate_ud(ctxt);
5595 rc = emulate_gp(ctxt, 0);
5599 /* Do instruction specific permission checks */
5600 if (ctxt->d & CheckPerm) {
5601 rc = ctxt->check_perm(ctxt);
5602 if (rc != X86EMUL_CONTINUE)
5606 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5607 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5608 X86_ICPT_POST_EXCEPT);
5609 if (rc != X86EMUL_CONTINUE)
5613 if (ctxt->rep_prefix && (ctxt->d & String)) {
5614 /* All REP prefixes have the same first termination condition */
5615 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5616 string_registers_quirk(ctxt);
5617 ctxt->eip = ctxt->_eip;
5618 ctxt->eflags &= ~X86_EFLAGS_RF;
5624 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5625 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5626 ctxt->src.valptr, ctxt->src.bytes);
5627 if (rc != X86EMUL_CONTINUE)
5629 ctxt->src.orig_val64 = ctxt->src.val64;
5632 if (ctxt->src2.type == OP_MEM) {
5633 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5634 &ctxt->src2.val, ctxt->src2.bytes);
5635 if (rc != X86EMUL_CONTINUE)
5639 if ((ctxt->d & DstMask) == ImplicitOps)
5643 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5644 /* optimisation - avoid slow emulated read if Mov */
5645 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5646 &ctxt->dst.val, ctxt->dst.bytes);
5647 if (rc != X86EMUL_CONTINUE) {
5648 if (!(ctxt->d & NoWrite) &&
5649 rc == X86EMUL_PROPAGATE_FAULT &&
5650 ctxt->exception.vector == PF_VECTOR)
5651 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5655 /* Copy full 64-bit value for CMPXCHG8B. */
5656 ctxt->dst.orig_val64 = ctxt->dst.val64;
5660 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5661 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5662 X86_ICPT_POST_MEMACCESS);
5663 if (rc != X86EMUL_CONTINUE)
5667 if (ctxt->rep_prefix && (ctxt->d & String))
5668 ctxt->eflags |= X86_EFLAGS_RF;
5670 ctxt->eflags &= ~X86_EFLAGS_RF;
5672 if (ctxt->execute) {
5673 if (ctxt->d & Fastop)
5674 rc = fastop(ctxt, ctxt->fop);
5676 rc = ctxt->execute(ctxt);
5677 if (rc != X86EMUL_CONTINUE)
5682 if (ctxt->opcode_len == 2)
5684 else if (ctxt->opcode_len == 3)
5685 goto threebyte_insn;
5688 case 0x70 ... 0x7f: /* jcc (short) */
5689 if (test_cc(ctxt->b, ctxt->eflags))
5690 rc = jmp_rel(ctxt, ctxt->src.val);
5692 case 0x8d: /* lea r16/r32, m */
5693 ctxt->dst.val = ctxt->src.addr.mem.ea;
5695 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5696 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5697 ctxt->dst.type = OP_NONE;
5701 case 0x98: /* cbw/cwde/cdqe */
5702 switch (ctxt->op_bytes) {
5703 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5704 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5705 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5708 case 0xcc: /* int3 */
5709 rc = emulate_int(ctxt, 3);
5711 case 0xcd: /* int n */
5712 rc = emulate_int(ctxt, ctxt->src.val);
5714 case 0xce: /* into */
5715 if (ctxt->eflags & X86_EFLAGS_OF)
5716 rc = emulate_int(ctxt, 4);
5718 case 0xe9: /* jmp rel */
5719 case 0xeb: /* jmp rel short */
5720 rc = jmp_rel(ctxt, ctxt->src.val);
5721 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5723 case 0xf4: /* hlt */
5724 ctxt->ops->halt(ctxt);
5726 case 0xf5: /* cmc */
5727 /* complement carry flag from eflags reg */
5728 ctxt->eflags ^= X86_EFLAGS_CF;
5730 case 0xf8: /* clc */
5731 ctxt->eflags &= ~X86_EFLAGS_CF;
5733 case 0xf9: /* stc */
5734 ctxt->eflags |= X86_EFLAGS_CF;
5736 case 0xfc: /* cld */
5737 ctxt->eflags &= ~X86_EFLAGS_DF;
5739 case 0xfd: /* std */
5740 ctxt->eflags |= X86_EFLAGS_DF;
5743 goto cannot_emulate;
5746 if (rc != X86EMUL_CONTINUE)
5750 if (ctxt->d & SrcWrite) {
5751 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5752 rc = writeback(ctxt, &ctxt->src);
5753 if (rc != X86EMUL_CONTINUE)
5756 if (!(ctxt->d & NoWrite)) {
5757 rc = writeback(ctxt, &ctxt->dst);
5758 if (rc != X86EMUL_CONTINUE)
5763 * restore dst type in case the decoding will be reused
5764 * (happens for string instruction )
5766 ctxt->dst.type = saved_dst_type;
5768 if ((ctxt->d & SrcMask) == SrcSI)
5769 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5771 if ((ctxt->d & DstMask) == DstDI)
5772 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5774 if (ctxt->rep_prefix && (ctxt->d & String)) {
5776 struct read_cache *r = &ctxt->io_read;
5777 if ((ctxt->d & SrcMask) == SrcSI)
5778 count = ctxt->src.count;
5780 count = ctxt->dst.count;
5781 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5783 if (!string_insn_completed(ctxt)) {
5785 * Re-enter guest when pio read ahead buffer is empty
5786 * or, if it is not used, after each 1024 iteration.
5788 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5789 (r->end == 0 || r->end != r->pos)) {
5791 * Reset read cache. Usually happens before
5792 * decode, but since instruction is restarted
5793 * we have to do it here.
5795 ctxt->mem_read.end = 0;
5796 writeback_registers(ctxt);
5797 return EMULATION_RESTART;
5799 goto done; /* skip rip writeback */
5801 ctxt->eflags &= ~X86_EFLAGS_RF;
5804 ctxt->eip = ctxt->_eip;
5805 if (ctxt->mode != X86EMUL_MODE_PROT64)
5806 ctxt->eip = (u32)ctxt->_eip;
5809 if (rc == X86EMUL_PROPAGATE_FAULT) {
5810 WARN_ON(ctxt->exception.vector > 0x1f);
5811 ctxt->have_exception = true;
5813 if (rc == X86EMUL_INTERCEPTED)
5814 return EMULATION_INTERCEPTED;
5816 if (rc == X86EMUL_CONTINUE)
5817 writeback_registers(ctxt);
5819 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5823 case 0x09: /* wbinvd */
5824 (ctxt->ops->wbinvd)(ctxt);
5826 case 0x08: /* invd */
5827 case 0x0d: /* GrpP (prefetch) */
5828 case 0x18: /* Grp16 (prefetch/nop) */
5829 case 0x1f: /* nop */
5831 case 0x20: /* mov cr, reg */
5832 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5834 case 0x21: /* mov from dr to reg */
5835 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5837 case 0x40 ... 0x4f: /* cmov */
5838 if (test_cc(ctxt->b, ctxt->eflags))
5839 ctxt->dst.val = ctxt->src.val;
5840 else if (ctxt->op_bytes != 4)
5841 ctxt->dst.type = OP_NONE; /* no writeback */
5843 case 0x80 ... 0x8f: /* jnz rel, etc*/
5844 if (test_cc(ctxt->b, ctxt->eflags))
5845 rc = jmp_rel(ctxt, ctxt->src.val);
5847 case 0x90 ... 0x9f: /* setcc r/m8 */
5848 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5850 case 0xb6 ... 0xb7: /* movzx */
5851 ctxt->dst.bytes = ctxt->op_bytes;
5852 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5853 : (u16) ctxt->src.val;
5855 case 0xbe ... 0xbf: /* movsx */
5856 ctxt->dst.bytes = ctxt->op_bytes;
5857 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5858 (s16) ctxt->src.val;
5861 goto cannot_emulate;
5866 if (rc != X86EMUL_CONTINUE)
5872 return EMULATION_FAILED;
5875 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5877 invalidate_registers(ctxt);
5880 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5882 writeback_registers(ctxt);
5885 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5887 if (ctxt->rep_prefix && (ctxt->d & String))
5890 if (ctxt->d & TwoMemOp)