1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
167 #define NoBigReal ((u64)1 << 50) /* No big real mode */
168 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
170 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
172 #define X2(x...) x, x
173 #define X3(x...) X2(x), x
174 #define X4(x...) X2(x), X2(x)
175 #define X5(x...) X4(x), x
176 #define X6(x...) X4(x), X2(x)
177 #define X7(x...) X4(x), X3(x)
178 #define X8(x...) X4(x), X4(x)
179 #define X16(x...) X8(x), X8(x)
181 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
182 #define FASTOP_SIZE 8
185 * fastop functions have a special calling convention:
190 * flags: rflags (in/out)
191 * ex: rsi (in:fastop pointer, out:zero if exception)
193 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
194 * different operand sizes can be reached by calculation, rather than a jump
195 * table (which would be bigger than the code).
197 * fastop functions are declared as taking a never-defined fastop parameter,
198 * so they can't be called from C directly.
207 int (*execute)(struct x86_emulate_ctxt *ctxt);
208 const struct opcode *group;
209 const struct group_dual *gdual;
210 const struct gprefix *gprefix;
211 const struct escape *esc;
212 void (*fastop)(struct fastop *fake);
214 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
218 struct opcode mod012[8];
219 struct opcode mod3[8];
223 struct opcode pfx_no;
224 struct opcode pfx_66;
225 struct opcode pfx_f2;
226 struct opcode pfx_f3;
231 struct opcode high[64];
234 /* EFLAGS bit definitions. */
235 #define EFLG_ID (1<<21)
236 #define EFLG_VIP (1<<20)
237 #define EFLG_VIF (1<<19)
238 #define EFLG_AC (1<<18)
239 #define EFLG_VM (1<<17)
240 #define EFLG_RF (1<<16)
241 #define EFLG_IOPL (3<<12)
242 #define EFLG_NT (1<<14)
243 #define EFLG_OF (1<<11)
244 #define EFLG_DF (1<<10)
245 #define EFLG_IF (1<<9)
246 #define EFLG_TF (1<<8)
247 #define EFLG_SF (1<<7)
248 #define EFLG_ZF (1<<6)
249 #define EFLG_AF (1<<4)
250 #define EFLG_PF (1<<2)
251 #define EFLG_CF (1<<0)
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 #define EFLG_RESERVED_ONE_MASK 2
256 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 if (!(ctxt->regs_valid & (1 << nr))) {
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
262 return ctxt->_regs[nr];
265 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->regs_dirty |= 1 << nr;
269 return &ctxt->_regs[nr];
272 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 return reg_write(ctxt, nr);
278 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
282 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
283 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
286 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
288 ctxt->regs_dirty = 0;
289 ctxt->regs_valid = 0;
293 * These EFLAGS bits are restored from saved value during emulation, and
294 * any changes are written back to the saved value after emulation.
296 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
304 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
306 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
307 #define FOP_RET "ret \n\t"
309 #define FOP_START(op) \
310 extern void em_##op(struct fastop *fake); \
311 asm(".pushsection .text, \"ax\" \n\t" \
312 ".global em_" #op " \n\t" \
319 #define FOPNOP() FOP_ALIGN FOP_RET
321 #define FOP1E(op, dst) \
322 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
324 #define FOP1EEX(op, dst) \
325 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
327 #define FASTOP1(op) \
332 ON64(FOP1E(op##q, rax)) \
335 /* 1-operand, using src2 (for MUL/DIV r/m) */
336 #define FASTOP1SRC2(op, name) \
341 ON64(FOP1E(op, rcx)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
345 #define FASTOP1SRC2EX(op, name) \
350 ON64(FOP1EEX(op, rcx)) \
353 #define FOP2E(op, dst, src) \
354 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
356 #define FASTOP2(op) \
358 FOP2E(op##b, al, dl) \
359 FOP2E(op##w, ax, dx) \
360 FOP2E(op##l, eax, edx) \
361 ON64(FOP2E(op##q, rax, rdx)) \
364 /* 2 operand, word only */
365 #define FASTOP2W(op) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, src is CL */
374 #define FASTOP2CL(op) \
376 FOP2E(op##b, al, cl) \
377 FOP2E(op##w, ax, cl) \
378 FOP2E(op##l, eax, cl) \
379 ON64(FOP2E(op##q, rax, cl)) \
382 #define FOP3E(op, dst, src, src2) \
383 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
385 /* 3-operand, word-only, src2=cl */
386 #define FASTOP3WCL(op) \
389 FOP3E(op##w, ax, dx, cl) \
390 FOP3E(op##l, eax, edx, cl) \
391 ON64(FOP3E(op##q, rax, rdx, cl)) \
394 /* Special case for SETcc - 1 instruction per cc */
395 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
397 asm(".global kvm_fastop_exception \n"
398 "kvm_fastop_exception: xor %esi, %esi; ret");
419 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
422 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
423 enum x86_intercept intercept,
424 enum x86_intercept_stage stage)
426 struct x86_instruction_info info = {
427 .intercept = intercept,
428 .rep_prefix = ctxt->rep_prefix,
429 .modrm_mod = ctxt->modrm_mod,
430 .modrm_reg = ctxt->modrm_reg,
431 .modrm_rm = ctxt->modrm_rm,
432 .src_val = ctxt->src.val64,
433 .dst_val = ctxt->dst.val64,
434 .src_bytes = ctxt->src.bytes,
435 .dst_bytes = ctxt->dst.bytes,
436 .ad_bytes = ctxt->ad_bytes,
437 .next_rip = ctxt->eip,
440 return ctxt->ops->intercept(ctxt, &info, stage);
443 static void assign_masked(ulong *dest, ulong src, ulong mask)
445 *dest = (*dest & ~mask) | (src & mask);
448 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
450 return (1UL << (ctxt->ad_bytes << 3)) - 1;
453 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
456 struct desc_struct ss;
458 if (ctxt->mode == X86EMUL_MODE_PROT64)
460 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
461 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
464 static int stack_size(struct x86_emulate_ctxt *ctxt)
466 return (__fls(stack_mask(ctxt)) + 1) >> 3;
469 /* Access/update address held in a register, based on addressing mode. */
470 static inline unsigned long
471 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
473 if (ctxt->ad_bytes == sizeof(unsigned long))
476 return reg & ad_mask(ctxt);
479 static inline unsigned long
480 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
482 return address_mask(ctxt, reg);
485 static void masked_increment(ulong *reg, ulong mask, int inc)
487 assign_masked(reg, *reg + inc, mask);
491 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
495 if (ctxt->ad_bytes == sizeof(unsigned long))
498 mask = ad_mask(ctxt);
499 masked_increment(reg, mask, inc);
502 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
504 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
507 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
509 register_address_increment(ctxt, &ctxt->_eip, rel);
512 static u32 desc_limit_scaled(struct desc_struct *desc)
514 u32 limit = get_desc_limit(desc);
516 return desc->g ? (limit << 12) | 0xfff : limit;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
528 u32 error, bool valid)
530 ctxt->exception.vector = vec;
531 ctxt->exception.error_code = error;
532 ctxt->exception.error_code_valid = valid;
533 return X86EMUL_PROPAGATE_FAULT;
536 static int emulate_db(struct x86_emulate_ctxt *ctxt)
538 return emulate_exception(ctxt, DB_VECTOR, 0, false);
541 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
543 return emulate_exception(ctxt, GP_VECTOR, err, true);
546 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
548 return emulate_exception(ctxt, SS_VECTOR, err, true);
551 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
553 return emulate_exception(ctxt, UD_VECTOR, 0, false);
556 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
558 return emulate_exception(ctxt, TS_VECTOR, err, true);
561 static int emulate_de(struct x86_emulate_ctxt *ctxt)
563 return emulate_exception(ctxt, DE_VECTOR, 0, false);
566 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
568 return emulate_exception(ctxt, NM_VECTOR, 0, false);
571 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
574 struct desc_struct desc;
576 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
580 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
585 struct desc_struct desc;
587 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
588 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
592 * x86 defines three classes of vector instructions: explicitly
593 * aligned, explicitly unaligned, and the rest, which change behaviour
594 * depending on whether they're AVX encoded or not.
596 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
597 * subject to the same check.
599 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
601 if (likely(size < 16))
604 if (ctxt->d & Aligned)
606 else if (ctxt->d & Unaligned)
608 else if (ctxt->d & Avx)
614 static int __linearize(struct x86_emulate_ctxt *ctxt,
615 struct segmented_address addr,
616 unsigned size, bool write, bool fetch,
619 struct desc_struct desc;
626 la = seg_base(ctxt, addr.seg) + addr.ea;
627 switch (ctxt->mode) {
628 case X86EMUL_MODE_PROT64:
629 if (((signed long)la << 16) >> 16 != la)
630 return emulate_gp(ctxt, 0);
633 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
637 /* code segment in protected mode or read-only data segment */
638 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
639 || !(desc.type & 2)) && write)
641 /* unreadable code segment */
642 if (!fetch && (desc.type & 8) && !(desc.type & 2))
644 lim = desc_limit_scaled(&desc);
645 if ((ctxt->mode == X86EMUL_MODE_REAL) && !fetch &&
646 (ctxt->d & NoBigReal)) {
647 /* la is between zero and 0xffff */
648 if (la > 0xffff || (u32)(la + size - 1) > 0xffff)
650 } else if ((desc.type & 8) || !(desc.type & 4)) {
651 /* expand-up segment */
652 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
655 /* expand-down segment */
656 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
658 lim = desc.d ? 0xffffffff : 0xffff;
659 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
662 cpl = ctxt->ops->cpl(ctxt);
663 if (!(desc.type & 8)) {
667 } else if ((desc.type & 8) && !(desc.type & 4)) {
668 /* nonconforming code segment */
671 } else if ((desc.type & 8) && (desc.type & 4)) {
672 /* conforming code segment */
678 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
680 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
681 return emulate_gp(ctxt, 0);
683 return X86EMUL_CONTINUE;
685 if (addr.seg == VCPU_SREG_SS)
686 return emulate_ss(ctxt, sel);
688 return emulate_gp(ctxt, sel);
691 static int linearize(struct x86_emulate_ctxt *ctxt,
692 struct segmented_address addr,
693 unsigned size, bool write,
696 return __linearize(ctxt, addr, size, write, false, linear);
700 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
701 struct segmented_address addr,
708 rc = linearize(ctxt, addr, size, false, &linear);
709 if (rc != X86EMUL_CONTINUE)
711 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
715 * Prefetch the remaining bytes of the instruction without crossing page
716 * boundary if they are not in fetch_cache yet.
718 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
722 unsigned long linear;
723 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
724 struct segmented_address addr = { .seg = VCPU_SREG_CS,
725 .ea = ctxt->eip + cur_size };
727 size = 15UL ^ cur_size;
728 rc = __linearize(ctxt, addr, size, false, true, &linear);
729 if (unlikely(rc != X86EMUL_CONTINUE))
732 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
735 * One instruction can only straddle two pages,
736 * and one has been loaded at the beginning of
737 * x86_decode_insn. So, if not enough bytes
738 * still, we must have hit the 15-byte boundary.
740 if (unlikely(size < op_size))
741 return X86EMUL_UNHANDLEABLE;
742 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
743 size, &ctxt->exception);
744 if (unlikely(rc != X86EMUL_CONTINUE))
746 ctxt->fetch.end += size;
747 return X86EMUL_CONTINUE;
750 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
753 if (unlikely(ctxt->fetch.end - ctxt->fetch.ptr < size))
754 return __do_insn_fetch_bytes(ctxt, size);
756 return X86EMUL_CONTINUE;
759 /* Fetch next part of the instruction being emulated. */
760 #define insn_fetch(_type, _ctxt) \
763 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
764 if (rc != X86EMUL_CONTINUE) \
766 ctxt->_eip += sizeof(_type); \
767 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
768 ctxt->fetch.ptr += sizeof(_type); \
772 #define insn_fetch_arr(_arr, _size, _ctxt) \
774 rc = do_insn_fetch_bytes(_ctxt, _size); \
775 if (rc != X86EMUL_CONTINUE) \
777 ctxt->_eip += (_size); \
778 memcpy(_arr, ctxt->fetch.ptr, _size); \
779 ctxt->fetch.ptr += (_size); \
783 * Given the 'reg' portion of a ModRM byte, and a register block, return a
784 * pointer into the block that addresses the relevant register.
785 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
787 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
791 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
793 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
794 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
796 p = reg_rmw(ctxt, modrm_reg);
800 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
801 struct segmented_address addr,
802 u16 *size, unsigned long *address, int op_bytes)
809 rc = segmented_read_std(ctxt, addr, size, 2);
810 if (rc != X86EMUL_CONTINUE)
813 rc = segmented_read_std(ctxt, addr, address, op_bytes);
827 FASTOP1SRC2(mul, mul_ex);
828 FASTOP1SRC2(imul, imul_ex);
829 FASTOP1SRC2EX(div, div_ex);
830 FASTOP1SRC2EX(idiv, idiv_ex);
859 static u8 test_cc(unsigned int condition, unsigned long flags)
862 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
864 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
865 asm("push %[flags]; popf; call *%[fastop]"
866 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
870 static void fetch_register_operand(struct operand *op)
874 op->val = *(u8 *)op->addr.reg;
877 op->val = *(u16 *)op->addr.reg;
880 op->val = *(u32 *)op->addr.reg;
883 op->val = *(u64 *)op->addr.reg;
888 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
890 ctxt->ops->get_fpu(ctxt);
892 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
893 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
894 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
895 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
896 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
897 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
898 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
899 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
901 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
902 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
903 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
904 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
905 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
906 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
907 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
908 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
912 ctxt->ops->put_fpu(ctxt);
915 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
918 ctxt->ops->get_fpu(ctxt);
920 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
921 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
922 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
923 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
924 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
925 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
926 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
927 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
929 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
930 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
931 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
932 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
933 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
934 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
935 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
936 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
940 ctxt->ops->put_fpu(ctxt);
943 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
945 ctxt->ops->get_fpu(ctxt);
947 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
948 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
949 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
950 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
951 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
952 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
953 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
954 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
957 ctxt->ops->put_fpu(ctxt);
960 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
962 ctxt->ops->get_fpu(ctxt);
964 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
965 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
966 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
967 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
968 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
969 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
970 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
971 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
974 ctxt->ops->put_fpu(ctxt);
977 static int em_fninit(struct x86_emulate_ctxt *ctxt)
979 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
980 return emulate_nm(ctxt);
982 ctxt->ops->get_fpu(ctxt);
983 asm volatile("fninit");
984 ctxt->ops->put_fpu(ctxt);
985 return X86EMUL_CONTINUE;
988 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
992 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
993 return emulate_nm(ctxt);
995 ctxt->ops->get_fpu(ctxt);
996 asm volatile("fnstcw %0": "+m"(fcw));
997 ctxt->ops->put_fpu(ctxt);
999 /* force 2 byte destination */
1000 ctxt->dst.bytes = 2;
1001 ctxt->dst.val = fcw;
1003 return X86EMUL_CONTINUE;
1006 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1010 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1011 return emulate_nm(ctxt);
1013 ctxt->ops->get_fpu(ctxt);
1014 asm volatile("fnstsw %0": "+m"(fsw));
1015 ctxt->ops->put_fpu(ctxt);
1017 /* force 2 byte destination */
1018 ctxt->dst.bytes = 2;
1019 ctxt->dst.val = fsw;
1021 return X86EMUL_CONTINUE;
1024 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1027 unsigned reg = ctxt->modrm_reg;
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1032 if (ctxt->d & Sse) {
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1039 if (ctxt->d & Mmx) {
1048 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1049 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1051 fetch_register_operand(op);
1052 op->orig_val = op->val;
1055 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1057 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1058 ctxt->modrm_seg = VCPU_SREG_SS;
1061 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1065 int index_reg, base_reg, scale;
1066 int rc = X86EMUL_CONTINUE;
1069 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1070 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1071 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1073 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1074 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1075 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1076 ctxt->modrm_seg = VCPU_SREG_DS;
1078 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1080 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1081 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1083 if (ctxt->d & Sse) {
1086 op->addr.xmm = ctxt->modrm_rm;
1087 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1090 if (ctxt->d & Mmx) {
1093 op->addr.mm = ctxt->modrm_rm & 7;
1096 fetch_register_operand(op);
1102 if (ctxt->ad_bytes == 2) {
1103 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1104 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1105 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1106 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1108 /* 16-bit ModR/M decode. */
1109 switch (ctxt->modrm_mod) {
1111 if (ctxt->modrm_rm == 6)
1112 modrm_ea += insn_fetch(u16, ctxt);
1115 modrm_ea += insn_fetch(s8, ctxt);
1118 modrm_ea += insn_fetch(u16, ctxt);
1121 switch (ctxt->modrm_rm) {
1123 modrm_ea += bx + si;
1126 modrm_ea += bx + di;
1129 modrm_ea += bp + si;
1132 modrm_ea += bp + di;
1141 if (ctxt->modrm_mod != 0)
1148 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1149 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1150 ctxt->modrm_seg = VCPU_SREG_SS;
1151 modrm_ea = (u16)modrm_ea;
1153 /* 32/64-bit ModR/M decode. */
1154 if ((ctxt->modrm_rm & 7) == 4) {
1155 sib = insn_fetch(u8, ctxt);
1156 index_reg |= (sib >> 3) & 7;
1157 base_reg |= sib & 7;
1160 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1161 modrm_ea += insn_fetch(s32, ctxt);
1163 modrm_ea += reg_read(ctxt, base_reg);
1164 adjust_modrm_seg(ctxt, base_reg);
1167 modrm_ea += reg_read(ctxt, index_reg) << scale;
1168 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1169 if (ctxt->mode == X86EMUL_MODE_PROT64)
1170 ctxt->rip_relative = 1;
1172 base_reg = ctxt->modrm_rm;
1173 modrm_ea += reg_read(ctxt, base_reg);
1174 adjust_modrm_seg(ctxt, base_reg);
1176 switch (ctxt->modrm_mod) {
1178 if (ctxt->modrm_rm == 5)
1179 modrm_ea += insn_fetch(s32, ctxt);
1182 modrm_ea += insn_fetch(s8, ctxt);
1185 modrm_ea += insn_fetch(s32, ctxt);
1189 op->addr.mem.ea = modrm_ea;
1190 if (ctxt->ad_bytes != 8)
1191 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1197 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1200 int rc = X86EMUL_CONTINUE;
1203 switch (ctxt->ad_bytes) {
1205 op->addr.mem.ea = insn_fetch(u16, ctxt);
1208 op->addr.mem.ea = insn_fetch(u32, ctxt);
1211 op->addr.mem.ea = insn_fetch(u64, ctxt);
1218 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1222 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1223 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1225 if (ctxt->src.bytes == 2)
1226 sv = (s16)ctxt->src.val & (s16)mask;
1227 else if (ctxt->src.bytes == 4)
1228 sv = (s32)ctxt->src.val & (s32)mask;
1230 sv = (s64)ctxt->src.val & (s64)mask;
1232 ctxt->dst.addr.mem.ea += (sv >> 3);
1235 /* only subword offset */
1236 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1239 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1240 unsigned long addr, void *dest, unsigned size)
1243 struct read_cache *mc = &ctxt->mem_read;
1245 if (mc->pos < mc->end)
1248 WARN_ON((mc->end + size) >= sizeof(mc->data));
1250 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1252 if (rc != X86EMUL_CONTINUE)
1258 memcpy(dest, mc->data + mc->pos, size);
1260 return X86EMUL_CONTINUE;
1263 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1264 struct segmented_address addr,
1271 rc = linearize(ctxt, addr, size, false, &linear);
1272 if (rc != X86EMUL_CONTINUE)
1274 return read_emulated(ctxt, linear, data, size);
1277 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1278 struct segmented_address addr,
1285 rc = linearize(ctxt, addr, size, true, &linear);
1286 if (rc != X86EMUL_CONTINUE)
1288 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1292 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1293 struct segmented_address addr,
1294 const void *orig_data, const void *data,
1300 rc = linearize(ctxt, addr, size, true, &linear);
1301 if (rc != X86EMUL_CONTINUE)
1303 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1304 size, &ctxt->exception);
1307 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1308 unsigned int size, unsigned short port,
1311 struct read_cache *rc = &ctxt->io_read;
1313 if (rc->pos == rc->end) { /* refill pio read ahead */
1314 unsigned int in_page, n;
1315 unsigned int count = ctxt->rep_prefix ?
1316 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1317 in_page = (ctxt->eflags & EFLG_DF) ?
1318 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1319 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1320 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1323 rc->pos = rc->end = 0;
1324 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1329 if (ctxt->rep_prefix && (ctxt->d & String) &&
1330 !(ctxt->eflags & EFLG_DF)) {
1331 ctxt->dst.data = rc->data + rc->pos;
1332 ctxt->dst.type = OP_MEM_STR;
1333 ctxt->dst.count = (rc->end - rc->pos) / size;
1336 memcpy(dest, rc->data + rc->pos, size);
1342 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1343 u16 index, struct desc_struct *desc)
1348 ctxt->ops->get_idt(ctxt, &dt);
1350 if (dt.size < index * 8 + 7)
1351 return emulate_gp(ctxt, index << 3 | 0x2);
1353 addr = dt.address + index * 8;
1354 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1358 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1359 u16 selector, struct desc_ptr *dt)
1361 const struct x86_emulate_ops *ops = ctxt->ops;
1364 if (selector & 1 << 2) {
1365 struct desc_struct desc;
1368 memset (dt, 0, sizeof *dt);
1369 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1373 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1374 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1376 ops->get_gdt(ctxt, dt);
1379 /* allowed just for 8 bytes segments */
1380 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1381 u16 selector, struct desc_struct *desc,
1385 u16 index = selector >> 3;
1388 get_descriptor_table_ptr(ctxt, selector, &dt);
1390 if (dt.size < index * 8 + 7)
1391 return emulate_gp(ctxt, selector & 0xfffc);
1393 *desc_addr_p = addr = dt.address + index * 8;
1394 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1398 /* allowed just for 8 bytes segments */
1399 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1400 u16 selector, struct desc_struct *desc)
1403 u16 index = selector >> 3;
1406 get_descriptor_table_ptr(ctxt, selector, &dt);
1408 if (dt.size < index * 8 + 7)
1409 return emulate_gp(ctxt, selector & 0xfffc);
1411 addr = dt.address + index * 8;
1412 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1416 /* Does not support long mode */
1417 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1418 u16 selector, int seg, u8 cpl, bool in_task_switch)
1420 struct desc_struct seg_desc, old_desc;
1422 unsigned err_vec = GP_VECTOR;
1424 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1430 memset(&seg_desc, 0, sizeof seg_desc);
1432 if (ctxt->mode == X86EMUL_MODE_REAL) {
1433 /* set real mode segment descriptor (keep limit etc. for
1435 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1436 set_desc_base(&seg_desc, selector << 4);
1438 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1439 /* VM86 needs a clean new segment descriptor */
1440 set_desc_base(&seg_desc, selector << 4);
1441 set_desc_limit(&seg_desc, 0xffff);
1451 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1452 if ((seg == VCPU_SREG_CS
1453 || (seg == VCPU_SREG_SS
1454 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1455 || seg == VCPU_SREG_TR)
1459 /* TR should be in GDT only */
1460 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1463 if (null_selector) /* for NULL selector skip all following checks */
1466 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1467 if (ret != X86EMUL_CONTINUE)
1470 err_code = selector & 0xfffc;
1471 err_vec = GP_VECTOR;
1473 /* can't load system descriptor into segment selector */
1474 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1478 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1487 * segment is not a writable data segment or segment
1488 * selector's RPL != CPL or segment selector's RPL != CPL
1490 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1494 if (in_task_switch && rpl != dpl)
1497 if (!(seg_desc.type & 8))
1500 if (seg_desc.type & 4) {
1506 if (rpl > cpl || dpl != cpl)
1509 /* CS(RPL) <- CPL */
1510 selector = (selector & 0xfffc) | cpl;
1513 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1515 old_desc = seg_desc;
1516 seg_desc.type |= 2; /* busy */
1517 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1518 sizeof(seg_desc), &ctxt->exception);
1519 if (ret != X86EMUL_CONTINUE)
1522 case VCPU_SREG_LDTR:
1523 if (seg_desc.s || seg_desc.type != 2)
1526 default: /* DS, ES, FS, or GS */
1528 * segment is not a data or readable code segment or
1529 * ((segment is a data or nonconforming code segment)
1530 * and (both RPL and CPL > DPL))
1532 if ((seg_desc.type & 0xa) == 0x8 ||
1533 (((seg_desc.type & 0xc) != 0xc) &&
1534 (rpl > dpl && cpl > dpl)))
1540 /* mark segment as accessed */
1542 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1543 if (ret != X86EMUL_CONTINUE)
1545 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1546 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1547 sizeof(base3), &ctxt->exception);
1548 if (ret != X86EMUL_CONTINUE)
1552 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1553 return X86EMUL_CONTINUE;
1555 emulate_exception(ctxt, err_vec, err_code, true);
1556 return X86EMUL_PROPAGATE_FAULT;
1559 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1560 u16 selector, int seg)
1562 u8 cpl = ctxt->ops->cpl(ctxt);
1563 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1566 static void write_register_operand(struct operand *op)
1568 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1569 switch (op->bytes) {
1571 *(u8 *)op->addr.reg = (u8)op->val;
1574 *(u16 *)op->addr.reg = (u16)op->val;
1577 *op->addr.reg = (u32)op->val;
1578 break; /* 64b: zero-extend */
1580 *op->addr.reg = op->val;
1585 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1589 write_register_operand(op);
1592 if (ctxt->lock_prefix)
1593 return segmented_cmpxchg(ctxt,
1599 return segmented_write(ctxt,
1605 return segmented_write(ctxt,
1608 op->bytes * op->count);
1611 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1614 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1622 return X86EMUL_CONTINUE;
1625 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1627 struct segmented_address addr;
1629 rsp_increment(ctxt, -bytes);
1630 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1631 addr.seg = VCPU_SREG_SS;
1633 return segmented_write(ctxt, addr, data, bytes);
1636 static int em_push(struct x86_emulate_ctxt *ctxt)
1638 /* Disable writeback. */
1639 ctxt->dst.type = OP_NONE;
1640 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1643 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1644 void *dest, int len)
1647 struct segmented_address addr;
1649 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1650 addr.seg = VCPU_SREG_SS;
1651 rc = segmented_read(ctxt, addr, dest, len);
1652 if (rc != X86EMUL_CONTINUE)
1655 rsp_increment(ctxt, len);
1659 static int em_pop(struct x86_emulate_ctxt *ctxt)
1661 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1664 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1665 void *dest, int len)
1668 unsigned long val, change_mask;
1669 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1670 int cpl = ctxt->ops->cpl(ctxt);
1672 rc = emulate_pop(ctxt, &val, len);
1673 if (rc != X86EMUL_CONTINUE)
1676 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1677 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1679 switch(ctxt->mode) {
1680 case X86EMUL_MODE_PROT64:
1681 case X86EMUL_MODE_PROT32:
1682 case X86EMUL_MODE_PROT16:
1684 change_mask |= EFLG_IOPL;
1686 change_mask |= EFLG_IF;
1688 case X86EMUL_MODE_VM86:
1690 return emulate_gp(ctxt, 0);
1691 change_mask |= EFLG_IF;
1693 default: /* real mode */
1694 change_mask |= (EFLG_IOPL | EFLG_IF);
1698 *(unsigned long *)dest =
1699 (ctxt->eflags & ~change_mask) | (val & change_mask);
1704 static int em_popf(struct x86_emulate_ctxt *ctxt)
1706 ctxt->dst.type = OP_REG;
1707 ctxt->dst.addr.reg = &ctxt->eflags;
1708 ctxt->dst.bytes = ctxt->op_bytes;
1709 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1712 static int em_enter(struct x86_emulate_ctxt *ctxt)
1715 unsigned frame_size = ctxt->src.val;
1716 unsigned nesting_level = ctxt->src2.val & 31;
1720 return X86EMUL_UNHANDLEABLE;
1722 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1723 rc = push(ctxt, &rbp, stack_size(ctxt));
1724 if (rc != X86EMUL_CONTINUE)
1726 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1728 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1729 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1731 return X86EMUL_CONTINUE;
1734 static int em_leave(struct x86_emulate_ctxt *ctxt)
1736 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1738 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1741 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1743 int seg = ctxt->src2.val;
1745 ctxt->src.val = get_segment_selector(ctxt, seg);
1747 return em_push(ctxt);
1750 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1752 int seg = ctxt->src2.val;
1753 unsigned long selector;
1756 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1757 if (rc != X86EMUL_CONTINUE)
1760 if (ctxt->modrm_reg == VCPU_SREG_SS)
1761 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1763 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1767 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1769 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1770 int rc = X86EMUL_CONTINUE;
1771 int reg = VCPU_REGS_RAX;
1773 while (reg <= VCPU_REGS_RDI) {
1774 (reg == VCPU_REGS_RSP) ?
1775 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1778 if (rc != X86EMUL_CONTINUE)
1787 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1789 ctxt->src.val = (unsigned long)ctxt->eflags;
1790 return em_push(ctxt);
1793 static int em_popa(struct x86_emulate_ctxt *ctxt)
1795 int rc = X86EMUL_CONTINUE;
1796 int reg = VCPU_REGS_RDI;
1798 while (reg >= VCPU_REGS_RAX) {
1799 if (reg == VCPU_REGS_RSP) {
1800 rsp_increment(ctxt, ctxt->op_bytes);
1804 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1805 if (rc != X86EMUL_CONTINUE)
1812 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1814 const struct x86_emulate_ops *ops = ctxt->ops;
1821 /* TODO: Add limit checks */
1822 ctxt->src.val = ctxt->eflags;
1824 if (rc != X86EMUL_CONTINUE)
1827 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1829 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1831 if (rc != X86EMUL_CONTINUE)
1834 ctxt->src.val = ctxt->_eip;
1836 if (rc != X86EMUL_CONTINUE)
1839 ops->get_idt(ctxt, &dt);
1841 eip_addr = dt.address + (irq << 2);
1842 cs_addr = dt.address + (irq << 2) + 2;
1844 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1845 if (rc != X86EMUL_CONTINUE)
1848 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1849 if (rc != X86EMUL_CONTINUE)
1852 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1853 if (rc != X86EMUL_CONTINUE)
1861 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1865 invalidate_registers(ctxt);
1866 rc = __emulate_int_real(ctxt, irq);
1867 if (rc == X86EMUL_CONTINUE)
1868 writeback_registers(ctxt);
1872 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1874 switch(ctxt->mode) {
1875 case X86EMUL_MODE_REAL:
1876 return __emulate_int_real(ctxt, irq);
1877 case X86EMUL_MODE_VM86:
1878 case X86EMUL_MODE_PROT16:
1879 case X86EMUL_MODE_PROT32:
1880 case X86EMUL_MODE_PROT64:
1882 /* Protected mode interrupts unimplemented yet */
1883 return X86EMUL_UNHANDLEABLE;
1887 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1889 int rc = X86EMUL_CONTINUE;
1890 unsigned long temp_eip = 0;
1891 unsigned long temp_eflags = 0;
1892 unsigned long cs = 0;
1893 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1894 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1895 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1896 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1898 /* TODO: Add stack limit check */
1900 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1902 if (rc != X86EMUL_CONTINUE)
1905 if (temp_eip & ~0xffff)
1906 return emulate_gp(ctxt, 0);
1908 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1910 if (rc != X86EMUL_CONTINUE)
1913 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1915 if (rc != X86EMUL_CONTINUE)
1918 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1920 if (rc != X86EMUL_CONTINUE)
1923 ctxt->_eip = temp_eip;
1926 if (ctxt->op_bytes == 4)
1927 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1928 else if (ctxt->op_bytes == 2) {
1929 ctxt->eflags &= ~0xffff;
1930 ctxt->eflags |= temp_eflags;
1933 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1934 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1939 static int em_iret(struct x86_emulate_ctxt *ctxt)
1941 switch(ctxt->mode) {
1942 case X86EMUL_MODE_REAL:
1943 return emulate_iret_real(ctxt);
1944 case X86EMUL_MODE_VM86:
1945 case X86EMUL_MODE_PROT16:
1946 case X86EMUL_MODE_PROT32:
1947 case X86EMUL_MODE_PROT64:
1949 /* iret from protected mode unimplemented yet */
1950 return X86EMUL_UNHANDLEABLE;
1954 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1959 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1961 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1962 if (rc != X86EMUL_CONTINUE)
1966 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1967 return X86EMUL_CONTINUE;
1970 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1972 int rc = X86EMUL_CONTINUE;
1974 switch (ctxt->modrm_reg) {
1975 case 2: /* call near abs */ {
1977 old_eip = ctxt->_eip;
1978 ctxt->_eip = ctxt->src.val;
1979 ctxt->src.val = old_eip;
1983 case 4: /* jmp abs */
1984 ctxt->_eip = ctxt->src.val;
1986 case 5: /* jmp far */
1987 rc = em_jmp_far(ctxt);
1996 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1998 u64 old = ctxt->dst.orig_val64;
2000 if (ctxt->dst.bytes == 16)
2001 return X86EMUL_UNHANDLEABLE;
2003 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2004 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2005 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2006 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2007 ctxt->eflags &= ~EFLG_ZF;
2009 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2010 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2012 ctxt->eflags |= EFLG_ZF;
2014 return X86EMUL_CONTINUE;
2017 static int em_ret(struct x86_emulate_ctxt *ctxt)
2019 ctxt->dst.type = OP_REG;
2020 ctxt->dst.addr.reg = &ctxt->_eip;
2021 ctxt->dst.bytes = ctxt->op_bytes;
2022 return em_pop(ctxt);
2025 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2029 int cpl = ctxt->ops->cpl(ctxt);
2031 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2032 if (rc != X86EMUL_CONTINUE)
2034 if (ctxt->op_bytes == 4)
2035 ctxt->_eip = (u32)ctxt->_eip;
2036 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2037 if (rc != X86EMUL_CONTINUE)
2039 /* Outer-privilege level return is not implemented */
2040 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2041 return X86EMUL_UNHANDLEABLE;
2042 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2046 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2050 rc = em_ret_far(ctxt);
2051 if (rc != X86EMUL_CONTINUE)
2053 rsp_increment(ctxt, ctxt->src.val);
2054 return X86EMUL_CONTINUE;
2057 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2059 /* Save real source value, then compare EAX against destination. */
2060 ctxt->dst.orig_val = ctxt->dst.val;
2061 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2062 ctxt->src.orig_val = ctxt->src.val;
2063 ctxt->src.val = ctxt->dst.orig_val;
2064 fastop(ctxt, em_cmp);
2066 if (ctxt->eflags & EFLG_ZF) {
2067 /* Success: write back to memory. */
2068 ctxt->dst.val = ctxt->src.orig_val;
2070 /* Failure: write the value we saw to EAX. */
2071 ctxt->dst.type = OP_REG;
2072 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2073 ctxt->dst.val = ctxt->dst.orig_val;
2075 return X86EMUL_CONTINUE;
2078 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2080 int seg = ctxt->src2.val;
2084 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2086 rc = load_segment_descriptor(ctxt, sel, seg);
2087 if (rc != X86EMUL_CONTINUE)
2090 ctxt->dst.val = ctxt->src.val;
2095 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2096 struct desc_struct *cs, struct desc_struct *ss)
2098 cs->l = 0; /* will be adjusted later */
2099 set_desc_base(cs, 0); /* flat segment */
2100 cs->g = 1; /* 4kb granularity */
2101 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2102 cs->type = 0x0b; /* Read, Execute, Accessed */
2104 cs->dpl = 0; /* will be adjusted later */
2109 set_desc_base(ss, 0); /* flat segment */
2110 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2111 ss->g = 1; /* 4kb granularity */
2113 ss->type = 0x03; /* Read/Write, Accessed */
2114 ss->d = 1; /* 32bit stack segment */
2121 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2123 u32 eax, ebx, ecx, edx;
2126 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2127 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2128 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2129 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2132 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2134 const struct x86_emulate_ops *ops = ctxt->ops;
2135 u32 eax, ebx, ecx, edx;
2138 * syscall should always be enabled in longmode - so only become
2139 * vendor specific (cpuid) if other modes are active...
2141 if (ctxt->mode == X86EMUL_MODE_PROT64)
2146 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2148 * Intel ("GenuineIntel")
2149 * remark: Intel CPUs only support "syscall" in 64bit
2150 * longmode. Also an 64bit guest with a
2151 * 32bit compat-app running will #UD !! While this
2152 * behaviour can be fixed (by emulating) into AMD
2153 * response - CPUs of AMD can't behave like Intel.
2155 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2156 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2157 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2160 /* AMD ("AuthenticAMD") */
2161 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2162 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2163 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2166 /* AMD ("AMDisbetter!") */
2167 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2168 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2169 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2172 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2176 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2178 const struct x86_emulate_ops *ops = ctxt->ops;
2179 struct desc_struct cs, ss;
2184 /* syscall is not available in real mode */
2185 if (ctxt->mode == X86EMUL_MODE_REAL ||
2186 ctxt->mode == X86EMUL_MODE_VM86)
2187 return emulate_ud(ctxt);
2189 if (!(em_syscall_is_enabled(ctxt)))
2190 return emulate_ud(ctxt);
2192 ops->get_msr(ctxt, MSR_EFER, &efer);
2193 setup_syscalls_segments(ctxt, &cs, &ss);
2195 if (!(efer & EFER_SCE))
2196 return emulate_ud(ctxt);
2198 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2200 cs_sel = (u16)(msr_data & 0xfffc);
2201 ss_sel = (u16)(msr_data + 8);
2203 if (efer & EFER_LMA) {
2207 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2208 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2210 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2211 if (efer & EFER_LMA) {
2212 #ifdef CONFIG_X86_64
2213 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2216 ctxt->mode == X86EMUL_MODE_PROT64 ?
2217 MSR_LSTAR : MSR_CSTAR, &msr_data);
2218 ctxt->_eip = msr_data;
2220 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2221 ctxt->eflags &= ~msr_data;
2225 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2226 ctxt->_eip = (u32)msr_data;
2228 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2231 return X86EMUL_CONTINUE;
2234 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2236 const struct x86_emulate_ops *ops = ctxt->ops;
2237 struct desc_struct cs, ss;
2242 ops->get_msr(ctxt, MSR_EFER, &efer);
2243 /* inject #GP if in real mode */
2244 if (ctxt->mode == X86EMUL_MODE_REAL)
2245 return emulate_gp(ctxt, 0);
2248 * Not recognized on AMD in compat mode (but is recognized in legacy
2251 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2252 && !vendor_intel(ctxt))
2253 return emulate_ud(ctxt);
2255 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2256 * Therefore, we inject an #UD.
2258 if (ctxt->mode == X86EMUL_MODE_PROT64)
2259 return emulate_ud(ctxt);
2261 setup_syscalls_segments(ctxt, &cs, &ss);
2263 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2264 switch (ctxt->mode) {
2265 case X86EMUL_MODE_PROT32:
2266 if ((msr_data & 0xfffc) == 0x0)
2267 return emulate_gp(ctxt, 0);
2269 case X86EMUL_MODE_PROT64:
2270 if (msr_data == 0x0)
2271 return emulate_gp(ctxt, 0);
2277 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2278 cs_sel = (u16)msr_data;
2279 cs_sel &= ~SELECTOR_RPL_MASK;
2280 ss_sel = cs_sel + 8;
2281 ss_sel &= ~SELECTOR_RPL_MASK;
2282 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2287 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2288 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2290 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2291 ctxt->_eip = msr_data;
2293 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2294 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2296 return X86EMUL_CONTINUE;
2299 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2301 const struct x86_emulate_ops *ops = ctxt->ops;
2302 struct desc_struct cs, ss;
2305 u16 cs_sel = 0, ss_sel = 0;
2307 /* inject #GP if in real mode or Virtual 8086 mode */
2308 if (ctxt->mode == X86EMUL_MODE_REAL ||
2309 ctxt->mode == X86EMUL_MODE_VM86)
2310 return emulate_gp(ctxt, 0);
2312 setup_syscalls_segments(ctxt, &cs, &ss);
2314 if ((ctxt->rex_prefix & 0x8) != 0x0)
2315 usermode = X86EMUL_MODE_PROT64;
2317 usermode = X86EMUL_MODE_PROT32;
2321 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2323 case X86EMUL_MODE_PROT32:
2324 cs_sel = (u16)(msr_data + 16);
2325 if ((msr_data & 0xfffc) == 0x0)
2326 return emulate_gp(ctxt, 0);
2327 ss_sel = (u16)(msr_data + 24);
2329 case X86EMUL_MODE_PROT64:
2330 cs_sel = (u16)(msr_data + 32);
2331 if (msr_data == 0x0)
2332 return emulate_gp(ctxt, 0);
2333 ss_sel = cs_sel + 8;
2338 cs_sel |= SELECTOR_RPL_MASK;
2339 ss_sel |= SELECTOR_RPL_MASK;
2341 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2342 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2344 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2345 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2347 return X86EMUL_CONTINUE;
2350 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2353 if (ctxt->mode == X86EMUL_MODE_REAL)
2355 if (ctxt->mode == X86EMUL_MODE_VM86)
2357 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2358 return ctxt->ops->cpl(ctxt) > iopl;
2361 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2364 const struct x86_emulate_ops *ops = ctxt->ops;
2365 struct desc_struct tr_seg;
2368 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2369 unsigned mask = (1 << len) - 1;
2372 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2375 if (desc_limit_scaled(&tr_seg) < 103)
2377 base = get_desc_base(&tr_seg);
2378 #ifdef CONFIG_X86_64
2379 base |= ((u64)base3) << 32;
2381 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2382 if (r != X86EMUL_CONTINUE)
2384 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2386 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2387 if (r != X86EMUL_CONTINUE)
2389 if ((perm >> bit_idx) & mask)
2394 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2400 if (emulator_bad_iopl(ctxt))
2401 if (!emulator_io_port_access_allowed(ctxt, port, len))
2404 ctxt->perm_ok = true;
2409 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2410 struct tss_segment_16 *tss)
2412 tss->ip = ctxt->_eip;
2413 tss->flag = ctxt->eflags;
2414 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2415 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2416 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2417 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2418 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2419 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2420 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2421 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2423 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2424 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2425 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2426 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2427 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2430 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2431 struct tss_segment_16 *tss)
2436 ctxt->_eip = tss->ip;
2437 ctxt->eflags = tss->flag | 2;
2438 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2439 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2440 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2441 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2442 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2443 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2444 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2445 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2448 * SDM says that segment selectors are loaded before segment
2451 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2452 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2453 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2454 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2455 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2460 * Now load segment descriptors. If fault happens at this stage
2461 * it is handled in a context of new task
2463 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2464 if (ret != X86EMUL_CONTINUE)
2466 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2467 if (ret != X86EMUL_CONTINUE)
2469 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2470 if (ret != X86EMUL_CONTINUE)
2472 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2473 if (ret != X86EMUL_CONTINUE)
2475 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2476 if (ret != X86EMUL_CONTINUE)
2479 return X86EMUL_CONTINUE;
2482 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2483 u16 tss_selector, u16 old_tss_sel,
2484 ulong old_tss_base, struct desc_struct *new_desc)
2486 const struct x86_emulate_ops *ops = ctxt->ops;
2487 struct tss_segment_16 tss_seg;
2489 u32 new_tss_base = get_desc_base(new_desc);
2491 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2493 if (ret != X86EMUL_CONTINUE)
2494 /* FIXME: need to provide precise fault address */
2497 save_state_to_tss16(ctxt, &tss_seg);
2499 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2501 if (ret != X86EMUL_CONTINUE)
2502 /* FIXME: need to provide precise fault address */
2505 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2507 if (ret != X86EMUL_CONTINUE)
2508 /* FIXME: need to provide precise fault address */
2511 if (old_tss_sel != 0xffff) {
2512 tss_seg.prev_task_link = old_tss_sel;
2514 ret = ops->write_std(ctxt, new_tss_base,
2515 &tss_seg.prev_task_link,
2516 sizeof tss_seg.prev_task_link,
2518 if (ret != X86EMUL_CONTINUE)
2519 /* FIXME: need to provide precise fault address */
2523 return load_state_from_tss16(ctxt, &tss_seg);
2526 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2527 struct tss_segment_32 *tss)
2529 /* CR3 and ldt selector are not saved intentionally */
2530 tss->eip = ctxt->_eip;
2531 tss->eflags = ctxt->eflags;
2532 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2533 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2534 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2535 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2536 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2537 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2538 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2539 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2541 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2542 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2543 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2544 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2545 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2546 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2549 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2550 struct tss_segment_32 *tss)
2555 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2556 return emulate_gp(ctxt, 0);
2557 ctxt->_eip = tss->eip;
2558 ctxt->eflags = tss->eflags | 2;
2560 /* General purpose registers */
2561 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2562 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2563 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2564 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2565 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2566 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2567 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2568 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2571 * SDM says that segment selectors are loaded before segment
2572 * descriptors. This is important because CPL checks will
2575 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2576 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2577 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2578 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2579 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2580 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2581 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2584 * If we're switching between Protected Mode and VM86, we need to make
2585 * sure to update the mode before loading the segment descriptors so
2586 * that the selectors are interpreted correctly.
2588 if (ctxt->eflags & X86_EFLAGS_VM) {
2589 ctxt->mode = X86EMUL_MODE_VM86;
2592 ctxt->mode = X86EMUL_MODE_PROT32;
2597 * Now load segment descriptors. If fault happenes at this stage
2598 * it is handled in a context of new task
2600 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2601 if (ret != X86EMUL_CONTINUE)
2603 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2604 if (ret != X86EMUL_CONTINUE)
2606 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2607 if (ret != X86EMUL_CONTINUE)
2609 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2610 if (ret != X86EMUL_CONTINUE)
2612 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2613 if (ret != X86EMUL_CONTINUE)
2615 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2616 if (ret != X86EMUL_CONTINUE)
2618 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2619 if (ret != X86EMUL_CONTINUE)
2622 return X86EMUL_CONTINUE;
2625 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2626 u16 tss_selector, u16 old_tss_sel,
2627 ulong old_tss_base, struct desc_struct *new_desc)
2629 const struct x86_emulate_ops *ops = ctxt->ops;
2630 struct tss_segment_32 tss_seg;
2632 u32 new_tss_base = get_desc_base(new_desc);
2633 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2634 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2636 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2638 if (ret != X86EMUL_CONTINUE)
2639 /* FIXME: need to provide precise fault address */
2642 save_state_to_tss32(ctxt, &tss_seg);
2644 /* Only GP registers and segment selectors are saved */
2645 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2646 ldt_sel_offset - eip_offset, &ctxt->exception);
2647 if (ret != X86EMUL_CONTINUE)
2648 /* FIXME: need to provide precise fault address */
2651 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2653 if (ret != X86EMUL_CONTINUE)
2654 /* FIXME: need to provide precise fault address */
2657 if (old_tss_sel != 0xffff) {
2658 tss_seg.prev_task_link = old_tss_sel;
2660 ret = ops->write_std(ctxt, new_tss_base,
2661 &tss_seg.prev_task_link,
2662 sizeof tss_seg.prev_task_link,
2664 if (ret != X86EMUL_CONTINUE)
2665 /* FIXME: need to provide precise fault address */
2669 return load_state_from_tss32(ctxt, &tss_seg);
2672 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2673 u16 tss_selector, int idt_index, int reason,
2674 bool has_error_code, u32 error_code)
2676 const struct x86_emulate_ops *ops = ctxt->ops;
2677 struct desc_struct curr_tss_desc, next_tss_desc;
2679 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2680 ulong old_tss_base =
2681 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2685 /* FIXME: old_tss_base == ~0 ? */
2687 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2688 if (ret != X86EMUL_CONTINUE)
2690 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2691 if (ret != X86EMUL_CONTINUE)
2694 /* FIXME: check that next_tss_desc is tss */
2697 * Check privileges. The three cases are task switch caused by...
2699 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2700 * 2. Exception/IRQ/iret: No check is performed
2701 * 3. jmp/call to TSS: Check against DPL of the TSS
2703 if (reason == TASK_SWITCH_GATE) {
2704 if (idt_index != -1) {
2705 /* Software interrupts */
2706 struct desc_struct task_gate_desc;
2709 ret = read_interrupt_descriptor(ctxt, idt_index,
2711 if (ret != X86EMUL_CONTINUE)
2714 dpl = task_gate_desc.dpl;
2715 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2716 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2718 } else if (reason != TASK_SWITCH_IRET) {
2719 int dpl = next_tss_desc.dpl;
2720 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2721 return emulate_gp(ctxt, tss_selector);
2725 desc_limit = desc_limit_scaled(&next_tss_desc);
2726 if (!next_tss_desc.p ||
2727 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2728 desc_limit < 0x2b)) {
2729 emulate_ts(ctxt, tss_selector & 0xfffc);
2730 return X86EMUL_PROPAGATE_FAULT;
2733 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2734 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2735 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2738 if (reason == TASK_SWITCH_IRET)
2739 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2741 /* set back link to prev task only if NT bit is set in eflags
2742 note that old_tss_sel is not used after this point */
2743 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2744 old_tss_sel = 0xffff;
2746 if (next_tss_desc.type & 8)
2747 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2748 old_tss_base, &next_tss_desc);
2750 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2751 old_tss_base, &next_tss_desc);
2752 if (ret != X86EMUL_CONTINUE)
2755 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2756 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2758 if (reason != TASK_SWITCH_IRET) {
2759 next_tss_desc.type |= (1 << 1); /* set busy flag */
2760 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2763 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2764 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2766 if (has_error_code) {
2767 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2768 ctxt->lock_prefix = 0;
2769 ctxt->src.val = (unsigned long) error_code;
2770 ret = em_push(ctxt);
2776 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2777 u16 tss_selector, int idt_index, int reason,
2778 bool has_error_code, u32 error_code)
2782 invalidate_registers(ctxt);
2783 ctxt->_eip = ctxt->eip;
2784 ctxt->dst.type = OP_NONE;
2786 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2787 has_error_code, error_code);
2789 if (rc == X86EMUL_CONTINUE) {
2790 ctxt->eip = ctxt->_eip;
2791 writeback_registers(ctxt);
2794 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2797 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2800 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2802 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2803 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2806 static int em_das(struct x86_emulate_ctxt *ctxt)
2809 bool af, cf, old_cf;
2811 cf = ctxt->eflags & X86_EFLAGS_CF;
2817 af = ctxt->eflags & X86_EFLAGS_AF;
2818 if ((al & 0x0f) > 9 || af) {
2820 cf = old_cf | (al >= 250);
2825 if (old_al > 0x99 || old_cf) {
2831 /* Set PF, ZF, SF */
2832 ctxt->src.type = OP_IMM;
2834 ctxt->src.bytes = 1;
2835 fastop(ctxt, em_or);
2836 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2838 ctxt->eflags |= X86_EFLAGS_CF;
2840 ctxt->eflags |= X86_EFLAGS_AF;
2841 return X86EMUL_CONTINUE;
2844 static int em_aam(struct x86_emulate_ctxt *ctxt)
2848 if (ctxt->src.val == 0)
2849 return emulate_de(ctxt);
2851 al = ctxt->dst.val & 0xff;
2852 ah = al / ctxt->src.val;
2853 al %= ctxt->src.val;
2855 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2857 /* Set PF, ZF, SF */
2858 ctxt->src.type = OP_IMM;
2860 ctxt->src.bytes = 1;
2861 fastop(ctxt, em_or);
2863 return X86EMUL_CONTINUE;
2866 static int em_aad(struct x86_emulate_ctxt *ctxt)
2868 u8 al = ctxt->dst.val & 0xff;
2869 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2871 al = (al + (ah * ctxt->src.val)) & 0xff;
2873 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2875 /* Set PF, ZF, SF */
2876 ctxt->src.type = OP_IMM;
2878 ctxt->src.bytes = 1;
2879 fastop(ctxt, em_or);
2881 return X86EMUL_CONTINUE;
2884 static int em_call(struct x86_emulate_ctxt *ctxt)
2886 long rel = ctxt->src.val;
2888 ctxt->src.val = (unsigned long)ctxt->_eip;
2890 return em_push(ctxt);
2893 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2899 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2900 old_eip = ctxt->_eip;
2902 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2903 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2904 return X86EMUL_CONTINUE;
2907 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2909 ctxt->src.val = old_cs;
2911 if (rc != X86EMUL_CONTINUE)
2914 ctxt->src.val = old_eip;
2915 return em_push(ctxt);
2918 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2922 ctxt->dst.type = OP_REG;
2923 ctxt->dst.addr.reg = &ctxt->_eip;
2924 ctxt->dst.bytes = ctxt->op_bytes;
2925 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2926 if (rc != X86EMUL_CONTINUE)
2928 rsp_increment(ctxt, ctxt->src.val);
2929 return X86EMUL_CONTINUE;
2932 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2934 /* Write back the register source. */
2935 ctxt->src.val = ctxt->dst.val;
2936 write_register_operand(&ctxt->src);
2938 /* Write back the memory destination with implicit LOCK prefix. */
2939 ctxt->dst.val = ctxt->src.orig_val;
2940 ctxt->lock_prefix = 1;
2941 return X86EMUL_CONTINUE;
2944 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2946 ctxt->dst.val = ctxt->src2.val;
2947 return fastop(ctxt, em_imul);
2950 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2952 ctxt->dst.type = OP_REG;
2953 ctxt->dst.bytes = ctxt->src.bytes;
2954 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2955 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2957 return X86EMUL_CONTINUE;
2960 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2964 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2965 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2966 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2967 return X86EMUL_CONTINUE;
2970 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2974 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2975 return emulate_gp(ctxt, 0);
2976 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2977 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2978 return X86EMUL_CONTINUE;
2981 static int em_mov(struct x86_emulate_ctxt *ctxt)
2983 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
2984 return X86EMUL_CONTINUE;
2987 #define FFL(x) bit(X86_FEATURE_##x)
2989 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2991 u32 ebx, ecx, edx, eax = 1;
2995 * Check MOVBE is set in the guest-visible CPUID leaf.
2997 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2998 if (!(ecx & FFL(MOVBE)))
2999 return emulate_ud(ctxt);
3001 switch (ctxt->op_bytes) {
3004 * From MOVBE definition: "...When the operand size is 16 bits,
3005 * the upper word of the destination register remains unchanged
3008 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3009 * rules so we have to do the operation almost per hand.
3011 tmp = (u16)ctxt->src.val;
3012 ctxt->dst.val &= ~0xffffUL;
3013 ctxt->dst.val |= (unsigned long)swab16(tmp);
3016 ctxt->dst.val = swab32((u32)ctxt->src.val);
3019 ctxt->dst.val = swab64(ctxt->src.val);
3022 return X86EMUL_PROPAGATE_FAULT;
3024 return X86EMUL_CONTINUE;
3027 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3029 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3030 return emulate_gp(ctxt, 0);
3032 /* Disable writeback. */
3033 ctxt->dst.type = OP_NONE;
3034 return X86EMUL_CONTINUE;
3037 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3041 if (ctxt->mode == X86EMUL_MODE_PROT64)
3042 val = ctxt->src.val & ~0ULL;
3044 val = ctxt->src.val & ~0U;
3046 /* #UD condition is already handled. */
3047 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3048 return emulate_gp(ctxt, 0);
3050 /* Disable writeback. */
3051 ctxt->dst.type = OP_NONE;
3052 return X86EMUL_CONTINUE;
3055 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3059 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3060 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3061 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3062 return emulate_gp(ctxt, 0);
3064 return X86EMUL_CONTINUE;
3067 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3071 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3072 return emulate_gp(ctxt, 0);
3074 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3075 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3076 return X86EMUL_CONTINUE;
3079 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3081 if (ctxt->modrm_reg > VCPU_SREG_GS)
3082 return emulate_ud(ctxt);
3084 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3085 return X86EMUL_CONTINUE;
3088 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3090 u16 sel = ctxt->src.val;
3092 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3093 return emulate_ud(ctxt);
3095 if (ctxt->modrm_reg == VCPU_SREG_SS)
3096 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3098 /* Disable writeback. */
3099 ctxt->dst.type = OP_NONE;
3100 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3103 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3105 u16 sel = ctxt->src.val;
3107 /* Disable writeback. */
3108 ctxt->dst.type = OP_NONE;
3109 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3112 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3114 u16 sel = ctxt->src.val;
3116 /* Disable writeback. */
3117 ctxt->dst.type = OP_NONE;
3118 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3121 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3126 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3127 if (rc == X86EMUL_CONTINUE)
3128 ctxt->ops->invlpg(ctxt, linear);
3129 /* Disable writeback. */
3130 ctxt->dst.type = OP_NONE;
3131 return X86EMUL_CONTINUE;
3134 static int em_clts(struct x86_emulate_ctxt *ctxt)
3138 cr0 = ctxt->ops->get_cr(ctxt, 0);
3140 ctxt->ops->set_cr(ctxt, 0, cr0);
3141 return X86EMUL_CONTINUE;
3144 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3148 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3149 return X86EMUL_UNHANDLEABLE;
3151 rc = ctxt->ops->fix_hypercall(ctxt);
3152 if (rc != X86EMUL_CONTINUE)
3155 /* Let the processor re-execute the fixed hypercall */
3156 ctxt->_eip = ctxt->eip;
3157 /* Disable writeback. */
3158 ctxt->dst.type = OP_NONE;
3159 return X86EMUL_CONTINUE;
3162 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3163 void (*get)(struct x86_emulate_ctxt *ctxt,
3164 struct desc_ptr *ptr))
3166 struct desc_ptr desc_ptr;
3168 if (ctxt->mode == X86EMUL_MODE_PROT64)
3170 get(ctxt, &desc_ptr);
3171 if (ctxt->op_bytes == 2) {
3173 desc_ptr.address &= 0x00ffffff;
3175 /* Disable writeback. */
3176 ctxt->dst.type = OP_NONE;
3177 return segmented_write(ctxt, ctxt->dst.addr.mem,
3178 &desc_ptr, 2 + ctxt->op_bytes);
3181 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3183 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3186 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3188 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3191 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3193 struct desc_ptr desc_ptr;
3196 if (ctxt->mode == X86EMUL_MODE_PROT64)
3198 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3199 &desc_ptr.size, &desc_ptr.address,
3201 if (rc != X86EMUL_CONTINUE)
3203 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3204 /* Disable writeback. */
3205 ctxt->dst.type = OP_NONE;
3206 return X86EMUL_CONTINUE;
3209 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3213 rc = ctxt->ops->fix_hypercall(ctxt);
3215 /* Disable writeback. */
3216 ctxt->dst.type = OP_NONE;
3220 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3222 struct desc_ptr desc_ptr;
3225 if (ctxt->mode == X86EMUL_MODE_PROT64)
3227 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3228 &desc_ptr.size, &desc_ptr.address,
3230 if (rc != X86EMUL_CONTINUE)
3232 ctxt->ops->set_idt(ctxt, &desc_ptr);
3233 /* Disable writeback. */
3234 ctxt->dst.type = OP_NONE;
3235 return X86EMUL_CONTINUE;
3238 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3240 if (ctxt->dst.type == OP_MEM)
3241 ctxt->dst.bytes = 2;
3242 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3243 return X86EMUL_CONTINUE;
3246 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3248 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3249 | (ctxt->src.val & 0x0f));
3250 ctxt->dst.type = OP_NONE;
3251 return X86EMUL_CONTINUE;
3254 static int em_loop(struct x86_emulate_ctxt *ctxt)
3256 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3257 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3258 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3259 jmp_rel(ctxt, ctxt->src.val);
3261 return X86EMUL_CONTINUE;
3264 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3266 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3267 jmp_rel(ctxt, ctxt->src.val);
3269 return X86EMUL_CONTINUE;
3272 static int em_in(struct x86_emulate_ctxt *ctxt)
3274 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3276 return X86EMUL_IO_NEEDED;
3278 return X86EMUL_CONTINUE;
3281 static int em_out(struct x86_emulate_ctxt *ctxt)
3283 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3285 /* Disable writeback. */
3286 ctxt->dst.type = OP_NONE;
3287 return X86EMUL_CONTINUE;
3290 static int em_cli(struct x86_emulate_ctxt *ctxt)
3292 if (emulator_bad_iopl(ctxt))
3293 return emulate_gp(ctxt, 0);
3295 ctxt->eflags &= ~X86_EFLAGS_IF;
3296 return X86EMUL_CONTINUE;
3299 static int em_sti(struct x86_emulate_ctxt *ctxt)
3301 if (emulator_bad_iopl(ctxt))
3302 return emulate_gp(ctxt, 0);
3304 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3305 ctxt->eflags |= X86_EFLAGS_IF;
3306 return X86EMUL_CONTINUE;
3309 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3311 u32 eax, ebx, ecx, edx;
3313 eax = reg_read(ctxt, VCPU_REGS_RAX);
3314 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3315 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3316 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3317 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3318 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3319 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3320 return X86EMUL_CONTINUE;
3323 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3327 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3328 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3330 ctxt->eflags &= ~0xffUL;
3331 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3332 return X86EMUL_CONTINUE;
3335 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3337 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3338 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3339 return X86EMUL_CONTINUE;
3342 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3344 switch (ctxt->op_bytes) {
3345 #ifdef CONFIG_X86_64
3347 asm("bswap %0" : "+r"(ctxt->dst.val));
3351 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3354 return X86EMUL_CONTINUE;
3357 static bool valid_cr(int nr)
3369 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3371 if (!valid_cr(ctxt->modrm_reg))
3372 return emulate_ud(ctxt);
3374 return X86EMUL_CONTINUE;
3377 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3379 u64 new_val = ctxt->src.val64;
3380 int cr = ctxt->modrm_reg;
3383 static u64 cr_reserved_bits[] = {
3384 0xffffffff00000000ULL,
3385 0, 0, 0, /* CR3 checked later */
3392 return emulate_ud(ctxt);
3394 if (new_val & cr_reserved_bits[cr])
3395 return emulate_gp(ctxt, 0);
3400 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3401 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3402 return emulate_gp(ctxt, 0);
3404 cr4 = ctxt->ops->get_cr(ctxt, 4);
3405 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3407 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3408 !(cr4 & X86_CR4_PAE))
3409 return emulate_gp(ctxt, 0);
3416 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3417 if (efer & EFER_LMA)
3418 rsvd = CR3_L_MODE_RESERVED_BITS;
3421 return emulate_gp(ctxt, 0);
3426 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3428 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3429 return emulate_gp(ctxt, 0);
3435 return X86EMUL_CONTINUE;
3438 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3442 ctxt->ops->get_dr(ctxt, 7, &dr7);
3444 /* Check if DR7.Global_Enable is set */
3445 return dr7 & (1 << 13);
3448 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3450 int dr = ctxt->modrm_reg;
3454 return emulate_ud(ctxt);
3456 cr4 = ctxt->ops->get_cr(ctxt, 4);
3457 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3458 return emulate_ud(ctxt);
3460 if (check_dr7_gd(ctxt))
3461 return emulate_db(ctxt);
3463 return X86EMUL_CONTINUE;
3466 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3468 u64 new_val = ctxt->src.val64;
3469 int dr = ctxt->modrm_reg;
3471 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3472 return emulate_gp(ctxt, 0);
3474 return check_dr_read(ctxt);
3477 static int check_svme(struct x86_emulate_ctxt *ctxt)
3481 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3483 if (!(efer & EFER_SVME))
3484 return emulate_ud(ctxt);
3486 return X86EMUL_CONTINUE;
3489 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3491 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3493 /* Valid physical address? */
3494 if (rax & 0xffff000000000000ULL)
3495 return emulate_gp(ctxt, 0);
3497 return check_svme(ctxt);
3500 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3502 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3504 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3505 return emulate_ud(ctxt);
3507 return X86EMUL_CONTINUE;
3510 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3512 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3513 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3515 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3516 ctxt->ops->check_pmc(ctxt, rcx))
3517 return emulate_gp(ctxt, 0);
3519 return X86EMUL_CONTINUE;
3522 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3524 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3525 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3526 return emulate_gp(ctxt, 0);
3528 return X86EMUL_CONTINUE;
3531 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3533 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3534 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3535 return emulate_gp(ctxt, 0);
3537 return X86EMUL_CONTINUE;
3540 #define D(_y) { .flags = (_y) }
3541 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3542 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3543 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3544 #define N D(NotImpl)
3545 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3546 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3547 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3548 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3549 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3550 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3551 #define II(_f, _e, _i) \
3552 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3553 #define IIP(_f, _e, _i, _p) \
3554 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3555 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3556 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3558 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3559 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3560 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3561 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3562 #define I2bvIP(_f, _e, _i, _p) \
3563 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3565 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3566 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3567 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3569 static const struct opcode group7_rm1[] = {
3570 DI(SrcNone | Priv, monitor),
3571 DI(SrcNone | Priv, mwait),
3575 static const struct opcode group7_rm3[] = {
3576 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3577 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3578 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3579 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3580 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3581 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3582 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3583 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3586 static const struct opcode group7_rm7[] = {
3588 DIP(SrcNone, rdtscp, check_rdtsc),
3592 static const struct opcode group1[] = {
3594 F(Lock | PageTable, em_or),
3597 F(Lock | PageTable, em_and),
3603 static const struct opcode group1A[] = {
3604 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3607 static const struct opcode group2[] = {
3608 F(DstMem | ModRM, em_rol),
3609 F(DstMem | ModRM, em_ror),
3610 F(DstMem | ModRM, em_rcl),
3611 F(DstMem | ModRM, em_rcr),
3612 F(DstMem | ModRM, em_shl),
3613 F(DstMem | ModRM, em_shr),
3614 F(DstMem | ModRM, em_shl),
3615 F(DstMem | ModRM, em_sar),
3618 static const struct opcode group3[] = {
3619 F(DstMem | SrcImm | NoWrite, em_test),
3620 F(DstMem | SrcImm | NoWrite, em_test),
3621 F(DstMem | SrcNone | Lock, em_not),
3622 F(DstMem | SrcNone | Lock, em_neg),
3623 F(DstXacc | Src2Mem, em_mul_ex),
3624 F(DstXacc | Src2Mem, em_imul_ex),
3625 F(DstXacc | Src2Mem, em_div_ex),
3626 F(DstXacc | Src2Mem, em_idiv_ex),
3629 static const struct opcode group4[] = {
3630 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3631 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3635 static const struct opcode group5[] = {
3636 F(DstMem | SrcNone | Lock, em_inc),
3637 F(DstMem | SrcNone | Lock, em_dec),
3638 I(SrcMem | Stack, em_grp45),
3639 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3640 I(SrcMem | Stack, em_grp45),
3641 I(SrcMemFAddr | ImplicitOps, em_grp45),
3642 I(SrcMem | Stack, em_grp45), D(Undefined),
3645 static const struct opcode group6[] = {
3648 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3649 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3653 static const struct group_dual group7 = { {
3654 II(Mov | DstMem, em_sgdt, sgdt),
3655 II(Mov | DstMem, em_sidt, sidt),
3656 II(SrcMem | Priv, em_lgdt, lgdt),
3657 II(SrcMem | Priv, em_lidt, lidt),
3658 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3659 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3660 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3662 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3664 N, EXT(0, group7_rm3),
3665 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3666 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3670 static const struct opcode group8[] = {
3672 F(DstMem | SrcImmByte | NoWrite, em_bt),
3673 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3674 F(DstMem | SrcImmByte | Lock, em_btr),
3675 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3678 static const struct group_dual group9 = { {
3679 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3681 N, N, N, N, N, N, N, N,
3684 static const struct opcode group11[] = {
3685 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3689 static const struct gprefix pfx_0f_6f_0f_7f = {
3690 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3693 static const struct gprefix pfx_vmovntpx = {
3694 I(0, em_mov), N, N, N,
3697 static const struct gprefix pfx_0f_28_0f_29 = {
3698 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3701 static const struct escape escape_d9 = { {
3702 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3705 N, N, N, N, N, N, N, N,
3707 N, N, N, N, N, N, N, N,
3709 N, N, N, N, N, N, N, N,
3711 N, N, N, N, N, N, N, N,
3713 N, N, N, N, N, N, N, N,
3715 N, N, N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3719 N, N, N, N, N, N, N, N,
3722 static const struct escape escape_db = { {
3723 N, N, N, N, N, N, N, N,
3726 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3730 N, N, N, N, N, N, N, N,
3732 N, N, N, N, N, N, N, N,
3734 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3736 N, N, N, N, N, N, N, N,
3738 N, N, N, N, N, N, N, N,
3740 N, N, N, N, N, N, N, N,
3743 static const struct escape escape_dd = { {
3744 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3747 N, N, N, N, N, N, N, N,
3749 N, N, N, N, N, N, N, N,
3751 N, N, N, N, N, N, N, N,
3753 N, N, N, N, N, N, N, N,
3755 N, N, N, N, N, N, N, N,
3757 N, N, N, N, N, N, N, N,
3759 N, N, N, N, N, N, N, N,
3761 N, N, N, N, N, N, N, N,
3764 static const struct opcode opcode_table[256] = {
3766 F6ALU(Lock, em_add),
3767 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3768 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3770 F6ALU(Lock | PageTable, em_or),
3771 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3774 F6ALU(Lock, em_adc),
3775 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3776 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3778 F6ALU(Lock, em_sbb),
3779 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3780 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3782 F6ALU(Lock | PageTable, em_and), N, N,
3784 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3786 F6ALU(Lock, em_xor), N, N,
3788 F6ALU(NoWrite, em_cmp), N, N,
3790 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3792 X8(I(SrcReg | Stack, em_push)),
3794 X8(I(DstReg | Stack, em_pop)),
3796 I(ImplicitOps | Stack | No64, em_pusha),
3797 I(ImplicitOps | Stack | No64, em_popa),
3798 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3801 I(SrcImm | Mov | Stack, em_push),
3802 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3803 I(SrcImmByte | Mov | Stack, em_push),
3804 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3805 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3806 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3810 G(ByteOp | DstMem | SrcImm, group1),
3811 G(DstMem | SrcImm, group1),
3812 G(ByteOp | DstMem | SrcImm | No64, group1),
3813 G(DstMem | SrcImmByte, group1),
3814 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3815 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3817 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3818 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3819 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3820 D(ModRM | SrcMem | NoAccess | DstReg),
3821 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3824 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3826 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3827 I(SrcImmFAddr | No64, em_call_far), N,
3828 II(ImplicitOps | Stack, em_pushf, pushf),
3829 II(ImplicitOps | Stack, em_popf, popf),
3830 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3832 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3833 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3834 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3835 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3837 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3838 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3839 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3840 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3842 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3844 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3846 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3847 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3848 I(ImplicitOps | Stack, em_ret),
3849 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3850 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3851 G(ByteOp, group11), G(0, group11),
3853 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3854 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3855 I(ImplicitOps | Stack, em_ret_far),
3856 D(ImplicitOps), DI(SrcImmByte, intn),
3857 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3859 G(Src2One | ByteOp, group2), G(Src2One, group2),
3860 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3861 I(DstAcc | SrcImmUByte | No64, em_aam),
3862 I(DstAcc | SrcImmUByte | No64, em_aad),
3863 F(DstAcc | ByteOp | No64, em_salc),
3864 I(DstAcc | SrcXLat | ByteOp, em_mov),
3866 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3868 X3(I(SrcImmByte, em_loop)),
3869 I(SrcImmByte, em_jcxz),
3870 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3871 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3873 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3874 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3875 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3876 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3878 N, DI(ImplicitOps, icebp), N, N,
3879 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3880 G(ByteOp, group3), G(0, group3),
3882 D(ImplicitOps), D(ImplicitOps),
3883 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3884 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3887 static const struct opcode twobyte_table[256] = {
3889 G(0, group6), GD(0, &group7), N, N,
3890 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3891 II(ImplicitOps | Priv, em_clts, clts), N,
3892 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3893 N, D(ImplicitOps | ModRM), N, N,
3895 N, N, N, N, N, N, N, N,
3896 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3898 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3899 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3900 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3902 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3905 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3906 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3907 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3910 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3911 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3912 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3913 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3914 I(ImplicitOps | EmulateOnUD, em_sysenter),
3915 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3917 N, N, N, N, N, N, N, N,
3919 X16(D(DstReg | SrcMem | ModRM)),
3921 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3926 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3931 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3935 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3937 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3938 II(ImplicitOps, em_cpuid, cpuid),
3939 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3940 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3941 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3943 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3944 DI(ImplicitOps, rsm),
3945 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3946 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3947 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3948 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3950 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3951 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3952 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3953 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3954 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3955 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3959 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3960 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3961 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3963 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3964 N, D(DstMem | SrcReg | ModRM | Mov),
3965 N, N, N, GD(0, &group9),
3967 X8(I(DstReg, em_bswap)),
3969 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3971 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3976 static const struct gprefix three_byte_0f_38_f0 = {
3977 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3980 static const struct gprefix three_byte_0f_38_f1 = {
3981 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3985 * Insns below are selected by the prefix which indexed by the third opcode
3988 static const struct opcode opcode_map_0f_38[256] = {
3990 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3992 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3994 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3995 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4014 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4018 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4024 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4025 unsigned size, bool sign_extension)
4027 int rc = X86EMUL_CONTINUE;
4031 op->addr.mem.ea = ctxt->_eip;
4032 /* NB. Immediates are sign-extended as necessary. */
4033 switch (op->bytes) {
4035 op->val = insn_fetch(s8, ctxt);
4038 op->val = insn_fetch(s16, ctxt);
4041 op->val = insn_fetch(s32, ctxt);
4044 op->val = insn_fetch(s64, ctxt);
4047 if (!sign_extension) {
4048 switch (op->bytes) {
4056 op->val &= 0xffffffff;
4064 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4067 int rc = X86EMUL_CONTINUE;
4071 decode_register_operand(ctxt, op);
4074 rc = decode_imm(ctxt, op, 1, false);
4077 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4081 if (ctxt->d & BitOp)
4082 fetch_bit_operand(ctxt);
4083 op->orig_val = op->val;
4086 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4090 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4091 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4092 fetch_register_operand(op);
4093 op->orig_val = op->val;
4097 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4098 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4099 fetch_register_operand(op);
4100 op->orig_val = op->val;
4103 if (ctxt->d & ByteOp) {
4108 op->bytes = ctxt->op_bytes;
4109 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4110 fetch_register_operand(op);
4111 op->orig_val = op->val;
4115 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4117 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4118 op->addr.mem.seg = VCPU_SREG_ES;
4125 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4126 fetch_register_operand(op);
4130 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4133 rc = decode_imm(ctxt, op, 1, true);
4140 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4143 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4146 ctxt->memop.bytes = 1;
4147 if (ctxt->memop.type == OP_REG) {
4148 ctxt->memop.addr.reg = decode_register(ctxt,
4149 ctxt->modrm_rm, true);
4150 fetch_register_operand(&ctxt->memop);
4154 ctxt->memop.bytes = 2;
4157 ctxt->memop.bytes = 4;
4160 rc = decode_imm(ctxt, op, 2, false);
4163 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4167 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4169 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4170 op->addr.mem.seg = ctxt->seg_override;
4176 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4178 register_address(ctxt,
4179 reg_read(ctxt, VCPU_REGS_RBX) +
4180 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4181 op->addr.mem.seg = ctxt->seg_override;
4186 op->addr.mem.ea = ctxt->_eip;
4187 op->bytes = ctxt->op_bytes + 2;
4188 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4191 ctxt->memop.bytes = ctxt->op_bytes + 2;
4194 op->val = VCPU_SREG_ES;
4197 op->val = VCPU_SREG_CS;
4200 op->val = VCPU_SREG_SS;
4203 op->val = VCPU_SREG_DS;
4206 op->val = VCPU_SREG_FS;
4209 op->val = VCPU_SREG_GS;
4212 /* Special instructions do their own operand decoding. */
4214 op->type = OP_NONE; /* Disable writeback. */
4222 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4224 int rc = X86EMUL_CONTINUE;
4225 int mode = ctxt->mode;
4226 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4227 bool op_prefix = false;
4228 bool has_seg_override = false;
4229 struct opcode opcode;
4231 ctxt->memop.type = OP_NONE;
4232 ctxt->memopp = NULL;
4233 ctxt->_eip = ctxt->eip;
4234 ctxt->fetch.ptr = ctxt->fetch.data;
4235 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4236 ctxt->opcode_len = 1;
4238 memcpy(ctxt->fetch.data, insn, insn_len);
4240 rc = __do_insn_fetch_bytes(ctxt, 1);
4241 if (rc != X86EMUL_CONTINUE)
4246 case X86EMUL_MODE_REAL:
4247 case X86EMUL_MODE_VM86:
4248 case X86EMUL_MODE_PROT16:
4249 def_op_bytes = def_ad_bytes = 2;
4251 case X86EMUL_MODE_PROT32:
4252 def_op_bytes = def_ad_bytes = 4;
4254 #ifdef CONFIG_X86_64
4255 case X86EMUL_MODE_PROT64:
4261 return EMULATION_FAILED;
4264 ctxt->op_bytes = def_op_bytes;
4265 ctxt->ad_bytes = def_ad_bytes;
4267 /* Legacy prefixes. */
4269 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4270 case 0x66: /* operand-size override */
4272 /* switch between 2/4 bytes */
4273 ctxt->op_bytes = def_op_bytes ^ 6;
4275 case 0x67: /* address-size override */
4276 if (mode == X86EMUL_MODE_PROT64)
4277 /* switch between 4/8 bytes */
4278 ctxt->ad_bytes = def_ad_bytes ^ 12;
4280 /* switch between 2/4 bytes */
4281 ctxt->ad_bytes = def_ad_bytes ^ 6;
4283 case 0x26: /* ES override */
4284 case 0x2e: /* CS override */
4285 case 0x36: /* SS override */
4286 case 0x3e: /* DS override */
4287 has_seg_override = true;
4288 ctxt->seg_override = (ctxt->b >> 3) & 3;
4290 case 0x64: /* FS override */
4291 case 0x65: /* GS override */
4292 has_seg_override = true;
4293 ctxt->seg_override = ctxt->b & 7;
4295 case 0x40 ... 0x4f: /* REX */
4296 if (mode != X86EMUL_MODE_PROT64)
4298 ctxt->rex_prefix = ctxt->b;
4300 case 0xf0: /* LOCK */
4301 ctxt->lock_prefix = 1;
4303 case 0xf2: /* REPNE/REPNZ */
4304 case 0xf3: /* REP/REPE/REPZ */
4305 ctxt->rep_prefix = ctxt->b;
4311 /* Any legacy prefix after a REX prefix nullifies its effect. */
4313 ctxt->rex_prefix = 0;
4319 if (ctxt->rex_prefix & 8)
4320 ctxt->op_bytes = 8; /* REX.W */
4322 /* Opcode byte(s). */
4323 opcode = opcode_table[ctxt->b];
4324 /* Two-byte opcode? */
4325 if (ctxt->b == 0x0f) {
4326 ctxt->opcode_len = 2;
4327 ctxt->b = insn_fetch(u8, ctxt);
4328 opcode = twobyte_table[ctxt->b];
4330 /* 0F_38 opcode map */
4331 if (ctxt->b == 0x38) {
4332 ctxt->opcode_len = 3;
4333 ctxt->b = insn_fetch(u8, ctxt);
4334 opcode = opcode_map_0f_38[ctxt->b];
4337 ctxt->d = opcode.flags;
4339 if (ctxt->d & ModRM)
4340 ctxt->modrm = insn_fetch(u8, ctxt);
4342 /* vex-prefix instructions are not implemented */
4343 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4344 (mode == X86EMUL_MODE_PROT64 ||
4345 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4349 while (ctxt->d & GroupMask) {
4350 switch (ctxt->d & GroupMask) {
4352 goffset = (ctxt->modrm >> 3) & 7;
4353 opcode = opcode.u.group[goffset];
4356 goffset = (ctxt->modrm >> 3) & 7;
4357 if ((ctxt->modrm >> 6) == 3)
4358 opcode = opcode.u.gdual->mod3[goffset];
4360 opcode = opcode.u.gdual->mod012[goffset];
4363 goffset = ctxt->modrm & 7;
4364 opcode = opcode.u.group[goffset];
4367 if (ctxt->rep_prefix && op_prefix)
4368 return EMULATION_FAILED;
4369 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4370 switch (simd_prefix) {
4371 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4372 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4373 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4374 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4378 if (ctxt->modrm > 0xbf)
4379 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4381 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4384 return EMULATION_FAILED;
4387 ctxt->d &= ~(u64)GroupMask;
4388 ctxt->d |= opcode.flags;
4393 return EMULATION_FAILED;
4395 ctxt->execute = opcode.u.execute;
4397 if (unlikely(ctxt->d &
4398 (NotImpl|EmulateOnUD|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4400 * These are copied unconditionally here, and checked unconditionally
4401 * in x86_emulate_insn.
4403 ctxt->check_perm = opcode.check_perm;
4404 ctxt->intercept = opcode.intercept;
4406 if (ctxt->d & NotImpl)
4407 return EMULATION_FAILED;
4409 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4410 return EMULATION_FAILED;
4412 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4415 if (ctxt->d & Op3264) {
4416 if (mode == X86EMUL_MODE_PROT64)
4423 ctxt->op_bytes = 16;
4424 else if (ctxt->d & Mmx)
4428 /* ModRM and SIB bytes. */
4429 if (ctxt->d & ModRM) {
4430 rc = decode_modrm(ctxt, &ctxt->memop);
4431 if (!has_seg_override) {
4432 has_seg_override = true;
4433 ctxt->seg_override = ctxt->modrm_seg;
4435 } else if (ctxt->d & MemAbs)
4436 rc = decode_abs(ctxt, &ctxt->memop);
4437 if (rc != X86EMUL_CONTINUE)
4440 if (!has_seg_override)
4441 ctxt->seg_override = VCPU_SREG_DS;
4443 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4446 * Decode and fetch the source operand: register, memory
4449 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4450 if (rc != X86EMUL_CONTINUE)
4454 * Decode and fetch the second source operand: register, memory
4457 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4458 if (rc != X86EMUL_CONTINUE)
4461 /* Decode and fetch the destination operand: register or memory. */
4462 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4465 if (ctxt->rip_relative)
4466 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4468 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4471 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4473 return ctxt->d & PageTable;
4476 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4478 /* The second termination condition only applies for REPE
4479 * and REPNE. Test if the repeat string operation prefix is
4480 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4481 * corresponding termination condition according to:
4482 * - if REPE/REPZ and ZF = 0 then done
4483 * - if REPNE/REPNZ and ZF = 1 then done
4485 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4486 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4487 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4488 ((ctxt->eflags & EFLG_ZF) == 0))
4489 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4490 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4496 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4500 ctxt->ops->get_fpu(ctxt);
4501 asm volatile("1: fwait \n\t"
4503 ".pushsection .fixup,\"ax\" \n\t"
4505 "movb $1, %[fault] \n\t"
4508 _ASM_EXTABLE(1b, 3b)
4509 : [fault]"+qm"(fault));
4510 ctxt->ops->put_fpu(ctxt);
4512 if (unlikely(fault))
4513 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4515 return X86EMUL_CONTINUE;
4518 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4521 if (op->type == OP_MM)
4522 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4525 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4527 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4528 if (!(ctxt->d & ByteOp))
4529 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4530 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4531 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4533 : "c"(ctxt->src2.val));
4534 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4535 if (!fop) /* exception is returned in fop variable */
4536 return emulate_de(ctxt);
4537 return X86EMUL_CONTINUE;
4540 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4542 memset(&ctxt->rip_relative, 0,
4543 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4545 ctxt->io_read.pos = 0;
4546 ctxt->io_read.end = 0;
4547 ctxt->mem_read.end = 0;
4550 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4552 const struct x86_emulate_ops *ops = ctxt->ops;
4553 int rc = X86EMUL_CONTINUE;
4554 int saved_dst_type = ctxt->dst.type;
4556 ctxt->mem_read.pos = 0;
4558 /* LOCK prefix is allowed only with some instructions */
4559 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4560 rc = emulate_ud(ctxt);
4564 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4565 rc = emulate_ud(ctxt);
4569 if (unlikely(ctxt->d &
4570 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4571 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4572 (ctxt->d & Undefined)) {
4573 rc = emulate_ud(ctxt);
4577 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4578 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4579 rc = emulate_ud(ctxt);
4583 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4584 rc = emulate_nm(ctxt);
4588 if (ctxt->d & Mmx) {
4589 rc = flush_pending_x87_faults(ctxt);
4590 if (rc != X86EMUL_CONTINUE)
4593 * Now that we know the fpu is exception safe, we can fetch
4596 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4597 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4598 if (!(ctxt->d & Mov))
4599 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4602 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4603 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4604 X86_ICPT_PRE_EXCEPT);
4605 if (rc != X86EMUL_CONTINUE)
4609 /* Privileged instruction can be executed only in CPL=0 */
4610 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4611 if (ctxt->d & PrivUD)
4612 rc = emulate_ud(ctxt);
4614 rc = emulate_gp(ctxt, 0);
4618 /* Instruction can only be executed in protected mode */
4619 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4620 rc = emulate_ud(ctxt);
4624 /* Do instruction specific permission checks */
4625 if (ctxt->d & CheckPerm) {
4626 rc = ctxt->check_perm(ctxt);
4627 if (rc != X86EMUL_CONTINUE)
4631 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4632 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4633 X86_ICPT_POST_EXCEPT);
4634 if (rc != X86EMUL_CONTINUE)
4638 if (ctxt->rep_prefix && (ctxt->d & String)) {
4639 /* All REP prefixes have the same first termination condition */
4640 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4641 ctxt->eip = ctxt->_eip;
4642 ctxt->eflags &= ~EFLG_RF;
4648 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4649 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4650 ctxt->src.valptr, ctxt->src.bytes);
4651 if (rc != X86EMUL_CONTINUE)
4653 ctxt->src.orig_val64 = ctxt->src.val64;
4656 if (ctxt->src2.type == OP_MEM) {
4657 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4658 &ctxt->src2.val, ctxt->src2.bytes);
4659 if (rc != X86EMUL_CONTINUE)
4663 if ((ctxt->d & DstMask) == ImplicitOps)
4667 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4668 /* optimisation - avoid slow emulated read if Mov */
4669 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4670 &ctxt->dst.val, ctxt->dst.bytes);
4671 if (rc != X86EMUL_CONTINUE)
4674 ctxt->dst.orig_val = ctxt->dst.val;
4678 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4679 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4680 X86_ICPT_POST_MEMACCESS);
4681 if (rc != X86EMUL_CONTINUE)
4685 if (ctxt->rep_prefix && (ctxt->d & String))
4686 ctxt->eflags |= EFLG_RF;
4688 ctxt->eflags &= ~EFLG_RF;
4690 if (ctxt->execute) {
4691 if (ctxt->d & Fastop) {
4692 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4693 rc = fastop(ctxt, fop);
4694 if (rc != X86EMUL_CONTINUE)
4698 rc = ctxt->execute(ctxt);
4699 if (rc != X86EMUL_CONTINUE)
4704 if (ctxt->opcode_len == 2)
4706 else if (ctxt->opcode_len == 3)
4707 goto threebyte_insn;
4710 case 0x63: /* movsxd */
4711 if (ctxt->mode != X86EMUL_MODE_PROT64)
4712 goto cannot_emulate;
4713 ctxt->dst.val = (s32) ctxt->src.val;
4715 case 0x70 ... 0x7f: /* jcc (short) */
4716 if (test_cc(ctxt->b, ctxt->eflags))
4717 jmp_rel(ctxt, ctxt->src.val);
4719 case 0x8d: /* lea r16/r32, m */
4720 ctxt->dst.val = ctxt->src.addr.mem.ea;
4722 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4723 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4724 ctxt->dst.type = OP_NONE;
4728 case 0x98: /* cbw/cwde/cdqe */
4729 switch (ctxt->op_bytes) {
4730 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4731 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4732 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4735 case 0xcc: /* int3 */
4736 rc = emulate_int(ctxt, 3);
4738 case 0xcd: /* int n */
4739 rc = emulate_int(ctxt, ctxt->src.val);
4741 case 0xce: /* into */
4742 if (ctxt->eflags & EFLG_OF)
4743 rc = emulate_int(ctxt, 4);
4745 case 0xe9: /* jmp rel */
4746 case 0xeb: /* jmp rel short */
4747 jmp_rel(ctxt, ctxt->src.val);
4748 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4750 case 0xf4: /* hlt */
4751 ctxt->ops->halt(ctxt);
4753 case 0xf5: /* cmc */
4754 /* complement carry flag from eflags reg */
4755 ctxt->eflags ^= EFLG_CF;
4757 case 0xf8: /* clc */
4758 ctxt->eflags &= ~EFLG_CF;
4760 case 0xf9: /* stc */
4761 ctxt->eflags |= EFLG_CF;
4763 case 0xfc: /* cld */
4764 ctxt->eflags &= ~EFLG_DF;
4766 case 0xfd: /* std */
4767 ctxt->eflags |= EFLG_DF;
4770 goto cannot_emulate;
4773 if (rc != X86EMUL_CONTINUE)
4777 if (ctxt->d & SrcWrite) {
4778 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4779 rc = writeback(ctxt, &ctxt->src);
4780 if (rc != X86EMUL_CONTINUE)
4783 if (!(ctxt->d & NoWrite)) {
4784 rc = writeback(ctxt, &ctxt->dst);
4785 if (rc != X86EMUL_CONTINUE)
4790 * restore dst type in case the decoding will be reused
4791 * (happens for string instruction )
4793 ctxt->dst.type = saved_dst_type;
4795 if ((ctxt->d & SrcMask) == SrcSI)
4796 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4798 if ((ctxt->d & DstMask) == DstDI)
4799 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4801 if (ctxt->rep_prefix && (ctxt->d & String)) {
4803 struct read_cache *r = &ctxt->io_read;
4804 if ((ctxt->d & SrcMask) == SrcSI)
4805 count = ctxt->src.count;
4807 count = ctxt->dst.count;
4808 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4811 if (!string_insn_completed(ctxt)) {
4813 * Re-enter guest when pio read ahead buffer is empty
4814 * or, if it is not used, after each 1024 iteration.
4816 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4817 (r->end == 0 || r->end != r->pos)) {
4819 * Reset read cache. Usually happens before
4820 * decode, but since instruction is restarted
4821 * we have to do it here.
4823 ctxt->mem_read.end = 0;
4824 writeback_registers(ctxt);
4825 return EMULATION_RESTART;
4827 goto done; /* skip rip writeback */
4829 ctxt->eflags &= ~EFLG_RF;
4832 ctxt->eip = ctxt->_eip;
4835 if (rc == X86EMUL_PROPAGATE_FAULT)
4836 ctxt->have_exception = true;
4837 if (rc == X86EMUL_INTERCEPTED)
4838 return EMULATION_INTERCEPTED;
4840 if (rc == X86EMUL_CONTINUE)
4841 writeback_registers(ctxt);
4843 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4847 case 0x09: /* wbinvd */
4848 (ctxt->ops->wbinvd)(ctxt);
4850 case 0x08: /* invd */
4851 case 0x0d: /* GrpP (prefetch) */
4852 case 0x18: /* Grp16 (prefetch/nop) */
4853 case 0x1f: /* nop */
4855 case 0x20: /* mov cr, reg */
4856 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4858 case 0x21: /* mov from dr to reg */
4859 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4861 case 0x40 ... 0x4f: /* cmov */
4862 if (test_cc(ctxt->b, ctxt->eflags))
4863 ctxt->dst.val = ctxt->src.val;
4864 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4865 ctxt->op_bytes != 4)
4866 ctxt->dst.type = OP_NONE; /* no writeback */
4868 case 0x80 ... 0x8f: /* jnz rel, etc*/
4869 if (test_cc(ctxt->b, ctxt->eflags))
4870 jmp_rel(ctxt, ctxt->src.val);
4872 case 0x90 ... 0x9f: /* setcc r/m8 */
4873 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4875 case 0xae: /* clflush */
4877 case 0xb6 ... 0xb7: /* movzx */
4878 ctxt->dst.bytes = ctxt->op_bytes;
4879 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4880 : (u16) ctxt->src.val;
4882 case 0xbe ... 0xbf: /* movsx */
4883 ctxt->dst.bytes = ctxt->op_bytes;
4884 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4885 (s16) ctxt->src.val;
4887 case 0xc3: /* movnti */
4888 ctxt->dst.bytes = ctxt->op_bytes;
4889 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4890 (u32) ctxt->src.val;
4893 goto cannot_emulate;
4898 if (rc != X86EMUL_CONTINUE)
4904 return EMULATION_FAILED;
4907 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4909 invalidate_registers(ctxt);
4912 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4914 writeback_registers(ctxt);