1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define GroupMask (7<<14) /* Opcode uses one of the group mechanisms */
77 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
78 #define GroupDual (2<<14) /* Alternate decoding of mod == 3 */
79 #define Prefix (3<<14) /* Instruction varies with 66/f2/f3 prefix */
80 #define RMExt (4<<14) /* Opcode extension in ModRM r/m if mod == 3 */
81 #define Sse (1<<17) /* SSE Vector instruction */
83 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
84 #define VendorSpecific (1<<22) /* Vendor specific instruction */
85 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
86 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Imm (4<<29)
97 #define Src2Mask (7<<29)
100 #define X3(x...) X2(x), x
101 #define X4(x...) X2(x), X2(x)
102 #define X5(x...) X4(x), x
103 #define X6(x...) X4(x), X2(x)
104 #define X7(x...) X4(x), X3(x)
105 #define X8(x...) X4(x), X4(x)
106 #define X16(x...) X8(x), X8(x)
112 int (*execute)(struct x86_emulate_ctxt *ctxt);
113 struct opcode *group;
114 struct group_dual *gdual;
115 struct gprefix *gprefix;
117 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
121 struct opcode mod012[8];
122 struct opcode mod3[8];
126 struct opcode pfx_no;
127 struct opcode pfx_66;
128 struct opcode pfx_f2;
129 struct opcode pfx_f3;
132 /* EFLAGS bit definitions. */
133 #define EFLG_ID (1<<21)
134 #define EFLG_VIP (1<<20)
135 #define EFLG_VIF (1<<19)
136 #define EFLG_AC (1<<18)
137 #define EFLG_VM (1<<17)
138 #define EFLG_RF (1<<16)
139 #define EFLG_IOPL (3<<12)
140 #define EFLG_NT (1<<14)
141 #define EFLG_OF (1<<11)
142 #define EFLG_DF (1<<10)
143 #define EFLG_IF (1<<9)
144 #define EFLG_TF (1<<8)
145 #define EFLG_SF (1<<7)
146 #define EFLG_ZF (1<<6)
147 #define EFLG_AF (1<<4)
148 #define EFLG_PF (1<<2)
149 #define EFLG_CF (1<<0)
151 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
152 #define EFLG_RESERVED_ONE_MASK 2
155 * Instruction emulation:
156 * Most instructions are emulated directly via a fragment of inline assembly
157 * code. This allows us to save/restore EFLAGS and thus very easily pick up
158 * any modified flags.
161 #if defined(CONFIG_X86_64)
162 #define _LO32 "k" /* force 32-bit operand */
163 #define _STK "%%rsp" /* stack pointer */
164 #elif defined(__i386__)
165 #define _LO32 "" /* force 32-bit operand */
166 #define _STK "%%esp" /* stack pointer */
170 * These EFLAGS bits are restored from saved value during emulation, and
171 * any changes are written back to the saved value after emulation.
173 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
175 /* Before executing instruction: restore necessary bits in EFLAGS. */
176 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
177 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
178 "movl %"_sav",%"_LO32 _tmp"; " \
181 "movl %"_msk",%"_LO32 _tmp"; " \
182 "andl %"_LO32 _tmp",("_STK"); " \
184 "notl %"_LO32 _tmp"; " \
185 "andl %"_LO32 _tmp",("_STK"); " \
186 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
188 "orl %"_LO32 _tmp",("_STK"); " \
192 /* After executing instruction: write-back necessary bits in EFLAGS. */
193 #define _POST_EFLAGS(_sav, _msk, _tmp) \
194 /* _sav |= EFLAGS & _msk; */ \
197 "andl %"_msk",%"_LO32 _tmp"; " \
198 "orl %"_LO32 _tmp",%"_sav"; "
206 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
208 __asm__ __volatile__ ( \
209 _PRE_EFLAGS("0", "4", "2") \
210 _op _suffix " %"_x"3,%1; " \
211 _POST_EFLAGS("0", "4", "2") \
212 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
214 : _y ((_src).val), "i" (EFLAGS_MASK)); \
218 /* Raw emulation: instruction has two explicit operands. */
219 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
221 unsigned long _tmp; \
223 switch ((_dst).bytes) { \
225 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
228 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
231 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
236 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
238 unsigned long _tmp; \
239 switch ((_dst).bytes) { \
241 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
244 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
245 _wx, _wy, _lx, _ly, _qx, _qy); \
250 /* Source operand is byte-sized and may be restricted to just %cl. */
251 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
252 __emulate_2op(_op, _src, _dst, _eflags, \
253 "b", "c", "b", "c", "b", "c", "b", "c")
255 /* Source operand is byte, word, long or quad sized. */
256 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
257 __emulate_2op(_op, _src, _dst, _eflags, \
258 "b", "q", "w", "r", _LO32, "r", "", "r")
260 /* Source operand is word, long or quad sized. */
261 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
262 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
263 "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (_cl).val; \
270 _type _srcv = (_src).val; \
271 _type _dstv = (_dst).val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (_cl).val = (unsigned long) _clv; \
282 (_src).val = (unsigned long) _srcv; \
283 (_dst).val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
288 switch ((_dst).bytes) { \
290 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
291 "w", unsigned short); \
294 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
295 "l", unsigned int); \
298 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
299 "q", unsigned long)); \
304 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
306 unsigned long _tmp; \
308 __asm__ __volatile__ ( \
309 _PRE_EFLAGS("0", "3", "2") \
310 _op _suffix " %1; " \
311 _POST_EFLAGS("0", "3", "2") \
312 : "=m" (_eflags), "+m" ((_dst).val), \
314 : "i" (EFLAGS_MASK)); \
317 /* Instruction has only one explicit operand (no source operand). */
318 #define emulate_1op(_op, _dst, _eflags) \
320 switch ((_dst).bytes) { \
321 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
322 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
323 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
324 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
328 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
330 unsigned long _tmp; \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "1") \
334 _op _suffix " %5; " \
335 _POST_EFLAGS("0", "4", "1") \
336 : "=m" (_eflags), "=&r" (_tmp), \
337 "+a" (_rax), "+d" (_rdx) \
338 : "i" (EFLAGS_MASK), "m" ((_src).val), \
339 "a" (_rax), "d" (_rdx)); \
342 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
344 unsigned long _tmp; \
346 __asm__ __volatile__ ( \
347 _PRE_EFLAGS("0", "5", "1") \
349 _op _suffix " %6; " \
351 _POST_EFLAGS("0", "5", "1") \
352 ".pushsection .fixup,\"ax\" \n\t" \
353 "3: movb $1, %4 \n\t" \
356 _ASM_EXTABLE(1b, 3b) \
357 : "=m" (_eflags), "=&r" (_tmp), \
358 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
359 : "i" (EFLAGS_MASK), "m" ((_src).val), \
360 "a" (_rax), "d" (_rdx)); \
363 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
364 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
366 switch((_src).bytes) { \
368 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
372 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
376 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
380 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
386 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
388 switch((_src).bytes) { \
390 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
391 _eflags, "b", _ex); \
394 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
395 _eflags, "w", _ex); \
398 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
399 _eflags, "l", _ex); \
402 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
403 _eflags, "q", _ex)); \
408 /* Fetch next part of the instruction being emulated. */
409 #define insn_fetch(_type, _size, _eip) \
410 ({ unsigned long _x; \
411 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
412 if (rc != X86EMUL_CONTINUE) \
418 #define insn_fetch_arr(_arr, _size, _eip) \
419 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
420 if (rc != X86EMUL_CONTINUE) \
425 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
426 enum x86_intercept intercept,
427 enum x86_intercept_stage stage)
429 struct x86_instruction_info info = {
430 .intercept = intercept,
431 .rep_prefix = ctxt->decode.rep_prefix,
432 .modrm_mod = ctxt->decode.modrm_mod,
433 .modrm_reg = ctxt->decode.modrm_reg,
434 .modrm_rm = ctxt->decode.modrm_rm,
435 .src_val = ctxt->decode.src.val64,
436 .src_bytes = ctxt->decode.src.bytes,
437 .dst_bytes = ctxt->decode.dst.bytes,
438 .ad_bytes = ctxt->decode.ad_bytes,
439 .next_rip = ctxt->eip,
442 return ctxt->ops->intercept(ctxt, &info, stage);
445 static inline unsigned long ad_mask(struct decode_cache *c)
447 return (1UL << (c->ad_bytes << 3)) - 1;
450 /* Access/update address held in a register, based on addressing mode. */
451 static inline unsigned long
452 address_mask(struct decode_cache *c, unsigned long reg)
454 if (c->ad_bytes == sizeof(unsigned long))
457 return reg & ad_mask(c);
460 static inline unsigned long
461 register_address(struct decode_cache *c, unsigned long reg)
463 return address_mask(c, reg);
467 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
469 if (c->ad_bytes == sizeof(unsigned long))
472 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
475 static inline void jmp_rel(struct decode_cache *c, int rel)
477 register_address_increment(c, &c->eip, rel);
480 static u32 desc_limit_scaled(struct desc_struct *desc)
482 u32 limit = get_desc_limit(desc);
484 return desc->g ? (limit << 12) | 0xfff : limit;
487 static void set_seg_override(struct decode_cache *c, int seg)
489 c->has_seg_override = true;
490 c->seg_override = seg;
493 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
494 struct x86_emulate_ops *ops, int seg)
496 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
499 return ops->get_cached_segment_base(ctxt, seg);
502 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
503 struct decode_cache *c)
505 if (!c->has_seg_override)
508 return c->seg_override;
511 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
512 u32 error, bool valid)
514 ctxt->exception.vector = vec;
515 ctxt->exception.error_code = error;
516 ctxt->exception.error_code_valid = valid;
517 return X86EMUL_PROPAGATE_FAULT;
520 static int emulate_db(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, DB_VECTOR, 0, false);
525 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, GP_VECTOR, err, true);
530 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
532 return emulate_exception(ctxt, SS_VECTOR, err, true);
535 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, UD_VECTOR, 0, false);
540 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, TS_VECTOR, err, true);
545 static int emulate_de(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DE_VECTOR, 0, false);
550 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
552 return emulate_exception(ctxt, NM_VECTOR, 0, false);
555 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
558 struct desc_struct desc;
560 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
564 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
569 struct desc_struct desc;
571 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
572 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
575 static int __linearize(struct x86_emulate_ctxt *ctxt,
576 struct segmented_address addr,
577 unsigned size, bool write, bool fetch,
580 struct decode_cache *c = &ctxt->decode;
581 struct desc_struct desc;
588 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
589 switch (ctxt->mode) {
590 case X86EMUL_MODE_REAL:
592 case X86EMUL_MODE_PROT64:
593 if (((signed long)la << 16) >> 16 != la)
594 return emulate_gp(ctxt, 0);
597 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
601 /* code segment or read-only data segment */
602 if (((desc.type & 8) || !(desc.type & 2)) && write)
604 /* unreadable code segment */
605 if (!fetch && (desc.type & 8) && !(desc.type & 2))
607 lim = desc_limit_scaled(&desc);
608 if ((desc.type & 8) || !(desc.type & 4)) {
609 /* expand-up segment */
610 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
613 /* exapand-down segment */
614 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
616 lim = desc.d ? 0xffffffff : 0xffff;
617 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
620 cpl = ctxt->ops->cpl(ctxt);
623 if (!(desc.type & 8)) {
627 } else if ((desc.type & 8) && !(desc.type & 4)) {
628 /* nonconforming code segment */
631 } else if ((desc.type & 8) && (desc.type & 4)) {
632 /* conforming code segment */
638 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
641 return X86EMUL_CONTINUE;
643 if (addr.seg == VCPU_SREG_SS)
644 return emulate_ss(ctxt, addr.seg);
646 return emulate_gp(ctxt, addr.seg);
649 static int linearize(struct x86_emulate_ctxt *ctxt,
650 struct segmented_address addr,
651 unsigned size, bool write,
654 return __linearize(ctxt, addr, size, write, false, linear);
658 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
659 struct segmented_address addr,
666 rc = linearize(ctxt, addr, size, false, &linear);
667 if (rc != X86EMUL_CONTINUE)
669 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
672 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
673 struct x86_emulate_ops *ops,
674 unsigned long eip, u8 *dest)
676 struct fetch_cache *fc = &ctxt->decode.fetch;
680 if (eip == fc->end) {
681 unsigned long linear;
682 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
683 cur_size = fc->end - fc->start;
684 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
685 rc = __linearize(ctxt, addr, size, false, true, &linear);
686 if (rc != X86EMUL_CONTINUE)
688 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
689 size, &ctxt->exception);
690 if (rc != X86EMUL_CONTINUE)
694 *dest = fc->data[eip - fc->start];
695 return X86EMUL_CONTINUE;
698 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
699 struct x86_emulate_ops *ops,
700 unsigned long eip, void *dest, unsigned size)
704 /* x86 instructions are limited to 15 bytes. */
705 if (eip + size - ctxt->eip > 15)
706 return X86EMUL_UNHANDLEABLE;
708 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
709 if (rc != X86EMUL_CONTINUE)
712 return X86EMUL_CONTINUE;
716 * Given the 'reg' portion of a ModRM byte, and a register block, return a
717 * pointer into the block that addresses the relevant register.
718 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
720 static void *decode_register(u8 modrm_reg, unsigned long *regs,
725 p = ®s[modrm_reg];
726 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
727 p = (unsigned char *)®s[modrm_reg & 3] + 1;
731 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 u16 *size, unsigned long *address, int op_bytes)
740 rc = segmented_read_std(ctxt, addr, size, 2);
741 if (rc != X86EMUL_CONTINUE)
744 rc = segmented_read_std(ctxt, addr, address, op_bytes);
748 static int test_cc(unsigned int condition, unsigned int flags)
752 switch ((condition & 15) >> 1) {
754 rc |= (flags & EFLG_OF);
756 case 1: /* b/c/nae */
757 rc |= (flags & EFLG_CF);
760 rc |= (flags & EFLG_ZF);
763 rc |= (flags & (EFLG_CF|EFLG_ZF));
766 rc |= (flags & EFLG_SF);
769 rc |= (flags & EFLG_PF);
772 rc |= (flags & EFLG_ZF);
775 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
779 /* Odd condition identifiers (lsb == 1) have inverted sense. */
780 return (!!rc ^ (condition & 1));
783 static void fetch_register_operand(struct operand *op)
787 op->val = *(u8 *)op->addr.reg;
790 op->val = *(u16 *)op->addr.reg;
793 op->val = *(u32 *)op->addr.reg;
796 op->val = *(u64 *)op->addr.reg;
801 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
803 ctxt->ops->get_fpu(ctxt);
805 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
806 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
807 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
808 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
809 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
810 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
811 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
812 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
814 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
815 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
816 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
817 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
818 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
819 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
820 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
821 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
825 ctxt->ops->put_fpu(ctxt);
828 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
831 ctxt->ops->get_fpu(ctxt);
833 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
834 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
835 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
836 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
837 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
838 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
839 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
840 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
842 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
843 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
844 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
845 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
846 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
847 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
848 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
849 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
853 ctxt->ops->put_fpu(ctxt);
856 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
858 struct decode_cache *c,
861 unsigned reg = c->modrm_reg;
862 int highbyte_regs = c->rex_prefix == 0;
865 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
871 read_sse_reg(ctxt, &op->vec_val, reg);
876 if ((c->d & ByteOp) && !inhibit_bytereg) {
877 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
880 op->addr.reg = decode_register(reg, c->regs, 0);
881 op->bytes = c->op_bytes;
883 fetch_register_operand(op);
884 op->orig_val = op->val;
887 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
888 struct x86_emulate_ops *ops,
891 struct decode_cache *c = &ctxt->decode;
893 int index_reg = 0, base_reg = 0, scale;
894 int rc = X86EMUL_CONTINUE;
898 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
899 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
900 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
903 c->modrm = insn_fetch(u8, 1, c->eip);
904 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
905 c->modrm_reg |= (c->modrm & 0x38) >> 3;
906 c->modrm_rm |= (c->modrm & 0x07);
907 c->modrm_seg = VCPU_SREG_DS;
909 if (c->modrm_mod == 3) {
911 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
912 op->addr.reg = decode_register(c->modrm_rm,
913 c->regs, c->d & ByteOp);
917 op->addr.xmm = c->modrm_rm;
918 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
921 fetch_register_operand(op);
927 if (c->ad_bytes == 2) {
928 unsigned bx = c->regs[VCPU_REGS_RBX];
929 unsigned bp = c->regs[VCPU_REGS_RBP];
930 unsigned si = c->regs[VCPU_REGS_RSI];
931 unsigned di = c->regs[VCPU_REGS_RDI];
933 /* 16-bit ModR/M decode. */
934 switch (c->modrm_mod) {
936 if (c->modrm_rm == 6)
937 modrm_ea += insn_fetch(u16, 2, c->eip);
940 modrm_ea += insn_fetch(s8, 1, c->eip);
943 modrm_ea += insn_fetch(u16, 2, c->eip);
946 switch (c->modrm_rm) {
966 if (c->modrm_mod != 0)
973 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
974 (c->modrm_rm == 6 && c->modrm_mod != 0))
975 c->modrm_seg = VCPU_SREG_SS;
976 modrm_ea = (u16)modrm_ea;
978 /* 32/64-bit ModR/M decode. */
979 if ((c->modrm_rm & 7) == 4) {
980 sib = insn_fetch(u8, 1, c->eip);
981 index_reg |= (sib >> 3) & 7;
985 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
986 modrm_ea += insn_fetch(s32, 4, c->eip);
988 modrm_ea += c->regs[base_reg];
990 modrm_ea += c->regs[index_reg] << scale;
991 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
992 if (ctxt->mode == X86EMUL_MODE_PROT64)
995 modrm_ea += c->regs[c->modrm_rm];
996 switch (c->modrm_mod) {
998 if (c->modrm_rm == 5)
999 modrm_ea += insn_fetch(s32, 4, c->eip);
1002 modrm_ea += insn_fetch(s8, 1, c->eip);
1005 modrm_ea += insn_fetch(s32, 4, c->eip);
1009 op->addr.mem.ea = modrm_ea;
1014 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1015 struct x86_emulate_ops *ops,
1018 struct decode_cache *c = &ctxt->decode;
1019 int rc = X86EMUL_CONTINUE;
1022 switch (c->ad_bytes) {
1024 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1027 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1030 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1037 static void fetch_bit_operand(struct decode_cache *c)
1041 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1042 mask = ~(c->dst.bytes * 8 - 1);
1044 if (c->src.bytes == 2)
1045 sv = (s16)c->src.val & (s16)mask;
1046 else if (c->src.bytes == 4)
1047 sv = (s32)c->src.val & (s32)mask;
1049 c->dst.addr.mem.ea += (sv >> 3);
1052 /* only subword offset */
1053 c->src.val &= (c->dst.bytes << 3) - 1;
1056 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1057 struct x86_emulate_ops *ops,
1058 unsigned long addr, void *dest, unsigned size)
1061 struct read_cache *mc = &ctxt->decode.mem_read;
1064 int n = min(size, 8u);
1066 if (mc->pos < mc->end)
1069 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1071 if (rc != X86EMUL_CONTINUE)
1076 memcpy(dest, mc->data + mc->pos, n);
1081 return X86EMUL_CONTINUE;
1084 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1085 struct segmented_address addr,
1092 rc = linearize(ctxt, addr, size, false, &linear);
1093 if (rc != X86EMUL_CONTINUE)
1095 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1098 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1099 struct segmented_address addr,
1106 rc = linearize(ctxt, addr, size, true, &linear);
1107 if (rc != X86EMUL_CONTINUE)
1109 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1113 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1114 struct segmented_address addr,
1115 const void *orig_data, const void *data,
1121 rc = linearize(ctxt, addr, size, true, &linear);
1122 if (rc != X86EMUL_CONTINUE)
1124 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1125 size, &ctxt->exception);
1128 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1129 struct x86_emulate_ops *ops,
1130 unsigned int size, unsigned short port,
1133 struct read_cache *rc = &ctxt->decode.io_read;
1135 if (rc->pos == rc->end) { /* refill pio read ahead */
1136 struct decode_cache *c = &ctxt->decode;
1137 unsigned int in_page, n;
1138 unsigned int count = c->rep_prefix ?
1139 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1140 in_page = (ctxt->eflags & EFLG_DF) ?
1141 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1142 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1143 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1147 rc->pos = rc->end = 0;
1148 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1153 memcpy(dest, rc->data + rc->pos, size);
1158 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1159 struct x86_emulate_ops *ops,
1160 u16 selector, struct desc_ptr *dt)
1162 if (selector & 1 << 2) {
1163 struct desc_struct desc;
1166 memset (dt, 0, sizeof *dt);
1167 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1170 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1171 dt->address = get_desc_base(&desc);
1173 ops->get_gdt(ctxt, dt);
1176 /* allowed just for 8 bytes segments */
1177 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1178 struct x86_emulate_ops *ops,
1179 u16 selector, struct desc_struct *desc)
1182 u16 index = selector >> 3;
1186 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1188 if (dt.size < index * 8 + 7)
1189 return emulate_gp(ctxt, selector & 0xfffc);
1190 addr = dt.address + index * 8;
1191 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1196 /* allowed just for 8 bytes segments */
1197 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1198 struct x86_emulate_ops *ops,
1199 u16 selector, struct desc_struct *desc)
1202 u16 index = selector >> 3;
1206 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1208 if (dt.size < index * 8 + 7)
1209 return emulate_gp(ctxt, selector & 0xfffc);
1211 addr = dt.address + index * 8;
1212 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1217 /* Does not support long mode */
1218 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1219 struct x86_emulate_ops *ops,
1220 u16 selector, int seg)
1222 struct desc_struct seg_desc;
1224 unsigned err_vec = GP_VECTOR;
1226 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1229 memset(&seg_desc, 0, sizeof seg_desc);
1231 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1232 || ctxt->mode == X86EMUL_MODE_REAL) {
1233 /* set real mode segment descriptor */
1234 set_desc_base(&seg_desc, selector << 4);
1235 set_desc_limit(&seg_desc, 0xffff);
1242 /* NULL selector is not valid for TR, CS and SS */
1243 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1247 /* TR should be in GDT only */
1248 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1251 if (null_selector) /* for NULL selector skip all following checks */
1254 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1255 if (ret != X86EMUL_CONTINUE)
1258 err_code = selector & 0xfffc;
1259 err_vec = GP_VECTOR;
1261 /* can't load system descriptor into segment selecor */
1262 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1266 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1272 cpl = ops->cpl(ctxt);
1277 * segment is not a writable data segment or segment
1278 * selector's RPL != CPL or segment selector's RPL != CPL
1280 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1284 if (!(seg_desc.type & 8))
1287 if (seg_desc.type & 4) {
1293 if (rpl > cpl || dpl != cpl)
1296 /* CS(RPL) <- CPL */
1297 selector = (selector & 0xfffc) | cpl;
1300 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1303 case VCPU_SREG_LDTR:
1304 if (seg_desc.s || seg_desc.type != 2)
1307 default: /* DS, ES, FS, or GS */
1309 * segment is not a data or readable code segment or
1310 * ((segment is a data or nonconforming code segment)
1311 * and (both RPL and CPL > DPL))
1313 if ((seg_desc.type & 0xa) == 0x8 ||
1314 (((seg_desc.type & 0xc) != 0xc) &&
1315 (rpl > dpl && cpl > dpl)))
1321 /* mark segment as accessed */
1323 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1324 if (ret != X86EMUL_CONTINUE)
1328 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1329 return X86EMUL_CONTINUE;
1331 emulate_exception(ctxt, err_vec, err_code, true);
1332 return X86EMUL_PROPAGATE_FAULT;
1335 static void write_register_operand(struct operand *op)
1337 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1338 switch (op->bytes) {
1340 *(u8 *)op->addr.reg = (u8)op->val;
1343 *(u16 *)op->addr.reg = (u16)op->val;
1346 *op->addr.reg = (u32)op->val;
1347 break; /* 64b: zero-extend */
1349 *op->addr.reg = op->val;
1354 static int writeback(struct x86_emulate_ctxt *ctxt)
1357 struct decode_cache *c = &ctxt->decode;
1359 switch (c->dst.type) {
1361 write_register_operand(&c->dst);
1365 rc = segmented_cmpxchg(ctxt,
1371 rc = segmented_write(ctxt,
1375 if (rc != X86EMUL_CONTINUE)
1379 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1387 return X86EMUL_CONTINUE;
1390 static int em_push(struct x86_emulate_ctxt *ctxt)
1392 struct decode_cache *c = &ctxt->decode;
1393 struct segmented_address addr;
1395 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1396 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1397 addr.seg = VCPU_SREG_SS;
1399 /* Disable writeback. */
1400 c->dst.type = OP_NONE;
1401 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1404 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1405 void *dest, int len)
1407 struct decode_cache *c = &ctxt->decode;
1409 struct segmented_address addr;
1411 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1412 addr.seg = VCPU_SREG_SS;
1413 rc = segmented_read(ctxt, addr, dest, len);
1414 if (rc != X86EMUL_CONTINUE)
1417 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1421 static int em_pop(struct x86_emulate_ctxt *ctxt)
1423 struct decode_cache *c = &ctxt->decode;
1425 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1428 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1429 struct x86_emulate_ops *ops,
1430 void *dest, int len)
1433 unsigned long val, change_mask;
1434 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1435 int cpl = ops->cpl(ctxt);
1437 rc = emulate_pop(ctxt, &val, len);
1438 if (rc != X86EMUL_CONTINUE)
1441 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1442 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1444 switch(ctxt->mode) {
1445 case X86EMUL_MODE_PROT64:
1446 case X86EMUL_MODE_PROT32:
1447 case X86EMUL_MODE_PROT16:
1449 change_mask |= EFLG_IOPL;
1451 change_mask |= EFLG_IF;
1453 case X86EMUL_MODE_VM86:
1455 return emulate_gp(ctxt, 0);
1456 change_mask |= EFLG_IF;
1458 default: /* real mode */
1459 change_mask |= (EFLG_IOPL | EFLG_IF);
1463 *(unsigned long *)dest =
1464 (ctxt->eflags & ~change_mask) | (val & change_mask);
1469 static int em_popf(struct x86_emulate_ctxt *ctxt)
1471 struct decode_cache *c = &ctxt->decode;
1473 c->dst.type = OP_REG;
1474 c->dst.addr.reg = &ctxt->eflags;
1475 c->dst.bytes = c->op_bytes;
1476 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1479 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1480 struct x86_emulate_ops *ops, int seg)
1482 struct decode_cache *c = &ctxt->decode;
1484 c->src.val = get_segment_selector(ctxt, seg);
1486 return em_push(ctxt);
1489 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1490 struct x86_emulate_ops *ops, int seg)
1492 struct decode_cache *c = &ctxt->decode;
1493 unsigned long selector;
1496 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1497 if (rc != X86EMUL_CONTINUE)
1500 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1504 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1506 struct decode_cache *c = &ctxt->decode;
1507 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1508 int rc = X86EMUL_CONTINUE;
1509 int reg = VCPU_REGS_RAX;
1511 while (reg <= VCPU_REGS_RDI) {
1512 (reg == VCPU_REGS_RSP) ?
1513 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1516 if (rc != X86EMUL_CONTINUE)
1525 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1527 struct decode_cache *c = &ctxt->decode;
1529 c->src.val = (unsigned long)ctxt->eflags;
1530 return em_push(ctxt);
1533 static int em_popa(struct x86_emulate_ctxt *ctxt)
1535 struct decode_cache *c = &ctxt->decode;
1536 int rc = X86EMUL_CONTINUE;
1537 int reg = VCPU_REGS_RDI;
1539 while (reg >= VCPU_REGS_RAX) {
1540 if (reg == VCPU_REGS_RSP) {
1541 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1546 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1547 if (rc != X86EMUL_CONTINUE)
1554 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1555 struct x86_emulate_ops *ops, int irq)
1557 struct decode_cache *c = &ctxt->decode;
1564 /* TODO: Add limit checks */
1565 c->src.val = ctxt->eflags;
1567 if (rc != X86EMUL_CONTINUE)
1570 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1572 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1574 if (rc != X86EMUL_CONTINUE)
1577 c->src.val = c->eip;
1579 if (rc != X86EMUL_CONTINUE)
1582 ops->get_idt(ctxt, &dt);
1584 eip_addr = dt.address + (irq << 2);
1585 cs_addr = dt.address + (irq << 2) + 2;
1587 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1588 if (rc != X86EMUL_CONTINUE)
1591 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1592 if (rc != X86EMUL_CONTINUE)
1595 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1596 if (rc != X86EMUL_CONTINUE)
1604 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1605 struct x86_emulate_ops *ops, int irq)
1607 switch(ctxt->mode) {
1608 case X86EMUL_MODE_REAL:
1609 return emulate_int_real(ctxt, ops, irq);
1610 case X86EMUL_MODE_VM86:
1611 case X86EMUL_MODE_PROT16:
1612 case X86EMUL_MODE_PROT32:
1613 case X86EMUL_MODE_PROT64:
1615 /* Protected mode interrupts unimplemented yet */
1616 return X86EMUL_UNHANDLEABLE;
1620 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1621 struct x86_emulate_ops *ops)
1623 struct decode_cache *c = &ctxt->decode;
1624 int rc = X86EMUL_CONTINUE;
1625 unsigned long temp_eip = 0;
1626 unsigned long temp_eflags = 0;
1627 unsigned long cs = 0;
1628 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1629 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1630 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1631 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1633 /* TODO: Add stack limit check */
1635 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1637 if (rc != X86EMUL_CONTINUE)
1640 if (temp_eip & ~0xffff)
1641 return emulate_gp(ctxt, 0);
1643 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1648 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1653 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1655 if (rc != X86EMUL_CONTINUE)
1661 if (c->op_bytes == 4)
1662 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1663 else if (c->op_bytes == 2) {
1664 ctxt->eflags &= ~0xffff;
1665 ctxt->eflags |= temp_eflags;
1668 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1669 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1674 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1675 struct x86_emulate_ops* ops)
1677 switch(ctxt->mode) {
1678 case X86EMUL_MODE_REAL:
1679 return emulate_iret_real(ctxt, ops);
1680 case X86EMUL_MODE_VM86:
1681 case X86EMUL_MODE_PROT16:
1682 case X86EMUL_MODE_PROT32:
1683 case X86EMUL_MODE_PROT64:
1685 /* iret from protected mode unimplemented yet */
1686 return X86EMUL_UNHANDLEABLE;
1690 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1692 struct decode_cache *c = &ctxt->decode;
1694 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1697 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1699 struct decode_cache *c = &ctxt->decode;
1700 switch (c->modrm_reg) {
1702 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1705 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1708 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1711 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1713 case 4: /* sal/shl */
1714 case 6: /* sal/shl */
1715 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1718 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1721 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1724 return X86EMUL_CONTINUE;
1727 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1729 struct decode_cache *c = &ctxt->decode;
1730 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1731 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1734 switch (c->modrm_reg) {
1735 case 0 ... 1: /* test */
1736 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1739 c->dst.val = ~c->dst.val;
1742 emulate_1op("neg", c->dst, ctxt->eflags);
1745 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1748 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1751 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1755 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1759 return X86EMUL_UNHANDLEABLE;
1762 return emulate_de(ctxt);
1763 return X86EMUL_CONTINUE;
1766 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1768 struct decode_cache *c = &ctxt->decode;
1769 int rc = X86EMUL_CONTINUE;
1771 switch (c->modrm_reg) {
1773 emulate_1op("inc", c->dst, ctxt->eflags);
1776 emulate_1op("dec", c->dst, ctxt->eflags);
1778 case 2: /* call near abs */ {
1781 c->eip = c->src.val;
1782 c->src.val = old_eip;
1786 case 4: /* jmp abs */
1787 c->eip = c->src.val;
1796 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1798 struct decode_cache *c = &ctxt->decode;
1799 u64 old = c->dst.orig_val64;
1801 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1802 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1803 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1804 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1805 ctxt->eflags &= ~EFLG_ZF;
1807 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1808 (u32) c->regs[VCPU_REGS_RBX];
1810 ctxt->eflags |= EFLG_ZF;
1812 return X86EMUL_CONTINUE;
1815 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1816 struct x86_emulate_ops *ops)
1818 struct decode_cache *c = &ctxt->decode;
1822 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1823 if (rc != X86EMUL_CONTINUE)
1825 if (c->op_bytes == 4)
1826 c->eip = (u32)c->eip;
1827 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1828 if (rc != X86EMUL_CONTINUE)
1830 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1834 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1835 struct x86_emulate_ops *ops, int seg)
1837 struct decode_cache *c = &ctxt->decode;
1841 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1843 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1844 if (rc != X86EMUL_CONTINUE)
1847 c->dst.val = c->src.val;
1852 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1853 struct x86_emulate_ops *ops, struct desc_struct *cs,
1854 struct desc_struct *ss)
1858 memset(cs, 0, sizeof(struct desc_struct));
1859 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1860 memset(ss, 0, sizeof(struct desc_struct));
1862 cs->l = 0; /* will be adjusted later */
1863 set_desc_base(cs, 0); /* flat segment */
1864 cs->g = 1; /* 4kb granularity */
1865 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1866 cs->type = 0x0b; /* Read, Execute, Accessed */
1868 cs->dpl = 0; /* will be adjusted later */
1872 set_desc_base(ss, 0); /* flat segment */
1873 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1874 ss->g = 1; /* 4kb granularity */
1876 ss->type = 0x03; /* Read/Write, Accessed */
1877 ss->d = 1; /* 32bit stack segment */
1883 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1885 struct decode_cache *c = &ctxt->decode;
1886 struct desc_struct cs, ss;
1891 /* syscall is not available in real mode */
1892 if (ctxt->mode == X86EMUL_MODE_REAL ||
1893 ctxt->mode == X86EMUL_MODE_VM86)
1894 return emulate_ud(ctxt);
1896 ops->get_msr(ctxt, MSR_EFER, &efer);
1897 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1899 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1901 cs_sel = (u16)(msr_data & 0xfffc);
1902 ss_sel = (u16)(msr_data + 8);
1904 if (efer & EFER_LMA) {
1908 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1909 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1911 c->regs[VCPU_REGS_RCX] = c->eip;
1912 if (efer & EFER_LMA) {
1913 #ifdef CONFIG_X86_64
1914 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1917 ctxt->mode == X86EMUL_MODE_PROT64 ?
1918 MSR_LSTAR : MSR_CSTAR, &msr_data);
1921 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1922 ctxt->eflags &= ~(msr_data | EFLG_RF);
1926 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1927 c->eip = (u32)msr_data;
1929 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1932 return X86EMUL_CONTINUE;
1936 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1938 struct decode_cache *c = &ctxt->decode;
1939 struct desc_struct cs, ss;
1944 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1945 /* inject #GP if in real mode */
1946 if (ctxt->mode == X86EMUL_MODE_REAL)
1947 return emulate_gp(ctxt, 0);
1949 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1950 * Therefore, we inject an #UD.
1952 if (ctxt->mode == X86EMUL_MODE_PROT64)
1953 return emulate_ud(ctxt);
1955 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1957 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1958 switch (ctxt->mode) {
1959 case X86EMUL_MODE_PROT32:
1960 if ((msr_data & 0xfffc) == 0x0)
1961 return emulate_gp(ctxt, 0);
1963 case X86EMUL_MODE_PROT64:
1964 if (msr_data == 0x0)
1965 return emulate_gp(ctxt, 0);
1969 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1970 cs_sel = (u16)msr_data;
1971 cs_sel &= ~SELECTOR_RPL_MASK;
1972 ss_sel = cs_sel + 8;
1973 ss_sel &= ~SELECTOR_RPL_MASK;
1974 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1979 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1980 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1982 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1985 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1986 c->regs[VCPU_REGS_RSP] = msr_data;
1988 return X86EMUL_CONTINUE;
1992 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1994 struct decode_cache *c = &ctxt->decode;
1995 struct desc_struct cs, ss;
2000 /* inject #GP if in real mode or Virtual 8086 mode */
2001 if (ctxt->mode == X86EMUL_MODE_REAL ||
2002 ctxt->mode == X86EMUL_MODE_VM86)
2003 return emulate_gp(ctxt, 0);
2005 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2007 if ((c->rex_prefix & 0x8) != 0x0)
2008 usermode = X86EMUL_MODE_PROT64;
2010 usermode = X86EMUL_MODE_PROT32;
2014 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2016 case X86EMUL_MODE_PROT32:
2017 cs_sel = (u16)(msr_data + 16);
2018 if ((msr_data & 0xfffc) == 0x0)
2019 return emulate_gp(ctxt, 0);
2020 ss_sel = (u16)(msr_data + 24);
2022 case X86EMUL_MODE_PROT64:
2023 cs_sel = (u16)(msr_data + 32);
2024 if (msr_data == 0x0)
2025 return emulate_gp(ctxt, 0);
2026 ss_sel = cs_sel + 8;
2031 cs_sel |= SELECTOR_RPL_MASK;
2032 ss_sel |= SELECTOR_RPL_MASK;
2034 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2035 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2037 c->eip = c->regs[VCPU_REGS_RDX];
2038 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2040 return X86EMUL_CONTINUE;
2043 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2044 struct x86_emulate_ops *ops)
2047 if (ctxt->mode == X86EMUL_MODE_REAL)
2049 if (ctxt->mode == X86EMUL_MODE_VM86)
2051 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2052 return ops->cpl(ctxt) > iopl;
2055 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2056 struct x86_emulate_ops *ops,
2059 struct desc_struct tr_seg;
2062 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2063 unsigned mask = (1 << len) - 1;
2066 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2069 if (desc_limit_scaled(&tr_seg) < 103)
2071 base = get_desc_base(&tr_seg);
2072 #ifdef CONFIG_X86_64
2073 base |= ((u64)base3) << 32;
2075 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2076 if (r != X86EMUL_CONTINUE)
2078 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2080 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2081 if (r != X86EMUL_CONTINUE)
2083 if ((perm >> bit_idx) & mask)
2088 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2089 struct x86_emulate_ops *ops,
2095 if (emulator_bad_iopl(ctxt, ops))
2096 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2099 ctxt->perm_ok = true;
2104 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2105 struct x86_emulate_ops *ops,
2106 struct tss_segment_16 *tss)
2108 struct decode_cache *c = &ctxt->decode;
2111 tss->flag = ctxt->eflags;
2112 tss->ax = c->regs[VCPU_REGS_RAX];
2113 tss->cx = c->regs[VCPU_REGS_RCX];
2114 tss->dx = c->regs[VCPU_REGS_RDX];
2115 tss->bx = c->regs[VCPU_REGS_RBX];
2116 tss->sp = c->regs[VCPU_REGS_RSP];
2117 tss->bp = c->regs[VCPU_REGS_RBP];
2118 tss->si = c->regs[VCPU_REGS_RSI];
2119 tss->di = c->regs[VCPU_REGS_RDI];
2121 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2122 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2123 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2124 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2125 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2128 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2129 struct x86_emulate_ops *ops,
2130 struct tss_segment_16 *tss)
2132 struct decode_cache *c = &ctxt->decode;
2136 ctxt->eflags = tss->flag | 2;
2137 c->regs[VCPU_REGS_RAX] = tss->ax;
2138 c->regs[VCPU_REGS_RCX] = tss->cx;
2139 c->regs[VCPU_REGS_RDX] = tss->dx;
2140 c->regs[VCPU_REGS_RBX] = tss->bx;
2141 c->regs[VCPU_REGS_RSP] = tss->sp;
2142 c->regs[VCPU_REGS_RBP] = tss->bp;
2143 c->regs[VCPU_REGS_RSI] = tss->si;
2144 c->regs[VCPU_REGS_RDI] = tss->di;
2147 * SDM says that segment selectors are loaded before segment
2150 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2151 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2152 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2153 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2154 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2157 * Now load segment descriptors. If fault happenes at this stage
2158 * it is handled in a context of new task
2160 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2161 if (ret != X86EMUL_CONTINUE)
2163 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2164 if (ret != X86EMUL_CONTINUE)
2166 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2167 if (ret != X86EMUL_CONTINUE)
2169 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2170 if (ret != X86EMUL_CONTINUE)
2172 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2173 if (ret != X86EMUL_CONTINUE)
2176 return X86EMUL_CONTINUE;
2179 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2180 struct x86_emulate_ops *ops,
2181 u16 tss_selector, u16 old_tss_sel,
2182 ulong old_tss_base, struct desc_struct *new_desc)
2184 struct tss_segment_16 tss_seg;
2186 u32 new_tss_base = get_desc_base(new_desc);
2188 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2190 if (ret != X86EMUL_CONTINUE)
2191 /* FIXME: need to provide precise fault address */
2194 save_state_to_tss16(ctxt, ops, &tss_seg);
2196 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2198 if (ret != X86EMUL_CONTINUE)
2199 /* FIXME: need to provide precise fault address */
2202 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2204 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: need to provide precise fault address */
2208 if (old_tss_sel != 0xffff) {
2209 tss_seg.prev_task_link = old_tss_sel;
2211 ret = ops->write_std(ctxt, new_tss_base,
2212 &tss_seg.prev_task_link,
2213 sizeof tss_seg.prev_task_link,
2215 if (ret != X86EMUL_CONTINUE)
2216 /* FIXME: need to provide precise fault address */
2220 return load_state_from_tss16(ctxt, ops, &tss_seg);
2223 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2224 struct x86_emulate_ops *ops,
2225 struct tss_segment_32 *tss)
2227 struct decode_cache *c = &ctxt->decode;
2229 tss->cr3 = ops->get_cr(ctxt, 3);
2231 tss->eflags = ctxt->eflags;
2232 tss->eax = c->regs[VCPU_REGS_RAX];
2233 tss->ecx = c->regs[VCPU_REGS_RCX];
2234 tss->edx = c->regs[VCPU_REGS_RDX];
2235 tss->ebx = c->regs[VCPU_REGS_RBX];
2236 tss->esp = c->regs[VCPU_REGS_RSP];
2237 tss->ebp = c->regs[VCPU_REGS_RBP];
2238 tss->esi = c->regs[VCPU_REGS_RSI];
2239 tss->edi = c->regs[VCPU_REGS_RDI];
2241 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2242 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2243 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2244 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2245 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2246 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2247 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2250 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2251 struct x86_emulate_ops *ops,
2252 struct tss_segment_32 *tss)
2254 struct decode_cache *c = &ctxt->decode;
2257 if (ops->set_cr(ctxt, 3, tss->cr3))
2258 return emulate_gp(ctxt, 0);
2260 ctxt->eflags = tss->eflags | 2;
2261 c->regs[VCPU_REGS_RAX] = tss->eax;
2262 c->regs[VCPU_REGS_RCX] = tss->ecx;
2263 c->regs[VCPU_REGS_RDX] = tss->edx;
2264 c->regs[VCPU_REGS_RBX] = tss->ebx;
2265 c->regs[VCPU_REGS_RSP] = tss->esp;
2266 c->regs[VCPU_REGS_RBP] = tss->ebp;
2267 c->regs[VCPU_REGS_RSI] = tss->esi;
2268 c->regs[VCPU_REGS_RDI] = tss->edi;
2271 * SDM says that segment selectors are loaded before segment
2274 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2275 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2276 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2277 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2278 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2279 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2280 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2283 * Now load segment descriptors. If fault happenes at this stage
2284 * it is handled in a context of new task
2286 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2287 if (ret != X86EMUL_CONTINUE)
2289 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2290 if (ret != X86EMUL_CONTINUE)
2292 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2293 if (ret != X86EMUL_CONTINUE)
2295 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2296 if (ret != X86EMUL_CONTINUE)
2298 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2299 if (ret != X86EMUL_CONTINUE)
2301 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2302 if (ret != X86EMUL_CONTINUE)
2304 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2305 if (ret != X86EMUL_CONTINUE)
2308 return X86EMUL_CONTINUE;
2311 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2312 struct x86_emulate_ops *ops,
2313 u16 tss_selector, u16 old_tss_sel,
2314 ulong old_tss_base, struct desc_struct *new_desc)
2316 struct tss_segment_32 tss_seg;
2318 u32 new_tss_base = get_desc_base(new_desc);
2320 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2322 if (ret != X86EMUL_CONTINUE)
2323 /* FIXME: need to provide precise fault address */
2326 save_state_to_tss32(ctxt, ops, &tss_seg);
2328 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2330 if (ret != X86EMUL_CONTINUE)
2331 /* FIXME: need to provide precise fault address */
2334 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2336 if (ret != X86EMUL_CONTINUE)
2337 /* FIXME: need to provide precise fault address */
2340 if (old_tss_sel != 0xffff) {
2341 tss_seg.prev_task_link = old_tss_sel;
2343 ret = ops->write_std(ctxt, new_tss_base,
2344 &tss_seg.prev_task_link,
2345 sizeof tss_seg.prev_task_link,
2347 if (ret != X86EMUL_CONTINUE)
2348 /* FIXME: need to provide precise fault address */
2352 return load_state_from_tss32(ctxt, ops, &tss_seg);
2355 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2356 struct x86_emulate_ops *ops,
2357 u16 tss_selector, int reason,
2358 bool has_error_code, u32 error_code)
2360 struct desc_struct curr_tss_desc, next_tss_desc;
2362 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2363 ulong old_tss_base =
2364 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2367 /* FIXME: old_tss_base == ~0 ? */
2369 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2370 if (ret != X86EMUL_CONTINUE)
2372 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2373 if (ret != X86EMUL_CONTINUE)
2376 /* FIXME: check that next_tss_desc is tss */
2378 if (reason != TASK_SWITCH_IRET) {
2379 if ((tss_selector & 3) > next_tss_desc.dpl ||
2380 ops->cpl(ctxt) > next_tss_desc.dpl)
2381 return emulate_gp(ctxt, 0);
2384 desc_limit = desc_limit_scaled(&next_tss_desc);
2385 if (!next_tss_desc.p ||
2386 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2387 desc_limit < 0x2b)) {
2388 emulate_ts(ctxt, tss_selector & 0xfffc);
2389 return X86EMUL_PROPAGATE_FAULT;
2392 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2393 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2394 write_segment_descriptor(ctxt, ops, old_tss_sel,
2398 if (reason == TASK_SWITCH_IRET)
2399 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2401 /* set back link to prev task only if NT bit is set in eflags
2402 note that old_tss_sel is not used afetr this point */
2403 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2404 old_tss_sel = 0xffff;
2406 if (next_tss_desc.type & 8)
2407 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2408 old_tss_base, &next_tss_desc);
2410 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2411 old_tss_base, &next_tss_desc);
2412 if (ret != X86EMUL_CONTINUE)
2415 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2416 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2418 if (reason != TASK_SWITCH_IRET) {
2419 next_tss_desc.type |= (1 << 1); /* set busy flag */
2420 write_segment_descriptor(ctxt, ops, tss_selector,
2424 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2425 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2427 if (has_error_code) {
2428 struct decode_cache *c = &ctxt->decode;
2430 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2432 c->src.val = (unsigned long) error_code;
2433 ret = em_push(ctxt);
2439 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2440 u16 tss_selector, int reason,
2441 bool has_error_code, u32 error_code)
2443 struct x86_emulate_ops *ops = ctxt->ops;
2444 struct decode_cache *c = &ctxt->decode;
2448 c->dst.type = OP_NONE;
2450 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2451 has_error_code, error_code);
2453 if (rc == X86EMUL_CONTINUE)
2456 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2459 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2460 int reg, struct operand *op)
2462 struct decode_cache *c = &ctxt->decode;
2463 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2465 register_address_increment(c, &c->regs[reg], df * op->bytes);
2466 op->addr.mem.ea = register_address(c, c->regs[reg]);
2467 op->addr.mem.seg = seg;
2470 static int em_das(struct x86_emulate_ctxt *ctxt)
2472 struct decode_cache *c = &ctxt->decode;
2474 bool af, cf, old_cf;
2476 cf = ctxt->eflags & X86_EFLAGS_CF;
2482 af = ctxt->eflags & X86_EFLAGS_AF;
2483 if ((al & 0x0f) > 9 || af) {
2485 cf = old_cf | (al >= 250);
2490 if (old_al > 0x99 || old_cf) {
2496 /* Set PF, ZF, SF */
2497 c->src.type = OP_IMM;
2500 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2501 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2503 ctxt->eflags |= X86_EFLAGS_CF;
2505 ctxt->eflags |= X86_EFLAGS_AF;
2506 return X86EMUL_CONTINUE;
2509 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2511 struct decode_cache *c = &ctxt->decode;
2516 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2519 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2520 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2521 return X86EMUL_CONTINUE;
2524 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2526 c->src.val = old_cs;
2528 if (rc != X86EMUL_CONTINUE)
2531 c->src.val = old_eip;
2532 return em_push(ctxt);
2535 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2537 struct decode_cache *c = &ctxt->decode;
2540 c->dst.type = OP_REG;
2541 c->dst.addr.reg = &c->eip;
2542 c->dst.bytes = c->op_bytes;
2543 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2544 if (rc != X86EMUL_CONTINUE)
2546 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2547 return X86EMUL_CONTINUE;
2550 static int em_add(struct x86_emulate_ctxt *ctxt)
2552 struct decode_cache *c = &ctxt->decode;
2554 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2555 return X86EMUL_CONTINUE;
2558 static int em_or(struct x86_emulate_ctxt *ctxt)
2560 struct decode_cache *c = &ctxt->decode;
2562 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2563 return X86EMUL_CONTINUE;
2566 static int em_adc(struct x86_emulate_ctxt *ctxt)
2568 struct decode_cache *c = &ctxt->decode;
2570 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2571 return X86EMUL_CONTINUE;
2574 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2576 struct decode_cache *c = &ctxt->decode;
2578 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2579 return X86EMUL_CONTINUE;
2582 static int em_and(struct x86_emulate_ctxt *ctxt)
2584 struct decode_cache *c = &ctxt->decode;
2586 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2587 return X86EMUL_CONTINUE;
2590 static int em_sub(struct x86_emulate_ctxt *ctxt)
2592 struct decode_cache *c = &ctxt->decode;
2594 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2595 return X86EMUL_CONTINUE;
2598 static int em_xor(struct x86_emulate_ctxt *ctxt)
2600 struct decode_cache *c = &ctxt->decode;
2602 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2603 return X86EMUL_CONTINUE;
2606 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2608 struct decode_cache *c = &ctxt->decode;
2610 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2611 /* Disable writeback. */
2612 c->dst.type = OP_NONE;
2613 return X86EMUL_CONTINUE;
2616 static int em_imul(struct x86_emulate_ctxt *ctxt)
2618 struct decode_cache *c = &ctxt->decode;
2620 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2621 return X86EMUL_CONTINUE;
2624 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2626 struct decode_cache *c = &ctxt->decode;
2628 c->dst.val = c->src2.val;
2629 return em_imul(ctxt);
2632 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2634 struct decode_cache *c = &ctxt->decode;
2636 c->dst.type = OP_REG;
2637 c->dst.bytes = c->src.bytes;
2638 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2639 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2641 return X86EMUL_CONTINUE;
2644 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2646 struct decode_cache *c = &ctxt->decode;
2649 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2650 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2651 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2652 return X86EMUL_CONTINUE;
2655 static int em_mov(struct x86_emulate_ctxt *ctxt)
2657 struct decode_cache *c = &ctxt->decode;
2658 c->dst.val = c->src.val;
2659 return X86EMUL_CONTINUE;
2662 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2664 struct decode_cache *c = &ctxt->decode;
2665 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2666 return X86EMUL_CONTINUE;
2669 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2671 struct decode_cache *c = &ctxt->decode;
2675 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2676 if (rc == X86EMUL_CONTINUE)
2677 ctxt->ops->invlpg(ctxt, linear);
2678 /* Disable writeback. */
2679 c->dst.type = OP_NONE;
2680 return X86EMUL_CONTINUE;
2683 static int em_clts(struct x86_emulate_ctxt *ctxt)
2687 cr0 = ctxt->ops->get_cr(ctxt, 0);
2689 ctxt->ops->set_cr(ctxt, 0, cr0);
2690 return X86EMUL_CONTINUE;
2693 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2695 struct decode_cache *c = &ctxt->decode;
2698 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2699 return X86EMUL_UNHANDLEABLE;
2701 rc = ctxt->ops->fix_hypercall(ctxt);
2702 if (rc != X86EMUL_CONTINUE)
2705 /* Let the processor re-execute the fixed hypercall */
2707 /* Disable writeback. */
2708 c->dst.type = OP_NONE;
2709 return X86EMUL_CONTINUE;
2712 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2714 struct decode_cache *c = &ctxt->decode;
2715 struct desc_ptr desc_ptr;
2718 rc = read_descriptor(ctxt, c->src.addr.mem,
2719 &desc_ptr.size, &desc_ptr.address,
2721 if (rc != X86EMUL_CONTINUE)
2723 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2724 /* Disable writeback. */
2725 c->dst.type = OP_NONE;
2726 return X86EMUL_CONTINUE;
2729 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2731 struct decode_cache *c = &ctxt->decode;
2734 rc = ctxt->ops->fix_hypercall(ctxt);
2736 /* Disable writeback. */
2737 c->dst.type = OP_NONE;
2741 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2743 struct decode_cache *c = &ctxt->decode;
2744 struct desc_ptr desc_ptr;
2747 rc = read_descriptor(ctxt, c->src.addr.mem,
2748 &desc_ptr.size, &desc_ptr.address,
2750 if (rc != X86EMUL_CONTINUE)
2752 ctxt->ops->set_idt(ctxt, &desc_ptr);
2753 /* Disable writeback. */
2754 c->dst.type = OP_NONE;
2755 return X86EMUL_CONTINUE;
2758 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2760 struct decode_cache *c = &ctxt->decode;
2763 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2764 return X86EMUL_CONTINUE;
2767 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2769 struct decode_cache *c = &ctxt->decode;
2770 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2771 | (c->src.val & 0x0f));
2772 c->dst.type = OP_NONE;
2773 return X86EMUL_CONTINUE;
2776 static bool valid_cr(int nr)
2788 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2790 struct decode_cache *c = &ctxt->decode;
2792 if (!valid_cr(c->modrm_reg))
2793 return emulate_ud(ctxt);
2795 return X86EMUL_CONTINUE;
2798 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2800 struct decode_cache *c = &ctxt->decode;
2801 u64 new_val = c->src.val64;
2802 int cr = c->modrm_reg;
2805 static u64 cr_reserved_bits[] = {
2806 0xffffffff00000000ULL,
2807 0, 0, 0, /* CR3 checked later */
2814 return emulate_ud(ctxt);
2816 if (new_val & cr_reserved_bits[cr])
2817 return emulate_gp(ctxt, 0);
2822 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2823 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2824 return emulate_gp(ctxt, 0);
2826 cr4 = ctxt->ops->get_cr(ctxt, 4);
2827 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2829 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2830 !(cr4 & X86_CR4_PAE))
2831 return emulate_gp(ctxt, 0);
2838 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2839 if (efer & EFER_LMA)
2840 rsvd = CR3_L_MODE_RESERVED_BITS;
2841 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2842 rsvd = CR3_PAE_RESERVED_BITS;
2843 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2844 rsvd = CR3_NONPAE_RESERVED_BITS;
2847 return emulate_gp(ctxt, 0);
2854 cr4 = ctxt->ops->get_cr(ctxt, 4);
2855 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2857 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2858 return emulate_gp(ctxt, 0);
2864 return X86EMUL_CONTINUE;
2867 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2871 ctxt->ops->get_dr(ctxt, 7, &dr7);
2873 /* Check if DR7.Global_Enable is set */
2874 return dr7 & (1 << 13);
2877 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2879 struct decode_cache *c = &ctxt->decode;
2880 int dr = c->modrm_reg;
2884 return emulate_ud(ctxt);
2886 cr4 = ctxt->ops->get_cr(ctxt, 4);
2887 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2888 return emulate_ud(ctxt);
2890 if (check_dr7_gd(ctxt))
2891 return emulate_db(ctxt);
2893 return X86EMUL_CONTINUE;
2896 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2898 struct decode_cache *c = &ctxt->decode;
2899 u64 new_val = c->src.val64;
2900 int dr = c->modrm_reg;
2902 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2903 return emulate_gp(ctxt, 0);
2905 return check_dr_read(ctxt);
2908 static int check_svme(struct x86_emulate_ctxt *ctxt)
2912 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2914 if (!(efer & EFER_SVME))
2915 return emulate_ud(ctxt);
2917 return X86EMUL_CONTINUE;
2920 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2922 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2924 /* Valid physical address? */
2925 if (rax & 0xffff000000000000ULL)
2926 return emulate_gp(ctxt, 0);
2928 return check_svme(ctxt);
2931 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2933 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2935 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2936 return emulate_ud(ctxt);
2938 return X86EMUL_CONTINUE;
2941 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2943 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2944 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2946 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2948 return emulate_gp(ctxt, 0);
2950 return X86EMUL_CONTINUE;
2953 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2955 struct decode_cache *c = &ctxt->decode;
2957 c->dst.bytes = min(c->dst.bytes, 4u);
2958 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2959 return emulate_gp(ctxt, 0);
2961 return X86EMUL_CONTINUE;
2964 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2966 struct decode_cache *c = &ctxt->decode;
2968 c->src.bytes = min(c->src.bytes, 4u);
2969 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2970 return emulate_gp(ctxt, 0);
2972 return X86EMUL_CONTINUE;
2975 #define D(_y) { .flags = (_y) }
2976 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2977 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2978 .check_perm = (_p) }
2980 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2981 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2982 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2983 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2984 #define II(_f, _e, _i) \
2985 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2986 #define IIP(_f, _e, _i, _p) \
2987 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2988 .check_perm = (_p) }
2989 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2991 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2992 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2993 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2995 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2996 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2997 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2999 static struct opcode group7_rm1[] = {
3000 DI(SrcNone | ModRM | Priv, monitor),
3001 DI(SrcNone | ModRM | Priv, mwait),
3005 static struct opcode group7_rm3[] = {
3006 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3007 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3008 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3009 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3010 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3011 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3012 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3013 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3016 static struct opcode group7_rm7[] = {
3018 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3022 static struct opcode group1[] = {
3033 static struct opcode group1A[] = {
3034 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3037 static struct opcode group3[] = {
3038 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3039 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3040 X4(D(SrcMem | ModRM)),
3043 static struct opcode group4[] = {
3044 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3048 static struct opcode group5[] = {
3049 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3050 D(SrcMem | ModRM | Stack),
3051 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3052 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3053 D(SrcMem | ModRM | Stack), N,
3056 static struct opcode group6[] = {
3057 DI(ModRM | Prot, sldt),
3058 DI(ModRM | Prot, str),
3059 DI(ModRM | Prot | Priv, lldt),
3060 DI(ModRM | Prot | Priv, ltr),
3064 static struct group_dual group7 = { {
3065 DI(ModRM | Mov | DstMem | Priv, sgdt),
3066 DI(ModRM | Mov | DstMem | Priv, sidt),
3067 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3068 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3069 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3070 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3071 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3073 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3075 N, EXT(0, group7_rm3),
3076 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3077 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3080 static struct opcode group8[] = {
3082 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3083 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3086 static struct group_dual group9 = { {
3087 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3089 N, N, N, N, N, N, N, N,
3092 static struct opcode group11[] = {
3093 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3096 static struct gprefix pfx_0f_6f_0f_7f = {
3097 N, N, N, I(Sse, em_movdqu),
3100 static struct opcode opcode_table[256] = {
3102 I6ALU(Lock, em_add),
3103 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3106 D(ImplicitOps | Stack | No64), N,
3108 I6ALU(Lock, em_adc),
3109 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3111 I6ALU(Lock, em_sbb),
3112 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3114 I6ALU(Lock, em_and), N, N,
3116 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3118 I6ALU(Lock, em_xor), N, N,
3120 I6ALU(0, em_cmp), N, N,
3124 X8(I(SrcReg | Stack, em_push)),
3126 X8(I(DstReg | Stack, em_pop)),
3128 I(ImplicitOps | Stack | No64, em_pusha),
3129 I(ImplicitOps | Stack | No64, em_popa),
3130 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3133 I(SrcImm | Mov | Stack, em_push),
3134 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3135 I(SrcImmByte | Mov | Stack, em_push),
3136 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3137 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3138 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3142 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3143 G(DstMem | SrcImm | ModRM | Group, group1),
3144 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3145 G(DstMem | SrcImmByte | ModRM | Group, group1),
3146 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3148 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3149 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3150 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3151 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3153 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3155 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3156 I(SrcImmFAddr | No64, em_call_far), N,
3157 II(ImplicitOps | Stack, em_pushf, pushf),
3158 II(ImplicitOps | Stack, em_popf, popf), N, N,
3160 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3161 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3162 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3163 I2bv(SrcSI | DstDI | String, em_cmp),
3165 D2bv(DstAcc | SrcImm),
3166 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3167 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3168 I2bv(SrcAcc | DstDI | String, em_cmp),
3170 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3172 X8(I(DstReg | SrcImm | Mov, em_mov)),
3174 D2bv(DstMem | SrcImmByte | ModRM),
3175 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3176 D(ImplicitOps | Stack),
3177 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3178 G(ByteOp, group11), G(0, group11),
3180 N, N, N, D(ImplicitOps | Stack),
3181 D(ImplicitOps), DI(SrcImmByte, intn),
3182 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3184 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3187 N, N, N, N, N, N, N, N,
3190 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3191 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3193 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3194 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3195 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3196 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3198 N, DI(ImplicitOps, icebp), N, N,
3199 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3200 G(ByteOp, group3), G(0, group3),
3202 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3203 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3206 static struct opcode twobyte_table[256] = {
3208 G(0, group6), GD(0, &group7), N, N,
3209 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3210 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3211 N, D(ImplicitOps | ModRM), N, N,
3213 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3215 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3216 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3217 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3218 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3220 N, N, N, N, N, N, N, N,
3222 DI(ImplicitOps | Priv, wrmsr),
3223 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3224 DI(ImplicitOps | Priv, rdmsr),
3225 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3226 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3228 N, N, N, N, N, N, N, N,
3230 X16(D(DstReg | SrcMem | ModRM | Mov)),
3232 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3237 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3242 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3246 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3248 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3249 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3250 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3251 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3253 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3254 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3255 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3256 D(DstMem | SrcReg | Src2CL | ModRM),
3257 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3259 D2bv(DstMem | SrcReg | ModRM | Lock),
3260 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3261 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3262 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3265 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3266 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3267 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3269 D2bv(DstMem | SrcReg | ModRM | Lock),
3270 N, D(DstMem | SrcReg | ModRM | Mov),
3271 N, N, N, GD(0, &group9),
3272 N, N, N, N, N, N, N, N,
3274 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3276 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3278 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3294 static unsigned imm_size(struct decode_cache *c)
3298 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3304 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3305 unsigned size, bool sign_extension)
3307 struct decode_cache *c = &ctxt->decode;
3308 struct x86_emulate_ops *ops = ctxt->ops;
3309 int rc = X86EMUL_CONTINUE;
3313 op->addr.mem.ea = c->eip;
3314 /* NB. Immediates are sign-extended as necessary. */
3315 switch (op->bytes) {
3317 op->val = insn_fetch(s8, 1, c->eip);
3320 op->val = insn_fetch(s16, 2, c->eip);
3323 op->val = insn_fetch(s32, 4, c->eip);
3326 if (!sign_extension) {
3327 switch (op->bytes) {
3335 op->val &= 0xffffffff;
3344 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3346 struct x86_emulate_ops *ops = ctxt->ops;
3347 struct decode_cache *c = &ctxt->decode;
3348 int rc = X86EMUL_CONTINUE;
3349 int mode = ctxt->mode;
3350 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3351 bool op_prefix = false;
3352 struct opcode opcode;
3353 struct operand memop = { .type = OP_NONE };
3356 c->fetch.start = c->eip;
3357 c->fetch.end = c->fetch.start + insn_len;
3359 memcpy(c->fetch.data, insn, insn_len);
3362 case X86EMUL_MODE_REAL:
3363 case X86EMUL_MODE_VM86:
3364 case X86EMUL_MODE_PROT16:
3365 def_op_bytes = def_ad_bytes = 2;
3367 case X86EMUL_MODE_PROT32:
3368 def_op_bytes = def_ad_bytes = 4;
3370 #ifdef CONFIG_X86_64
3371 case X86EMUL_MODE_PROT64:
3380 c->op_bytes = def_op_bytes;
3381 c->ad_bytes = def_ad_bytes;
3383 /* Legacy prefixes. */
3385 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3386 case 0x66: /* operand-size override */
3388 /* switch between 2/4 bytes */
3389 c->op_bytes = def_op_bytes ^ 6;
3391 case 0x67: /* address-size override */
3392 if (mode == X86EMUL_MODE_PROT64)
3393 /* switch between 4/8 bytes */
3394 c->ad_bytes = def_ad_bytes ^ 12;
3396 /* switch between 2/4 bytes */
3397 c->ad_bytes = def_ad_bytes ^ 6;
3399 case 0x26: /* ES override */
3400 case 0x2e: /* CS override */
3401 case 0x36: /* SS override */
3402 case 0x3e: /* DS override */
3403 set_seg_override(c, (c->b >> 3) & 3);
3405 case 0x64: /* FS override */
3406 case 0x65: /* GS override */
3407 set_seg_override(c, c->b & 7);
3409 case 0x40 ... 0x4f: /* REX */
3410 if (mode != X86EMUL_MODE_PROT64)
3412 c->rex_prefix = c->b;
3414 case 0xf0: /* LOCK */
3417 case 0xf2: /* REPNE/REPNZ */
3418 case 0xf3: /* REP/REPE/REPZ */
3419 c->rep_prefix = c->b;
3425 /* Any legacy prefix after a REX prefix nullifies its effect. */
3433 if (c->rex_prefix & 8)
3434 c->op_bytes = 8; /* REX.W */
3436 /* Opcode byte(s). */
3437 opcode = opcode_table[c->b];
3438 /* Two-byte opcode? */
3441 c->b = insn_fetch(u8, 1, c->eip);
3442 opcode = twobyte_table[c->b];
3444 c->d = opcode.flags;
3446 while (c->d & GroupMask) {
3447 switch (c->d & GroupMask) {
3449 c->modrm = insn_fetch(u8, 1, c->eip);
3451 goffset = (c->modrm >> 3) & 7;
3452 opcode = opcode.u.group[goffset];
3455 c->modrm = insn_fetch(u8, 1, c->eip);
3457 goffset = (c->modrm >> 3) & 7;
3458 if ((c->modrm >> 6) == 3)
3459 opcode = opcode.u.gdual->mod3[goffset];
3461 opcode = opcode.u.gdual->mod012[goffset];
3464 goffset = c->modrm & 7;
3465 opcode = opcode.u.group[goffset];
3468 if (c->rep_prefix && op_prefix)
3469 return X86EMUL_UNHANDLEABLE;
3470 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3471 switch (simd_prefix) {
3472 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3473 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3474 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3475 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3479 return X86EMUL_UNHANDLEABLE;
3483 c->d |= opcode.flags;
3486 c->execute = opcode.u.execute;
3487 c->check_perm = opcode.check_perm;
3488 c->intercept = opcode.intercept;
3491 if (c->d == 0 || (c->d & Undefined))
3494 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3497 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3500 if (c->d & Op3264) {
3501 if (mode == X86EMUL_MODE_PROT64)
3510 /* ModRM and SIB bytes. */
3512 rc = decode_modrm(ctxt, ops, &memop);
3513 if (!c->has_seg_override)
3514 set_seg_override(c, c->modrm_seg);
3515 } else if (c->d & MemAbs)
3516 rc = decode_abs(ctxt, ops, &memop);
3517 if (rc != X86EMUL_CONTINUE)
3520 if (!c->has_seg_override)
3521 set_seg_override(c, VCPU_SREG_DS);
3523 memop.addr.mem.seg = seg_override(ctxt, c);
3525 if (memop.type == OP_MEM && c->ad_bytes != 8)
3526 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3528 if (memop.type == OP_MEM && c->rip_relative)
3529 memop.addr.mem.ea += c->eip;
3532 * Decode and fetch the source operand: register, memory
3535 switch (c->d & SrcMask) {
3539 decode_register_operand(ctxt, &c->src, c, 0);
3548 memop.bytes = (c->d & ByteOp) ? 1 :
3554 rc = decode_imm(ctxt, &c->src, 2, false);
3557 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3560 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3563 rc = decode_imm(ctxt, &c->src, 1, true);
3566 rc = decode_imm(ctxt, &c->src, 1, false);
3569 c->src.type = OP_REG;
3570 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3571 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3572 fetch_register_operand(&c->src);
3579 c->src.type = OP_MEM;
3580 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3581 c->src.addr.mem.ea =
3582 register_address(c, c->regs[VCPU_REGS_RSI]);
3583 c->src.addr.mem.seg = seg_override(ctxt, c);
3587 c->src.type = OP_IMM;
3588 c->src.addr.mem.ea = c->eip;
3589 c->src.bytes = c->op_bytes + 2;
3590 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3593 memop.bytes = c->op_bytes + 2;
3598 if (rc != X86EMUL_CONTINUE)
3602 * Decode and fetch the second source operand: register, memory
3605 switch (c->d & Src2Mask) {
3610 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3613 rc = decode_imm(ctxt, &c->src2, 1, true);
3620 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3624 if (rc != X86EMUL_CONTINUE)
3627 /* Decode and fetch the destination operand: register or memory. */
3628 switch (c->d & DstMask) {
3630 decode_register_operand(ctxt, &c->dst, c,
3631 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3634 c->dst.type = OP_IMM;
3635 c->dst.addr.mem.ea = c->eip;
3637 c->dst.val = insn_fetch(u8, 1, c->eip);
3642 if ((c->d & DstMask) == DstMem64)
3645 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3647 fetch_bit_operand(c);
3648 c->dst.orig_val = c->dst.val;
3651 c->dst.type = OP_REG;
3652 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3653 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3654 fetch_register_operand(&c->dst);
3655 c->dst.orig_val = c->dst.val;
3658 c->dst.type = OP_MEM;
3659 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3660 c->dst.addr.mem.ea =
3661 register_address(c, c->regs[VCPU_REGS_RDI]);
3662 c->dst.addr.mem.seg = VCPU_SREG_ES;
3666 /* Special instructions do their own operand decoding. */
3668 c->dst.type = OP_NONE; /* Disable writeback. */
3673 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3676 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3678 struct decode_cache *c = &ctxt->decode;
3680 /* The second termination condition only applies for REPE
3681 * and REPNE. Test if the repeat string operation prefix is
3682 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3683 * corresponding termination condition according to:
3684 * - if REPE/REPZ and ZF = 0 then done
3685 * - if REPNE/REPNZ and ZF = 1 then done
3687 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3688 (c->b == 0xae) || (c->b == 0xaf))
3689 && (((c->rep_prefix == REPE_PREFIX) &&
3690 ((ctxt->eflags & EFLG_ZF) == 0))
3691 || ((c->rep_prefix == REPNE_PREFIX) &&
3692 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3699 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3701 struct x86_emulate_ops *ops = ctxt->ops;
3703 struct decode_cache *c = &ctxt->decode;
3704 int rc = X86EMUL_CONTINUE;
3705 int saved_dst_type = c->dst.type;
3706 int irq; /* Used for int 3, int, and into */
3708 ctxt->decode.mem_read.pos = 0;
3710 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3711 rc = emulate_ud(ctxt);
3715 /* LOCK prefix is allowed only with some instructions */
3716 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3717 rc = emulate_ud(ctxt);
3721 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3722 rc = emulate_ud(ctxt);
3727 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3728 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3729 rc = emulate_ud(ctxt);
3733 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3734 rc = emulate_nm(ctxt);
3738 if (unlikely(ctxt->guest_mode) && c->intercept) {
3739 rc = emulator_check_intercept(ctxt, c->intercept,
3740 X86_ICPT_PRE_EXCEPT);
3741 if (rc != X86EMUL_CONTINUE)
3745 /* Privileged instruction can be executed only in CPL=0 */
3746 if ((c->d & Priv) && ops->cpl(ctxt)) {
3747 rc = emulate_gp(ctxt, 0);
3751 /* Instruction can only be executed in protected mode */
3752 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3753 rc = emulate_ud(ctxt);
3757 /* Do instruction specific permission checks */
3758 if (c->check_perm) {
3759 rc = c->check_perm(ctxt);
3760 if (rc != X86EMUL_CONTINUE)
3764 if (unlikely(ctxt->guest_mode) && c->intercept) {
3765 rc = emulator_check_intercept(ctxt, c->intercept,
3766 X86_ICPT_POST_EXCEPT);
3767 if (rc != X86EMUL_CONTINUE)
3771 if (c->rep_prefix && (c->d & String)) {
3772 /* All REP prefixes have the same first termination condition */
3773 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3779 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3780 rc = segmented_read(ctxt, c->src.addr.mem,
3781 c->src.valptr, c->src.bytes);
3782 if (rc != X86EMUL_CONTINUE)
3784 c->src.orig_val64 = c->src.val64;
3787 if (c->src2.type == OP_MEM) {
3788 rc = segmented_read(ctxt, c->src2.addr.mem,
3789 &c->src2.val, c->src2.bytes);
3790 if (rc != X86EMUL_CONTINUE)
3794 if ((c->d & DstMask) == ImplicitOps)
3798 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3799 /* optimisation - avoid slow emulated read if Mov */
3800 rc = segmented_read(ctxt, c->dst.addr.mem,
3801 &c->dst.val, c->dst.bytes);
3802 if (rc != X86EMUL_CONTINUE)
3805 c->dst.orig_val = c->dst.val;
3809 if (unlikely(ctxt->guest_mode) && c->intercept) {
3810 rc = emulator_check_intercept(ctxt, c->intercept,
3811 X86_ICPT_POST_MEMACCESS);
3812 if (rc != X86EMUL_CONTINUE)
3817 rc = c->execute(ctxt);
3818 if (rc != X86EMUL_CONTINUE)
3827 case 0x06: /* push es */
3828 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3830 case 0x07: /* pop es */
3831 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3833 case 0x0e: /* push cs */
3834 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3836 case 0x16: /* push ss */
3837 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3839 case 0x17: /* pop ss */
3840 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3842 case 0x1e: /* push ds */
3843 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3845 case 0x1f: /* pop ds */
3846 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3848 case 0x40 ... 0x47: /* inc r16/r32 */
3849 emulate_1op("inc", c->dst, ctxt->eflags);
3851 case 0x48 ... 0x4f: /* dec r16/r32 */
3852 emulate_1op("dec", c->dst, ctxt->eflags);
3854 case 0x63: /* movsxd */
3855 if (ctxt->mode != X86EMUL_MODE_PROT64)
3856 goto cannot_emulate;
3857 c->dst.val = (s32) c->src.val;
3859 case 0x6c: /* insb */
3860 case 0x6d: /* insw/insd */
3861 c->src.val = c->regs[VCPU_REGS_RDX];
3863 case 0x6e: /* outsb */
3864 case 0x6f: /* outsw/outsd */
3865 c->dst.val = c->regs[VCPU_REGS_RDX];
3868 case 0x70 ... 0x7f: /* jcc (short) */
3869 if (test_cc(c->b, ctxt->eflags))
3870 jmp_rel(c, c->src.val);
3874 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3876 case 0x86 ... 0x87: /* xchg */
3878 /* Write back the register source. */
3879 c->src.val = c->dst.val;
3880 write_register_operand(&c->src);
3882 * Write back the memory destination with implicit LOCK
3885 c->dst.val = c->src.orig_val;
3888 case 0x8c: /* mov r/m, sreg */
3889 if (c->modrm_reg > VCPU_SREG_GS) {
3890 rc = emulate_ud(ctxt);
3893 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3895 case 0x8d: /* lea r16/r32, m */
3896 c->dst.val = c->src.addr.mem.ea;
3898 case 0x8e: { /* mov seg, r/m16 */
3903 if (c->modrm_reg == VCPU_SREG_CS ||
3904 c->modrm_reg > VCPU_SREG_GS) {
3905 rc = emulate_ud(ctxt);
3909 if (c->modrm_reg == VCPU_SREG_SS)
3910 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3912 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3914 c->dst.type = OP_NONE; /* Disable writeback. */
3917 case 0x8f: /* pop (sole member of Grp1a) */
3918 rc = em_grp1a(ctxt);
3920 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3921 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3924 case 0x98: /* cbw/cwde/cdqe */
3925 switch (c->op_bytes) {
3926 case 2: c->dst.val = (s8)c->dst.val; break;
3927 case 4: c->dst.val = (s16)c->dst.val; break;
3928 case 8: c->dst.val = (s32)c->dst.val; break;
3931 case 0xa8 ... 0xa9: /* test ax, imm */
3936 case 0xc3: /* ret */
3937 c->dst.type = OP_REG;
3938 c->dst.addr.reg = &c->eip;
3939 c->dst.bytes = c->op_bytes;
3942 case 0xc4: /* les */
3943 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3945 case 0xc5: /* lds */
3946 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3948 case 0xcb: /* ret far */
3949 rc = emulate_ret_far(ctxt, ops);
3951 case 0xcc: /* int3 */
3954 case 0xcd: /* int n */
3957 rc = emulate_int(ctxt, ops, irq);
3959 case 0xce: /* into */
3960 if (ctxt->eflags & EFLG_OF) {
3965 case 0xcf: /* iret */
3966 rc = emulate_iret(ctxt, ops);
3968 case 0xd0 ... 0xd1: /* Grp2 */
3971 case 0xd2 ... 0xd3: /* Grp2 */
3972 c->src.val = c->regs[VCPU_REGS_RCX];
3975 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3976 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3977 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3978 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3979 jmp_rel(c, c->src.val);
3981 case 0xe3: /* jcxz/jecxz/jrcxz */
3982 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3983 jmp_rel(c, c->src.val);
3985 case 0xe4: /* inb */
3988 case 0xe6: /* outb */
3989 case 0xe7: /* out */
3991 case 0xe8: /* call (near) */ {
3992 long int rel = c->src.val;
3993 c->src.val = (unsigned long) c->eip;
3998 case 0xe9: /* jmp rel */
4000 case 0xea: { /* jmp far */
4003 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
4005 rc = load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS);
4006 if (rc != X86EMUL_CONTINUE)
4010 memcpy(&c->eip, c->src.valptr, c->op_bytes);
4014 jmp: /* jmp rel short */
4015 jmp_rel(c, c->src.val);
4016 c->dst.type = OP_NONE; /* Disable writeback. */
4018 case 0xec: /* in al,dx */
4019 case 0xed: /* in (e/r)ax,dx */
4020 c->src.val = c->regs[VCPU_REGS_RDX];
4022 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4024 goto done; /* IO is needed */
4026 case 0xee: /* out dx,al */
4027 case 0xef: /* out dx,(e/r)ax */
4028 c->dst.val = c->regs[VCPU_REGS_RDX];
4030 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4032 c->dst.type = OP_NONE; /* Disable writeback. */
4034 case 0xf4: /* hlt */
4035 ctxt->ops->halt(ctxt);
4037 case 0xf5: /* cmc */
4038 /* complement carry flag from eflags reg */
4039 ctxt->eflags ^= EFLG_CF;
4041 case 0xf6 ... 0xf7: /* Grp3 */
4044 case 0xf8: /* clc */
4045 ctxt->eflags &= ~EFLG_CF;
4047 case 0xf9: /* stc */
4048 ctxt->eflags |= EFLG_CF;
4050 case 0xfa: /* cli */
4051 if (emulator_bad_iopl(ctxt, ops)) {
4052 rc = emulate_gp(ctxt, 0);
4055 ctxt->eflags &= ~X86_EFLAGS_IF;
4057 case 0xfb: /* sti */
4058 if (emulator_bad_iopl(ctxt, ops)) {
4059 rc = emulate_gp(ctxt, 0);
4062 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4063 ctxt->eflags |= X86_EFLAGS_IF;
4066 case 0xfc: /* cld */
4067 ctxt->eflags &= ~EFLG_DF;
4069 case 0xfd: /* std */
4070 ctxt->eflags |= EFLG_DF;
4072 case 0xfe: /* Grp4 */
4073 rc = em_grp45(ctxt);
4075 case 0xff: /* Grp5 */
4076 if (c->modrm_reg == 5)
4078 rc = em_grp45(ctxt);
4081 goto cannot_emulate;
4084 if (rc != X86EMUL_CONTINUE)
4088 rc = writeback(ctxt);
4089 if (rc != X86EMUL_CONTINUE)
4093 * restore dst type in case the decoding will be reused
4094 * (happens for string instruction )
4096 c->dst.type = saved_dst_type;
4098 if ((c->d & SrcMask) == SrcSI)
4099 string_addr_inc(ctxt, seg_override(ctxt, c),
4100 VCPU_REGS_RSI, &c->src);
4102 if ((c->d & DstMask) == DstDI)
4103 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4106 if (c->rep_prefix && (c->d & String)) {
4107 struct read_cache *r = &ctxt->decode.io_read;
4108 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4110 if (!string_insn_completed(ctxt)) {
4112 * Re-enter guest when pio read ahead buffer is empty
4113 * or, if it is not used, after each 1024 iteration.
4115 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4116 (r->end == 0 || r->end != r->pos)) {
4118 * Reset read cache. Usually happens before
4119 * decode, but since instruction is restarted
4120 * we have to do it here.
4122 ctxt->decode.mem_read.end = 0;
4123 return EMULATION_RESTART;
4125 goto done; /* skip rip writeback */
4132 if (rc == X86EMUL_PROPAGATE_FAULT)
4133 ctxt->have_exception = true;
4134 if (rc == X86EMUL_INTERCEPTED)
4135 return EMULATION_INTERCEPTED;
4137 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4141 case 0x05: /* syscall */
4142 rc = emulate_syscall(ctxt, ops);
4147 case 0x09: /* wbinvd */
4148 (ctxt->ops->wbinvd)(ctxt);
4150 case 0x08: /* invd */
4151 case 0x0d: /* GrpP (prefetch) */
4152 case 0x18: /* Grp16 (prefetch/nop) */
4154 case 0x20: /* mov cr, reg */
4155 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4157 case 0x21: /* mov from dr to reg */
4158 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4160 case 0x22: /* mov reg, cr */
4161 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4162 emulate_gp(ctxt, 0);
4163 rc = X86EMUL_PROPAGATE_FAULT;
4166 c->dst.type = OP_NONE;
4168 case 0x23: /* mov from reg to dr */
4169 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4170 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4171 ~0ULL : ~0U)) < 0) {
4172 /* #UD condition is already handled by the code above */
4173 emulate_gp(ctxt, 0);
4174 rc = X86EMUL_PROPAGATE_FAULT;
4178 c->dst.type = OP_NONE; /* no writeback */
4182 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4183 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4184 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4185 emulate_gp(ctxt, 0);
4186 rc = X86EMUL_PROPAGATE_FAULT;
4189 rc = X86EMUL_CONTINUE;
4193 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4194 emulate_gp(ctxt, 0);
4195 rc = X86EMUL_PROPAGATE_FAULT;
4198 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4199 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4201 rc = X86EMUL_CONTINUE;
4203 case 0x34: /* sysenter */
4204 rc = emulate_sysenter(ctxt, ops);
4206 case 0x35: /* sysexit */
4207 rc = emulate_sysexit(ctxt, ops);
4209 case 0x40 ... 0x4f: /* cmov */
4210 c->dst.val = c->dst.orig_val = c->src.val;
4211 if (!test_cc(c->b, ctxt->eflags))
4212 c->dst.type = OP_NONE; /* no writeback */
4214 case 0x80 ... 0x8f: /* jnz rel, etc*/
4215 if (test_cc(c->b, ctxt->eflags))
4216 jmp_rel(c, c->src.val);
4218 case 0x90 ... 0x9f: /* setcc r/m8 */
4219 c->dst.val = test_cc(c->b, ctxt->eflags);
4221 case 0xa0: /* push fs */
4222 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4224 case 0xa1: /* pop fs */
4225 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4229 c->dst.type = OP_NONE;
4230 /* only subword offset */
4231 c->src.val &= (c->dst.bytes << 3) - 1;
4232 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4234 case 0xa4: /* shld imm8, r, r/m */
4235 case 0xa5: /* shld cl, r, r/m */
4236 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4238 case 0xa8: /* push gs */
4239 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4241 case 0xa9: /* pop gs */
4242 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4246 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4248 case 0xac: /* shrd imm8, r, r/m */
4249 case 0xad: /* shrd cl, r, r/m */
4250 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4252 case 0xae: /* clflush */
4254 case 0xb0 ... 0xb1: /* cmpxchg */
4256 * Save real source value, then compare EAX against
4259 c->src.orig_val = c->src.val;
4260 c->src.val = c->regs[VCPU_REGS_RAX];
4261 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4262 if (ctxt->eflags & EFLG_ZF) {
4263 /* Success: write back to memory. */
4264 c->dst.val = c->src.orig_val;
4266 /* Failure: write the value we saw to EAX. */
4267 c->dst.type = OP_REG;
4268 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4271 case 0xb2: /* lss */
4272 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4276 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4278 case 0xb4: /* lfs */
4279 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4281 case 0xb5: /* lgs */
4282 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4284 case 0xb6 ... 0xb7: /* movzx */
4285 c->dst.bytes = c->op_bytes;
4286 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4289 case 0xba: /* Grp8 */
4290 switch (c->modrm_reg & 3) {
4303 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4305 case 0xbc: { /* bsf */
4307 __asm__ ("bsf %2, %0; setz %1"
4308 : "=r"(c->dst.val), "=q"(zf)
4310 ctxt->eflags &= ~X86_EFLAGS_ZF;
4312 ctxt->eflags |= X86_EFLAGS_ZF;
4313 c->dst.type = OP_NONE; /* Disable writeback. */
4317 case 0xbd: { /* bsr */
4319 __asm__ ("bsr %2, %0; setz %1"
4320 : "=r"(c->dst.val), "=q"(zf)
4322 ctxt->eflags &= ~X86_EFLAGS_ZF;
4324 ctxt->eflags |= X86_EFLAGS_ZF;
4325 c->dst.type = OP_NONE; /* Disable writeback. */
4329 case 0xbe ... 0xbf: /* movsx */
4330 c->dst.bytes = c->op_bytes;
4331 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4334 case 0xc0 ... 0xc1: /* xadd */
4335 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4336 /* Write back the register source. */
4337 c->src.val = c->dst.orig_val;
4338 write_register_operand(&c->src);
4340 case 0xc3: /* movnti */
4341 c->dst.bytes = c->op_bytes;
4342 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4345 case 0xc7: /* Grp9 (cmpxchg8b) */
4349 goto cannot_emulate;
4352 if (rc != X86EMUL_CONTINUE)
4358 return EMULATION_FAILED;