1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
81 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
82 #define VendorSpecific (1<<22) /* Vendor specific instruction */
83 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
84 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
85 #define Undefined (1<<25) /* No Such Instruction */
86 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
87 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
89 /* Source 2 operand type */
90 #define Src2None (0<<29)
91 #define Src2CL (1<<29)
92 #define Src2ImmByte (2<<29)
93 #define Src2One (3<<29)
94 #define Src2Imm (4<<29)
95 #define Src2Mask (7<<29)
98 #define X3(x...) X2(x), x
99 #define X4(x...) X2(x), X2(x)
100 #define X5(x...) X4(x), x
101 #define X6(x...) X4(x), X2(x)
102 #define X7(x...) X4(x), X3(x)
103 #define X8(x...) X4(x), X4(x)
104 #define X16(x...) X8(x), X8(x)
110 int (*execute)(struct x86_emulate_ctxt *ctxt);
111 struct opcode *group;
112 struct group_dual *gdual;
113 struct gprefix *gprefix;
115 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
119 struct opcode mod012[8];
120 struct opcode mod3[8];
124 struct opcode pfx_no;
125 struct opcode pfx_66;
126 struct opcode pfx_f2;
127 struct opcode pfx_f3;
130 /* EFLAGS bit definitions. */
131 #define EFLG_ID (1<<21)
132 #define EFLG_VIP (1<<20)
133 #define EFLG_VIF (1<<19)
134 #define EFLG_AC (1<<18)
135 #define EFLG_VM (1<<17)
136 #define EFLG_RF (1<<16)
137 #define EFLG_IOPL (3<<12)
138 #define EFLG_NT (1<<14)
139 #define EFLG_OF (1<<11)
140 #define EFLG_DF (1<<10)
141 #define EFLG_IF (1<<9)
142 #define EFLG_TF (1<<8)
143 #define EFLG_SF (1<<7)
144 #define EFLG_ZF (1<<6)
145 #define EFLG_AF (1<<4)
146 #define EFLG_PF (1<<2)
147 #define EFLG_CF (1<<0)
149 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
150 #define EFLG_RESERVED_ONE_MASK 2
153 * Instruction emulation:
154 * Most instructions are emulated directly via a fragment of inline assembly
155 * code. This allows us to save/restore EFLAGS and thus very easily pick up
156 * any modified flags.
159 #if defined(CONFIG_X86_64)
160 #define _LO32 "k" /* force 32-bit operand */
161 #define _STK "%%rsp" /* stack pointer */
162 #elif defined(__i386__)
163 #define _LO32 "" /* force 32-bit operand */
164 #define _STK "%%esp" /* stack pointer */
168 * These EFLAGS bits are restored from saved value during emulation, and
169 * any changes are written back to the saved value after emulation.
171 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
173 /* Before executing instruction: restore necessary bits in EFLAGS. */
174 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
175 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
176 "movl %"_sav",%"_LO32 _tmp"; " \
179 "movl %"_msk",%"_LO32 _tmp"; " \
180 "andl %"_LO32 _tmp",("_STK"); " \
182 "notl %"_LO32 _tmp"; " \
183 "andl %"_LO32 _tmp",("_STK"); " \
184 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
186 "orl %"_LO32 _tmp",("_STK"); " \
190 /* After executing instruction: write-back necessary bits in EFLAGS. */
191 #define _POST_EFLAGS(_sav, _msk, _tmp) \
192 /* _sav |= EFLAGS & _msk; */ \
195 "andl %"_msk",%"_LO32 _tmp"; " \
196 "orl %"_LO32 _tmp",%"_sav"; "
204 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
206 __asm__ __volatile__ ( \
207 _PRE_EFLAGS("0", "4", "2") \
208 _op _suffix " %"_x"3,%1; " \
209 _POST_EFLAGS("0", "4", "2") \
210 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
212 : _y ((_src).val), "i" (EFLAGS_MASK)); \
216 /* Raw emulation: instruction has two explicit operands. */
217 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
219 unsigned long _tmp; \
221 switch ((_dst).bytes) { \
223 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
226 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
229 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
234 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
236 unsigned long _tmp; \
237 switch ((_dst).bytes) { \
239 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
242 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
243 _wx, _wy, _lx, _ly, _qx, _qy); \
248 /* Source operand is byte-sized and may be restricted to just %cl. */
249 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
250 __emulate_2op(_op, _src, _dst, _eflags, \
251 "b", "c", "b", "c", "b", "c", "b", "c")
253 /* Source operand is byte, word, long or quad sized. */
254 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
255 __emulate_2op(_op, _src, _dst, _eflags, \
256 "b", "q", "w", "r", _LO32, "r", "", "r")
258 /* Source operand is word, long or quad sized. */
259 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
260 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
261 "w", "r", _LO32, "r", "", "r")
263 /* Instruction has three operands and one operand is stored in ECX register */
264 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
266 unsigned long _tmp; \
267 _type _clv = (_cl).val; \
268 _type _srcv = (_src).val; \
269 _type _dstv = (_dst).val; \
271 __asm__ __volatile__ ( \
272 _PRE_EFLAGS("0", "5", "2") \
273 _op _suffix " %4,%1 \n" \
274 _POST_EFLAGS("0", "5", "2") \
275 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
276 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
279 (_cl).val = (unsigned long) _clv; \
280 (_src).val = (unsigned long) _srcv; \
281 (_dst).val = (unsigned long) _dstv; \
284 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
286 switch ((_dst).bytes) { \
288 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
289 "w", unsigned short); \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "l", unsigned int); \
296 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "q", unsigned long)); \
302 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
304 unsigned long _tmp; \
306 __asm__ __volatile__ ( \
307 _PRE_EFLAGS("0", "3", "2") \
308 _op _suffix " %1; " \
309 _POST_EFLAGS("0", "3", "2") \
310 : "=m" (_eflags), "+m" ((_dst).val), \
312 : "i" (EFLAGS_MASK)); \
315 /* Instruction has only one explicit operand (no source operand). */
316 #define emulate_1op(_op, _dst, _eflags) \
318 switch ((_dst).bytes) { \
319 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
320 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
321 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
322 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
326 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
328 unsigned long _tmp; \
330 __asm__ __volatile__ ( \
331 _PRE_EFLAGS("0", "4", "1") \
332 _op _suffix " %5; " \
333 _POST_EFLAGS("0", "4", "1") \
334 : "=m" (_eflags), "=&r" (_tmp), \
335 "+a" (_rax), "+d" (_rdx) \
336 : "i" (EFLAGS_MASK), "m" ((_src).val), \
337 "a" (_rax), "d" (_rdx)); \
340 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
342 unsigned long _tmp; \
344 __asm__ __volatile__ ( \
345 _PRE_EFLAGS("0", "5", "1") \
347 _op _suffix " %6; " \
349 _POST_EFLAGS("0", "5", "1") \
350 ".pushsection .fixup,\"ax\" \n\t" \
351 "3: movb $1, %4 \n\t" \
354 _ASM_EXTABLE(1b, 3b) \
355 : "=m" (_eflags), "=&r" (_tmp), \
356 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
357 : "i" (EFLAGS_MASK), "m" ((_src).val), \
358 "a" (_rax), "d" (_rdx)); \
361 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
362 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
364 switch((_src).bytes) { \
365 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
366 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
367 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
368 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
372 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
374 switch((_src).bytes) { \
376 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
377 _eflags, "b", _ex); \
380 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
381 _eflags, "w", _ex); \
384 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
385 _eflags, "l", _ex); \
388 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
389 _eflags, "q", _ex)); \
394 /* Fetch next part of the instruction being emulated. */
395 #define insn_fetch(_type, _size, _eip) \
396 ({ unsigned long _x; \
397 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
398 if (rc != X86EMUL_CONTINUE) \
404 #define insn_fetch_arr(_arr, _size, _eip) \
405 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
406 if (rc != X86EMUL_CONTINUE) \
411 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
412 enum x86_intercept intercept,
413 enum x86_intercept_stage stage)
415 struct x86_instruction_info info = {
416 .intercept = intercept,
417 .rep_prefix = ctxt->decode.rep_prefix,
418 .modrm_mod = ctxt->decode.modrm_mod,
419 .modrm_reg = ctxt->decode.modrm_reg,
420 .modrm_rm = ctxt->decode.modrm_rm,
421 .src_val = ctxt->decode.src.val64,
422 .src_bytes = ctxt->decode.src.bytes,
423 .dst_bytes = ctxt->decode.dst.bytes,
424 .ad_bytes = ctxt->decode.ad_bytes,
425 .next_rip = ctxt->eip,
428 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
431 static inline unsigned long ad_mask(struct decode_cache *c)
433 return (1UL << (c->ad_bytes << 3)) - 1;
436 /* Access/update address held in a register, based on addressing mode. */
437 static inline unsigned long
438 address_mask(struct decode_cache *c, unsigned long reg)
440 if (c->ad_bytes == sizeof(unsigned long))
443 return reg & ad_mask(c);
446 static inline unsigned long
447 register_address(struct decode_cache *c, unsigned long reg)
449 return address_mask(c, reg);
453 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
455 if (c->ad_bytes == sizeof(unsigned long))
458 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
461 static inline void jmp_rel(struct decode_cache *c, int rel)
463 register_address_increment(c, &c->eip, rel);
466 static void set_seg_override(struct decode_cache *c, int seg)
468 c->has_seg_override = true;
469 c->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
473 struct x86_emulate_ops *ops, int seg)
475 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
478 return ops->get_cached_segment_base(seg, ctxt->vcpu);
481 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
482 struct x86_emulate_ops *ops,
483 struct decode_cache *c)
485 if (!c->has_seg_override)
488 return c->seg_override;
491 static ulong linear(struct x86_emulate_ctxt *ctxt,
492 struct segmented_address addr)
494 struct decode_cache *c = &ctxt->decode;
497 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
498 if (c->ad_bytes != 8)
503 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
504 u32 error, bool valid)
506 ctxt->exception.vector = vec;
507 ctxt->exception.error_code = error;
508 ctxt->exception.error_code_valid = valid;
509 return X86EMUL_PROPAGATE_FAULT;
512 static int emulate_db(struct x86_emulate_ctxt *ctxt)
514 return emulate_exception(ctxt, DB_VECTOR, 0, false);
517 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
519 return emulate_exception(ctxt, GP_VECTOR, err, true);
522 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, UD_VECTOR, 0, false);
527 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
529 return emulate_exception(ctxt, TS_VECTOR, err, true);
532 static int emulate_de(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, DE_VECTOR, 0, false);
537 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
539 return emulate_exception(ctxt, NM_VECTOR, 0, false);
542 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
543 struct x86_emulate_ops *ops,
544 unsigned long eip, u8 *dest)
546 struct fetch_cache *fc = &ctxt->decode.fetch;
550 if (eip == fc->end) {
551 cur_size = fc->end - fc->start;
552 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
553 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
554 size, ctxt->vcpu, &ctxt->exception);
555 if (rc != X86EMUL_CONTINUE)
559 *dest = fc->data[eip - fc->start];
560 return X86EMUL_CONTINUE;
563 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
564 struct x86_emulate_ops *ops,
565 unsigned long eip, void *dest, unsigned size)
569 /* x86 instructions are limited to 15 bytes. */
570 if (eip + size - ctxt->eip > 15)
571 return X86EMUL_UNHANDLEABLE;
573 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
574 if (rc != X86EMUL_CONTINUE)
577 return X86EMUL_CONTINUE;
581 * Given the 'reg' portion of a ModRM byte, and a register block, return a
582 * pointer into the block that addresses the relevant register.
583 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
585 static void *decode_register(u8 modrm_reg, unsigned long *regs,
590 p = ®s[modrm_reg];
591 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
592 p = (unsigned char *)®s[modrm_reg & 3] + 1;
596 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
597 struct x86_emulate_ops *ops,
598 struct segmented_address addr,
599 u16 *size, unsigned long *address, int op_bytes)
606 rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2,
607 ctxt->vcpu, &ctxt->exception);
608 if (rc != X86EMUL_CONTINUE)
611 rc = ops->read_std(linear(ctxt, addr), address, op_bytes,
612 ctxt->vcpu, &ctxt->exception);
616 static int test_cc(unsigned int condition, unsigned int flags)
620 switch ((condition & 15) >> 1) {
622 rc |= (flags & EFLG_OF);
624 case 1: /* b/c/nae */
625 rc |= (flags & EFLG_CF);
628 rc |= (flags & EFLG_ZF);
631 rc |= (flags & (EFLG_CF|EFLG_ZF));
634 rc |= (flags & EFLG_SF);
637 rc |= (flags & EFLG_PF);
640 rc |= (flags & EFLG_ZF);
643 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
647 /* Odd condition identifiers (lsb == 1) have inverted sense. */
648 return (!!rc ^ (condition & 1));
651 static void fetch_register_operand(struct operand *op)
655 op->val = *(u8 *)op->addr.reg;
658 op->val = *(u16 *)op->addr.reg;
661 op->val = *(u32 *)op->addr.reg;
664 op->val = *(u64 *)op->addr.reg;
669 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
671 ctxt->ops->get_fpu(ctxt);
673 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
674 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
675 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
676 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
677 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
678 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
679 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
680 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
682 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
683 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
684 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
685 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
686 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
687 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
688 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
689 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
693 ctxt->ops->put_fpu(ctxt);
696 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
699 ctxt->ops->get_fpu(ctxt);
701 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
702 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
703 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
704 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
705 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
706 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
707 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
708 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
710 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
711 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
712 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
713 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
714 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
715 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
716 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
717 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
721 ctxt->ops->put_fpu(ctxt);
724 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
726 struct decode_cache *c,
729 unsigned reg = c->modrm_reg;
730 int highbyte_regs = c->rex_prefix == 0;
733 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
739 read_sse_reg(ctxt, &op->vec_val, reg);
744 if ((c->d & ByteOp) && !inhibit_bytereg) {
745 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
748 op->addr.reg = decode_register(reg, c->regs, 0);
749 op->bytes = c->op_bytes;
751 fetch_register_operand(op);
752 op->orig_val = op->val;
755 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
756 struct x86_emulate_ops *ops,
759 struct decode_cache *c = &ctxt->decode;
761 int index_reg = 0, base_reg = 0, scale;
762 int rc = X86EMUL_CONTINUE;
766 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
767 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
768 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
771 c->modrm = insn_fetch(u8, 1, c->eip);
772 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
773 c->modrm_reg |= (c->modrm & 0x38) >> 3;
774 c->modrm_rm |= (c->modrm & 0x07);
775 c->modrm_seg = VCPU_SREG_DS;
777 if (c->modrm_mod == 3) {
779 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
780 op->addr.reg = decode_register(c->modrm_rm,
781 c->regs, c->d & ByteOp);
785 op->addr.xmm = c->modrm_rm;
786 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
789 fetch_register_operand(op);
795 if (c->ad_bytes == 2) {
796 unsigned bx = c->regs[VCPU_REGS_RBX];
797 unsigned bp = c->regs[VCPU_REGS_RBP];
798 unsigned si = c->regs[VCPU_REGS_RSI];
799 unsigned di = c->regs[VCPU_REGS_RDI];
801 /* 16-bit ModR/M decode. */
802 switch (c->modrm_mod) {
804 if (c->modrm_rm == 6)
805 modrm_ea += insn_fetch(u16, 2, c->eip);
808 modrm_ea += insn_fetch(s8, 1, c->eip);
811 modrm_ea += insn_fetch(u16, 2, c->eip);
814 switch (c->modrm_rm) {
834 if (c->modrm_mod != 0)
841 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
842 (c->modrm_rm == 6 && c->modrm_mod != 0))
843 c->modrm_seg = VCPU_SREG_SS;
844 modrm_ea = (u16)modrm_ea;
846 /* 32/64-bit ModR/M decode. */
847 if ((c->modrm_rm & 7) == 4) {
848 sib = insn_fetch(u8, 1, c->eip);
849 index_reg |= (sib >> 3) & 7;
853 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
854 modrm_ea += insn_fetch(s32, 4, c->eip);
856 modrm_ea += c->regs[base_reg];
858 modrm_ea += c->regs[index_reg] << scale;
859 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
860 if (ctxt->mode == X86EMUL_MODE_PROT64)
863 modrm_ea += c->regs[c->modrm_rm];
864 switch (c->modrm_mod) {
866 if (c->modrm_rm == 5)
867 modrm_ea += insn_fetch(s32, 4, c->eip);
870 modrm_ea += insn_fetch(s8, 1, c->eip);
873 modrm_ea += insn_fetch(s32, 4, c->eip);
877 op->addr.mem.ea = modrm_ea;
882 static int decode_abs(struct x86_emulate_ctxt *ctxt,
883 struct x86_emulate_ops *ops,
886 struct decode_cache *c = &ctxt->decode;
887 int rc = X86EMUL_CONTINUE;
890 switch (c->ad_bytes) {
892 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
895 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
898 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
905 static void fetch_bit_operand(struct decode_cache *c)
909 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
910 mask = ~(c->dst.bytes * 8 - 1);
912 if (c->src.bytes == 2)
913 sv = (s16)c->src.val & (s16)mask;
914 else if (c->src.bytes == 4)
915 sv = (s32)c->src.val & (s32)mask;
917 c->dst.addr.mem.ea += (sv >> 3);
920 /* only subword offset */
921 c->src.val &= (c->dst.bytes << 3) - 1;
924 static int read_emulated(struct x86_emulate_ctxt *ctxt,
925 struct x86_emulate_ops *ops,
926 unsigned long addr, void *dest, unsigned size)
929 struct read_cache *mc = &ctxt->decode.mem_read;
932 int n = min(size, 8u);
934 if (mc->pos < mc->end)
937 rc = ops->read_emulated(addr, mc->data + mc->end, n,
938 &ctxt->exception, ctxt->vcpu);
939 if (rc != X86EMUL_CONTINUE)
944 memcpy(dest, mc->data + mc->pos, n);
949 return X86EMUL_CONTINUE;
952 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
953 struct x86_emulate_ops *ops,
954 unsigned int size, unsigned short port,
957 struct read_cache *rc = &ctxt->decode.io_read;
959 if (rc->pos == rc->end) { /* refill pio read ahead */
960 struct decode_cache *c = &ctxt->decode;
961 unsigned int in_page, n;
962 unsigned int count = c->rep_prefix ?
963 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
964 in_page = (ctxt->eflags & EFLG_DF) ?
965 offset_in_page(c->regs[VCPU_REGS_RDI]) :
966 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
967 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
971 rc->pos = rc->end = 0;
972 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
977 memcpy(dest, rc->data + rc->pos, size);
982 static u32 desc_limit_scaled(struct desc_struct *desc)
984 u32 limit = get_desc_limit(desc);
986 return desc->g ? (limit << 12) | 0xfff : limit;
989 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
990 struct x86_emulate_ops *ops,
991 u16 selector, struct desc_ptr *dt)
993 if (selector & 1 << 2) {
994 struct desc_struct desc;
995 memset (dt, 0, sizeof *dt);
996 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1000 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1001 dt->address = get_desc_base(&desc);
1003 ops->get_gdt(dt, ctxt->vcpu);
1006 /* allowed just for 8 bytes segments */
1007 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1008 struct x86_emulate_ops *ops,
1009 u16 selector, struct desc_struct *desc)
1012 u16 index = selector >> 3;
1016 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1018 if (dt.size < index * 8 + 7)
1019 return emulate_gp(ctxt, selector & 0xfffc);
1020 addr = dt.address + index * 8;
1021 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1027 /* allowed just for 8 bytes segments */
1028 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1029 struct x86_emulate_ops *ops,
1030 u16 selector, struct desc_struct *desc)
1033 u16 index = selector >> 3;
1037 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1039 if (dt.size < index * 8 + 7)
1040 return emulate_gp(ctxt, selector & 0xfffc);
1042 addr = dt.address + index * 8;
1043 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1049 /* Does not support long mode */
1050 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1051 struct x86_emulate_ops *ops,
1052 u16 selector, int seg)
1054 struct desc_struct seg_desc;
1056 unsigned err_vec = GP_VECTOR;
1058 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1061 memset(&seg_desc, 0, sizeof seg_desc);
1063 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1064 || ctxt->mode == X86EMUL_MODE_REAL) {
1065 /* set real mode segment descriptor */
1066 set_desc_base(&seg_desc, selector << 4);
1067 set_desc_limit(&seg_desc, 0xffff);
1074 /* NULL selector is not valid for TR, CS and SS */
1075 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1079 /* TR should be in GDT only */
1080 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1083 if (null_selector) /* for NULL selector skip all following checks */
1086 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1087 if (ret != X86EMUL_CONTINUE)
1090 err_code = selector & 0xfffc;
1091 err_vec = GP_VECTOR;
1093 /* can't load system descriptor into segment selecor */
1094 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1098 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1104 cpl = ops->cpl(ctxt->vcpu);
1109 * segment is not a writable data segment or segment
1110 * selector's RPL != CPL or segment selector's RPL != CPL
1112 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1116 if (!(seg_desc.type & 8))
1119 if (seg_desc.type & 4) {
1125 if (rpl > cpl || dpl != cpl)
1128 /* CS(RPL) <- CPL */
1129 selector = (selector & 0xfffc) | cpl;
1132 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1135 case VCPU_SREG_LDTR:
1136 if (seg_desc.s || seg_desc.type != 2)
1139 default: /* DS, ES, FS, or GS */
1141 * segment is not a data or readable code segment or
1142 * ((segment is a data or nonconforming code segment)
1143 * and (both RPL and CPL > DPL))
1145 if ((seg_desc.type & 0xa) == 0x8 ||
1146 (((seg_desc.type & 0xc) != 0xc) &&
1147 (rpl > dpl && cpl > dpl)))
1153 /* mark segment as accessed */
1155 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1156 if (ret != X86EMUL_CONTINUE)
1160 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1161 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1162 return X86EMUL_CONTINUE;
1164 emulate_exception(ctxt, err_vec, err_code, true);
1165 return X86EMUL_PROPAGATE_FAULT;
1168 static void write_register_operand(struct operand *op)
1170 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1171 switch (op->bytes) {
1173 *(u8 *)op->addr.reg = (u8)op->val;
1176 *(u16 *)op->addr.reg = (u16)op->val;
1179 *op->addr.reg = (u32)op->val;
1180 break; /* 64b: zero-extend */
1182 *op->addr.reg = op->val;
1187 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1188 struct x86_emulate_ops *ops)
1191 struct decode_cache *c = &ctxt->decode;
1193 switch (c->dst.type) {
1195 write_register_operand(&c->dst);
1199 rc = ops->cmpxchg_emulated(
1200 linear(ctxt, c->dst.addr.mem),
1207 rc = ops->write_emulated(
1208 linear(ctxt, c->dst.addr.mem),
1213 if (rc != X86EMUL_CONTINUE)
1217 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1225 return X86EMUL_CONTINUE;
1228 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1229 struct x86_emulate_ops *ops)
1231 struct decode_cache *c = &ctxt->decode;
1233 c->dst.type = OP_MEM;
1234 c->dst.bytes = c->op_bytes;
1235 c->dst.val = c->src.val;
1236 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1237 c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1238 c->dst.addr.mem.seg = VCPU_SREG_SS;
1241 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1242 struct x86_emulate_ops *ops,
1243 void *dest, int len)
1245 struct decode_cache *c = &ctxt->decode;
1247 struct segmented_address addr;
1249 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1250 addr.seg = VCPU_SREG_SS;
1251 rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len);
1252 if (rc != X86EMUL_CONTINUE)
1255 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1259 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1260 struct x86_emulate_ops *ops,
1261 void *dest, int len)
1264 unsigned long val, change_mask;
1265 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1266 int cpl = ops->cpl(ctxt->vcpu);
1268 rc = emulate_pop(ctxt, ops, &val, len);
1269 if (rc != X86EMUL_CONTINUE)
1272 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1273 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1275 switch(ctxt->mode) {
1276 case X86EMUL_MODE_PROT64:
1277 case X86EMUL_MODE_PROT32:
1278 case X86EMUL_MODE_PROT16:
1280 change_mask |= EFLG_IOPL;
1282 change_mask |= EFLG_IF;
1284 case X86EMUL_MODE_VM86:
1286 return emulate_gp(ctxt, 0);
1287 change_mask |= EFLG_IF;
1289 default: /* real mode */
1290 change_mask |= (EFLG_IOPL | EFLG_IF);
1294 *(unsigned long *)dest =
1295 (ctxt->eflags & ~change_mask) | (val & change_mask);
1300 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1301 struct x86_emulate_ops *ops, int seg)
1303 struct decode_cache *c = &ctxt->decode;
1305 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1307 emulate_push(ctxt, ops);
1310 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1311 struct x86_emulate_ops *ops, int seg)
1313 struct decode_cache *c = &ctxt->decode;
1314 unsigned long selector;
1317 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1318 if (rc != X86EMUL_CONTINUE)
1321 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1325 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1326 struct x86_emulate_ops *ops)
1328 struct decode_cache *c = &ctxt->decode;
1329 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1330 int rc = X86EMUL_CONTINUE;
1331 int reg = VCPU_REGS_RAX;
1333 while (reg <= VCPU_REGS_RDI) {
1334 (reg == VCPU_REGS_RSP) ?
1335 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1337 emulate_push(ctxt, ops);
1339 rc = writeback(ctxt, ops);
1340 if (rc != X86EMUL_CONTINUE)
1346 /* Disable writeback. */
1347 c->dst.type = OP_NONE;
1352 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1353 struct x86_emulate_ops *ops)
1355 struct decode_cache *c = &ctxt->decode;
1356 int rc = X86EMUL_CONTINUE;
1357 int reg = VCPU_REGS_RDI;
1359 while (reg >= VCPU_REGS_RAX) {
1360 if (reg == VCPU_REGS_RSP) {
1361 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1366 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1367 if (rc != X86EMUL_CONTINUE)
1374 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1375 struct x86_emulate_ops *ops, int irq)
1377 struct decode_cache *c = &ctxt->decode;
1384 /* TODO: Add limit checks */
1385 c->src.val = ctxt->eflags;
1386 emulate_push(ctxt, ops);
1387 rc = writeback(ctxt, ops);
1388 if (rc != X86EMUL_CONTINUE)
1391 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1393 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1394 emulate_push(ctxt, ops);
1395 rc = writeback(ctxt, ops);
1396 if (rc != X86EMUL_CONTINUE)
1399 c->src.val = c->eip;
1400 emulate_push(ctxt, ops);
1401 rc = writeback(ctxt, ops);
1402 if (rc != X86EMUL_CONTINUE)
1405 c->dst.type = OP_NONE;
1407 ops->get_idt(&dt, ctxt->vcpu);
1409 eip_addr = dt.address + (irq << 2);
1410 cs_addr = dt.address + (irq << 2) + 2;
1412 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1413 if (rc != X86EMUL_CONTINUE)
1416 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1417 if (rc != X86EMUL_CONTINUE)
1420 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1421 if (rc != X86EMUL_CONTINUE)
1429 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1430 struct x86_emulate_ops *ops, int irq)
1432 switch(ctxt->mode) {
1433 case X86EMUL_MODE_REAL:
1434 return emulate_int_real(ctxt, ops, irq);
1435 case X86EMUL_MODE_VM86:
1436 case X86EMUL_MODE_PROT16:
1437 case X86EMUL_MODE_PROT32:
1438 case X86EMUL_MODE_PROT64:
1440 /* Protected mode interrupts unimplemented yet */
1441 return X86EMUL_UNHANDLEABLE;
1445 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1446 struct x86_emulate_ops *ops)
1448 struct decode_cache *c = &ctxt->decode;
1449 int rc = X86EMUL_CONTINUE;
1450 unsigned long temp_eip = 0;
1451 unsigned long temp_eflags = 0;
1452 unsigned long cs = 0;
1453 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1454 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1455 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1456 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1458 /* TODO: Add stack limit check */
1460 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1462 if (rc != X86EMUL_CONTINUE)
1465 if (temp_eip & ~0xffff)
1466 return emulate_gp(ctxt, 0);
1468 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1470 if (rc != X86EMUL_CONTINUE)
1473 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1475 if (rc != X86EMUL_CONTINUE)
1478 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1480 if (rc != X86EMUL_CONTINUE)
1486 if (c->op_bytes == 4)
1487 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1488 else if (c->op_bytes == 2) {
1489 ctxt->eflags &= ~0xffff;
1490 ctxt->eflags |= temp_eflags;
1493 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1494 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1499 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1500 struct x86_emulate_ops* ops)
1502 switch(ctxt->mode) {
1503 case X86EMUL_MODE_REAL:
1504 return emulate_iret_real(ctxt, ops);
1505 case X86EMUL_MODE_VM86:
1506 case X86EMUL_MODE_PROT16:
1507 case X86EMUL_MODE_PROT32:
1508 case X86EMUL_MODE_PROT64:
1510 /* iret from protected mode unimplemented yet */
1511 return X86EMUL_UNHANDLEABLE;
1515 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1516 struct x86_emulate_ops *ops)
1518 struct decode_cache *c = &ctxt->decode;
1520 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1523 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1525 struct decode_cache *c = &ctxt->decode;
1526 switch (c->modrm_reg) {
1528 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1531 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1534 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1537 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1539 case 4: /* sal/shl */
1540 case 6: /* sal/shl */
1541 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1544 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1547 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1552 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1553 struct x86_emulate_ops *ops)
1555 struct decode_cache *c = &ctxt->decode;
1556 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1557 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1560 switch (c->modrm_reg) {
1561 case 0 ... 1: /* test */
1562 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1565 c->dst.val = ~c->dst.val;
1568 emulate_1op("neg", c->dst, ctxt->eflags);
1571 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1574 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1577 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1581 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1585 return X86EMUL_UNHANDLEABLE;
1588 return emulate_de(ctxt);
1589 return X86EMUL_CONTINUE;
1592 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1593 struct x86_emulate_ops *ops)
1595 struct decode_cache *c = &ctxt->decode;
1597 switch (c->modrm_reg) {
1599 emulate_1op("inc", c->dst, ctxt->eflags);
1602 emulate_1op("dec", c->dst, ctxt->eflags);
1604 case 2: /* call near abs */ {
1607 c->eip = c->src.val;
1608 c->src.val = old_eip;
1609 emulate_push(ctxt, ops);
1612 case 4: /* jmp abs */
1613 c->eip = c->src.val;
1616 emulate_push(ctxt, ops);
1619 return X86EMUL_CONTINUE;
1622 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1623 struct x86_emulate_ops *ops)
1625 struct decode_cache *c = &ctxt->decode;
1626 u64 old = c->dst.orig_val64;
1628 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1629 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1630 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1631 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1632 ctxt->eflags &= ~EFLG_ZF;
1634 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1635 (u32) c->regs[VCPU_REGS_RBX];
1637 ctxt->eflags |= EFLG_ZF;
1639 return X86EMUL_CONTINUE;
1642 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1643 struct x86_emulate_ops *ops)
1645 struct decode_cache *c = &ctxt->decode;
1649 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1652 if (c->op_bytes == 4)
1653 c->eip = (u32)c->eip;
1654 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1655 if (rc != X86EMUL_CONTINUE)
1657 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1661 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1662 struct x86_emulate_ops *ops, int seg)
1664 struct decode_cache *c = &ctxt->decode;
1668 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1670 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1671 if (rc != X86EMUL_CONTINUE)
1674 c->dst.val = c->src.val;
1679 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1680 struct x86_emulate_ops *ops, struct desc_struct *cs,
1681 struct desc_struct *ss)
1683 memset(cs, 0, sizeof(struct desc_struct));
1684 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1685 memset(ss, 0, sizeof(struct desc_struct));
1687 cs->l = 0; /* will be adjusted later */
1688 set_desc_base(cs, 0); /* flat segment */
1689 cs->g = 1; /* 4kb granularity */
1690 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1691 cs->type = 0x0b; /* Read, Execute, Accessed */
1693 cs->dpl = 0; /* will be adjusted later */
1697 set_desc_base(ss, 0); /* flat segment */
1698 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1699 ss->g = 1; /* 4kb granularity */
1701 ss->type = 0x03; /* Read/Write, Accessed */
1702 ss->d = 1; /* 32bit stack segment */
1708 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1710 struct decode_cache *c = &ctxt->decode;
1711 struct desc_struct cs, ss;
1715 /* syscall is not available in real mode */
1716 if (ctxt->mode == X86EMUL_MODE_REAL ||
1717 ctxt->mode == X86EMUL_MODE_VM86)
1718 return emulate_ud(ctxt);
1720 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1722 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1724 cs_sel = (u16)(msr_data & 0xfffc);
1725 ss_sel = (u16)(msr_data + 8);
1727 if (is_long_mode(ctxt->vcpu)) {
1731 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1732 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1733 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1734 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1736 c->regs[VCPU_REGS_RCX] = c->eip;
1737 if (is_long_mode(ctxt->vcpu)) {
1738 #ifdef CONFIG_X86_64
1739 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1741 ops->get_msr(ctxt->vcpu,
1742 ctxt->mode == X86EMUL_MODE_PROT64 ?
1743 MSR_LSTAR : MSR_CSTAR, &msr_data);
1746 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1747 ctxt->eflags &= ~(msr_data | EFLG_RF);
1751 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1752 c->eip = (u32)msr_data;
1754 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1757 return X86EMUL_CONTINUE;
1761 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1763 struct decode_cache *c = &ctxt->decode;
1764 struct desc_struct cs, ss;
1768 /* inject #GP if in real mode */
1769 if (ctxt->mode == X86EMUL_MODE_REAL)
1770 return emulate_gp(ctxt, 0);
1772 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1773 * Therefore, we inject an #UD.
1775 if (ctxt->mode == X86EMUL_MODE_PROT64)
1776 return emulate_ud(ctxt);
1778 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1780 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1781 switch (ctxt->mode) {
1782 case X86EMUL_MODE_PROT32:
1783 if ((msr_data & 0xfffc) == 0x0)
1784 return emulate_gp(ctxt, 0);
1786 case X86EMUL_MODE_PROT64:
1787 if (msr_data == 0x0)
1788 return emulate_gp(ctxt, 0);
1792 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1793 cs_sel = (u16)msr_data;
1794 cs_sel &= ~SELECTOR_RPL_MASK;
1795 ss_sel = cs_sel + 8;
1796 ss_sel &= ~SELECTOR_RPL_MASK;
1797 if (ctxt->mode == X86EMUL_MODE_PROT64
1798 || is_long_mode(ctxt->vcpu)) {
1803 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1804 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1805 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1806 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1808 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1811 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1812 c->regs[VCPU_REGS_RSP] = msr_data;
1814 return X86EMUL_CONTINUE;
1818 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1820 struct decode_cache *c = &ctxt->decode;
1821 struct desc_struct cs, ss;
1826 /* inject #GP if in real mode or Virtual 8086 mode */
1827 if (ctxt->mode == X86EMUL_MODE_REAL ||
1828 ctxt->mode == X86EMUL_MODE_VM86)
1829 return emulate_gp(ctxt, 0);
1831 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1833 if ((c->rex_prefix & 0x8) != 0x0)
1834 usermode = X86EMUL_MODE_PROT64;
1836 usermode = X86EMUL_MODE_PROT32;
1840 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1842 case X86EMUL_MODE_PROT32:
1843 cs_sel = (u16)(msr_data + 16);
1844 if ((msr_data & 0xfffc) == 0x0)
1845 return emulate_gp(ctxt, 0);
1846 ss_sel = (u16)(msr_data + 24);
1848 case X86EMUL_MODE_PROT64:
1849 cs_sel = (u16)(msr_data + 32);
1850 if (msr_data == 0x0)
1851 return emulate_gp(ctxt, 0);
1852 ss_sel = cs_sel + 8;
1857 cs_sel |= SELECTOR_RPL_MASK;
1858 ss_sel |= SELECTOR_RPL_MASK;
1860 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1861 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1862 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1863 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1865 c->eip = c->regs[VCPU_REGS_RDX];
1866 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1868 return X86EMUL_CONTINUE;
1871 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1872 struct x86_emulate_ops *ops)
1875 if (ctxt->mode == X86EMUL_MODE_REAL)
1877 if (ctxt->mode == X86EMUL_MODE_VM86)
1879 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1880 return ops->cpl(ctxt->vcpu) > iopl;
1883 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1884 struct x86_emulate_ops *ops,
1887 struct desc_struct tr_seg;
1890 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
1891 unsigned mask = (1 << len) - 1;
1894 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
1897 if (desc_limit_scaled(&tr_seg) < 103)
1899 base = get_desc_base(&tr_seg);
1900 #ifdef CONFIG_X86_64
1901 base |= ((u64)base3) << 32;
1903 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
1904 if (r != X86EMUL_CONTINUE)
1906 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
1908 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
1910 if (r != X86EMUL_CONTINUE)
1912 if ((perm >> bit_idx) & mask)
1917 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1918 struct x86_emulate_ops *ops,
1924 if (emulator_bad_iopl(ctxt, ops))
1925 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1928 ctxt->perm_ok = true;
1933 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
1934 struct x86_emulate_ops *ops,
1935 struct tss_segment_16 *tss)
1937 struct decode_cache *c = &ctxt->decode;
1940 tss->flag = ctxt->eflags;
1941 tss->ax = c->regs[VCPU_REGS_RAX];
1942 tss->cx = c->regs[VCPU_REGS_RCX];
1943 tss->dx = c->regs[VCPU_REGS_RDX];
1944 tss->bx = c->regs[VCPU_REGS_RBX];
1945 tss->sp = c->regs[VCPU_REGS_RSP];
1946 tss->bp = c->regs[VCPU_REGS_RBP];
1947 tss->si = c->regs[VCPU_REGS_RSI];
1948 tss->di = c->regs[VCPU_REGS_RDI];
1950 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
1951 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1952 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
1953 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
1954 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
1957 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
1958 struct x86_emulate_ops *ops,
1959 struct tss_segment_16 *tss)
1961 struct decode_cache *c = &ctxt->decode;
1965 ctxt->eflags = tss->flag | 2;
1966 c->regs[VCPU_REGS_RAX] = tss->ax;
1967 c->regs[VCPU_REGS_RCX] = tss->cx;
1968 c->regs[VCPU_REGS_RDX] = tss->dx;
1969 c->regs[VCPU_REGS_RBX] = tss->bx;
1970 c->regs[VCPU_REGS_RSP] = tss->sp;
1971 c->regs[VCPU_REGS_RBP] = tss->bp;
1972 c->regs[VCPU_REGS_RSI] = tss->si;
1973 c->regs[VCPU_REGS_RDI] = tss->di;
1976 * SDM says that segment selectors are loaded before segment
1979 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
1980 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
1981 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
1982 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
1983 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
1986 * Now load segment descriptors. If fault happenes at this stage
1987 * it is handled in a context of new task
1989 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
1990 if (ret != X86EMUL_CONTINUE)
1992 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
1993 if (ret != X86EMUL_CONTINUE)
1995 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
1996 if (ret != X86EMUL_CONTINUE)
1998 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
1999 if (ret != X86EMUL_CONTINUE)
2001 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2002 if (ret != X86EMUL_CONTINUE)
2005 return X86EMUL_CONTINUE;
2008 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2009 struct x86_emulate_ops *ops,
2010 u16 tss_selector, u16 old_tss_sel,
2011 ulong old_tss_base, struct desc_struct *new_desc)
2013 struct tss_segment_16 tss_seg;
2015 u32 new_tss_base = get_desc_base(new_desc);
2017 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2019 if (ret != X86EMUL_CONTINUE)
2020 /* FIXME: need to provide precise fault address */
2023 save_state_to_tss16(ctxt, ops, &tss_seg);
2025 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2027 if (ret != X86EMUL_CONTINUE)
2028 /* FIXME: need to provide precise fault address */
2031 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2033 if (ret != X86EMUL_CONTINUE)
2034 /* FIXME: need to provide precise fault address */
2037 if (old_tss_sel != 0xffff) {
2038 tss_seg.prev_task_link = old_tss_sel;
2040 ret = ops->write_std(new_tss_base,
2041 &tss_seg.prev_task_link,
2042 sizeof tss_seg.prev_task_link,
2043 ctxt->vcpu, &ctxt->exception);
2044 if (ret != X86EMUL_CONTINUE)
2045 /* FIXME: need to provide precise fault address */
2049 return load_state_from_tss16(ctxt, ops, &tss_seg);
2052 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2053 struct x86_emulate_ops *ops,
2054 struct tss_segment_32 *tss)
2056 struct decode_cache *c = &ctxt->decode;
2058 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2060 tss->eflags = ctxt->eflags;
2061 tss->eax = c->regs[VCPU_REGS_RAX];
2062 tss->ecx = c->regs[VCPU_REGS_RCX];
2063 tss->edx = c->regs[VCPU_REGS_RDX];
2064 tss->ebx = c->regs[VCPU_REGS_RBX];
2065 tss->esp = c->regs[VCPU_REGS_RSP];
2066 tss->ebp = c->regs[VCPU_REGS_RBP];
2067 tss->esi = c->regs[VCPU_REGS_RSI];
2068 tss->edi = c->regs[VCPU_REGS_RDI];
2070 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2071 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2072 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2073 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2074 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2075 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2076 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2079 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2080 struct x86_emulate_ops *ops,
2081 struct tss_segment_32 *tss)
2083 struct decode_cache *c = &ctxt->decode;
2086 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2087 return emulate_gp(ctxt, 0);
2089 ctxt->eflags = tss->eflags | 2;
2090 c->regs[VCPU_REGS_RAX] = tss->eax;
2091 c->regs[VCPU_REGS_RCX] = tss->ecx;
2092 c->regs[VCPU_REGS_RDX] = tss->edx;
2093 c->regs[VCPU_REGS_RBX] = tss->ebx;
2094 c->regs[VCPU_REGS_RSP] = tss->esp;
2095 c->regs[VCPU_REGS_RBP] = tss->ebp;
2096 c->regs[VCPU_REGS_RSI] = tss->esi;
2097 c->regs[VCPU_REGS_RDI] = tss->edi;
2100 * SDM says that segment selectors are loaded before segment
2103 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2104 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2105 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2106 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2107 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2108 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2109 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2112 * Now load segment descriptors. If fault happenes at this stage
2113 * it is handled in a context of new task
2115 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2116 if (ret != X86EMUL_CONTINUE)
2118 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2119 if (ret != X86EMUL_CONTINUE)
2121 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2122 if (ret != X86EMUL_CONTINUE)
2124 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2125 if (ret != X86EMUL_CONTINUE)
2127 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2128 if (ret != X86EMUL_CONTINUE)
2130 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2131 if (ret != X86EMUL_CONTINUE)
2133 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2134 if (ret != X86EMUL_CONTINUE)
2137 return X86EMUL_CONTINUE;
2140 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2141 struct x86_emulate_ops *ops,
2142 u16 tss_selector, u16 old_tss_sel,
2143 ulong old_tss_base, struct desc_struct *new_desc)
2145 struct tss_segment_32 tss_seg;
2147 u32 new_tss_base = get_desc_base(new_desc);
2149 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2151 if (ret != X86EMUL_CONTINUE)
2152 /* FIXME: need to provide precise fault address */
2155 save_state_to_tss32(ctxt, ops, &tss_seg);
2157 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2159 if (ret != X86EMUL_CONTINUE)
2160 /* FIXME: need to provide precise fault address */
2163 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2165 if (ret != X86EMUL_CONTINUE)
2166 /* FIXME: need to provide precise fault address */
2169 if (old_tss_sel != 0xffff) {
2170 tss_seg.prev_task_link = old_tss_sel;
2172 ret = ops->write_std(new_tss_base,
2173 &tss_seg.prev_task_link,
2174 sizeof tss_seg.prev_task_link,
2175 ctxt->vcpu, &ctxt->exception);
2176 if (ret != X86EMUL_CONTINUE)
2177 /* FIXME: need to provide precise fault address */
2181 return load_state_from_tss32(ctxt, ops, &tss_seg);
2184 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2185 struct x86_emulate_ops *ops,
2186 u16 tss_selector, int reason,
2187 bool has_error_code, u32 error_code)
2189 struct desc_struct curr_tss_desc, next_tss_desc;
2191 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2192 ulong old_tss_base =
2193 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2196 /* FIXME: old_tss_base == ~0 ? */
2198 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2199 if (ret != X86EMUL_CONTINUE)
2201 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2202 if (ret != X86EMUL_CONTINUE)
2205 /* FIXME: check that next_tss_desc is tss */
2207 if (reason != TASK_SWITCH_IRET) {
2208 if ((tss_selector & 3) > next_tss_desc.dpl ||
2209 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2210 return emulate_gp(ctxt, 0);
2213 desc_limit = desc_limit_scaled(&next_tss_desc);
2214 if (!next_tss_desc.p ||
2215 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2216 desc_limit < 0x2b)) {
2217 emulate_ts(ctxt, tss_selector & 0xfffc);
2218 return X86EMUL_PROPAGATE_FAULT;
2221 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2222 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2223 write_segment_descriptor(ctxt, ops, old_tss_sel,
2227 if (reason == TASK_SWITCH_IRET)
2228 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2230 /* set back link to prev task only if NT bit is set in eflags
2231 note that old_tss_sel is not used afetr this point */
2232 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2233 old_tss_sel = 0xffff;
2235 if (next_tss_desc.type & 8)
2236 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2237 old_tss_base, &next_tss_desc);
2239 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2240 old_tss_base, &next_tss_desc);
2241 if (ret != X86EMUL_CONTINUE)
2244 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2245 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2247 if (reason != TASK_SWITCH_IRET) {
2248 next_tss_desc.type |= (1 << 1); /* set busy flag */
2249 write_segment_descriptor(ctxt, ops, tss_selector,
2253 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2254 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2255 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2257 if (has_error_code) {
2258 struct decode_cache *c = &ctxt->decode;
2260 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2262 c->src.val = (unsigned long) error_code;
2263 emulate_push(ctxt, ops);
2269 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2270 u16 tss_selector, int reason,
2271 bool has_error_code, u32 error_code)
2273 struct x86_emulate_ops *ops = ctxt->ops;
2274 struct decode_cache *c = &ctxt->decode;
2278 c->dst.type = OP_NONE;
2280 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2281 has_error_code, error_code);
2283 if (rc == X86EMUL_CONTINUE) {
2284 rc = writeback(ctxt, ops);
2285 if (rc == X86EMUL_CONTINUE)
2289 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2292 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2293 int reg, struct operand *op)
2295 struct decode_cache *c = &ctxt->decode;
2296 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2298 register_address_increment(c, &c->regs[reg], df * op->bytes);
2299 op->addr.mem.ea = register_address(c, c->regs[reg]);
2300 op->addr.mem.seg = seg;
2303 static int em_push(struct x86_emulate_ctxt *ctxt)
2305 emulate_push(ctxt, ctxt->ops);
2306 return X86EMUL_CONTINUE;
2309 static int em_das(struct x86_emulate_ctxt *ctxt)
2311 struct decode_cache *c = &ctxt->decode;
2313 bool af, cf, old_cf;
2315 cf = ctxt->eflags & X86_EFLAGS_CF;
2321 af = ctxt->eflags & X86_EFLAGS_AF;
2322 if ((al & 0x0f) > 9 || af) {
2324 cf = old_cf | (al >= 250);
2329 if (old_al > 0x99 || old_cf) {
2335 /* Set PF, ZF, SF */
2336 c->src.type = OP_IMM;
2339 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2340 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2342 ctxt->eflags |= X86_EFLAGS_CF;
2344 ctxt->eflags |= X86_EFLAGS_AF;
2345 return X86EMUL_CONTINUE;
2348 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2350 struct decode_cache *c = &ctxt->decode;
2355 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2358 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2359 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2360 return X86EMUL_CONTINUE;
2363 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2365 c->src.val = old_cs;
2366 emulate_push(ctxt, ctxt->ops);
2367 rc = writeback(ctxt, ctxt->ops);
2368 if (rc != X86EMUL_CONTINUE)
2371 c->src.val = old_eip;
2372 emulate_push(ctxt, ctxt->ops);
2373 rc = writeback(ctxt, ctxt->ops);
2374 if (rc != X86EMUL_CONTINUE)
2377 c->dst.type = OP_NONE;
2379 return X86EMUL_CONTINUE;
2382 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2384 struct decode_cache *c = &ctxt->decode;
2387 c->dst.type = OP_REG;
2388 c->dst.addr.reg = &c->eip;
2389 c->dst.bytes = c->op_bytes;
2390 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2391 if (rc != X86EMUL_CONTINUE)
2393 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2394 return X86EMUL_CONTINUE;
2397 static int em_imul(struct x86_emulate_ctxt *ctxt)
2399 struct decode_cache *c = &ctxt->decode;
2401 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2402 return X86EMUL_CONTINUE;
2405 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2407 struct decode_cache *c = &ctxt->decode;
2409 c->dst.val = c->src2.val;
2410 return em_imul(ctxt);
2413 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2415 struct decode_cache *c = &ctxt->decode;
2417 c->dst.type = OP_REG;
2418 c->dst.bytes = c->src.bytes;
2419 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2420 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2422 return X86EMUL_CONTINUE;
2425 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2427 unsigned cpl = ctxt->ops->cpl(ctxt->vcpu);
2428 struct decode_cache *c = &ctxt->decode;
2431 if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD))
2432 return emulate_gp(ctxt, 0);
2433 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2434 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2435 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2436 return X86EMUL_CONTINUE;
2439 static int em_mov(struct x86_emulate_ctxt *ctxt)
2441 struct decode_cache *c = &ctxt->decode;
2442 c->dst.val = c->src.val;
2443 return X86EMUL_CONTINUE;
2446 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2448 struct decode_cache *c = &ctxt->decode;
2449 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2450 return X86EMUL_CONTINUE;
2453 static bool valid_cr(int nr)
2465 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2467 struct decode_cache *c = &ctxt->decode;
2469 if (!valid_cr(c->modrm_reg))
2470 return emulate_ud(ctxt);
2472 return X86EMUL_CONTINUE;
2475 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2477 struct decode_cache *c = &ctxt->decode;
2478 u64 new_val = c->src.val64;
2479 int cr = c->modrm_reg;
2481 static u64 cr_reserved_bits[] = {
2482 0xffffffff00000000ULL,
2483 0, 0, 0, /* CR3 checked later */
2490 return emulate_ud(ctxt);
2492 if (new_val & cr_reserved_bits[cr])
2493 return emulate_gp(ctxt, 0);
2498 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2499 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2500 return emulate_gp(ctxt, 0);
2502 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2503 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2505 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2506 !(cr4 & X86_CR4_PAE))
2507 return emulate_gp(ctxt, 0);
2514 if (is_long_mode(ctxt->vcpu))
2515 rsvd = CR3_L_MODE_RESERVED_BITS;
2516 else if (is_pae(ctxt->vcpu))
2517 rsvd = CR3_PAE_RESERVED_BITS;
2518 else if (is_paging(ctxt->vcpu))
2519 rsvd = CR3_NONPAE_RESERVED_BITS;
2522 return emulate_gp(ctxt, 0);
2529 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2530 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2532 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2533 return emulate_gp(ctxt, 0);
2539 return X86EMUL_CONTINUE;
2542 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2546 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2548 /* Check if DR7.Global_Enable is set */
2549 return dr7 & (1 << 13);
2552 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2554 struct decode_cache *c = &ctxt->decode;
2555 int dr = c->modrm_reg;
2559 return emulate_ud(ctxt);
2561 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2562 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2563 return emulate_ud(ctxt);
2565 if (check_dr7_gd(ctxt))
2566 return emulate_db(ctxt);
2568 return X86EMUL_CONTINUE;
2571 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2573 struct decode_cache *c = &ctxt->decode;
2574 u64 new_val = c->src.val64;
2575 int dr = c->modrm_reg;
2577 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2578 return emulate_gp(ctxt, 0);
2580 return check_dr_read(ctxt);
2583 #define D(_y) { .flags = (_y) }
2584 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2585 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2586 .check_perm = (_p) }
2588 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2589 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2590 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2591 #define II(_f, _e, _i) \
2592 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2593 #define IIP(_f, _e, _i, _p) \
2594 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2595 .check_perm = (_p) }
2596 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2598 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2599 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2601 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2602 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2603 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2606 static struct opcode group1[] = {
2610 static struct opcode group1A[] = {
2611 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2614 static struct opcode group3[] = {
2615 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2616 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2617 X4(D(SrcMem | ModRM)),
2620 static struct opcode group4[] = {
2621 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2625 static struct opcode group5[] = {
2626 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2627 D(SrcMem | ModRM | Stack),
2628 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2629 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2630 D(SrcMem | ModRM | Stack), N,
2633 static struct opcode group6[] = {
2634 DI(ModRM | Prot, sldt),
2635 DI(ModRM | Prot, str),
2636 DI(ModRM | Prot | Priv, lldt),
2637 DI(ModRM | Prot | Priv, ltr),
2641 static struct group_dual group7 = { {
2642 DI(ModRM | Mov | DstMem | Priv, sgdt),
2643 DI(ModRM | Mov | DstMem | Priv, sidt),
2644 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2645 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2646 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2647 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2649 D(SrcNone | ModRM | Priv | VendorSpecific), N,
2650 N, D(SrcNone | ModRM | Priv | VendorSpecific),
2651 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2652 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), N,
2655 static struct opcode group8[] = {
2657 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2658 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2661 static struct group_dual group9 = { {
2662 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2664 N, N, N, N, N, N, N, N,
2667 static struct opcode group11[] = {
2668 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2671 static struct gprefix pfx_0f_6f_0f_7f = {
2672 N, N, N, I(Sse, em_movdqu),
2675 static struct opcode opcode_table[256] = {
2678 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2681 D(ImplicitOps | Stack | No64), N,
2684 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2687 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2691 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2699 X8(I(SrcReg | Stack, em_push)),
2701 X8(D(DstReg | Stack)),
2703 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2704 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2707 I(SrcImm | Mov | Stack, em_push),
2708 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2709 I(SrcImmByte | Mov | Stack, em_push),
2710 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2711 D2bv(DstDI | Mov | String), /* insb, insw/insd */
2712 D2bv(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
2716 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2717 G(DstMem | SrcImm | ModRM | Group, group1),
2718 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2719 G(DstMem | SrcImmByte | ModRM | Group, group1),
2720 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2722 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2723 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2724 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2725 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2727 X8(D(SrcAcc | DstReg)),
2729 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2730 I(SrcImmFAddr | No64, em_call_far), N,
2731 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2733 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2734 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2735 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2736 D2bv(SrcSI | DstDI | String),
2738 D2bv(DstAcc | SrcImm),
2739 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2740 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2741 D2bv(SrcAcc | DstDI | String),
2743 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2745 X8(I(DstReg | SrcImm | Mov, em_mov)),
2747 D2bv(DstMem | SrcImmByte | ModRM),
2748 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2749 D(ImplicitOps | Stack),
2750 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2751 G(ByteOp, group11), G(0, group11),
2753 N, N, N, D(ImplicitOps | Stack),
2754 D(ImplicitOps), DI(SrcImmByte, intn),
2755 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2757 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2760 N, N, N, N, N, N, N, N,
2763 D2bv(SrcImmUByte | DstAcc), D2bv(SrcAcc | DstImmUByte),
2765 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2766 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2767 D2bv(SrcNone | DstAcc), D2bv(SrcAcc | ImplicitOps),
2770 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2771 G(ByteOp, group3), G(0, group3),
2773 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2774 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2777 static struct opcode twobyte_table[256] = {
2779 G(0, group6), GD(0, &group7), N, N,
2780 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2781 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2782 N, D(ImplicitOps | ModRM), N, N,
2784 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2786 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2787 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2788 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2789 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
2791 N, N, N, N, N, N, N, N,
2793 D(ImplicitOps | Priv), II(ImplicitOps, em_rdtsc, rdtsc),
2794 D(ImplicitOps | Priv), N,
2795 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2797 N, N, N, N, N, N, N, N,
2799 X16(D(DstReg | SrcMem | ModRM | Mov)),
2801 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2806 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
2811 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
2815 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
2817 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2818 N, D(DstMem | SrcReg | ModRM | BitOp),
2819 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2820 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
2822 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2823 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
2824 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2825 D(DstMem | SrcReg | Src2CL | ModRM),
2826 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
2828 D2bv(DstMem | SrcReg | ModRM | Lock),
2829 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2830 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
2831 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2834 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2835 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
2836 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2838 D2bv(DstMem | SrcReg | ModRM | Lock),
2839 N, D(DstMem | SrcReg | ModRM | Mov),
2840 N, N, N, GD(0, &group9),
2841 N, N, N, N, N, N, N, N,
2843 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2845 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2847 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
2861 static unsigned imm_size(struct decode_cache *c)
2865 size = (c->d & ByteOp) ? 1 : c->op_bytes;
2871 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
2872 unsigned size, bool sign_extension)
2874 struct decode_cache *c = &ctxt->decode;
2875 struct x86_emulate_ops *ops = ctxt->ops;
2876 int rc = X86EMUL_CONTINUE;
2880 op->addr.mem.ea = c->eip;
2881 /* NB. Immediates are sign-extended as necessary. */
2882 switch (op->bytes) {
2884 op->val = insn_fetch(s8, 1, c->eip);
2887 op->val = insn_fetch(s16, 2, c->eip);
2890 op->val = insn_fetch(s32, 4, c->eip);
2893 if (!sign_extension) {
2894 switch (op->bytes) {
2902 op->val &= 0xffffffff;
2911 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
2913 struct x86_emulate_ops *ops = ctxt->ops;
2914 struct decode_cache *c = &ctxt->decode;
2915 int rc = X86EMUL_CONTINUE;
2916 int mode = ctxt->mode;
2917 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
2918 bool op_prefix = false;
2919 struct opcode opcode, *g_mod012, *g_mod3;
2920 struct operand memop = { .type = OP_NONE };
2923 c->fetch.start = c->eip;
2924 c->fetch.end = c->fetch.start + insn_len;
2926 memcpy(c->fetch.data, insn, insn_len);
2927 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
2930 case X86EMUL_MODE_REAL:
2931 case X86EMUL_MODE_VM86:
2932 case X86EMUL_MODE_PROT16:
2933 def_op_bytes = def_ad_bytes = 2;
2935 case X86EMUL_MODE_PROT32:
2936 def_op_bytes = def_ad_bytes = 4;
2938 #ifdef CONFIG_X86_64
2939 case X86EMUL_MODE_PROT64:
2948 c->op_bytes = def_op_bytes;
2949 c->ad_bytes = def_ad_bytes;
2951 /* Legacy prefixes. */
2953 switch (c->b = insn_fetch(u8, 1, c->eip)) {
2954 case 0x66: /* operand-size override */
2956 /* switch between 2/4 bytes */
2957 c->op_bytes = def_op_bytes ^ 6;
2959 case 0x67: /* address-size override */
2960 if (mode == X86EMUL_MODE_PROT64)
2961 /* switch between 4/8 bytes */
2962 c->ad_bytes = def_ad_bytes ^ 12;
2964 /* switch between 2/4 bytes */
2965 c->ad_bytes = def_ad_bytes ^ 6;
2967 case 0x26: /* ES override */
2968 case 0x2e: /* CS override */
2969 case 0x36: /* SS override */
2970 case 0x3e: /* DS override */
2971 set_seg_override(c, (c->b >> 3) & 3);
2973 case 0x64: /* FS override */
2974 case 0x65: /* GS override */
2975 set_seg_override(c, c->b & 7);
2977 case 0x40 ... 0x4f: /* REX */
2978 if (mode != X86EMUL_MODE_PROT64)
2980 c->rex_prefix = c->b;
2982 case 0xf0: /* LOCK */
2985 case 0xf2: /* REPNE/REPNZ */
2986 case 0xf3: /* REP/REPE/REPZ */
2987 c->rep_prefix = c->b;
2993 /* Any legacy prefix after a REX prefix nullifies its effect. */
3001 if (c->rex_prefix & 8)
3002 c->op_bytes = 8; /* REX.W */
3004 /* Opcode byte(s). */
3005 opcode = opcode_table[c->b];
3006 /* Two-byte opcode? */
3009 c->b = insn_fetch(u8, 1, c->eip);
3010 opcode = twobyte_table[c->b];
3012 c->d = opcode.flags;
3015 dual = c->d & GroupDual;
3016 c->modrm = insn_fetch(u8, 1, c->eip);
3019 if (c->d & GroupDual) {
3020 g_mod012 = opcode.u.gdual->mod012;
3021 g_mod3 = opcode.u.gdual->mod3;
3023 g_mod012 = g_mod3 = opcode.u.group;
3025 c->d &= ~(Group | GroupDual);
3027 goffset = (c->modrm >> 3) & 7;
3029 if ((c->modrm >> 6) == 3)
3030 opcode = g_mod3[goffset];
3032 opcode = g_mod012[goffset];
3033 c->d |= opcode.flags;
3036 if (c->d & Prefix) {
3037 if (c->rep_prefix && op_prefix)
3038 return X86EMUL_UNHANDLEABLE;
3039 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3040 switch (simd_prefix) {
3041 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3042 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3043 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3044 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3046 c->d |= opcode.flags;
3049 c->execute = opcode.u.execute;
3050 c->check_perm = opcode.check_perm;
3051 c->intercept = opcode.intercept;
3054 if (c->d == 0 || (c->d & Undefined))
3057 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3060 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3063 if (c->d & Op3264) {
3064 if (mode == X86EMUL_MODE_PROT64)
3073 /* ModRM and SIB bytes. */
3075 rc = decode_modrm(ctxt, ops, &memop);
3076 if (!c->has_seg_override)
3077 set_seg_override(c, c->modrm_seg);
3078 } else if (c->d & MemAbs)
3079 rc = decode_abs(ctxt, ops, &memop);
3080 if (rc != X86EMUL_CONTINUE)
3083 if (!c->has_seg_override)
3084 set_seg_override(c, VCPU_SREG_DS);
3086 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3088 if (memop.type == OP_MEM && c->ad_bytes != 8)
3089 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3091 if (memop.type == OP_MEM && c->rip_relative)
3092 memop.addr.mem.ea += c->eip;
3095 * Decode and fetch the source operand: register, memory
3098 switch (c->d & SrcMask) {
3102 decode_register_operand(ctxt, &c->src, c, 0);
3111 memop.bytes = (c->d & ByteOp) ? 1 :
3117 rc = decode_imm(ctxt, &c->src, 2, false);
3120 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3123 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3126 rc = decode_imm(ctxt, &c->src, 1, true);
3129 rc = decode_imm(ctxt, &c->src, 1, false);
3132 c->src.type = OP_REG;
3133 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3134 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3135 fetch_register_operand(&c->src);
3142 c->src.type = OP_MEM;
3143 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3144 c->src.addr.mem.ea =
3145 register_address(c, c->regs[VCPU_REGS_RSI]);
3146 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3150 c->src.type = OP_IMM;
3151 c->src.addr.mem.ea = c->eip;
3152 c->src.bytes = c->op_bytes + 2;
3153 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3156 memop.bytes = c->op_bytes + 2;
3161 if (rc != X86EMUL_CONTINUE)
3165 * Decode and fetch the second source operand: register, memory
3168 switch (c->d & Src2Mask) {
3173 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3176 rc = decode_imm(ctxt, &c->src2, 1, true);
3183 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3187 if (rc != X86EMUL_CONTINUE)
3190 /* Decode and fetch the destination operand: register or memory. */
3191 switch (c->d & DstMask) {
3193 decode_register_operand(ctxt, &c->dst, c,
3194 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3197 c->dst.type = OP_IMM;
3198 c->dst.addr.mem.ea = c->eip;
3200 c->dst.val = insn_fetch(u8, 1, c->eip);
3205 if ((c->d & DstMask) == DstMem64)
3208 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3210 fetch_bit_operand(c);
3211 c->dst.orig_val = c->dst.val;
3214 c->dst.type = OP_REG;
3215 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3216 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3217 fetch_register_operand(&c->dst);
3218 c->dst.orig_val = c->dst.val;
3221 c->dst.type = OP_MEM;
3222 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3223 c->dst.addr.mem.ea =
3224 register_address(c, c->regs[VCPU_REGS_RDI]);
3225 c->dst.addr.mem.seg = VCPU_SREG_ES;
3229 /* Special instructions do their own operand decoding. */
3231 c->dst.type = OP_NONE; /* Disable writeback. */
3236 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3239 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3241 struct decode_cache *c = &ctxt->decode;
3243 /* The second termination condition only applies for REPE
3244 * and REPNE. Test if the repeat string operation prefix is
3245 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3246 * corresponding termination condition according to:
3247 * - if REPE/REPZ and ZF = 0 then done
3248 * - if REPNE/REPNZ and ZF = 1 then done
3250 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3251 (c->b == 0xae) || (c->b == 0xaf))
3252 && (((c->rep_prefix == REPE_PREFIX) &&
3253 ((ctxt->eflags & EFLG_ZF) == 0))
3254 || ((c->rep_prefix == REPNE_PREFIX) &&
3255 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3262 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3264 struct x86_emulate_ops *ops = ctxt->ops;
3266 struct decode_cache *c = &ctxt->decode;
3267 int rc = X86EMUL_CONTINUE;
3268 int saved_dst_type = c->dst.type;
3269 int irq; /* Used for int 3, int, and into */
3271 ctxt->decode.mem_read.pos = 0;
3273 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3274 rc = emulate_ud(ctxt);
3278 /* LOCK prefix is allowed only with some instructions */
3279 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3280 rc = emulate_ud(ctxt);
3284 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3285 rc = emulate_ud(ctxt);
3290 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3291 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3292 rc = emulate_ud(ctxt);
3296 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3297 rc = emulate_nm(ctxt);
3301 if (unlikely(ctxt->guest_mode) && c->intercept) {
3302 rc = emulator_check_intercept(ctxt, c->intercept,
3303 X86_ICPT_PRE_EXCEPT);
3304 if (rc != X86EMUL_CONTINUE)
3308 /* Privileged instruction can be executed only in CPL=0 */
3309 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3310 rc = emulate_gp(ctxt, 0);
3314 /* Instruction can only be executed in protected mode */
3315 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3316 rc = emulate_ud(ctxt);
3320 /* Do instruction specific permission checks */
3321 if (c->check_perm) {
3322 rc = c->check_perm(ctxt);
3323 if (rc != X86EMUL_CONTINUE)
3327 if (unlikely(ctxt->guest_mode) && c->intercept) {
3328 rc = emulator_check_intercept(ctxt, c->intercept,
3329 X86_ICPT_POST_EXCEPT);
3330 if (rc != X86EMUL_CONTINUE)
3334 if (c->rep_prefix && (c->d & String)) {
3335 /* All REP prefixes have the same first termination condition */
3336 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3342 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3343 rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem),
3344 c->src.valptr, c->src.bytes);
3345 if (rc != X86EMUL_CONTINUE)
3347 c->src.orig_val64 = c->src.val64;
3350 if (c->src2.type == OP_MEM) {
3351 rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem),
3352 &c->src2.val, c->src2.bytes);
3353 if (rc != X86EMUL_CONTINUE)
3357 if ((c->d & DstMask) == ImplicitOps)
3361 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3362 /* optimisation - avoid slow emulated read if Mov */
3363 rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem),
3364 &c->dst.val, c->dst.bytes);
3365 if (rc != X86EMUL_CONTINUE)
3368 c->dst.orig_val = c->dst.val;
3372 if (unlikely(ctxt->guest_mode) && c->intercept) {
3373 rc = emulator_check_intercept(ctxt, c->intercept,
3374 X86_ICPT_POST_MEMACCESS);
3375 if (rc != X86EMUL_CONTINUE)
3380 rc = c->execute(ctxt);
3381 if (rc != X86EMUL_CONTINUE)
3392 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3394 case 0x06: /* push es */
3395 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3397 case 0x07: /* pop es */
3398 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3402 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3404 case 0x0e: /* push cs */
3405 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3409 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3411 case 0x16: /* push ss */
3412 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3414 case 0x17: /* pop ss */
3415 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3419 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3421 case 0x1e: /* push ds */
3422 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3424 case 0x1f: /* pop ds */
3425 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3429 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3433 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3437 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3441 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3443 case 0x40 ... 0x47: /* inc r16/r32 */
3444 emulate_1op("inc", c->dst, ctxt->eflags);
3446 case 0x48 ... 0x4f: /* dec r16/r32 */
3447 emulate_1op("dec", c->dst, ctxt->eflags);
3449 case 0x58 ... 0x5f: /* pop reg */
3451 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3453 case 0x60: /* pusha */
3454 rc = emulate_pusha(ctxt, ops);
3456 case 0x61: /* popa */
3457 rc = emulate_popa(ctxt, ops);
3459 case 0x63: /* movsxd */
3460 if (ctxt->mode != X86EMUL_MODE_PROT64)
3461 goto cannot_emulate;
3462 c->dst.val = (s32) c->src.val;
3464 case 0x6c: /* insb */
3465 case 0x6d: /* insw/insd */
3466 c->src.val = c->regs[VCPU_REGS_RDX];
3468 case 0x6e: /* outsb */
3469 case 0x6f: /* outsw/outsd */
3470 c->dst.val = c->regs[VCPU_REGS_RDX];
3473 case 0x70 ... 0x7f: /* jcc (short) */
3474 if (test_cc(c->b, ctxt->eflags))
3475 jmp_rel(c, c->src.val);
3477 case 0x80 ... 0x83: /* Grp1 */
3478 switch (c->modrm_reg) {
3499 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3501 case 0x86 ... 0x87: /* xchg */
3503 /* Write back the register source. */
3504 c->src.val = c->dst.val;
3505 write_register_operand(&c->src);
3507 * Write back the memory destination with implicit LOCK
3510 c->dst.val = c->src.orig_val;
3513 case 0x8c: /* mov r/m, sreg */
3514 if (c->modrm_reg > VCPU_SREG_GS) {
3515 rc = emulate_ud(ctxt);
3518 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3520 case 0x8d: /* lea r16/r32, m */
3521 c->dst.val = c->src.addr.mem.ea;
3523 case 0x8e: { /* mov seg, r/m16 */
3528 if (c->modrm_reg == VCPU_SREG_CS ||
3529 c->modrm_reg > VCPU_SREG_GS) {
3530 rc = emulate_ud(ctxt);
3534 if (c->modrm_reg == VCPU_SREG_SS)
3535 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3537 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3539 c->dst.type = OP_NONE; /* Disable writeback. */
3542 case 0x8f: /* pop (sole member of Grp1a) */
3543 rc = emulate_grp1a(ctxt, ops);
3545 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3546 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3549 case 0x98: /* cbw/cwde/cdqe */
3550 switch (c->op_bytes) {
3551 case 2: c->dst.val = (s8)c->dst.val; break;
3552 case 4: c->dst.val = (s16)c->dst.val; break;
3553 case 8: c->dst.val = (s32)c->dst.val; break;
3556 case 0x9c: /* pushf */
3557 c->src.val = (unsigned long) ctxt->eflags;
3558 emulate_push(ctxt, ops);
3560 case 0x9d: /* popf */
3561 c->dst.type = OP_REG;
3562 c->dst.addr.reg = &ctxt->eflags;
3563 c->dst.bytes = c->op_bytes;
3564 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3566 case 0xa6 ... 0xa7: /* cmps */
3567 c->dst.type = OP_NONE; /* Disable writeback. */
3569 case 0xa8 ... 0xa9: /* test ax, imm */
3571 case 0xae ... 0xaf: /* scas */
3576 case 0xc3: /* ret */
3577 c->dst.type = OP_REG;
3578 c->dst.addr.reg = &c->eip;
3579 c->dst.bytes = c->op_bytes;
3580 goto pop_instruction;
3581 case 0xc4: /* les */
3582 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3584 case 0xc5: /* lds */
3585 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3587 case 0xcb: /* ret far */
3588 rc = emulate_ret_far(ctxt, ops);
3590 case 0xcc: /* int3 */
3593 case 0xcd: /* int n */
3596 rc = emulate_int(ctxt, ops, irq);
3598 case 0xce: /* into */
3599 if (ctxt->eflags & EFLG_OF) {
3604 case 0xcf: /* iret */
3605 rc = emulate_iret(ctxt, ops);
3607 case 0xd0 ... 0xd1: /* Grp2 */
3610 case 0xd2 ... 0xd3: /* Grp2 */
3611 c->src.val = c->regs[VCPU_REGS_RCX];
3614 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3615 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3616 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3617 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3618 jmp_rel(c, c->src.val);
3620 case 0xe3: /* jcxz/jecxz/jrcxz */
3621 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3622 jmp_rel(c, c->src.val);
3624 case 0xe4: /* inb */
3627 case 0xe6: /* outb */
3628 case 0xe7: /* out */
3630 case 0xe8: /* call (near) */ {
3631 long int rel = c->src.val;
3632 c->src.val = (unsigned long) c->eip;
3634 emulate_push(ctxt, ops);
3637 case 0xe9: /* jmp rel */
3639 case 0xea: { /* jmp far */
3642 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3644 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3648 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3652 jmp: /* jmp rel short */
3653 jmp_rel(c, c->src.val);
3654 c->dst.type = OP_NONE; /* Disable writeback. */
3656 case 0xec: /* in al,dx */
3657 case 0xed: /* in (e/r)ax,dx */
3658 c->src.val = c->regs[VCPU_REGS_RDX];
3660 c->dst.bytes = min(c->dst.bytes, 4u);
3661 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3662 rc = emulate_gp(ctxt, 0);
3665 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3667 goto done; /* IO is needed */
3669 case 0xee: /* out dx,al */
3670 case 0xef: /* out dx,(e/r)ax */
3671 c->dst.val = c->regs[VCPU_REGS_RDX];
3673 c->src.bytes = min(c->src.bytes, 4u);
3674 if (!emulator_io_permited(ctxt, ops, c->dst.val,
3676 rc = emulate_gp(ctxt, 0);
3679 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3680 &c->src.val, 1, ctxt->vcpu);
3681 c->dst.type = OP_NONE; /* Disable writeback. */
3683 case 0xf4: /* hlt */
3684 ctxt->vcpu->arch.halt_request = 1;
3686 case 0xf5: /* cmc */
3687 /* complement carry flag from eflags reg */
3688 ctxt->eflags ^= EFLG_CF;
3690 case 0xf6 ... 0xf7: /* Grp3 */
3691 rc = emulate_grp3(ctxt, ops);
3693 case 0xf8: /* clc */
3694 ctxt->eflags &= ~EFLG_CF;
3696 case 0xf9: /* stc */
3697 ctxt->eflags |= EFLG_CF;
3699 case 0xfa: /* cli */
3700 if (emulator_bad_iopl(ctxt, ops)) {
3701 rc = emulate_gp(ctxt, 0);
3704 ctxt->eflags &= ~X86_EFLAGS_IF;
3706 case 0xfb: /* sti */
3707 if (emulator_bad_iopl(ctxt, ops)) {
3708 rc = emulate_gp(ctxt, 0);
3711 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3712 ctxt->eflags |= X86_EFLAGS_IF;
3715 case 0xfc: /* cld */
3716 ctxt->eflags &= ~EFLG_DF;
3718 case 0xfd: /* std */
3719 ctxt->eflags |= EFLG_DF;
3721 case 0xfe: /* Grp4 */
3723 rc = emulate_grp45(ctxt, ops);
3725 case 0xff: /* Grp5 */
3726 if (c->modrm_reg == 5)
3730 goto cannot_emulate;
3733 if (rc != X86EMUL_CONTINUE)
3737 rc = writeback(ctxt, ops);
3738 if (rc != X86EMUL_CONTINUE)
3742 * restore dst type in case the decoding will be reused
3743 * (happens for string instruction )
3745 c->dst.type = saved_dst_type;
3747 if ((c->d & SrcMask) == SrcSI)
3748 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3749 VCPU_REGS_RSI, &c->src);
3751 if ((c->d & DstMask) == DstDI)
3752 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3755 if (c->rep_prefix && (c->d & String)) {
3756 struct read_cache *r = &ctxt->decode.io_read;
3757 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3759 if (!string_insn_completed(ctxt)) {
3761 * Re-enter guest when pio read ahead buffer is empty
3762 * or, if it is not used, after each 1024 iteration.
3764 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3765 (r->end == 0 || r->end != r->pos)) {
3767 * Reset read cache. Usually happens before
3768 * decode, but since instruction is restarted
3769 * we have to do it here.
3771 ctxt->decode.mem_read.end = 0;
3772 return EMULATION_RESTART;
3774 goto done; /* skip rip writeback */
3781 if (rc == X86EMUL_PROPAGATE_FAULT)
3782 ctxt->have_exception = true;
3783 if (rc == X86EMUL_INTERCEPTED)
3784 return EMULATION_INTERCEPTED;
3786 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3790 case 0x01: /* lgdt, lidt, lmsw */
3791 switch (c->modrm_reg) {
3793 unsigned long address;
3795 case 0: /* vmcall */
3796 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3797 goto cannot_emulate;
3799 rc = kvm_fix_hypercall(ctxt->vcpu);
3800 if (rc != X86EMUL_CONTINUE)
3803 /* Let the processor re-execute the fixed hypercall */
3805 /* Disable writeback. */
3806 c->dst.type = OP_NONE;
3809 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3810 &size, &address, c->op_bytes);
3811 if (rc != X86EMUL_CONTINUE)
3813 realmode_lgdt(ctxt->vcpu, size, address);
3814 /* Disable writeback. */
3815 c->dst.type = OP_NONE;
3817 case 3: /* lidt/vmmcall */
3818 if (c->modrm_mod == 3) {
3819 switch (c->modrm_rm) {
3821 rc = kvm_fix_hypercall(ctxt->vcpu);
3824 goto cannot_emulate;
3827 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3830 if (rc != X86EMUL_CONTINUE)
3832 realmode_lidt(ctxt->vcpu, size, address);
3834 /* Disable writeback. */
3835 c->dst.type = OP_NONE;
3839 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3842 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
3843 (c->src.val & 0x0f), ctxt->vcpu);
3844 c->dst.type = OP_NONE;
3846 case 5: /* not defined */
3848 rc = X86EMUL_PROPAGATE_FAULT;
3851 emulate_invlpg(ctxt->vcpu,
3852 linear(ctxt, c->src.addr.mem));
3853 /* Disable writeback. */
3854 c->dst.type = OP_NONE;
3857 goto cannot_emulate;
3860 case 0x05: /* syscall */
3861 rc = emulate_syscall(ctxt, ops);
3864 emulate_clts(ctxt->vcpu);
3866 case 0x09: /* wbinvd */
3867 kvm_emulate_wbinvd(ctxt->vcpu);
3869 case 0x08: /* invd */
3870 case 0x0d: /* GrpP (prefetch) */
3871 case 0x18: /* Grp16 (prefetch/nop) */
3873 case 0x20: /* mov cr, reg */
3874 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3876 case 0x21: /* mov from dr to reg */
3877 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
3879 case 0x22: /* mov reg, cr */
3880 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
3881 emulate_gp(ctxt, 0);
3882 rc = X86EMUL_PROPAGATE_FAULT;
3885 c->dst.type = OP_NONE;
3887 case 0x23: /* mov from reg to dr */
3888 if (ops->set_dr(c->modrm_reg, c->src.val &
3889 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3890 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3891 /* #UD condition is already handled by the code above */
3892 emulate_gp(ctxt, 0);
3893 rc = X86EMUL_PROPAGATE_FAULT;
3897 c->dst.type = OP_NONE; /* no writeback */
3901 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3902 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3903 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3904 emulate_gp(ctxt, 0);
3905 rc = X86EMUL_PROPAGATE_FAULT;
3908 rc = X86EMUL_CONTINUE;
3912 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3913 emulate_gp(ctxt, 0);
3914 rc = X86EMUL_PROPAGATE_FAULT;
3917 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3918 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3920 rc = X86EMUL_CONTINUE;
3922 case 0x34: /* sysenter */
3923 rc = emulate_sysenter(ctxt, ops);
3925 case 0x35: /* sysexit */
3926 rc = emulate_sysexit(ctxt, ops);
3928 case 0x40 ... 0x4f: /* cmov */
3929 c->dst.val = c->dst.orig_val = c->src.val;
3930 if (!test_cc(c->b, ctxt->eflags))
3931 c->dst.type = OP_NONE; /* no writeback */
3933 case 0x80 ... 0x8f: /* jnz rel, etc*/
3934 if (test_cc(c->b, ctxt->eflags))
3935 jmp_rel(c, c->src.val);
3937 case 0x90 ... 0x9f: /* setcc r/m8 */
3938 c->dst.val = test_cc(c->b, ctxt->eflags);
3940 case 0xa0: /* push fs */
3941 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3943 case 0xa1: /* pop fs */
3944 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3948 c->dst.type = OP_NONE;
3949 /* only subword offset */
3950 c->src.val &= (c->dst.bytes << 3) - 1;
3951 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3953 case 0xa4: /* shld imm8, r, r/m */
3954 case 0xa5: /* shld cl, r, r/m */
3955 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3957 case 0xa8: /* push gs */
3958 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3960 case 0xa9: /* pop gs */
3961 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3965 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3967 case 0xac: /* shrd imm8, r, r/m */
3968 case 0xad: /* shrd cl, r, r/m */
3969 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3971 case 0xae: /* clflush */
3973 case 0xb0 ... 0xb1: /* cmpxchg */
3975 * Save real source value, then compare EAX against
3978 c->src.orig_val = c->src.val;
3979 c->src.val = c->regs[VCPU_REGS_RAX];
3980 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3981 if (ctxt->eflags & EFLG_ZF) {
3982 /* Success: write back to memory. */
3983 c->dst.val = c->src.orig_val;
3985 /* Failure: write the value we saw to EAX. */
3986 c->dst.type = OP_REG;
3987 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3990 case 0xb2: /* lss */
3991 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
3995 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3997 case 0xb4: /* lfs */
3998 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4000 case 0xb5: /* lgs */
4001 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4003 case 0xb6 ... 0xb7: /* movzx */
4004 c->dst.bytes = c->op_bytes;
4005 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4008 case 0xba: /* Grp8 */
4009 switch (c->modrm_reg & 3) {
4022 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4024 case 0xbc: { /* bsf */
4026 __asm__ ("bsf %2, %0; setz %1"
4027 : "=r"(c->dst.val), "=q"(zf)
4029 ctxt->eflags &= ~X86_EFLAGS_ZF;
4031 ctxt->eflags |= X86_EFLAGS_ZF;
4032 c->dst.type = OP_NONE; /* Disable writeback. */
4036 case 0xbd: { /* bsr */
4038 __asm__ ("bsr %2, %0; setz %1"
4039 : "=r"(c->dst.val), "=q"(zf)
4041 ctxt->eflags &= ~X86_EFLAGS_ZF;
4043 ctxt->eflags |= X86_EFLAGS_ZF;
4044 c->dst.type = OP_NONE; /* Disable writeback. */
4048 case 0xbe ... 0xbf: /* movsx */
4049 c->dst.bytes = c->op_bytes;
4050 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4053 case 0xc0 ... 0xc1: /* xadd */
4054 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4055 /* Write back the register source. */
4056 c->src.val = c->dst.orig_val;
4057 write_register_operand(&c->src);
4059 case 0xc3: /* movnti */
4060 c->dst.bytes = c->op_bytes;
4061 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4064 case 0xc7: /* Grp9 (cmpxchg8b) */
4065 rc = emulate_grp9(ctxt, ops);
4068 goto cannot_emulate;
4071 if (rc != X86EMUL_CONTINUE)