1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
81 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
82 #define VendorSpecific (1<<22) /* Vendor specific instruction */
83 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
84 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
85 #define Undefined (1<<25) /* No Such Instruction */
86 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
87 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
89 /* Source 2 operand type */
90 #define Src2None (0<<29)
91 #define Src2CL (1<<29)
92 #define Src2ImmByte (2<<29)
93 #define Src2One (3<<29)
94 #define Src2Imm (4<<29)
95 #define Src2Mask (7<<29)
98 #define X3(x...) X2(x), x
99 #define X4(x...) X2(x), X2(x)
100 #define X5(x...) X4(x), x
101 #define X6(x...) X4(x), X2(x)
102 #define X7(x...) X4(x), X3(x)
103 #define X8(x...) X4(x), X4(x)
104 #define X16(x...) X8(x), X8(x)
110 int (*execute)(struct x86_emulate_ctxt *ctxt);
111 struct opcode *group;
112 struct group_dual *gdual;
113 struct gprefix *gprefix;
115 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
119 struct opcode mod012[8];
120 struct opcode mod3[8];
124 struct opcode pfx_no;
125 struct opcode pfx_66;
126 struct opcode pfx_f2;
127 struct opcode pfx_f3;
130 /* EFLAGS bit definitions. */
131 #define EFLG_ID (1<<21)
132 #define EFLG_VIP (1<<20)
133 #define EFLG_VIF (1<<19)
134 #define EFLG_AC (1<<18)
135 #define EFLG_VM (1<<17)
136 #define EFLG_RF (1<<16)
137 #define EFLG_IOPL (3<<12)
138 #define EFLG_NT (1<<14)
139 #define EFLG_OF (1<<11)
140 #define EFLG_DF (1<<10)
141 #define EFLG_IF (1<<9)
142 #define EFLG_TF (1<<8)
143 #define EFLG_SF (1<<7)
144 #define EFLG_ZF (1<<6)
145 #define EFLG_AF (1<<4)
146 #define EFLG_PF (1<<2)
147 #define EFLG_CF (1<<0)
149 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
150 #define EFLG_RESERVED_ONE_MASK 2
153 * Instruction emulation:
154 * Most instructions are emulated directly via a fragment of inline assembly
155 * code. This allows us to save/restore EFLAGS and thus very easily pick up
156 * any modified flags.
159 #if defined(CONFIG_X86_64)
160 #define _LO32 "k" /* force 32-bit operand */
161 #define _STK "%%rsp" /* stack pointer */
162 #elif defined(__i386__)
163 #define _LO32 "" /* force 32-bit operand */
164 #define _STK "%%esp" /* stack pointer */
168 * These EFLAGS bits are restored from saved value during emulation, and
169 * any changes are written back to the saved value after emulation.
171 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
173 /* Before executing instruction: restore necessary bits in EFLAGS. */
174 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
175 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
176 "movl %"_sav",%"_LO32 _tmp"; " \
179 "movl %"_msk",%"_LO32 _tmp"; " \
180 "andl %"_LO32 _tmp",("_STK"); " \
182 "notl %"_LO32 _tmp"; " \
183 "andl %"_LO32 _tmp",("_STK"); " \
184 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
186 "orl %"_LO32 _tmp",("_STK"); " \
190 /* After executing instruction: write-back necessary bits in EFLAGS. */
191 #define _POST_EFLAGS(_sav, _msk, _tmp) \
192 /* _sav |= EFLAGS & _msk; */ \
195 "andl %"_msk",%"_LO32 _tmp"; " \
196 "orl %"_LO32 _tmp",%"_sav"; "
204 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
206 __asm__ __volatile__ ( \
207 _PRE_EFLAGS("0", "4", "2") \
208 _op _suffix " %"_x"3,%1; " \
209 _POST_EFLAGS("0", "4", "2") \
210 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
212 : _y ((_src).val), "i" (EFLAGS_MASK)); \
216 /* Raw emulation: instruction has two explicit operands. */
217 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
219 unsigned long _tmp; \
221 switch ((_dst).bytes) { \
223 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
226 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
229 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
234 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
236 unsigned long _tmp; \
237 switch ((_dst).bytes) { \
239 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
242 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
243 _wx, _wy, _lx, _ly, _qx, _qy); \
248 /* Source operand is byte-sized and may be restricted to just %cl. */
249 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
250 __emulate_2op(_op, _src, _dst, _eflags, \
251 "b", "c", "b", "c", "b", "c", "b", "c")
253 /* Source operand is byte, word, long or quad sized. */
254 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
255 __emulate_2op(_op, _src, _dst, _eflags, \
256 "b", "q", "w", "r", _LO32, "r", "", "r")
258 /* Source operand is word, long or quad sized. */
259 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
260 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
261 "w", "r", _LO32, "r", "", "r")
263 /* Instruction has three operands and one operand is stored in ECX register */
264 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
266 unsigned long _tmp; \
267 _type _clv = (_cl).val; \
268 _type _srcv = (_src).val; \
269 _type _dstv = (_dst).val; \
271 __asm__ __volatile__ ( \
272 _PRE_EFLAGS("0", "5", "2") \
273 _op _suffix " %4,%1 \n" \
274 _POST_EFLAGS("0", "5", "2") \
275 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
276 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
279 (_cl).val = (unsigned long) _clv; \
280 (_src).val = (unsigned long) _srcv; \
281 (_dst).val = (unsigned long) _dstv; \
284 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
286 switch ((_dst).bytes) { \
288 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
289 "w", unsigned short); \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "l", unsigned int); \
296 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "q", unsigned long)); \
302 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
304 unsigned long _tmp; \
306 __asm__ __volatile__ ( \
307 _PRE_EFLAGS("0", "3", "2") \
308 _op _suffix " %1; " \
309 _POST_EFLAGS("0", "3", "2") \
310 : "=m" (_eflags), "+m" ((_dst).val), \
312 : "i" (EFLAGS_MASK)); \
315 /* Instruction has only one explicit operand (no source operand). */
316 #define emulate_1op(_op, _dst, _eflags) \
318 switch ((_dst).bytes) { \
319 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
320 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
321 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
322 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
326 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
328 unsigned long _tmp; \
330 __asm__ __volatile__ ( \
331 _PRE_EFLAGS("0", "4", "1") \
332 _op _suffix " %5; " \
333 _POST_EFLAGS("0", "4", "1") \
334 : "=m" (_eflags), "=&r" (_tmp), \
335 "+a" (_rax), "+d" (_rdx) \
336 : "i" (EFLAGS_MASK), "m" ((_src).val), \
337 "a" (_rax), "d" (_rdx)); \
340 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
342 unsigned long _tmp; \
344 __asm__ __volatile__ ( \
345 _PRE_EFLAGS("0", "5", "1") \
347 _op _suffix " %6; " \
349 _POST_EFLAGS("0", "5", "1") \
350 ".pushsection .fixup,\"ax\" \n\t" \
351 "3: movb $1, %4 \n\t" \
354 _ASM_EXTABLE(1b, 3b) \
355 : "=m" (_eflags), "=&r" (_tmp), \
356 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
357 : "i" (EFLAGS_MASK), "m" ((_src).val), \
358 "a" (_rax), "d" (_rdx)); \
361 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
362 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
364 switch((_src).bytes) { \
365 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
366 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
367 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
368 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
372 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
374 switch((_src).bytes) { \
376 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
377 _eflags, "b", _ex); \
380 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
381 _eflags, "w", _ex); \
384 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
385 _eflags, "l", _ex); \
388 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
389 _eflags, "q", _ex)); \
394 /* Fetch next part of the instruction being emulated. */
395 #define insn_fetch(_type, _size, _eip) \
396 ({ unsigned long _x; \
397 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
398 if (rc != X86EMUL_CONTINUE) \
404 #define insn_fetch_arr(_arr, _size, _eip) \
405 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
406 if (rc != X86EMUL_CONTINUE) \
411 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
412 enum x86_intercept intercept,
413 enum x86_intercept_stage stage)
415 struct x86_instruction_info info = {
416 .intercept = intercept,
417 .rep_prefix = ctxt->decode.rep_prefix,
418 .modrm_mod = ctxt->decode.modrm_mod,
419 .modrm_reg = ctxt->decode.modrm_reg,
420 .modrm_rm = ctxt->decode.modrm_rm,
421 .src_val = ctxt->decode.src.val64,
422 .src_bytes = ctxt->decode.src.bytes,
423 .dst_bytes = ctxt->decode.dst.bytes,
424 .ad_bytes = ctxt->decode.ad_bytes,
425 .next_rip = ctxt->eip,
428 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
431 static inline unsigned long ad_mask(struct decode_cache *c)
433 return (1UL << (c->ad_bytes << 3)) - 1;
436 /* Access/update address held in a register, based on addressing mode. */
437 static inline unsigned long
438 address_mask(struct decode_cache *c, unsigned long reg)
440 if (c->ad_bytes == sizeof(unsigned long))
443 return reg & ad_mask(c);
446 static inline unsigned long
447 register_address(struct decode_cache *c, unsigned long reg)
449 return address_mask(c, reg);
453 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
455 if (c->ad_bytes == sizeof(unsigned long))
458 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
461 static inline void jmp_rel(struct decode_cache *c, int rel)
463 register_address_increment(c, &c->eip, rel);
466 static void set_seg_override(struct decode_cache *c, int seg)
468 c->has_seg_override = true;
469 c->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
473 struct x86_emulate_ops *ops, int seg)
475 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
478 return ops->get_cached_segment_base(seg, ctxt->vcpu);
481 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
482 struct x86_emulate_ops *ops,
483 struct decode_cache *c)
485 if (!c->has_seg_override)
488 return c->seg_override;
491 static ulong linear(struct x86_emulate_ctxt *ctxt,
492 struct segmented_address addr)
494 struct decode_cache *c = &ctxt->decode;
497 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
498 if (c->ad_bytes != 8)
503 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
504 u32 error, bool valid)
506 ctxt->exception.vector = vec;
507 ctxt->exception.error_code = error;
508 ctxt->exception.error_code_valid = valid;
509 return X86EMUL_PROPAGATE_FAULT;
512 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
514 return emulate_exception(ctxt, GP_VECTOR, err, true);
517 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519 return emulate_exception(ctxt, UD_VECTOR, 0, false);
522 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524 return emulate_exception(ctxt, TS_VECTOR, err, true);
527 static int emulate_de(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, DE_VECTOR, 0, false);
532 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, NM_VECTOR, 0, false);
537 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
538 struct x86_emulate_ops *ops,
539 unsigned long eip, u8 *dest)
541 struct fetch_cache *fc = &ctxt->decode.fetch;
545 if (eip == fc->end) {
546 cur_size = fc->end - fc->start;
547 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
548 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
549 size, ctxt->vcpu, &ctxt->exception);
550 if (rc != X86EMUL_CONTINUE)
554 *dest = fc->data[eip - fc->start];
555 return X86EMUL_CONTINUE;
558 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
559 struct x86_emulate_ops *ops,
560 unsigned long eip, void *dest, unsigned size)
564 /* x86 instructions are limited to 15 bytes. */
565 if (eip + size - ctxt->eip > 15)
566 return X86EMUL_UNHANDLEABLE;
568 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
569 if (rc != X86EMUL_CONTINUE)
572 return X86EMUL_CONTINUE;
576 * Given the 'reg' portion of a ModRM byte, and a register block, return a
577 * pointer into the block that addresses the relevant register.
578 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
580 static void *decode_register(u8 modrm_reg, unsigned long *regs,
585 p = ®s[modrm_reg];
586 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
587 p = (unsigned char *)®s[modrm_reg & 3] + 1;
591 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
592 struct x86_emulate_ops *ops,
593 struct segmented_address addr,
594 u16 *size, unsigned long *address, int op_bytes)
601 rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2,
602 ctxt->vcpu, &ctxt->exception);
603 if (rc != X86EMUL_CONTINUE)
606 rc = ops->read_std(linear(ctxt, addr), address, op_bytes,
607 ctxt->vcpu, &ctxt->exception);
611 static int test_cc(unsigned int condition, unsigned int flags)
615 switch ((condition & 15) >> 1) {
617 rc |= (flags & EFLG_OF);
619 case 1: /* b/c/nae */
620 rc |= (flags & EFLG_CF);
623 rc |= (flags & EFLG_ZF);
626 rc |= (flags & (EFLG_CF|EFLG_ZF));
629 rc |= (flags & EFLG_SF);
632 rc |= (flags & EFLG_PF);
635 rc |= (flags & EFLG_ZF);
638 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
642 /* Odd condition identifiers (lsb == 1) have inverted sense. */
643 return (!!rc ^ (condition & 1));
646 static void fetch_register_operand(struct operand *op)
650 op->val = *(u8 *)op->addr.reg;
653 op->val = *(u16 *)op->addr.reg;
656 op->val = *(u32 *)op->addr.reg;
659 op->val = *(u64 *)op->addr.reg;
664 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
666 ctxt->ops->get_fpu(ctxt);
668 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
669 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
670 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
671 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
672 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
673 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
674 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
675 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
677 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
678 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
679 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
680 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
681 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
682 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
683 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
684 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
688 ctxt->ops->put_fpu(ctxt);
691 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
694 ctxt->ops->get_fpu(ctxt);
696 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
697 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
698 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
699 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
700 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
701 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
702 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
703 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
705 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
706 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
707 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
708 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
709 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
710 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
711 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
712 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
716 ctxt->ops->put_fpu(ctxt);
719 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
721 struct decode_cache *c,
724 unsigned reg = c->modrm_reg;
725 int highbyte_regs = c->rex_prefix == 0;
728 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
734 read_sse_reg(ctxt, &op->vec_val, reg);
739 if ((c->d & ByteOp) && !inhibit_bytereg) {
740 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
743 op->addr.reg = decode_register(reg, c->regs, 0);
744 op->bytes = c->op_bytes;
746 fetch_register_operand(op);
747 op->orig_val = op->val;
750 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
751 struct x86_emulate_ops *ops,
754 struct decode_cache *c = &ctxt->decode;
756 int index_reg = 0, base_reg = 0, scale;
757 int rc = X86EMUL_CONTINUE;
761 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
762 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
763 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
766 c->modrm = insn_fetch(u8, 1, c->eip);
767 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
768 c->modrm_reg |= (c->modrm & 0x38) >> 3;
769 c->modrm_rm |= (c->modrm & 0x07);
770 c->modrm_seg = VCPU_SREG_DS;
772 if (c->modrm_mod == 3) {
774 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
775 op->addr.reg = decode_register(c->modrm_rm,
776 c->regs, c->d & ByteOp);
780 op->addr.xmm = c->modrm_rm;
781 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
784 fetch_register_operand(op);
790 if (c->ad_bytes == 2) {
791 unsigned bx = c->regs[VCPU_REGS_RBX];
792 unsigned bp = c->regs[VCPU_REGS_RBP];
793 unsigned si = c->regs[VCPU_REGS_RSI];
794 unsigned di = c->regs[VCPU_REGS_RDI];
796 /* 16-bit ModR/M decode. */
797 switch (c->modrm_mod) {
799 if (c->modrm_rm == 6)
800 modrm_ea += insn_fetch(u16, 2, c->eip);
803 modrm_ea += insn_fetch(s8, 1, c->eip);
806 modrm_ea += insn_fetch(u16, 2, c->eip);
809 switch (c->modrm_rm) {
829 if (c->modrm_mod != 0)
836 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
837 (c->modrm_rm == 6 && c->modrm_mod != 0))
838 c->modrm_seg = VCPU_SREG_SS;
839 modrm_ea = (u16)modrm_ea;
841 /* 32/64-bit ModR/M decode. */
842 if ((c->modrm_rm & 7) == 4) {
843 sib = insn_fetch(u8, 1, c->eip);
844 index_reg |= (sib >> 3) & 7;
848 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
849 modrm_ea += insn_fetch(s32, 4, c->eip);
851 modrm_ea += c->regs[base_reg];
853 modrm_ea += c->regs[index_reg] << scale;
854 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
855 if (ctxt->mode == X86EMUL_MODE_PROT64)
858 modrm_ea += c->regs[c->modrm_rm];
859 switch (c->modrm_mod) {
861 if (c->modrm_rm == 5)
862 modrm_ea += insn_fetch(s32, 4, c->eip);
865 modrm_ea += insn_fetch(s8, 1, c->eip);
868 modrm_ea += insn_fetch(s32, 4, c->eip);
872 op->addr.mem.ea = modrm_ea;
877 static int decode_abs(struct x86_emulate_ctxt *ctxt,
878 struct x86_emulate_ops *ops,
881 struct decode_cache *c = &ctxt->decode;
882 int rc = X86EMUL_CONTINUE;
885 switch (c->ad_bytes) {
887 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
890 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
893 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
900 static void fetch_bit_operand(struct decode_cache *c)
904 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
905 mask = ~(c->dst.bytes * 8 - 1);
907 if (c->src.bytes == 2)
908 sv = (s16)c->src.val & (s16)mask;
909 else if (c->src.bytes == 4)
910 sv = (s32)c->src.val & (s32)mask;
912 c->dst.addr.mem.ea += (sv >> 3);
915 /* only subword offset */
916 c->src.val &= (c->dst.bytes << 3) - 1;
919 static int read_emulated(struct x86_emulate_ctxt *ctxt,
920 struct x86_emulate_ops *ops,
921 unsigned long addr, void *dest, unsigned size)
924 struct read_cache *mc = &ctxt->decode.mem_read;
927 int n = min(size, 8u);
929 if (mc->pos < mc->end)
932 rc = ops->read_emulated(addr, mc->data + mc->end, n,
933 &ctxt->exception, ctxt->vcpu);
934 if (rc != X86EMUL_CONTINUE)
939 memcpy(dest, mc->data + mc->pos, n);
944 return X86EMUL_CONTINUE;
947 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
948 struct x86_emulate_ops *ops,
949 unsigned int size, unsigned short port,
952 struct read_cache *rc = &ctxt->decode.io_read;
954 if (rc->pos == rc->end) { /* refill pio read ahead */
955 struct decode_cache *c = &ctxt->decode;
956 unsigned int in_page, n;
957 unsigned int count = c->rep_prefix ?
958 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
959 in_page = (ctxt->eflags & EFLG_DF) ?
960 offset_in_page(c->regs[VCPU_REGS_RDI]) :
961 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
962 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
966 rc->pos = rc->end = 0;
967 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
972 memcpy(dest, rc->data + rc->pos, size);
977 static u32 desc_limit_scaled(struct desc_struct *desc)
979 u32 limit = get_desc_limit(desc);
981 return desc->g ? (limit << 12) | 0xfff : limit;
984 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
985 struct x86_emulate_ops *ops,
986 u16 selector, struct desc_ptr *dt)
988 if (selector & 1 << 2) {
989 struct desc_struct desc;
990 memset (dt, 0, sizeof *dt);
991 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
995 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
996 dt->address = get_desc_base(&desc);
998 ops->get_gdt(dt, ctxt->vcpu);
1001 /* allowed just for 8 bytes segments */
1002 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1003 struct x86_emulate_ops *ops,
1004 u16 selector, struct desc_struct *desc)
1007 u16 index = selector >> 3;
1011 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1013 if (dt.size < index * 8 + 7)
1014 return emulate_gp(ctxt, selector & 0xfffc);
1015 addr = dt.address + index * 8;
1016 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1022 /* allowed just for 8 bytes segments */
1023 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1024 struct x86_emulate_ops *ops,
1025 u16 selector, struct desc_struct *desc)
1028 u16 index = selector >> 3;
1032 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1034 if (dt.size < index * 8 + 7)
1035 return emulate_gp(ctxt, selector & 0xfffc);
1037 addr = dt.address + index * 8;
1038 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1044 /* Does not support long mode */
1045 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1046 struct x86_emulate_ops *ops,
1047 u16 selector, int seg)
1049 struct desc_struct seg_desc;
1051 unsigned err_vec = GP_VECTOR;
1053 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1056 memset(&seg_desc, 0, sizeof seg_desc);
1058 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1059 || ctxt->mode == X86EMUL_MODE_REAL) {
1060 /* set real mode segment descriptor */
1061 set_desc_base(&seg_desc, selector << 4);
1062 set_desc_limit(&seg_desc, 0xffff);
1069 /* NULL selector is not valid for TR, CS and SS */
1070 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1074 /* TR should be in GDT only */
1075 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1078 if (null_selector) /* for NULL selector skip all following checks */
1081 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1082 if (ret != X86EMUL_CONTINUE)
1085 err_code = selector & 0xfffc;
1086 err_vec = GP_VECTOR;
1088 /* can't load system descriptor into segment selecor */
1089 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1093 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1099 cpl = ops->cpl(ctxt->vcpu);
1104 * segment is not a writable data segment or segment
1105 * selector's RPL != CPL or segment selector's RPL != CPL
1107 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1111 if (!(seg_desc.type & 8))
1114 if (seg_desc.type & 4) {
1120 if (rpl > cpl || dpl != cpl)
1123 /* CS(RPL) <- CPL */
1124 selector = (selector & 0xfffc) | cpl;
1127 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1130 case VCPU_SREG_LDTR:
1131 if (seg_desc.s || seg_desc.type != 2)
1134 default: /* DS, ES, FS, or GS */
1136 * segment is not a data or readable code segment or
1137 * ((segment is a data or nonconforming code segment)
1138 * and (both RPL and CPL > DPL))
1140 if ((seg_desc.type & 0xa) == 0x8 ||
1141 (((seg_desc.type & 0xc) != 0xc) &&
1142 (rpl > dpl && cpl > dpl)))
1148 /* mark segment as accessed */
1150 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1151 if (ret != X86EMUL_CONTINUE)
1155 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1156 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1157 return X86EMUL_CONTINUE;
1159 emulate_exception(ctxt, err_vec, err_code, true);
1160 return X86EMUL_PROPAGATE_FAULT;
1163 static void write_register_operand(struct operand *op)
1165 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1166 switch (op->bytes) {
1168 *(u8 *)op->addr.reg = (u8)op->val;
1171 *(u16 *)op->addr.reg = (u16)op->val;
1174 *op->addr.reg = (u32)op->val;
1175 break; /* 64b: zero-extend */
1177 *op->addr.reg = op->val;
1182 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1183 struct x86_emulate_ops *ops)
1186 struct decode_cache *c = &ctxt->decode;
1188 switch (c->dst.type) {
1190 write_register_operand(&c->dst);
1194 rc = ops->cmpxchg_emulated(
1195 linear(ctxt, c->dst.addr.mem),
1202 rc = ops->write_emulated(
1203 linear(ctxt, c->dst.addr.mem),
1208 if (rc != X86EMUL_CONTINUE)
1212 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1220 return X86EMUL_CONTINUE;
1223 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1224 struct x86_emulate_ops *ops)
1226 struct decode_cache *c = &ctxt->decode;
1228 c->dst.type = OP_MEM;
1229 c->dst.bytes = c->op_bytes;
1230 c->dst.val = c->src.val;
1231 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1232 c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1233 c->dst.addr.mem.seg = VCPU_SREG_SS;
1236 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1237 struct x86_emulate_ops *ops,
1238 void *dest, int len)
1240 struct decode_cache *c = &ctxt->decode;
1242 struct segmented_address addr;
1244 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1245 addr.seg = VCPU_SREG_SS;
1246 rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len);
1247 if (rc != X86EMUL_CONTINUE)
1250 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1254 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1255 struct x86_emulate_ops *ops,
1256 void *dest, int len)
1259 unsigned long val, change_mask;
1260 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1261 int cpl = ops->cpl(ctxt->vcpu);
1263 rc = emulate_pop(ctxt, ops, &val, len);
1264 if (rc != X86EMUL_CONTINUE)
1267 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1268 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1270 switch(ctxt->mode) {
1271 case X86EMUL_MODE_PROT64:
1272 case X86EMUL_MODE_PROT32:
1273 case X86EMUL_MODE_PROT16:
1275 change_mask |= EFLG_IOPL;
1277 change_mask |= EFLG_IF;
1279 case X86EMUL_MODE_VM86:
1281 return emulate_gp(ctxt, 0);
1282 change_mask |= EFLG_IF;
1284 default: /* real mode */
1285 change_mask |= (EFLG_IOPL | EFLG_IF);
1289 *(unsigned long *)dest =
1290 (ctxt->eflags & ~change_mask) | (val & change_mask);
1295 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1296 struct x86_emulate_ops *ops, int seg)
1298 struct decode_cache *c = &ctxt->decode;
1300 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1302 emulate_push(ctxt, ops);
1305 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1306 struct x86_emulate_ops *ops, int seg)
1308 struct decode_cache *c = &ctxt->decode;
1309 unsigned long selector;
1312 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1313 if (rc != X86EMUL_CONTINUE)
1316 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1320 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1321 struct x86_emulate_ops *ops)
1323 struct decode_cache *c = &ctxt->decode;
1324 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1325 int rc = X86EMUL_CONTINUE;
1326 int reg = VCPU_REGS_RAX;
1328 while (reg <= VCPU_REGS_RDI) {
1329 (reg == VCPU_REGS_RSP) ?
1330 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1332 emulate_push(ctxt, ops);
1334 rc = writeback(ctxt, ops);
1335 if (rc != X86EMUL_CONTINUE)
1341 /* Disable writeback. */
1342 c->dst.type = OP_NONE;
1347 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1348 struct x86_emulate_ops *ops)
1350 struct decode_cache *c = &ctxt->decode;
1351 int rc = X86EMUL_CONTINUE;
1352 int reg = VCPU_REGS_RDI;
1354 while (reg >= VCPU_REGS_RAX) {
1355 if (reg == VCPU_REGS_RSP) {
1356 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1361 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1362 if (rc != X86EMUL_CONTINUE)
1369 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1370 struct x86_emulate_ops *ops, int irq)
1372 struct decode_cache *c = &ctxt->decode;
1379 /* TODO: Add limit checks */
1380 c->src.val = ctxt->eflags;
1381 emulate_push(ctxt, ops);
1382 rc = writeback(ctxt, ops);
1383 if (rc != X86EMUL_CONTINUE)
1386 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1388 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1389 emulate_push(ctxt, ops);
1390 rc = writeback(ctxt, ops);
1391 if (rc != X86EMUL_CONTINUE)
1394 c->src.val = c->eip;
1395 emulate_push(ctxt, ops);
1396 rc = writeback(ctxt, ops);
1397 if (rc != X86EMUL_CONTINUE)
1400 c->dst.type = OP_NONE;
1402 ops->get_idt(&dt, ctxt->vcpu);
1404 eip_addr = dt.address + (irq << 2);
1405 cs_addr = dt.address + (irq << 2) + 2;
1407 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1408 if (rc != X86EMUL_CONTINUE)
1411 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1412 if (rc != X86EMUL_CONTINUE)
1415 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1416 if (rc != X86EMUL_CONTINUE)
1424 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1425 struct x86_emulate_ops *ops, int irq)
1427 switch(ctxt->mode) {
1428 case X86EMUL_MODE_REAL:
1429 return emulate_int_real(ctxt, ops, irq);
1430 case X86EMUL_MODE_VM86:
1431 case X86EMUL_MODE_PROT16:
1432 case X86EMUL_MODE_PROT32:
1433 case X86EMUL_MODE_PROT64:
1435 /* Protected mode interrupts unimplemented yet */
1436 return X86EMUL_UNHANDLEABLE;
1440 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1441 struct x86_emulate_ops *ops)
1443 struct decode_cache *c = &ctxt->decode;
1444 int rc = X86EMUL_CONTINUE;
1445 unsigned long temp_eip = 0;
1446 unsigned long temp_eflags = 0;
1447 unsigned long cs = 0;
1448 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1449 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1450 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1451 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1453 /* TODO: Add stack limit check */
1455 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1457 if (rc != X86EMUL_CONTINUE)
1460 if (temp_eip & ~0xffff)
1461 return emulate_gp(ctxt, 0);
1463 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1465 if (rc != X86EMUL_CONTINUE)
1468 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1470 if (rc != X86EMUL_CONTINUE)
1473 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1475 if (rc != X86EMUL_CONTINUE)
1481 if (c->op_bytes == 4)
1482 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1483 else if (c->op_bytes == 2) {
1484 ctxt->eflags &= ~0xffff;
1485 ctxt->eflags |= temp_eflags;
1488 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1489 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1494 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1495 struct x86_emulate_ops* ops)
1497 switch(ctxt->mode) {
1498 case X86EMUL_MODE_REAL:
1499 return emulate_iret_real(ctxt, ops);
1500 case X86EMUL_MODE_VM86:
1501 case X86EMUL_MODE_PROT16:
1502 case X86EMUL_MODE_PROT32:
1503 case X86EMUL_MODE_PROT64:
1505 /* iret from protected mode unimplemented yet */
1506 return X86EMUL_UNHANDLEABLE;
1510 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1511 struct x86_emulate_ops *ops)
1513 struct decode_cache *c = &ctxt->decode;
1515 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1518 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1520 struct decode_cache *c = &ctxt->decode;
1521 switch (c->modrm_reg) {
1523 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1526 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1529 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1532 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1534 case 4: /* sal/shl */
1535 case 6: /* sal/shl */
1536 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1539 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1542 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1547 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1548 struct x86_emulate_ops *ops)
1550 struct decode_cache *c = &ctxt->decode;
1551 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1552 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1555 switch (c->modrm_reg) {
1556 case 0 ... 1: /* test */
1557 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1560 c->dst.val = ~c->dst.val;
1563 emulate_1op("neg", c->dst, ctxt->eflags);
1566 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1569 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1572 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1576 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1580 return X86EMUL_UNHANDLEABLE;
1583 return emulate_de(ctxt);
1584 return X86EMUL_CONTINUE;
1587 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1588 struct x86_emulate_ops *ops)
1590 struct decode_cache *c = &ctxt->decode;
1592 switch (c->modrm_reg) {
1594 emulate_1op("inc", c->dst, ctxt->eflags);
1597 emulate_1op("dec", c->dst, ctxt->eflags);
1599 case 2: /* call near abs */ {
1602 c->eip = c->src.val;
1603 c->src.val = old_eip;
1604 emulate_push(ctxt, ops);
1607 case 4: /* jmp abs */
1608 c->eip = c->src.val;
1611 emulate_push(ctxt, ops);
1614 return X86EMUL_CONTINUE;
1617 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1618 struct x86_emulate_ops *ops)
1620 struct decode_cache *c = &ctxt->decode;
1621 u64 old = c->dst.orig_val64;
1623 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1624 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1625 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1626 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1627 ctxt->eflags &= ~EFLG_ZF;
1629 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1630 (u32) c->regs[VCPU_REGS_RBX];
1632 ctxt->eflags |= EFLG_ZF;
1634 return X86EMUL_CONTINUE;
1637 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1638 struct x86_emulate_ops *ops)
1640 struct decode_cache *c = &ctxt->decode;
1644 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1647 if (c->op_bytes == 4)
1648 c->eip = (u32)c->eip;
1649 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1652 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1656 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1657 struct x86_emulate_ops *ops, int seg)
1659 struct decode_cache *c = &ctxt->decode;
1663 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1665 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1666 if (rc != X86EMUL_CONTINUE)
1669 c->dst.val = c->src.val;
1674 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1675 struct x86_emulate_ops *ops, struct desc_struct *cs,
1676 struct desc_struct *ss)
1678 memset(cs, 0, sizeof(struct desc_struct));
1679 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1680 memset(ss, 0, sizeof(struct desc_struct));
1682 cs->l = 0; /* will be adjusted later */
1683 set_desc_base(cs, 0); /* flat segment */
1684 cs->g = 1; /* 4kb granularity */
1685 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1686 cs->type = 0x0b; /* Read, Execute, Accessed */
1688 cs->dpl = 0; /* will be adjusted later */
1692 set_desc_base(ss, 0); /* flat segment */
1693 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1694 ss->g = 1; /* 4kb granularity */
1696 ss->type = 0x03; /* Read/Write, Accessed */
1697 ss->d = 1; /* 32bit stack segment */
1703 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1705 struct decode_cache *c = &ctxt->decode;
1706 struct desc_struct cs, ss;
1710 /* syscall is not available in real mode */
1711 if (ctxt->mode == X86EMUL_MODE_REAL ||
1712 ctxt->mode == X86EMUL_MODE_VM86)
1713 return emulate_ud(ctxt);
1715 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1717 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1719 cs_sel = (u16)(msr_data & 0xfffc);
1720 ss_sel = (u16)(msr_data + 8);
1722 if (is_long_mode(ctxt->vcpu)) {
1726 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1727 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1728 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1729 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1731 c->regs[VCPU_REGS_RCX] = c->eip;
1732 if (is_long_mode(ctxt->vcpu)) {
1733 #ifdef CONFIG_X86_64
1734 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1736 ops->get_msr(ctxt->vcpu,
1737 ctxt->mode == X86EMUL_MODE_PROT64 ?
1738 MSR_LSTAR : MSR_CSTAR, &msr_data);
1741 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1742 ctxt->eflags &= ~(msr_data | EFLG_RF);
1746 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1747 c->eip = (u32)msr_data;
1749 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1752 return X86EMUL_CONTINUE;
1756 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1758 struct decode_cache *c = &ctxt->decode;
1759 struct desc_struct cs, ss;
1763 /* inject #GP if in real mode */
1764 if (ctxt->mode == X86EMUL_MODE_REAL)
1765 return emulate_gp(ctxt, 0);
1767 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1768 * Therefore, we inject an #UD.
1770 if (ctxt->mode == X86EMUL_MODE_PROT64)
1771 return emulate_ud(ctxt);
1773 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1775 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1776 switch (ctxt->mode) {
1777 case X86EMUL_MODE_PROT32:
1778 if ((msr_data & 0xfffc) == 0x0)
1779 return emulate_gp(ctxt, 0);
1781 case X86EMUL_MODE_PROT64:
1782 if (msr_data == 0x0)
1783 return emulate_gp(ctxt, 0);
1787 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1788 cs_sel = (u16)msr_data;
1789 cs_sel &= ~SELECTOR_RPL_MASK;
1790 ss_sel = cs_sel + 8;
1791 ss_sel &= ~SELECTOR_RPL_MASK;
1792 if (ctxt->mode == X86EMUL_MODE_PROT64
1793 || is_long_mode(ctxt->vcpu)) {
1798 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1799 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1800 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1801 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1803 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1806 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1807 c->regs[VCPU_REGS_RSP] = msr_data;
1809 return X86EMUL_CONTINUE;
1813 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1815 struct decode_cache *c = &ctxt->decode;
1816 struct desc_struct cs, ss;
1821 /* inject #GP if in real mode or Virtual 8086 mode */
1822 if (ctxt->mode == X86EMUL_MODE_REAL ||
1823 ctxt->mode == X86EMUL_MODE_VM86)
1824 return emulate_gp(ctxt, 0);
1826 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1828 if ((c->rex_prefix & 0x8) != 0x0)
1829 usermode = X86EMUL_MODE_PROT64;
1831 usermode = X86EMUL_MODE_PROT32;
1835 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1837 case X86EMUL_MODE_PROT32:
1838 cs_sel = (u16)(msr_data + 16);
1839 if ((msr_data & 0xfffc) == 0x0)
1840 return emulate_gp(ctxt, 0);
1841 ss_sel = (u16)(msr_data + 24);
1843 case X86EMUL_MODE_PROT64:
1844 cs_sel = (u16)(msr_data + 32);
1845 if (msr_data == 0x0)
1846 return emulate_gp(ctxt, 0);
1847 ss_sel = cs_sel + 8;
1852 cs_sel |= SELECTOR_RPL_MASK;
1853 ss_sel |= SELECTOR_RPL_MASK;
1855 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1856 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1857 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1858 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1860 c->eip = c->regs[VCPU_REGS_RDX];
1861 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1863 return X86EMUL_CONTINUE;
1866 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1867 struct x86_emulate_ops *ops)
1870 if (ctxt->mode == X86EMUL_MODE_REAL)
1872 if (ctxt->mode == X86EMUL_MODE_VM86)
1874 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1875 return ops->cpl(ctxt->vcpu) > iopl;
1878 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1879 struct x86_emulate_ops *ops,
1882 struct desc_struct tr_seg;
1885 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
1886 unsigned mask = (1 << len) - 1;
1889 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
1892 if (desc_limit_scaled(&tr_seg) < 103)
1894 base = get_desc_base(&tr_seg);
1895 #ifdef CONFIG_X86_64
1896 base |= ((u64)base3) << 32;
1898 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
1899 if (r != X86EMUL_CONTINUE)
1901 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
1903 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
1905 if (r != X86EMUL_CONTINUE)
1907 if ((perm >> bit_idx) & mask)
1912 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1913 struct x86_emulate_ops *ops,
1919 if (emulator_bad_iopl(ctxt, ops))
1920 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1923 ctxt->perm_ok = true;
1928 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
1929 struct x86_emulate_ops *ops,
1930 struct tss_segment_16 *tss)
1932 struct decode_cache *c = &ctxt->decode;
1935 tss->flag = ctxt->eflags;
1936 tss->ax = c->regs[VCPU_REGS_RAX];
1937 tss->cx = c->regs[VCPU_REGS_RCX];
1938 tss->dx = c->regs[VCPU_REGS_RDX];
1939 tss->bx = c->regs[VCPU_REGS_RBX];
1940 tss->sp = c->regs[VCPU_REGS_RSP];
1941 tss->bp = c->regs[VCPU_REGS_RBP];
1942 tss->si = c->regs[VCPU_REGS_RSI];
1943 tss->di = c->regs[VCPU_REGS_RDI];
1945 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
1946 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1947 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
1948 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
1949 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
1952 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
1953 struct x86_emulate_ops *ops,
1954 struct tss_segment_16 *tss)
1956 struct decode_cache *c = &ctxt->decode;
1960 ctxt->eflags = tss->flag | 2;
1961 c->regs[VCPU_REGS_RAX] = tss->ax;
1962 c->regs[VCPU_REGS_RCX] = tss->cx;
1963 c->regs[VCPU_REGS_RDX] = tss->dx;
1964 c->regs[VCPU_REGS_RBX] = tss->bx;
1965 c->regs[VCPU_REGS_RSP] = tss->sp;
1966 c->regs[VCPU_REGS_RBP] = tss->bp;
1967 c->regs[VCPU_REGS_RSI] = tss->si;
1968 c->regs[VCPU_REGS_RDI] = tss->di;
1971 * SDM says that segment selectors are loaded before segment
1974 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
1975 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
1976 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
1977 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
1978 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
1981 * Now load segment descriptors. If fault happenes at this stage
1982 * it is handled in a context of new task
1984 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
1985 if (ret != X86EMUL_CONTINUE)
1987 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
1988 if (ret != X86EMUL_CONTINUE)
1990 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
1991 if (ret != X86EMUL_CONTINUE)
1993 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
1994 if (ret != X86EMUL_CONTINUE)
1996 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
1997 if (ret != X86EMUL_CONTINUE)
2000 return X86EMUL_CONTINUE;
2003 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2004 struct x86_emulate_ops *ops,
2005 u16 tss_selector, u16 old_tss_sel,
2006 ulong old_tss_base, struct desc_struct *new_desc)
2008 struct tss_segment_16 tss_seg;
2010 u32 new_tss_base = get_desc_base(new_desc);
2012 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2014 if (ret != X86EMUL_CONTINUE)
2015 /* FIXME: need to provide precise fault address */
2018 save_state_to_tss16(ctxt, ops, &tss_seg);
2020 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2022 if (ret != X86EMUL_CONTINUE)
2023 /* FIXME: need to provide precise fault address */
2026 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2028 if (ret != X86EMUL_CONTINUE)
2029 /* FIXME: need to provide precise fault address */
2032 if (old_tss_sel != 0xffff) {
2033 tss_seg.prev_task_link = old_tss_sel;
2035 ret = ops->write_std(new_tss_base,
2036 &tss_seg.prev_task_link,
2037 sizeof tss_seg.prev_task_link,
2038 ctxt->vcpu, &ctxt->exception);
2039 if (ret != X86EMUL_CONTINUE)
2040 /* FIXME: need to provide precise fault address */
2044 return load_state_from_tss16(ctxt, ops, &tss_seg);
2047 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2048 struct x86_emulate_ops *ops,
2049 struct tss_segment_32 *tss)
2051 struct decode_cache *c = &ctxt->decode;
2053 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2055 tss->eflags = ctxt->eflags;
2056 tss->eax = c->regs[VCPU_REGS_RAX];
2057 tss->ecx = c->regs[VCPU_REGS_RCX];
2058 tss->edx = c->regs[VCPU_REGS_RDX];
2059 tss->ebx = c->regs[VCPU_REGS_RBX];
2060 tss->esp = c->regs[VCPU_REGS_RSP];
2061 tss->ebp = c->regs[VCPU_REGS_RBP];
2062 tss->esi = c->regs[VCPU_REGS_RSI];
2063 tss->edi = c->regs[VCPU_REGS_RDI];
2065 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2066 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2067 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2068 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2069 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2070 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2071 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2074 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2075 struct x86_emulate_ops *ops,
2076 struct tss_segment_32 *tss)
2078 struct decode_cache *c = &ctxt->decode;
2081 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2082 return emulate_gp(ctxt, 0);
2084 ctxt->eflags = tss->eflags | 2;
2085 c->regs[VCPU_REGS_RAX] = tss->eax;
2086 c->regs[VCPU_REGS_RCX] = tss->ecx;
2087 c->regs[VCPU_REGS_RDX] = tss->edx;
2088 c->regs[VCPU_REGS_RBX] = tss->ebx;
2089 c->regs[VCPU_REGS_RSP] = tss->esp;
2090 c->regs[VCPU_REGS_RBP] = tss->ebp;
2091 c->regs[VCPU_REGS_RSI] = tss->esi;
2092 c->regs[VCPU_REGS_RDI] = tss->edi;
2095 * SDM says that segment selectors are loaded before segment
2098 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2099 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2100 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2101 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2102 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2103 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2104 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2107 * Now load segment descriptors. If fault happenes at this stage
2108 * it is handled in a context of new task
2110 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2111 if (ret != X86EMUL_CONTINUE)
2113 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2114 if (ret != X86EMUL_CONTINUE)
2116 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2117 if (ret != X86EMUL_CONTINUE)
2119 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2120 if (ret != X86EMUL_CONTINUE)
2122 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2123 if (ret != X86EMUL_CONTINUE)
2125 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2126 if (ret != X86EMUL_CONTINUE)
2128 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2129 if (ret != X86EMUL_CONTINUE)
2132 return X86EMUL_CONTINUE;
2135 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2136 struct x86_emulate_ops *ops,
2137 u16 tss_selector, u16 old_tss_sel,
2138 ulong old_tss_base, struct desc_struct *new_desc)
2140 struct tss_segment_32 tss_seg;
2142 u32 new_tss_base = get_desc_base(new_desc);
2144 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2146 if (ret != X86EMUL_CONTINUE)
2147 /* FIXME: need to provide precise fault address */
2150 save_state_to_tss32(ctxt, ops, &tss_seg);
2152 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2154 if (ret != X86EMUL_CONTINUE)
2155 /* FIXME: need to provide precise fault address */
2158 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2160 if (ret != X86EMUL_CONTINUE)
2161 /* FIXME: need to provide precise fault address */
2164 if (old_tss_sel != 0xffff) {
2165 tss_seg.prev_task_link = old_tss_sel;
2167 ret = ops->write_std(new_tss_base,
2168 &tss_seg.prev_task_link,
2169 sizeof tss_seg.prev_task_link,
2170 ctxt->vcpu, &ctxt->exception);
2171 if (ret != X86EMUL_CONTINUE)
2172 /* FIXME: need to provide precise fault address */
2176 return load_state_from_tss32(ctxt, ops, &tss_seg);
2179 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2180 struct x86_emulate_ops *ops,
2181 u16 tss_selector, int reason,
2182 bool has_error_code, u32 error_code)
2184 struct desc_struct curr_tss_desc, next_tss_desc;
2186 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2187 ulong old_tss_base =
2188 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2191 /* FIXME: old_tss_base == ~0 ? */
2193 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2194 if (ret != X86EMUL_CONTINUE)
2196 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2197 if (ret != X86EMUL_CONTINUE)
2200 /* FIXME: check that next_tss_desc is tss */
2202 if (reason != TASK_SWITCH_IRET) {
2203 if ((tss_selector & 3) > next_tss_desc.dpl ||
2204 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2205 return emulate_gp(ctxt, 0);
2208 desc_limit = desc_limit_scaled(&next_tss_desc);
2209 if (!next_tss_desc.p ||
2210 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2211 desc_limit < 0x2b)) {
2212 emulate_ts(ctxt, tss_selector & 0xfffc);
2213 return X86EMUL_PROPAGATE_FAULT;
2216 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2217 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2218 write_segment_descriptor(ctxt, ops, old_tss_sel,
2222 if (reason == TASK_SWITCH_IRET)
2223 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2225 /* set back link to prev task only if NT bit is set in eflags
2226 note that old_tss_sel is not used afetr this point */
2227 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2228 old_tss_sel = 0xffff;
2230 if (next_tss_desc.type & 8)
2231 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2232 old_tss_base, &next_tss_desc);
2234 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2235 old_tss_base, &next_tss_desc);
2236 if (ret != X86EMUL_CONTINUE)
2239 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2240 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2242 if (reason != TASK_SWITCH_IRET) {
2243 next_tss_desc.type |= (1 << 1); /* set busy flag */
2244 write_segment_descriptor(ctxt, ops, tss_selector,
2248 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2249 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2250 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2252 if (has_error_code) {
2253 struct decode_cache *c = &ctxt->decode;
2255 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2257 c->src.val = (unsigned long) error_code;
2258 emulate_push(ctxt, ops);
2264 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2265 u16 tss_selector, int reason,
2266 bool has_error_code, u32 error_code)
2268 struct x86_emulate_ops *ops = ctxt->ops;
2269 struct decode_cache *c = &ctxt->decode;
2273 c->dst.type = OP_NONE;
2275 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2276 has_error_code, error_code);
2278 if (rc == X86EMUL_CONTINUE) {
2279 rc = writeback(ctxt, ops);
2280 if (rc == X86EMUL_CONTINUE)
2284 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2287 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2288 int reg, struct operand *op)
2290 struct decode_cache *c = &ctxt->decode;
2291 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2293 register_address_increment(c, &c->regs[reg], df * op->bytes);
2294 op->addr.mem.ea = register_address(c, c->regs[reg]);
2295 op->addr.mem.seg = seg;
2298 static int em_push(struct x86_emulate_ctxt *ctxt)
2300 emulate_push(ctxt, ctxt->ops);
2301 return X86EMUL_CONTINUE;
2304 static int em_das(struct x86_emulate_ctxt *ctxt)
2306 struct decode_cache *c = &ctxt->decode;
2308 bool af, cf, old_cf;
2310 cf = ctxt->eflags & X86_EFLAGS_CF;
2316 af = ctxt->eflags & X86_EFLAGS_AF;
2317 if ((al & 0x0f) > 9 || af) {
2319 cf = old_cf | (al >= 250);
2324 if (old_al > 0x99 || old_cf) {
2330 /* Set PF, ZF, SF */
2331 c->src.type = OP_IMM;
2334 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2335 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2337 ctxt->eflags |= X86_EFLAGS_CF;
2339 ctxt->eflags |= X86_EFLAGS_AF;
2340 return X86EMUL_CONTINUE;
2343 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2345 struct decode_cache *c = &ctxt->decode;
2350 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2353 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2354 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2355 return X86EMUL_CONTINUE;
2358 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2360 c->src.val = old_cs;
2361 emulate_push(ctxt, ctxt->ops);
2362 rc = writeback(ctxt, ctxt->ops);
2363 if (rc != X86EMUL_CONTINUE)
2366 c->src.val = old_eip;
2367 emulate_push(ctxt, ctxt->ops);
2368 rc = writeback(ctxt, ctxt->ops);
2369 if (rc != X86EMUL_CONTINUE)
2372 c->dst.type = OP_NONE;
2374 return X86EMUL_CONTINUE;
2377 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2379 struct decode_cache *c = &ctxt->decode;
2382 c->dst.type = OP_REG;
2383 c->dst.addr.reg = &c->eip;
2384 c->dst.bytes = c->op_bytes;
2385 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2386 if (rc != X86EMUL_CONTINUE)
2388 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2389 return X86EMUL_CONTINUE;
2392 static int em_imul(struct x86_emulate_ctxt *ctxt)
2394 struct decode_cache *c = &ctxt->decode;
2396 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2397 return X86EMUL_CONTINUE;
2400 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2402 struct decode_cache *c = &ctxt->decode;
2404 c->dst.val = c->src2.val;
2405 return em_imul(ctxt);
2408 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2410 struct decode_cache *c = &ctxt->decode;
2412 c->dst.type = OP_REG;
2413 c->dst.bytes = c->src.bytes;
2414 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2415 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2417 return X86EMUL_CONTINUE;
2420 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2422 unsigned cpl = ctxt->ops->cpl(ctxt->vcpu);
2423 struct decode_cache *c = &ctxt->decode;
2426 if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD))
2427 return emulate_gp(ctxt, 0);
2428 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2429 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2430 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2431 return X86EMUL_CONTINUE;
2434 static int em_mov(struct x86_emulate_ctxt *ctxt)
2436 struct decode_cache *c = &ctxt->decode;
2437 c->dst.val = c->src.val;
2438 return X86EMUL_CONTINUE;
2441 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2443 struct decode_cache *c = &ctxt->decode;
2444 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2445 return X86EMUL_CONTINUE;
2448 static bool valid_cr(int nr)
2460 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2462 struct decode_cache *c = &ctxt->decode;
2464 if (!valid_cr(c->modrm_reg))
2465 return emulate_ud(ctxt);
2467 return X86EMUL_CONTINUE;
2470 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2472 struct decode_cache *c = &ctxt->decode;
2473 u64 new_val = c->src.val64;
2474 int cr = c->modrm_reg;
2476 static u64 cr_reserved_bits[] = {
2477 0xffffffff00000000ULL,
2478 0, 0, 0, /* CR3 checked later */
2485 return emulate_ud(ctxt);
2487 if (new_val & cr_reserved_bits[cr])
2488 return emulate_gp(ctxt, 0);
2493 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2494 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2495 return emulate_gp(ctxt, 0);
2497 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2498 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2500 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2501 !(cr4 & X86_CR4_PAE))
2502 return emulate_gp(ctxt, 0);
2509 if (is_long_mode(ctxt->vcpu))
2510 rsvd = CR3_L_MODE_RESERVED_BITS;
2511 else if (is_pae(ctxt->vcpu))
2512 rsvd = CR3_PAE_RESERVED_BITS;
2513 else if (is_paging(ctxt->vcpu))
2514 rsvd = CR3_NONPAE_RESERVED_BITS;
2517 return emulate_gp(ctxt, 0);
2524 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2525 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2527 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2528 return emulate_gp(ctxt, 0);
2534 return X86EMUL_CONTINUE;
2537 #define D(_y) { .flags = (_y) }
2538 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2539 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2540 .check_perm = (_p) }
2542 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2543 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2544 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2545 #define II(_f, _e, _i) \
2546 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2547 #define IIP(_f, _e, _i, _p) \
2548 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2549 .check_perm = (_p) }
2550 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2552 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2553 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2555 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2556 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2557 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2560 static struct opcode group1[] = {
2564 static struct opcode group1A[] = {
2565 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2568 static struct opcode group3[] = {
2569 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2570 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2571 X4(D(SrcMem | ModRM)),
2574 static struct opcode group4[] = {
2575 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2579 static struct opcode group5[] = {
2580 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2581 D(SrcMem | ModRM | Stack),
2582 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2583 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2584 D(SrcMem | ModRM | Stack), N,
2587 static struct group_dual group7 = { {
2588 N, N, DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2589 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2590 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2591 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2593 D(SrcNone | ModRM | Priv | VendorSpecific), N,
2594 N, D(SrcNone | ModRM | Priv | VendorSpecific),
2595 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2596 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), N,
2599 static struct opcode group8[] = {
2601 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2602 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2605 static struct group_dual group9 = { {
2606 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2608 N, N, N, N, N, N, N, N,
2611 static struct opcode group11[] = {
2612 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2615 static struct gprefix pfx_0f_6f_0f_7f = {
2616 N, N, N, I(Sse, em_movdqu),
2619 static struct opcode opcode_table[256] = {
2622 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2625 D(ImplicitOps | Stack | No64), N,
2628 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2631 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2635 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2643 X8(I(SrcReg | Stack, em_push)),
2645 X8(D(DstReg | Stack)),
2647 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2648 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2651 I(SrcImm | Mov | Stack, em_push),
2652 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2653 I(SrcImmByte | Mov | Stack, em_push),
2654 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2655 D2bv(DstDI | Mov | String), /* insb, insw/insd */
2656 D2bv(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
2660 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2661 G(DstMem | SrcImm | ModRM | Group, group1),
2662 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2663 G(DstMem | SrcImmByte | ModRM | Group, group1),
2664 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2666 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2667 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2668 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2669 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2671 X8(D(SrcAcc | DstReg)),
2673 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2674 I(SrcImmFAddr | No64, em_call_far), N,
2675 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2677 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2678 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2679 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2680 D2bv(SrcSI | DstDI | String),
2682 D2bv(DstAcc | SrcImm),
2683 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2684 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2685 D2bv(SrcAcc | DstDI | String),
2687 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2689 X8(I(DstReg | SrcImm | Mov, em_mov)),
2691 D2bv(DstMem | SrcImmByte | ModRM),
2692 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2693 D(ImplicitOps | Stack),
2694 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2695 G(ByteOp, group11), G(0, group11),
2697 N, N, N, D(ImplicitOps | Stack),
2698 D(ImplicitOps), DI(SrcImmByte, intn),
2699 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2701 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2704 N, N, N, N, N, N, N, N,
2707 D2bv(SrcImmUByte | DstAcc), D2bv(SrcAcc | DstImmUByte),
2709 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2710 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2711 D2bv(SrcNone | DstAcc), D2bv(SrcAcc | ImplicitOps),
2714 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2715 G(ByteOp, group3), G(0, group3),
2717 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2718 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2721 static struct opcode twobyte_table[256] = {
2723 N, GD(0, &group7), N, N,
2724 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2725 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2726 N, D(ImplicitOps | ModRM), N, N,
2728 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2730 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2731 D(ModRM | DstMem | Priv | Op3264),
2732 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2733 D(ModRM | SrcMem | Priv | Op3264),
2735 N, N, N, N, N, N, N, N,
2737 D(ImplicitOps | Priv), II(ImplicitOps, em_rdtsc, rdtsc),
2738 D(ImplicitOps | Priv), N,
2739 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2741 N, N, N, N, N, N, N, N,
2743 X16(D(DstReg | SrcMem | ModRM | Mov)),
2745 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2750 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
2755 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
2759 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
2761 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2762 N, D(DstMem | SrcReg | ModRM | BitOp),
2763 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2764 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
2766 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2767 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
2768 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2769 D(DstMem | SrcReg | Src2CL | ModRM),
2770 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
2772 D2bv(DstMem | SrcReg | ModRM | Lock),
2773 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2774 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
2775 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2778 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2779 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
2780 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2782 D2bv(DstMem | SrcReg | ModRM | Lock),
2783 N, D(DstMem | SrcReg | ModRM | Mov),
2784 N, N, N, GD(0, &group9),
2785 N, N, N, N, N, N, N, N,
2787 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2789 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2791 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
2805 static unsigned imm_size(struct decode_cache *c)
2809 size = (c->d & ByteOp) ? 1 : c->op_bytes;
2815 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
2816 unsigned size, bool sign_extension)
2818 struct decode_cache *c = &ctxt->decode;
2819 struct x86_emulate_ops *ops = ctxt->ops;
2820 int rc = X86EMUL_CONTINUE;
2824 op->addr.mem.ea = c->eip;
2825 /* NB. Immediates are sign-extended as necessary. */
2826 switch (op->bytes) {
2828 op->val = insn_fetch(s8, 1, c->eip);
2831 op->val = insn_fetch(s16, 2, c->eip);
2834 op->val = insn_fetch(s32, 4, c->eip);
2837 if (!sign_extension) {
2838 switch (op->bytes) {
2846 op->val &= 0xffffffff;
2855 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
2857 struct x86_emulate_ops *ops = ctxt->ops;
2858 struct decode_cache *c = &ctxt->decode;
2859 int rc = X86EMUL_CONTINUE;
2860 int mode = ctxt->mode;
2861 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
2862 bool op_prefix = false;
2863 struct opcode opcode, *g_mod012, *g_mod3;
2864 struct operand memop = { .type = OP_NONE };
2867 c->fetch.start = c->eip;
2868 c->fetch.end = c->fetch.start + insn_len;
2870 memcpy(c->fetch.data, insn, insn_len);
2871 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
2874 case X86EMUL_MODE_REAL:
2875 case X86EMUL_MODE_VM86:
2876 case X86EMUL_MODE_PROT16:
2877 def_op_bytes = def_ad_bytes = 2;
2879 case X86EMUL_MODE_PROT32:
2880 def_op_bytes = def_ad_bytes = 4;
2882 #ifdef CONFIG_X86_64
2883 case X86EMUL_MODE_PROT64:
2892 c->op_bytes = def_op_bytes;
2893 c->ad_bytes = def_ad_bytes;
2895 /* Legacy prefixes. */
2897 switch (c->b = insn_fetch(u8, 1, c->eip)) {
2898 case 0x66: /* operand-size override */
2900 /* switch between 2/4 bytes */
2901 c->op_bytes = def_op_bytes ^ 6;
2903 case 0x67: /* address-size override */
2904 if (mode == X86EMUL_MODE_PROT64)
2905 /* switch between 4/8 bytes */
2906 c->ad_bytes = def_ad_bytes ^ 12;
2908 /* switch between 2/4 bytes */
2909 c->ad_bytes = def_ad_bytes ^ 6;
2911 case 0x26: /* ES override */
2912 case 0x2e: /* CS override */
2913 case 0x36: /* SS override */
2914 case 0x3e: /* DS override */
2915 set_seg_override(c, (c->b >> 3) & 3);
2917 case 0x64: /* FS override */
2918 case 0x65: /* GS override */
2919 set_seg_override(c, c->b & 7);
2921 case 0x40 ... 0x4f: /* REX */
2922 if (mode != X86EMUL_MODE_PROT64)
2924 c->rex_prefix = c->b;
2926 case 0xf0: /* LOCK */
2929 case 0xf2: /* REPNE/REPNZ */
2930 case 0xf3: /* REP/REPE/REPZ */
2931 c->rep_prefix = c->b;
2937 /* Any legacy prefix after a REX prefix nullifies its effect. */
2945 if (c->rex_prefix & 8)
2946 c->op_bytes = 8; /* REX.W */
2948 /* Opcode byte(s). */
2949 opcode = opcode_table[c->b];
2950 /* Two-byte opcode? */
2953 c->b = insn_fetch(u8, 1, c->eip);
2954 opcode = twobyte_table[c->b];
2956 c->d = opcode.flags;
2959 dual = c->d & GroupDual;
2960 c->modrm = insn_fetch(u8, 1, c->eip);
2963 if (c->d & GroupDual) {
2964 g_mod012 = opcode.u.gdual->mod012;
2965 g_mod3 = opcode.u.gdual->mod3;
2967 g_mod012 = g_mod3 = opcode.u.group;
2969 c->d &= ~(Group | GroupDual);
2971 goffset = (c->modrm >> 3) & 7;
2973 if ((c->modrm >> 6) == 3)
2974 opcode = g_mod3[goffset];
2976 opcode = g_mod012[goffset];
2977 c->d |= opcode.flags;
2980 if (c->d & Prefix) {
2981 if (c->rep_prefix && op_prefix)
2982 return X86EMUL_UNHANDLEABLE;
2983 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
2984 switch (simd_prefix) {
2985 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
2986 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
2987 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
2988 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
2990 c->d |= opcode.flags;
2993 c->execute = opcode.u.execute;
2994 c->check_perm = opcode.check_perm;
2995 c->intercept = opcode.intercept;
2998 if (c->d == 0 || (c->d & Undefined))
3001 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3004 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3007 if (c->d & Op3264) {
3008 if (mode == X86EMUL_MODE_PROT64)
3017 /* ModRM and SIB bytes. */
3019 rc = decode_modrm(ctxt, ops, &memop);
3020 if (!c->has_seg_override)
3021 set_seg_override(c, c->modrm_seg);
3022 } else if (c->d & MemAbs)
3023 rc = decode_abs(ctxt, ops, &memop);
3024 if (rc != X86EMUL_CONTINUE)
3027 if (!c->has_seg_override)
3028 set_seg_override(c, VCPU_SREG_DS);
3030 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3032 if (memop.type == OP_MEM && c->ad_bytes != 8)
3033 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3035 if (memop.type == OP_MEM && c->rip_relative)
3036 memop.addr.mem.ea += c->eip;
3039 * Decode and fetch the source operand: register, memory
3042 switch (c->d & SrcMask) {
3046 decode_register_operand(ctxt, &c->src, c, 0);
3055 memop.bytes = (c->d & ByteOp) ? 1 :
3061 rc = decode_imm(ctxt, &c->src, 2, false);
3064 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3067 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3070 rc = decode_imm(ctxt, &c->src, 1, true);
3073 rc = decode_imm(ctxt, &c->src, 1, false);
3076 c->src.type = OP_REG;
3077 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3078 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3079 fetch_register_operand(&c->src);
3086 c->src.type = OP_MEM;
3087 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3088 c->src.addr.mem.ea =
3089 register_address(c, c->regs[VCPU_REGS_RSI]);
3090 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3094 c->src.type = OP_IMM;
3095 c->src.addr.mem.ea = c->eip;
3096 c->src.bytes = c->op_bytes + 2;
3097 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3100 memop.bytes = c->op_bytes + 2;
3105 if (rc != X86EMUL_CONTINUE)
3109 * Decode and fetch the second source operand: register, memory
3112 switch (c->d & Src2Mask) {
3117 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3120 rc = decode_imm(ctxt, &c->src2, 1, true);
3127 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3131 if (rc != X86EMUL_CONTINUE)
3134 /* Decode and fetch the destination operand: register or memory. */
3135 switch (c->d & DstMask) {
3137 decode_register_operand(ctxt, &c->dst, c,
3138 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3141 c->dst.type = OP_IMM;
3142 c->dst.addr.mem.ea = c->eip;
3144 c->dst.val = insn_fetch(u8, 1, c->eip);
3149 if ((c->d & DstMask) == DstMem64)
3152 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3154 fetch_bit_operand(c);
3155 c->dst.orig_val = c->dst.val;
3158 c->dst.type = OP_REG;
3159 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3160 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3161 fetch_register_operand(&c->dst);
3162 c->dst.orig_val = c->dst.val;
3165 c->dst.type = OP_MEM;
3166 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3167 c->dst.addr.mem.ea =
3168 register_address(c, c->regs[VCPU_REGS_RDI]);
3169 c->dst.addr.mem.seg = VCPU_SREG_ES;
3173 /* Special instructions do their own operand decoding. */
3175 c->dst.type = OP_NONE; /* Disable writeback. */
3180 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3183 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3185 struct decode_cache *c = &ctxt->decode;
3187 /* The second termination condition only applies for REPE
3188 * and REPNE. Test if the repeat string operation prefix is
3189 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3190 * corresponding termination condition according to:
3191 * - if REPE/REPZ and ZF = 0 then done
3192 * - if REPNE/REPNZ and ZF = 1 then done
3194 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3195 (c->b == 0xae) || (c->b == 0xaf))
3196 && (((c->rep_prefix == REPE_PREFIX) &&
3197 ((ctxt->eflags & EFLG_ZF) == 0))
3198 || ((c->rep_prefix == REPNE_PREFIX) &&
3199 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3206 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3208 struct x86_emulate_ops *ops = ctxt->ops;
3210 struct decode_cache *c = &ctxt->decode;
3211 int rc = X86EMUL_CONTINUE;
3212 int saved_dst_type = c->dst.type;
3213 int irq; /* Used for int 3, int, and into */
3215 ctxt->decode.mem_read.pos = 0;
3217 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3218 rc = emulate_ud(ctxt);
3222 /* LOCK prefix is allowed only with some instructions */
3223 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3224 rc = emulate_ud(ctxt);
3228 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3229 rc = emulate_ud(ctxt);
3234 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3235 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3236 rc = emulate_ud(ctxt);
3240 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3241 rc = emulate_nm(ctxt);
3245 if (unlikely(ctxt->guest_mode) && c->intercept) {
3246 rc = emulator_check_intercept(ctxt, c->intercept,
3247 X86_ICPT_PRE_EXCEPT);
3248 if (rc != X86EMUL_CONTINUE)
3252 /* Privileged instruction can be executed only in CPL=0 */
3253 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3254 rc = emulate_gp(ctxt, 0);
3258 /* Instruction can only be executed in protected mode */
3259 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3260 rc = emulate_ud(ctxt);
3264 /* Do instruction specific permission checks */
3265 if (c->check_perm) {
3266 rc = c->check_perm(ctxt);
3267 if (rc != X86EMUL_CONTINUE)
3271 if (unlikely(ctxt->guest_mode) && c->intercept) {
3272 rc = emulator_check_intercept(ctxt, c->intercept,
3273 X86_ICPT_POST_EXCEPT);
3274 if (rc != X86EMUL_CONTINUE)
3278 if (c->rep_prefix && (c->d & String)) {
3279 /* All REP prefixes have the same first termination condition */
3280 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3286 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3287 rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem),
3288 c->src.valptr, c->src.bytes);
3289 if (rc != X86EMUL_CONTINUE)
3291 c->src.orig_val64 = c->src.val64;
3294 if (c->src2.type == OP_MEM) {
3295 rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem),
3296 &c->src2.val, c->src2.bytes);
3297 if (rc != X86EMUL_CONTINUE)
3301 if ((c->d & DstMask) == ImplicitOps)
3305 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3306 /* optimisation - avoid slow emulated read if Mov */
3307 rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem),
3308 &c->dst.val, c->dst.bytes);
3309 if (rc != X86EMUL_CONTINUE)
3312 c->dst.orig_val = c->dst.val;
3316 if (unlikely(ctxt->guest_mode) && c->intercept) {
3317 rc = emulator_check_intercept(ctxt, c->intercept,
3318 X86_ICPT_POST_MEMACCESS);
3319 if (rc != X86EMUL_CONTINUE)
3324 rc = c->execute(ctxt);
3325 if (rc != X86EMUL_CONTINUE)
3336 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3338 case 0x06: /* push es */
3339 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3341 case 0x07: /* pop es */
3342 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3346 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3348 case 0x0e: /* push cs */
3349 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3353 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3355 case 0x16: /* push ss */
3356 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3358 case 0x17: /* pop ss */
3359 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3363 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3365 case 0x1e: /* push ds */
3366 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3368 case 0x1f: /* pop ds */
3369 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3373 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3377 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3381 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3385 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3387 case 0x40 ... 0x47: /* inc r16/r32 */
3388 emulate_1op("inc", c->dst, ctxt->eflags);
3390 case 0x48 ... 0x4f: /* dec r16/r32 */
3391 emulate_1op("dec", c->dst, ctxt->eflags);
3393 case 0x58 ... 0x5f: /* pop reg */
3395 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3397 case 0x60: /* pusha */
3398 rc = emulate_pusha(ctxt, ops);
3400 case 0x61: /* popa */
3401 rc = emulate_popa(ctxt, ops);
3403 case 0x63: /* movsxd */
3404 if (ctxt->mode != X86EMUL_MODE_PROT64)
3405 goto cannot_emulate;
3406 c->dst.val = (s32) c->src.val;
3408 case 0x6c: /* insb */
3409 case 0x6d: /* insw/insd */
3410 c->src.val = c->regs[VCPU_REGS_RDX];
3412 case 0x6e: /* outsb */
3413 case 0x6f: /* outsw/outsd */
3414 c->dst.val = c->regs[VCPU_REGS_RDX];
3417 case 0x70 ... 0x7f: /* jcc (short) */
3418 if (test_cc(c->b, ctxt->eflags))
3419 jmp_rel(c, c->src.val);
3421 case 0x80 ... 0x83: /* Grp1 */
3422 switch (c->modrm_reg) {
3443 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3445 case 0x86 ... 0x87: /* xchg */
3447 /* Write back the register source. */
3448 c->src.val = c->dst.val;
3449 write_register_operand(&c->src);
3451 * Write back the memory destination with implicit LOCK
3454 c->dst.val = c->src.orig_val;
3457 case 0x8c: /* mov r/m, sreg */
3458 if (c->modrm_reg > VCPU_SREG_GS) {
3459 rc = emulate_ud(ctxt);
3462 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3464 case 0x8d: /* lea r16/r32, m */
3465 c->dst.val = c->src.addr.mem.ea;
3467 case 0x8e: { /* mov seg, r/m16 */
3472 if (c->modrm_reg == VCPU_SREG_CS ||
3473 c->modrm_reg > VCPU_SREG_GS) {
3474 rc = emulate_ud(ctxt);
3478 if (c->modrm_reg == VCPU_SREG_SS)
3479 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3481 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3483 c->dst.type = OP_NONE; /* Disable writeback. */
3486 case 0x8f: /* pop (sole member of Grp1a) */
3487 rc = emulate_grp1a(ctxt, ops);
3489 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3490 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3493 case 0x98: /* cbw/cwde/cdqe */
3494 switch (c->op_bytes) {
3495 case 2: c->dst.val = (s8)c->dst.val; break;
3496 case 4: c->dst.val = (s16)c->dst.val; break;
3497 case 8: c->dst.val = (s32)c->dst.val; break;
3500 case 0x9c: /* pushf */
3501 c->src.val = (unsigned long) ctxt->eflags;
3502 emulate_push(ctxt, ops);
3504 case 0x9d: /* popf */
3505 c->dst.type = OP_REG;
3506 c->dst.addr.reg = &ctxt->eflags;
3507 c->dst.bytes = c->op_bytes;
3508 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3510 case 0xa6 ... 0xa7: /* cmps */
3511 c->dst.type = OP_NONE; /* Disable writeback. */
3513 case 0xa8 ... 0xa9: /* test ax, imm */
3515 case 0xae ... 0xaf: /* scas */
3520 case 0xc3: /* ret */
3521 c->dst.type = OP_REG;
3522 c->dst.addr.reg = &c->eip;
3523 c->dst.bytes = c->op_bytes;
3524 goto pop_instruction;
3525 case 0xc4: /* les */
3526 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3528 case 0xc5: /* lds */
3529 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3531 case 0xcb: /* ret far */
3532 rc = emulate_ret_far(ctxt, ops);
3534 case 0xcc: /* int3 */
3537 case 0xcd: /* int n */
3540 rc = emulate_int(ctxt, ops, irq);
3542 case 0xce: /* into */
3543 if (ctxt->eflags & EFLG_OF) {
3548 case 0xcf: /* iret */
3549 rc = emulate_iret(ctxt, ops);
3551 case 0xd0 ... 0xd1: /* Grp2 */
3554 case 0xd2 ... 0xd3: /* Grp2 */
3555 c->src.val = c->regs[VCPU_REGS_RCX];
3558 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3559 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3560 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3561 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3562 jmp_rel(c, c->src.val);
3564 case 0xe3: /* jcxz/jecxz/jrcxz */
3565 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3566 jmp_rel(c, c->src.val);
3568 case 0xe4: /* inb */
3571 case 0xe6: /* outb */
3572 case 0xe7: /* out */
3574 case 0xe8: /* call (near) */ {
3575 long int rel = c->src.val;
3576 c->src.val = (unsigned long) c->eip;
3578 emulate_push(ctxt, ops);
3581 case 0xe9: /* jmp rel */
3583 case 0xea: { /* jmp far */
3586 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3588 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3592 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3596 jmp: /* jmp rel short */
3597 jmp_rel(c, c->src.val);
3598 c->dst.type = OP_NONE; /* Disable writeback. */
3600 case 0xec: /* in al,dx */
3601 case 0xed: /* in (e/r)ax,dx */
3602 c->src.val = c->regs[VCPU_REGS_RDX];
3604 c->dst.bytes = min(c->dst.bytes, 4u);
3605 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3606 rc = emulate_gp(ctxt, 0);
3609 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3611 goto done; /* IO is needed */
3613 case 0xee: /* out dx,al */
3614 case 0xef: /* out dx,(e/r)ax */
3615 c->dst.val = c->regs[VCPU_REGS_RDX];
3617 c->src.bytes = min(c->src.bytes, 4u);
3618 if (!emulator_io_permited(ctxt, ops, c->dst.val,
3620 rc = emulate_gp(ctxt, 0);
3623 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3624 &c->src.val, 1, ctxt->vcpu);
3625 c->dst.type = OP_NONE; /* Disable writeback. */
3627 case 0xf4: /* hlt */
3628 ctxt->vcpu->arch.halt_request = 1;
3630 case 0xf5: /* cmc */
3631 /* complement carry flag from eflags reg */
3632 ctxt->eflags ^= EFLG_CF;
3634 case 0xf6 ... 0xf7: /* Grp3 */
3635 rc = emulate_grp3(ctxt, ops);
3637 case 0xf8: /* clc */
3638 ctxt->eflags &= ~EFLG_CF;
3640 case 0xf9: /* stc */
3641 ctxt->eflags |= EFLG_CF;
3643 case 0xfa: /* cli */
3644 if (emulator_bad_iopl(ctxt, ops)) {
3645 rc = emulate_gp(ctxt, 0);
3648 ctxt->eflags &= ~X86_EFLAGS_IF;
3650 case 0xfb: /* sti */
3651 if (emulator_bad_iopl(ctxt, ops)) {
3652 rc = emulate_gp(ctxt, 0);
3655 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3656 ctxt->eflags |= X86_EFLAGS_IF;
3659 case 0xfc: /* cld */
3660 ctxt->eflags &= ~EFLG_DF;
3662 case 0xfd: /* std */
3663 ctxt->eflags |= EFLG_DF;
3665 case 0xfe: /* Grp4 */
3667 rc = emulate_grp45(ctxt, ops);
3669 case 0xff: /* Grp5 */
3670 if (c->modrm_reg == 5)
3674 goto cannot_emulate;
3677 if (rc != X86EMUL_CONTINUE)
3681 rc = writeback(ctxt, ops);
3682 if (rc != X86EMUL_CONTINUE)
3686 * restore dst type in case the decoding will be reused
3687 * (happens for string instruction )
3689 c->dst.type = saved_dst_type;
3691 if ((c->d & SrcMask) == SrcSI)
3692 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3693 VCPU_REGS_RSI, &c->src);
3695 if ((c->d & DstMask) == DstDI)
3696 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3699 if (c->rep_prefix && (c->d & String)) {
3700 struct read_cache *r = &ctxt->decode.io_read;
3701 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3703 if (!string_insn_completed(ctxt)) {
3705 * Re-enter guest when pio read ahead buffer is empty
3706 * or, if it is not used, after each 1024 iteration.
3708 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3709 (r->end == 0 || r->end != r->pos)) {
3711 * Reset read cache. Usually happens before
3712 * decode, but since instruction is restarted
3713 * we have to do it here.
3715 ctxt->decode.mem_read.end = 0;
3716 return EMULATION_RESTART;
3718 goto done; /* skip rip writeback */
3725 if (rc == X86EMUL_PROPAGATE_FAULT)
3726 ctxt->have_exception = true;
3727 if (rc == X86EMUL_INTERCEPTED)
3728 return EMULATION_INTERCEPTED;
3730 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3734 case 0x01: /* lgdt, lidt, lmsw */
3735 switch (c->modrm_reg) {
3737 unsigned long address;
3739 case 0: /* vmcall */
3740 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3741 goto cannot_emulate;
3743 rc = kvm_fix_hypercall(ctxt->vcpu);
3744 if (rc != X86EMUL_CONTINUE)
3747 /* Let the processor re-execute the fixed hypercall */
3749 /* Disable writeback. */
3750 c->dst.type = OP_NONE;
3753 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3754 &size, &address, c->op_bytes);
3755 if (rc != X86EMUL_CONTINUE)
3757 realmode_lgdt(ctxt->vcpu, size, address);
3758 /* Disable writeback. */
3759 c->dst.type = OP_NONE;
3761 case 3: /* lidt/vmmcall */
3762 if (c->modrm_mod == 3) {
3763 switch (c->modrm_rm) {
3765 rc = kvm_fix_hypercall(ctxt->vcpu);
3768 goto cannot_emulate;
3771 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3774 if (rc != X86EMUL_CONTINUE)
3776 realmode_lidt(ctxt->vcpu, size, address);
3778 /* Disable writeback. */
3779 c->dst.type = OP_NONE;
3783 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3786 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
3787 (c->src.val & 0x0f), ctxt->vcpu);
3788 c->dst.type = OP_NONE;
3790 case 5: /* not defined */
3792 rc = X86EMUL_PROPAGATE_FAULT;
3795 emulate_invlpg(ctxt->vcpu,
3796 linear(ctxt, c->src.addr.mem));
3797 /* Disable writeback. */
3798 c->dst.type = OP_NONE;
3801 goto cannot_emulate;
3804 case 0x05: /* syscall */
3805 rc = emulate_syscall(ctxt, ops);
3808 emulate_clts(ctxt->vcpu);
3810 case 0x09: /* wbinvd */
3811 kvm_emulate_wbinvd(ctxt->vcpu);
3813 case 0x08: /* invd */
3814 case 0x0d: /* GrpP (prefetch) */
3815 case 0x18: /* Grp16 (prefetch/nop) */
3817 case 0x20: /* mov cr, reg */
3818 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3820 case 0x21: /* mov from dr to reg */
3821 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3822 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3824 rc = X86EMUL_PROPAGATE_FAULT;
3827 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
3829 case 0x22: /* mov reg, cr */
3830 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
3831 emulate_gp(ctxt, 0);
3832 rc = X86EMUL_PROPAGATE_FAULT;
3835 c->dst.type = OP_NONE;
3837 case 0x23: /* mov from reg to dr */
3838 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3839 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3841 rc = X86EMUL_PROPAGATE_FAULT;
3845 if (ops->set_dr(c->modrm_reg, c->src.val &
3846 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3847 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3848 /* #UD condition is already handled by the code above */
3849 emulate_gp(ctxt, 0);
3850 rc = X86EMUL_PROPAGATE_FAULT;
3854 c->dst.type = OP_NONE; /* no writeback */
3858 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3859 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3860 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3861 emulate_gp(ctxt, 0);
3862 rc = X86EMUL_PROPAGATE_FAULT;
3865 rc = X86EMUL_CONTINUE;
3869 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3870 emulate_gp(ctxt, 0);
3871 rc = X86EMUL_PROPAGATE_FAULT;
3874 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3875 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3877 rc = X86EMUL_CONTINUE;
3879 case 0x34: /* sysenter */
3880 rc = emulate_sysenter(ctxt, ops);
3882 case 0x35: /* sysexit */
3883 rc = emulate_sysexit(ctxt, ops);
3885 case 0x40 ... 0x4f: /* cmov */
3886 c->dst.val = c->dst.orig_val = c->src.val;
3887 if (!test_cc(c->b, ctxt->eflags))
3888 c->dst.type = OP_NONE; /* no writeback */
3890 case 0x80 ... 0x8f: /* jnz rel, etc*/
3891 if (test_cc(c->b, ctxt->eflags))
3892 jmp_rel(c, c->src.val);
3894 case 0x90 ... 0x9f: /* setcc r/m8 */
3895 c->dst.val = test_cc(c->b, ctxt->eflags);
3897 case 0xa0: /* push fs */
3898 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3900 case 0xa1: /* pop fs */
3901 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3905 c->dst.type = OP_NONE;
3906 /* only subword offset */
3907 c->src.val &= (c->dst.bytes << 3) - 1;
3908 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3910 case 0xa4: /* shld imm8, r, r/m */
3911 case 0xa5: /* shld cl, r, r/m */
3912 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3914 case 0xa8: /* push gs */
3915 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3917 case 0xa9: /* pop gs */
3918 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3922 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3924 case 0xac: /* shrd imm8, r, r/m */
3925 case 0xad: /* shrd cl, r, r/m */
3926 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3928 case 0xae: /* clflush */
3930 case 0xb0 ... 0xb1: /* cmpxchg */
3932 * Save real source value, then compare EAX against
3935 c->src.orig_val = c->src.val;
3936 c->src.val = c->regs[VCPU_REGS_RAX];
3937 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3938 if (ctxt->eflags & EFLG_ZF) {
3939 /* Success: write back to memory. */
3940 c->dst.val = c->src.orig_val;
3942 /* Failure: write the value we saw to EAX. */
3943 c->dst.type = OP_REG;
3944 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3947 case 0xb2: /* lss */
3948 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
3952 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3954 case 0xb4: /* lfs */
3955 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
3957 case 0xb5: /* lgs */
3958 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
3960 case 0xb6 ... 0xb7: /* movzx */
3961 c->dst.bytes = c->op_bytes;
3962 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3965 case 0xba: /* Grp8 */
3966 switch (c->modrm_reg & 3) {
3979 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3981 case 0xbc: { /* bsf */
3983 __asm__ ("bsf %2, %0; setz %1"
3984 : "=r"(c->dst.val), "=q"(zf)
3986 ctxt->eflags &= ~X86_EFLAGS_ZF;
3988 ctxt->eflags |= X86_EFLAGS_ZF;
3989 c->dst.type = OP_NONE; /* Disable writeback. */
3993 case 0xbd: { /* bsr */
3995 __asm__ ("bsr %2, %0; setz %1"
3996 : "=r"(c->dst.val), "=q"(zf)
3998 ctxt->eflags &= ~X86_EFLAGS_ZF;
4000 ctxt->eflags |= X86_EFLAGS_ZF;
4001 c->dst.type = OP_NONE; /* Disable writeback. */
4005 case 0xbe ... 0xbf: /* movsx */
4006 c->dst.bytes = c->op_bytes;
4007 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4010 case 0xc0 ... 0xc1: /* xadd */
4011 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4012 /* Write back the register source. */
4013 c->src.val = c->dst.orig_val;
4014 write_register_operand(&c->src);
4016 case 0xc3: /* movnti */
4017 c->dst.bytes = c->op_bytes;
4018 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4021 case 0xc7: /* Grp9 (cmpxchg8b) */
4022 rc = emulate_grp9(ctxt, ops);
4025 goto cannot_emulate;
4028 if (rc != X86EMUL_CONTINUE)