1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
81 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
82 #define VendorSpecific (1<<22) /* Vendor specific instruction */
83 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
84 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
85 #define Undefined (1<<25) /* No Such Instruction */
86 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
87 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
89 /* Source 2 operand type */
90 #define Src2None (0<<29)
91 #define Src2CL (1<<29)
92 #define Src2ImmByte (2<<29)
93 #define Src2One (3<<29)
94 #define Src2Imm (4<<29)
95 #define Src2Mask (7<<29)
98 #define X3(x...) X2(x), x
99 #define X4(x...) X2(x), X2(x)
100 #define X5(x...) X4(x), x
101 #define X6(x...) X4(x), X2(x)
102 #define X7(x...) X4(x), X3(x)
103 #define X8(x...) X4(x), X4(x)
104 #define X16(x...) X8(x), X8(x)
110 int (*execute)(struct x86_emulate_ctxt *ctxt);
111 struct opcode *group;
112 struct group_dual *gdual;
113 struct gprefix *gprefix;
115 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
119 struct opcode mod012[8];
120 struct opcode mod3[8];
124 struct opcode pfx_no;
125 struct opcode pfx_66;
126 struct opcode pfx_f2;
127 struct opcode pfx_f3;
130 /* EFLAGS bit definitions. */
131 #define EFLG_ID (1<<21)
132 #define EFLG_VIP (1<<20)
133 #define EFLG_VIF (1<<19)
134 #define EFLG_AC (1<<18)
135 #define EFLG_VM (1<<17)
136 #define EFLG_RF (1<<16)
137 #define EFLG_IOPL (3<<12)
138 #define EFLG_NT (1<<14)
139 #define EFLG_OF (1<<11)
140 #define EFLG_DF (1<<10)
141 #define EFLG_IF (1<<9)
142 #define EFLG_TF (1<<8)
143 #define EFLG_SF (1<<7)
144 #define EFLG_ZF (1<<6)
145 #define EFLG_AF (1<<4)
146 #define EFLG_PF (1<<2)
147 #define EFLG_CF (1<<0)
149 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
150 #define EFLG_RESERVED_ONE_MASK 2
153 * Instruction emulation:
154 * Most instructions are emulated directly via a fragment of inline assembly
155 * code. This allows us to save/restore EFLAGS and thus very easily pick up
156 * any modified flags.
159 #if defined(CONFIG_X86_64)
160 #define _LO32 "k" /* force 32-bit operand */
161 #define _STK "%%rsp" /* stack pointer */
162 #elif defined(__i386__)
163 #define _LO32 "" /* force 32-bit operand */
164 #define _STK "%%esp" /* stack pointer */
168 * These EFLAGS bits are restored from saved value during emulation, and
169 * any changes are written back to the saved value after emulation.
171 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
173 /* Before executing instruction: restore necessary bits in EFLAGS. */
174 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
175 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
176 "movl %"_sav",%"_LO32 _tmp"; " \
179 "movl %"_msk",%"_LO32 _tmp"; " \
180 "andl %"_LO32 _tmp",("_STK"); " \
182 "notl %"_LO32 _tmp"; " \
183 "andl %"_LO32 _tmp",("_STK"); " \
184 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
186 "orl %"_LO32 _tmp",("_STK"); " \
190 /* After executing instruction: write-back necessary bits in EFLAGS. */
191 #define _POST_EFLAGS(_sav, _msk, _tmp) \
192 /* _sav |= EFLAGS & _msk; */ \
195 "andl %"_msk",%"_LO32 _tmp"; " \
196 "orl %"_LO32 _tmp",%"_sav"; "
204 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
206 __asm__ __volatile__ ( \
207 _PRE_EFLAGS("0", "4", "2") \
208 _op _suffix " %"_x"3,%1; " \
209 _POST_EFLAGS("0", "4", "2") \
210 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
212 : _y ((_src).val), "i" (EFLAGS_MASK)); \
216 /* Raw emulation: instruction has two explicit operands. */
217 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
219 unsigned long _tmp; \
221 switch ((_dst).bytes) { \
223 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
226 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
229 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
234 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
236 unsigned long _tmp; \
237 switch ((_dst).bytes) { \
239 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
242 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
243 _wx, _wy, _lx, _ly, _qx, _qy); \
248 /* Source operand is byte-sized and may be restricted to just %cl. */
249 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
250 __emulate_2op(_op, _src, _dst, _eflags, \
251 "b", "c", "b", "c", "b", "c", "b", "c")
253 /* Source operand is byte, word, long or quad sized. */
254 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
255 __emulate_2op(_op, _src, _dst, _eflags, \
256 "b", "q", "w", "r", _LO32, "r", "", "r")
258 /* Source operand is word, long or quad sized. */
259 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
260 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
261 "w", "r", _LO32, "r", "", "r")
263 /* Instruction has three operands and one operand is stored in ECX register */
264 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
266 unsigned long _tmp; \
267 _type _clv = (_cl).val; \
268 _type _srcv = (_src).val; \
269 _type _dstv = (_dst).val; \
271 __asm__ __volatile__ ( \
272 _PRE_EFLAGS("0", "5", "2") \
273 _op _suffix " %4,%1 \n" \
274 _POST_EFLAGS("0", "5", "2") \
275 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
276 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
279 (_cl).val = (unsigned long) _clv; \
280 (_src).val = (unsigned long) _srcv; \
281 (_dst).val = (unsigned long) _dstv; \
284 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
286 switch ((_dst).bytes) { \
288 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
289 "w", unsigned short); \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "l", unsigned int); \
296 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "q", unsigned long)); \
302 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
304 unsigned long _tmp; \
306 __asm__ __volatile__ ( \
307 _PRE_EFLAGS("0", "3", "2") \
308 _op _suffix " %1; " \
309 _POST_EFLAGS("0", "3", "2") \
310 : "=m" (_eflags), "+m" ((_dst).val), \
312 : "i" (EFLAGS_MASK)); \
315 /* Instruction has only one explicit operand (no source operand). */
316 #define emulate_1op(_op, _dst, _eflags) \
318 switch ((_dst).bytes) { \
319 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
320 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
321 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
322 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
326 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
328 unsigned long _tmp; \
330 __asm__ __volatile__ ( \
331 _PRE_EFLAGS("0", "4", "1") \
332 _op _suffix " %5; " \
333 _POST_EFLAGS("0", "4", "1") \
334 : "=m" (_eflags), "=&r" (_tmp), \
335 "+a" (_rax), "+d" (_rdx) \
336 : "i" (EFLAGS_MASK), "m" ((_src).val), \
337 "a" (_rax), "d" (_rdx)); \
340 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
342 unsigned long _tmp; \
344 __asm__ __volatile__ ( \
345 _PRE_EFLAGS("0", "5", "1") \
347 _op _suffix " %6; " \
349 _POST_EFLAGS("0", "5", "1") \
350 ".pushsection .fixup,\"ax\" \n\t" \
351 "3: movb $1, %4 \n\t" \
354 _ASM_EXTABLE(1b, 3b) \
355 : "=m" (_eflags), "=&r" (_tmp), \
356 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
357 : "i" (EFLAGS_MASK), "m" ((_src).val), \
358 "a" (_rax), "d" (_rdx)); \
361 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
362 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
364 switch((_src).bytes) { \
365 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
366 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
367 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
368 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
372 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
374 switch((_src).bytes) { \
376 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
377 _eflags, "b", _ex); \
380 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
381 _eflags, "w", _ex); \
384 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
385 _eflags, "l", _ex); \
388 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
389 _eflags, "q", _ex)); \
394 /* Fetch next part of the instruction being emulated. */
395 #define insn_fetch(_type, _size, _eip) \
396 ({ unsigned long _x; \
397 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
398 if (rc != X86EMUL_CONTINUE) \
404 #define insn_fetch_arr(_arr, _size, _eip) \
405 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
406 if (rc != X86EMUL_CONTINUE) \
411 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
412 enum x86_intercept intercept,
413 enum x86_intercept_stage stage)
415 struct x86_instruction_info info = {
416 .intercept = intercept,
417 .rep_prefix = ctxt->decode.rep_prefix,
418 .modrm_mod = ctxt->decode.modrm_mod,
419 .modrm_reg = ctxt->decode.modrm_reg,
420 .modrm_rm = ctxt->decode.modrm_rm,
421 .src_val = ctxt->decode.src.val64,
422 .src_bytes = ctxt->decode.src.bytes,
423 .dst_bytes = ctxt->decode.dst.bytes,
424 .ad_bytes = ctxt->decode.ad_bytes,
425 .next_rip = ctxt->eip,
428 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
431 static inline unsigned long ad_mask(struct decode_cache *c)
433 return (1UL << (c->ad_bytes << 3)) - 1;
436 /* Access/update address held in a register, based on addressing mode. */
437 static inline unsigned long
438 address_mask(struct decode_cache *c, unsigned long reg)
440 if (c->ad_bytes == sizeof(unsigned long))
443 return reg & ad_mask(c);
446 static inline unsigned long
447 register_address(struct decode_cache *c, unsigned long reg)
449 return address_mask(c, reg);
453 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
455 if (c->ad_bytes == sizeof(unsigned long))
458 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
461 static inline void jmp_rel(struct decode_cache *c, int rel)
463 register_address_increment(c, &c->eip, rel);
466 static void set_seg_override(struct decode_cache *c, int seg)
468 c->has_seg_override = true;
469 c->seg_override = seg;
472 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
473 struct x86_emulate_ops *ops, int seg)
475 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
478 return ops->get_cached_segment_base(seg, ctxt->vcpu);
481 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
482 struct x86_emulate_ops *ops,
483 struct decode_cache *c)
485 if (!c->has_seg_override)
488 return c->seg_override;
491 static ulong linear(struct x86_emulate_ctxt *ctxt,
492 struct segmented_address addr)
494 struct decode_cache *c = &ctxt->decode;
497 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
498 if (c->ad_bytes != 8)
503 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
504 u32 error, bool valid)
506 ctxt->exception.vector = vec;
507 ctxt->exception.error_code = error;
508 ctxt->exception.error_code_valid = valid;
509 return X86EMUL_PROPAGATE_FAULT;
512 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
514 return emulate_exception(ctxt, GP_VECTOR, err, true);
517 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
519 return emulate_exception(ctxt, UD_VECTOR, 0, false);
522 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
524 return emulate_exception(ctxt, TS_VECTOR, err, true);
527 static int emulate_de(struct x86_emulate_ctxt *ctxt)
529 return emulate_exception(ctxt, DE_VECTOR, 0, false);
532 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, NM_VECTOR, 0, false);
537 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
538 struct x86_emulate_ops *ops,
539 unsigned long eip, u8 *dest)
541 struct fetch_cache *fc = &ctxt->decode.fetch;
545 if (eip == fc->end) {
546 cur_size = fc->end - fc->start;
547 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
548 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
549 size, ctxt->vcpu, &ctxt->exception);
550 if (rc != X86EMUL_CONTINUE)
554 *dest = fc->data[eip - fc->start];
555 return X86EMUL_CONTINUE;
558 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
559 struct x86_emulate_ops *ops,
560 unsigned long eip, void *dest, unsigned size)
564 /* x86 instructions are limited to 15 bytes. */
565 if (eip + size - ctxt->eip > 15)
566 return X86EMUL_UNHANDLEABLE;
568 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
569 if (rc != X86EMUL_CONTINUE)
572 return X86EMUL_CONTINUE;
576 * Given the 'reg' portion of a ModRM byte, and a register block, return a
577 * pointer into the block that addresses the relevant register.
578 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
580 static void *decode_register(u8 modrm_reg, unsigned long *regs,
585 p = ®s[modrm_reg];
586 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
587 p = (unsigned char *)®s[modrm_reg & 3] + 1;
591 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
592 struct x86_emulate_ops *ops,
593 struct segmented_address addr,
594 u16 *size, unsigned long *address, int op_bytes)
601 rc = ops->read_std(linear(ctxt, addr), (unsigned long *)size, 2,
602 ctxt->vcpu, &ctxt->exception);
603 if (rc != X86EMUL_CONTINUE)
606 rc = ops->read_std(linear(ctxt, addr), address, op_bytes,
607 ctxt->vcpu, &ctxt->exception);
611 static int test_cc(unsigned int condition, unsigned int flags)
615 switch ((condition & 15) >> 1) {
617 rc |= (flags & EFLG_OF);
619 case 1: /* b/c/nae */
620 rc |= (flags & EFLG_CF);
623 rc |= (flags & EFLG_ZF);
626 rc |= (flags & (EFLG_CF|EFLG_ZF));
629 rc |= (flags & EFLG_SF);
632 rc |= (flags & EFLG_PF);
635 rc |= (flags & EFLG_ZF);
638 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
642 /* Odd condition identifiers (lsb == 1) have inverted sense. */
643 return (!!rc ^ (condition & 1));
646 static void fetch_register_operand(struct operand *op)
650 op->val = *(u8 *)op->addr.reg;
653 op->val = *(u16 *)op->addr.reg;
656 op->val = *(u32 *)op->addr.reg;
659 op->val = *(u64 *)op->addr.reg;
664 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
666 ctxt->ops->get_fpu(ctxt);
668 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
669 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
670 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
671 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
672 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
673 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
674 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
675 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
677 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
678 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
679 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
680 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
681 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
682 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
683 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
684 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
688 ctxt->ops->put_fpu(ctxt);
691 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
694 ctxt->ops->get_fpu(ctxt);
696 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
697 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
698 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
699 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
700 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
701 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
702 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
703 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
705 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
706 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
707 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
708 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
709 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
710 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
711 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
712 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
716 ctxt->ops->put_fpu(ctxt);
719 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
721 struct decode_cache *c,
724 unsigned reg = c->modrm_reg;
725 int highbyte_regs = c->rex_prefix == 0;
728 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
734 read_sse_reg(ctxt, &op->vec_val, reg);
739 if ((c->d & ByteOp) && !inhibit_bytereg) {
740 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
743 op->addr.reg = decode_register(reg, c->regs, 0);
744 op->bytes = c->op_bytes;
746 fetch_register_operand(op);
747 op->orig_val = op->val;
750 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
751 struct x86_emulate_ops *ops,
754 struct decode_cache *c = &ctxt->decode;
756 int index_reg = 0, base_reg = 0, scale;
757 int rc = X86EMUL_CONTINUE;
761 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
762 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
763 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
766 c->modrm = insn_fetch(u8, 1, c->eip);
767 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
768 c->modrm_reg |= (c->modrm & 0x38) >> 3;
769 c->modrm_rm |= (c->modrm & 0x07);
770 c->modrm_seg = VCPU_SREG_DS;
772 if (c->modrm_mod == 3) {
774 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
775 op->addr.reg = decode_register(c->modrm_rm,
776 c->regs, c->d & ByteOp);
780 op->addr.xmm = c->modrm_rm;
781 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
784 fetch_register_operand(op);
790 if (c->ad_bytes == 2) {
791 unsigned bx = c->regs[VCPU_REGS_RBX];
792 unsigned bp = c->regs[VCPU_REGS_RBP];
793 unsigned si = c->regs[VCPU_REGS_RSI];
794 unsigned di = c->regs[VCPU_REGS_RDI];
796 /* 16-bit ModR/M decode. */
797 switch (c->modrm_mod) {
799 if (c->modrm_rm == 6)
800 modrm_ea += insn_fetch(u16, 2, c->eip);
803 modrm_ea += insn_fetch(s8, 1, c->eip);
806 modrm_ea += insn_fetch(u16, 2, c->eip);
809 switch (c->modrm_rm) {
829 if (c->modrm_mod != 0)
836 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
837 (c->modrm_rm == 6 && c->modrm_mod != 0))
838 c->modrm_seg = VCPU_SREG_SS;
839 modrm_ea = (u16)modrm_ea;
841 /* 32/64-bit ModR/M decode. */
842 if ((c->modrm_rm & 7) == 4) {
843 sib = insn_fetch(u8, 1, c->eip);
844 index_reg |= (sib >> 3) & 7;
848 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
849 modrm_ea += insn_fetch(s32, 4, c->eip);
851 modrm_ea += c->regs[base_reg];
853 modrm_ea += c->regs[index_reg] << scale;
854 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
855 if (ctxt->mode == X86EMUL_MODE_PROT64)
858 modrm_ea += c->regs[c->modrm_rm];
859 switch (c->modrm_mod) {
861 if (c->modrm_rm == 5)
862 modrm_ea += insn_fetch(s32, 4, c->eip);
865 modrm_ea += insn_fetch(s8, 1, c->eip);
868 modrm_ea += insn_fetch(s32, 4, c->eip);
872 op->addr.mem.ea = modrm_ea;
877 static int decode_abs(struct x86_emulate_ctxt *ctxt,
878 struct x86_emulate_ops *ops,
881 struct decode_cache *c = &ctxt->decode;
882 int rc = X86EMUL_CONTINUE;
885 switch (c->ad_bytes) {
887 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
890 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
893 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
900 static void fetch_bit_operand(struct decode_cache *c)
904 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
905 mask = ~(c->dst.bytes * 8 - 1);
907 if (c->src.bytes == 2)
908 sv = (s16)c->src.val & (s16)mask;
909 else if (c->src.bytes == 4)
910 sv = (s32)c->src.val & (s32)mask;
912 c->dst.addr.mem.ea += (sv >> 3);
915 /* only subword offset */
916 c->src.val &= (c->dst.bytes << 3) - 1;
919 static int read_emulated(struct x86_emulate_ctxt *ctxt,
920 struct x86_emulate_ops *ops,
921 unsigned long addr, void *dest, unsigned size)
924 struct read_cache *mc = &ctxt->decode.mem_read;
927 int n = min(size, 8u);
929 if (mc->pos < mc->end)
932 rc = ops->read_emulated(addr, mc->data + mc->end, n,
933 &ctxt->exception, ctxt->vcpu);
934 if (rc != X86EMUL_CONTINUE)
939 memcpy(dest, mc->data + mc->pos, n);
944 return X86EMUL_CONTINUE;
947 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
948 struct x86_emulate_ops *ops,
949 unsigned int size, unsigned short port,
952 struct read_cache *rc = &ctxt->decode.io_read;
954 if (rc->pos == rc->end) { /* refill pio read ahead */
955 struct decode_cache *c = &ctxt->decode;
956 unsigned int in_page, n;
957 unsigned int count = c->rep_prefix ?
958 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
959 in_page = (ctxt->eflags & EFLG_DF) ?
960 offset_in_page(c->regs[VCPU_REGS_RDI]) :
961 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
962 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
966 rc->pos = rc->end = 0;
967 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
972 memcpy(dest, rc->data + rc->pos, size);
977 static u32 desc_limit_scaled(struct desc_struct *desc)
979 u32 limit = get_desc_limit(desc);
981 return desc->g ? (limit << 12) | 0xfff : limit;
984 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
985 struct x86_emulate_ops *ops,
986 u16 selector, struct desc_ptr *dt)
988 if (selector & 1 << 2) {
989 struct desc_struct desc;
990 memset (dt, 0, sizeof *dt);
991 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
995 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
996 dt->address = get_desc_base(&desc);
998 ops->get_gdt(dt, ctxt->vcpu);
1001 /* allowed just for 8 bytes segments */
1002 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1003 struct x86_emulate_ops *ops,
1004 u16 selector, struct desc_struct *desc)
1007 u16 index = selector >> 3;
1011 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1013 if (dt.size < index * 8 + 7)
1014 return emulate_gp(ctxt, selector & 0xfffc);
1015 addr = dt.address + index * 8;
1016 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1022 /* allowed just for 8 bytes segments */
1023 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1024 struct x86_emulate_ops *ops,
1025 u16 selector, struct desc_struct *desc)
1028 u16 index = selector >> 3;
1032 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1034 if (dt.size < index * 8 + 7)
1035 return emulate_gp(ctxt, selector & 0xfffc);
1037 addr = dt.address + index * 8;
1038 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1044 /* Does not support long mode */
1045 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1046 struct x86_emulate_ops *ops,
1047 u16 selector, int seg)
1049 struct desc_struct seg_desc;
1051 unsigned err_vec = GP_VECTOR;
1053 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1056 memset(&seg_desc, 0, sizeof seg_desc);
1058 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1059 || ctxt->mode == X86EMUL_MODE_REAL) {
1060 /* set real mode segment descriptor */
1061 set_desc_base(&seg_desc, selector << 4);
1062 set_desc_limit(&seg_desc, 0xffff);
1069 /* NULL selector is not valid for TR, CS and SS */
1070 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1074 /* TR should be in GDT only */
1075 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1078 if (null_selector) /* for NULL selector skip all following checks */
1081 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1082 if (ret != X86EMUL_CONTINUE)
1085 err_code = selector & 0xfffc;
1086 err_vec = GP_VECTOR;
1088 /* can't load system descriptor into segment selecor */
1089 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1093 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1099 cpl = ops->cpl(ctxt->vcpu);
1104 * segment is not a writable data segment or segment
1105 * selector's RPL != CPL or segment selector's RPL != CPL
1107 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1111 if (!(seg_desc.type & 8))
1114 if (seg_desc.type & 4) {
1120 if (rpl > cpl || dpl != cpl)
1123 /* CS(RPL) <- CPL */
1124 selector = (selector & 0xfffc) | cpl;
1127 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1130 case VCPU_SREG_LDTR:
1131 if (seg_desc.s || seg_desc.type != 2)
1134 default: /* DS, ES, FS, or GS */
1136 * segment is not a data or readable code segment or
1137 * ((segment is a data or nonconforming code segment)
1138 * and (both RPL and CPL > DPL))
1140 if ((seg_desc.type & 0xa) == 0x8 ||
1141 (((seg_desc.type & 0xc) != 0xc) &&
1142 (rpl > dpl && cpl > dpl)))
1148 /* mark segment as accessed */
1150 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1151 if (ret != X86EMUL_CONTINUE)
1155 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1156 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1157 return X86EMUL_CONTINUE;
1159 emulate_exception(ctxt, err_vec, err_code, true);
1160 return X86EMUL_PROPAGATE_FAULT;
1163 static void write_register_operand(struct operand *op)
1165 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1166 switch (op->bytes) {
1168 *(u8 *)op->addr.reg = (u8)op->val;
1171 *(u16 *)op->addr.reg = (u16)op->val;
1174 *op->addr.reg = (u32)op->val;
1175 break; /* 64b: zero-extend */
1177 *op->addr.reg = op->val;
1182 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1183 struct x86_emulate_ops *ops)
1186 struct decode_cache *c = &ctxt->decode;
1188 switch (c->dst.type) {
1190 write_register_operand(&c->dst);
1194 rc = ops->cmpxchg_emulated(
1195 linear(ctxt, c->dst.addr.mem),
1202 rc = ops->write_emulated(
1203 linear(ctxt, c->dst.addr.mem),
1208 if (rc != X86EMUL_CONTINUE)
1212 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1220 return X86EMUL_CONTINUE;
1223 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1224 struct x86_emulate_ops *ops)
1226 struct decode_cache *c = &ctxt->decode;
1228 c->dst.type = OP_MEM;
1229 c->dst.bytes = c->op_bytes;
1230 c->dst.val = c->src.val;
1231 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1232 c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1233 c->dst.addr.mem.seg = VCPU_SREG_SS;
1236 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1237 struct x86_emulate_ops *ops,
1238 void *dest, int len)
1240 struct decode_cache *c = &ctxt->decode;
1242 struct segmented_address addr;
1244 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1245 addr.seg = VCPU_SREG_SS;
1246 rc = read_emulated(ctxt, ops, linear(ctxt, addr), dest, len);
1247 if (rc != X86EMUL_CONTINUE)
1250 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1254 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1255 struct x86_emulate_ops *ops,
1256 void *dest, int len)
1259 unsigned long val, change_mask;
1260 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1261 int cpl = ops->cpl(ctxt->vcpu);
1263 rc = emulate_pop(ctxt, ops, &val, len);
1264 if (rc != X86EMUL_CONTINUE)
1267 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1268 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1270 switch(ctxt->mode) {
1271 case X86EMUL_MODE_PROT64:
1272 case X86EMUL_MODE_PROT32:
1273 case X86EMUL_MODE_PROT16:
1275 change_mask |= EFLG_IOPL;
1277 change_mask |= EFLG_IF;
1279 case X86EMUL_MODE_VM86:
1281 return emulate_gp(ctxt, 0);
1282 change_mask |= EFLG_IF;
1284 default: /* real mode */
1285 change_mask |= (EFLG_IOPL | EFLG_IF);
1289 *(unsigned long *)dest =
1290 (ctxt->eflags & ~change_mask) | (val & change_mask);
1295 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1296 struct x86_emulate_ops *ops, int seg)
1298 struct decode_cache *c = &ctxt->decode;
1300 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1302 emulate_push(ctxt, ops);
1305 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1306 struct x86_emulate_ops *ops, int seg)
1308 struct decode_cache *c = &ctxt->decode;
1309 unsigned long selector;
1312 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1313 if (rc != X86EMUL_CONTINUE)
1316 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1320 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1321 struct x86_emulate_ops *ops)
1323 struct decode_cache *c = &ctxt->decode;
1324 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1325 int rc = X86EMUL_CONTINUE;
1326 int reg = VCPU_REGS_RAX;
1328 while (reg <= VCPU_REGS_RDI) {
1329 (reg == VCPU_REGS_RSP) ?
1330 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1332 emulate_push(ctxt, ops);
1334 rc = writeback(ctxt, ops);
1335 if (rc != X86EMUL_CONTINUE)
1341 /* Disable writeback. */
1342 c->dst.type = OP_NONE;
1347 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1348 struct x86_emulate_ops *ops)
1350 struct decode_cache *c = &ctxt->decode;
1351 int rc = X86EMUL_CONTINUE;
1352 int reg = VCPU_REGS_RDI;
1354 while (reg >= VCPU_REGS_RAX) {
1355 if (reg == VCPU_REGS_RSP) {
1356 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1361 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1362 if (rc != X86EMUL_CONTINUE)
1369 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1370 struct x86_emulate_ops *ops, int irq)
1372 struct decode_cache *c = &ctxt->decode;
1379 /* TODO: Add limit checks */
1380 c->src.val = ctxt->eflags;
1381 emulate_push(ctxt, ops);
1382 rc = writeback(ctxt, ops);
1383 if (rc != X86EMUL_CONTINUE)
1386 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1388 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1389 emulate_push(ctxt, ops);
1390 rc = writeback(ctxt, ops);
1391 if (rc != X86EMUL_CONTINUE)
1394 c->src.val = c->eip;
1395 emulate_push(ctxt, ops);
1396 rc = writeback(ctxt, ops);
1397 if (rc != X86EMUL_CONTINUE)
1400 c->dst.type = OP_NONE;
1402 ops->get_idt(&dt, ctxt->vcpu);
1404 eip_addr = dt.address + (irq << 2);
1405 cs_addr = dt.address + (irq << 2) + 2;
1407 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1408 if (rc != X86EMUL_CONTINUE)
1411 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1412 if (rc != X86EMUL_CONTINUE)
1415 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1416 if (rc != X86EMUL_CONTINUE)
1424 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1425 struct x86_emulate_ops *ops, int irq)
1427 switch(ctxt->mode) {
1428 case X86EMUL_MODE_REAL:
1429 return emulate_int_real(ctxt, ops, irq);
1430 case X86EMUL_MODE_VM86:
1431 case X86EMUL_MODE_PROT16:
1432 case X86EMUL_MODE_PROT32:
1433 case X86EMUL_MODE_PROT64:
1435 /* Protected mode interrupts unimplemented yet */
1436 return X86EMUL_UNHANDLEABLE;
1440 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1441 struct x86_emulate_ops *ops)
1443 struct decode_cache *c = &ctxt->decode;
1444 int rc = X86EMUL_CONTINUE;
1445 unsigned long temp_eip = 0;
1446 unsigned long temp_eflags = 0;
1447 unsigned long cs = 0;
1448 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1449 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1450 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1451 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1453 /* TODO: Add stack limit check */
1455 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1457 if (rc != X86EMUL_CONTINUE)
1460 if (temp_eip & ~0xffff)
1461 return emulate_gp(ctxt, 0);
1463 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1465 if (rc != X86EMUL_CONTINUE)
1468 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1470 if (rc != X86EMUL_CONTINUE)
1473 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1475 if (rc != X86EMUL_CONTINUE)
1481 if (c->op_bytes == 4)
1482 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1483 else if (c->op_bytes == 2) {
1484 ctxt->eflags &= ~0xffff;
1485 ctxt->eflags |= temp_eflags;
1488 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1489 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1494 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1495 struct x86_emulate_ops* ops)
1497 switch(ctxt->mode) {
1498 case X86EMUL_MODE_REAL:
1499 return emulate_iret_real(ctxt, ops);
1500 case X86EMUL_MODE_VM86:
1501 case X86EMUL_MODE_PROT16:
1502 case X86EMUL_MODE_PROT32:
1503 case X86EMUL_MODE_PROT64:
1505 /* iret from protected mode unimplemented yet */
1506 return X86EMUL_UNHANDLEABLE;
1510 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1511 struct x86_emulate_ops *ops)
1513 struct decode_cache *c = &ctxt->decode;
1515 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1518 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1520 struct decode_cache *c = &ctxt->decode;
1521 switch (c->modrm_reg) {
1523 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1526 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1529 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1532 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1534 case 4: /* sal/shl */
1535 case 6: /* sal/shl */
1536 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1539 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1542 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1547 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1548 struct x86_emulate_ops *ops)
1550 struct decode_cache *c = &ctxt->decode;
1551 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1552 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1555 switch (c->modrm_reg) {
1556 case 0 ... 1: /* test */
1557 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1560 c->dst.val = ~c->dst.val;
1563 emulate_1op("neg", c->dst, ctxt->eflags);
1566 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1569 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1572 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1576 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1580 return X86EMUL_UNHANDLEABLE;
1583 return emulate_de(ctxt);
1584 return X86EMUL_CONTINUE;
1587 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1588 struct x86_emulate_ops *ops)
1590 struct decode_cache *c = &ctxt->decode;
1592 switch (c->modrm_reg) {
1594 emulate_1op("inc", c->dst, ctxt->eflags);
1597 emulate_1op("dec", c->dst, ctxt->eflags);
1599 case 2: /* call near abs */ {
1602 c->eip = c->src.val;
1603 c->src.val = old_eip;
1604 emulate_push(ctxt, ops);
1607 case 4: /* jmp abs */
1608 c->eip = c->src.val;
1611 emulate_push(ctxt, ops);
1614 return X86EMUL_CONTINUE;
1617 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1618 struct x86_emulate_ops *ops)
1620 struct decode_cache *c = &ctxt->decode;
1621 u64 old = c->dst.orig_val64;
1623 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1624 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1625 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1626 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1627 ctxt->eflags &= ~EFLG_ZF;
1629 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1630 (u32) c->regs[VCPU_REGS_RBX];
1632 ctxt->eflags |= EFLG_ZF;
1634 return X86EMUL_CONTINUE;
1637 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1638 struct x86_emulate_ops *ops)
1640 struct decode_cache *c = &ctxt->decode;
1644 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1647 if (c->op_bytes == 4)
1648 c->eip = (u32)c->eip;
1649 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1652 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1656 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1657 struct x86_emulate_ops *ops, int seg)
1659 struct decode_cache *c = &ctxt->decode;
1663 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1665 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1666 if (rc != X86EMUL_CONTINUE)
1669 c->dst.val = c->src.val;
1674 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1675 struct x86_emulate_ops *ops, struct desc_struct *cs,
1676 struct desc_struct *ss)
1678 memset(cs, 0, sizeof(struct desc_struct));
1679 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1680 memset(ss, 0, sizeof(struct desc_struct));
1682 cs->l = 0; /* will be adjusted later */
1683 set_desc_base(cs, 0); /* flat segment */
1684 cs->g = 1; /* 4kb granularity */
1685 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1686 cs->type = 0x0b; /* Read, Execute, Accessed */
1688 cs->dpl = 0; /* will be adjusted later */
1692 set_desc_base(ss, 0); /* flat segment */
1693 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1694 ss->g = 1; /* 4kb granularity */
1696 ss->type = 0x03; /* Read/Write, Accessed */
1697 ss->d = 1; /* 32bit stack segment */
1703 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1705 struct decode_cache *c = &ctxt->decode;
1706 struct desc_struct cs, ss;
1710 /* syscall is not available in real mode */
1711 if (ctxt->mode == X86EMUL_MODE_REAL ||
1712 ctxt->mode == X86EMUL_MODE_VM86)
1713 return emulate_ud(ctxt);
1715 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1717 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1719 cs_sel = (u16)(msr_data & 0xfffc);
1720 ss_sel = (u16)(msr_data + 8);
1722 if (is_long_mode(ctxt->vcpu)) {
1726 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1727 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1728 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1729 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1731 c->regs[VCPU_REGS_RCX] = c->eip;
1732 if (is_long_mode(ctxt->vcpu)) {
1733 #ifdef CONFIG_X86_64
1734 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1736 ops->get_msr(ctxt->vcpu,
1737 ctxt->mode == X86EMUL_MODE_PROT64 ?
1738 MSR_LSTAR : MSR_CSTAR, &msr_data);
1741 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1742 ctxt->eflags &= ~(msr_data | EFLG_RF);
1746 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1747 c->eip = (u32)msr_data;
1749 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1752 return X86EMUL_CONTINUE;
1756 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1758 struct decode_cache *c = &ctxt->decode;
1759 struct desc_struct cs, ss;
1763 /* inject #GP if in real mode */
1764 if (ctxt->mode == X86EMUL_MODE_REAL)
1765 return emulate_gp(ctxt, 0);
1767 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1768 * Therefore, we inject an #UD.
1770 if (ctxt->mode == X86EMUL_MODE_PROT64)
1771 return emulate_ud(ctxt);
1773 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1775 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1776 switch (ctxt->mode) {
1777 case X86EMUL_MODE_PROT32:
1778 if ((msr_data & 0xfffc) == 0x0)
1779 return emulate_gp(ctxt, 0);
1781 case X86EMUL_MODE_PROT64:
1782 if (msr_data == 0x0)
1783 return emulate_gp(ctxt, 0);
1787 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1788 cs_sel = (u16)msr_data;
1789 cs_sel &= ~SELECTOR_RPL_MASK;
1790 ss_sel = cs_sel + 8;
1791 ss_sel &= ~SELECTOR_RPL_MASK;
1792 if (ctxt->mode == X86EMUL_MODE_PROT64
1793 || is_long_mode(ctxt->vcpu)) {
1798 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1799 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1800 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1801 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1803 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1806 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1807 c->regs[VCPU_REGS_RSP] = msr_data;
1809 return X86EMUL_CONTINUE;
1813 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1815 struct decode_cache *c = &ctxt->decode;
1816 struct desc_struct cs, ss;
1821 /* inject #GP if in real mode or Virtual 8086 mode */
1822 if (ctxt->mode == X86EMUL_MODE_REAL ||
1823 ctxt->mode == X86EMUL_MODE_VM86)
1824 return emulate_gp(ctxt, 0);
1826 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1828 if ((c->rex_prefix & 0x8) != 0x0)
1829 usermode = X86EMUL_MODE_PROT64;
1831 usermode = X86EMUL_MODE_PROT32;
1835 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1837 case X86EMUL_MODE_PROT32:
1838 cs_sel = (u16)(msr_data + 16);
1839 if ((msr_data & 0xfffc) == 0x0)
1840 return emulate_gp(ctxt, 0);
1841 ss_sel = (u16)(msr_data + 24);
1843 case X86EMUL_MODE_PROT64:
1844 cs_sel = (u16)(msr_data + 32);
1845 if (msr_data == 0x0)
1846 return emulate_gp(ctxt, 0);
1847 ss_sel = cs_sel + 8;
1852 cs_sel |= SELECTOR_RPL_MASK;
1853 ss_sel |= SELECTOR_RPL_MASK;
1855 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1856 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1857 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1858 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1860 c->eip = c->regs[VCPU_REGS_RDX];
1861 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1863 return X86EMUL_CONTINUE;
1866 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1867 struct x86_emulate_ops *ops)
1870 if (ctxt->mode == X86EMUL_MODE_REAL)
1872 if (ctxt->mode == X86EMUL_MODE_VM86)
1874 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1875 return ops->cpl(ctxt->vcpu) > iopl;
1878 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1879 struct x86_emulate_ops *ops,
1882 struct desc_struct tr_seg;
1885 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
1886 unsigned mask = (1 << len) - 1;
1889 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
1892 if (desc_limit_scaled(&tr_seg) < 103)
1894 base = get_desc_base(&tr_seg);
1895 #ifdef CONFIG_X86_64
1896 base |= ((u64)base3) << 32;
1898 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
1899 if (r != X86EMUL_CONTINUE)
1901 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
1903 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
1905 if (r != X86EMUL_CONTINUE)
1907 if ((perm >> bit_idx) & mask)
1912 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1913 struct x86_emulate_ops *ops,
1919 if (emulator_bad_iopl(ctxt, ops))
1920 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1923 ctxt->perm_ok = true;
1928 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
1929 struct x86_emulate_ops *ops,
1930 struct tss_segment_16 *tss)
1932 struct decode_cache *c = &ctxt->decode;
1935 tss->flag = ctxt->eflags;
1936 tss->ax = c->regs[VCPU_REGS_RAX];
1937 tss->cx = c->regs[VCPU_REGS_RCX];
1938 tss->dx = c->regs[VCPU_REGS_RDX];
1939 tss->bx = c->regs[VCPU_REGS_RBX];
1940 tss->sp = c->regs[VCPU_REGS_RSP];
1941 tss->bp = c->regs[VCPU_REGS_RBP];
1942 tss->si = c->regs[VCPU_REGS_RSI];
1943 tss->di = c->regs[VCPU_REGS_RDI];
1945 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
1946 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1947 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
1948 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
1949 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
1952 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
1953 struct x86_emulate_ops *ops,
1954 struct tss_segment_16 *tss)
1956 struct decode_cache *c = &ctxt->decode;
1960 ctxt->eflags = tss->flag | 2;
1961 c->regs[VCPU_REGS_RAX] = tss->ax;
1962 c->regs[VCPU_REGS_RCX] = tss->cx;
1963 c->regs[VCPU_REGS_RDX] = tss->dx;
1964 c->regs[VCPU_REGS_RBX] = tss->bx;
1965 c->regs[VCPU_REGS_RSP] = tss->sp;
1966 c->regs[VCPU_REGS_RBP] = tss->bp;
1967 c->regs[VCPU_REGS_RSI] = tss->si;
1968 c->regs[VCPU_REGS_RDI] = tss->di;
1971 * SDM says that segment selectors are loaded before segment
1974 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
1975 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
1976 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
1977 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
1978 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
1981 * Now load segment descriptors. If fault happenes at this stage
1982 * it is handled in a context of new task
1984 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
1985 if (ret != X86EMUL_CONTINUE)
1987 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
1988 if (ret != X86EMUL_CONTINUE)
1990 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
1991 if (ret != X86EMUL_CONTINUE)
1993 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
1994 if (ret != X86EMUL_CONTINUE)
1996 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
1997 if (ret != X86EMUL_CONTINUE)
2000 return X86EMUL_CONTINUE;
2003 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2004 struct x86_emulate_ops *ops,
2005 u16 tss_selector, u16 old_tss_sel,
2006 ulong old_tss_base, struct desc_struct *new_desc)
2008 struct tss_segment_16 tss_seg;
2010 u32 new_tss_base = get_desc_base(new_desc);
2012 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2014 if (ret != X86EMUL_CONTINUE)
2015 /* FIXME: need to provide precise fault address */
2018 save_state_to_tss16(ctxt, ops, &tss_seg);
2020 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2022 if (ret != X86EMUL_CONTINUE)
2023 /* FIXME: need to provide precise fault address */
2026 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2028 if (ret != X86EMUL_CONTINUE)
2029 /* FIXME: need to provide precise fault address */
2032 if (old_tss_sel != 0xffff) {
2033 tss_seg.prev_task_link = old_tss_sel;
2035 ret = ops->write_std(new_tss_base,
2036 &tss_seg.prev_task_link,
2037 sizeof tss_seg.prev_task_link,
2038 ctxt->vcpu, &ctxt->exception);
2039 if (ret != X86EMUL_CONTINUE)
2040 /* FIXME: need to provide precise fault address */
2044 return load_state_from_tss16(ctxt, ops, &tss_seg);
2047 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2048 struct x86_emulate_ops *ops,
2049 struct tss_segment_32 *tss)
2051 struct decode_cache *c = &ctxt->decode;
2053 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2055 tss->eflags = ctxt->eflags;
2056 tss->eax = c->regs[VCPU_REGS_RAX];
2057 tss->ecx = c->regs[VCPU_REGS_RCX];
2058 tss->edx = c->regs[VCPU_REGS_RDX];
2059 tss->ebx = c->regs[VCPU_REGS_RBX];
2060 tss->esp = c->regs[VCPU_REGS_RSP];
2061 tss->ebp = c->regs[VCPU_REGS_RBP];
2062 tss->esi = c->regs[VCPU_REGS_RSI];
2063 tss->edi = c->regs[VCPU_REGS_RDI];
2065 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2066 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2067 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2068 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2069 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2070 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2071 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2074 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2075 struct x86_emulate_ops *ops,
2076 struct tss_segment_32 *tss)
2078 struct decode_cache *c = &ctxt->decode;
2081 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2082 return emulate_gp(ctxt, 0);
2084 ctxt->eflags = tss->eflags | 2;
2085 c->regs[VCPU_REGS_RAX] = tss->eax;
2086 c->regs[VCPU_REGS_RCX] = tss->ecx;
2087 c->regs[VCPU_REGS_RDX] = tss->edx;
2088 c->regs[VCPU_REGS_RBX] = tss->ebx;
2089 c->regs[VCPU_REGS_RSP] = tss->esp;
2090 c->regs[VCPU_REGS_RBP] = tss->ebp;
2091 c->regs[VCPU_REGS_RSI] = tss->esi;
2092 c->regs[VCPU_REGS_RDI] = tss->edi;
2095 * SDM says that segment selectors are loaded before segment
2098 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2099 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2100 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2101 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2102 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2103 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2104 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2107 * Now load segment descriptors. If fault happenes at this stage
2108 * it is handled in a context of new task
2110 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2111 if (ret != X86EMUL_CONTINUE)
2113 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2114 if (ret != X86EMUL_CONTINUE)
2116 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2117 if (ret != X86EMUL_CONTINUE)
2119 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2120 if (ret != X86EMUL_CONTINUE)
2122 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2123 if (ret != X86EMUL_CONTINUE)
2125 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2126 if (ret != X86EMUL_CONTINUE)
2128 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2129 if (ret != X86EMUL_CONTINUE)
2132 return X86EMUL_CONTINUE;
2135 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2136 struct x86_emulate_ops *ops,
2137 u16 tss_selector, u16 old_tss_sel,
2138 ulong old_tss_base, struct desc_struct *new_desc)
2140 struct tss_segment_32 tss_seg;
2142 u32 new_tss_base = get_desc_base(new_desc);
2144 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2146 if (ret != X86EMUL_CONTINUE)
2147 /* FIXME: need to provide precise fault address */
2150 save_state_to_tss32(ctxt, ops, &tss_seg);
2152 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2154 if (ret != X86EMUL_CONTINUE)
2155 /* FIXME: need to provide precise fault address */
2158 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2160 if (ret != X86EMUL_CONTINUE)
2161 /* FIXME: need to provide precise fault address */
2164 if (old_tss_sel != 0xffff) {
2165 tss_seg.prev_task_link = old_tss_sel;
2167 ret = ops->write_std(new_tss_base,
2168 &tss_seg.prev_task_link,
2169 sizeof tss_seg.prev_task_link,
2170 ctxt->vcpu, &ctxt->exception);
2171 if (ret != X86EMUL_CONTINUE)
2172 /* FIXME: need to provide precise fault address */
2176 return load_state_from_tss32(ctxt, ops, &tss_seg);
2179 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2180 struct x86_emulate_ops *ops,
2181 u16 tss_selector, int reason,
2182 bool has_error_code, u32 error_code)
2184 struct desc_struct curr_tss_desc, next_tss_desc;
2186 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2187 ulong old_tss_base =
2188 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2191 /* FIXME: old_tss_base == ~0 ? */
2193 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2194 if (ret != X86EMUL_CONTINUE)
2196 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2197 if (ret != X86EMUL_CONTINUE)
2200 /* FIXME: check that next_tss_desc is tss */
2202 if (reason != TASK_SWITCH_IRET) {
2203 if ((tss_selector & 3) > next_tss_desc.dpl ||
2204 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2205 return emulate_gp(ctxt, 0);
2208 desc_limit = desc_limit_scaled(&next_tss_desc);
2209 if (!next_tss_desc.p ||
2210 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2211 desc_limit < 0x2b)) {
2212 emulate_ts(ctxt, tss_selector & 0xfffc);
2213 return X86EMUL_PROPAGATE_FAULT;
2216 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2217 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2218 write_segment_descriptor(ctxt, ops, old_tss_sel,
2222 if (reason == TASK_SWITCH_IRET)
2223 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2225 /* set back link to prev task only if NT bit is set in eflags
2226 note that old_tss_sel is not used afetr this point */
2227 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2228 old_tss_sel = 0xffff;
2230 if (next_tss_desc.type & 8)
2231 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2232 old_tss_base, &next_tss_desc);
2234 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2235 old_tss_base, &next_tss_desc);
2236 if (ret != X86EMUL_CONTINUE)
2239 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2240 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2242 if (reason != TASK_SWITCH_IRET) {
2243 next_tss_desc.type |= (1 << 1); /* set busy flag */
2244 write_segment_descriptor(ctxt, ops, tss_selector,
2248 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2249 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2250 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2252 if (has_error_code) {
2253 struct decode_cache *c = &ctxt->decode;
2255 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2257 c->src.val = (unsigned long) error_code;
2258 emulate_push(ctxt, ops);
2264 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2265 u16 tss_selector, int reason,
2266 bool has_error_code, u32 error_code)
2268 struct x86_emulate_ops *ops = ctxt->ops;
2269 struct decode_cache *c = &ctxt->decode;
2273 c->dst.type = OP_NONE;
2275 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2276 has_error_code, error_code);
2278 if (rc == X86EMUL_CONTINUE) {
2279 rc = writeback(ctxt, ops);
2280 if (rc == X86EMUL_CONTINUE)
2284 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2287 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2288 int reg, struct operand *op)
2290 struct decode_cache *c = &ctxt->decode;
2291 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2293 register_address_increment(c, &c->regs[reg], df * op->bytes);
2294 op->addr.mem.ea = register_address(c, c->regs[reg]);
2295 op->addr.mem.seg = seg;
2298 static int em_push(struct x86_emulate_ctxt *ctxt)
2300 emulate_push(ctxt, ctxt->ops);
2301 return X86EMUL_CONTINUE;
2304 static int em_das(struct x86_emulate_ctxt *ctxt)
2306 struct decode_cache *c = &ctxt->decode;
2308 bool af, cf, old_cf;
2310 cf = ctxt->eflags & X86_EFLAGS_CF;
2316 af = ctxt->eflags & X86_EFLAGS_AF;
2317 if ((al & 0x0f) > 9 || af) {
2319 cf = old_cf | (al >= 250);
2324 if (old_al > 0x99 || old_cf) {
2330 /* Set PF, ZF, SF */
2331 c->src.type = OP_IMM;
2334 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2335 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2337 ctxt->eflags |= X86_EFLAGS_CF;
2339 ctxt->eflags |= X86_EFLAGS_AF;
2340 return X86EMUL_CONTINUE;
2343 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2345 struct decode_cache *c = &ctxt->decode;
2350 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2353 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2354 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2355 return X86EMUL_CONTINUE;
2358 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2360 c->src.val = old_cs;
2361 emulate_push(ctxt, ctxt->ops);
2362 rc = writeback(ctxt, ctxt->ops);
2363 if (rc != X86EMUL_CONTINUE)
2366 c->src.val = old_eip;
2367 emulate_push(ctxt, ctxt->ops);
2368 rc = writeback(ctxt, ctxt->ops);
2369 if (rc != X86EMUL_CONTINUE)
2372 c->dst.type = OP_NONE;
2374 return X86EMUL_CONTINUE;
2377 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2379 struct decode_cache *c = &ctxt->decode;
2382 c->dst.type = OP_REG;
2383 c->dst.addr.reg = &c->eip;
2384 c->dst.bytes = c->op_bytes;
2385 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2386 if (rc != X86EMUL_CONTINUE)
2388 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2389 return X86EMUL_CONTINUE;
2392 static int em_imul(struct x86_emulate_ctxt *ctxt)
2394 struct decode_cache *c = &ctxt->decode;
2396 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2397 return X86EMUL_CONTINUE;
2400 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2402 struct decode_cache *c = &ctxt->decode;
2404 c->dst.val = c->src2.val;
2405 return em_imul(ctxt);
2408 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2410 struct decode_cache *c = &ctxt->decode;
2412 c->dst.type = OP_REG;
2413 c->dst.bytes = c->src.bytes;
2414 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2415 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2417 return X86EMUL_CONTINUE;
2420 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2422 unsigned cpl = ctxt->ops->cpl(ctxt->vcpu);
2423 struct decode_cache *c = &ctxt->decode;
2426 if (cpl > 0 && (ctxt->ops->get_cr(4, ctxt->vcpu) & X86_CR4_TSD))
2427 return emulate_gp(ctxt, 0);
2428 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2429 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2430 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2431 return X86EMUL_CONTINUE;
2434 static int em_mov(struct x86_emulate_ctxt *ctxt)
2436 struct decode_cache *c = &ctxt->decode;
2437 c->dst.val = c->src.val;
2438 return X86EMUL_CONTINUE;
2441 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2443 struct decode_cache *c = &ctxt->decode;
2444 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2445 return X86EMUL_CONTINUE;
2448 #define D(_y) { .flags = (_y) }
2449 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2450 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2451 .check_perm = (_p) }
2453 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2454 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2455 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2456 #define II(_f, _e, _i) \
2457 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2458 #define IIP(_f, _e, _i, _p) \
2459 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2460 .check_perm = (_p) }
2461 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2463 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2464 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2466 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2467 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2468 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2471 static struct opcode group1[] = {
2475 static struct opcode group1A[] = {
2476 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2479 static struct opcode group3[] = {
2480 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2481 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2482 X4(D(SrcMem | ModRM)),
2485 static struct opcode group4[] = {
2486 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2490 static struct opcode group5[] = {
2491 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2492 D(SrcMem | ModRM | Stack),
2493 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2494 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2495 D(SrcMem | ModRM | Stack), N,
2498 static struct group_dual group7 = { {
2499 N, N, DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2500 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2501 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2502 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2504 D(SrcNone | ModRM | Priv | VendorSpecific), N,
2505 N, D(SrcNone | ModRM | Priv | VendorSpecific),
2506 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2507 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), N,
2510 static struct opcode group8[] = {
2512 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2513 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2516 static struct group_dual group9 = { {
2517 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2519 N, N, N, N, N, N, N, N,
2522 static struct opcode group11[] = {
2523 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2526 static struct gprefix pfx_0f_6f_0f_7f = {
2527 N, N, N, I(Sse, em_movdqu),
2530 static struct opcode opcode_table[256] = {
2533 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2536 D(ImplicitOps | Stack | No64), N,
2539 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2542 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2546 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2554 X8(I(SrcReg | Stack, em_push)),
2556 X8(D(DstReg | Stack)),
2558 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2559 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2562 I(SrcImm | Mov | Stack, em_push),
2563 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2564 I(SrcImmByte | Mov | Stack, em_push),
2565 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2566 D2bv(DstDI | Mov | String), /* insb, insw/insd */
2567 D2bv(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
2571 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2572 G(DstMem | SrcImm | ModRM | Group, group1),
2573 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2574 G(DstMem | SrcImmByte | ModRM | Group, group1),
2575 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2577 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2578 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2579 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2580 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2582 X8(D(SrcAcc | DstReg)),
2584 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2585 I(SrcImmFAddr | No64, em_call_far), N,
2586 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2588 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2589 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2590 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2591 D2bv(SrcSI | DstDI | String),
2593 D2bv(DstAcc | SrcImm),
2594 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2595 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2596 D2bv(SrcAcc | DstDI | String),
2598 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2600 X8(I(DstReg | SrcImm | Mov, em_mov)),
2602 D2bv(DstMem | SrcImmByte | ModRM),
2603 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2604 D(ImplicitOps | Stack),
2605 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2606 G(ByteOp, group11), G(0, group11),
2608 N, N, N, D(ImplicitOps | Stack),
2609 D(ImplicitOps), DI(SrcImmByte, intn),
2610 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2612 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2615 N, N, N, N, N, N, N, N,
2618 D2bv(SrcImmUByte | DstAcc), D2bv(SrcAcc | DstImmUByte),
2620 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2621 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2622 D2bv(SrcNone | DstAcc), D2bv(SrcAcc | ImplicitOps),
2625 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2626 G(ByteOp, group3), G(0, group3),
2628 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2629 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2632 static struct opcode twobyte_table[256] = {
2634 N, GD(0, &group7), N, N,
2635 N, D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv), N,
2636 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2637 N, D(ImplicitOps | ModRM), N, N,
2639 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2641 D(ModRM | DstMem | Priv | Op3264), D(ModRM | DstMem | Priv | Op3264),
2642 D(ModRM | SrcMem | Priv | Op3264), D(ModRM | SrcMem | Priv | Op3264),
2644 N, N, N, N, N, N, N, N,
2646 D(ImplicitOps | Priv), II(ImplicitOps, em_rdtsc, rdtsc),
2647 D(ImplicitOps | Priv), N,
2648 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2650 N, N, N, N, N, N, N, N,
2652 X16(D(DstReg | SrcMem | ModRM | Mov)),
2654 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2659 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
2664 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
2668 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
2670 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2671 N, D(DstMem | SrcReg | ModRM | BitOp),
2672 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2673 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
2675 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2676 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
2677 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2678 D(DstMem | SrcReg | Src2CL | ModRM),
2679 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
2681 D2bv(DstMem | SrcReg | ModRM | Lock),
2682 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2683 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
2684 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2687 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2688 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
2689 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2691 D2bv(DstMem | SrcReg | ModRM | Lock),
2692 N, D(DstMem | SrcReg | ModRM | Mov),
2693 N, N, N, GD(0, &group9),
2694 N, N, N, N, N, N, N, N,
2696 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2698 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2700 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
2714 static unsigned imm_size(struct decode_cache *c)
2718 size = (c->d & ByteOp) ? 1 : c->op_bytes;
2724 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
2725 unsigned size, bool sign_extension)
2727 struct decode_cache *c = &ctxt->decode;
2728 struct x86_emulate_ops *ops = ctxt->ops;
2729 int rc = X86EMUL_CONTINUE;
2733 op->addr.mem.ea = c->eip;
2734 /* NB. Immediates are sign-extended as necessary. */
2735 switch (op->bytes) {
2737 op->val = insn_fetch(s8, 1, c->eip);
2740 op->val = insn_fetch(s16, 2, c->eip);
2743 op->val = insn_fetch(s32, 4, c->eip);
2746 if (!sign_extension) {
2747 switch (op->bytes) {
2755 op->val &= 0xffffffff;
2764 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
2766 struct x86_emulate_ops *ops = ctxt->ops;
2767 struct decode_cache *c = &ctxt->decode;
2768 int rc = X86EMUL_CONTINUE;
2769 int mode = ctxt->mode;
2770 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
2771 bool op_prefix = false;
2772 struct opcode opcode, *g_mod012, *g_mod3;
2773 struct operand memop = { .type = OP_NONE };
2776 c->fetch.start = c->eip;
2777 c->fetch.end = c->fetch.start + insn_len;
2779 memcpy(c->fetch.data, insn, insn_len);
2780 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
2783 case X86EMUL_MODE_REAL:
2784 case X86EMUL_MODE_VM86:
2785 case X86EMUL_MODE_PROT16:
2786 def_op_bytes = def_ad_bytes = 2;
2788 case X86EMUL_MODE_PROT32:
2789 def_op_bytes = def_ad_bytes = 4;
2791 #ifdef CONFIG_X86_64
2792 case X86EMUL_MODE_PROT64:
2801 c->op_bytes = def_op_bytes;
2802 c->ad_bytes = def_ad_bytes;
2804 /* Legacy prefixes. */
2806 switch (c->b = insn_fetch(u8, 1, c->eip)) {
2807 case 0x66: /* operand-size override */
2809 /* switch between 2/4 bytes */
2810 c->op_bytes = def_op_bytes ^ 6;
2812 case 0x67: /* address-size override */
2813 if (mode == X86EMUL_MODE_PROT64)
2814 /* switch between 4/8 bytes */
2815 c->ad_bytes = def_ad_bytes ^ 12;
2817 /* switch between 2/4 bytes */
2818 c->ad_bytes = def_ad_bytes ^ 6;
2820 case 0x26: /* ES override */
2821 case 0x2e: /* CS override */
2822 case 0x36: /* SS override */
2823 case 0x3e: /* DS override */
2824 set_seg_override(c, (c->b >> 3) & 3);
2826 case 0x64: /* FS override */
2827 case 0x65: /* GS override */
2828 set_seg_override(c, c->b & 7);
2830 case 0x40 ... 0x4f: /* REX */
2831 if (mode != X86EMUL_MODE_PROT64)
2833 c->rex_prefix = c->b;
2835 case 0xf0: /* LOCK */
2838 case 0xf2: /* REPNE/REPNZ */
2839 case 0xf3: /* REP/REPE/REPZ */
2840 c->rep_prefix = c->b;
2846 /* Any legacy prefix after a REX prefix nullifies its effect. */
2854 if (c->rex_prefix & 8)
2855 c->op_bytes = 8; /* REX.W */
2857 /* Opcode byte(s). */
2858 opcode = opcode_table[c->b];
2859 /* Two-byte opcode? */
2862 c->b = insn_fetch(u8, 1, c->eip);
2863 opcode = twobyte_table[c->b];
2865 c->d = opcode.flags;
2868 dual = c->d & GroupDual;
2869 c->modrm = insn_fetch(u8, 1, c->eip);
2872 if (c->d & GroupDual) {
2873 g_mod012 = opcode.u.gdual->mod012;
2874 g_mod3 = opcode.u.gdual->mod3;
2876 g_mod012 = g_mod3 = opcode.u.group;
2878 c->d &= ~(Group | GroupDual);
2880 goffset = (c->modrm >> 3) & 7;
2882 if ((c->modrm >> 6) == 3)
2883 opcode = g_mod3[goffset];
2885 opcode = g_mod012[goffset];
2886 c->d |= opcode.flags;
2889 if (c->d & Prefix) {
2890 if (c->rep_prefix && op_prefix)
2891 return X86EMUL_UNHANDLEABLE;
2892 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
2893 switch (simd_prefix) {
2894 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
2895 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
2896 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
2897 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
2899 c->d |= opcode.flags;
2902 c->execute = opcode.u.execute;
2903 c->check_perm = opcode.check_perm;
2904 c->intercept = opcode.intercept;
2907 if (c->d == 0 || (c->d & Undefined))
2910 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
2913 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
2916 if (c->d & Op3264) {
2917 if (mode == X86EMUL_MODE_PROT64)
2926 /* ModRM and SIB bytes. */
2928 rc = decode_modrm(ctxt, ops, &memop);
2929 if (!c->has_seg_override)
2930 set_seg_override(c, c->modrm_seg);
2931 } else if (c->d & MemAbs)
2932 rc = decode_abs(ctxt, ops, &memop);
2933 if (rc != X86EMUL_CONTINUE)
2936 if (!c->has_seg_override)
2937 set_seg_override(c, VCPU_SREG_DS);
2939 memop.addr.mem.seg = seg_override(ctxt, ops, c);
2941 if (memop.type == OP_MEM && c->ad_bytes != 8)
2942 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
2944 if (memop.type == OP_MEM && c->rip_relative)
2945 memop.addr.mem.ea += c->eip;
2948 * Decode and fetch the source operand: register, memory
2951 switch (c->d & SrcMask) {
2955 decode_register_operand(ctxt, &c->src, c, 0);
2964 memop.bytes = (c->d & ByteOp) ? 1 :
2970 rc = decode_imm(ctxt, &c->src, 2, false);
2973 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
2976 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
2979 rc = decode_imm(ctxt, &c->src, 1, true);
2982 rc = decode_imm(ctxt, &c->src, 1, false);
2985 c->src.type = OP_REG;
2986 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2987 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
2988 fetch_register_operand(&c->src);
2995 c->src.type = OP_MEM;
2996 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2997 c->src.addr.mem.ea =
2998 register_address(c, c->regs[VCPU_REGS_RSI]);
2999 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3003 c->src.type = OP_IMM;
3004 c->src.addr.mem.ea = c->eip;
3005 c->src.bytes = c->op_bytes + 2;
3006 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3009 memop.bytes = c->op_bytes + 2;
3014 if (rc != X86EMUL_CONTINUE)
3018 * Decode and fetch the second source operand: register, memory
3021 switch (c->d & Src2Mask) {
3026 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3029 rc = decode_imm(ctxt, &c->src2, 1, true);
3036 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3040 if (rc != X86EMUL_CONTINUE)
3043 /* Decode and fetch the destination operand: register or memory. */
3044 switch (c->d & DstMask) {
3046 decode_register_operand(ctxt, &c->dst, c,
3047 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3050 c->dst.type = OP_IMM;
3051 c->dst.addr.mem.ea = c->eip;
3053 c->dst.val = insn_fetch(u8, 1, c->eip);
3058 if ((c->d & DstMask) == DstMem64)
3061 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3063 fetch_bit_operand(c);
3064 c->dst.orig_val = c->dst.val;
3067 c->dst.type = OP_REG;
3068 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3069 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3070 fetch_register_operand(&c->dst);
3071 c->dst.orig_val = c->dst.val;
3074 c->dst.type = OP_MEM;
3075 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3076 c->dst.addr.mem.ea =
3077 register_address(c, c->regs[VCPU_REGS_RDI]);
3078 c->dst.addr.mem.seg = VCPU_SREG_ES;
3082 /* Special instructions do their own operand decoding. */
3084 c->dst.type = OP_NONE; /* Disable writeback. */
3089 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3092 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3094 struct decode_cache *c = &ctxt->decode;
3096 /* The second termination condition only applies for REPE
3097 * and REPNE. Test if the repeat string operation prefix is
3098 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3099 * corresponding termination condition according to:
3100 * - if REPE/REPZ and ZF = 0 then done
3101 * - if REPNE/REPNZ and ZF = 1 then done
3103 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3104 (c->b == 0xae) || (c->b == 0xaf))
3105 && (((c->rep_prefix == REPE_PREFIX) &&
3106 ((ctxt->eflags & EFLG_ZF) == 0))
3107 || ((c->rep_prefix == REPNE_PREFIX) &&
3108 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3115 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3117 struct x86_emulate_ops *ops = ctxt->ops;
3119 struct decode_cache *c = &ctxt->decode;
3120 int rc = X86EMUL_CONTINUE;
3121 int saved_dst_type = c->dst.type;
3122 int irq; /* Used for int 3, int, and into */
3124 ctxt->decode.mem_read.pos = 0;
3126 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3127 rc = emulate_ud(ctxt);
3131 /* LOCK prefix is allowed only with some instructions */
3132 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3133 rc = emulate_ud(ctxt);
3137 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3138 rc = emulate_ud(ctxt);
3143 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3144 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3145 rc = emulate_ud(ctxt);
3149 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3150 rc = emulate_nm(ctxt);
3154 if (unlikely(ctxt->guest_mode) && c->intercept) {
3155 rc = emulator_check_intercept(ctxt, c->intercept,
3156 X86_ICPT_PRE_EXCEPT);
3157 if (rc != X86EMUL_CONTINUE)
3161 /* Privileged instruction can be executed only in CPL=0 */
3162 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3163 rc = emulate_gp(ctxt, 0);
3167 /* Instruction can only be executed in protected mode */
3168 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3169 rc = emulate_ud(ctxt);
3173 /* Do instruction specific permission checks */
3174 if (c->check_perm) {
3175 rc = c->check_perm(ctxt);
3176 if (rc != X86EMUL_CONTINUE)
3180 if (unlikely(ctxt->guest_mode) && c->intercept) {
3181 rc = emulator_check_intercept(ctxt, c->intercept,
3182 X86_ICPT_POST_EXCEPT);
3183 if (rc != X86EMUL_CONTINUE)
3187 if (c->rep_prefix && (c->d & String)) {
3188 /* All REP prefixes have the same first termination condition */
3189 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3195 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3196 rc = read_emulated(ctxt, ops, linear(ctxt, c->src.addr.mem),
3197 c->src.valptr, c->src.bytes);
3198 if (rc != X86EMUL_CONTINUE)
3200 c->src.orig_val64 = c->src.val64;
3203 if (c->src2.type == OP_MEM) {
3204 rc = read_emulated(ctxt, ops, linear(ctxt, c->src2.addr.mem),
3205 &c->src2.val, c->src2.bytes);
3206 if (rc != X86EMUL_CONTINUE)
3210 if ((c->d & DstMask) == ImplicitOps)
3214 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3215 /* optimisation - avoid slow emulated read if Mov */
3216 rc = read_emulated(ctxt, ops, linear(ctxt, c->dst.addr.mem),
3217 &c->dst.val, c->dst.bytes);
3218 if (rc != X86EMUL_CONTINUE)
3221 c->dst.orig_val = c->dst.val;
3225 if (unlikely(ctxt->guest_mode) && c->intercept) {
3226 rc = emulator_check_intercept(ctxt, c->intercept,
3227 X86_ICPT_POST_MEMACCESS);
3228 if (rc != X86EMUL_CONTINUE)
3233 rc = c->execute(ctxt);
3234 if (rc != X86EMUL_CONTINUE)
3245 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3247 case 0x06: /* push es */
3248 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3250 case 0x07: /* pop es */
3251 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3255 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3257 case 0x0e: /* push cs */
3258 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3262 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3264 case 0x16: /* push ss */
3265 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3267 case 0x17: /* pop ss */
3268 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3272 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3274 case 0x1e: /* push ds */
3275 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3277 case 0x1f: /* pop ds */
3278 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3282 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3286 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3290 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3294 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3296 case 0x40 ... 0x47: /* inc r16/r32 */
3297 emulate_1op("inc", c->dst, ctxt->eflags);
3299 case 0x48 ... 0x4f: /* dec r16/r32 */
3300 emulate_1op("dec", c->dst, ctxt->eflags);
3302 case 0x58 ... 0x5f: /* pop reg */
3304 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3306 case 0x60: /* pusha */
3307 rc = emulate_pusha(ctxt, ops);
3309 case 0x61: /* popa */
3310 rc = emulate_popa(ctxt, ops);
3312 case 0x63: /* movsxd */
3313 if (ctxt->mode != X86EMUL_MODE_PROT64)
3314 goto cannot_emulate;
3315 c->dst.val = (s32) c->src.val;
3317 case 0x6c: /* insb */
3318 case 0x6d: /* insw/insd */
3319 c->src.val = c->regs[VCPU_REGS_RDX];
3321 case 0x6e: /* outsb */
3322 case 0x6f: /* outsw/outsd */
3323 c->dst.val = c->regs[VCPU_REGS_RDX];
3326 case 0x70 ... 0x7f: /* jcc (short) */
3327 if (test_cc(c->b, ctxt->eflags))
3328 jmp_rel(c, c->src.val);
3330 case 0x80 ... 0x83: /* Grp1 */
3331 switch (c->modrm_reg) {
3352 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3354 case 0x86 ... 0x87: /* xchg */
3356 /* Write back the register source. */
3357 c->src.val = c->dst.val;
3358 write_register_operand(&c->src);
3360 * Write back the memory destination with implicit LOCK
3363 c->dst.val = c->src.orig_val;
3366 case 0x8c: /* mov r/m, sreg */
3367 if (c->modrm_reg > VCPU_SREG_GS) {
3368 rc = emulate_ud(ctxt);
3371 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3373 case 0x8d: /* lea r16/r32, m */
3374 c->dst.val = c->src.addr.mem.ea;
3376 case 0x8e: { /* mov seg, r/m16 */
3381 if (c->modrm_reg == VCPU_SREG_CS ||
3382 c->modrm_reg > VCPU_SREG_GS) {
3383 rc = emulate_ud(ctxt);
3387 if (c->modrm_reg == VCPU_SREG_SS)
3388 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3390 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3392 c->dst.type = OP_NONE; /* Disable writeback. */
3395 case 0x8f: /* pop (sole member of Grp1a) */
3396 rc = emulate_grp1a(ctxt, ops);
3398 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3399 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3402 case 0x98: /* cbw/cwde/cdqe */
3403 switch (c->op_bytes) {
3404 case 2: c->dst.val = (s8)c->dst.val; break;
3405 case 4: c->dst.val = (s16)c->dst.val; break;
3406 case 8: c->dst.val = (s32)c->dst.val; break;
3409 case 0x9c: /* pushf */
3410 c->src.val = (unsigned long) ctxt->eflags;
3411 emulate_push(ctxt, ops);
3413 case 0x9d: /* popf */
3414 c->dst.type = OP_REG;
3415 c->dst.addr.reg = &ctxt->eflags;
3416 c->dst.bytes = c->op_bytes;
3417 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3419 case 0xa6 ... 0xa7: /* cmps */
3420 c->dst.type = OP_NONE; /* Disable writeback. */
3422 case 0xa8 ... 0xa9: /* test ax, imm */
3424 case 0xae ... 0xaf: /* scas */
3429 case 0xc3: /* ret */
3430 c->dst.type = OP_REG;
3431 c->dst.addr.reg = &c->eip;
3432 c->dst.bytes = c->op_bytes;
3433 goto pop_instruction;
3434 case 0xc4: /* les */
3435 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3437 case 0xc5: /* lds */
3438 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3440 case 0xcb: /* ret far */
3441 rc = emulate_ret_far(ctxt, ops);
3443 case 0xcc: /* int3 */
3446 case 0xcd: /* int n */
3449 rc = emulate_int(ctxt, ops, irq);
3451 case 0xce: /* into */
3452 if (ctxt->eflags & EFLG_OF) {
3457 case 0xcf: /* iret */
3458 rc = emulate_iret(ctxt, ops);
3460 case 0xd0 ... 0xd1: /* Grp2 */
3463 case 0xd2 ... 0xd3: /* Grp2 */
3464 c->src.val = c->regs[VCPU_REGS_RCX];
3467 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3468 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3469 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3470 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3471 jmp_rel(c, c->src.val);
3473 case 0xe3: /* jcxz/jecxz/jrcxz */
3474 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3475 jmp_rel(c, c->src.val);
3477 case 0xe4: /* inb */
3480 case 0xe6: /* outb */
3481 case 0xe7: /* out */
3483 case 0xe8: /* call (near) */ {
3484 long int rel = c->src.val;
3485 c->src.val = (unsigned long) c->eip;
3487 emulate_push(ctxt, ops);
3490 case 0xe9: /* jmp rel */
3492 case 0xea: { /* jmp far */
3495 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3497 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3501 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3505 jmp: /* jmp rel short */
3506 jmp_rel(c, c->src.val);
3507 c->dst.type = OP_NONE; /* Disable writeback. */
3509 case 0xec: /* in al,dx */
3510 case 0xed: /* in (e/r)ax,dx */
3511 c->src.val = c->regs[VCPU_REGS_RDX];
3513 c->dst.bytes = min(c->dst.bytes, 4u);
3514 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3515 rc = emulate_gp(ctxt, 0);
3518 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3520 goto done; /* IO is needed */
3522 case 0xee: /* out dx,al */
3523 case 0xef: /* out dx,(e/r)ax */
3524 c->dst.val = c->regs[VCPU_REGS_RDX];
3526 c->src.bytes = min(c->src.bytes, 4u);
3527 if (!emulator_io_permited(ctxt, ops, c->dst.val,
3529 rc = emulate_gp(ctxt, 0);
3532 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3533 &c->src.val, 1, ctxt->vcpu);
3534 c->dst.type = OP_NONE; /* Disable writeback. */
3536 case 0xf4: /* hlt */
3537 ctxt->vcpu->arch.halt_request = 1;
3539 case 0xf5: /* cmc */
3540 /* complement carry flag from eflags reg */
3541 ctxt->eflags ^= EFLG_CF;
3543 case 0xf6 ... 0xf7: /* Grp3 */
3544 rc = emulate_grp3(ctxt, ops);
3546 case 0xf8: /* clc */
3547 ctxt->eflags &= ~EFLG_CF;
3549 case 0xf9: /* stc */
3550 ctxt->eflags |= EFLG_CF;
3552 case 0xfa: /* cli */
3553 if (emulator_bad_iopl(ctxt, ops)) {
3554 rc = emulate_gp(ctxt, 0);
3557 ctxt->eflags &= ~X86_EFLAGS_IF;
3559 case 0xfb: /* sti */
3560 if (emulator_bad_iopl(ctxt, ops)) {
3561 rc = emulate_gp(ctxt, 0);
3564 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3565 ctxt->eflags |= X86_EFLAGS_IF;
3568 case 0xfc: /* cld */
3569 ctxt->eflags &= ~EFLG_DF;
3571 case 0xfd: /* std */
3572 ctxt->eflags |= EFLG_DF;
3574 case 0xfe: /* Grp4 */
3576 rc = emulate_grp45(ctxt, ops);
3578 case 0xff: /* Grp5 */
3579 if (c->modrm_reg == 5)
3583 goto cannot_emulate;
3586 if (rc != X86EMUL_CONTINUE)
3590 rc = writeback(ctxt, ops);
3591 if (rc != X86EMUL_CONTINUE)
3595 * restore dst type in case the decoding will be reused
3596 * (happens for string instruction )
3598 c->dst.type = saved_dst_type;
3600 if ((c->d & SrcMask) == SrcSI)
3601 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3602 VCPU_REGS_RSI, &c->src);
3604 if ((c->d & DstMask) == DstDI)
3605 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3608 if (c->rep_prefix && (c->d & String)) {
3609 struct read_cache *r = &ctxt->decode.io_read;
3610 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3612 if (!string_insn_completed(ctxt)) {
3614 * Re-enter guest when pio read ahead buffer is empty
3615 * or, if it is not used, after each 1024 iteration.
3617 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3618 (r->end == 0 || r->end != r->pos)) {
3620 * Reset read cache. Usually happens before
3621 * decode, but since instruction is restarted
3622 * we have to do it here.
3624 ctxt->decode.mem_read.end = 0;
3625 return EMULATION_RESTART;
3627 goto done; /* skip rip writeback */
3634 if (rc == X86EMUL_PROPAGATE_FAULT)
3635 ctxt->have_exception = true;
3636 if (rc == X86EMUL_INTERCEPTED)
3637 return EMULATION_INTERCEPTED;
3639 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3643 case 0x01: /* lgdt, lidt, lmsw */
3644 switch (c->modrm_reg) {
3646 unsigned long address;
3648 case 0: /* vmcall */
3649 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3650 goto cannot_emulate;
3652 rc = kvm_fix_hypercall(ctxt->vcpu);
3653 if (rc != X86EMUL_CONTINUE)
3656 /* Let the processor re-execute the fixed hypercall */
3658 /* Disable writeback. */
3659 c->dst.type = OP_NONE;
3662 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3663 &size, &address, c->op_bytes);
3664 if (rc != X86EMUL_CONTINUE)
3666 realmode_lgdt(ctxt->vcpu, size, address);
3667 /* Disable writeback. */
3668 c->dst.type = OP_NONE;
3670 case 3: /* lidt/vmmcall */
3671 if (c->modrm_mod == 3) {
3672 switch (c->modrm_rm) {
3674 rc = kvm_fix_hypercall(ctxt->vcpu);
3677 goto cannot_emulate;
3680 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3683 if (rc != X86EMUL_CONTINUE)
3685 realmode_lidt(ctxt->vcpu, size, address);
3687 /* Disable writeback. */
3688 c->dst.type = OP_NONE;
3692 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3695 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
3696 (c->src.val & 0x0f), ctxt->vcpu);
3697 c->dst.type = OP_NONE;
3699 case 5: /* not defined */
3701 rc = X86EMUL_PROPAGATE_FAULT;
3704 emulate_invlpg(ctxt->vcpu,
3705 linear(ctxt, c->src.addr.mem));
3706 /* Disable writeback. */
3707 c->dst.type = OP_NONE;
3710 goto cannot_emulate;
3713 case 0x05: /* syscall */
3714 rc = emulate_syscall(ctxt, ops);
3717 emulate_clts(ctxt->vcpu);
3719 case 0x09: /* wbinvd */
3720 kvm_emulate_wbinvd(ctxt->vcpu);
3722 case 0x08: /* invd */
3723 case 0x0d: /* GrpP (prefetch) */
3724 case 0x18: /* Grp16 (prefetch/nop) */
3726 case 0x20: /* mov cr, reg */
3727 switch (c->modrm_reg) {
3732 rc = X86EMUL_PROPAGATE_FAULT;
3735 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3737 case 0x21: /* mov from dr to reg */
3738 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3739 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3741 rc = X86EMUL_PROPAGATE_FAULT;
3744 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
3746 case 0x22: /* mov reg, cr */
3747 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
3748 emulate_gp(ctxt, 0);
3749 rc = X86EMUL_PROPAGATE_FAULT;
3752 c->dst.type = OP_NONE;
3754 case 0x23: /* mov from reg to dr */
3755 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3756 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3758 rc = X86EMUL_PROPAGATE_FAULT;
3762 if (ops->set_dr(c->modrm_reg, c->src.val &
3763 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3764 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3765 /* #UD condition is already handled by the code above */
3766 emulate_gp(ctxt, 0);
3767 rc = X86EMUL_PROPAGATE_FAULT;
3771 c->dst.type = OP_NONE; /* no writeback */
3775 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3776 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3777 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3778 emulate_gp(ctxt, 0);
3779 rc = X86EMUL_PROPAGATE_FAULT;
3782 rc = X86EMUL_CONTINUE;
3786 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3787 emulate_gp(ctxt, 0);
3788 rc = X86EMUL_PROPAGATE_FAULT;
3791 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3792 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3794 rc = X86EMUL_CONTINUE;
3796 case 0x34: /* sysenter */
3797 rc = emulate_sysenter(ctxt, ops);
3799 case 0x35: /* sysexit */
3800 rc = emulate_sysexit(ctxt, ops);
3802 case 0x40 ... 0x4f: /* cmov */
3803 c->dst.val = c->dst.orig_val = c->src.val;
3804 if (!test_cc(c->b, ctxt->eflags))
3805 c->dst.type = OP_NONE; /* no writeback */
3807 case 0x80 ... 0x8f: /* jnz rel, etc*/
3808 if (test_cc(c->b, ctxt->eflags))
3809 jmp_rel(c, c->src.val);
3811 case 0x90 ... 0x9f: /* setcc r/m8 */
3812 c->dst.val = test_cc(c->b, ctxt->eflags);
3814 case 0xa0: /* push fs */
3815 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3817 case 0xa1: /* pop fs */
3818 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3822 c->dst.type = OP_NONE;
3823 /* only subword offset */
3824 c->src.val &= (c->dst.bytes << 3) - 1;
3825 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3827 case 0xa4: /* shld imm8, r, r/m */
3828 case 0xa5: /* shld cl, r, r/m */
3829 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3831 case 0xa8: /* push gs */
3832 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3834 case 0xa9: /* pop gs */
3835 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3839 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3841 case 0xac: /* shrd imm8, r, r/m */
3842 case 0xad: /* shrd cl, r, r/m */
3843 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3845 case 0xae: /* clflush */
3847 case 0xb0 ... 0xb1: /* cmpxchg */
3849 * Save real source value, then compare EAX against
3852 c->src.orig_val = c->src.val;
3853 c->src.val = c->regs[VCPU_REGS_RAX];
3854 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3855 if (ctxt->eflags & EFLG_ZF) {
3856 /* Success: write back to memory. */
3857 c->dst.val = c->src.orig_val;
3859 /* Failure: write the value we saw to EAX. */
3860 c->dst.type = OP_REG;
3861 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3864 case 0xb2: /* lss */
3865 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
3869 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3871 case 0xb4: /* lfs */
3872 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
3874 case 0xb5: /* lgs */
3875 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
3877 case 0xb6 ... 0xb7: /* movzx */
3878 c->dst.bytes = c->op_bytes;
3879 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3882 case 0xba: /* Grp8 */
3883 switch (c->modrm_reg & 3) {
3896 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3898 case 0xbc: { /* bsf */
3900 __asm__ ("bsf %2, %0; setz %1"
3901 : "=r"(c->dst.val), "=q"(zf)
3903 ctxt->eflags &= ~X86_EFLAGS_ZF;
3905 ctxt->eflags |= X86_EFLAGS_ZF;
3906 c->dst.type = OP_NONE; /* Disable writeback. */
3910 case 0xbd: { /* bsr */
3912 __asm__ ("bsr %2, %0; setz %1"
3913 : "=r"(c->dst.val), "=q"(zf)
3915 ctxt->eflags &= ~X86_EFLAGS_ZF;
3917 ctxt->eflags |= X86_EFLAGS_ZF;
3918 c->dst.type = OP_NONE; /* Disable writeback. */
3922 case 0xbe ... 0xbf: /* movsx */
3923 c->dst.bytes = c->op_bytes;
3924 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3927 case 0xc0 ... 0xc1: /* xadd */
3928 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3929 /* Write back the register source. */
3930 c->src.val = c->dst.orig_val;
3931 write_register_operand(&c->src);
3933 case 0xc3: /* movnti */
3934 c->dst.bytes = c->op_bytes;
3935 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3938 case 0xc7: /* Grp9 (cmpxchg8b) */
3939 rc = emulate_grp9(ctxt, ops);
3942 goto cannot_emulate;
3945 if (rc != X86EMUL_CONTINUE)