1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct decode_cache *c, int seg)
476 c->has_seg_override = true;
477 c->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
481 struct x86_emulate_ops *ops, int seg)
483 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
486 return ops->get_cached_segment_base(seg, ctxt->vcpu);
489 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
490 struct x86_emulate_ops *ops,
491 struct decode_cache *c)
493 if (!c->has_seg_override)
496 return c->seg_override;
499 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
500 u32 error, bool valid)
502 ctxt->exception.vector = vec;
503 ctxt->exception.error_code = error;
504 ctxt->exception.error_code_valid = valid;
505 return X86EMUL_PROPAGATE_FAULT;
508 static int emulate_db(struct x86_emulate_ctxt *ctxt)
510 return emulate_exception(ctxt, DB_VECTOR, 0, false);
513 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, GP_VECTOR, err, true);
518 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, SS_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int linearize(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
545 unsigned size, bool write,
548 struct decode_cache *c = &ctxt->decode;
549 struct desc_struct desc;
555 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
556 switch (ctxt->mode) {
557 case X86EMUL_MODE_REAL:
559 case X86EMUL_MODE_PROT64:
560 if (((signed long)la << 16) >> 16 != la)
561 return emulate_gp(ctxt, 0);
564 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
568 /* code segment or read-only data segment */
569 if (((desc.type & 8) || !(desc.type & 2)) && write)
571 /* unreadable code segment */
572 if ((desc.type & 8) && !(desc.type & 2))
574 lim = desc_limit_scaled(&desc);
575 if ((desc.type & 8) || !(desc.type & 4)) {
576 /* expand-up segment */
577 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
580 /* exapand-down segment */
581 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
583 lim = desc.d ? 0xffffffff : 0xffff;
584 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
587 cpl = ctxt->ops->cpl(ctxt->vcpu);
588 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
590 if (!(desc.type & 8)) {
594 } else if ((desc.type & 8) && !(desc.type & 4)) {
595 /* nonconforming code segment */
598 } else if ((desc.type & 8) && (desc.type & 4)) {
599 /* conforming code segment */
605 if (c->ad_bytes != 8)
608 return X86EMUL_CONTINUE;
610 if (addr.seg == VCPU_SREG_SS)
611 return emulate_ss(ctxt, addr.seg);
613 return emulate_gp(ctxt, addr.seg);
616 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
617 struct segmented_address addr,
624 rc = linearize(ctxt, addr, size, false, &linear);
625 if (rc != X86EMUL_CONTINUE)
627 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
631 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops,
633 unsigned long eip, u8 *dest)
635 struct fetch_cache *fc = &ctxt->decode.fetch;
639 if (eip == fc->end) {
640 unsigned long linear = eip + ctxt->cs_base;
641 if (ctxt->mode != X86EMUL_MODE_PROT64)
643 cur_size = fc->end - fc->start;
644 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
645 rc = ops->fetch(linear, fc->data + cur_size,
646 size, ctxt->vcpu, &ctxt->exception);
647 if (rc != X86EMUL_CONTINUE)
651 *dest = fc->data[eip - fc->start];
652 return X86EMUL_CONTINUE;
655 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
656 struct x86_emulate_ops *ops,
657 unsigned long eip, void *dest, unsigned size)
661 /* x86 instructions are limited to 15 bytes. */
662 if (eip + size - ctxt->eip > 15)
663 return X86EMUL_UNHANDLEABLE;
665 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
666 if (rc != X86EMUL_CONTINUE)
669 return X86EMUL_CONTINUE;
673 * Given the 'reg' portion of a ModRM byte, and a register block, return a
674 * pointer into the block that addresses the relevant register.
675 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
677 static void *decode_register(u8 modrm_reg, unsigned long *regs,
682 p = ®s[modrm_reg];
683 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
684 p = (unsigned char *)®s[modrm_reg & 3] + 1;
688 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
689 struct x86_emulate_ops *ops,
690 struct segmented_address addr,
691 u16 *size, unsigned long *address, int op_bytes)
698 rc = segmented_read_std(ctxt, addr, size, 2);
699 if (rc != X86EMUL_CONTINUE)
702 rc = segmented_read_std(ctxt, addr, address, op_bytes);
706 static int test_cc(unsigned int condition, unsigned int flags)
710 switch ((condition & 15) >> 1) {
712 rc |= (flags & EFLG_OF);
714 case 1: /* b/c/nae */
715 rc |= (flags & EFLG_CF);
718 rc |= (flags & EFLG_ZF);
721 rc |= (flags & (EFLG_CF|EFLG_ZF));
724 rc |= (flags & EFLG_SF);
727 rc |= (flags & EFLG_PF);
730 rc |= (flags & EFLG_ZF);
733 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
737 /* Odd condition identifiers (lsb == 1) have inverted sense. */
738 return (!!rc ^ (condition & 1));
741 static void fetch_register_operand(struct operand *op)
745 op->val = *(u8 *)op->addr.reg;
748 op->val = *(u16 *)op->addr.reg;
751 op->val = *(u32 *)op->addr.reg;
754 op->val = *(u64 *)op->addr.reg;
759 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
761 ctxt->ops->get_fpu(ctxt);
763 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
764 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
765 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
766 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
767 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
768 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
769 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
770 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
772 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
773 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
774 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
775 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
776 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
777 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
778 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
779 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
783 ctxt->ops->put_fpu(ctxt);
786 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
789 ctxt->ops->get_fpu(ctxt);
791 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
792 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
793 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
794 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
795 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
796 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
797 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
798 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
800 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
801 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
802 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
803 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
804 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
805 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
806 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
807 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
811 ctxt->ops->put_fpu(ctxt);
814 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
816 struct decode_cache *c,
819 unsigned reg = c->modrm_reg;
820 int highbyte_regs = c->rex_prefix == 0;
823 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
829 read_sse_reg(ctxt, &op->vec_val, reg);
834 if ((c->d & ByteOp) && !inhibit_bytereg) {
835 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
838 op->addr.reg = decode_register(reg, c->regs, 0);
839 op->bytes = c->op_bytes;
841 fetch_register_operand(op);
842 op->orig_val = op->val;
845 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
846 struct x86_emulate_ops *ops,
849 struct decode_cache *c = &ctxt->decode;
851 int index_reg = 0, base_reg = 0, scale;
852 int rc = X86EMUL_CONTINUE;
856 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
857 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
858 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
861 c->modrm = insn_fetch(u8, 1, c->eip);
862 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
863 c->modrm_reg |= (c->modrm & 0x38) >> 3;
864 c->modrm_rm |= (c->modrm & 0x07);
865 c->modrm_seg = VCPU_SREG_DS;
867 if (c->modrm_mod == 3) {
869 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
870 op->addr.reg = decode_register(c->modrm_rm,
871 c->regs, c->d & ByteOp);
875 op->addr.xmm = c->modrm_rm;
876 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
879 fetch_register_operand(op);
885 if (c->ad_bytes == 2) {
886 unsigned bx = c->regs[VCPU_REGS_RBX];
887 unsigned bp = c->regs[VCPU_REGS_RBP];
888 unsigned si = c->regs[VCPU_REGS_RSI];
889 unsigned di = c->regs[VCPU_REGS_RDI];
891 /* 16-bit ModR/M decode. */
892 switch (c->modrm_mod) {
894 if (c->modrm_rm == 6)
895 modrm_ea += insn_fetch(u16, 2, c->eip);
898 modrm_ea += insn_fetch(s8, 1, c->eip);
901 modrm_ea += insn_fetch(u16, 2, c->eip);
904 switch (c->modrm_rm) {
924 if (c->modrm_mod != 0)
931 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
932 (c->modrm_rm == 6 && c->modrm_mod != 0))
933 c->modrm_seg = VCPU_SREG_SS;
934 modrm_ea = (u16)modrm_ea;
936 /* 32/64-bit ModR/M decode. */
937 if ((c->modrm_rm & 7) == 4) {
938 sib = insn_fetch(u8, 1, c->eip);
939 index_reg |= (sib >> 3) & 7;
943 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
944 modrm_ea += insn_fetch(s32, 4, c->eip);
946 modrm_ea += c->regs[base_reg];
948 modrm_ea += c->regs[index_reg] << scale;
949 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
950 if (ctxt->mode == X86EMUL_MODE_PROT64)
953 modrm_ea += c->regs[c->modrm_rm];
954 switch (c->modrm_mod) {
956 if (c->modrm_rm == 5)
957 modrm_ea += insn_fetch(s32, 4, c->eip);
960 modrm_ea += insn_fetch(s8, 1, c->eip);
963 modrm_ea += insn_fetch(s32, 4, c->eip);
967 op->addr.mem.ea = modrm_ea;
972 static int decode_abs(struct x86_emulate_ctxt *ctxt,
973 struct x86_emulate_ops *ops,
976 struct decode_cache *c = &ctxt->decode;
977 int rc = X86EMUL_CONTINUE;
980 switch (c->ad_bytes) {
982 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
985 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
988 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
995 static void fetch_bit_operand(struct decode_cache *c)
999 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1000 mask = ~(c->dst.bytes * 8 - 1);
1002 if (c->src.bytes == 2)
1003 sv = (s16)c->src.val & (s16)mask;
1004 else if (c->src.bytes == 4)
1005 sv = (s32)c->src.val & (s32)mask;
1007 c->dst.addr.mem.ea += (sv >> 3);
1010 /* only subword offset */
1011 c->src.val &= (c->dst.bytes << 3) - 1;
1014 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1015 struct x86_emulate_ops *ops,
1016 unsigned long addr, void *dest, unsigned size)
1019 struct read_cache *mc = &ctxt->decode.mem_read;
1022 int n = min(size, 8u);
1024 if (mc->pos < mc->end)
1027 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1028 &ctxt->exception, ctxt->vcpu);
1029 if (rc != X86EMUL_CONTINUE)
1034 memcpy(dest, mc->data + mc->pos, n);
1039 return X86EMUL_CONTINUE;
1042 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1043 struct segmented_address addr,
1050 rc = linearize(ctxt, addr, size, false, &linear);
1051 if (rc != X86EMUL_CONTINUE)
1053 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1056 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1057 struct segmented_address addr,
1064 rc = linearize(ctxt, addr, size, true, &linear);
1065 if (rc != X86EMUL_CONTINUE)
1067 return ctxt->ops->write_emulated(linear, data, size,
1068 &ctxt->exception, ctxt->vcpu);
1071 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1072 struct segmented_address addr,
1073 const void *orig_data, const void *data,
1079 rc = linearize(ctxt, addr, size, true, &linear);
1080 if (rc != X86EMUL_CONTINUE)
1082 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1083 size, &ctxt->exception, ctxt->vcpu);
1086 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1087 struct x86_emulate_ops *ops,
1088 unsigned int size, unsigned short port,
1091 struct read_cache *rc = &ctxt->decode.io_read;
1093 if (rc->pos == rc->end) { /* refill pio read ahead */
1094 struct decode_cache *c = &ctxt->decode;
1095 unsigned int in_page, n;
1096 unsigned int count = c->rep_prefix ?
1097 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1098 in_page = (ctxt->eflags & EFLG_DF) ?
1099 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1100 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1101 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1105 rc->pos = rc->end = 0;
1106 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1111 memcpy(dest, rc->data + rc->pos, size);
1116 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1117 struct x86_emulate_ops *ops,
1118 u16 selector, struct desc_ptr *dt)
1120 if (selector & 1 << 2) {
1121 struct desc_struct desc;
1122 memset (dt, 0, sizeof *dt);
1123 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1127 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1128 dt->address = get_desc_base(&desc);
1130 ops->get_gdt(dt, ctxt->vcpu);
1133 /* allowed just for 8 bytes segments */
1134 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1135 struct x86_emulate_ops *ops,
1136 u16 selector, struct desc_struct *desc)
1139 u16 index = selector >> 3;
1143 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1145 if (dt.size < index * 8 + 7)
1146 return emulate_gp(ctxt, selector & 0xfffc);
1147 addr = dt.address + index * 8;
1148 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1154 /* allowed just for 8 bytes segments */
1155 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1156 struct x86_emulate_ops *ops,
1157 u16 selector, struct desc_struct *desc)
1160 u16 index = selector >> 3;
1164 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1166 if (dt.size < index * 8 + 7)
1167 return emulate_gp(ctxt, selector & 0xfffc);
1169 addr = dt.address + index * 8;
1170 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1176 /* Does not support long mode */
1177 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1178 struct x86_emulate_ops *ops,
1179 u16 selector, int seg)
1181 struct desc_struct seg_desc;
1183 unsigned err_vec = GP_VECTOR;
1185 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1188 memset(&seg_desc, 0, sizeof seg_desc);
1190 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1191 || ctxt->mode == X86EMUL_MODE_REAL) {
1192 /* set real mode segment descriptor */
1193 set_desc_base(&seg_desc, selector << 4);
1194 set_desc_limit(&seg_desc, 0xffff);
1201 /* NULL selector is not valid for TR, CS and SS */
1202 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1206 /* TR should be in GDT only */
1207 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1210 if (null_selector) /* for NULL selector skip all following checks */
1213 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1214 if (ret != X86EMUL_CONTINUE)
1217 err_code = selector & 0xfffc;
1218 err_vec = GP_VECTOR;
1220 /* can't load system descriptor into segment selecor */
1221 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1225 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1231 cpl = ops->cpl(ctxt->vcpu);
1236 * segment is not a writable data segment or segment
1237 * selector's RPL != CPL or segment selector's RPL != CPL
1239 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1243 if (!(seg_desc.type & 8))
1246 if (seg_desc.type & 4) {
1252 if (rpl > cpl || dpl != cpl)
1255 /* CS(RPL) <- CPL */
1256 selector = (selector & 0xfffc) | cpl;
1259 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1262 case VCPU_SREG_LDTR:
1263 if (seg_desc.s || seg_desc.type != 2)
1266 default: /* DS, ES, FS, or GS */
1268 * segment is not a data or readable code segment or
1269 * ((segment is a data or nonconforming code segment)
1270 * and (both RPL and CPL > DPL))
1272 if ((seg_desc.type & 0xa) == 0x8 ||
1273 (((seg_desc.type & 0xc) != 0xc) &&
1274 (rpl > dpl && cpl > dpl)))
1280 /* mark segment as accessed */
1282 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1283 if (ret != X86EMUL_CONTINUE)
1287 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1288 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1289 return X86EMUL_CONTINUE;
1291 emulate_exception(ctxt, err_vec, err_code, true);
1292 return X86EMUL_PROPAGATE_FAULT;
1295 static void write_register_operand(struct operand *op)
1297 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1298 switch (op->bytes) {
1300 *(u8 *)op->addr.reg = (u8)op->val;
1303 *(u16 *)op->addr.reg = (u16)op->val;
1306 *op->addr.reg = (u32)op->val;
1307 break; /* 64b: zero-extend */
1309 *op->addr.reg = op->val;
1314 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1315 struct x86_emulate_ops *ops)
1318 struct decode_cache *c = &ctxt->decode;
1320 switch (c->dst.type) {
1322 write_register_operand(&c->dst);
1326 rc = segmented_cmpxchg(ctxt,
1332 rc = segmented_write(ctxt,
1336 if (rc != X86EMUL_CONTINUE)
1340 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1348 return X86EMUL_CONTINUE;
1351 static int em_push(struct x86_emulate_ctxt *ctxt)
1353 struct decode_cache *c = &ctxt->decode;
1354 struct segmented_address addr;
1356 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1357 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1358 addr.seg = VCPU_SREG_SS;
1360 /* Disable writeback. */
1361 c->dst.type = OP_NONE;
1362 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1365 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1366 struct x86_emulate_ops *ops,
1367 void *dest, int len)
1369 struct decode_cache *c = &ctxt->decode;
1371 struct segmented_address addr;
1373 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1374 addr.seg = VCPU_SREG_SS;
1375 rc = segmented_read(ctxt, addr, dest, len);
1376 if (rc != X86EMUL_CONTINUE)
1379 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1383 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1384 struct x86_emulate_ops *ops,
1385 void *dest, int len)
1388 unsigned long val, change_mask;
1389 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1390 int cpl = ops->cpl(ctxt->vcpu);
1392 rc = emulate_pop(ctxt, ops, &val, len);
1393 if (rc != X86EMUL_CONTINUE)
1396 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1397 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1399 switch(ctxt->mode) {
1400 case X86EMUL_MODE_PROT64:
1401 case X86EMUL_MODE_PROT32:
1402 case X86EMUL_MODE_PROT16:
1404 change_mask |= EFLG_IOPL;
1406 change_mask |= EFLG_IF;
1408 case X86EMUL_MODE_VM86:
1410 return emulate_gp(ctxt, 0);
1411 change_mask |= EFLG_IF;
1413 default: /* real mode */
1414 change_mask |= (EFLG_IOPL | EFLG_IF);
1418 *(unsigned long *)dest =
1419 (ctxt->eflags & ~change_mask) | (val & change_mask);
1424 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1425 struct x86_emulate_ops *ops, int seg)
1427 struct decode_cache *c = &ctxt->decode;
1429 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1431 return em_push(ctxt);
1434 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1435 struct x86_emulate_ops *ops, int seg)
1437 struct decode_cache *c = &ctxt->decode;
1438 unsigned long selector;
1441 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1442 if (rc != X86EMUL_CONTINUE)
1445 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1449 static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1451 struct decode_cache *c = &ctxt->decode;
1452 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1453 int rc = X86EMUL_CONTINUE;
1454 int reg = VCPU_REGS_RAX;
1456 while (reg <= VCPU_REGS_RDI) {
1457 (reg == VCPU_REGS_RSP) ?
1458 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1461 if (rc != X86EMUL_CONTINUE)
1470 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1471 struct x86_emulate_ops *ops)
1473 struct decode_cache *c = &ctxt->decode;
1474 int rc = X86EMUL_CONTINUE;
1475 int reg = VCPU_REGS_RDI;
1477 while (reg >= VCPU_REGS_RAX) {
1478 if (reg == VCPU_REGS_RSP) {
1479 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1484 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1485 if (rc != X86EMUL_CONTINUE)
1492 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1493 struct x86_emulate_ops *ops, int irq)
1495 struct decode_cache *c = &ctxt->decode;
1502 /* TODO: Add limit checks */
1503 c->src.val = ctxt->eflags;
1505 if (rc != X86EMUL_CONTINUE)
1508 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1510 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1512 if (rc != X86EMUL_CONTINUE)
1515 c->src.val = c->eip;
1517 if (rc != X86EMUL_CONTINUE)
1520 ops->get_idt(&dt, ctxt->vcpu);
1522 eip_addr = dt.address + (irq << 2);
1523 cs_addr = dt.address + (irq << 2) + 2;
1525 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1526 if (rc != X86EMUL_CONTINUE)
1529 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1530 if (rc != X86EMUL_CONTINUE)
1533 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1534 if (rc != X86EMUL_CONTINUE)
1542 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1543 struct x86_emulate_ops *ops, int irq)
1545 switch(ctxt->mode) {
1546 case X86EMUL_MODE_REAL:
1547 return emulate_int_real(ctxt, ops, irq);
1548 case X86EMUL_MODE_VM86:
1549 case X86EMUL_MODE_PROT16:
1550 case X86EMUL_MODE_PROT32:
1551 case X86EMUL_MODE_PROT64:
1553 /* Protected mode interrupts unimplemented yet */
1554 return X86EMUL_UNHANDLEABLE;
1558 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1559 struct x86_emulate_ops *ops)
1561 struct decode_cache *c = &ctxt->decode;
1562 int rc = X86EMUL_CONTINUE;
1563 unsigned long temp_eip = 0;
1564 unsigned long temp_eflags = 0;
1565 unsigned long cs = 0;
1566 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1567 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1568 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1569 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1571 /* TODO: Add stack limit check */
1573 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1575 if (rc != X86EMUL_CONTINUE)
1578 if (temp_eip & ~0xffff)
1579 return emulate_gp(ctxt, 0);
1581 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1583 if (rc != X86EMUL_CONTINUE)
1586 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1588 if (rc != X86EMUL_CONTINUE)
1591 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1593 if (rc != X86EMUL_CONTINUE)
1599 if (c->op_bytes == 4)
1600 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1601 else if (c->op_bytes == 2) {
1602 ctxt->eflags &= ~0xffff;
1603 ctxt->eflags |= temp_eflags;
1606 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1607 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1612 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1613 struct x86_emulate_ops* ops)
1615 switch(ctxt->mode) {
1616 case X86EMUL_MODE_REAL:
1617 return emulate_iret_real(ctxt, ops);
1618 case X86EMUL_MODE_VM86:
1619 case X86EMUL_MODE_PROT16:
1620 case X86EMUL_MODE_PROT32:
1621 case X86EMUL_MODE_PROT64:
1623 /* iret from protected mode unimplemented yet */
1624 return X86EMUL_UNHANDLEABLE;
1628 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1629 struct x86_emulate_ops *ops)
1631 struct decode_cache *c = &ctxt->decode;
1633 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1636 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1638 struct decode_cache *c = &ctxt->decode;
1639 switch (c->modrm_reg) {
1641 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1644 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1647 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1650 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1652 case 4: /* sal/shl */
1653 case 6: /* sal/shl */
1654 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1657 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1660 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1665 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1666 struct x86_emulate_ops *ops)
1668 struct decode_cache *c = &ctxt->decode;
1669 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1670 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1673 switch (c->modrm_reg) {
1674 case 0 ... 1: /* test */
1675 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1678 c->dst.val = ~c->dst.val;
1681 emulate_1op("neg", c->dst, ctxt->eflags);
1684 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1687 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1690 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1694 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1698 return X86EMUL_UNHANDLEABLE;
1701 return emulate_de(ctxt);
1702 return X86EMUL_CONTINUE;
1705 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1707 struct decode_cache *c = &ctxt->decode;
1708 int rc = X86EMUL_CONTINUE;
1710 switch (c->modrm_reg) {
1712 emulate_1op("inc", c->dst, ctxt->eflags);
1715 emulate_1op("dec", c->dst, ctxt->eflags);
1717 case 2: /* call near abs */ {
1720 c->eip = c->src.val;
1721 c->src.val = old_eip;
1725 case 4: /* jmp abs */
1726 c->eip = c->src.val;
1735 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1736 struct x86_emulate_ops *ops)
1738 struct decode_cache *c = &ctxt->decode;
1739 u64 old = c->dst.orig_val64;
1741 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1742 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1743 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1744 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1745 ctxt->eflags &= ~EFLG_ZF;
1747 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1748 (u32) c->regs[VCPU_REGS_RBX];
1750 ctxt->eflags |= EFLG_ZF;
1752 return X86EMUL_CONTINUE;
1755 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1756 struct x86_emulate_ops *ops)
1758 struct decode_cache *c = &ctxt->decode;
1762 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1763 if (rc != X86EMUL_CONTINUE)
1765 if (c->op_bytes == 4)
1766 c->eip = (u32)c->eip;
1767 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1768 if (rc != X86EMUL_CONTINUE)
1770 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1774 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1775 struct x86_emulate_ops *ops, int seg)
1777 struct decode_cache *c = &ctxt->decode;
1781 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1783 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1784 if (rc != X86EMUL_CONTINUE)
1787 c->dst.val = c->src.val;
1792 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1793 struct x86_emulate_ops *ops, struct desc_struct *cs,
1794 struct desc_struct *ss)
1796 memset(cs, 0, sizeof(struct desc_struct));
1797 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1798 memset(ss, 0, sizeof(struct desc_struct));
1800 cs->l = 0; /* will be adjusted later */
1801 set_desc_base(cs, 0); /* flat segment */
1802 cs->g = 1; /* 4kb granularity */
1803 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1804 cs->type = 0x0b; /* Read, Execute, Accessed */
1806 cs->dpl = 0; /* will be adjusted later */
1810 set_desc_base(ss, 0); /* flat segment */
1811 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1812 ss->g = 1; /* 4kb granularity */
1814 ss->type = 0x03; /* Read/Write, Accessed */
1815 ss->d = 1; /* 32bit stack segment */
1821 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1823 struct decode_cache *c = &ctxt->decode;
1824 struct desc_struct cs, ss;
1828 /* syscall is not available in real mode */
1829 if (ctxt->mode == X86EMUL_MODE_REAL ||
1830 ctxt->mode == X86EMUL_MODE_VM86)
1831 return emulate_ud(ctxt);
1833 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1835 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1837 cs_sel = (u16)(msr_data & 0xfffc);
1838 ss_sel = (u16)(msr_data + 8);
1840 if (is_long_mode(ctxt->vcpu)) {
1844 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1845 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1846 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1847 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1849 c->regs[VCPU_REGS_RCX] = c->eip;
1850 if (is_long_mode(ctxt->vcpu)) {
1851 #ifdef CONFIG_X86_64
1852 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1854 ops->get_msr(ctxt->vcpu,
1855 ctxt->mode == X86EMUL_MODE_PROT64 ?
1856 MSR_LSTAR : MSR_CSTAR, &msr_data);
1859 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1860 ctxt->eflags &= ~(msr_data | EFLG_RF);
1864 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1865 c->eip = (u32)msr_data;
1867 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1870 return X86EMUL_CONTINUE;
1874 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1876 struct decode_cache *c = &ctxt->decode;
1877 struct desc_struct cs, ss;
1881 /* inject #GP if in real mode */
1882 if (ctxt->mode == X86EMUL_MODE_REAL)
1883 return emulate_gp(ctxt, 0);
1885 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1886 * Therefore, we inject an #UD.
1888 if (ctxt->mode == X86EMUL_MODE_PROT64)
1889 return emulate_ud(ctxt);
1891 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1893 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1894 switch (ctxt->mode) {
1895 case X86EMUL_MODE_PROT32:
1896 if ((msr_data & 0xfffc) == 0x0)
1897 return emulate_gp(ctxt, 0);
1899 case X86EMUL_MODE_PROT64:
1900 if (msr_data == 0x0)
1901 return emulate_gp(ctxt, 0);
1905 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1906 cs_sel = (u16)msr_data;
1907 cs_sel &= ~SELECTOR_RPL_MASK;
1908 ss_sel = cs_sel + 8;
1909 ss_sel &= ~SELECTOR_RPL_MASK;
1910 if (ctxt->mode == X86EMUL_MODE_PROT64
1911 || is_long_mode(ctxt->vcpu)) {
1916 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1917 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1918 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1919 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1921 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1924 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1925 c->regs[VCPU_REGS_RSP] = msr_data;
1927 return X86EMUL_CONTINUE;
1931 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1933 struct decode_cache *c = &ctxt->decode;
1934 struct desc_struct cs, ss;
1939 /* inject #GP if in real mode or Virtual 8086 mode */
1940 if (ctxt->mode == X86EMUL_MODE_REAL ||
1941 ctxt->mode == X86EMUL_MODE_VM86)
1942 return emulate_gp(ctxt, 0);
1944 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1946 if ((c->rex_prefix & 0x8) != 0x0)
1947 usermode = X86EMUL_MODE_PROT64;
1949 usermode = X86EMUL_MODE_PROT32;
1953 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1955 case X86EMUL_MODE_PROT32:
1956 cs_sel = (u16)(msr_data + 16);
1957 if ((msr_data & 0xfffc) == 0x0)
1958 return emulate_gp(ctxt, 0);
1959 ss_sel = (u16)(msr_data + 24);
1961 case X86EMUL_MODE_PROT64:
1962 cs_sel = (u16)(msr_data + 32);
1963 if (msr_data == 0x0)
1964 return emulate_gp(ctxt, 0);
1965 ss_sel = cs_sel + 8;
1970 cs_sel |= SELECTOR_RPL_MASK;
1971 ss_sel |= SELECTOR_RPL_MASK;
1973 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1974 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1975 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1976 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1978 c->eip = c->regs[VCPU_REGS_RDX];
1979 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1981 return X86EMUL_CONTINUE;
1984 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1985 struct x86_emulate_ops *ops)
1988 if (ctxt->mode == X86EMUL_MODE_REAL)
1990 if (ctxt->mode == X86EMUL_MODE_VM86)
1992 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1993 return ops->cpl(ctxt->vcpu) > iopl;
1996 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1997 struct x86_emulate_ops *ops,
2000 struct desc_struct tr_seg;
2003 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2004 unsigned mask = (1 << len) - 1;
2007 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2010 if (desc_limit_scaled(&tr_seg) < 103)
2012 base = get_desc_base(&tr_seg);
2013 #ifdef CONFIG_X86_64
2014 base |= ((u64)base3) << 32;
2016 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2017 if (r != X86EMUL_CONTINUE)
2019 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2021 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2023 if (r != X86EMUL_CONTINUE)
2025 if ((perm >> bit_idx) & mask)
2030 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2031 struct x86_emulate_ops *ops,
2037 if (emulator_bad_iopl(ctxt, ops))
2038 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2041 ctxt->perm_ok = true;
2046 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2047 struct x86_emulate_ops *ops,
2048 struct tss_segment_16 *tss)
2050 struct decode_cache *c = &ctxt->decode;
2053 tss->flag = ctxt->eflags;
2054 tss->ax = c->regs[VCPU_REGS_RAX];
2055 tss->cx = c->regs[VCPU_REGS_RCX];
2056 tss->dx = c->regs[VCPU_REGS_RDX];
2057 tss->bx = c->regs[VCPU_REGS_RBX];
2058 tss->sp = c->regs[VCPU_REGS_RSP];
2059 tss->bp = c->regs[VCPU_REGS_RBP];
2060 tss->si = c->regs[VCPU_REGS_RSI];
2061 tss->di = c->regs[VCPU_REGS_RDI];
2063 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2064 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2065 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2066 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2067 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2070 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2071 struct x86_emulate_ops *ops,
2072 struct tss_segment_16 *tss)
2074 struct decode_cache *c = &ctxt->decode;
2078 ctxt->eflags = tss->flag | 2;
2079 c->regs[VCPU_REGS_RAX] = tss->ax;
2080 c->regs[VCPU_REGS_RCX] = tss->cx;
2081 c->regs[VCPU_REGS_RDX] = tss->dx;
2082 c->regs[VCPU_REGS_RBX] = tss->bx;
2083 c->regs[VCPU_REGS_RSP] = tss->sp;
2084 c->regs[VCPU_REGS_RBP] = tss->bp;
2085 c->regs[VCPU_REGS_RSI] = tss->si;
2086 c->regs[VCPU_REGS_RDI] = tss->di;
2089 * SDM says that segment selectors are loaded before segment
2092 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2093 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2094 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2095 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2096 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2099 * Now load segment descriptors. If fault happenes at this stage
2100 * it is handled in a context of new task
2102 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2103 if (ret != X86EMUL_CONTINUE)
2105 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2106 if (ret != X86EMUL_CONTINUE)
2108 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2109 if (ret != X86EMUL_CONTINUE)
2111 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2112 if (ret != X86EMUL_CONTINUE)
2114 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2115 if (ret != X86EMUL_CONTINUE)
2118 return X86EMUL_CONTINUE;
2121 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2122 struct x86_emulate_ops *ops,
2123 u16 tss_selector, u16 old_tss_sel,
2124 ulong old_tss_base, struct desc_struct *new_desc)
2126 struct tss_segment_16 tss_seg;
2128 u32 new_tss_base = get_desc_base(new_desc);
2130 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2132 if (ret != X86EMUL_CONTINUE)
2133 /* FIXME: need to provide precise fault address */
2136 save_state_to_tss16(ctxt, ops, &tss_seg);
2138 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2140 if (ret != X86EMUL_CONTINUE)
2141 /* FIXME: need to provide precise fault address */
2144 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2146 if (ret != X86EMUL_CONTINUE)
2147 /* FIXME: need to provide precise fault address */
2150 if (old_tss_sel != 0xffff) {
2151 tss_seg.prev_task_link = old_tss_sel;
2153 ret = ops->write_std(new_tss_base,
2154 &tss_seg.prev_task_link,
2155 sizeof tss_seg.prev_task_link,
2156 ctxt->vcpu, &ctxt->exception);
2157 if (ret != X86EMUL_CONTINUE)
2158 /* FIXME: need to provide precise fault address */
2162 return load_state_from_tss16(ctxt, ops, &tss_seg);
2165 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2166 struct x86_emulate_ops *ops,
2167 struct tss_segment_32 *tss)
2169 struct decode_cache *c = &ctxt->decode;
2171 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2173 tss->eflags = ctxt->eflags;
2174 tss->eax = c->regs[VCPU_REGS_RAX];
2175 tss->ecx = c->regs[VCPU_REGS_RCX];
2176 tss->edx = c->regs[VCPU_REGS_RDX];
2177 tss->ebx = c->regs[VCPU_REGS_RBX];
2178 tss->esp = c->regs[VCPU_REGS_RSP];
2179 tss->ebp = c->regs[VCPU_REGS_RBP];
2180 tss->esi = c->regs[VCPU_REGS_RSI];
2181 tss->edi = c->regs[VCPU_REGS_RDI];
2183 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2184 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2185 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2186 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2187 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2188 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2189 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2192 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2193 struct x86_emulate_ops *ops,
2194 struct tss_segment_32 *tss)
2196 struct decode_cache *c = &ctxt->decode;
2199 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2200 return emulate_gp(ctxt, 0);
2202 ctxt->eflags = tss->eflags | 2;
2203 c->regs[VCPU_REGS_RAX] = tss->eax;
2204 c->regs[VCPU_REGS_RCX] = tss->ecx;
2205 c->regs[VCPU_REGS_RDX] = tss->edx;
2206 c->regs[VCPU_REGS_RBX] = tss->ebx;
2207 c->regs[VCPU_REGS_RSP] = tss->esp;
2208 c->regs[VCPU_REGS_RBP] = tss->ebp;
2209 c->regs[VCPU_REGS_RSI] = tss->esi;
2210 c->regs[VCPU_REGS_RDI] = tss->edi;
2213 * SDM says that segment selectors are loaded before segment
2216 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2217 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2218 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2219 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2220 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2221 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2222 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2225 * Now load segment descriptors. If fault happenes at this stage
2226 * it is handled in a context of new task
2228 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2229 if (ret != X86EMUL_CONTINUE)
2231 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2232 if (ret != X86EMUL_CONTINUE)
2234 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2235 if (ret != X86EMUL_CONTINUE)
2237 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2238 if (ret != X86EMUL_CONTINUE)
2240 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2241 if (ret != X86EMUL_CONTINUE)
2243 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2244 if (ret != X86EMUL_CONTINUE)
2246 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2247 if (ret != X86EMUL_CONTINUE)
2250 return X86EMUL_CONTINUE;
2253 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2254 struct x86_emulate_ops *ops,
2255 u16 tss_selector, u16 old_tss_sel,
2256 ulong old_tss_base, struct desc_struct *new_desc)
2258 struct tss_segment_32 tss_seg;
2260 u32 new_tss_base = get_desc_base(new_desc);
2262 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2264 if (ret != X86EMUL_CONTINUE)
2265 /* FIXME: need to provide precise fault address */
2268 save_state_to_tss32(ctxt, ops, &tss_seg);
2270 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2272 if (ret != X86EMUL_CONTINUE)
2273 /* FIXME: need to provide precise fault address */
2276 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2278 if (ret != X86EMUL_CONTINUE)
2279 /* FIXME: need to provide precise fault address */
2282 if (old_tss_sel != 0xffff) {
2283 tss_seg.prev_task_link = old_tss_sel;
2285 ret = ops->write_std(new_tss_base,
2286 &tss_seg.prev_task_link,
2287 sizeof tss_seg.prev_task_link,
2288 ctxt->vcpu, &ctxt->exception);
2289 if (ret != X86EMUL_CONTINUE)
2290 /* FIXME: need to provide precise fault address */
2294 return load_state_from_tss32(ctxt, ops, &tss_seg);
2297 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2298 struct x86_emulate_ops *ops,
2299 u16 tss_selector, int reason,
2300 bool has_error_code, u32 error_code)
2302 struct desc_struct curr_tss_desc, next_tss_desc;
2304 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2305 ulong old_tss_base =
2306 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2309 /* FIXME: old_tss_base == ~0 ? */
2311 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2312 if (ret != X86EMUL_CONTINUE)
2314 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2315 if (ret != X86EMUL_CONTINUE)
2318 /* FIXME: check that next_tss_desc is tss */
2320 if (reason != TASK_SWITCH_IRET) {
2321 if ((tss_selector & 3) > next_tss_desc.dpl ||
2322 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2323 return emulate_gp(ctxt, 0);
2326 desc_limit = desc_limit_scaled(&next_tss_desc);
2327 if (!next_tss_desc.p ||
2328 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2329 desc_limit < 0x2b)) {
2330 emulate_ts(ctxt, tss_selector & 0xfffc);
2331 return X86EMUL_PROPAGATE_FAULT;
2334 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2335 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2336 write_segment_descriptor(ctxt, ops, old_tss_sel,
2340 if (reason == TASK_SWITCH_IRET)
2341 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2343 /* set back link to prev task only if NT bit is set in eflags
2344 note that old_tss_sel is not used afetr this point */
2345 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2346 old_tss_sel = 0xffff;
2348 if (next_tss_desc.type & 8)
2349 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2350 old_tss_base, &next_tss_desc);
2352 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2353 old_tss_base, &next_tss_desc);
2354 if (ret != X86EMUL_CONTINUE)
2357 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2358 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2360 if (reason != TASK_SWITCH_IRET) {
2361 next_tss_desc.type |= (1 << 1); /* set busy flag */
2362 write_segment_descriptor(ctxt, ops, tss_selector,
2366 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2367 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2368 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2370 if (has_error_code) {
2371 struct decode_cache *c = &ctxt->decode;
2373 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2375 c->src.val = (unsigned long) error_code;
2376 ret = em_push(ctxt);
2382 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2383 u16 tss_selector, int reason,
2384 bool has_error_code, u32 error_code)
2386 struct x86_emulate_ops *ops = ctxt->ops;
2387 struct decode_cache *c = &ctxt->decode;
2391 c->dst.type = OP_NONE;
2393 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2394 has_error_code, error_code);
2396 if (rc == X86EMUL_CONTINUE)
2399 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2402 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2403 int reg, struct operand *op)
2405 struct decode_cache *c = &ctxt->decode;
2406 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2408 register_address_increment(c, &c->regs[reg], df * op->bytes);
2409 op->addr.mem.ea = register_address(c, c->regs[reg]);
2410 op->addr.mem.seg = seg;
2413 static int em_das(struct x86_emulate_ctxt *ctxt)
2415 struct decode_cache *c = &ctxt->decode;
2417 bool af, cf, old_cf;
2419 cf = ctxt->eflags & X86_EFLAGS_CF;
2425 af = ctxt->eflags & X86_EFLAGS_AF;
2426 if ((al & 0x0f) > 9 || af) {
2428 cf = old_cf | (al >= 250);
2433 if (old_al > 0x99 || old_cf) {
2439 /* Set PF, ZF, SF */
2440 c->src.type = OP_IMM;
2443 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2444 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2446 ctxt->eflags |= X86_EFLAGS_CF;
2448 ctxt->eflags |= X86_EFLAGS_AF;
2449 return X86EMUL_CONTINUE;
2452 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2454 struct decode_cache *c = &ctxt->decode;
2459 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2462 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2463 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2464 return X86EMUL_CONTINUE;
2467 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2469 c->src.val = old_cs;
2471 if (rc != X86EMUL_CONTINUE)
2474 c->src.val = old_eip;
2475 return em_push(ctxt);
2478 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2480 struct decode_cache *c = &ctxt->decode;
2483 c->dst.type = OP_REG;
2484 c->dst.addr.reg = &c->eip;
2485 c->dst.bytes = c->op_bytes;
2486 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2487 if (rc != X86EMUL_CONTINUE)
2489 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2490 return X86EMUL_CONTINUE;
2493 static int em_imul(struct x86_emulate_ctxt *ctxt)
2495 struct decode_cache *c = &ctxt->decode;
2497 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2498 return X86EMUL_CONTINUE;
2501 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2503 struct decode_cache *c = &ctxt->decode;
2505 c->dst.val = c->src2.val;
2506 return em_imul(ctxt);
2509 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2511 struct decode_cache *c = &ctxt->decode;
2513 c->dst.type = OP_REG;
2514 c->dst.bytes = c->src.bytes;
2515 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2516 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2518 return X86EMUL_CONTINUE;
2521 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2523 struct decode_cache *c = &ctxt->decode;
2526 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2527 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2528 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2529 return X86EMUL_CONTINUE;
2532 static int em_mov(struct x86_emulate_ctxt *ctxt)
2534 struct decode_cache *c = &ctxt->decode;
2535 c->dst.val = c->src.val;
2536 return X86EMUL_CONTINUE;
2539 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2541 struct decode_cache *c = &ctxt->decode;
2542 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2543 return X86EMUL_CONTINUE;
2546 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2548 struct decode_cache *c = &ctxt->decode;
2552 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2553 if (rc == X86EMUL_CONTINUE)
2554 emulate_invlpg(ctxt->vcpu, linear);
2555 /* Disable writeback. */
2556 c->dst.type = OP_NONE;
2557 return X86EMUL_CONTINUE;
2560 static bool valid_cr(int nr)
2572 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2574 struct decode_cache *c = &ctxt->decode;
2576 if (!valid_cr(c->modrm_reg))
2577 return emulate_ud(ctxt);
2579 return X86EMUL_CONTINUE;
2582 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2584 struct decode_cache *c = &ctxt->decode;
2585 u64 new_val = c->src.val64;
2586 int cr = c->modrm_reg;
2588 static u64 cr_reserved_bits[] = {
2589 0xffffffff00000000ULL,
2590 0, 0, 0, /* CR3 checked later */
2597 return emulate_ud(ctxt);
2599 if (new_val & cr_reserved_bits[cr])
2600 return emulate_gp(ctxt, 0);
2605 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2606 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2607 return emulate_gp(ctxt, 0);
2609 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2610 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2612 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2613 !(cr4 & X86_CR4_PAE))
2614 return emulate_gp(ctxt, 0);
2621 if (is_long_mode(ctxt->vcpu))
2622 rsvd = CR3_L_MODE_RESERVED_BITS;
2623 else if (is_pae(ctxt->vcpu))
2624 rsvd = CR3_PAE_RESERVED_BITS;
2625 else if (is_paging(ctxt->vcpu))
2626 rsvd = CR3_NONPAE_RESERVED_BITS;
2629 return emulate_gp(ctxt, 0);
2636 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2637 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2639 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2640 return emulate_gp(ctxt, 0);
2646 return X86EMUL_CONTINUE;
2649 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2653 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2655 /* Check if DR7.Global_Enable is set */
2656 return dr7 & (1 << 13);
2659 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2661 struct decode_cache *c = &ctxt->decode;
2662 int dr = c->modrm_reg;
2666 return emulate_ud(ctxt);
2668 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2669 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2670 return emulate_ud(ctxt);
2672 if (check_dr7_gd(ctxt))
2673 return emulate_db(ctxt);
2675 return X86EMUL_CONTINUE;
2678 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2680 struct decode_cache *c = &ctxt->decode;
2681 u64 new_val = c->src.val64;
2682 int dr = c->modrm_reg;
2684 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2685 return emulate_gp(ctxt, 0);
2687 return check_dr_read(ctxt);
2690 static int check_svme(struct x86_emulate_ctxt *ctxt)
2694 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2696 if (!(efer & EFER_SVME))
2697 return emulate_ud(ctxt);
2699 return X86EMUL_CONTINUE;
2702 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2704 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2706 /* Valid physical address? */
2707 if (rax & 0xffff000000000000)
2708 return emulate_gp(ctxt, 0);
2710 return check_svme(ctxt);
2713 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2715 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2717 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2718 return emulate_ud(ctxt);
2720 return X86EMUL_CONTINUE;
2723 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2725 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2726 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2728 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2730 return emulate_gp(ctxt, 0);
2732 return X86EMUL_CONTINUE;
2735 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2737 struct decode_cache *c = &ctxt->decode;
2739 c->dst.bytes = min(c->dst.bytes, 4u);
2740 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2741 return emulate_gp(ctxt, 0);
2743 return X86EMUL_CONTINUE;
2746 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2748 struct decode_cache *c = &ctxt->decode;
2750 c->src.bytes = min(c->src.bytes, 4u);
2751 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2752 return emulate_gp(ctxt, 0);
2754 return X86EMUL_CONTINUE;
2757 #define D(_y) { .flags = (_y) }
2758 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2759 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2760 .check_perm = (_p) }
2762 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2763 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2764 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2765 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2766 #define II(_f, _e, _i) \
2767 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2768 #define IIP(_f, _e, _i, _p) \
2769 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2770 .check_perm = (_p) }
2771 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2773 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2774 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2775 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2777 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2778 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2779 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2781 static struct opcode group7_rm1[] = {
2782 DI(SrcNone | ModRM | Priv, monitor),
2783 DI(SrcNone | ModRM | Priv, mwait),
2787 static struct opcode group7_rm3[] = {
2788 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2789 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2790 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2791 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2792 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2793 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2794 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2795 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2798 static struct opcode group7_rm7[] = {
2800 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2803 static struct opcode group1[] = {
2807 static struct opcode group1A[] = {
2808 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2811 static struct opcode group3[] = {
2812 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2813 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2814 X4(D(SrcMem | ModRM)),
2817 static struct opcode group4[] = {
2818 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2822 static struct opcode group5[] = {
2823 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2824 D(SrcMem | ModRM | Stack),
2825 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2826 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2827 D(SrcMem | ModRM | Stack), N,
2830 static struct opcode group6[] = {
2831 DI(ModRM | Prot, sldt),
2832 DI(ModRM | Prot, str),
2833 DI(ModRM | Prot | Priv, lldt),
2834 DI(ModRM | Prot | Priv, ltr),
2838 static struct group_dual group7 = { {
2839 DI(ModRM | Mov | DstMem | Priv, sgdt),
2840 DI(ModRM | Mov | DstMem | Priv, sidt),
2841 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2842 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2843 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2844 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2846 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2847 N, EXT(0, group7_rm3),
2848 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2849 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2852 static struct opcode group8[] = {
2854 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2855 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2858 static struct group_dual group9 = { {
2859 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2861 N, N, N, N, N, N, N, N,
2864 static struct opcode group11[] = {
2865 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2868 static struct gprefix pfx_0f_6f_0f_7f = {
2869 N, N, N, I(Sse, em_movdqu),
2872 static struct opcode opcode_table[256] = {
2875 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2878 D(ImplicitOps | Stack | No64), N,
2881 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2884 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2888 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2896 X8(I(SrcReg | Stack, em_push)),
2898 X8(D(DstReg | Stack)),
2900 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2901 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2904 I(SrcImm | Mov | Stack, em_push),
2905 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2906 I(SrcImmByte | Mov | Stack, em_push),
2907 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2908 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2909 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2913 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2914 G(DstMem | SrcImm | ModRM | Group, group1),
2915 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2916 G(DstMem | SrcImmByte | ModRM | Group, group1),
2917 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2919 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2920 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2921 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2922 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2924 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2926 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2927 I(SrcImmFAddr | No64, em_call_far), N,
2928 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2930 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2931 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2932 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2933 D2bv(SrcSI | DstDI | String),
2935 D2bv(DstAcc | SrcImm),
2936 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2937 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2938 D2bv(SrcAcc | DstDI | String),
2940 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2942 X8(I(DstReg | SrcImm | Mov, em_mov)),
2944 D2bv(DstMem | SrcImmByte | ModRM),
2945 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2946 D(ImplicitOps | Stack),
2947 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2948 G(ByteOp, group11), G(0, group11),
2950 N, N, N, D(ImplicitOps | Stack),
2951 D(ImplicitOps), DI(SrcImmByte, intn),
2952 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2954 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2957 N, N, N, N, N, N, N, N,
2960 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2961 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2963 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2964 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2965 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2966 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2968 N, DI(ImplicitOps, icebp), N, N,
2969 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2970 G(ByteOp, group3), G(0, group3),
2972 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2973 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2976 static struct opcode twobyte_table[256] = {
2978 G(0, group6), GD(0, &group7), N, N,
2979 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2980 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2981 N, D(ImplicitOps | ModRM), N, N,
2983 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2985 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2986 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2987 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2988 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
2990 N, N, N, N, N, N, N, N,
2992 DI(ImplicitOps | Priv, wrmsr),
2993 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
2994 DI(ImplicitOps | Priv, rdmsr),
2995 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
2996 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2998 N, N, N, N, N, N, N, N,
3000 X16(D(DstReg | SrcMem | ModRM | Mov)),
3002 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3007 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3012 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3016 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3018 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3019 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3020 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3021 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3023 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3024 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3025 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3026 D(DstMem | SrcReg | Src2CL | ModRM),
3027 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3029 D2bv(DstMem | SrcReg | ModRM | Lock),
3030 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3031 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3032 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3035 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3036 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3037 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3039 D2bv(DstMem | SrcReg | ModRM | Lock),
3040 N, D(DstMem | SrcReg | ModRM | Mov),
3041 N, N, N, GD(0, &group9),
3042 N, N, N, N, N, N, N, N,
3044 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3046 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3048 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3064 static unsigned imm_size(struct decode_cache *c)
3068 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3074 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3075 unsigned size, bool sign_extension)
3077 struct decode_cache *c = &ctxt->decode;
3078 struct x86_emulate_ops *ops = ctxt->ops;
3079 int rc = X86EMUL_CONTINUE;
3083 op->addr.mem.ea = c->eip;
3084 /* NB. Immediates are sign-extended as necessary. */
3085 switch (op->bytes) {
3087 op->val = insn_fetch(s8, 1, c->eip);
3090 op->val = insn_fetch(s16, 2, c->eip);
3093 op->val = insn_fetch(s32, 4, c->eip);
3096 if (!sign_extension) {
3097 switch (op->bytes) {
3105 op->val &= 0xffffffff;
3114 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3116 struct x86_emulate_ops *ops = ctxt->ops;
3117 struct decode_cache *c = &ctxt->decode;
3118 int rc = X86EMUL_CONTINUE;
3119 int mode = ctxt->mode;
3120 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3121 bool op_prefix = false;
3122 struct opcode opcode, *g_mod012, *g_mod3;
3123 struct operand memop = { .type = OP_NONE };
3126 c->fetch.start = c->eip;
3127 c->fetch.end = c->fetch.start + insn_len;
3129 memcpy(c->fetch.data, insn, insn_len);
3130 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
3133 case X86EMUL_MODE_REAL:
3134 case X86EMUL_MODE_VM86:
3135 case X86EMUL_MODE_PROT16:
3136 def_op_bytes = def_ad_bytes = 2;
3138 case X86EMUL_MODE_PROT32:
3139 def_op_bytes = def_ad_bytes = 4;
3141 #ifdef CONFIG_X86_64
3142 case X86EMUL_MODE_PROT64:
3151 c->op_bytes = def_op_bytes;
3152 c->ad_bytes = def_ad_bytes;
3154 /* Legacy prefixes. */
3156 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3157 case 0x66: /* operand-size override */
3159 /* switch between 2/4 bytes */
3160 c->op_bytes = def_op_bytes ^ 6;
3162 case 0x67: /* address-size override */
3163 if (mode == X86EMUL_MODE_PROT64)
3164 /* switch between 4/8 bytes */
3165 c->ad_bytes = def_ad_bytes ^ 12;
3167 /* switch between 2/4 bytes */
3168 c->ad_bytes = def_ad_bytes ^ 6;
3170 case 0x26: /* ES override */
3171 case 0x2e: /* CS override */
3172 case 0x36: /* SS override */
3173 case 0x3e: /* DS override */
3174 set_seg_override(c, (c->b >> 3) & 3);
3176 case 0x64: /* FS override */
3177 case 0x65: /* GS override */
3178 set_seg_override(c, c->b & 7);
3180 case 0x40 ... 0x4f: /* REX */
3181 if (mode != X86EMUL_MODE_PROT64)
3183 c->rex_prefix = c->b;
3185 case 0xf0: /* LOCK */
3188 case 0xf2: /* REPNE/REPNZ */
3189 case 0xf3: /* REP/REPE/REPZ */
3190 c->rep_prefix = c->b;
3196 /* Any legacy prefix after a REX prefix nullifies its effect. */
3204 if (c->rex_prefix & 8)
3205 c->op_bytes = 8; /* REX.W */
3207 /* Opcode byte(s). */
3208 opcode = opcode_table[c->b];
3209 /* Two-byte opcode? */
3212 c->b = insn_fetch(u8, 1, c->eip);
3213 opcode = twobyte_table[c->b];
3215 c->d = opcode.flags;
3218 dual = c->d & GroupDual;
3219 c->modrm = insn_fetch(u8, 1, c->eip);
3222 if (c->d & GroupDual) {
3223 g_mod012 = opcode.u.gdual->mod012;
3224 g_mod3 = opcode.u.gdual->mod3;
3226 g_mod012 = g_mod3 = opcode.u.group;
3228 c->d &= ~(Group | GroupDual);
3230 goffset = (c->modrm >> 3) & 7;
3232 if ((c->modrm >> 6) == 3)
3233 opcode = g_mod3[goffset];
3235 opcode = g_mod012[goffset];
3237 if (opcode.flags & RMExt) {
3238 goffset = c->modrm & 7;
3239 opcode = opcode.u.group[goffset];
3242 c->d |= opcode.flags;
3245 if (c->d & Prefix) {
3246 if (c->rep_prefix && op_prefix)
3247 return X86EMUL_UNHANDLEABLE;
3248 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3249 switch (simd_prefix) {
3250 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3251 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3252 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3253 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3255 c->d |= opcode.flags;
3258 c->execute = opcode.u.execute;
3259 c->check_perm = opcode.check_perm;
3260 c->intercept = opcode.intercept;
3263 if (c->d == 0 || (c->d & Undefined))
3266 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3269 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3272 if (c->d & Op3264) {
3273 if (mode == X86EMUL_MODE_PROT64)
3282 /* ModRM and SIB bytes. */
3284 rc = decode_modrm(ctxt, ops, &memop);
3285 if (!c->has_seg_override)
3286 set_seg_override(c, c->modrm_seg);
3287 } else if (c->d & MemAbs)
3288 rc = decode_abs(ctxt, ops, &memop);
3289 if (rc != X86EMUL_CONTINUE)
3292 if (!c->has_seg_override)
3293 set_seg_override(c, VCPU_SREG_DS);
3295 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3297 if (memop.type == OP_MEM && c->ad_bytes != 8)
3298 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3300 if (memop.type == OP_MEM && c->rip_relative)
3301 memop.addr.mem.ea += c->eip;
3304 * Decode and fetch the source operand: register, memory
3307 switch (c->d & SrcMask) {
3311 decode_register_operand(ctxt, &c->src, c, 0);
3320 memop.bytes = (c->d & ByteOp) ? 1 :
3326 rc = decode_imm(ctxt, &c->src, 2, false);
3329 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3332 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3335 rc = decode_imm(ctxt, &c->src, 1, true);
3338 rc = decode_imm(ctxt, &c->src, 1, false);
3341 c->src.type = OP_REG;
3342 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3343 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3344 fetch_register_operand(&c->src);
3351 c->src.type = OP_MEM;
3352 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3353 c->src.addr.mem.ea =
3354 register_address(c, c->regs[VCPU_REGS_RSI]);
3355 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3359 c->src.type = OP_IMM;
3360 c->src.addr.mem.ea = c->eip;
3361 c->src.bytes = c->op_bytes + 2;
3362 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3365 memop.bytes = c->op_bytes + 2;
3370 if (rc != X86EMUL_CONTINUE)
3374 * Decode and fetch the second source operand: register, memory
3377 switch (c->d & Src2Mask) {
3382 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3385 rc = decode_imm(ctxt, &c->src2, 1, true);
3392 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3396 if (rc != X86EMUL_CONTINUE)
3399 /* Decode and fetch the destination operand: register or memory. */
3400 switch (c->d & DstMask) {
3402 decode_register_operand(ctxt, &c->dst, c,
3403 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3406 c->dst.type = OP_IMM;
3407 c->dst.addr.mem.ea = c->eip;
3409 c->dst.val = insn_fetch(u8, 1, c->eip);
3414 if ((c->d & DstMask) == DstMem64)
3417 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3419 fetch_bit_operand(c);
3420 c->dst.orig_val = c->dst.val;
3423 c->dst.type = OP_REG;
3424 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3425 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3426 fetch_register_operand(&c->dst);
3427 c->dst.orig_val = c->dst.val;
3430 c->dst.type = OP_MEM;
3431 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3432 c->dst.addr.mem.ea =
3433 register_address(c, c->regs[VCPU_REGS_RDI]);
3434 c->dst.addr.mem.seg = VCPU_SREG_ES;
3438 /* Special instructions do their own operand decoding. */
3440 c->dst.type = OP_NONE; /* Disable writeback. */
3445 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3448 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3450 struct decode_cache *c = &ctxt->decode;
3452 /* The second termination condition only applies for REPE
3453 * and REPNE. Test if the repeat string operation prefix is
3454 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3455 * corresponding termination condition according to:
3456 * - if REPE/REPZ and ZF = 0 then done
3457 * - if REPNE/REPNZ and ZF = 1 then done
3459 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3460 (c->b == 0xae) || (c->b == 0xaf))
3461 && (((c->rep_prefix == REPE_PREFIX) &&
3462 ((ctxt->eflags & EFLG_ZF) == 0))
3463 || ((c->rep_prefix == REPNE_PREFIX) &&
3464 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3471 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3473 struct x86_emulate_ops *ops = ctxt->ops;
3475 struct decode_cache *c = &ctxt->decode;
3476 int rc = X86EMUL_CONTINUE;
3477 int saved_dst_type = c->dst.type;
3478 int irq; /* Used for int 3, int, and into */
3480 ctxt->decode.mem_read.pos = 0;
3482 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3483 rc = emulate_ud(ctxt);
3487 /* LOCK prefix is allowed only with some instructions */
3488 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3489 rc = emulate_ud(ctxt);
3493 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3494 rc = emulate_ud(ctxt);
3499 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3500 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3501 rc = emulate_ud(ctxt);
3505 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3506 rc = emulate_nm(ctxt);
3510 if (unlikely(ctxt->guest_mode) && c->intercept) {
3511 rc = emulator_check_intercept(ctxt, c->intercept,
3512 X86_ICPT_PRE_EXCEPT);
3513 if (rc != X86EMUL_CONTINUE)
3517 /* Privileged instruction can be executed only in CPL=0 */
3518 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3519 rc = emulate_gp(ctxt, 0);
3523 /* Instruction can only be executed in protected mode */
3524 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3525 rc = emulate_ud(ctxt);
3529 /* Do instruction specific permission checks */
3530 if (c->check_perm) {
3531 rc = c->check_perm(ctxt);
3532 if (rc != X86EMUL_CONTINUE)
3536 if (unlikely(ctxt->guest_mode) && c->intercept) {
3537 rc = emulator_check_intercept(ctxt, c->intercept,
3538 X86_ICPT_POST_EXCEPT);
3539 if (rc != X86EMUL_CONTINUE)
3543 if (c->rep_prefix && (c->d & String)) {
3544 /* All REP prefixes have the same first termination condition */
3545 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3551 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3552 rc = segmented_read(ctxt, c->src.addr.mem,
3553 c->src.valptr, c->src.bytes);
3554 if (rc != X86EMUL_CONTINUE)
3556 c->src.orig_val64 = c->src.val64;
3559 if (c->src2.type == OP_MEM) {
3560 rc = segmented_read(ctxt, c->src2.addr.mem,
3561 &c->src2.val, c->src2.bytes);
3562 if (rc != X86EMUL_CONTINUE)
3566 if ((c->d & DstMask) == ImplicitOps)
3570 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3571 /* optimisation - avoid slow emulated read if Mov */
3572 rc = segmented_read(ctxt, c->dst.addr.mem,
3573 &c->dst.val, c->dst.bytes);
3574 if (rc != X86EMUL_CONTINUE)
3577 c->dst.orig_val = c->dst.val;
3581 if (unlikely(ctxt->guest_mode) && c->intercept) {
3582 rc = emulator_check_intercept(ctxt, c->intercept,
3583 X86_ICPT_POST_MEMACCESS);
3584 if (rc != X86EMUL_CONTINUE)
3589 rc = c->execute(ctxt);
3590 if (rc != X86EMUL_CONTINUE)
3601 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3603 case 0x06: /* push es */
3604 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3606 case 0x07: /* pop es */
3607 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3611 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3613 case 0x0e: /* push cs */
3614 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3618 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3620 case 0x16: /* push ss */
3621 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3623 case 0x17: /* pop ss */
3624 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3628 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3630 case 0x1e: /* push ds */
3631 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3633 case 0x1f: /* pop ds */
3634 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3638 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3642 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3646 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3650 c->dst.type = OP_NONE; /* Disable writeback. */
3651 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3653 case 0x40 ... 0x47: /* inc r16/r32 */
3654 emulate_1op("inc", c->dst, ctxt->eflags);
3656 case 0x48 ... 0x4f: /* dec r16/r32 */
3657 emulate_1op("dec", c->dst, ctxt->eflags);
3659 case 0x58 ... 0x5f: /* pop reg */
3661 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3663 case 0x60: /* pusha */
3664 rc = emulate_pusha(ctxt);
3666 case 0x61: /* popa */
3667 rc = emulate_popa(ctxt, ops);
3669 case 0x63: /* movsxd */
3670 if (ctxt->mode != X86EMUL_MODE_PROT64)
3671 goto cannot_emulate;
3672 c->dst.val = (s32) c->src.val;
3674 case 0x6c: /* insb */
3675 case 0x6d: /* insw/insd */
3676 c->src.val = c->regs[VCPU_REGS_RDX];
3678 case 0x6e: /* outsb */
3679 case 0x6f: /* outsw/outsd */
3680 c->dst.val = c->regs[VCPU_REGS_RDX];
3683 case 0x70 ... 0x7f: /* jcc (short) */
3684 if (test_cc(c->b, ctxt->eflags))
3685 jmp_rel(c, c->src.val);
3687 case 0x80 ... 0x83: /* Grp1 */
3688 switch (c->modrm_reg) {
3709 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3711 case 0x86 ... 0x87: /* xchg */
3713 /* Write back the register source. */
3714 c->src.val = c->dst.val;
3715 write_register_operand(&c->src);
3717 * Write back the memory destination with implicit LOCK
3720 c->dst.val = c->src.orig_val;
3723 case 0x8c: /* mov r/m, sreg */
3724 if (c->modrm_reg > VCPU_SREG_GS) {
3725 rc = emulate_ud(ctxt);
3728 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3730 case 0x8d: /* lea r16/r32, m */
3731 c->dst.val = c->src.addr.mem.ea;
3733 case 0x8e: { /* mov seg, r/m16 */
3738 if (c->modrm_reg == VCPU_SREG_CS ||
3739 c->modrm_reg > VCPU_SREG_GS) {
3740 rc = emulate_ud(ctxt);
3744 if (c->modrm_reg == VCPU_SREG_SS)
3745 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3747 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3749 c->dst.type = OP_NONE; /* Disable writeback. */
3752 case 0x8f: /* pop (sole member of Grp1a) */
3753 rc = emulate_grp1a(ctxt, ops);
3755 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3756 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3759 case 0x98: /* cbw/cwde/cdqe */
3760 switch (c->op_bytes) {
3761 case 2: c->dst.val = (s8)c->dst.val; break;
3762 case 4: c->dst.val = (s16)c->dst.val; break;
3763 case 8: c->dst.val = (s32)c->dst.val; break;
3766 case 0x9c: /* pushf */
3767 c->src.val = (unsigned long) ctxt->eflags;
3770 case 0x9d: /* popf */
3771 c->dst.type = OP_REG;
3772 c->dst.addr.reg = &ctxt->eflags;
3773 c->dst.bytes = c->op_bytes;
3774 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3776 case 0xa6 ... 0xa7: /* cmps */
3778 case 0xa8 ... 0xa9: /* test ax, imm */
3780 case 0xae ... 0xaf: /* scas */
3785 case 0xc3: /* ret */
3786 c->dst.type = OP_REG;
3787 c->dst.addr.reg = &c->eip;
3788 c->dst.bytes = c->op_bytes;
3789 goto pop_instruction;
3790 case 0xc4: /* les */
3791 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3793 case 0xc5: /* lds */
3794 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3796 case 0xcb: /* ret far */
3797 rc = emulate_ret_far(ctxt, ops);
3799 case 0xcc: /* int3 */
3802 case 0xcd: /* int n */
3805 rc = emulate_int(ctxt, ops, irq);
3807 case 0xce: /* into */
3808 if (ctxt->eflags & EFLG_OF) {
3813 case 0xcf: /* iret */
3814 rc = emulate_iret(ctxt, ops);
3816 case 0xd0 ... 0xd1: /* Grp2 */
3819 case 0xd2 ... 0xd3: /* Grp2 */
3820 c->src.val = c->regs[VCPU_REGS_RCX];
3823 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3824 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3825 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3826 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3827 jmp_rel(c, c->src.val);
3829 case 0xe3: /* jcxz/jecxz/jrcxz */
3830 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3831 jmp_rel(c, c->src.val);
3833 case 0xe4: /* inb */
3836 case 0xe6: /* outb */
3837 case 0xe7: /* out */
3839 case 0xe8: /* call (near) */ {
3840 long int rel = c->src.val;
3841 c->src.val = (unsigned long) c->eip;
3846 case 0xe9: /* jmp rel */
3848 case 0xea: { /* jmp far */
3851 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3853 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3857 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3861 jmp: /* jmp rel short */
3862 jmp_rel(c, c->src.val);
3863 c->dst.type = OP_NONE; /* Disable writeback. */
3865 case 0xec: /* in al,dx */
3866 case 0xed: /* in (e/r)ax,dx */
3867 c->src.val = c->regs[VCPU_REGS_RDX];
3869 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3871 goto done; /* IO is needed */
3873 case 0xee: /* out dx,al */
3874 case 0xef: /* out dx,(e/r)ax */
3875 c->dst.val = c->regs[VCPU_REGS_RDX];
3877 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3878 &c->src.val, 1, ctxt->vcpu);
3879 c->dst.type = OP_NONE; /* Disable writeback. */
3881 case 0xf4: /* hlt */
3882 ctxt->vcpu->arch.halt_request = 1;
3884 case 0xf5: /* cmc */
3885 /* complement carry flag from eflags reg */
3886 ctxt->eflags ^= EFLG_CF;
3888 case 0xf6 ... 0xf7: /* Grp3 */
3889 rc = emulate_grp3(ctxt, ops);
3891 case 0xf8: /* clc */
3892 ctxt->eflags &= ~EFLG_CF;
3894 case 0xf9: /* stc */
3895 ctxt->eflags |= EFLG_CF;
3897 case 0xfa: /* cli */
3898 if (emulator_bad_iopl(ctxt, ops)) {
3899 rc = emulate_gp(ctxt, 0);
3902 ctxt->eflags &= ~X86_EFLAGS_IF;
3904 case 0xfb: /* sti */
3905 if (emulator_bad_iopl(ctxt, ops)) {
3906 rc = emulate_gp(ctxt, 0);
3909 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3910 ctxt->eflags |= X86_EFLAGS_IF;
3913 case 0xfc: /* cld */
3914 ctxt->eflags &= ~EFLG_DF;
3916 case 0xfd: /* std */
3917 ctxt->eflags |= EFLG_DF;
3919 case 0xfe: /* Grp4 */
3921 rc = emulate_grp45(ctxt);
3923 case 0xff: /* Grp5 */
3924 if (c->modrm_reg == 5)
3928 goto cannot_emulate;
3931 if (rc != X86EMUL_CONTINUE)
3935 rc = writeback(ctxt, ops);
3936 if (rc != X86EMUL_CONTINUE)
3940 * restore dst type in case the decoding will be reused
3941 * (happens for string instruction )
3943 c->dst.type = saved_dst_type;
3945 if ((c->d & SrcMask) == SrcSI)
3946 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3947 VCPU_REGS_RSI, &c->src);
3949 if ((c->d & DstMask) == DstDI)
3950 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3953 if (c->rep_prefix && (c->d & String)) {
3954 struct read_cache *r = &ctxt->decode.io_read;
3955 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3957 if (!string_insn_completed(ctxt)) {
3959 * Re-enter guest when pio read ahead buffer is empty
3960 * or, if it is not used, after each 1024 iteration.
3962 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3963 (r->end == 0 || r->end != r->pos)) {
3965 * Reset read cache. Usually happens before
3966 * decode, but since instruction is restarted
3967 * we have to do it here.
3969 ctxt->decode.mem_read.end = 0;
3970 return EMULATION_RESTART;
3972 goto done; /* skip rip writeback */
3979 if (rc == X86EMUL_PROPAGATE_FAULT)
3980 ctxt->have_exception = true;
3981 if (rc == X86EMUL_INTERCEPTED)
3982 return EMULATION_INTERCEPTED;
3984 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3988 case 0x01: /* lgdt, lidt, lmsw */
3989 switch (c->modrm_reg) {
3991 unsigned long address;
3993 case 0: /* vmcall */
3994 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3995 goto cannot_emulate;
3997 rc = kvm_fix_hypercall(ctxt->vcpu);
3998 if (rc != X86EMUL_CONTINUE)
4001 /* Let the processor re-execute the fixed hypercall */
4003 /* Disable writeback. */
4004 c->dst.type = OP_NONE;
4007 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4008 &size, &address, c->op_bytes);
4009 if (rc != X86EMUL_CONTINUE)
4011 realmode_lgdt(ctxt->vcpu, size, address);
4012 /* Disable writeback. */
4013 c->dst.type = OP_NONE;
4015 case 3: /* lidt/vmmcall */
4016 if (c->modrm_mod == 3) {
4017 switch (c->modrm_rm) {
4019 rc = kvm_fix_hypercall(ctxt->vcpu);
4022 goto cannot_emulate;
4025 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4028 if (rc != X86EMUL_CONTINUE)
4030 realmode_lidt(ctxt->vcpu, size, address);
4032 /* Disable writeback. */
4033 c->dst.type = OP_NONE;
4037 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4040 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4041 (c->src.val & 0x0f), ctxt->vcpu);
4042 c->dst.type = OP_NONE;
4044 case 5: /* not defined */
4046 rc = X86EMUL_PROPAGATE_FAULT;
4049 rc = em_invlpg(ctxt);
4052 goto cannot_emulate;
4055 case 0x05: /* syscall */
4056 rc = emulate_syscall(ctxt, ops);
4059 emulate_clts(ctxt->vcpu);
4061 case 0x09: /* wbinvd */
4062 kvm_emulate_wbinvd(ctxt->vcpu);
4064 case 0x08: /* invd */
4065 case 0x0d: /* GrpP (prefetch) */
4066 case 0x18: /* Grp16 (prefetch/nop) */
4068 case 0x20: /* mov cr, reg */
4069 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4071 case 0x21: /* mov from dr to reg */
4072 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4074 case 0x22: /* mov reg, cr */
4075 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4076 emulate_gp(ctxt, 0);
4077 rc = X86EMUL_PROPAGATE_FAULT;
4080 c->dst.type = OP_NONE;
4082 case 0x23: /* mov from reg to dr */
4083 if (ops->set_dr(c->modrm_reg, c->src.val &
4084 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4085 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4086 /* #UD condition is already handled by the code above */
4087 emulate_gp(ctxt, 0);
4088 rc = X86EMUL_PROPAGATE_FAULT;
4092 c->dst.type = OP_NONE; /* no writeback */
4096 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4097 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4098 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4099 emulate_gp(ctxt, 0);
4100 rc = X86EMUL_PROPAGATE_FAULT;
4103 rc = X86EMUL_CONTINUE;
4107 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4108 emulate_gp(ctxt, 0);
4109 rc = X86EMUL_PROPAGATE_FAULT;
4112 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4113 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4115 rc = X86EMUL_CONTINUE;
4117 case 0x34: /* sysenter */
4118 rc = emulate_sysenter(ctxt, ops);
4120 case 0x35: /* sysexit */
4121 rc = emulate_sysexit(ctxt, ops);
4123 case 0x40 ... 0x4f: /* cmov */
4124 c->dst.val = c->dst.orig_val = c->src.val;
4125 if (!test_cc(c->b, ctxt->eflags))
4126 c->dst.type = OP_NONE; /* no writeback */
4128 case 0x80 ... 0x8f: /* jnz rel, etc*/
4129 if (test_cc(c->b, ctxt->eflags))
4130 jmp_rel(c, c->src.val);
4132 case 0x90 ... 0x9f: /* setcc r/m8 */
4133 c->dst.val = test_cc(c->b, ctxt->eflags);
4135 case 0xa0: /* push fs */
4136 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4138 case 0xa1: /* pop fs */
4139 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4143 c->dst.type = OP_NONE;
4144 /* only subword offset */
4145 c->src.val &= (c->dst.bytes << 3) - 1;
4146 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4148 case 0xa4: /* shld imm8, r, r/m */
4149 case 0xa5: /* shld cl, r, r/m */
4150 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4152 case 0xa8: /* push gs */
4153 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4155 case 0xa9: /* pop gs */
4156 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4160 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4162 case 0xac: /* shrd imm8, r, r/m */
4163 case 0xad: /* shrd cl, r, r/m */
4164 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4166 case 0xae: /* clflush */
4168 case 0xb0 ... 0xb1: /* cmpxchg */
4170 * Save real source value, then compare EAX against
4173 c->src.orig_val = c->src.val;
4174 c->src.val = c->regs[VCPU_REGS_RAX];
4175 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4176 if (ctxt->eflags & EFLG_ZF) {
4177 /* Success: write back to memory. */
4178 c->dst.val = c->src.orig_val;
4180 /* Failure: write the value we saw to EAX. */
4181 c->dst.type = OP_REG;
4182 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4185 case 0xb2: /* lss */
4186 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4190 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4192 case 0xb4: /* lfs */
4193 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4195 case 0xb5: /* lgs */
4196 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4198 case 0xb6 ... 0xb7: /* movzx */
4199 c->dst.bytes = c->op_bytes;
4200 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4203 case 0xba: /* Grp8 */
4204 switch (c->modrm_reg & 3) {
4217 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4219 case 0xbc: { /* bsf */
4221 __asm__ ("bsf %2, %0; setz %1"
4222 : "=r"(c->dst.val), "=q"(zf)
4224 ctxt->eflags &= ~X86_EFLAGS_ZF;
4226 ctxt->eflags |= X86_EFLAGS_ZF;
4227 c->dst.type = OP_NONE; /* Disable writeback. */
4231 case 0xbd: { /* bsr */
4233 __asm__ ("bsr %2, %0; setz %1"
4234 : "=r"(c->dst.val), "=q"(zf)
4236 ctxt->eflags &= ~X86_EFLAGS_ZF;
4238 ctxt->eflags |= X86_EFLAGS_ZF;
4239 c->dst.type = OP_NONE; /* Disable writeback. */
4243 case 0xbe ... 0xbf: /* movsx */
4244 c->dst.bytes = c->op_bytes;
4245 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4248 case 0xc0 ... 0xc1: /* xadd */
4249 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4250 /* Write back the register source. */
4251 c->src.val = c->dst.orig_val;
4252 write_register_operand(&c->src);
4254 case 0xc3: /* movnti */
4255 c->dst.bytes = c->op_bytes;
4256 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4259 case 0xc7: /* Grp9 (cmpxchg8b) */
4260 rc = emulate_grp9(ctxt, ops);
4263 goto cannot_emulate;
4266 if (rc != X86EMUL_CONTINUE)
4272 return EMULATION_FAILED;