1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
367 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
371 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
375 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
379 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
385 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
387 switch((_src).bytes) { \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "b", _ex); \
393 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
394 _eflags, "w", _ex); \
397 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
398 _eflags, "l", _ex); \
401 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
402 _eflags, "q", _ex)); \
407 /* Fetch next part of the instruction being emulated. */
408 #define insn_fetch(_type, _size, _eip) \
409 ({ unsigned long _x; \
410 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
411 if (rc != X86EMUL_CONTINUE) \
417 #define insn_fetch_arr(_arr, _size, _eip) \
418 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
419 if (rc != X86EMUL_CONTINUE) \
424 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
425 enum x86_intercept intercept,
426 enum x86_intercept_stage stage)
428 struct x86_instruction_info info = {
429 .intercept = intercept,
430 .rep_prefix = ctxt->decode.rep_prefix,
431 .modrm_mod = ctxt->decode.modrm_mod,
432 .modrm_reg = ctxt->decode.modrm_reg,
433 .modrm_rm = ctxt->decode.modrm_rm,
434 .src_val = ctxt->decode.src.val64,
435 .src_bytes = ctxt->decode.src.bytes,
436 .dst_bytes = ctxt->decode.dst.bytes,
437 .ad_bytes = ctxt->decode.ad_bytes,
438 .next_rip = ctxt->eip,
441 return ctxt->ops->intercept(ctxt, &info, stage);
444 static inline unsigned long ad_mask(struct decode_cache *c)
446 return (1UL << (c->ad_bytes << 3)) - 1;
449 /* Access/update address held in a register, based on addressing mode. */
450 static inline unsigned long
451 address_mask(struct decode_cache *c, unsigned long reg)
453 if (c->ad_bytes == sizeof(unsigned long))
456 return reg & ad_mask(c);
459 static inline unsigned long
460 register_address(struct decode_cache *c, unsigned long reg)
462 return address_mask(c, reg);
466 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
468 if (c->ad_bytes == sizeof(unsigned long))
471 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
474 static inline void jmp_rel(struct decode_cache *c, int rel)
476 register_address_increment(c, &c->eip, rel);
479 static u32 desc_limit_scaled(struct desc_struct *desc)
481 u32 limit = get_desc_limit(desc);
483 return desc->g ? (limit << 12) | 0xfff : limit;
486 static void set_seg_override(struct decode_cache *c, int seg)
488 c->has_seg_override = true;
489 c->seg_override = seg;
492 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
493 struct x86_emulate_ops *ops, int seg)
495 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
498 return ops->get_cached_segment_base(ctxt, seg);
501 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
502 struct x86_emulate_ops *ops,
503 struct decode_cache *c)
505 if (!c->has_seg_override)
508 return c->seg_override;
511 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
512 u32 error, bool valid)
514 ctxt->exception.vector = vec;
515 ctxt->exception.error_code = error;
516 ctxt->exception.error_code_valid = valid;
517 return X86EMUL_PROPAGATE_FAULT;
520 static int emulate_db(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, DB_VECTOR, 0, false);
525 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, GP_VECTOR, err, true);
530 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
532 return emulate_exception(ctxt, SS_VECTOR, err, true);
535 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, UD_VECTOR, 0, false);
540 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, TS_VECTOR, err, true);
545 static int emulate_de(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DE_VECTOR, 0, false);
550 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
552 return emulate_exception(ctxt, NM_VECTOR, 0, false);
555 static int __linearize(struct x86_emulate_ctxt *ctxt,
556 struct segmented_address addr,
557 unsigned size, bool write, bool fetch,
560 struct decode_cache *c = &ctxt->decode;
561 struct desc_struct desc;
567 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
568 switch (ctxt->mode) {
569 case X86EMUL_MODE_REAL:
571 case X86EMUL_MODE_PROT64:
572 if (((signed long)la << 16) >> 16 != la)
573 return emulate_gp(ctxt, 0);
576 usable = ctxt->ops->get_cached_descriptor(ctxt, &desc, NULL,
580 /* code segment or read-only data segment */
581 if (((desc.type & 8) || !(desc.type & 2)) && write)
583 /* unreadable code segment */
584 if (!fetch && (desc.type & 8) && !(desc.type & 2))
586 lim = desc_limit_scaled(&desc);
587 if ((desc.type & 8) || !(desc.type & 4)) {
588 /* expand-up segment */
589 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
592 /* exapand-down segment */
593 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
595 lim = desc.d ? 0xffffffff : 0xffff;
596 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
599 cpl = ctxt->ops->cpl(ctxt);
600 rpl = ctxt->ops->get_segment_selector(ctxt, addr.seg) & 3;
602 if (!(desc.type & 8)) {
606 } else if ((desc.type & 8) && !(desc.type & 4)) {
607 /* nonconforming code segment */
610 } else if ((desc.type & 8) && (desc.type & 4)) {
611 /* conforming code segment */
617 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
620 return X86EMUL_CONTINUE;
622 if (addr.seg == VCPU_SREG_SS)
623 return emulate_ss(ctxt, addr.seg);
625 return emulate_gp(ctxt, addr.seg);
628 static int linearize(struct x86_emulate_ctxt *ctxt,
629 struct segmented_address addr,
630 unsigned size, bool write,
633 return __linearize(ctxt, addr, size, write, false, linear);
637 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
638 struct segmented_address addr,
645 rc = linearize(ctxt, addr, size, false, &linear);
646 if (rc != X86EMUL_CONTINUE)
648 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
651 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
652 struct x86_emulate_ops *ops,
653 unsigned long eip, u8 *dest)
655 struct fetch_cache *fc = &ctxt->decode.fetch;
659 if (eip == fc->end) {
660 unsigned long linear;
661 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
662 cur_size = fc->end - fc->start;
663 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
664 rc = __linearize(ctxt, addr, size, false, true, &linear);
665 if (rc != X86EMUL_CONTINUE)
667 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
668 size, &ctxt->exception);
669 if (rc != X86EMUL_CONTINUE)
673 *dest = fc->data[eip - fc->start];
674 return X86EMUL_CONTINUE;
677 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
678 struct x86_emulate_ops *ops,
679 unsigned long eip, void *dest, unsigned size)
683 /* x86 instructions are limited to 15 bytes. */
684 if (eip + size - ctxt->eip > 15)
685 return X86EMUL_UNHANDLEABLE;
687 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
688 if (rc != X86EMUL_CONTINUE)
691 return X86EMUL_CONTINUE;
695 * Given the 'reg' portion of a ModRM byte, and a register block, return a
696 * pointer into the block that addresses the relevant register.
697 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
699 static void *decode_register(u8 modrm_reg, unsigned long *regs,
704 p = ®s[modrm_reg];
705 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
706 p = (unsigned char *)®s[modrm_reg & 3] + 1;
710 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
711 struct x86_emulate_ops *ops,
712 struct segmented_address addr,
713 u16 *size, unsigned long *address, int op_bytes)
720 rc = segmented_read_std(ctxt, addr, size, 2);
721 if (rc != X86EMUL_CONTINUE)
724 rc = segmented_read_std(ctxt, addr, address, op_bytes);
728 static int test_cc(unsigned int condition, unsigned int flags)
732 switch ((condition & 15) >> 1) {
734 rc |= (flags & EFLG_OF);
736 case 1: /* b/c/nae */
737 rc |= (flags & EFLG_CF);
740 rc |= (flags & EFLG_ZF);
743 rc |= (flags & (EFLG_CF|EFLG_ZF));
746 rc |= (flags & EFLG_SF);
749 rc |= (flags & EFLG_PF);
752 rc |= (flags & EFLG_ZF);
755 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
759 /* Odd condition identifiers (lsb == 1) have inverted sense. */
760 return (!!rc ^ (condition & 1));
763 static void fetch_register_operand(struct operand *op)
767 op->val = *(u8 *)op->addr.reg;
770 op->val = *(u16 *)op->addr.reg;
773 op->val = *(u32 *)op->addr.reg;
776 op->val = *(u64 *)op->addr.reg;
781 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
783 ctxt->ops->get_fpu(ctxt);
785 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
786 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
787 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
788 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
789 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
790 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
791 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
792 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
794 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
795 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
796 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
797 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
798 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
799 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
800 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
801 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
805 ctxt->ops->put_fpu(ctxt);
808 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
811 ctxt->ops->get_fpu(ctxt);
813 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
814 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
815 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
816 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
817 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
818 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
819 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
820 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
822 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
823 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
824 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
825 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
826 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
827 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
828 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
829 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
833 ctxt->ops->put_fpu(ctxt);
836 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
838 struct decode_cache *c,
841 unsigned reg = c->modrm_reg;
842 int highbyte_regs = c->rex_prefix == 0;
845 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
851 read_sse_reg(ctxt, &op->vec_val, reg);
856 if ((c->d & ByteOp) && !inhibit_bytereg) {
857 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
860 op->addr.reg = decode_register(reg, c->regs, 0);
861 op->bytes = c->op_bytes;
863 fetch_register_operand(op);
864 op->orig_val = op->val;
867 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
868 struct x86_emulate_ops *ops,
871 struct decode_cache *c = &ctxt->decode;
873 int index_reg = 0, base_reg = 0, scale;
874 int rc = X86EMUL_CONTINUE;
878 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
879 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
880 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
883 c->modrm = insn_fetch(u8, 1, c->eip);
884 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
885 c->modrm_reg |= (c->modrm & 0x38) >> 3;
886 c->modrm_rm |= (c->modrm & 0x07);
887 c->modrm_seg = VCPU_SREG_DS;
889 if (c->modrm_mod == 3) {
891 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
892 op->addr.reg = decode_register(c->modrm_rm,
893 c->regs, c->d & ByteOp);
897 op->addr.xmm = c->modrm_rm;
898 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
901 fetch_register_operand(op);
907 if (c->ad_bytes == 2) {
908 unsigned bx = c->regs[VCPU_REGS_RBX];
909 unsigned bp = c->regs[VCPU_REGS_RBP];
910 unsigned si = c->regs[VCPU_REGS_RSI];
911 unsigned di = c->regs[VCPU_REGS_RDI];
913 /* 16-bit ModR/M decode. */
914 switch (c->modrm_mod) {
916 if (c->modrm_rm == 6)
917 modrm_ea += insn_fetch(u16, 2, c->eip);
920 modrm_ea += insn_fetch(s8, 1, c->eip);
923 modrm_ea += insn_fetch(u16, 2, c->eip);
926 switch (c->modrm_rm) {
946 if (c->modrm_mod != 0)
953 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
954 (c->modrm_rm == 6 && c->modrm_mod != 0))
955 c->modrm_seg = VCPU_SREG_SS;
956 modrm_ea = (u16)modrm_ea;
958 /* 32/64-bit ModR/M decode. */
959 if ((c->modrm_rm & 7) == 4) {
960 sib = insn_fetch(u8, 1, c->eip);
961 index_reg |= (sib >> 3) & 7;
965 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
966 modrm_ea += insn_fetch(s32, 4, c->eip);
968 modrm_ea += c->regs[base_reg];
970 modrm_ea += c->regs[index_reg] << scale;
971 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
972 if (ctxt->mode == X86EMUL_MODE_PROT64)
975 modrm_ea += c->regs[c->modrm_rm];
976 switch (c->modrm_mod) {
978 if (c->modrm_rm == 5)
979 modrm_ea += insn_fetch(s32, 4, c->eip);
982 modrm_ea += insn_fetch(s8, 1, c->eip);
985 modrm_ea += insn_fetch(s32, 4, c->eip);
989 op->addr.mem.ea = modrm_ea;
994 static int decode_abs(struct x86_emulate_ctxt *ctxt,
995 struct x86_emulate_ops *ops,
998 struct decode_cache *c = &ctxt->decode;
999 int rc = X86EMUL_CONTINUE;
1002 switch (c->ad_bytes) {
1004 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1007 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1010 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1017 static void fetch_bit_operand(struct decode_cache *c)
1021 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1022 mask = ~(c->dst.bytes * 8 - 1);
1024 if (c->src.bytes == 2)
1025 sv = (s16)c->src.val & (s16)mask;
1026 else if (c->src.bytes == 4)
1027 sv = (s32)c->src.val & (s32)mask;
1029 c->dst.addr.mem.ea += (sv >> 3);
1032 /* only subword offset */
1033 c->src.val &= (c->dst.bytes << 3) - 1;
1036 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1037 struct x86_emulate_ops *ops,
1038 unsigned long addr, void *dest, unsigned size)
1041 struct read_cache *mc = &ctxt->decode.mem_read;
1044 int n = min(size, 8u);
1046 if (mc->pos < mc->end)
1049 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1051 if (rc != X86EMUL_CONTINUE)
1056 memcpy(dest, mc->data + mc->pos, n);
1061 return X86EMUL_CONTINUE;
1064 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1065 struct segmented_address addr,
1072 rc = linearize(ctxt, addr, size, false, &linear);
1073 if (rc != X86EMUL_CONTINUE)
1075 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1078 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1079 struct segmented_address addr,
1086 rc = linearize(ctxt, addr, size, true, &linear);
1087 if (rc != X86EMUL_CONTINUE)
1089 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1093 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1094 struct segmented_address addr,
1095 const void *orig_data, const void *data,
1101 rc = linearize(ctxt, addr, size, true, &linear);
1102 if (rc != X86EMUL_CONTINUE)
1104 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1105 size, &ctxt->exception);
1108 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1109 struct x86_emulate_ops *ops,
1110 unsigned int size, unsigned short port,
1113 struct read_cache *rc = &ctxt->decode.io_read;
1115 if (rc->pos == rc->end) { /* refill pio read ahead */
1116 struct decode_cache *c = &ctxt->decode;
1117 unsigned int in_page, n;
1118 unsigned int count = c->rep_prefix ?
1119 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1120 in_page = (ctxt->eflags & EFLG_DF) ?
1121 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1122 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1123 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1127 rc->pos = rc->end = 0;
1128 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1133 memcpy(dest, rc->data + rc->pos, size);
1138 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1139 struct x86_emulate_ops *ops,
1140 u16 selector, struct desc_ptr *dt)
1142 if (selector & 1 << 2) {
1143 struct desc_struct desc;
1144 memset (dt, 0, sizeof *dt);
1145 if (!ops->get_cached_descriptor(ctxt, &desc, NULL,
1149 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1150 dt->address = get_desc_base(&desc);
1152 ops->get_gdt(ctxt, dt);
1155 /* allowed just for 8 bytes segments */
1156 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1157 struct x86_emulate_ops *ops,
1158 u16 selector, struct desc_struct *desc)
1161 u16 index = selector >> 3;
1165 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1167 if (dt.size < index * 8 + 7)
1168 return emulate_gp(ctxt, selector & 0xfffc);
1169 addr = dt.address + index * 8;
1170 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1175 /* allowed just for 8 bytes segments */
1176 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1177 struct x86_emulate_ops *ops,
1178 u16 selector, struct desc_struct *desc)
1181 u16 index = selector >> 3;
1185 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1187 if (dt.size < index * 8 + 7)
1188 return emulate_gp(ctxt, selector & 0xfffc);
1190 addr = dt.address + index * 8;
1191 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1196 /* Does not support long mode */
1197 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1198 struct x86_emulate_ops *ops,
1199 u16 selector, int seg)
1201 struct desc_struct seg_desc;
1203 unsigned err_vec = GP_VECTOR;
1205 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1208 memset(&seg_desc, 0, sizeof seg_desc);
1210 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1211 || ctxt->mode == X86EMUL_MODE_REAL) {
1212 /* set real mode segment descriptor */
1213 set_desc_base(&seg_desc, selector << 4);
1214 set_desc_limit(&seg_desc, 0xffff);
1221 /* NULL selector is not valid for TR, CS and SS */
1222 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1226 /* TR should be in GDT only */
1227 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1230 if (null_selector) /* for NULL selector skip all following checks */
1233 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1234 if (ret != X86EMUL_CONTINUE)
1237 err_code = selector & 0xfffc;
1238 err_vec = GP_VECTOR;
1240 /* can't load system descriptor into segment selecor */
1241 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1245 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1251 cpl = ops->cpl(ctxt);
1256 * segment is not a writable data segment or segment
1257 * selector's RPL != CPL or segment selector's RPL != CPL
1259 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1263 if (!(seg_desc.type & 8))
1266 if (seg_desc.type & 4) {
1272 if (rpl > cpl || dpl != cpl)
1275 /* CS(RPL) <- CPL */
1276 selector = (selector & 0xfffc) | cpl;
1279 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1282 case VCPU_SREG_LDTR:
1283 if (seg_desc.s || seg_desc.type != 2)
1286 default: /* DS, ES, FS, or GS */
1288 * segment is not a data or readable code segment or
1289 * ((segment is a data or nonconforming code segment)
1290 * and (both RPL and CPL > DPL))
1292 if ((seg_desc.type & 0xa) == 0x8 ||
1293 (((seg_desc.type & 0xc) != 0xc) &&
1294 (rpl > dpl && cpl > dpl)))
1300 /* mark segment as accessed */
1302 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1303 if (ret != X86EMUL_CONTINUE)
1307 ops->set_segment_selector(ctxt, selector, seg);
1308 ops->set_cached_descriptor(ctxt, &seg_desc, 0, seg);
1309 return X86EMUL_CONTINUE;
1311 emulate_exception(ctxt, err_vec, err_code, true);
1312 return X86EMUL_PROPAGATE_FAULT;
1315 static void write_register_operand(struct operand *op)
1317 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1318 switch (op->bytes) {
1320 *(u8 *)op->addr.reg = (u8)op->val;
1323 *(u16 *)op->addr.reg = (u16)op->val;
1326 *op->addr.reg = (u32)op->val;
1327 break; /* 64b: zero-extend */
1329 *op->addr.reg = op->val;
1334 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1335 struct x86_emulate_ops *ops)
1338 struct decode_cache *c = &ctxt->decode;
1340 switch (c->dst.type) {
1342 write_register_operand(&c->dst);
1346 rc = segmented_cmpxchg(ctxt,
1352 rc = segmented_write(ctxt,
1356 if (rc != X86EMUL_CONTINUE)
1360 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1368 return X86EMUL_CONTINUE;
1371 static int em_push(struct x86_emulate_ctxt *ctxt)
1373 struct decode_cache *c = &ctxt->decode;
1374 struct segmented_address addr;
1376 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1377 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1378 addr.seg = VCPU_SREG_SS;
1380 /* Disable writeback. */
1381 c->dst.type = OP_NONE;
1382 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1385 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1386 struct x86_emulate_ops *ops,
1387 void *dest, int len)
1389 struct decode_cache *c = &ctxt->decode;
1391 struct segmented_address addr;
1393 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1394 addr.seg = VCPU_SREG_SS;
1395 rc = segmented_read(ctxt, addr, dest, len);
1396 if (rc != X86EMUL_CONTINUE)
1399 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1403 static int em_pop(struct x86_emulate_ctxt *ctxt)
1405 struct decode_cache *c = &ctxt->decode;
1407 return emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1410 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1411 struct x86_emulate_ops *ops,
1412 void *dest, int len)
1415 unsigned long val, change_mask;
1416 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1417 int cpl = ops->cpl(ctxt);
1419 rc = emulate_pop(ctxt, ops, &val, len);
1420 if (rc != X86EMUL_CONTINUE)
1423 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1424 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1426 switch(ctxt->mode) {
1427 case X86EMUL_MODE_PROT64:
1428 case X86EMUL_MODE_PROT32:
1429 case X86EMUL_MODE_PROT16:
1431 change_mask |= EFLG_IOPL;
1433 change_mask |= EFLG_IF;
1435 case X86EMUL_MODE_VM86:
1437 return emulate_gp(ctxt, 0);
1438 change_mask |= EFLG_IF;
1440 default: /* real mode */
1441 change_mask |= (EFLG_IOPL | EFLG_IF);
1445 *(unsigned long *)dest =
1446 (ctxt->eflags & ~change_mask) | (val & change_mask);
1451 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1452 struct x86_emulate_ops *ops, int seg)
1454 struct decode_cache *c = &ctxt->decode;
1456 c->src.val = ops->get_segment_selector(ctxt, seg);
1458 return em_push(ctxt);
1461 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1462 struct x86_emulate_ops *ops, int seg)
1464 struct decode_cache *c = &ctxt->decode;
1465 unsigned long selector;
1468 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1469 if (rc != X86EMUL_CONTINUE)
1472 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1476 static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1478 struct decode_cache *c = &ctxt->decode;
1479 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1480 int rc = X86EMUL_CONTINUE;
1481 int reg = VCPU_REGS_RAX;
1483 while (reg <= VCPU_REGS_RDI) {
1484 (reg == VCPU_REGS_RSP) ?
1485 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1488 if (rc != X86EMUL_CONTINUE)
1497 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1498 struct x86_emulate_ops *ops)
1500 struct decode_cache *c = &ctxt->decode;
1501 int rc = X86EMUL_CONTINUE;
1502 int reg = VCPU_REGS_RDI;
1504 while (reg >= VCPU_REGS_RAX) {
1505 if (reg == VCPU_REGS_RSP) {
1506 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1511 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1512 if (rc != X86EMUL_CONTINUE)
1519 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1520 struct x86_emulate_ops *ops, int irq)
1522 struct decode_cache *c = &ctxt->decode;
1529 /* TODO: Add limit checks */
1530 c->src.val = ctxt->eflags;
1532 if (rc != X86EMUL_CONTINUE)
1535 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1537 c->src.val = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
1539 if (rc != X86EMUL_CONTINUE)
1542 c->src.val = c->eip;
1544 if (rc != X86EMUL_CONTINUE)
1547 ops->get_idt(ctxt, &dt);
1549 eip_addr = dt.address + (irq << 2);
1550 cs_addr = dt.address + (irq << 2) + 2;
1552 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1553 if (rc != X86EMUL_CONTINUE)
1556 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1557 if (rc != X86EMUL_CONTINUE)
1560 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1561 if (rc != X86EMUL_CONTINUE)
1569 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1570 struct x86_emulate_ops *ops, int irq)
1572 switch(ctxt->mode) {
1573 case X86EMUL_MODE_REAL:
1574 return emulate_int_real(ctxt, ops, irq);
1575 case X86EMUL_MODE_VM86:
1576 case X86EMUL_MODE_PROT16:
1577 case X86EMUL_MODE_PROT32:
1578 case X86EMUL_MODE_PROT64:
1580 /* Protected mode interrupts unimplemented yet */
1581 return X86EMUL_UNHANDLEABLE;
1585 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1586 struct x86_emulate_ops *ops)
1588 struct decode_cache *c = &ctxt->decode;
1589 int rc = X86EMUL_CONTINUE;
1590 unsigned long temp_eip = 0;
1591 unsigned long temp_eflags = 0;
1592 unsigned long cs = 0;
1593 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1594 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1595 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1596 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1598 /* TODO: Add stack limit check */
1600 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1602 if (rc != X86EMUL_CONTINUE)
1605 if (temp_eip & ~0xffff)
1606 return emulate_gp(ctxt, 0);
1608 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1610 if (rc != X86EMUL_CONTINUE)
1613 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1615 if (rc != X86EMUL_CONTINUE)
1618 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1620 if (rc != X86EMUL_CONTINUE)
1626 if (c->op_bytes == 4)
1627 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1628 else if (c->op_bytes == 2) {
1629 ctxt->eflags &= ~0xffff;
1630 ctxt->eflags |= temp_eflags;
1633 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1634 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1639 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1640 struct x86_emulate_ops* ops)
1642 switch(ctxt->mode) {
1643 case X86EMUL_MODE_REAL:
1644 return emulate_iret_real(ctxt, ops);
1645 case X86EMUL_MODE_VM86:
1646 case X86EMUL_MODE_PROT16:
1647 case X86EMUL_MODE_PROT32:
1648 case X86EMUL_MODE_PROT64:
1650 /* iret from protected mode unimplemented yet */
1651 return X86EMUL_UNHANDLEABLE;
1655 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1656 struct x86_emulate_ops *ops)
1658 struct decode_cache *c = &ctxt->decode;
1660 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1663 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1665 struct decode_cache *c = &ctxt->decode;
1666 switch (c->modrm_reg) {
1668 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1671 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1674 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1677 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1679 case 4: /* sal/shl */
1680 case 6: /* sal/shl */
1681 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1684 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1687 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1692 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1693 struct x86_emulate_ops *ops)
1695 struct decode_cache *c = &ctxt->decode;
1696 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1697 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1700 switch (c->modrm_reg) {
1701 case 0 ... 1: /* test */
1702 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1705 c->dst.val = ~c->dst.val;
1708 emulate_1op("neg", c->dst, ctxt->eflags);
1711 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1714 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1717 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1721 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1725 return X86EMUL_UNHANDLEABLE;
1728 return emulate_de(ctxt);
1729 return X86EMUL_CONTINUE;
1732 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1734 struct decode_cache *c = &ctxt->decode;
1735 int rc = X86EMUL_CONTINUE;
1737 switch (c->modrm_reg) {
1739 emulate_1op("inc", c->dst, ctxt->eflags);
1742 emulate_1op("dec", c->dst, ctxt->eflags);
1744 case 2: /* call near abs */ {
1747 c->eip = c->src.val;
1748 c->src.val = old_eip;
1752 case 4: /* jmp abs */
1753 c->eip = c->src.val;
1762 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1763 struct x86_emulate_ops *ops)
1765 struct decode_cache *c = &ctxt->decode;
1766 u64 old = c->dst.orig_val64;
1768 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1769 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1770 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1771 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1772 ctxt->eflags &= ~EFLG_ZF;
1774 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1775 (u32) c->regs[VCPU_REGS_RBX];
1777 ctxt->eflags |= EFLG_ZF;
1779 return X86EMUL_CONTINUE;
1782 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1783 struct x86_emulate_ops *ops)
1785 struct decode_cache *c = &ctxt->decode;
1789 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1790 if (rc != X86EMUL_CONTINUE)
1792 if (c->op_bytes == 4)
1793 c->eip = (u32)c->eip;
1794 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1795 if (rc != X86EMUL_CONTINUE)
1797 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1801 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1802 struct x86_emulate_ops *ops, int seg)
1804 struct decode_cache *c = &ctxt->decode;
1808 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1810 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1811 if (rc != X86EMUL_CONTINUE)
1814 c->dst.val = c->src.val;
1819 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1820 struct x86_emulate_ops *ops, struct desc_struct *cs,
1821 struct desc_struct *ss)
1823 memset(cs, 0, sizeof(struct desc_struct));
1824 ops->get_cached_descriptor(ctxt, cs, NULL, VCPU_SREG_CS);
1825 memset(ss, 0, sizeof(struct desc_struct));
1827 cs->l = 0; /* will be adjusted later */
1828 set_desc_base(cs, 0); /* flat segment */
1829 cs->g = 1; /* 4kb granularity */
1830 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1831 cs->type = 0x0b; /* Read, Execute, Accessed */
1833 cs->dpl = 0; /* will be adjusted later */
1837 set_desc_base(ss, 0); /* flat segment */
1838 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1839 ss->g = 1; /* 4kb granularity */
1841 ss->type = 0x03; /* Read/Write, Accessed */
1842 ss->d = 1; /* 32bit stack segment */
1848 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1850 struct decode_cache *c = &ctxt->decode;
1851 struct desc_struct cs, ss;
1856 /* syscall is not available in real mode */
1857 if (ctxt->mode == X86EMUL_MODE_REAL ||
1858 ctxt->mode == X86EMUL_MODE_VM86)
1859 return emulate_ud(ctxt);
1861 ops->get_msr(ctxt, MSR_EFER, &efer);
1862 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1864 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1866 cs_sel = (u16)(msr_data & 0xfffc);
1867 ss_sel = (u16)(msr_data + 8);
1869 if (efer & EFER_LMA) {
1873 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
1874 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
1875 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
1876 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1878 c->regs[VCPU_REGS_RCX] = c->eip;
1879 if (efer & EFER_LMA) {
1880 #ifdef CONFIG_X86_64
1881 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1884 ctxt->mode == X86EMUL_MODE_PROT64 ?
1885 MSR_LSTAR : MSR_CSTAR, &msr_data);
1888 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1889 ctxt->eflags &= ~(msr_data | EFLG_RF);
1893 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1894 c->eip = (u32)msr_data;
1896 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1899 return X86EMUL_CONTINUE;
1903 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1905 struct decode_cache *c = &ctxt->decode;
1906 struct desc_struct cs, ss;
1911 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1912 /* inject #GP if in real mode */
1913 if (ctxt->mode == X86EMUL_MODE_REAL)
1914 return emulate_gp(ctxt, 0);
1916 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1917 * Therefore, we inject an #UD.
1919 if (ctxt->mode == X86EMUL_MODE_PROT64)
1920 return emulate_ud(ctxt);
1922 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1924 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1925 switch (ctxt->mode) {
1926 case X86EMUL_MODE_PROT32:
1927 if ((msr_data & 0xfffc) == 0x0)
1928 return emulate_gp(ctxt, 0);
1930 case X86EMUL_MODE_PROT64:
1931 if (msr_data == 0x0)
1932 return emulate_gp(ctxt, 0);
1936 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1937 cs_sel = (u16)msr_data;
1938 cs_sel &= ~SELECTOR_RPL_MASK;
1939 ss_sel = cs_sel + 8;
1940 ss_sel &= ~SELECTOR_RPL_MASK;
1941 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1946 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
1947 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
1948 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
1949 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
1951 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1954 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1955 c->regs[VCPU_REGS_RSP] = msr_data;
1957 return X86EMUL_CONTINUE;
1961 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1963 struct decode_cache *c = &ctxt->decode;
1964 struct desc_struct cs, ss;
1969 /* inject #GP if in real mode or Virtual 8086 mode */
1970 if (ctxt->mode == X86EMUL_MODE_REAL ||
1971 ctxt->mode == X86EMUL_MODE_VM86)
1972 return emulate_gp(ctxt, 0);
1974 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1976 if ((c->rex_prefix & 0x8) != 0x0)
1977 usermode = X86EMUL_MODE_PROT64;
1979 usermode = X86EMUL_MODE_PROT32;
1983 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1985 case X86EMUL_MODE_PROT32:
1986 cs_sel = (u16)(msr_data + 16);
1987 if ((msr_data & 0xfffc) == 0x0)
1988 return emulate_gp(ctxt, 0);
1989 ss_sel = (u16)(msr_data + 24);
1991 case X86EMUL_MODE_PROT64:
1992 cs_sel = (u16)(msr_data + 32);
1993 if (msr_data == 0x0)
1994 return emulate_gp(ctxt, 0);
1995 ss_sel = cs_sel + 8;
2000 cs_sel |= SELECTOR_RPL_MASK;
2001 ss_sel |= SELECTOR_RPL_MASK;
2003 ops->set_cached_descriptor(ctxt, &cs, 0, VCPU_SREG_CS);
2004 ops->set_segment_selector(ctxt, cs_sel, VCPU_SREG_CS);
2005 ops->set_cached_descriptor(ctxt, &ss, 0, VCPU_SREG_SS);
2006 ops->set_segment_selector(ctxt, ss_sel, VCPU_SREG_SS);
2008 c->eip = c->regs[VCPU_REGS_RDX];
2009 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2011 return X86EMUL_CONTINUE;
2014 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2015 struct x86_emulate_ops *ops)
2018 if (ctxt->mode == X86EMUL_MODE_REAL)
2020 if (ctxt->mode == X86EMUL_MODE_VM86)
2022 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2023 return ops->cpl(ctxt) > iopl;
2026 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2027 struct x86_emulate_ops *ops,
2030 struct desc_struct tr_seg;
2033 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2034 unsigned mask = (1 << len) - 1;
2037 ops->get_cached_descriptor(ctxt, &tr_seg, &base3, VCPU_SREG_TR);
2040 if (desc_limit_scaled(&tr_seg) < 103)
2042 base = get_desc_base(&tr_seg);
2043 #ifdef CONFIG_X86_64
2044 base |= ((u64)base3) << 32;
2046 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2047 if (r != X86EMUL_CONTINUE)
2049 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2051 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2052 if (r != X86EMUL_CONTINUE)
2054 if ((perm >> bit_idx) & mask)
2059 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2060 struct x86_emulate_ops *ops,
2066 if (emulator_bad_iopl(ctxt, ops))
2067 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2070 ctxt->perm_ok = true;
2075 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2076 struct x86_emulate_ops *ops,
2077 struct tss_segment_16 *tss)
2079 struct decode_cache *c = &ctxt->decode;
2082 tss->flag = ctxt->eflags;
2083 tss->ax = c->regs[VCPU_REGS_RAX];
2084 tss->cx = c->regs[VCPU_REGS_RCX];
2085 tss->dx = c->regs[VCPU_REGS_RDX];
2086 tss->bx = c->regs[VCPU_REGS_RBX];
2087 tss->sp = c->regs[VCPU_REGS_RSP];
2088 tss->bp = c->regs[VCPU_REGS_RBP];
2089 tss->si = c->regs[VCPU_REGS_RSI];
2090 tss->di = c->regs[VCPU_REGS_RDI];
2092 tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
2093 tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2094 tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
2095 tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
2096 tss->ldt = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2099 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2100 struct x86_emulate_ops *ops,
2101 struct tss_segment_16 *tss)
2103 struct decode_cache *c = &ctxt->decode;
2107 ctxt->eflags = tss->flag | 2;
2108 c->regs[VCPU_REGS_RAX] = tss->ax;
2109 c->regs[VCPU_REGS_RCX] = tss->cx;
2110 c->regs[VCPU_REGS_RDX] = tss->dx;
2111 c->regs[VCPU_REGS_RBX] = tss->bx;
2112 c->regs[VCPU_REGS_RSP] = tss->sp;
2113 c->regs[VCPU_REGS_RBP] = tss->bp;
2114 c->regs[VCPU_REGS_RSI] = tss->si;
2115 c->regs[VCPU_REGS_RDI] = tss->di;
2118 * SDM says that segment selectors are loaded before segment
2121 ops->set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2122 ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2123 ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2124 ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2125 ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2128 * Now load segment descriptors. If fault happenes at this stage
2129 * it is handled in a context of new task
2131 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2132 if (ret != X86EMUL_CONTINUE)
2134 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2135 if (ret != X86EMUL_CONTINUE)
2137 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2138 if (ret != X86EMUL_CONTINUE)
2140 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2141 if (ret != X86EMUL_CONTINUE)
2143 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2144 if (ret != X86EMUL_CONTINUE)
2147 return X86EMUL_CONTINUE;
2150 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2151 struct x86_emulate_ops *ops,
2152 u16 tss_selector, u16 old_tss_sel,
2153 ulong old_tss_base, struct desc_struct *new_desc)
2155 struct tss_segment_16 tss_seg;
2157 u32 new_tss_base = get_desc_base(new_desc);
2159 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2161 if (ret != X86EMUL_CONTINUE)
2162 /* FIXME: need to provide precise fault address */
2165 save_state_to_tss16(ctxt, ops, &tss_seg);
2167 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2169 if (ret != X86EMUL_CONTINUE)
2170 /* FIXME: need to provide precise fault address */
2173 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2175 if (ret != X86EMUL_CONTINUE)
2176 /* FIXME: need to provide precise fault address */
2179 if (old_tss_sel != 0xffff) {
2180 tss_seg.prev_task_link = old_tss_sel;
2182 ret = ops->write_std(ctxt, new_tss_base,
2183 &tss_seg.prev_task_link,
2184 sizeof tss_seg.prev_task_link,
2186 if (ret != X86EMUL_CONTINUE)
2187 /* FIXME: need to provide precise fault address */
2191 return load_state_from_tss16(ctxt, ops, &tss_seg);
2194 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2195 struct x86_emulate_ops *ops,
2196 struct tss_segment_32 *tss)
2198 struct decode_cache *c = &ctxt->decode;
2200 tss->cr3 = ops->get_cr(ctxt, 3);
2202 tss->eflags = ctxt->eflags;
2203 tss->eax = c->regs[VCPU_REGS_RAX];
2204 tss->ecx = c->regs[VCPU_REGS_RCX];
2205 tss->edx = c->regs[VCPU_REGS_RDX];
2206 tss->ebx = c->regs[VCPU_REGS_RBX];
2207 tss->esp = c->regs[VCPU_REGS_RSP];
2208 tss->ebp = c->regs[VCPU_REGS_RBP];
2209 tss->esi = c->regs[VCPU_REGS_RSI];
2210 tss->edi = c->regs[VCPU_REGS_RDI];
2212 tss->es = ops->get_segment_selector(ctxt, VCPU_SREG_ES);
2213 tss->cs = ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2214 tss->ss = ops->get_segment_selector(ctxt, VCPU_SREG_SS);
2215 tss->ds = ops->get_segment_selector(ctxt, VCPU_SREG_DS);
2216 tss->fs = ops->get_segment_selector(ctxt, VCPU_SREG_FS);
2217 tss->gs = ops->get_segment_selector(ctxt, VCPU_SREG_GS);
2218 tss->ldt_selector = ops->get_segment_selector(ctxt, VCPU_SREG_LDTR);
2221 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2222 struct x86_emulate_ops *ops,
2223 struct tss_segment_32 *tss)
2225 struct decode_cache *c = &ctxt->decode;
2228 if (ops->set_cr(ctxt, 3, tss->cr3))
2229 return emulate_gp(ctxt, 0);
2231 ctxt->eflags = tss->eflags | 2;
2232 c->regs[VCPU_REGS_RAX] = tss->eax;
2233 c->regs[VCPU_REGS_RCX] = tss->ecx;
2234 c->regs[VCPU_REGS_RDX] = tss->edx;
2235 c->regs[VCPU_REGS_RBX] = tss->ebx;
2236 c->regs[VCPU_REGS_RSP] = tss->esp;
2237 c->regs[VCPU_REGS_RBP] = tss->ebp;
2238 c->regs[VCPU_REGS_RSI] = tss->esi;
2239 c->regs[VCPU_REGS_RDI] = tss->edi;
2242 * SDM says that segment selectors are loaded before segment
2245 ops->set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2246 ops->set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2247 ops->set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2248 ops->set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2249 ops->set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2250 ops->set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2251 ops->set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2254 * Now load segment descriptors. If fault happenes at this stage
2255 * it is handled in a context of new task
2257 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2258 if (ret != X86EMUL_CONTINUE)
2260 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2261 if (ret != X86EMUL_CONTINUE)
2263 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2264 if (ret != X86EMUL_CONTINUE)
2266 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2267 if (ret != X86EMUL_CONTINUE)
2269 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2270 if (ret != X86EMUL_CONTINUE)
2272 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2273 if (ret != X86EMUL_CONTINUE)
2275 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2276 if (ret != X86EMUL_CONTINUE)
2279 return X86EMUL_CONTINUE;
2282 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2283 struct x86_emulate_ops *ops,
2284 u16 tss_selector, u16 old_tss_sel,
2285 ulong old_tss_base, struct desc_struct *new_desc)
2287 struct tss_segment_32 tss_seg;
2289 u32 new_tss_base = get_desc_base(new_desc);
2291 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2293 if (ret != X86EMUL_CONTINUE)
2294 /* FIXME: need to provide precise fault address */
2297 save_state_to_tss32(ctxt, ops, &tss_seg);
2299 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2301 if (ret != X86EMUL_CONTINUE)
2302 /* FIXME: need to provide precise fault address */
2305 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2307 if (ret != X86EMUL_CONTINUE)
2308 /* FIXME: need to provide precise fault address */
2311 if (old_tss_sel != 0xffff) {
2312 tss_seg.prev_task_link = old_tss_sel;
2314 ret = ops->write_std(ctxt, new_tss_base,
2315 &tss_seg.prev_task_link,
2316 sizeof tss_seg.prev_task_link,
2318 if (ret != X86EMUL_CONTINUE)
2319 /* FIXME: need to provide precise fault address */
2323 return load_state_from_tss32(ctxt, ops, &tss_seg);
2326 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2327 struct x86_emulate_ops *ops,
2328 u16 tss_selector, int reason,
2329 bool has_error_code, u32 error_code)
2331 struct desc_struct curr_tss_desc, next_tss_desc;
2333 u16 old_tss_sel = ops->get_segment_selector(ctxt, VCPU_SREG_TR);
2334 ulong old_tss_base =
2335 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2338 /* FIXME: old_tss_base == ~0 ? */
2340 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2341 if (ret != X86EMUL_CONTINUE)
2343 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2344 if (ret != X86EMUL_CONTINUE)
2347 /* FIXME: check that next_tss_desc is tss */
2349 if (reason != TASK_SWITCH_IRET) {
2350 if ((tss_selector & 3) > next_tss_desc.dpl ||
2351 ops->cpl(ctxt) > next_tss_desc.dpl)
2352 return emulate_gp(ctxt, 0);
2355 desc_limit = desc_limit_scaled(&next_tss_desc);
2356 if (!next_tss_desc.p ||
2357 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2358 desc_limit < 0x2b)) {
2359 emulate_ts(ctxt, tss_selector & 0xfffc);
2360 return X86EMUL_PROPAGATE_FAULT;
2363 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2364 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2365 write_segment_descriptor(ctxt, ops, old_tss_sel,
2369 if (reason == TASK_SWITCH_IRET)
2370 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2372 /* set back link to prev task only if NT bit is set in eflags
2373 note that old_tss_sel is not used afetr this point */
2374 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2375 old_tss_sel = 0xffff;
2377 if (next_tss_desc.type & 8)
2378 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2379 old_tss_base, &next_tss_desc);
2381 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2382 old_tss_base, &next_tss_desc);
2383 if (ret != X86EMUL_CONTINUE)
2386 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2387 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2389 if (reason != TASK_SWITCH_IRET) {
2390 next_tss_desc.type |= (1 << 1); /* set busy flag */
2391 write_segment_descriptor(ctxt, ops, tss_selector,
2395 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2396 ops->set_cached_descriptor(ctxt, &next_tss_desc, 0, VCPU_SREG_TR);
2397 ops->set_segment_selector(ctxt, tss_selector, VCPU_SREG_TR);
2399 if (has_error_code) {
2400 struct decode_cache *c = &ctxt->decode;
2402 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2404 c->src.val = (unsigned long) error_code;
2405 ret = em_push(ctxt);
2411 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2412 u16 tss_selector, int reason,
2413 bool has_error_code, u32 error_code)
2415 struct x86_emulate_ops *ops = ctxt->ops;
2416 struct decode_cache *c = &ctxt->decode;
2420 c->dst.type = OP_NONE;
2422 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2423 has_error_code, error_code);
2425 if (rc == X86EMUL_CONTINUE)
2428 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2431 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2432 int reg, struct operand *op)
2434 struct decode_cache *c = &ctxt->decode;
2435 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2437 register_address_increment(c, &c->regs[reg], df * op->bytes);
2438 op->addr.mem.ea = register_address(c, c->regs[reg]);
2439 op->addr.mem.seg = seg;
2442 static int em_das(struct x86_emulate_ctxt *ctxt)
2444 struct decode_cache *c = &ctxt->decode;
2446 bool af, cf, old_cf;
2448 cf = ctxt->eflags & X86_EFLAGS_CF;
2454 af = ctxt->eflags & X86_EFLAGS_AF;
2455 if ((al & 0x0f) > 9 || af) {
2457 cf = old_cf | (al >= 250);
2462 if (old_al > 0x99 || old_cf) {
2468 /* Set PF, ZF, SF */
2469 c->src.type = OP_IMM;
2472 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2473 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2475 ctxt->eflags |= X86_EFLAGS_CF;
2477 ctxt->eflags |= X86_EFLAGS_AF;
2478 return X86EMUL_CONTINUE;
2481 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2483 struct decode_cache *c = &ctxt->decode;
2488 old_cs = ctxt->ops->get_segment_selector(ctxt, VCPU_SREG_CS);
2491 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2492 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2493 return X86EMUL_CONTINUE;
2496 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2498 c->src.val = old_cs;
2500 if (rc != X86EMUL_CONTINUE)
2503 c->src.val = old_eip;
2504 return em_push(ctxt);
2507 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2509 struct decode_cache *c = &ctxt->decode;
2512 c->dst.type = OP_REG;
2513 c->dst.addr.reg = &c->eip;
2514 c->dst.bytes = c->op_bytes;
2515 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2516 if (rc != X86EMUL_CONTINUE)
2518 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2519 return X86EMUL_CONTINUE;
2522 static int em_add(struct x86_emulate_ctxt *ctxt)
2524 struct decode_cache *c = &ctxt->decode;
2526 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2527 return X86EMUL_CONTINUE;
2530 static int em_or(struct x86_emulate_ctxt *ctxt)
2532 struct decode_cache *c = &ctxt->decode;
2534 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2535 return X86EMUL_CONTINUE;
2538 static int em_adc(struct x86_emulate_ctxt *ctxt)
2540 struct decode_cache *c = &ctxt->decode;
2542 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2543 return X86EMUL_CONTINUE;
2546 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2548 struct decode_cache *c = &ctxt->decode;
2550 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2551 return X86EMUL_CONTINUE;
2554 static int em_and(struct x86_emulate_ctxt *ctxt)
2556 struct decode_cache *c = &ctxt->decode;
2558 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2559 return X86EMUL_CONTINUE;
2562 static int em_sub(struct x86_emulate_ctxt *ctxt)
2564 struct decode_cache *c = &ctxt->decode;
2566 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2567 return X86EMUL_CONTINUE;
2570 static int em_xor(struct x86_emulate_ctxt *ctxt)
2572 struct decode_cache *c = &ctxt->decode;
2574 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2575 return X86EMUL_CONTINUE;
2578 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2580 struct decode_cache *c = &ctxt->decode;
2582 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2583 /* Disable writeback. */
2584 c->dst.type = OP_NONE;
2585 return X86EMUL_CONTINUE;
2588 static int em_imul(struct x86_emulate_ctxt *ctxt)
2590 struct decode_cache *c = &ctxt->decode;
2592 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2593 return X86EMUL_CONTINUE;
2596 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2598 struct decode_cache *c = &ctxt->decode;
2600 c->dst.val = c->src2.val;
2601 return em_imul(ctxt);
2604 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2606 struct decode_cache *c = &ctxt->decode;
2608 c->dst.type = OP_REG;
2609 c->dst.bytes = c->src.bytes;
2610 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2611 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2613 return X86EMUL_CONTINUE;
2616 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2618 struct decode_cache *c = &ctxt->decode;
2621 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2622 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2623 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2624 return X86EMUL_CONTINUE;
2627 static int em_mov(struct x86_emulate_ctxt *ctxt)
2629 struct decode_cache *c = &ctxt->decode;
2630 c->dst.val = c->src.val;
2631 return X86EMUL_CONTINUE;
2634 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2636 struct decode_cache *c = &ctxt->decode;
2637 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2638 return X86EMUL_CONTINUE;
2641 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2643 struct decode_cache *c = &ctxt->decode;
2647 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2648 if (rc == X86EMUL_CONTINUE)
2649 ctxt->ops->invlpg(ctxt, linear);
2650 /* Disable writeback. */
2651 c->dst.type = OP_NONE;
2652 return X86EMUL_CONTINUE;
2655 static int em_clts(struct x86_emulate_ctxt *ctxt)
2659 cr0 = ctxt->ops->get_cr(ctxt, 0);
2661 ctxt->ops->set_cr(ctxt, 0, cr0);
2662 return X86EMUL_CONTINUE;
2665 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2667 struct decode_cache *c = &ctxt->decode;
2670 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2671 return X86EMUL_UNHANDLEABLE;
2673 rc = ctxt->ops->fix_hypercall(ctxt);
2674 if (rc != X86EMUL_CONTINUE)
2677 /* Let the processor re-execute the fixed hypercall */
2679 /* Disable writeback. */
2680 c->dst.type = OP_NONE;
2681 return X86EMUL_CONTINUE;
2684 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2686 struct decode_cache *c = &ctxt->decode;
2687 struct desc_ptr desc_ptr;
2690 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2691 &desc_ptr.size, &desc_ptr.address,
2693 if (rc != X86EMUL_CONTINUE)
2695 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2696 /* Disable writeback. */
2697 c->dst.type = OP_NONE;
2698 return X86EMUL_CONTINUE;
2701 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2703 struct decode_cache *c = &ctxt->decode;
2706 rc = ctxt->ops->fix_hypercall(ctxt);
2708 /* Disable writeback. */
2709 c->dst.type = OP_NONE;
2713 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2715 struct decode_cache *c = &ctxt->decode;
2716 struct desc_ptr desc_ptr;
2719 rc = read_descriptor(ctxt, ctxt->ops, c->src.addr.mem,
2723 if (rc != X86EMUL_CONTINUE)
2725 ctxt->ops->set_idt(ctxt, &desc_ptr);
2726 /* Disable writeback. */
2727 c->dst.type = OP_NONE;
2728 return X86EMUL_CONTINUE;
2731 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2733 struct decode_cache *c = &ctxt->decode;
2736 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2737 return X86EMUL_CONTINUE;
2740 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2742 struct decode_cache *c = &ctxt->decode;
2743 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2744 | (c->src.val & 0x0f));
2745 c->dst.type = OP_NONE;
2746 return X86EMUL_CONTINUE;
2749 static bool valid_cr(int nr)
2761 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2763 struct decode_cache *c = &ctxt->decode;
2765 if (!valid_cr(c->modrm_reg))
2766 return emulate_ud(ctxt);
2768 return X86EMUL_CONTINUE;
2771 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2773 struct decode_cache *c = &ctxt->decode;
2774 u64 new_val = c->src.val64;
2775 int cr = c->modrm_reg;
2778 static u64 cr_reserved_bits[] = {
2779 0xffffffff00000000ULL,
2780 0, 0, 0, /* CR3 checked later */
2787 return emulate_ud(ctxt);
2789 if (new_val & cr_reserved_bits[cr])
2790 return emulate_gp(ctxt, 0);
2795 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2796 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2797 return emulate_gp(ctxt, 0);
2799 cr4 = ctxt->ops->get_cr(ctxt, 4);
2800 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2802 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2803 !(cr4 & X86_CR4_PAE))
2804 return emulate_gp(ctxt, 0);
2811 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2812 if (efer & EFER_LMA)
2813 rsvd = CR3_L_MODE_RESERVED_BITS;
2814 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2815 rsvd = CR3_PAE_RESERVED_BITS;
2816 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2817 rsvd = CR3_NONPAE_RESERVED_BITS;
2820 return emulate_gp(ctxt, 0);
2827 cr4 = ctxt->ops->get_cr(ctxt, 4);
2828 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2830 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2831 return emulate_gp(ctxt, 0);
2837 return X86EMUL_CONTINUE;
2840 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2844 ctxt->ops->get_dr(ctxt, 7, &dr7);
2846 /* Check if DR7.Global_Enable is set */
2847 return dr7 & (1 << 13);
2850 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2852 struct decode_cache *c = &ctxt->decode;
2853 int dr = c->modrm_reg;
2857 return emulate_ud(ctxt);
2859 cr4 = ctxt->ops->get_cr(ctxt, 4);
2860 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2861 return emulate_ud(ctxt);
2863 if (check_dr7_gd(ctxt))
2864 return emulate_db(ctxt);
2866 return X86EMUL_CONTINUE;
2869 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2871 struct decode_cache *c = &ctxt->decode;
2872 u64 new_val = c->src.val64;
2873 int dr = c->modrm_reg;
2875 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2876 return emulate_gp(ctxt, 0);
2878 return check_dr_read(ctxt);
2881 static int check_svme(struct x86_emulate_ctxt *ctxt)
2885 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2887 if (!(efer & EFER_SVME))
2888 return emulate_ud(ctxt);
2890 return X86EMUL_CONTINUE;
2893 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2895 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2897 /* Valid physical address? */
2898 if (rax & 0xffff000000000000ULL)
2899 return emulate_gp(ctxt, 0);
2901 return check_svme(ctxt);
2904 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2906 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2908 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2909 return emulate_ud(ctxt);
2911 return X86EMUL_CONTINUE;
2914 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2916 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2917 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2919 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2921 return emulate_gp(ctxt, 0);
2923 return X86EMUL_CONTINUE;
2926 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2928 struct decode_cache *c = &ctxt->decode;
2930 c->dst.bytes = min(c->dst.bytes, 4u);
2931 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2932 return emulate_gp(ctxt, 0);
2934 return X86EMUL_CONTINUE;
2937 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2939 struct decode_cache *c = &ctxt->decode;
2941 c->src.bytes = min(c->src.bytes, 4u);
2942 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2943 return emulate_gp(ctxt, 0);
2945 return X86EMUL_CONTINUE;
2948 #define D(_y) { .flags = (_y) }
2949 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2950 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2951 .check_perm = (_p) }
2953 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2954 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2955 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2956 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2957 #define II(_f, _e, _i) \
2958 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2959 #define IIP(_f, _e, _i, _p) \
2960 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2961 .check_perm = (_p) }
2962 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2964 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2965 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2966 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2968 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2969 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2970 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2972 static struct opcode group7_rm1[] = {
2973 DI(SrcNone | ModRM | Priv, monitor),
2974 DI(SrcNone | ModRM | Priv, mwait),
2978 static struct opcode group7_rm3[] = {
2979 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2980 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2981 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2982 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2983 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2984 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2985 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2986 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2989 static struct opcode group7_rm7[] = {
2991 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2995 static struct opcode group1[] = {
3006 static struct opcode group1A[] = {
3007 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3010 static struct opcode group3[] = {
3011 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3012 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3013 X4(D(SrcMem | ModRM)),
3016 static struct opcode group4[] = {
3017 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3021 static struct opcode group5[] = {
3022 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3023 D(SrcMem | ModRM | Stack),
3024 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3025 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3026 D(SrcMem | ModRM | Stack), N,
3029 static struct opcode group6[] = {
3030 DI(ModRM | Prot, sldt),
3031 DI(ModRM | Prot, str),
3032 DI(ModRM | Prot | Priv, lldt),
3033 DI(ModRM | Prot | Priv, ltr),
3037 static struct group_dual group7 = { {
3038 DI(ModRM | Mov | DstMem | Priv, sgdt),
3039 DI(ModRM | Mov | DstMem | Priv, sidt),
3040 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3041 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3042 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3043 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3044 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3046 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3048 N, EXT(0, group7_rm3),
3049 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3050 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3053 static struct opcode group8[] = {
3055 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3056 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3059 static struct group_dual group9 = { {
3060 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3062 N, N, N, N, N, N, N, N,
3065 static struct opcode group11[] = {
3066 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3069 static struct gprefix pfx_0f_6f_0f_7f = {
3070 N, N, N, I(Sse, em_movdqu),
3073 static struct opcode opcode_table[256] = {
3075 I6ALU(Lock, em_add),
3076 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3079 D(ImplicitOps | Stack | No64), N,
3081 I6ALU(Lock, em_adc),
3082 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3084 I6ALU(Lock, em_sbb),
3085 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3087 I6ALU(Lock, em_and), N, N,
3089 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3091 I6ALU(Lock, em_xor), N, N,
3093 I6ALU(0, em_cmp), N, N,
3097 X8(I(SrcReg | Stack, em_push)),
3099 X8(I(DstReg | Stack, em_pop)),
3101 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3102 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3105 I(SrcImm | Mov | Stack, em_push),
3106 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3107 I(SrcImmByte | Mov | Stack, em_push),
3108 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3109 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3110 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3114 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3115 G(DstMem | SrcImm | ModRM | Group, group1),
3116 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3117 G(DstMem | SrcImmByte | ModRM | Group, group1),
3118 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3120 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3121 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3122 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3123 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3125 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3127 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3128 I(SrcImmFAddr | No64, em_call_far), N,
3129 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
3131 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3132 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3133 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3134 I2bv(SrcSI | DstDI | String, em_cmp),
3136 D2bv(DstAcc | SrcImm),
3137 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3138 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3139 I2bv(SrcAcc | DstDI | String, em_cmp),
3141 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3143 X8(I(DstReg | SrcImm | Mov, em_mov)),
3145 D2bv(DstMem | SrcImmByte | ModRM),
3146 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3147 D(ImplicitOps | Stack),
3148 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3149 G(ByteOp, group11), G(0, group11),
3151 N, N, N, D(ImplicitOps | Stack),
3152 D(ImplicitOps), DI(SrcImmByte, intn),
3153 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3155 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3158 N, N, N, N, N, N, N, N,
3161 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3162 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3164 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3165 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3166 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3167 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3169 N, DI(ImplicitOps, icebp), N, N,
3170 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3171 G(ByteOp, group3), G(0, group3),
3173 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3174 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3177 static struct opcode twobyte_table[256] = {
3179 G(0, group6), GD(0, &group7), N, N,
3180 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3181 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3182 N, D(ImplicitOps | ModRM), N, N,
3184 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3186 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3187 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3188 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3189 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3191 N, N, N, N, N, N, N, N,
3193 DI(ImplicitOps | Priv, wrmsr),
3194 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3195 DI(ImplicitOps | Priv, rdmsr),
3196 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3197 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3199 N, N, N, N, N, N, N, N,
3201 X16(D(DstReg | SrcMem | ModRM | Mov)),
3203 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3208 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3213 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3217 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3219 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3220 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3221 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3222 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3224 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3225 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3226 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3227 D(DstMem | SrcReg | Src2CL | ModRM),
3228 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3230 D2bv(DstMem | SrcReg | ModRM | Lock),
3231 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3232 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3233 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3236 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3237 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3238 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3240 D2bv(DstMem | SrcReg | ModRM | Lock),
3241 N, D(DstMem | SrcReg | ModRM | Mov),
3242 N, N, N, GD(0, &group9),
3243 N, N, N, N, N, N, N, N,
3245 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3247 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3249 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3265 static unsigned imm_size(struct decode_cache *c)
3269 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3275 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3276 unsigned size, bool sign_extension)
3278 struct decode_cache *c = &ctxt->decode;
3279 struct x86_emulate_ops *ops = ctxt->ops;
3280 int rc = X86EMUL_CONTINUE;
3284 op->addr.mem.ea = c->eip;
3285 /* NB. Immediates are sign-extended as necessary. */
3286 switch (op->bytes) {
3288 op->val = insn_fetch(s8, 1, c->eip);
3291 op->val = insn_fetch(s16, 2, c->eip);
3294 op->val = insn_fetch(s32, 4, c->eip);
3297 if (!sign_extension) {
3298 switch (op->bytes) {
3306 op->val &= 0xffffffff;
3315 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3317 struct x86_emulate_ops *ops = ctxt->ops;
3318 struct decode_cache *c = &ctxt->decode;
3319 int rc = X86EMUL_CONTINUE;
3320 int mode = ctxt->mode;
3321 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3322 bool op_prefix = false;
3323 struct opcode opcode, *g_mod012, *g_mod3;
3324 struct operand memop = { .type = OP_NONE };
3327 c->fetch.start = c->eip;
3328 c->fetch.end = c->fetch.start + insn_len;
3330 memcpy(c->fetch.data, insn, insn_len);
3333 case X86EMUL_MODE_REAL:
3334 case X86EMUL_MODE_VM86:
3335 case X86EMUL_MODE_PROT16:
3336 def_op_bytes = def_ad_bytes = 2;
3338 case X86EMUL_MODE_PROT32:
3339 def_op_bytes = def_ad_bytes = 4;
3341 #ifdef CONFIG_X86_64
3342 case X86EMUL_MODE_PROT64:
3351 c->op_bytes = def_op_bytes;
3352 c->ad_bytes = def_ad_bytes;
3354 /* Legacy prefixes. */
3356 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3357 case 0x66: /* operand-size override */
3359 /* switch between 2/4 bytes */
3360 c->op_bytes = def_op_bytes ^ 6;
3362 case 0x67: /* address-size override */
3363 if (mode == X86EMUL_MODE_PROT64)
3364 /* switch between 4/8 bytes */
3365 c->ad_bytes = def_ad_bytes ^ 12;
3367 /* switch between 2/4 bytes */
3368 c->ad_bytes = def_ad_bytes ^ 6;
3370 case 0x26: /* ES override */
3371 case 0x2e: /* CS override */
3372 case 0x36: /* SS override */
3373 case 0x3e: /* DS override */
3374 set_seg_override(c, (c->b >> 3) & 3);
3376 case 0x64: /* FS override */
3377 case 0x65: /* GS override */
3378 set_seg_override(c, c->b & 7);
3380 case 0x40 ... 0x4f: /* REX */
3381 if (mode != X86EMUL_MODE_PROT64)
3383 c->rex_prefix = c->b;
3385 case 0xf0: /* LOCK */
3388 case 0xf2: /* REPNE/REPNZ */
3389 case 0xf3: /* REP/REPE/REPZ */
3390 c->rep_prefix = c->b;
3396 /* Any legacy prefix after a REX prefix nullifies its effect. */
3404 if (c->rex_prefix & 8)
3405 c->op_bytes = 8; /* REX.W */
3407 /* Opcode byte(s). */
3408 opcode = opcode_table[c->b];
3409 /* Two-byte opcode? */
3412 c->b = insn_fetch(u8, 1, c->eip);
3413 opcode = twobyte_table[c->b];
3415 c->d = opcode.flags;
3418 dual = c->d & GroupDual;
3419 c->modrm = insn_fetch(u8, 1, c->eip);
3422 if (c->d & GroupDual) {
3423 g_mod012 = opcode.u.gdual->mod012;
3424 g_mod3 = opcode.u.gdual->mod3;
3426 g_mod012 = g_mod3 = opcode.u.group;
3428 c->d &= ~(Group | GroupDual);
3430 goffset = (c->modrm >> 3) & 7;
3432 if ((c->modrm >> 6) == 3)
3433 opcode = g_mod3[goffset];
3435 opcode = g_mod012[goffset];
3437 if (opcode.flags & RMExt) {
3438 goffset = c->modrm & 7;
3439 opcode = opcode.u.group[goffset];
3442 c->d |= opcode.flags;
3445 if (c->d & Prefix) {
3446 if (c->rep_prefix && op_prefix)
3447 return X86EMUL_UNHANDLEABLE;
3448 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3449 switch (simd_prefix) {
3450 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3451 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3452 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3453 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3455 c->d |= opcode.flags;
3458 c->execute = opcode.u.execute;
3459 c->check_perm = opcode.check_perm;
3460 c->intercept = opcode.intercept;
3463 if (c->d == 0 || (c->d & Undefined))
3466 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3469 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3472 if (c->d & Op3264) {
3473 if (mode == X86EMUL_MODE_PROT64)
3482 /* ModRM and SIB bytes. */
3484 rc = decode_modrm(ctxt, ops, &memop);
3485 if (!c->has_seg_override)
3486 set_seg_override(c, c->modrm_seg);
3487 } else if (c->d & MemAbs)
3488 rc = decode_abs(ctxt, ops, &memop);
3489 if (rc != X86EMUL_CONTINUE)
3492 if (!c->has_seg_override)
3493 set_seg_override(c, VCPU_SREG_DS);
3495 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3497 if (memop.type == OP_MEM && c->ad_bytes != 8)
3498 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3500 if (memop.type == OP_MEM && c->rip_relative)
3501 memop.addr.mem.ea += c->eip;
3504 * Decode and fetch the source operand: register, memory
3507 switch (c->d & SrcMask) {
3511 decode_register_operand(ctxt, &c->src, c, 0);
3520 memop.bytes = (c->d & ByteOp) ? 1 :
3526 rc = decode_imm(ctxt, &c->src, 2, false);
3529 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3532 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3535 rc = decode_imm(ctxt, &c->src, 1, true);
3538 rc = decode_imm(ctxt, &c->src, 1, false);
3541 c->src.type = OP_REG;
3542 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3543 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3544 fetch_register_operand(&c->src);
3551 c->src.type = OP_MEM;
3552 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3553 c->src.addr.mem.ea =
3554 register_address(c, c->regs[VCPU_REGS_RSI]);
3555 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3559 c->src.type = OP_IMM;
3560 c->src.addr.mem.ea = c->eip;
3561 c->src.bytes = c->op_bytes + 2;
3562 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3565 memop.bytes = c->op_bytes + 2;
3570 if (rc != X86EMUL_CONTINUE)
3574 * Decode and fetch the second source operand: register, memory
3577 switch (c->d & Src2Mask) {
3582 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3585 rc = decode_imm(ctxt, &c->src2, 1, true);
3592 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3596 if (rc != X86EMUL_CONTINUE)
3599 /* Decode and fetch the destination operand: register or memory. */
3600 switch (c->d & DstMask) {
3602 decode_register_operand(ctxt, &c->dst, c,
3603 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3606 c->dst.type = OP_IMM;
3607 c->dst.addr.mem.ea = c->eip;
3609 c->dst.val = insn_fetch(u8, 1, c->eip);
3614 if ((c->d & DstMask) == DstMem64)
3617 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3619 fetch_bit_operand(c);
3620 c->dst.orig_val = c->dst.val;
3623 c->dst.type = OP_REG;
3624 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3625 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3626 fetch_register_operand(&c->dst);
3627 c->dst.orig_val = c->dst.val;
3630 c->dst.type = OP_MEM;
3631 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3632 c->dst.addr.mem.ea =
3633 register_address(c, c->regs[VCPU_REGS_RDI]);
3634 c->dst.addr.mem.seg = VCPU_SREG_ES;
3638 /* Special instructions do their own operand decoding. */
3640 c->dst.type = OP_NONE; /* Disable writeback. */
3645 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3648 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3650 struct decode_cache *c = &ctxt->decode;
3652 /* The second termination condition only applies for REPE
3653 * and REPNE. Test if the repeat string operation prefix is
3654 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3655 * corresponding termination condition according to:
3656 * - if REPE/REPZ and ZF = 0 then done
3657 * - if REPNE/REPNZ and ZF = 1 then done
3659 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3660 (c->b == 0xae) || (c->b == 0xaf))
3661 && (((c->rep_prefix == REPE_PREFIX) &&
3662 ((ctxt->eflags & EFLG_ZF) == 0))
3663 || ((c->rep_prefix == REPNE_PREFIX) &&
3664 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3671 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3673 struct x86_emulate_ops *ops = ctxt->ops;
3675 struct decode_cache *c = &ctxt->decode;
3676 int rc = X86EMUL_CONTINUE;
3677 int saved_dst_type = c->dst.type;
3678 int irq; /* Used for int 3, int, and into */
3680 ctxt->decode.mem_read.pos = 0;
3682 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3683 rc = emulate_ud(ctxt);
3687 /* LOCK prefix is allowed only with some instructions */
3688 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3689 rc = emulate_ud(ctxt);
3693 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3694 rc = emulate_ud(ctxt);
3699 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3700 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3701 rc = emulate_ud(ctxt);
3705 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3706 rc = emulate_nm(ctxt);
3710 if (unlikely(ctxt->guest_mode) && c->intercept) {
3711 rc = emulator_check_intercept(ctxt, c->intercept,
3712 X86_ICPT_PRE_EXCEPT);
3713 if (rc != X86EMUL_CONTINUE)
3717 /* Privileged instruction can be executed only in CPL=0 */
3718 if ((c->d & Priv) && ops->cpl(ctxt)) {
3719 rc = emulate_gp(ctxt, 0);
3723 /* Instruction can only be executed in protected mode */
3724 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3725 rc = emulate_ud(ctxt);
3729 /* Do instruction specific permission checks */
3730 if (c->check_perm) {
3731 rc = c->check_perm(ctxt);
3732 if (rc != X86EMUL_CONTINUE)
3736 if (unlikely(ctxt->guest_mode) && c->intercept) {
3737 rc = emulator_check_intercept(ctxt, c->intercept,
3738 X86_ICPT_POST_EXCEPT);
3739 if (rc != X86EMUL_CONTINUE)
3743 if (c->rep_prefix && (c->d & String)) {
3744 /* All REP prefixes have the same first termination condition */
3745 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3751 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3752 rc = segmented_read(ctxt, c->src.addr.mem,
3753 c->src.valptr, c->src.bytes);
3754 if (rc != X86EMUL_CONTINUE)
3756 c->src.orig_val64 = c->src.val64;
3759 if (c->src2.type == OP_MEM) {
3760 rc = segmented_read(ctxt, c->src2.addr.mem,
3761 &c->src2.val, c->src2.bytes);
3762 if (rc != X86EMUL_CONTINUE)
3766 if ((c->d & DstMask) == ImplicitOps)
3770 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3771 /* optimisation - avoid slow emulated read if Mov */
3772 rc = segmented_read(ctxt, c->dst.addr.mem,
3773 &c->dst.val, c->dst.bytes);
3774 if (rc != X86EMUL_CONTINUE)
3777 c->dst.orig_val = c->dst.val;
3781 if (unlikely(ctxt->guest_mode) && c->intercept) {
3782 rc = emulator_check_intercept(ctxt, c->intercept,
3783 X86_ICPT_POST_MEMACCESS);
3784 if (rc != X86EMUL_CONTINUE)
3789 rc = c->execute(ctxt);
3790 if (rc != X86EMUL_CONTINUE)
3799 case 0x06: /* push es */
3800 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3802 case 0x07: /* pop es */
3803 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3805 case 0x0e: /* push cs */
3806 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3808 case 0x16: /* push ss */
3809 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3811 case 0x17: /* pop ss */
3812 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3814 case 0x1e: /* push ds */
3815 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3817 case 0x1f: /* pop ds */
3818 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3820 case 0x40 ... 0x47: /* inc r16/r32 */
3821 emulate_1op("inc", c->dst, ctxt->eflags);
3823 case 0x48 ... 0x4f: /* dec r16/r32 */
3824 emulate_1op("dec", c->dst, ctxt->eflags);
3826 case 0x60: /* pusha */
3827 rc = emulate_pusha(ctxt);
3829 case 0x61: /* popa */
3830 rc = emulate_popa(ctxt, ops);
3832 case 0x63: /* movsxd */
3833 if (ctxt->mode != X86EMUL_MODE_PROT64)
3834 goto cannot_emulate;
3835 c->dst.val = (s32) c->src.val;
3837 case 0x6c: /* insb */
3838 case 0x6d: /* insw/insd */
3839 c->src.val = c->regs[VCPU_REGS_RDX];
3841 case 0x6e: /* outsb */
3842 case 0x6f: /* outsw/outsd */
3843 c->dst.val = c->regs[VCPU_REGS_RDX];
3846 case 0x70 ... 0x7f: /* jcc (short) */
3847 if (test_cc(c->b, ctxt->eflags))
3848 jmp_rel(c, c->src.val);
3852 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3854 case 0x86 ... 0x87: /* xchg */
3856 /* Write back the register source. */
3857 c->src.val = c->dst.val;
3858 write_register_operand(&c->src);
3860 * Write back the memory destination with implicit LOCK
3863 c->dst.val = c->src.orig_val;
3866 case 0x8c: /* mov r/m, sreg */
3867 if (c->modrm_reg > VCPU_SREG_GS) {
3868 rc = emulate_ud(ctxt);
3871 c->dst.val = ops->get_segment_selector(ctxt, c->modrm_reg);
3873 case 0x8d: /* lea r16/r32, m */
3874 c->dst.val = c->src.addr.mem.ea;
3876 case 0x8e: { /* mov seg, r/m16 */
3881 if (c->modrm_reg == VCPU_SREG_CS ||
3882 c->modrm_reg > VCPU_SREG_GS) {
3883 rc = emulate_ud(ctxt);
3887 if (c->modrm_reg == VCPU_SREG_SS)
3888 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3890 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3892 c->dst.type = OP_NONE; /* Disable writeback. */
3895 case 0x8f: /* pop (sole member of Grp1a) */
3896 rc = emulate_grp1a(ctxt, ops);
3898 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3899 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3902 case 0x98: /* cbw/cwde/cdqe */
3903 switch (c->op_bytes) {
3904 case 2: c->dst.val = (s8)c->dst.val; break;
3905 case 4: c->dst.val = (s16)c->dst.val; break;
3906 case 8: c->dst.val = (s32)c->dst.val; break;
3909 case 0x9c: /* pushf */
3910 c->src.val = (unsigned long) ctxt->eflags;
3913 case 0x9d: /* popf */
3914 c->dst.type = OP_REG;
3915 c->dst.addr.reg = &ctxt->eflags;
3916 c->dst.bytes = c->op_bytes;
3917 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3919 case 0xa8 ... 0xa9: /* test ax, imm */
3924 case 0xc3: /* ret */
3925 c->dst.type = OP_REG;
3926 c->dst.addr.reg = &c->eip;
3927 c->dst.bytes = c->op_bytes;
3930 case 0xc4: /* les */
3931 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3933 case 0xc5: /* lds */
3934 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3936 case 0xcb: /* ret far */
3937 rc = emulate_ret_far(ctxt, ops);
3939 case 0xcc: /* int3 */
3942 case 0xcd: /* int n */
3945 rc = emulate_int(ctxt, ops, irq);
3947 case 0xce: /* into */
3948 if (ctxt->eflags & EFLG_OF) {
3953 case 0xcf: /* iret */
3954 rc = emulate_iret(ctxt, ops);
3956 case 0xd0 ... 0xd1: /* Grp2 */
3959 case 0xd2 ... 0xd3: /* Grp2 */
3960 c->src.val = c->regs[VCPU_REGS_RCX];
3963 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3964 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3965 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3966 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3967 jmp_rel(c, c->src.val);
3969 case 0xe3: /* jcxz/jecxz/jrcxz */
3970 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3971 jmp_rel(c, c->src.val);
3973 case 0xe4: /* inb */
3976 case 0xe6: /* outb */
3977 case 0xe7: /* out */
3979 case 0xe8: /* call (near) */ {
3980 long int rel = c->src.val;
3981 c->src.val = (unsigned long) c->eip;
3986 case 0xe9: /* jmp rel */
3988 case 0xea: { /* jmp far */
3991 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3993 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3997 memcpy(&c->eip, c->src.valptr, c->op_bytes);
4001 jmp: /* jmp rel short */
4002 jmp_rel(c, c->src.val);
4003 c->dst.type = OP_NONE; /* Disable writeback. */
4005 case 0xec: /* in al,dx */
4006 case 0xed: /* in (e/r)ax,dx */
4007 c->src.val = c->regs[VCPU_REGS_RDX];
4009 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4011 goto done; /* IO is needed */
4013 case 0xee: /* out dx,al */
4014 case 0xef: /* out dx,(e/r)ax */
4015 c->dst.val = c->regs[VCPU_REGS_RDX];
4017 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4019 c->dst.type = OP_NONE; /* Disable writeback. */
4021 case 0xf4: /* hlt */
4022 ctxt->ops->halt(ctxt);
4024 case 0xf5: /* cmc */
4025 /* complement carry flag from eflags reg */
4026 ctxt->eflags ^= EFLG_CF;
4028 case 0xf6 ... 0xf7: /* Grp3 */
4029 rc = emulate_grp3(ctxt, ops);
4031 case 0xf8: /* clc */
4032 ctxt->eflags &= ~EFLG_CF;
4034 case 0xf9: /* stc */
4035 ctxt->eflags |= EFLG_CF;
4037 case 0xfa: /* cli */
4038 if (emulator_bad_iopl(ctxt, ops)) {
4039 rc = emulate_gp(ctxt, 0);
4042 ctxt->eflags &= ~X86_EFLAGS_IF;
4044 case 0xfb: /* sti */
4045 if (emulator_bad_iopl(ctxt, ops)) {
4046 rc = emulate_gp(ctxt, 0);
4049 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4050 ctxt->eflags |= X86_EFLAGS_IF;
4053 case 0xfc: /* cld */
4054 ctxt->eflags &= ~EFLG_DF;
4056 case 0xfd: /* std */
4057 ctxt->eflags |= EFLG_DF;
4059 case 0xfe: /* Grp4 */
4061 rc = emulate_grp45(ctxt);
4063 case 0xff: /* Grp5 */
4064 if (c->modrm_reg == 5)
4068 goto cannot_emulate;
4071 if (rc != X86EMUL_CONTINUE)
4075 rc = writeback(ctxt, ops);
4076 if (rc != X86EMUL_CONTINUE)
4080 * restore dst type in case the decoding will be reused
4081 * (happens for string instruction )
4083 c->dst.type = saved_dst_type;
4085 if ((c->d & SrcMask) == SrcSI)
4086 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
4087 VCPU_REGS_RSI, &c->src);
4089 if ((c->d & DstMask) == DstDI)
4090 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4093 if (c->rep_prefix && (c->d & String)) {
4094 struct read_cache *r = &ctxt->decode.io_read;
4095 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4097 if (!string_insn_completed(ctxt)) {
4099 * Re-enter guest when pio read ahead buffer is empty
4100 * or, if it is not used, after each 1024 iteration.
4102 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4103 (r->end == 0 || r->end != r->pos)) {
4105 * Reset read cache. Usually happens before
4106 * decode, but since instruction is restarted
4107 * we have to do it here.
4109 ctxt->decode.mem_read.end = 0;
4110 return EMULATION_RESTART;
4112 goto done; /* skip rip writeback */
4119 if (rc == X86EMUL_PROPAGATE_FAULT)
4120 ctxt->have_exception = true;
4121 if (rc == X86EMUL_INTERCEPTED)
4122 return EMULATION_INTERCEPTED;
4124 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4128 case 0x05: /* syscall */
4129 rc = emulate_syscall(ctxt, ops);
4134 case 0x09: /* wbinvd */
4135 (ctxt->ops->wbinvd)(ctxt);
4137 case 0x08: /* invd */
4138 case 0x0d: /* GrpP (prefetch) */
4139 case 0x18: /* Grp16 (prefetch/nop) */
4141 case 0x20: /* mov cr, reg */
4142 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4144 case 0x21: /* mov from dr to reg */
4145 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4147 case 0x22: /* mov reg, cr */
4148 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4149 emulate_gp(ctxt, 0);
4150 rc = X86EMUL_PROPAGATE_FAULT;
4153 c->dst.type = OP_NONE;
4155 case 0x23: /* mov from reg to dr */
4156 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4157 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4158 ~0ULL : ~0U)) < 0) {
4159 /* #UD condition is already handled by the code above */
4160 emulate_gp(ctxt, 0);
4161 rc = X86EMUL_PROPAGATE_FAULT;
4165 c->dst.type = OP_NONE; /* no writeback */
4169 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4170 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4171 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4172 emulate_gp(ctxt, 0);
4173 rc = X86EMUL_PROPAGATE_FAULT;
4176 rc = X86EMUL_CONTINUE;
4180 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4181 emulate_gp(ctxt, 0);
4182 rc = X86EMUL_PROPAGATE_FAULT;
4185 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4186 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4188 rc = X86EMUL_CONTINUE;
4190 case 0x34: /* sysenter */
4191 rc = emulate_sysenter(ctxt, ops);
4193 case 0x35: /* sysexit */
4194 rc = emulate_sysexit(ctxt, ops);
4196 case 0x40 ... 0x4f: /* cmov */
4197 c->dst.val = c->dst.orig_val = c->src.val;
4198 if (!test_cc(c->b, ctxt->eflags))
4199 c->dst.type = OP_NONE; /* no writeback */
4201 case 0x80 ... 0x8f: /* jnz rel, etc*/
4202 if (test_cc(c->b, ctxt->eflags))
4203 jmp_rel(c, c->src.val);
4205 case 0x90 ... 0x9f: /* setcc r/m8 */
4206 c->dst.val = test_cc(c->b, ctxt->eflags);
4208 case 0xa0: /* push fs */
4209 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4211 case 0xa1: /* pop fs */
4212 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4216 c->dst.type = OP_NONE;
4217 /* only subword offset */
4218 c->src.val &= (c->dst.bytes << 3) - 1;
4219 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4221 case 0xa4: /* shld imm8, r, r/m */
4222 case 0xa5: /* shld cl, r, r/m */
4223 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4225 case 0xa8: /* push gs */
4226 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4228 case 0xa9: /* pop gs */
4229 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4233 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4235 case 0xac: /* shrd imm8, r, r/m */
4236 case 0xad: /* shrd cl, r, r/m */
4237 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4239 case 0xae: /* clflush */
4241 case 0xb0 ... 0xb1: /* cmpxchg */
4243 * Save real source value, then compare EAX against
4246 c->src.orig_val = c->src.val;
4247 c->src.val = c->regs[VCPU_REGS_RAX];
4248 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4249 if (ctxt->eflags & EFLG_ZF) {
4250 /* Success: write back to memory. */
4251 c->dst.val = c->src.orig_val;
4253 /* Failure: write the value we saw to EAX. */
4254 c->dst.type = OP_REG;
4255 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4258 case 0xb2: /* lss */
4259 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4263 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4265 case 0xb4: /* lfs */
4266 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4268 case 0xb5: /* lgs */
4269 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4271 case 0xb6 ... 0xb7: /* movzx */
4272 c->dst.bytes = c->op_bytes;
4273 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4276 case 0xba: /* Grp8 */
4277 switch (c->modrm_reg & 3) {
4290 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4292 case 0xbc: { /* bsf */
4294 __asm__ ("bsf %2, %0; setz %1"
4295 : "=r"(c->dst.val), "=q"(zf)
4297 ctxt->eflags &= ~X86_EFLAGS_ZF;
4299 ctxt->eflags |= X86_EFLAGS_ZF;
4300 c->dst.type = OP_NONE; /* Disable writeback. */
4304 case 0xbd: { /* bsr */
4306 __asm__ ("bsr %2, %0; setz %1"
4307 : "=r"(c->dst.val), "=q"(zf)
4309 ctxt->eflags &= ~X86_EFLAGS_ZF;
4311 ctxt->eflags |= X86_EFLAGS_ZF;
4312 c->dst.type = OP_NONE; /* Disable writeback. */
4316 case 0xbe ... 0xbf: /* movsx */
4317 c->dst.bytes = c->op_bytes;
4318 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4321 case 0xc0 ... 0xc1: /* xadd */
4322 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4323 /* Write back the register source. */
4324 c->src.val = c->dst.orig_val;
4325 write_register_operand(&c->src);
4327 case 0xc3: /* movnti */
4328 c->dst.bytes = c->op_bytes;
4329 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4332 case 0xc7: /* Grp9 (cmpxchg8b) */
4333 rc = emulate_grp9(ctxt, ops);
4336 goto cannot_emulate;
4339 if (rc != X86EMUL_CONTINUE)
4345 return EMULATION_FAILED;