1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct decode_cache *c, int seg)
476 c->has_seg_override = true;
477 c->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
481 struct x86_emulate_ops *ops, int seg)
483 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
486 return ops->get_cached_segment_base(seg, ctxt->vcpu);
489 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
490 struct x86_emulate_ops *ops,
491 struct decode_cache *c)
493 if (!c->has_seg_override)
496 return c->seg_override;
499 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
500 u32 error, bool valid)
502 ctxt->exception.vector = vec;
503 ctxt->exception.error_code = error;
504 ctxt->exception.error_code_valid = valid;
505 return X86EMUL_PROPAGATE_FAULT;
508 static int emulate_db(struct x86_emulate_ctxt *ctxt)
510 return emulate_exception(ctxt, DB_VECTOR, 0, false);
513 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, GP_VECTOR, err, true);
518 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, SS_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int __linearize(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
545 unsigned size, bool write, bool fetch,
548 struct decode_cache *c = &ctxt->decode;
549 struct desc_struct desc;
555 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
556 switch (ctxt->mode) {
557 case X86EMUL_MODE_REAL:
559 case X86EMUL_MODE_PROT64:
560 if (((signed long)la << 16) >> 16 != la)
561 return emulate_gp(ctxt, 0);
564 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
568 /* code segment or read-only data segment */
569 if (((desc.type & 8) || !(desc.type & 2)) && write)
571 /* unreadable code segment */
572 if (!fetch && (desc.type & 8) && !(desc.type & 2))
574 lim = desc_limit_scaled(&desc);
575 if ((desc.type & 8) || !(desc.type & 4)) {
576 /* expand-up segment */
577 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
580 /* exapand-down segment */
581 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
583 lim = desc.d ? 0xffffffff : 0xffff;
584 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
587 cpl = ctxt->ops->cpl(ctxt->vcpu);
588 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
590 if (!(desc.type & 8)) {
594 } else if ((desc.type & 8) && !(desc.type & 4)) {
595 /* nonconforming code segment */
598 } else if ((desc.type & 8) && (desc.type & 4)) {
599 /* conforming code segment */
605 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
608 return X86EMUL_CONTINUE;
610 if (addr.seg == VCPU_SREG_SS)
611 return emulate_ss(ctxt, addr.seg);
613 return emulate_gp(ctxt, addr.seg);
616 static int linearize(struct x86_emulate_ctxt *ctxt,
617 struct segmented_address addr,
618 unsigned size, bool write,
621 return __linearize(ctxt, addr, size, write, false, linear);
625 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
626 struct segmented_address addr,
633 rc = linearize(ctxt, addr, size, false, &linear);
634 if (rc != X86EMUL_CONTINUE)
636 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
640 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
641 struct x86_emulate_ops *ops,
642 unsigned long eip, u8 *dest)
644 struct fetch_cache *fc = &ctxt->decode.fetch;
648 if (eip == fc->end) {
649 unsigned long linear;
650 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
651 cur_size = fc->end - fc->start;
652 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
653 rc = __linearize(ctxt, addr, size, false, true, &linear);
654 if (rc != X86EMUL_CONTINUE)
656 rc = ops->fetch(linear, fc->data + cur_size,
657 size, ctxt->vcpu, &ctxt->exception);
658 if (rc != X86EMUL_CONTINUE)
662 *dest = fc->data[eip - fc->start];
663 return X86EMUL_CONTINUE;
666 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
667 struct x86_emulate_ops *ops,
668 unsigned long eip, void *dest, unsigned size)
672 /* x86 instructions are limited to 15 bytes. */
673 if (eip + size - ctxt->eip > 15)
674 return X86EMUL_UNHANDLEABLE;
676 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
677 if (rc != X86EMUL_CONTINUE)
680 return X86EMUL_CONTINUE;
684 * Given the 'reg' portion of a ModRM byte, and a register block, return a
685 * pointer into the block that addresses the relevant register.
686 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
688 static void *decode_register(u8 modrm_reg, unsigned long *regs,
693 p = ®s[modrm_reg];
694 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
695 p = (unsigned char *)®s[modrm_reg & 3] + 1;
699 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
700 struct x86_emulate_ops *ops,
701 struct segmented_address addr,
702 u16 *size, unsigned long *address, int op_bytes)
709 rc = segmented_read_std(ctxt, addr, size, 2);
710 if (rc != X86EMUL_CONTINUE)
713 rc = segmented_read_std(ctxt, addr, address, op_bytes);
717 static int test_cc(unsigned int condition, unsigned int flags)
721 switch ((condition & 15) >> 1) {
723 rc |= (flags & EFLG_OF);
725 case 1: /* b/c/nae */
726 rc |= (flags & EFLG_CF);
729 rc |= (flags & EFLG_ZF);
732 rc |= (flags & (EFLG_CF|EFLG_ZF));
735 rc |= (flags & EFLG_SF);
738 rc |= (flags & EFLG_PF);
741 rc |= (flags & EFLG_ZF);
744 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
748 /* Odd condition identifiers (lsb == 1) have inverted sense. */
749 return (!!rc ^ (condition & 1));
752 static void fetch_register_operand(struct operand *op)
756 op->val = *(u8 *)op->addr.reg;
759 op->val = *(u16 *)op->addr.reg;
762 op->val = *(u32 *)op->addr.reg;
765 op->val = *(u64 *)op->addr.reg;
770 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
772 ctxt->ops->get_fpu(ctxt);
774 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
775 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
776 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
777 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
778 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
779 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
780 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
781 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
783 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
784 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
785 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
786 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
787 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
788 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
789 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
790 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
794 ctxt->ops->put_fpu(ctxt);
797 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
800 ctxt->ops->get_fpu(ctxt);
802 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
803 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
804 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
805 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
806 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
807 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
808 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
809 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
811 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
812 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
813 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
814 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
815 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
816 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
817 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
818 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
822 ctxt->ops->put_fpu(ctxt);
825 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
827 struct decode_cache *c,
830 unsigned reg = c->modrm_reg;
831 int highbyte_regs = c->rex_prefix == 0;
834 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
840 read_sse_reg(ctxt, &op->vec_val, reg);
845 if ((c->d & ByteOp) && !inhibit_bytereg) {
846 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
849 op->addr.reg = decode_register(reg, c->regs, 0);
850 op->bytes = c->op_bytes;
852 fetch_register_operand(op);
853 op->orig_val = op->val;
856 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
857 struct x86_emulate_ops *ops,
860 struct decode_cache *c = &ctxt->decode;
862 int index_reg = 0, base_reg = 0, scale;
863 int rc = X86EMUL_CONTINUE;
867 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
868 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
869 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
872 c->modrm = insn_fetch(u8, 1, c->eip);
873 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
874 c->modrm_reg |= (c->modrm & 0x38) >> 3;
875 c->modrm_rm |= (c->modrm & 0x07);
876 c->modrm_seg = VCPU_SREG_DS;
878 if (c->modrm_mod == 3) {
880 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
881 op->addr.reg = decode_register(c->modrm_rm,
882 c->regs, c->d & ByteOp);
886 op->addr.xmm = c->modrm_rm;
887 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
890 fetch_register_operand(op);
896 if (c->ad_bytes == 2) {
897 unsigned bx = c->regs[VCPU_REGS_RBX];
898 unsigned bp = c->regs[VCPU_REGS_RBP];
899 unsigned si = c->regs[VCPU_REGS_RSI];
900 unsigned di = c->regs[VCPU_REGS_RDI];
902 /* 16-bit ModR/M decode. */
903 switch (c->modrm_mod) {
905 if (c->modrm_rm == 6)
906 modrm_ea += insn_fetch(u16, 2, c->eip);
909 modrm_ea += insn_fetch(s8, 1, c->eip);
912 modrm_ea += insn_fetch(u16, 2, c->eip);
915 switch (c->modrm_rm) {
935 if (c->modrm_mod != 0)
942 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
943 (c->modrm_rm == 6 && c->modrm_mod != 0))
944 c->modrm_seg = VCPU_SREG_SS;
945 modrm_ea = (u16)modrm_ea;
947 /* 32/64-bit ModR/M decode. */
948 if ((c->modrm_rm & 7) == 4) {
949 sib = insn_fetch(u8, 1, c->eip);
950 index_reg |= (sib >> 3) & 7;
954 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
955 modrm_ea += insn_fetch(s32, 4, c->eip);
957 modrm_ea += c->regs[base_reg];
959 modrm_ea += c->regs[index_reg] << scale;
960 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
961 if (ctxt->mode == X86EMUL_MODE_PROT64)
964 modrm_ea += c->regs[c->modrm_rm];
965 switch (c->modrm_mod) {
967 if (c->modrm_rm == 5)
968 modrm_ea += insn_fetch(s32, 4, c->eip);
971 modrm_ea += insn_fetch(s8, 1, c->eip);
974 modrm_ea += insn_fetch(s32, 4, c->eip);
978 op->addr.mem.ea = modrm_ea;
983 static int decode_abs(struct x86_emulate_ctxt *ctxt,
984 struct x86_emulate_ops *ops,
987 struct decode_cache *c = &ctxt->decode;
988 int rc = X86EMUL_CONTINUE;
991 switch (c->ad_bytes) {
993 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
996 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
999 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1006 static void fetch_bit_operand(struct decode_cache *c)
1010 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1011 mask = ~(c->dst.bytes * 8 - 1);
1013 if (c->src.bytes == 2)
1014 sv = (s16)c->src.val & (s16)mask;
1015 else if (c->src.bytes == 4)
1016 sv = (s32)c->src.val & (s32)mask;
1018 c->dst.addr.mem.ea += (sv >> 3);
1021 /* only subword offset */
1022 c->src.val &= (c->dst.bytes << 3) - 1;
1025 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1026 struct x86_emulate_ops *ops,
1027 unsigned long addr, void *dest, unsigned size)
1030 struct read_cache *mc = &ctxt->decode.mem_read;
1033 int n = min(size, 8u);
1035 if (mc->pos < mc->end)
1038 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1039 &ctxt->exception, ctxt->vcpu);
1040 if (rc != X86EMUL_CONTINUE)
1045 memcpy(dest, mc->data + mc->pos, n);
1050 return X86EMUL_CONTINUE;
1053 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1054 struct segmented_address addr,
1061 rc = linearize(ctxt, addr, size, false, &linear);
1062 if (rc != X86EMUL_CONTINUE)
1064 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1067 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1068 struct segmented_address addr,
1075 rc = linearize(ctxt, addr, size, true, &linear);
1076 if (rc != X86EMUL_CONTINUE)
1078 return ctxt->ops->write_emulated(linear, data, size,
1079 &ctxt->exception, ctxt->vcpu);
1082 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1084 const void *orig_data, const void *data,
1090 rc = linearize(ctxt, addr, size, true, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1094 size, &ctxt->exception, ctxt->vcpu);
1097 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1098 struct x86_emulate_ops *ops,
1099 unsigned int size, unsigned short port,
1102 struct read_cache *rc = &ctxt->decode.io_read;
1104 if (rc->pos == rc->end) { /* refill pio read ahead */
1105 struct decode_cache *c = &ctxt->decode;
1106 unsigned int in_page, n;
1107 unsigned int count = c->rep_prefix ?
1108 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1109 in_page = (ctxt->eflags & EFLG_DF) ?
1110 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1111 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1112 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1116 rc->pos = rc->end = 0;
1117 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1122 memcpy(dest, rc->data + rc->pos, size);
1127 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1128 struct x86_emulate_ops *ops,
1129 u16 selector, struct desc_ptr *dt)
1131 if (selector & 1 << 2) {
1132 struct desc_struct desc;
1133 memset (dt, 0, sizeof *dt);
1134 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1138 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1139 dt->address = get_desc_base(&desc);
1141 ops->get_gdt(dt, ctxt->vcpu);
1144 /* allowed just for 8 bytes segments */
1145 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1146 struct x86_emulate_ops *ops,
1147 u16 selector, struct desc_struct *desc)
1150 u16 index = selector >> 3;
1154 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1156 if (dt.size < index * 8 + 7)
1157 return emulate_gp(ctxt, selector & 0xfffc);
1158 addr = dt.address + index * 8;
1159 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1165 /* allowed just for 8 bytes segments */
1166 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1167 struct x86_emulate_ops *ops,
1168 u16 selector, struct desc_struct *desc)
1171 u16 index = selector >> 3;
1175 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1177 if (dt.size < index * 8 + 7)
1178 return emulate_gp(ctxt, selector & 0xfffc);
1180 addr = dt.address + index * 8;
1181 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1187 /* Does not support long mode */
1188 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1189 struct x86_emulate_ops *ops,
1190 u16 selector, int seg)
1192 struct desc_struct seg_desc;
1194 unsigned err_vec = GP_VECTOR;
1196 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1199 memset(&seg_desc, 0, sizeof seg_desc);
1201 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1202 || ctxt->mode == X86EMUL_MODE_REAL) {
1203 /* set real mode segment descriptor */
1204 set_desc_base(&seg_desc, selector << 4);
1205 set_desc_limit(&seg_desc, 0xffff);
1212 /* NULL selector is not valid for TR, CS and SS */
1213 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1217 /* TR should be in GDT only */
1218 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1221 if (null_selector) /* for NULL selector skip all following checks */
1224 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1225 if (ret != X86EMUL_CONTINUE)
1228 err_code = selector & 0xfffc;
1229 err_vec = GP_VECTOR;
1231 /* can't load system descriptor into segment selecor */
1232 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1236 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1242 cpl = ops->cpl(ctxt->vcpu);
1247 * segment is not a writable data segment or segment
1248 * selector's RPL != CPL or segment selector's RPL != CPL
1250 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1254 if (!(seg_desc.type & 8))
1257 if (seg_desc.type & 4) {
1263 if (rpl > cpl || dpl != cpl)
1266 /* CS(RPL) <- CPL */
1267 selector = (selector & 0xfffc) | cpl;
1270 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1273 case VCPU_SREG_LDTR:
1274 if (seg_desc.s || seg_desc.type != 2)
1277 default: /* DS, ES, FS, or GS */
1279 * segment is not a data or readable code segment or
1280 * ((segment is a data or nonconforming code segment)
1281 * and (both RPL and CPL > DPL))
1283 if ((seg_desc.type & 0xa) == 0x8 ||
1284 (((seg_desc.type & 0xc) != 0xc) &&
1285 (rpl > dpl && cpl > dpl)))
1291 /* mark segment as accessed */
1293 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1294 if (ret != X86EMUL_CONTINUE)
1298 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1299 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1300 return X86EMUL_CONTINUE;
1302 emulate_exception(ctxt, err_vec, err_code, true);
1303 return X86EMUL_PROPAGATE_FAULT;
1306 static void write_register_operand(struct operand *op)
1308 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1309 switch (op->bytes) {
1311 *(u8 *)op->addr.reg = (u8)op->val;
1314 *(u16 *)op->addr.reg = (u16)op->val;
1317 *op->addr.reg = (u32)op->val;
1318 break; /* 64b: zero-extend */
1320 *op->addr.reg = op->val;
1325 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1326 struct x86_emulate_ops *ops)
1329 struct decode_cache *c = &ctxt->decode;
1331 switch (c->dst.type) {
1333 write_register_operand(&c->dst);
1337 rc = segmented_cmpxchg(ctxt,
1343 rc = segmented_write(ctxt,
1347 if (rc != X86EMUL_CONTINUE)
1351 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1359 return X86EMUL_CONTINUE;
1362 static int em_push(struct x86_emulate_ctxt *ctxt)
1364 struct decode_cache *c = &ctxt->decode;
1365 struct segmented_address addr;
1367 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1368 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1369 addr.seg = VCPU_SREG_SS;
1371 /* Disable writeback. */
1372 c->dst.type = OP_NONE;
1373 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1376 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1377 struct x86_emulate_ops *ops,
1378 void *dest, int len)
1380 struct decode_cache *c = &ctxt->decode;
1382 struct segmented_address addr;
1384 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1385 addr.seg = VCPU_SREG_SS;
1386 rc = segmented_read(ctxt, addr, dest, len);
1387 if (rc != X86EMUL_CONTINUE)
1390 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1394 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1395 struct x86_emulate_ops *ops,
1396 void *dest, int len)
1399 unsigned long val, change_mask;
1400 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1401 int cpl = ops->cpl(ctxt->vcpu);
1403 rc = emulate_pop(ctxt, ops, &val, len);
1404 if (rc != X86EMUL_CONTINUE)
1407 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1408 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1410 switch(ctxt->mode) {
1411 case X86EMUL_MODE_PROT64:
1412 case X86EMUL_MODE_PROT32:
1413 case X86EMUL_MODE_PROT16:
1415 change_mask |= EFLG_IOPL;
1417 change_mask |= EFLG_IF;
1419 case X86EMUL_MODE_VM86:
1421 return emulate_gp(ctxt, 0);
1422 change_mask |= EFLG_IF;
1424 default: /* real mode */
1425 change_mask |= (EFLG_IOPL | EFLG_IF);
1429 *(unsigned long *)dest =
1430 (ctxt->eflags & ~change_mask) | (val & change_mask);
1435 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1436 struct x86_emulate_ops *ops, int seg)
1438 struct decode_cache *c = &ctxt->decode;
1440 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1442 return em_push(ctxt);
1445 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1446 struct x86_emulate_ops *ops, int seg)
1448 struct decode_cache *c = &ctxt->decode;
1449 unsigned long selector;
1452 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1453 if (rc != X86EMUL_CONTINUE)
1456 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1460 static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1462 struct decode_cache *c = &ctxt->decode;
1463 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1464 int rc = X86EMUL_CONTINUE;
1465 int reg = VCPU_REGS_RAX;
1467 while (reg <= VCPU_REGS_RDI) {
1468 (reg == VCPU_REGS_RSP) ?
1469 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1472 if (rc != X86EMUL_CONTINUE)
1481 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1482 struct x86_emulate_ops *ops)
1484 struct decode_cache *c = &ctxt->decode;
1485 int rc = X86EMUL_CONTINUE;
1486 int reg = VCPU_REGS_RDI;
1488 while (reg >= VCPU_REGS_RAX) {
1489 if (reg == VCPU_REGS_RSP) {
1490 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1495 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1496 if (rc != X86EMUL_CONTINUE)
1503 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1504 struct x86_emulate_ops *ops, int irq)
1506 struct decode_cache *c = &ctxt->decode;
1513 /* TODO: Add limit checks */
1514 c->src.val = ctxt->eflags;
1516 if (rc != X86EMUL_CONTINUE)
1519 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1521 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1523 if (rc != X86EMUL_CONTINUE)
1526 c->src.val = c->eip;
1528 if (rc != X86EMUL_CONTINUE)
1531 ops->get_idt(&dt, ctxt->vcpu);
1533 eip_addr = dt.address + (irq << 2);
1534 cs_addr = dt.address + (irq << 2) + 2;
1536 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1537 if (rc != X86EMUL_CONTINUE)
1540 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1541 if (rc != X86EMUL_CONTINUE)
1544 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1545 if (rc != X86EMUL_CONTINUE)
1553 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1554 struct x86_emulate_ops *ops, int irq)
1556 switch(ctxt->mode) {
1557 case X86EMUL_MODE_REAL:
1558 return emulate_int_real(ctxt, ops, irq);
1559 case X86EMUL_MODE_VM86:
1560 case X86EMUL_MODE_PROT16:
1561 case X86EMUL_MODE_PROT32:
1562 case X86EMUL_MODE_PROT64:
1564 /* Protected mode interrupts unimplemented yet */
1565 return X86EMUL_UNHANDLEABLE;
1569 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1570 struct x86_emulate_ops *ops)
1572 struct decode_cache *c = &ctxt->decode;
1573 int rc = X86EMUL_CONTINUE;
1574 unsigned long temp_eip = 0;
1575 unsigned long temp_eflags = 0;
1576 unsigned long cs = 0;
1577 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1578 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1579 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1580 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1582 /* TODO: Add stack limit check */
1584 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1586 if (rc != X86EMUL_CONTINUE)
1589 if (temp_eip & ~0xffff)
1590 return emulate_gp(ctxt, 0);
1592 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1599 if (rc != X86EMUL_CONTINUE)
1602 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1604 if (rc != X86EMUL_CONTINUE)
1610 if (c->op_bytes == 4)
1611 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1612 else if (c->op_bytes == 2) {
1613 ctxt->eflags &= ~0xffff;
1614 ctxt->eflags |= temp_eflags;
1617 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1618 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1623 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1624 struct x86_emulate_ops* ops)
1626 switch(ctxt->mode) {
1627 case X86EMUL_MODE_REAL:
1628 return emulate_iret_real(ctxt, ops);
1629 case X86EMUL_MODE_VM86:
1630 case X86EMUL_MODE_PROT16:
1631 case X86EMUL_MODE_PROT32:
1632 case X86EMUL_MODE_PROT64:
1634 /* iret from protected mode unimplemented yet */
1635 return X86EMUL_UNHANDLEABLE;
1639 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1640 struct x86_emulate_ops *ops)
1642 struct decode_cache *c = &ctxt->decode;
1644 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1647 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1649 struct decode_cache *c = &ctxt->decode;
1650 switch (c->modrm_reg) {
1652 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1655 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1658 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1661 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1663 case 4: /* sal/shl */
1664 case 6: /* sal/shl */
1665 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1668 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1671 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1676 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1677 struct x86_emulate_ops *ops)
1679 struct decode_cache *c = &ctxt->decode;
1680 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1681 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1684 switch (c->modrm_reg) {
1685 case 0 ... 1: /* test */
1686 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1689 c->dst.val = ~c->dst.val;
1692 emulate_1op("neg", c->dst, ctxt->eflags);
1695 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1698 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1701 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1705 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1709 return X86EMUL_UNHANDLEABLE;
1712 return emulate_de(ctxt);
1713 return X86EMUL_CONTINUE;
1716 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1718 struct decode_cache *c = &ctxt->decode;
1719 int rc = X86EMUL_CONTINUE;
1721 switch (c->modrm_reg) {
1723 emulate_1op("inc", c->dst, ctxt->eflags);
1726 emulate_1op("dec", c->dst, ctxt->eflags);
1728 case 2: /* call near abs */ {
1731 c->eip = c->src.val;
1732 c->src.val = old_eip;
1736 case 4: /* jmp abs */
1737 c->eip = c->src.val;
1746 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1747 struct x86_emulate_ops *ops)
1749 struct decode_cache *c = &ctxt->decode;
1750 u64 old = c->dst.orig_val64;
1752 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1753 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1754 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1755 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1756 ctxt->eflags &= ~EFLG_ZF;
1758 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1759 (u32) c->regs[VCPU_REGS_RBX];
1761 ctxt->eflags |= EFLG_ZF;
1763 return X86EMUL_CONTINUE;
1766 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1767 struct x86_emulate_ops *ops)
1769 struct decode_cache *c = &ctxt->decode;
1773 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1774 if (rc != X86EMUL_CONTINUE)
1776 if (c->op_bytes == 4)
1777 c->eip = (u32)c->eip;
1778 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1779 if (rc != X86EMUL_CONTINUE)
1781 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1785 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1786 struct x86_emulate_ops *ops, int seg)
1788 struct decode_cache *c = &ctxt->decode;
1792 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1794 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1795 if (rc != X86EMUL_CONTINUE)
1798 c->dst.val = c->src.val;
1803 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1804 struct x86_emulate_ops *ops, struct desc_struct *cs,
1805 struct desc_struct *ss)
1807 memset(cs, 0, sizeof(struct desc_struct));
1808 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1809 memset(ss, 0, sizeof(struct desc_struct));
1811 cs->l = 0; /* will be adjusted later */
1812 set_desc_base(cs, 0); /* flat segment */
1813 cs->g = 1; /* 4kb granularity */
1814 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1815 cs->type = 0x0b; /* Read, Execute, Accessed */
1817 cs->dpl = 0; /* will be adjusted later */
1821 set_desc_base(ss, 0); /* flat segment */
1822 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1823 ss->g = 1; /* 4kb granularity */
1825 ss->type = 0x03; /* Read/Write, Accessed */
1826 ss->d = 1; /* 32bit stack segment */
1832 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1834 struct decode_cache *c = &ctxt->decode;
1835 struct desc_struct cs, ss;
1839 /* syscall is not available in real mode */
1840 if (ctxt->mode == X86EMUL_MODE_REAL ||
1841 ctxt->mode == X86EMUL_MODE_VM86)
1842 return emulate_ud(ctxt);
1844 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1846 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1848 cs_sel = (u16)(msr_data & 0xfffc);
1849 ss_sel = (u16)(msr_data + 8);
1851 if (is_long_mode(ctxt->vcpu)) {
1855 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1856 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1857 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1858 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1860 c->regs[VCPU_REGS_RCX] = c->eip;
1861 if (is_long_mode(ctxt->vcpu)) {
1862 #ifdef CONFIG_X86_64
1863 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1865 ops->get_msr(ctxt->vcpu,
1866 ctxt->mode == X86EMUL_MODE_PROT64 ?
1867 MSR_LSTAR : MSR_CSTAR, &msr_data);
1870 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1871 ctxt->eflags &= ~(msr_data | EFLG_RF);
1875 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1876 c->eip = (u32)msr_data;
1878 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1881 return X86EMUL_CONTINUE;
1885 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1887 struct decode_cache *c = &ctxt->decode;
1888 struct desc_struct cs, ss;
1892 /* inject #GP if in real mode */
1893 if (ctxt->mode == X86EMUL_MODE_REAL)
1894 return emulate_gp(ctxt, 0);
1896 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1897 * Therefore, we inject an #UD.
1899 if (ctxt->mode == X86EMUL_MODE_PROT64)
1900 return emulate_ud(ctxt);
1902 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1904 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1905 switch (ctxt->mode) {
1906 case X86EMUL_MODE_PROT32:
1907 if ((msr_data & 0xfffc) == 0x0)
1908 return emulate_gp(ctxt, 0);
1910 case X86EMUL_MODE_PROT64:
1911 if (msr_data == 0x0)
1912 return emulate_gp(ctxt, 0);
1916 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1917 cs_sel = (u16)msr_data;
1918 cs_sel &= ~SELECTOR_RPL_MASK;
1919 ss_sel = cs_sel + 8;
1920 ss_sel &= ~SELECTOR_RPL_MASK;
1921 if (ctxt->mode == X86EMUL_MODE_PROT64
1922 || is_long_mode(ctxt->vcpu)) {
1927 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1928 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1929 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1930 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1932 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1935 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1936 c->regs[VCPU_REGS_RSP] = msr_data;
1938 return X86EMUL_CONTINUE;
1942 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1944 struct decode_cache *c = &ctxt->decode;
1945 struct desc_struct cs, ss;
1950 /* inject #GP if in real mode or Virtual 8086 mode */
1951 if (ctxt->mode == X86EMUL_MODE_REAL ||
1952 ctxt->mode == X86EMUL_MODE_VM86)
1953 return emulate_gp(ctxt, 0);
1955 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1957 if ((c->rex_prefix & 0x8) != 0x0)
1958 usermode = X86EMUL_MODE_PROT64;
1960 usermode = X86EMUL_MODE_PROT32;
1964 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1966 case X86EMUL_MODE_PROT32:
1967 cs_sel = (u16)(msr_data + 16);
1968 if ((msr_data & 0xfffc) == 0x0)
1969 return emulate_gp(ctxt, 0);
1970 ss_sel = (u16)(msr_data + 24);
1972 case X86EMUL_MODE_PROT64:
1973 cs_sel = (u16)(msr_data + 32);
1974 if (msr_data == 0x0)
1975 return emulate_gp(ctxt, 0);
1976 ss_sel = cs_sel + 8;
1981 cs_sel |= SELECTOR_RPL_MASK;
1982 ss_sel |= SELECTOR_RPL_MASK;
1984 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1985 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1986 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1987 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1989 c->eip = c->regs[VCPU_REGS_RDX];
1990 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1992 return X86EMUL_CONTINUE;
1995 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1996 struct x86_emulate_ops *ops)
1999 if (ctxt->mode == X86EMUL_MODE_REAL)
2001 if (ctxt->mode == X86EMUL_MODE_VM86)
2003 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2004 return ops->cpl(ctxt->vcpu) > iopl;
2007 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2008 struct x86_emulate_ops *ops,
2011 struct desc_struct tr_seg;
2014 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2015 unsigned mask = (1 << len) - 1;
2018 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2021 if (desc_limit_scaled(&tr_seg) < 103)
2023 base = get_desc_base(&tr_seg);
2024 #ifdef CONFIG_X86_64
2025 base |= ((u64)base3) << 32;
2027 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2028 if (r != X86EMUL_CONTINUE)
2030 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2032 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2034 if (r != X86EMUL_CONTINUE)
2036 if ((perm >> bit_idx) & mask)
2041 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2042 struct x86_emulate_ops *ops,
2048 if (emulator_bad_iopl(ctxt, ops))
2049 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2052 ctxt->perm_ok = true;
2057 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2058 struct x86_emulate_ops *ops,
2059 struct tss_segment_16 *tss)
2061 struct decode_cache *c = &ctxt->decode;
2064 tss->flag = ctxt->eflags;
2065 tss->ax = c->regs[VCPU_REGS_RAX];
2066 tss->cx = c->regs[VCPU_REGS_RCX];
2067 tss->dx = c->regs[VCPU_REGS_RDX];
2068 tss->bx = c->regs[VCPU_REGS_RBX];
2069 tss->sp = c->regs[VCPU_REGS_RSP];
2070 tss->bp = c->regs[VCPU_REGS_RBP];
2071 tss->si = c->regs[VCPU_REGS_RSI];
2072 tss->di = c->regs[VCPU_REGS_RDI];
2074 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2075 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2076 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2077 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2078 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2081 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2082 struct x86_emulate_ops *ops,
2083 struct tss_segment_16 *tss)
2085 struct decode_cache *c = &ctxt->decode;
2089 ctxt->eflags = tss->flag | 2;
2090 c->regs[VCPU_REGS_RAX] = tss->ax;
2091 c->regs[VCPU_REGS_RCX] = tss->cx;
2092 c->regs[VCPU_REGS_RDX] = tss->dx;
2093 c->regs[VCPU_REGS_RBX] = tss->bx;
2094 c->regs[VCPU_REGS_RSP] = tss->sp;
2095 c->regs[VCPU_REGS_RBP] = tss->bp;
2096 c->regs[VCPU_REGS_RSI] = tss->si;
2097 c->regs[VCPU_REGS_RDI] = tss->di;
2100 * SDM says that segment selectors are loaded before segment
2103 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2104 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2105 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2106 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2107 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2110 * Now load segment descriptors. If fault happenes at this stage
2111 * it is handled in a context of new task
2113 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2114 if (ret != X86EMUL_CONTINUE)
2116 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2117 if (ret != X86EMUL_CONTINUE)
2119 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2120 if (ret != X86EMUL_CONTINUE)
2122 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2123 if (ret != X86EMUL_CONTINUE)
2125 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2126 if (ret != X86EMUL_CONTINUE)
2129 return X86EMUL_CONTINUE;
2132 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2133 struct x86_emulate_ops *ops,
2134 u16 tss_selector, u16 old_tss_sel,
2135 ulong old_tss_base, struct desc_struct *new_desc)
2137 struct tss_segment_16 tss_seg;
2139 u32 new_tss_base = get_desc_base(new_desc);
2141 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2143 if (ret != X86EMUL_CONTINUE)
2144 /* FIXME: need to provide precise fault address */
2147 save_state_to_tss16(ctxt, ops, &tss_seg);
2149 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2151 if (ret != X86EMUL_CONTINUE)
2152 /* FIXME: need to provide precise fault address */
2155 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2157 if (ret != X86EMUL_CONTINUE)
2158 /* FIXME: need to provide precise fault address */
2161 if (old_tss_sel != 0xffff) {
2162 tss_seg.prev_task_link = old_tss_sel;
2164 ret = ops->write_std(new_tss_base,
2165 &tss_seg.prev_task_link,
2166 sizeof tss_seg.prev_task_link,
2167 ctxt->vcpu, &ctxt->exception);
2168 if (ret != X86EMUL_CONTINUE)
2169 /* FIXME: need to provide precise fault address */
2173 return load_state_from_tss16(ctxt, ops, &tss_seg);
2176 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2177 struct x86_emulate_ops *ops,
2178 struct tss_segment_32 *tss)
2180 struct decode_cache *c = &ctxt->decode;
2182 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2184 tss->eflags = ctxt->eflags;
2185 tss->eax = c->regs[VCPU_REGS_RAX];
2186 tss->ecx = c->regs[VCPU_REGS_RCX];
2187 tss->edx = c->regs[VCPU_REGS_RDX];
2188 tss->ebx = c->regs[VCPU_REGS_RBX];
2189 tss->esp = c->regs[VCPU_REGS_RSP];
2190 tss->ebp = c->regs[VCPU_REGS_RBP];
2191 tss->esi = c->regs[VCPU_REGS_RSI];
2192 tss->edi = c->regs[VCPU_REGS_RDI];
2194 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2195 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2196 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2197 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2198 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2199 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2200 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2203 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2204 struct x86_emulate_ops *ops,
2205 struct tss_segment_32 *tss)
2207 struct decode_cache *c = &ctxt->decode;
2210 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2211 return emulate_gp(ctxt, 0);
2213 ctxt->eflags = tss->eflags | 2;
2214 c->regs[VCPU_REGS_RAX] = tss->eax;
2215 c->regs[VCPU_REGS_RCX] = tss->ecx;
2216 c->regs[VCPU_REGS_RDX] = tss->edx;
2217 c->regs[VCPU_REGS_RBX] = tss->ebx;
2218 c->regs[VCPU_REGS_RSP] = tss->esp;
2219 c->regs[VCPU_REGS_RBP] = tss->ebp;
2220 c->regs[VCPU_REGS_RSI] = tss->esi;
2221 c->regs[VCPU_REGS_RDI] = tss->edi;
2224 * SDM says that segment selectors are loaded before segment
2227 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2228 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2229 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2230 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2231 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2232 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2233 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2236 * Now load segment descriptors. If fault happenes at this stage
2237 * it is handled in a context of new task
2239 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2240 if (ret != X86EMUL_CONTINUE)
2242 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2243 if (ret != X86EMUL_CONTINUE)
2245 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2246 if (ret != X86EMUL_CONTINUE)
2248 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2249 if (ret != X86EMUL_CONTINUE)
2251 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2252 if (ret != X86EMUL_CONTINUE)
2254 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2255 if (ret != X86EMUL_CONTINUE)
2257 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2258 if (ret != X86EMUL_CONTINUE)
2261 return X86EMUL_CONTINUE;
2264 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2265 struct x86_emulate_ops *ops,
2266 u16 tss_selector, u16 old_tss_sel,
2267 ulong old_tss_base, struct desc_struct *new_desc)
2269 struct tss_segment_32 tss_seg;
2271 u32 new_tss_base = get_desc_base(new_desc);
2273 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2275 if (ret != X86EMUL_CONTINUE)
2276 /* FIXME: need to provide precise fault address */
2279 save_state_to_tss32(ctxt, ops, &tss_seg);
2281 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2283 if (ret != X86EMUL_CONTINUE)
2284 /* FIXME: need to provide precise fault address */
2287 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2289 if (ret != X86EMUL_CONTINUE)
2290 /* FIXME: need to provide precise fault address */
2293 if (old_tss_sel != 0xffff) {
2294 tss_seg.prev_task_link = old_tss_sel;
2296 ret = ops->write_std(new_tss_base,
2297 &tss_seg.prev_task_link,
2298 sizeof tss_seg.prev_task_link,
2299 ctxt->vcpu, &ctxt->exception);
2300 if (ret != X86EMUL_CONTINUE)
2301 /* FIXME: need to provide precise fault address */
2305 return load_state_from_tss32(ctxt, ops, &tss_seg);
2308 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2309 struct x86_emulate_ops *ops,
2310 u16 tss_selector, int reason,
2311 bool has_error_code, u32 error_code)
2313 struct desc_struct curr_tss_desc, next_tss_desc;
2315 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2316 ulong old_tss_base =
2317 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2320 /* FIXME: old_tss_base == ~0 ? */
2322 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2323 if (ret != X86EMUL_CONTINUE)
2325 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2326 if (ret != X86EMUL_CONTINUE)
2329 /* FIXME: check that next_tss_desc is tss */
2331 if (reason != TASK_SWITCH_IRET) {
2332 if ((tss_selector & 3) > next_tss_desc.dpl ||
2333 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2334 return emulate_gp(ctxt, 0);
2337 desc_limit = desc_limit_scaled(&next_tss_desc);
2338 if (!next_tss_desc.p ||
2339 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2340 desc_limit < 0x2b)) {
2341 emulate_ts(ctxt, tss_selector & 0xfffc);
2342 return X86EMUL_PROPAGATE_FAULT;
2345 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2346 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2347 write_segment_descriptor(ctxt, ops, old_tss_sel,
2351 if (reason == TASK_SWITCH_IRET)
2352 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2354 /* set back link to prev task only if NT bit is set in eflags
2355 note that old_tss_sel is not used afetr this point */
2356 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2357 old_tss_sel = 0xffff;
2359 if (next_tss_desc.type & 8)
2360 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2361 old_tss_base, &next_tss_desc);
2363 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2364 old_tss_base, &next_tss_desc);
2365 if (ret != X86EMUL_CONTINUE)
2368 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2369 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2371 if (reason != TASK_SWITCH_IRET) {
2372 next_tss_desc.type |= (1 << 1); /* set busy flag */
2373 write_segment_descriptor(ctxt, ops, tss_selector,
2377 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2378 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2379 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2381 if (has_error_code) {
2382 struct decode_cache *c = &ctxt->decode;
2384 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2386 c->src.val = (unsigned long) error_code;
2387 ret = em_push(ctxt);
2393 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2394 u16 tss_selector, int reason,
2395 bool has_error_code, u32 error_code)
2397 struct x86_emulate_ops *ops = ctxt->ops;
2398 struct decode_cache *c = &ctxt->decode;
2402 c->dst.type = OP_NONE;
2404 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2405 has_error_code, error_code);
2407 if (rc == X86EMUL_CONTINUE)
2410 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2413 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2414 int reg, struct operand *op)
2416 struct decode_cache *c = &ctxt->decode;
2417 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2419 register_address_increment(c, &c->regs[reg], df * op->bytes);
2420 op->addr.mem.ea = register_address(c, c->regs[reg]);
2421 op->addr.mem.seg = seg;
2424 static int em_das(struct x86_emulate_ctxt *ctxt)
2426 struct decode_cache *c = &ctxt->decode;
2428 bool af, cf, old_cf;
2430 cf = ctxt->eflags & X86_EFLAGS_CF;
2436 af = ctxt->eflags & X86_EFLAGS_AF;
2437 if ((al & 0x0f) > 9 || af) {
2439 cf = old_cf | (al >= 250);
2444 if (old_al > 0x99 || old_cf) {
2450 /* Set PF, ZF, SF */
2451 c->src.type = OP_IMM;
2454 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2455 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2457 ctxt->eflags |= X86_EFLAGS_CF;
2459 ctxt->eflags |= X86_EFLAGS_AF;
2460 return X86EMUL_CONTINUE;
2463 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2465 struct decode_cache *c = &ctxt->decode;
2470 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2473 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2474 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2475 return X86EMUL_CONTINUE;
2478 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2480 c->src.val = old_cs;
2482 if (rc != X86EMUL_CONTINUE)
2485 c->src.val = old_eip;
2486 return em_push(ctxt);
2489 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2491 struct decode_cache *c = &ctxt->decode;
2494 c->dst.type = OP_REG;
2495 c->dst.addr.reg = &c->eip;
2496 c->dst.bytes = c->op_bytes;
2497 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2498 if (rc != X86EMUL_CONTINUE)
2500 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2501 return X86EMUL_CONTINUE;
2504 static int em_imul(struct x86_emulate_ctxt *ctxt)
2506 struct decode_cache *c = &ctxt->decode;
2508 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2509 return X86EMUL_CONTINUE;
2512 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2514 struct decode_cache *c = &ctxt->decode;
2516 c->dst.val = c->src2.val;
2517 return em_imul(ctxt);
2520 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2522 struct decode_cache *c = &ctxt->decode;
2524 c->dst.type = OP_REG;
2525 c->dst.bytes = c->src.bytes;
2526 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2527 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2529 return X86EMUL_CONTINUE;
2532 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2534 struct decode_cache *c = &ctxt->decode;
2537 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2538 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2539 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2540 return X86EMUL_CONTINUE;
2543 static int em_mov(struct x86_emulate_ctxt *ctxt)
2545 struct decode_cache *c = &ctxt->decode;
2546 c->dst.val = c->src.val;
2547 return X86EMUL_CONTINUE;
2550 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2552 struct decode_cache *c = &ctxt->decode;
2553 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2554 return X86EMUL_CONTINUE;
2557 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2559 struct decode_cache *c = &ctxt->decode;
2563 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2564 if (rc == X86EMUL_CONTINUE)
2565 emulate_invlpg(ctxt->vcpu, linear);
2566 /* Disable writeback. */
2567 c->dst.type = OP_NONE;
2568 return X86EMUL_CONTINUE;
2571 static bool valid_cr(int nr)
2583 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2585 struct decode_cache *c = &ctxt->decode;
2587 if (!valid_cr(c->modrm_reg))
2588 return emulate_ud(ctxt);
2590 return X86EMUL_CONTINUE;
2593 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2595 struct decode_cache *c = &ctxt->decode;
2596 u64 new_val = c->src.val64;
2597 int cr = c->modrm_reg;
2599 static u64 cr_reserved_bits[] = {
2600 0xffffffff00000000ULL,
2601 0, 0, 0, /* CR3 checked later */
2608 return emulate_ud(ctxt);
2610 if (new_val & cr_reserved_bits[cr])
2611 return emulate_gp(ctxt, 0);
2616 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2617 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2618 return emulate_gp(ctxt, 0);
2620 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2621 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2623 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2624 !(cr4 & X86_CR4_PAE))
2625 return emulate_gp(ctxt, 0);
2632 if (is_long_mode(ctxt->vcpu))
2633 rsvd = CR3_L_MODE_RESERVED_BITS;
2634 else if (is_pae(ctxt->vcpu))
2635 rsvd = CR3_PAE_RESERVED_BITS;
2636 else if (is_paging(ctxt->vcpu))
2637 rsvd = CR3_NONPAE_RESERVED_BITS;
2640 return emulate_gp(ctxt, 0);
2647 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2648 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2650 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2651 return emulate_gp(ctxt, 0);
2657 return X86EMUL_CONTINUE;
2660 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2664 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2666 /* Check if DR7.Global_Enable is set */
2667 return dr7 & (1 << 13);
2670 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2672 struct decode_cache *c = &ctxt->decode;
2673 int dr = c->modrm_reg;
2677 return emulate_ud(ctxt);
2679 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2680 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2681 return emulate_ud(ctxt);
2683 if (check_dr7_gd(ctxt))
2684 return emulate_db(ctxt);
2686 return X86EMUL_CONTINUE;
2689 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2691 struct decode_cache *c = &ctxt->decode;
2692 u64 new_val = c->src.val64;
2693 int dr = c->modrm_reg;
2695 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2696 return emulate_gp(ctxt, 0);
2698 return check_dr_read(ctxt);
2701 static int check_svme(struct x86_emulate_ctxt *ctxt)
2705 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2707 if (!(efer & EFER_SVME))
2708 return emulate_ud(ctxt);
2710 return X86EMUL_CONTINUE;
2713 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2715 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2717 /* Valid physical address? */
2718 if (rax & 0xffff000000000000)
2719 return emulate_gp(ctxt, 0);
2721 return check_svme(ctxt);
2724 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2726 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2728 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2729 return emulate_ud(ctxt);
2731 return X86EMUL_CONTINUE;
2734 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2736 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2737 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2739 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2741 return emulate_gp(ctxt, 0);
2743 return X86EMUL_CONTINUE;
2746 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2748 struct decode_cache *c = &ctxt->decode;
2750 c->dst.bytes = min(c->dst.bytes, 4u);
2751 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2752 return emulate_gp(ctxt, 0);
2754 return X86EMUL_CONTINUE;
2757 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2759 struct decode_cache *c = &ctxt->decode;
2761 c->src.bytes = min(c->src.bytes, 4u);
2762 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2763 return emulate_gp(ctxt, 0);
2765 return X86EMUL_CONTINUE;
2768 #define D(_y) { .flags = (_y) }
2769 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2770 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2771 .check_perm = (_p) }
2773 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2774 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2775 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2776 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2777 #define II(_f, _e, _i) \
2778 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2779 #define IIP(_f, _e, _i, _p) \
2780 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2781 .check_perm = (_p) }
2782 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2784 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2785 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2786 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2788 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2789 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2790 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2792 static struct opcode group7_rm1[] = {
2793 DI(SrcNone | ModRM | Priv, monitor),
2794 DI(SrcNone | ModRM | Priv, mwait),
2798 static struct opcode group7_rm3[] = {
2799 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2800 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2801 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2802 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2803 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2804 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2805 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2806 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2809 static struct opcode group7_rm7[] = {
2811 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2814 static struct opcode group1[] = {
2818 static struct opcode group1A[] = {
2819 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2822 static struct opcode group3[] = {
2823 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2824 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2825 X4(D(SrcMem | ModRM)),
2828 static struct opcode group4[] = {
2829 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2833 static struct opcode group5[] = {
2834 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2835 D(SrcMem | ModRM | Stack),
2836 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2837 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2838 D(SrcMem | ModRM | Stack), N,
2841 static struct opcode group6[] = {
2842 DI(ModRM | Prot, sldt),
2843 DI(ModRM | Prot, str),
2844 DI(ModRM | Prot | Priv, lldt),
2845 DI(ModRM | Prot | Priv, ltr),
2849 static struct group_dual group7 = { {
2850 DI(ModRM | Mov | DstMem | Priv, sgdt),
2851 DI(ModRM | Mov | DstMem | Priv, sidt),
2852 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2853 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2854 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2855 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2857 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2858 N, EXT(0, group7_rm3),
2859 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2860 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2863 static struct opcode group8[] = {
2865 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2866 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2869 static struct group_dual group9 = { {
2870 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2872 N, N, N, N, N, N, N, N,
2875 static struct opcode group11[] = {
2876 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2879 static struct gprefix pfx_0f_6f_0f_7f = {
2880 N, N, N, I(Sse, em_movdqu),
2883 static struct opcode opcode_table[256] = {
2886 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2889 D(ImplicitOps | Stack | No64), N,
2892 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2895 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2899 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2907 X8(I(SrcReg | Stack, em_push)),
2909 X8(D(DstReg | Stack)),
2911 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2912 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2915 I(SrcImm | Mov | Stack, em_push),
2916 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2917 I(SrcImmByte | Mov | Stack, em_push),
2918 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2919 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2920 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2924 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2925 G(DstMem | SrcImm | ModRM | Group, group1),
2926 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2927 G(DstMem | SrcImmByte | ModRM | Group, group1),
2928 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2930 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2931 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2932 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2933 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2935 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2937 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2938 I(SrcImmFAddr | No64, em_call_far), N,
2939 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2941 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2942 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2943 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2944 D2bv(SrcSI | DstDI | String),
2946 D2bv(DstAcc | SrcImm),
2947 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2948 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2949 D2bv(SrcAcc | DstDI | String),
2951 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2953 X8(I(DstReg | SrcImm | Mov, em_mov)),
2955 D2bv(DstMem | SrcImmByte | ModRM),
2956 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2957 D(ImplicitOps | Stack),
2958 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2959 G(ByteOp, group11), G(0, group11),
2961 N, N, N, D(ImplicitOps | Stack),
2962 D(ImplicitOps), DI(SrcImmByte, intn),
2963 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2965 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2968 N, N, N, N, N, N, N, N,
2971 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2972 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2974 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2975 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2976 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2977 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2979 N, DI(ImplicitOps, icebp), N, N,
2980 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2981 G(ByteOp, group3), G(0, group3),
2983 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2984 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2987 static struct opcode twobyte_table[256] = {
2989 G(0, group6), GD(0, &group7), N, N,
2990 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2991 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2992 N, D(ImplicitOps | ModRM), N, N,
2994 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2996 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2997 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2998 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2999 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3001 N, N, N, N, N, N, N, N,
3003 DI(ImplicitOps | Priv, wrmsr),
3004 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3005 DI(ImplicitOps | Priv, rdmsr),
3006 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3007 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3009 N, N, N, N, N, N, N, N,
3011 X16(D(DstReg | SrcMem | ModRM | Mov)),
3013 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3018 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3023 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3027 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3029 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3030 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3031 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3032 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3034 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3035 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3036 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3037 D(DstMem | SrcReg | Src2CL | ModRM),
3038 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3040 D2bv(DstMem | SrcReg | ModRM | Lock),
3041 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3042 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3043 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3046 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3047 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3048 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3050 D2bv(DstMem | SrcReg | ModRM | Lock),
3051 N, D(DstMem | SrcReg | ModRM | Mov),
3052 N, N, N, GD(0, &group9),
3053 N, N, N, N, N, N, N, N,
3055 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3057 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3059 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3075 static unsigned imm_size(struct decode_cache *c)
3079 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3085 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3086 unsigned size, bool sign_extension)
3088 struct decode_cache *c = &ctxt->decode;
3089 struct x86_emulate_ops *ops = ctxt->ops;
3090 int rc = X86EMUL_CONTINUE;
3094 op->addr.mem.ea = c->eip;
3095 /* NB. Immediates are sign-extended as necessary. */
3096 switch (op->bytes) {
3098 op->val = insn_fetch(s8, 1, c->eip);
3101 op->val = insn_fetch(s16, 2, c->eip);
3104 op->val = insn_fetch(s32, 4, c->eip);
3107 if (!sign_extension) {
3108 switch (op->bytes) {
3116 op->val &= 0xffffffff;
3125 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3127 struct x86_emulate_ops *ops = ctxt->ops;
3128 struct decode_cache *c = &ctxt->decode;
3129 int rc = X86EMUL_CONTINUE;
3130 int mode = ctxt->mode;
3131 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3132 bool op_prefix = false;
3133 struct opcode opcode, *g_mod012, *g_mod3;
3134 struct operand memop = { .type = OP_NONE };
3137 c->fetch.start = c->eip;
3138 c->fetch.end = c->fetch.start + insn_len;
3140 memcpy(c->fetch.data, insn, insn_len);
3143 case X86EMUL_MODE_REAL:
3144 case X86EMUL_MODE_VM86:
3145 case X86EMUL_MODE_PROT16:
3146 def_op_bytes = def_ad_bytes = 2;
3148 case X86EMUL_MODE_PROT32:
3149 def_op_bytes = def_ad_bytes = 4;
3151 #ifdef CONFIG_X86_64
3152 case X86EMUL_MODE_PROT64:
3161 c->op_bytes = def_op_bytes;
3162 c->ad_bytes = def_ad_bytes;
3164 /* Legacy prefixes. */
3166 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3167 case 0x66: /* operand-size override */
3169 /* switch between 2/4 bytes */
3170 c->op_bytes = def_op_bytes ^ 6;
3172 case 0x67: /* address-size override */
3173 if (mode == X86EMUL_MODE_PROT64)
3174 /* switch between 4/8 bytes */
3175 c->ad_bytes = def_ad_bytes ^ 12;
3177 /* switch between 2/4 bytes */
3178 c->ad_bytes = def_ad_bytes ^ 6;
3180 case 0x26: /* ES override */
3181 case 0x2e: /* CS override */
3182 case 0x36: /* SS override */
3183 case 0x3e: /* DS override */
3184 set_seg_override(c, (c->b >> 3) & 3);
3186 case 0x64: /* FS override */
3187 case 0x65: /* GS override */
3188 set_seg_override(c, c->b & 7);
3190 case 0x40 ... 0x4f: /* REX */
3191 if (mode != X86EMUL_MODE_PROT64)
3193 c->rex_prefix = c->b;
3195 case 0xf0: /* LOCK */
3198 case 0xf2: /* REPNE/REPNZ */
3199 case 0xf3: /* REP/REPE/REPZ */
3200 c->rep_prefix = c->b;
3206 /* Any legacy prefix after a REX prefix nullifies its effect. */
3214 if (c->rex_prefix & 8)
3215 c->op_bytes = 8; /* REX.W */
3217 /* Opcode byte(s). */
3218 opcode = opcode_table[c->b];
3219 /* Two-byte opcode? */
3222 c->b = insn_fetch(u8, 1, c->eip);
3223 opcode = twobyte_table[c->b];
3225 c->d = opcode.flags;
3228 dual = c->d & GroupDual;
3229 c->modrm = insn_fetch(u8, 1, c->eip);
3232 if (c->d & GroupDual) {
3233 g_mod012 = opcode.u.gdual->mod012;
3234 g_mod3 = opcode.u.gdual->mod3;
3236 g_mod012 = g_mod3 = opcode.u.group;
3238 c->d &= ~(Group | GroupDual);
3240 goffset = (c->modrm >> 3) & 7;
3242 if ((c->modrm >> 6) == 3)
3243 opcode = g_mod3[goffset];
3245 opcode = g_mod012[goffset];
3247 if (opcode.flags & RMExt) {
3248 goffset = c->modrm & 7;
3249 opcode = opcode.u.group[goffset];
3252 c->d |= opcode.flags;
3255 if (c->d & Prefix) {
3256 if (c->rep_prefix && op_prefix)
3257 return X86EMUL_UNHANDLEABLE;
3258 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3259 switch (simd_prefix) {
3260 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3261 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3262 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3263 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3265 c->d |= opcode.flags;
3268 c->execute = opcode.u.execute;
3269 c->check_perm = opcode.check_perm;
3270 c->intercept = opcode.intercept;
3273 if (c->d == 0 || (c->d & Undefined))
3276 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3279 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3282 if (c->d & Op3264) {
3283 if (mode == X86EMUL_MODE_PROT64)
3292 /* ModRM and SIB bytes. */
3294 rc = decode_modrm(ctxt, ops, &memop);
3295 if (!c->has_seg_override)
3296 set_seg_override(c, c->modrm_seg);
3297 } else if (c->d & MemAbs)
3298 rc = decode_abs(ctxt, ops, &memop);
3299 if (rc != X86EMUL_CONTINUE)
3302 if (!c->has_seg_override)
3303 set_seg_override(c, VCPU_SREG_DS);
3305 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3307 if (memop.type == OP_MEM && c->ad_bytes != 8)
3308 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3310 if (memop.type == OP_MEM && c->rip_relative)
3311 memop.addr.mem.ea += c->eip;
3314 * Decode and fetch the source operand: register, memory
3317 switch (c->d & SrcMask) {
3321 decode_register_operand(ctxt, &c->src, c, 0);
3330 memop.bytes = (c->d & ByteOp) ? 1 :
3336 rc = decode_imm(ctxt, &c->src, 2, false);
3339 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3342 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3345 rc = decode_imm(ctxt, &c->src, 1, true);
3348 rc = decode_imm(ctxt, &c->src, 1, false);
3351 c->src.type = OP_REG;
3352 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3353 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3354 fetch_register_operand(&c->src);
3361 c->src.type = OP_MEM;
3362 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3363 c->src.addr.mem.ea =
3364 register_address(c, c->regs[VCPU_REGS_RSI]);
3365 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3369 c->src.type = OP_IMM;
3370 c->src.addr.mem.ea = c->eip;
3371 c->src.bytes = c->op_bytes + 2;
3372 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3375 memop.bytes = c->op_bytes + 2;
3380 if (rc != X86EMUL_CONTINUE)
3384 * Decode and fetch the second source operand: register, memory
3387 switch (c->d & Src2Mask) {
3392 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3395 rc = decode_imm(ctxt, &c->src2, 1, true);
3402 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3406 if (rc != X86EMUL_CONTINUE)
3409 /* Decode and fetch the destination operand: register or memory. */
3410 switch (c->d & DstMask) {
3412 decode_register_operand(ctxt, &c->dst, c,
3413 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3416 c->dst.type = OP_IMM;
3417 c->dst.addr.mem.ea = c->eip;
3419 c->dst.val = insn_fetch(u8, 1, c->eip);
3424 if ((c->d & DstMask) == DstMem64)
3427 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3429 fetch_bit_operand(c);
3430 c->dst.orig_val = c->dst.val;
3433 c->dst.type = OP_REG;
3434 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3435 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3436 fetch_register_operand(&c->dst);
3437 c->dst.orig_val = c->dst.val;
3440 c->dst.type = OP_MEM;
3441 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3442 c->dst.addr.mem.ea =
3443 register_address(c, c->regs[VCPU_REGS_RDI]);
3444 c->dst.addr.mem.seg = VCPU_SREG_ES;
3448 /* Special instructions do their own operand decoding. */
3450 c->dst.type = OP_NONE; /* Disable writeback. */
3455 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3458 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3460 struct decode_cache *c = &ctxt->decode;
3462 /* The second termination condition only applies for REPE
3463 * and REPNE. Test if the repeat string operation prefix is
3464 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3465 * corresponding termination condition according to:
3466 * - if REPE/REPZ and ZF = 0 then done
3467 * - if REPNE/REPNZ and ZF = 1 then done
3469 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3470 (c->b == 0xae) || (c->b == 0xaf))
3471 && (((c->rep_prefix == REPE_PREFIX) &&
3472 ((ctxt->eflags & EFLG_ZF) == 0))
3473 || ((c->rep_prefix == REPNE_PREFIX) &&
3474 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3481 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3483 struct x86_emulate_ops *ops = ctxt->ops;
3485 struct decode_cache *c = &ctxt->decode;
3486 int rc = X86EMUL_CONTINUE;
3487 int saved_dst_type = c->dst.type;
3488 int irq; /* Used for int 3, int, and into */
3490 ctxt->decode.mem_read.pos = 0;
3492 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3493 rc = emulate_ud(ctxt);
3497 /* LOCK prefix is allowed only with some instructions */
3498 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3499 rc = emulate_ud(ctxt);
3503 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3504 rc = emulate_ud(ctxt);
3509 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3510 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3511 rc = emulate_ud(ctxt);
3515 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3516 rc = emulate_nm(ctxt);
3520 if (unlikely(ctxt->guest_mode) && c->intercept) {
3521 rc = emulator_check_intercept(ctxt, c->intercept,
3522 X86_ICPT_PRE_EXCEPT);
3523 if (rc != X86EMUL_CONTINUE)
3527 /* Privileged instruction can be executed only in CPL=0 */
3528 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3529 rc = emulate_gp(ctxt, 0);
3533 /* Instruction can only be executed in protected mode */
3534 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3535 rc = emulate_ud(ctxt);
3539 /* Do instruction specific permission checks */
3540 if (c->check_perm) {
3541 rc = c->check_perm(ctxt);
3542 if (rc != X86EMUL_CONTINUE)
3546 if (unlikely(ctxt->guest_mode) && c->intercept) {
3547 rc = emulator_check_intercept(ctxt, c->intercept,
3548 X86_ICPT_POST_EXCEPT);
3549 if (rc != X86EMUL_CONTINUE)
3553 if (c->rep_prefix && (c->d & String)) {
3554 /* All REP prefixes have the same first termination condition */
3555 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3561 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3562 rc = segmented_read(ctxt, c->src.addr.mem,
3563 c->src.valptr, c->src.bytes);
3564 if (rc != X86EMUL_CONTINUE)
3566 c->src.orig_val64 = c->src.val64;
3569 if (c->src2.type == OP_MEM) {
3570 rc = segmented_read(ctxt, c->src2.addr.mem,
3571 &c->src2.val, c->src2.bytes);
3572 if (rc != X86EMUL_CONTINUE)
3576 if ((c->d & DstMask) == ImplicitOps)
3580 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3581 /* optimisation - avoid slow emulated read if Mov */
3582 rc = segmented_read(ctxt, c->dst.addr.mem,
3583 &c->dst.val, c->dst.bytes);
3584 if (rc != X86EMUL_CONTINUE)
3587 c->dst.orig_val = c->dst.val;
3591 if (unlikely(ctxt->guest_mode) && c->intercept) {
3592 rc = emulator_check_intercept(ctxt, c->intercept,
3593 X86_ICPT_POST_MEMACCESS);
3594 if (rc != X86EMUL_CONTINUE)
3599 rc = c->execute(ctxt);
3600 if (rc != X86EMUL_CONTINUE)
3611 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3613 case 0x06: /* push es */
3614 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3616 case 0x07: /* pop es */
3617 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3621 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3623 case 0x0e: /* push cs */
3624 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3628 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3630 case 0x16: /* push ss */
3631 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3633 case 0x17: /* pop ss */
3634 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3638 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3640 case 0x1e: /* push ds */
3641 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3643 case 0x1f: /* pop ds */
3644 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3648 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3652 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3656 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3660 c->dst.type = OP_NONE; /* Disable writeback. */
3661 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3663 case 0x40 ... 0x47: /* inc r16/r32 */
3664 emulate_1op("inc", c->dst, ctxt->eflags);
3666 case 0x48 ... 0x4f: /* dec r16/r32 */
3667 emulate_1op("dec", c->dst, ctxt->eflags);
3669 case 0x58 ... 0x5f: /* pop reg */
3671 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3673 case 0x60: /* pusha */
3674 rc = emulate_pusha(ctxt);
3676 case 0x61: /* popa */
3677 rc = emulate_popa(ctxt, ops);
3679 case 0x63: /* movsxd */
3680 if (ctxt->mode != X86EMUL_MODE_PROT64)
3681 goto cannot_emulate;
3682 c->dst.val = (s32) c->src.val;
3684 case 0x6c: /* insb */
3685 case 0x6d: /* insw/insd */
3686 c->src.val = c->regs[VCPU_REGS_RDX];
3688 case 0x6e: /* outsb */
3689 case 0x6f: /* outsw/outsd */
3690 c->dst.val = c->regs[VCPU_REGS_RDX];
3693 case 0x70 ... 0x7f: /* jcc (short) */
3694 if (test_cc(c->b, ctxt->eflags))
3695 jmp_rel(c, c->src.val);
3697 case 0x80 ... 0x83: /* Grp1 */
3698 switch (c->modrm_reg) {
3719 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3721 case 0x86 ... 0x87: /* xchg */
3723 /* Write back the register source. */
3724 c->src.val = c->dst.val;
3725 write_register_operand(&c->src);
3727 * Write back the memory destination with implicit LOCK
3730 c->dst.val = c->src.orig_val;
3733 case 0x8c: /* mov r/m, sreg */
3734 if (c->modrm_reg > VCPU_SREG_GS) {
3735 rc = emulate_ud(ctxt);
3738 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3740 case 0x8d: /* lea r16/r32, m */
3741 c->dst.val = c->src.addr.mem.ea;
3743 case 0x8e: { /* mov seg, r/m16 */
3748 if (c->modrm_reg == VCPU_SREG_CS ||
3749 c->modrm_reg > VCPU_SREG_GS) {
3750 rc = emulate_ud(ctxt);
3754 if (c->modrm_reg == VCPU_SREG_SS)
3755 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3757 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3759 c->dst.type = OP_NONE; /* Disable writeback. */
3762 case 0x8f: /* pop (sole member of Grp1a) */
3763 rc = emulate_grp1a(ctxt, ops);
3765 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3766 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3769 case 0x98: /* cbw/cwde/cdqe */
3770 switch (c->op_bytes) {
3771 case 2: c->dst.val = (s8)c->dst.val; break;
3772 case 4: c->dst.val = (s16)c->dst.val; break;
3773 case 8: c->dst.val = (s32)c->dst.val; break;
3776 case 0x9c: /* pushf */
3777 c->src.val = (unsigned long) ctxt->eflags;
3780 case 0x9d: /* popf */
3781 c->dst.type = OP_REG;
3782 c->dst.addr.reg = &ctxt->eflags;
3783 c->dst.bytes = c->op_bytes;
3784 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3786 case 0xa6 ... 0xa7: /* cmps */
3788 case 0xa8 ... 0xa9: /* test ax, imm */
3790 case 0xae ... 0xaf: /* scas */
3795 case 0xc3: /* ret */
3796 c->dst.type = OP_REG;
3797 c->dst.addr.reg = &c->eip;
3798 c->dst.bytes = c->op_bytes;
3799 goto pop_instruction;
3800 case 0xc4: /* les */
3801 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3803 case 0xc5: /* lds */
3804 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3806 case 0xcb: /* ret far */
3807 rc = emulate_ret_far(ctxt, ops);
3809 case 0xcc: /* int3 */
3812 case 0xcd: /* int n */
3815 rc = emulate_int(ctxt, ops, irq);
3817 case 0xce: /* into */
3818 if (ctxt->eflags & EFLG_OF) {
3823 case 0xcf: /* iret */
3824 rc = emulate_iret(ctxt, ops);
3826 case 0xd0 ... 0xd1: /* Grp2 */
3829 case 0xd2 ... 0xd3: /* Grp2 */
3830 c->src.val = c->regs[VCPU_REGS_RCX];
3833 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3834 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3835 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3836 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3837 jmp_rel(c, c->src.val);
3839 case 0xe3: /* jcxz/jecxz/jrcxz */
3840 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3841 jmp_rel(c, c->src.val);
3843 case 0xe4: /* inb */
3846 case 0xe6: /* outb */
3847 case 0xe7: /* out */
3849 case 0xe8: /* call (near) */ {
3850 long int rel = c->src.val;
3851 c->src.val = (unsigned long) c->eip;
3856 case 0xe9: /* jmp rel */
3858 case 0xea: { /* jmp far */
3861 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3863 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3867 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3871 jmp: /* jmp rel short */
3872 jmp_rel(c, c->src.val);
3873 c->dst.type = OP_NONE; /* Disable writeback. */
3875 case 0xec: /* in al,dx */
3876 case 0xed: /* in (e/r)ax,dx */
3877 c->src.val = c->regs[VCPU_REGS_RDX];
3879 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3881 goto done; /* IO is needed */
3883 case 0xee: /* out dx,al */
3884 case 0xef: /* out dx,(e/r)ax */
3885 c->dst.val = c->regs[VCPU_REGS_RDX];
3887 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3888 &c->src.val, 1, ctxt->vcpu);
3889 c->dst.type = OP_NONE; /* Disable writeback. */
3891 case 0xf4: /* hlt */
3892 ctxt->vcpu->arch.halt_request = 1;
3894 case 0xf5: /* cmc */
3895 /* complement carry flag from eflags reg */
3896 ctxt->eflags ^= EFLG_CF;
3898 case 0xf6 ... 0xf7: /* Grp3 */
3899 rc = emulate_grp3(ctxt, ops);
3901 case 0xf8: /* clc */
3902 ctxt->eflags &= ~EFLG_CF;
3904 case 0xf9: /* stc */
3905 ctxt->eflags |= EFLG_CF;
3907 case 0xfa: /* cli */
3908 if (emulator_bad_iopl(ctxt, ops)) {
3909 rc = emulate_gp(ctxt, 0);
3912 ctxt->eflags &= ~X86_EFLAGS_IF;
3914 case 0xfb: /* sti */
3915 if (emulator_bad_iopl(ctxt, ops)) {
3916 rc = emulate_gp(ctxt, 0);
3919 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3920 ctxt->eflags |= X86_EFLAGS_IF;
3923 case 0xfc: /* cld */
3924 ctxt->eflags &= ~EFLG_DF;
3926 case 0xfd: /* std */
3927 ctxt->eflags |= EFLG_DF;
3929 case 0xfe: /* Grp4 */
3931 rc = emulate_grp45(ctxt);
3933 case 0xff: /* Grp5 */
3934 if (c->modrm_reg == 5)
3938 goto cannot_emulate;
3941 if (rc != X86EMUL_CONTINUE)
3945 rc = writeback(ctxt, ops);
3946 if (rc != X86EMUL_CONTINUE)
3950 * restore dst type in case the decoding will be reused
3951 * (happens for string instruction )
3953 c->dst.type = saved_dst_type;
3955 if ((c->d & SrcMask) == SrcSI)
3956 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3957 VCPU_REGS_RSI, &c->src);
3959 if ((c->d & DstMask) == DstDI)
3960 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3963 if (c->rep_prefix && (c->d & String)) {
3964 struct read_cache *r = &ctxt->decode.io_read;
3965 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3967 if (!string_insn_completed(ctxt)) {
3969 * Re-enter guest when pio read ahead buffer is empty
3970 * or, if it is not used, after each 1024 iteration.
3972 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3973 (r->end == 0 || r->end != r->pos)) {
3975 * Reset read cache. Usually happens before
3976 * decode, but since instruction is restarted
3977 * we have to do it here.
3979 ctxt->decode.mem_read.end = 0;
3980 return EMULATION_RESTART;
3982 goto done; /* skip rip writeback */
3989 if (rc == X86EMUL_PROPAGATE_FAULT)
3990 ctxt->have_exception = true;
3991 if (rc == X86EMUL_INTERCEPTED)
3992 return EMULATION_INTERCEPTED;
3994 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3998 case 0x01: /* lgdt, lidt, lmsw */
3999 switch (c->modrm_reg) {
4001 unsigned long address;
4003 case 0: /* vmcall */
4004 if (c->modrm_mod != 3 || c->modrm_rm != 1)
4005 goto cannot_emulate;
4007 rc = kvm_fix_hypercall(ctxt->vcpu);
4008 if (rc != X86EMUL_CONTINUE)
4011 /* Let the processor re-execute the fixed hypercall */
4013 /* Disable writeback. */
4014 c->dst.type = OP_NONE;
4017 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4018 &size, &address, c->op_bytes);
4019 if (rc != X86EMUL_CONTINUE)
4021 realmode_lgdt(ctxt->vcpu, size, address);
4022 /* Disable writeback. */
4023 c->dst.type = OP_NONE;
4025 case 3: /* lidt/vmmcall */
4026 if (c->modrm_mod == 3) {
4027 switch (c->modrm_rm) {
4029 rc = kvm_fix_hypercall(ctxt->vcpu);
4032 goto cannot_emulate;
4035 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4038 if (rc != X86EMUL_CONTINUE)
4040 realmode_lidt(ctxt->vcpu, size, address);
4042 /* Disable writeback. */
4043 c->dst.type = OP_NONE;
4047 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4050 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4051 (c->src.val & 0x0f), ctxt->vcpu);
4052 c->dst.type = OP_NONE;
4054 case 5: /* not defined */
4056 rc = X86EMUL_PROPAGATE_FAULT;
4059 rc = em_invlpg(ctxt);
4062 goto cannot_emulate;
4065 case 0x05: /* syscall */
4066 rc = emulate_syscall(ctxt, ops);
4069 emulate_clts(ctxt->vcpu);
4071 case 0x09: /* wbinvd */
4072 kvm_emulate_wbinvd(ctxt->vcpu);
4074 case 0x08: /* invd */
4075 case 0x0d: /* GrpP (prefetch) */
4076 case 0x18: /* Grp16 (prefetch/nop) */
4078 case 0x20: /* mov cr, reg */
4079 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4081 case 0x21: /* mov from dr to reg */
4082 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4084 case 0x22: /* mov reg, cr */
4085 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4086 emulate_gp(ctxt, 0);
4087 rc = X86EMUL_PROPAGATE_FAULT;
4090 c->dst.type = OP_NONE;
4092 case 0x23: /* mov from reg to dr */
4093 if (ops->set_dr(c->modrm_reg, c->src.val &
4094 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4095 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4096 /* #UD condition is already handled by the code above */
4097 emulate_gp(ctxt, 0);
4098 rc = X86EMUL_PROPAGATE_FAULT;
4102 c->dst.type = OP_NONE; /* no writeback */
4106 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4107 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4108 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4109 emulate_gp(ctxt, 0);
4110 rc = X86EMUL_PROPAGATE_FAULT;
4113 rc = X86EMUL_CONTINUE;
4117 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4118 emulate_gp(ctxt, 0);
4119 rc = X86EMUL_PROPAGATE_FAULT;
4122 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4123 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4125 rc = X86EMUL_CONTINUE;
4127 case 0x34: /* sysenter */
4128 rc = emulate_sysenter(ctxt, ops);
4130 case 0x35: /* sysexit */
4131 rc = emulate_sysexit(ctxt, ops);
4133 case 0x40 ... 0x4f: /* cmov */
4134 c->dst.val = c->dst.orig_val = c->src.val;
4135 if (!test_cc(c->b, ctxt->eflags))
4136 c->dst.type = OP_NONE; /* no writeback */
4138 case 0x80 ... 0x8f: /* jnz rel, etc*/
4139 if (test_cc(c->b, ctxt->eflags))
4140 jmp_rel(c, c->src.val);
4142 case 0x90 ... 0x9f: /* setcc r/m8 */
4143 c->dst.val = test_cc(c->b, ctxt->eflags);
4145 case 0xa0: /* push fs */
4146 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4148 case 0xa1: /* pop fs */
4149 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4153 c->dst.type = OP_NONE;
4154 /* only subword offset */
4155 c->src.val &= (c->dst.bytes << 3) - 1;
4156 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4158 case 0xa4: /* shld imm8, r, r/m */
4159 case 0xa5: /* shld cl, r, r/m */
4160 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4162 case 0xa8: /* push gs */
4163 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4165 case 0xa9: /* pop gs */
4166 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4170 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4172 case 0xac: /* shrd imm8, r, r/m */
4173 case 0xad: /* shrd cl, r, r/m */
4174 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4176 case 0xae: /* clflush */
4178 case 0xb0 ... 0xb1: /* cmpxchg */
4180 * Save real source value, then compare EAX against
4183 c->src.orig_val = c->src.val;
4184 c->src.val = c->regs[VCPU_REGS_RAX];
4185 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4186 if (ctxt->eflags & EFLG_ZF) {
4187 /* Success: write back to memory. */
4188 c->dst.val = c->src.orig_val;
4190 /* Failure: write the value we saw to EAX. */
4191 c->dst.type = OP_REG;
4192 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4195 case 0xb2: /* lss */
4196 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4200 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4202 case 0xb4: /* lfs */
4203 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4205 case 0xb5: /* lgs */
4206 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4208 case 0xb6 ... 0xb7: /* movzx */
4209 c->dst.bytes = c->op_bytes;
4210 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4213 case 0xba: /* Grp8 */
4214 switch (c->modrm_reg & 3) {
4227 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4229 case 0xbc: { /* bsf */
4231 __asm__ ("bsf %2, %0; setz %1"
4232 : "=r"(c->dst.val), "=q"(zf)
4234 ctxt->eflags &= ~X86_EFLAGS_ZF;
4236 ctxt->eflags |= X86_EFLAGS_ZF;
4237 c->dst.type = OP_NONE; /* Disable writeback. */
4241 case 0xbd: { /* bsr */
4243 __asm__ ("bsr %2, %0; setz %1"
4244 : "=r"(c->dst.val), "=q"(zf)
4246 ctxt->eflags &= ~X86_EFLAGS_ZF;
4248 ctxt->eflags |= X86_EFLAGS_ZF;
4249 c->dst.type = OP_NONE; /* Disable writeback. */
4253 case 0xbe ... 0xbf: /* movsx */
4254 c->dst.bytes = c->op_bytes;
4255 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4258 case 0xc0 ... 0xc1: /* xadd */
4259 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4260 /* Write back the register source. */
4261 c->src.val = c->dst.orig_val;
4262 write_register_operand(&c->src);
4264 case 0xc3: /* movnti */
4265 c->dst.bytes = c->op_bytes;
4266 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4269 case 0xc7: /* Grp9 (cmpxchg8b) */
4270 rc = emulate_grp9(ctxt, ops);
4273 goto cannot_emulate;
4276 if (rc != X86EMUL_CONTINUE)
4282 return EMULATION_FAILED;