1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct decode_cache *c, int seg)
476 c->has_seg_override = true;
477 c->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
481 struct x86_emulate_ops *ops, int seg)
483 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
486 return ops->get_cached_segment_base(seg, ctxt->vcpu);
489 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
490 struct x86_emulate_ops *ops,
491 struct decode_cache *c)
493 if (!c->has_seg_override)
496 return c->seg_override;
499 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
500 u32 error, bool valid)
502 ctxt->exception.vector = vec;
503 ctxt->exception.error_code = error;
504 ctxt->exception.error_code_valid = valid;
505 return X86EMUL_PROPAGATE_FAULT;
508 static int emulate_db(struct x86_emulate_ctxt *ctxt)
510 return emulate_exception(ctxt, DB_VECTOR, 0, false);
513 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, GP_VECTOR, err, true);
518 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, SS_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int linearize(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
545 unsigned size, bool write,
548 struct decode_cache *c = &ctxt->decode;
549 struct desc_struct desc;
555 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
556 switch (ctxt->mode) {
557 case X86EMUL_MODE_REAL:
559 case X86EMUL_MODE_PROT64:
560 if (((signed long)la << 16) >> 16 != la)
561 return emulate_gp(ctxt, 0);
564 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
568 /* code segment or read-only data segment */
569 if (((desc.type & 8) || !(desc.type & 2)) && write)
571 /* unreadable code segment */
572 if ((desc.type & 8) && !(desc.type & 2))
574 lim = desc_limit_scaled(&desc);
575 if ((desc.type & 8) || !(desc.type & 4)) {
576 /* expand-up segment */
577 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
580 /* exapand-down segment */
581 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
583 lim = desc.d ? 0xffffffff : 0xffff;
584 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
587 cpl = ctxt->ops->cpl(ctxt->vcpu);
588 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
590 if (!(desc.type & 8)) {
594 } else if ((desc.type & 8) && !(desc.type & 4)) {
595 /* nonconforming code segment */
598 } else if ((desc.type & 8) && (desc.type & 4)) {
599 /* conforming code segment */
605 if (c->ad_bytes != 8)
608 return X86EMUL_CONTINUE;
610 if (addr.seg == VCPU_SREG_SS)
611 return emulate_ss(ctxt, addr.seg);
613 return emulate_gp(ctxt, addr.seg);
616 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
617 struct segmented_address addr,
624 rc = linearize(ctxt, addr, size, false, &linear);
625 if (rc != X86EMUL_CONTINUE)
627 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
631 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops,
633 unsigned long eip, u8 *dest)
635 struct fetch_cache *fc = &ctxt->decode.fetch;
639 if (eip == fc->end) {
640 cur_size = fc->end - fc->start;
641 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
642 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
643 size, ctxt->vcpu, &ctxt->exception);
644 if (rc != X86EMUL_CONTINUE)
648 *dest = fc->data[eip - fc->start];
649 return X86EMUL_CONTINUE;
652 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
653 struct x86_emulate_ops *ops,
654 unsigned long eip, void *dest, unsigned size)
658 /* x86 instructions are limited to 15 bytes. */
659 if (eip + size - ctxt->eip > 15)
660 return X86EMUL_UNHANDLEABLE;
662 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
663 if (rc != X86EMUL_CONTINUE)
666 return X86EMUL_CONTINUE;
670 * Given the 'reg' portion of a ModRM byte, and a register block, return a
671 * pointer into the block that addresses the relevant register.
672 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
674 static void *decode_register(u8 modrm_reg, unsigned long *regs,
679 p = ®s[modrm_reg];
680 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
681 p = (unsigned char *)®s[modrm_reg & 3] + 1;
685 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
686 struct x86_emulate_ops *ops,
687 struct segmented_address addr,
688 u16 *size, unsigned long *address, int op_bytes)
695 rc = segmented_read_std(ctxt, addr, size, 2);
696 if (rc != X86EMUL_CONTINUE)
699 rc = segmented_read_std(ctxt, addr, address, op_bytes);
703 static int test_cc(unsigned int condition, unsigned int flags)
707 switch ((condition & 15) >> 1) {
709 rc |= (flags & EFLG_OF);
711 case 1: /* b/c/nae */
712 rc |= (flags & EFLG_CF);
715 rc |= (flags & EFLG_ZF);
718 rc |= (flags & (EFLG_CF|EFLG_ZF));
721 rc |= (flags & EFLG_SF);
724 rc |= (flags & EFLG_PF);
727 rc |= (flags & EFLG_ZF);
730 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
734 /* Odd condition identifiers (lsb == 1) have inverted sense. */
735 return (!!rc ^ (condition & 1));
738 static void fetch_register_operand(struct operand *op)
742 op->val = *(u8 *)op->addr.reg;
745 op->val = *(u16 *)op->addr.reg;
748 op->val = *(u32 *)op->addr.reg;
751 op->val = *(u64 *)op->addr.reg;
756 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
758 ctxt->ops->get_fpu(ctxt);
760 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
761 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
762 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
763 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
764 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
765 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
766 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
767 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
769 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
770 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
771 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
772 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
773 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
774 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
775 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
776 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
780 ctxt->ops->put_fpu(ctxt);
783 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
786 ctxt->ops->get_fpu(ctxt);
788 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
789 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
790 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
791 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
792 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
793 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
794 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
795 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
797 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
798 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
799 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
800 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
801 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
802 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
803 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
804 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
808 ctxt->ops->put_fpu(ctxt);
811 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
813 struct decode_cache *c,
816 unsigned reg = c->modrm_reg;
817 int highbyte_regs = c->rex_prefix == 0;
820 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
826 read_sse_reg(ctxt, &op->vec_val, reg);
831 if ((c->d & ByteOp) && !inhibit_bytereg) {
832 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
835 op->addr.reg = decode_register(reg, c->regs, 0);
836 op->bytes = c->op_bytes;
838 fetch_register_operand(op);
839 op->orig_val = op->val;
842 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
843 struct x86_emulate_ops *ops,
846 struct decode_cache *c = &ctxt->decode;
848 int index_reg = 0, base_reg = 0, scale;
849 int rc = X86EMUL_CONTINUE;
853 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
854 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
855 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
858 c->modrm = insn_fetch(u8, 1, c->eip);
859 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
860 c->modrm_reg |= (c->modrm & 0x38) >> 3;
861 c->modrm_rm |= (c->modrm & 0x07);
862 c->modrm_seg = VCPU_SREG_DS;
864 if (c->modrm_mod == 3) {
866 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
867 op->addr.reg = decode_register(c->modrm_rm,
868 c->regs, c->d & ByteOp);
872 op->addr.xmm = c->modrm_rm;
873 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
876 fetch_register_operand(op);
882 if (c->ad_bytes == 2) {
883 unsigned bx = c->regs[VCPU_REGS_RBX];
884 unsigned bp = c->regs[VCPU_REGS_RBP];
885 unsigned si = c->regs[VCPU_REGS_RSI];
886 unsigned di = c->regs[VCPU_REGS_RDI];
888 /* 16-bit ModR/M decode. */
889 switch (c->modrm_mod) {
891 if (c->modrm_rm == 6)
892 modrm_ea += insn_fetch(u16, 2, c->eip);
895 modrm_ea += insn_fetch(s8, 1, c->eip);
898 modrm_ea += insn_fetch(u16, 2, c->eip);
901 switch (c->modrm_rm) {
921 if (c->modrm_mod != 0)
928 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
929 (c->modrm_rm == 6 && c->modrm_mod != 0))
930 c->modrm_seg = VCPU_SREG_SS;
931 modrm_ea = (u16)modrm_ea;
933 /* 32/64-bit ModR/M decode. */
934 if ((c->modrm_rm & 7) == 4) {
935 sib = insn_fetch(u8, 1, c->eip);
936 index_reg |= (sib >> 3) & 7;
940 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
941 modrm_ea += insn_fetch(s32, 4, c->eip);
943 modrm_ea += c->regs[base_reg];
945 modrm_ea += c->regs[index_reg] << scale;
946 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
947 if (ctxt->mode == X86EMUL_MODE_PROT64)
950 modrm_ea += c->regs[c->modrm_rm];
951 switch (c->modrm_mod) {
953 if (c->modrm_rm == 5)
954 modrm_ea += insn_fetch(s32, 4, c->eip);
957 modrm_ea += insn_fetch(s8, 1, c->eip);
960 modrm_ea += insn_fetch(s32, 4, c->eip);
964 op->addr.mem.ea = modrm_ea;
969 static int decode_abs(struct x86_emulate_ctxt *ctxt,
970 struct x86_emulate_ops *ops,
973 struct decode_cache *c = &ctxt->decode;
974 int rc = X86EMUL_CONTINUE;
977 switch (c->ad_bytes) {
979 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
982 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
985 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
992 static void fetch_bit_operand(struct decode_cache *c)
996 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
997 mask = ~(c->dst.bytes * 8 - 1);
999 if (c->src.bytes == 2)
1000 sv = (s16)c->src.val & (s16)mask;
1001 else if (c->src.bytes == 4)
1002 sv = (s32)c->src.val & (s32)mask;
1004 c->dst.addr.mem.ea += (sv >> 3);
1007 /* only subword offset */
1008 c->src.val &= (c->dst.bytes << 3) - 1;
1011 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1012 struct x86_emulate_ops *ops,
1013 unsigned long addr, void *dest, unsigned size)
1016 struct read_cache *mc = &ctxt->decode.mem_read;
1019 int n = min(size, 8u);
1021 if (mc->pos < mc->end)
1024 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1025 &ctxt->exception, ctxt->vcpu);
1026 if (rc != X86EMUL_CONTINUE)
1031 memcpy(dest, mc->data + mc->pos, n);
1036 return X86EMUL_CONTINUE;
1039 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1040 struct segmented_address addr,
1047 rc = linearize(ctxt, addr, size, false, &linear);
1048 if (rc != X86EMUL_CONTINUE)
1050 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1053 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1054 struct segmented_address addr,
1061 rc = linearize(ctxt, addr, size, true, &linear);
1062 if (rc != X86EMUL_CONTINUE)
1064 return ctxt->ops->write_emulated(linear, data, size,
1065 &ctxt->exception, ctxt->vcpu);
1068 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1069 struct segmented_address addr,
1070 const void *orig_data, const void *data,
1076 rc = linearize(ctxt, addr, size, true, &linear);
1077 if (rc != X86EMUL_CONTINUE)
1079 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1080 size, &ctxt->exception, ctxt->vcpu);
1083 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1084 struct x86_emulate_ops *ops,
1085 unsigned int size, unsigned short port,
1088 struct read_cache *rc = &ctxt->decode.io_read;
1090 if (rc->pos == rc->end) { /* refill pio read ahead */
1091 struct decode_cache *c = &ctxt->decode;
1092 unsigned int in_page, n;
1093 unsigned int count = c->rep_prefix ?
1094 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1095 in_page = (ctxt->eflags & EFLG_DF) ?
1096 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1097 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1098 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1102 rc->pos = rc->end = 0;
1103 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1108 memcpy(dest, rc->data + rc->pos, size);
1113 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1114 struct x86_emulate_ops *ops,
1115 u16 selector, struct desc_ptr *dt)
1117 if (selector & 1 << 2) {
1118 struct desc_struct desc;
1119 memset (dt, 0, sizeof *dt);
1120 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1124 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1125 dt->address = get_desc_base(&desc);
1127 ops->get_gdt(dt, ctxt->vcpu);
1130 /* allowed just for 8 bytes segments */
1131 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1132 struct x86_emulate_ops *ops,
1133 u16 selector, struct desc_struct *desc)
1136 u16 index = selector >> 3;
1140 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1142 if (dt.size < index * 8 + 7)
1143 return emulate_gp(ctxt, selector & 0xfffc);
1144 addr = dt.address + index * 8;
1145 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1151 /* allowed just for 8 bytes segments */
1152 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1153 struct x86_emulate_ops *ops,
1154 u16 selector, struct desc_struct *desc)
1157 u16 index = selector >> 3;
1161 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1163 if (dt.size < index * 8 + 7)
1164 return emulate_gp(ctxt, selector & 0xfffc);
1166 addr = dt.address + index * 8;
1167 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1173 /* Does not support long mode */
1174 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 struct x86_emulate_ops *ops,
1176 u16 selector, int seg)
1178 struct desc_struct seg_desc;
1180 unsigned err_vec = GP_VECTOR;
1182 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1185 memset(&seg_desc, 0, sizeof seg_desc);
1187 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1188 || ctxt->mode == X86EMUL_MODE_REAL) {
1189 /* set real mode segment descriptor */
1190 set_desc_base(&seg_desc, selector << 4);
1191 set_desc_limit(&seg_desc, 0xffff);
1198 /* NULL selector is not valid for TR, CS and SS */
1199 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1203 /* TR should be in GDT only */
1204 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1207 if (null_selector) /* for NULL selector skip all following checks */
1210 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1211 if (ret != X86EMUL_CONTINUE)
1214 err_code = selector & 0xfffc;
1215 err_vec = GP_VECTOR;
1217 /* can't load system descriptor into segment selecor */
1218 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1222 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1228 cpl = ops->cpl(ctxt->vcpu);
1233 * segment is not a writable data segment or segment
1234 * selector's RPL != CPL or segment selector's RPL != CPL
1236 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1240 if (!(seg_desc.type & 8))
1243 if (seg_desc.type & 4) {
1249 if (rpl > cpl || dpl != cpl)
1252 /* CS(RPL) <- CPL */
1253 selector = (selector & 0xfffc) | cpl;
1256 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1259 case VCPU_SREG_LDTR:
1260 if (seg_desc.s || seg_desc.type != 2)
1263 default: /* DS, ES, FS, or GS */
1265 * segment is not a data or readable code segment or
1266 * ((segment is a data or nonconforming code segment)
1267 * and (both RPL and CPL > DPL))
1269 if ((seg_desc.type & 0xa) == 0x8 ||
1270 (((seg_desc.type & 0xc) != 0xc) &&
1271 (rpl > dpl && cpl > dpl)))
1277 /* mark segment as accessed */
1279 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1280 if (ret != X86EMUL_CONTINUE)
1284 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1285 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1286 return X86EMUL_CONTINUE;
1288 emulate_exception(ctxt, err_vec, err_code, true);
1289 return X86EMUL_PROPAGATE_FAULT;
1292 static void write_register_operand(struct operand *op)
1294 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1295 switch (op->bytes) {
1297 *(u8 *)op->addr.reg = (u8)op->val;
1300 *(u16 *)op->addr.reg = (u16)op->val;
1303 *op->addr.reg = (u32)op->val;
1304 break; /* 64b: zero-extend */
1306 *op->addr.reg = op->val;
1311 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1312 struct x86_emulate_ops *ops)
1315 struct decode_cache *c = &ctxt->decode;
1317 switch (c->dst.type) {
1319 write_register_operand(&c->dst);
1323 rc = segmented_cmpxchg(ctxt,
1329 rc = segmented_write(ctxt,
1333 if (rc != X86EMUL_CONTINUE)
1337 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1345 return X86EMUL_CONTINUE;
1348 static int emulate_push(struct x86_emulate_ctxt *ctxt,
1349 struct x86_emulate_ops *ops)
1351 struct decode_cache *c = &ctxt->decode;
1352 struct segmented_address addr;
1354 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1355 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1356 addr.seg = VCPU_SREG_SS;
1358 /* Disable writeback. */
1359 c->dst.type = OP_NONE;
1360 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1363 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1364 struct x86_emulate_ops *ops,
1365 void *dest, int len)
1367 struct decode_cache *c = &ctxt->decode;
1369 struct segmented_address addr;
1371 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1372 addr.seg = VCPU_SREG_SS;
1373 rc = segmented_read(ctxt, addr, dest, len);
1374 if (rc != X86EMUL_CONTINUE)
1377 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1381 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1382 struct x86_emulate_ops *ops,
1383 void *dest, int len)
1386 unsigned long val, change_mask;
1387 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1388 int cpl = ops->cpl(ctxt->vcpu);
1390 rc = emulate_pop(ctxt, ops, &val, len);
1391 if (rc != X86EMUL_CONTINUE)
1394 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1395 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1397 switch(ctxt->mode) {
1398 case X86EMUL_MODE_PROT64:
1399 case X86EMUL_MODE_PROT32:
1400 case X86EMUL_MODE_PROT16:
1402 change_mask |= EFLG_IOPL;
1404 change_mask |= EFLG_IF;
1406 case X86EMUL_MODE_VM86:
1408 return emulate_gp(ctxt, 0);
1409 change_mask |= EFLG_IF;
1411 default: /* real mode */
1412 change_mask |= (EFLG_IOPL | EFLG_IF);
1416 *(unsigned long *)dest =
1417 (ctxt->eflags & ~change_mask) | (val & change_mask);
1422 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1423 struct x86_emulate_ops *ops, int seg)
1425 struct decode_cache *c = &ctxt->decode;
1427 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1429 return emulate_push(ctxt, ops);
1432 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1433 struct x86_emulate_ops *ops, int seg)
1435 struct decode_cache *c = &ctxt->decode;
1436 unsigned long selector;
1439 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1440 if (rc != X86EMUL_CONTINUE)
1443 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1447 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1448 struct x86_emulate_ops *ops)
1450 struct decode_cache *c = &ctxt->decode;
1451 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1452 int rc = X86EMUL_CONTINUE;
1453 int reg = VCPU_REGS_RAX;
1455 while (reg <= VCPU_REGS_RDI) {
1456 (reg == VCPU_REGS_RSP) ?
1457 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1459 rc = emulate_push(ctxt, ops);
1460 if (rc != X86EMUL_CONTINUE)
1469 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1470 struct x86_emulate_ops *ops)
1472 struct decode_cache *c = &ctxt->decode;
1473 int rc = X86EMUL_CONTINUE;
1474 int reg = VCPU_REGS_RDI;
1476 while (reg >= VCPU_REGS_RAX) {
1477 if (reg == VCPU_REGS_RSP) {
1478 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1483 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1484 if (rc != X86EMUL_CONTINUE)
1491 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1492 struct x86_emulate_ops *ops, int irq)
1494 struct decode_cache *c = &ctxt->decode;
1501 /* TODO: Add limit checks */
1502 c->src.val = ctxt->eflags;
1503 rc = emulate_push(ctxt, ops);
1504 if (rc != X86EMUL_CONTINUE)
1507 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1509 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1510 rc = emulate_push(ctxt, ops);
1511 if (rc != X86EMUL_CONTINUE)
1514 c->src.val = c->eip;
1515 rc = emulate_push(ctxt, ops);
1516 if (rc != X86EMUL_CONTINUE)
1519 ops->get_idt(&dt, ctxt->vcpu);
1521 eip_addr = dt.address + (irq << 2);
1522 cs_addr = dt.address + (irq << 2) + 2;
1524 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1525 if (rc != X86EMUL_CONTINUE)
1528 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1529 if (rc != X86EMUL_CONTINUE)
1532 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1533 if (rc != X86EMUL_CONTINUE)
1541 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1542 struct x86_emulate_ops *ops, int irq)
1544 switch(ctxt->mode) {
1545 case X86EMUL_MODE_REAL:
1546 return emulate_int_real(ctxt, ops, irq);
1547 case X86EMUL_MODE_VM86:
1548 case X86EMUL_MODE_PROT16:
1549 case X86EMUL_MODE_PROT32:
1550 case X86EMUL_MODE_PROT64:
1552 /* Protected mode interrupts unimplemented yet */
1553 return X86EMUL_UNHANDLEABLE;
1557 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1558 struct x86_emulate_ops *ops)
1560 struct decode_cache *c = &ctxt->decode;
1561 int rc = X86EMUL_CONTINUE;
1562 unsigned long temp_eip = 0;
1563 unsigned long temp_eflags = 0;
1564 unsigned long cs = 0;
1565 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1566 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1567 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1568 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1570 /* TODO: Add stack limit check */
1572 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1574 if (rc != X86EMUL_CONTINUE)
1577 if (temp_eip & ~0xffff)
1578 return emulate_gp(ctxt, 0);
1580 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1582 if (rc != X86EMUL_CONTINUE)
1585 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1587 if (rc != X86EMUL_CONTINUE)
1590 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1592 if (rc != X86EMUL_CONTINUE)
1598 if (c->op_bytes == 4)
1599 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1600 else if (c->op_bytes == 2) {
1601 ctxt->eflags &= ~0xffff;
1602 ctxt->eflags |= temp_eflags;
1605 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1606 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1611 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1612 struct x86_emulate_ops* ops)
1614 switch(ctxt->mode) {
1615 case X86EMUL_MODE_REAL:
1616 return emulate_iret_real(ctxt, ops);
1617 case X86EMUL_MODE_VM86:
1618 case X86EMUL_MODE_PROT16:
1619 case X86EMUL_MODE_PROT32:
1620 case X86EMUL_MODE_PROT64:
1622 /* iret from protected mode unimplemented yet */
1623 return X86EMUL_UNHANDLEABLE;
1627 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1628 struct x86_emulate_ops *ops)
1630 struct decode_cache *c = &ctxt->decode;
1632 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1635 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1637 struct decode_cache *c = &ctxt->decode;
1638 switch (c->modrm_reg) {
1640 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1643 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1646 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1649 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1651 case 4: /* sal/shl */
1652 case 6: /* sal/shl */
1653 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1656 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1659 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1664 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1665 struct x86_emulate_ops *ops)
1667 struct decode_cache *c = &ctxt->decode;
1668 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1669 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1672 switch (c->modrm_reg) {
1673 case 0 ... 1: /* test */
1674 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1677 c->dst.val = ~c->dst.val;
1680 emulate_1op("neg", c->dst, ctxt->eflags);
1683 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1686 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1689 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1693 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1697 return X86EMUL_UNHANDLEABLE;
1700 return emulate_de(ctxt);
1701 return X86EMUL_CONTINUE;
1704 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1705 struct x86_emulate_ops *ops)
1707 struct decode_cache *c = &ctxt->decode;
1708 int rc = X86EMUL_CONTINUE;
1710 switch (c->modrm_reg) {
1712 emulate_1op("inc", c->dst, ctxt->eflags);
1715 emulate_1op("dec", c->dst, ctxt->eflags);
1717 case 2: /* call near abs */ {
1720 c->eip = c->src.val;
1721 c->src.val = old_eip;
1722 rc = emulate_push(ctxt, ops);
1725 case 4: /* jmp abs */
1726 c->eip = c->src.val;
1729 rc = emulate_push(ctxt, ops);
1735 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1736 struct x86_emulate_ops *ops)
1738 struct decode_cache *c = &ctxt->decode;
1739 u64 old = c->dst.orig_val64;
1741 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1742 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1743 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1744 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1745 ctxt->eflags &= ~EFLG_ZF;
1747 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1748 (u32) c->regs[VCPU_REGS_RBX];
1750 ctxt->eflags |= EFLG_ZF;
1752 return X86EMUL_CONTINUE;
1755 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1756 struct x86_emulate_ops *ops)
1758 struct decode_cache *c = &ctxt->decode;
1762 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1763 if (rc != X86EMUL_CONTINUE)
1765 if (c->op_bytes == 4)
1766 c->eip = (u32)c->eip;
1767 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1768 if (rc != X86EMUL_CONTINUE)
1770 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1774 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1775 struct x86_emulate_ops *ops, int seg)
1777 struct decode_cache *c = &ctxt->decode;
1781 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1783 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1784 if (rc != X86EMUL_CONTINUE)
1787 c->dst.val = c->src.val;
1792 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1793 struct x86_emulate_ops *ops, struct desc_struct *cs,
1794 struct desc_struct *ss)
1796 memset(cs, 0, sizeof(struct desc_struct));
1797 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1798 memset(ss, 0, sizeof(struct desc_struct));
1800 cs->l = 0; /* will be adjusted later */
1801 set_desc_base(cs, 0); /* flat segment */
1802 cs->g = 1; /* 4kb granularity */
1803 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1804 cs->type = 0x0b; /* Read, Execute, Accessed */
1806 cs->dpl = 0; /* will be adjusted later */
1810 set_desc_base(ss, 0); /* flat segment */
1811 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1812 ss->g = 1; /* 4kb granularity */
1814 ss->type = 0x03; /* Read/Write, Accessed */
1815 ss->d = 1; /* 32bit stack segment */
1821 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1823 struct decode_cache *c = &ctxt->decode;
1824 struct desc_struct cs, ss;
1828 /* syscall is not available in real mode */
1829 if (ctxt->mode == X86EMUL_MODE_REAL ||
1830 ctxt->mode == X86EMUL_MODE_VM86)
1831 return emulate_ud(ctxt);
1833 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1835 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1837 cs_sel = (u16)(msr_data & 0xfffc);
1838 ss_sel = (u16)(msr_data + 8);
1840 if (is_long_mode(ctxt->vcpu)) {
1844 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1845 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1846 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1847 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1849 c->regs[VCPU_REGS_RCX] = c->eip;
1850 if (is_long_mode(ctxt->vcpu)) {
1851 #ifdef CONFIG_X86_64
1852 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1854 ops->get_msr(ctxt->vcpu,
1855 ctxt->mode == X86EMUL_MODE_PROT64 ?
1856 MSR_LSTAR : MSR_CSTAR, &msr_data);
1859 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1860 ctxt->eflags &= ~(msr_data | EFLG_RF);
1864 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1865 c->eip = (u32)msr_data;
1867 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1870 return X86EMUL_CONTINUE;
1874 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1876 struct decode_cache *c = &ctxt->decode;
1877 struct desc_struct cs, ss;
1881 /* inject #GP if in real mode */
1882 if (ctxt->mode == X86EMUL_MODE_REAL)
1883 return emulate_gp(ctxt, 0);
1885 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1886 * Therefore, we inject an #UD.
1888 if (ctxt->mode == X86EMUL_MODE_PROT64)
1889 return emulate_ud(ctxt);
1891 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1893 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1894 switch (ctxt->mode) {
1895 case X86EMUL_MODE_PROT32:
1896 if ((msr_data & 0xfffc) == 0x0)
1897 return emulate_gp(ctxt, 0);
1899 case X86EMUL_MODE_PROT64:
1900 if (msr_data == 0x0)
1901 return emulate_gp(ctxt, 0);
1905 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1906 cs_sel = (u16)msr_data;
1907 cs_sel &= ~SELECTOR_RPL_MASK;
1908 ss_sel = cs_sel + 8;
1909 ss_sel &= ~SELECTOR_RPL_MASK;
1910 if (ctxt->mode == X86EMUL_MODE_PROT64
1911 || is_long_mode(ctxt->vcpu)) {
1916 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1917 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1918 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1919 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1921 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1924 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1925 c->regs[VCPU_REGS_RSP] = msr_data;
1927 return X86EMUL_CONTINUE;
1931 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1933 struct decode_cache *c = &ctxt->decode;
1934 struct desc_struct cs, ss;
1939 /* inject #GP if in real mode or Virtual 8086 mode */
1940 if (ctxt->mode == X86EMUL_MODE_REAL ||
1941 ctxt->mode == X86EMUL_MODE_VM86)
1942 return emulate_gp(ctxt, 0);
1944 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1946 if ((c->rex_prefix & 0x8) != 0x0)
1947 usermode = X86EMUL_MODE_PROT64;
1949 usermode = X86EMUL_MODE_PROT32;
1953 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1955 case X86EMUL_MODE_PROT32:
1956 cs_sel = (u16)(msr_data + 16);
1957 if ((msr_data & 0xfffc) == 0x0)
1958 return emulate_gp(ctxt, 0);
1959 ss_sel = (u16)(msr_data + 24);
1961 case X86EMUL_MODE_PROT64:
1962 cs_sel = (u16)(msr_data + 32);
1963 if (msr_data == 0x0)
1964 return emulate_gp(ctxt, 0);
1965 ss_sel = cs_sel + 8;
1970 cs_sel |= SELECTOR_RPL_MASK;
1971 ss_sel |= SELECTOR_RPL_MASK;
1973 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1974 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1975 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1976 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1978 c->eip = c->regs[VCPU_REGS_RDX];
1979 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1981 return X86EMUL_CONTINUE;
1984 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1985 struct x86_emulate_ops *ops)
1988 if (ctxt->mode == X86EMUL_MODE_REAL)
1990 if (ctxt->mode == X86EMUL_MODE_VM86)
1992 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1993 return ops->cpl(ctxt->vcpu) > iopl;
1996 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1997 struct x86_emulate_ops *ops,
2000 struct desc_struct tr_seg;
2003 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2004 unsigned mask = (1 << len) - 1;
2007 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2010 if (desc_limit_scaled(&tr_seg) < 103)
2012 base = get_desc_base(&tr_seg);
2013 #ifdef CONFIG_X86_64
2014 base |= ((u64)base3) << 32;
2016 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2017 if (r != X86EMUL_CONTINUE)
2019 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2021 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2023 if (r != X86EMUL_CONTINUE)
2025 if ((perm >> bit_idx) & mask)
2030 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2031 struct x86_emulate_ops *ops,
2037 if (emulator_bad_iopl(ctxt, ops))
2038 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2041 ctxt->perm_ok = true;
2046 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2047 struct x86_emulate_ops *ops,
2048 struct tss_segment_16 *tss)
2050 struct decode_cache *c = &ctxt->decode;
2053 tss->flag = ctxt->eflags;
2054 tss->ax = c->regs[VCPU_REGS_RAX];
2055 tss->cx = c->regs[VCPU_REGS_RCX];
2056 tss->dx = c->regs[VCPU_REGS_RDX];
2057 tss->bx = c->regs[VCPU_REGS_RBX];
2058 tss->sp = c->regs[VCPU_REGS_RSP];
2059 tss->bp = c->regs[VCPU_REGS_RBP];
2060 tss->si = c->regs[VCPU_REGS_RSI];
2061 tss->di = c->regs[VCPU_REGS_RDI];
2063 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2064 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2065 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2066 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2067 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2070 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2071 struct x86_emulate_ops *ops,
2072 struct tss_segment_16 *tss)
2074 struct decode_cache *c = &ctxt->decode;
2078 ctxt->eflags = tss->flag | 2;
2079 c->regs[VCPU_REGS_RAX] = tss->ax;
2080 c->regs[VCPU_REGS_RCX] = tss->cx;
2081 c->regs[VCPU_REGS_RDX] = tss->dx;
2082 c->regs[VCPU_REGS_RBX] = tss->bx;
2083 c->regs[VCPU_REGS_RSP] = tss->sp;
2084 c->regs[VCPU_REGS_RBP] = tss->bp;
2085 c->regs[VCPU_REGS_RSI] = tss->si;
2086 c->regs[VCPU_REGS_RDI] = tss->di;
2089 * SDM says that segment selectors are loaded before segment
2092 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2093 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2094 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2095 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2096 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2099 * Now load segment descriptors. If fault happenes at this stage
2100 * it is handled in a context of new task
2102 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2103 if (ret != X86EMUL_CONTINUE)
2105 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2106 if (ret != X86EMUL_CONTINUE)
2108 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2109 if (ret != X86EMUL_CONTINUE)
2111 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2112 if (ret != X86EMUL_CONTINUE)
2114 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2115 if (ret != X86EMUL_CONTINUE)
2118 return X86EMUL_CONTINUE;
2121 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2122 struct x86_emulate_ops *ops,
2123 u16 tss_selector, u16 old_tss_sel,
2124 ulong old_tss_base, struct desc_struct *new_desc)
2126 struct tss_segment_16 tss_seg;
2128 u32 new_tss_base = get_desc_base(new_desc);
2130 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2132 if (ret != X86EMUL_CONTINUE)
2133 /* FIXME: need to provide precise fault address */
2136 save_state_to_tss16(ctxt, ops, &tss_seg);
2138 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2140 if (ret != X86EMUL_CONTINUE)
2141 /* FIXME: need to provide precise fault address */
2144 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2146 if (ret != X86EMUL_CONTINUE)
2147 /* FIXME: need to provide precise fault address */
2150 if (old_tss_sel != 0xffff) {
2151 tss_seg.prev_task_link = old_tss_sel;
2153 ret = ops->write_std(new_tss_base,
2154 &tss_seg.prev_task_link,
2155 sizeof tss_seg.prev_task_link,
2156 ctxt->vcpu, &ctxt->exception);
2157 if (ret != X86EMUL_CONTINUE)
2158 /* FIXME: need to provide precise fault address */
2162 return load_state_from_tss16(ctxt, ops, &tss_seg);
2165 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2166 struct x86_emulate_ops *ops,
2167 struct tss_segment_32 *tss)
2169 struct decode_cache *c = &ctxt->decode;
2171 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2173 tss->eflags = ctxt->eflags;
2174 tss->eax = c->regs[VCPU_REGS_RAX];
2175 tss->ecx = c->regs[VCPU_REGS_RCX];
2176 tss->edx = c->regs[VCPU_REGS_RDX];
2177 tss->ebx = c->regs[VCPU_REGS_RBX];
2178 tss->esp = c->regs[VCPU_REGS_RSP];
2179 tss->ebp = c->regs[VCPU_REGS_RBP];
2180 tss->esi = c->regs[VCPU_REGS_RSI];
2181 tss->edi = c->regs[VCPU_REGS_RDI];
2183 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2184 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2185 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2186 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2187 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2188 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2189 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2192 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2193 struct x86_emulate_ops *ops,
2194 struct tss_segment_32 *tss)
2196 struct decode_cache *c = &ctxt->decode;
2199 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2200 return emulate_gp(ctxt, 0);
2202 ctxt->eflags = tss->eflags | 2;
2203 c->regs[VCPU_REGS_RAX] = tss->eax;
2204 c->regs[VCPU_REGS_RCX] = tss->ecx;
2205 c->regs[VCPU_REGS_RDX] = tss->edx;
2206 c->regs[VCPU_REGS_RBX] = tss->ebx;
2207 c->regs[VCPU_REGS_RSP] = tss->esp;
2208 c->regs[VCPU_REGS_RBP] = tss->ebp;
2209 c->regs[VCPU_REGS_RSI] = tss->esi;
2210 c->regs[VCPU_REGS_RDI] = tss->edi;
2213 * SDM says that segment selectors are loaded before segment
2216 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2217 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2218 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2219 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2220 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2221 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2222 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2225 * Now load segment descriptors. If fault happenes at this stage
2226 * it is handled in a context of new task
2228 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2229 if (ret != X86EMUL_CONTINUE)
2231 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2232 if (ret != X86EMUL_CONTINUE)
2234 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2235 if (ret != X86EMUL_CONTINUE)
2237 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2238 if (ret != X86EMUL_CONTINUE)
2240 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2241 if (ret != X86EMUL_CONTINUE)
2243 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2244 if (ret != X86EMUL_CONTINUE)
2246 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2247 if (ret != X86EMUL_CONTINUE)
2250 return X86EMUL_CONTINUE;
2253 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2254 struct x86_emulate_ops *ops,
2255 u16 tss_selector, u16 old_tss_sel,
2256 ulong old_tss_base, struct desc_struct *new_desc)
2258 struct tss_segment_32 tss_seg;
2260 u32 new_tss_base = get_desc_base(new_desc);
2262 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2264 if (ret != X86EMUL_CONTINUE)
2265 /* FIXME: need to provide precise fault address */
2268 save_state_to_tss32(ctxt, ops, &tss_seg);
2270 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2272 if (ret != X86EMUL_CONTINUE)
2273 /* FIXME: need to provide precise fault address */
2276 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2278 if (ret != X86EMUL_CONTINUE)
2279 /* FIXME: need to provide precise fault address */
2282 if (old_tss_sel != 0xffff) {
2283 tss_seg.prev_task_link = old_tss_sel;
2285 ret = ops->write_std(new_tss_base,
2286 &tss_seg.prev_task_link,
2287 sizeof tss_seg.prev_task_link,
2288 ctxt->vcpu, &ctxt->exception);
2289 if (ret != X86EMUL_CONTINUE)
2290 /* FIXME: need to provide precise fault address */
2294 return load_state_from_tss32(ctxt, ops, &tss_seg);
2297 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2298 struct x86_emulate_ops *ops,
2299 u16 tss_selector, int reason,
2300 bool has_error_code, u32 error_code)
2302 struct desc_struct curr_tss_desc, next_tss_desc;
2304 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2305 ulong old_tss_base =
2306 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2309 /* FIXME: old_tss_base == ~0 ? */
2311 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2312 if (ret != X86EMUL_CONTINUE)
2314 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2315 if (ret != X86EMUL_CONTINUE)
2318 /* FIXME: check that next_tss_desc is tss */
2320 if (reason != TASK_SWITCH_IRET) {
2321 if ((tss_selector & 3) > next_tss_desc.dpl ||
2322 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2323 return emulate_gp(ctxt, 0);
2326 desc_limit = desc_limit_scaled(&next_tss_desc);
2327 if (!next_tss_desc.p ||
2328 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2329 desc_limit < 0x2b)) {
2330 emulate_ts(ctxt, tss_selector & 0xfffc);
2331 return X86EMUL_PROPAGATE_FAULT;
2334 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2335 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2336 write_segment_descriptor(ctxt, ops, old_tss_sel,
2340 if (reason == TASK_SWITCH_IRET)
2341 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2343 /* set back link to prev task only if NT bit is set in eflags
2344 note that old_tss_sel is not used afetr this point */
2345 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2346 old_tss_sel = 0xffff;
2348 if (next_tss_desc.type & 8)
2349 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2350 old_tss_base, &next_tss_desc);
2352 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2353 old_tss_base, &next_tss_desc);
2354 if (ret != X86EMUL_CONTINUE)
2357 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2358 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2360 if (reason != TASK_SWITCH_IRET) {
2361 next_tss_desc.type |= (1 << 1); /* set busy flag */
2362 write_segment_descriptor(ctxt, ops, tss_selector,
2366 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2367 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2368 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2370 if (has_error_code) {
2371 struct decode_cache *c = &ctxt->decode;
2373 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2375 c->src.val = (unsigned long) error_code;
2376 ret = emulate_push(ctxt, ops);
2382 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2383 u16 tss_selector, int reason,
2384 bool has_error_code, u32 error_code)
2386 struct x86_emulate_ops *ops = ctxt->ops;
2387 struct decode_cache *c = &ctxt->decode;
2391 c->dst.type = OP_NONE;
2393 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2394 has_error_code, error_code);
2396 if (rc == X86EMUL_CONTINUE)
2399 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2402 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2403 int reg, struct operand *op)
2405 struct decode_cache *c = &ctxt->decode;
2406 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2408 register_address_increment(c, &c->regs[reg], df * op->bytes);
2409 op->addr.mem.ea = register_address(c, c->regs[reg]);
2410 op->addr.mem.seg = seg;
2413 static int em_push(struct x86_emulate_ctxt *ctxt)
2415 return emulate_push(ctxt, ctxt->ops);
2418 static int em_das(struct x86_emulate_ctxt *ctxt)
2420 struct decode_cache *c = &ctxt->decode;
2422 bool af, cf, old_cf;
2424 cf = ctxt->eflags & X86_EFLAGS_CF;
2430 af = ctxt->eflags & X86_EFLAGS_AF;
2431 if ((al & 0x0f) > 9 || af) {
2433 cf = old_cf | (al >= 250);
2438 if (old_al > 0x99 || old_cf) {
2444 /* Set PF, ZF, SF */
2445 c->src.type = OP_IMM;
2448 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2449 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2451 ctxt->eflags |= X86_EFLAGS_CF;
2453 ctxt->eflags |= X86_EFLAGS_AF;
2454 return X86EMUL_CONTINUE;
2457 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2459 struct decode_cache *c = &ctxt->decode;
2464 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2467 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2468 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2469 return X86EMUL_CONTINUE;
2472 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2474 c->src.val = old_cs;
2475 rc = emulate_push(ctxt, ctxt->ops);
2476 if (rc != X86EMUL_CONTINUE)
2479 c->src.val = old_eip;
2480 return emulate_push(ctxt, ctxt->ops);
2483 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2485 struct decode_cache *c = &ctxt->decode;
2488 c->dst.type = OP_REG;
2489 c->dst.addr.reg = &c->eip;
2490 c->dst.bytes = c->op_bytes;
2491 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2492 if (rc != X86EMUL_CONTINUE)
2494 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2495 return X86EMUL_CONTINUE;
2498 static int em_imul(struct x86_emulate_ctxt *ctxt)
2500 struct decode_cache *c = &ctxt->decode;
2502 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2503 return X86EMUL_CONTINUE;
2506 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2508 struct decode_cache *c = &ctxt->decode;
2510 c->dst.val = c->src2.val;
2511 return em_imul(ctxt);
2514 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2516 struct decode_cache *c = &ctxt->decode;
2518 c->dst.type = OP_REG;
2519 c->dst.bytes = c->src.bytes;
2520 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2521 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2523 return X86EMUL_CONTINUE;
2526 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2528 struct decode_cache *c = &ctxt->decode;
2531 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2532 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2533 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2534 return X86EMUL_CONTINUE;
2537 static int em_mov(struct x86_emulate_ctxt *ctxt)
2539 struct decode_cache *c = &ctxt->decode;
2540 c->dst.val = c->src.val;
2541 return X86EMUL_CONTINUE;
2544 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2546 struct decode_cache *c = &ctxt->decode;
2547 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2548 return X86EMUL_CONTINUE;
2551 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2553 struct decode_cache *c = &ctxt->decode;
2557 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2558 if (rc == X86EMUL_CONTINUE)
2559 emulate_invlpg(ctxt->vcpu, linear);
2560 /* Disable writeback. */
2561 c->dst.type = OP_NONE;
2562 return X86EMUL_CONTINUE;
2565 static bool valid_cr(int nr)
2577 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2579 struct decode_cache *c = &ctxt->decode;
2581 if (!valid_cr(c->modrm_reg))
2582 return emulate_ud(ctxt);
2584 return X86EMUL_CONTINUE;
2587 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2589 struct decode_cache *c = &ctxt->decode;
2590 u64 new_val = c->src.val64;
2591 int cr = c->modrm_reg;
2593 static u64 cr_reserved_bits[] = {
2594 0xffffffff00000000ULL,
2595 0, 0, 0, /* CR3 checked later */
2602 return emulate_ud(ctxt);
2604 if (new_val & cr_reserved_bits[cr])
2605 return emulate_gp(ctxt, 0);
2610 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2611 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2612 return emulate_gp(ctxt, 0);
2614 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2615 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2617 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2618 !(cr4 & X86_CR4_PAE))
2619 return emulate_gp(ctxt, 0);
2626 if (is_long_mode(ctxt->vcpu))
2627 rsvd = CR3_L_MODE_RESERVED_BITS;
2628 else if (is_pae(ctxt->vcpu))
2629 rsvd = CR3_PAE_RESERVED_BITS;
2630 else if (is_paging(ctxt->vcpu))
2631 rsvd = CR3_NONPAE_RESERVED_BITS;
2634 return emulate_gp(ctxt, 0);
2641 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2642 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2644 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2645 return emulate_gp(ctxt, 0);
2651 return X86EMUL_CONTINUE;
2654 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2658 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2660 /* Check if DR7.Global_Enable is set */
2661 return dr7 & (1 << 13);
2664 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2666 struct decode_cache *c = &ctxt->decode;
2667 int dr = c->modrm_reg;
2671 return emulate_ud(ctxt);
2673 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2674 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2675 return emulate_ud(ctxt);
2677 if (check_dr7_gd(ctxt))
2678 return emulate_db(ctxt);
2680 return X86EMUL_CONTINUE;
2683 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2685 struct decode_cache *c = &ctxt->decode;
2686 u64 new_val = c->src.val64;
2687 int dr = c->modrm_reg;
2689 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2690 return emulate_gp(ctxt, 0);
2692 return check_dr_read(ctxt);
2695 static int check_svme(struct x86_emulate_ctxt *ctxt)
2699 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2701 if (!(efer & EFER_SVME))
2702 return emulate_ud(ctxt);
2704 return X86EMUL_CONTINUE;
2707 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2709 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2711 /* Valid physical address? */
2712 if (rax & 0xffff000000000000)
2713 return emulate_gp(ctxt, 0);
2715 return check_svme(ctxt);
2718 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2720 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2722 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2723 return emulate_ud(ctxt);
2725 return X86EMUL_CONTINUE;
2728 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2730 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2731 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2733 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2735 return emulate_gp(ctxt, 0);
2737 return X86EMUL_CONTINUE;
2740 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2742 struct decode_cache *c = &ctxt->decode;
2744 c->dst.bytes = min(c->dst.bytes, 4u);
2745 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2746 return emulate_gp(ctxt, 0);
2748 return X86EMUL_CONTINUE;
2751 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2753 struct decode_cache *c = &ctxt->decode;
2755 c->src.bytes = min(c->src.bytes, 4u);
2756 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2757 return emulate_gp(ctxt, 0);
2759 return X86EMUL_CONTINUE;
2762 #define D(_y) { .flags = (_y) }
2763 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2764 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2765 .check_perm = (_p) }
2767 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2768 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2769 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2770 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2771 #define II(_f, _e, _i) \
2772 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2773 #define IIP(_f, _e, _i, _p) \
2774 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2775 .check_perm = (_p) }
2776 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2778 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2779 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2780 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2782 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2783 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2784 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2786 static struct opcode group7_rm1[] = {
2787 DI(SrcNone | ModRM | Priv, monitor),
2788 DI(SrcNone | ModRM | Priv, mwait),
2792 static struct opcode group7_rm3[] = {
2793 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2794 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2795 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2796 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2797 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2798 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2799 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2800 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2803 static struct opcode group7_rm7[] = {
2805 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2808 static struct opcode group1[] = {
2812 static struct opcode group1A[] = {
2813 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2816 static struct opcode group3[] = {
2817 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2818 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2819 X4(D(SrcMem | ModRM)),
2822 static struct opcode group4[] = {
2823 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2827 static struct opcode group5[] = {
2828 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2829 D(SrcMem | ModRM | Stack),
2830 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2831 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2832 D(SrcMem | ModRM | Stack), N,
2835 static struct opcode group6[] = {
2836 DI(ModRM | Prot, sldt),
2837 DI(ModRM | Prot, str),
2838 DI(ModRM | Prot | Priv, lldt),
2839 DI(ModRM | Prot | Priv, ltr),
2843 static struct group_dual group7 = { {
2844 DI(ModRM | Mov | DstMem | Priv, sgdt),
2845 DI(ModRM | Mov | DstMem | Priv, sidt),
2846 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2847 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2848 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2849 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2851 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2852 N, EXT(0, group7_rm3),
2853 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2854 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2857 static struct opcode group8[] = {
2859 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2860 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2863 static struct group_dual group9 = { {
2864 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2866 N, N, N, N, N, N, N, N,
2869 static struct opcode group11[] = {
2870 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2873 static struct gprefix pfx_0f_6f_0f_7f = {
2874 N, N, N, I(Sse, em_movdqu),
2877 static struct opcode opcode_table[256] = {
2880 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2883 D(ImplicitOps | Stack | No64), N,
2886 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2889 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2893 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2901 X8(I(SrcReg | Stack, em_push)),
2903 X8(D(DstReg | Stack)),
2905 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2906 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2909 I(SrcImm | Mov | Stack, em_push),
2910 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2911 I(SrcImmByte | Mov | Stack, em_push),
2912 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2913 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2914 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2918 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2919 G(DstMem | SrcImm | ModRM | Group, group1),
2920 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2921 G(DstMem | SrcImmByte | ModRM | Group, group1),
2922 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2924 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2925 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2926 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2927 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2929 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2931 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2932 I(SrcImmFAddr | No64, em_call_far), N,
2933 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2935 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2936 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2937 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2938 D2bv(SrcSI | DstDI | String),
2940 D2bv(DstAcc | SrcImm),
2941 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2942 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2943 D2bv(SrcAcc | DstDI | String),
2945 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2947 X8(I(DstReg | SrcImm | Mov, em_mov)),
2949 D2bv(DstMem | SrcImmByte | ModRM),
2950 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2951 D(ImplicitOps | Stack),
2952 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2953 G(ByteOp, group11), G(0, group11),
2955 N, N, N, D(ImplicitOps | Stack),
2956 D(ImplicitOps), DI(SrcImmByte, intn),
2957 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2959 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2962 N, N, N, N, N, N, N, N,
2965 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2966 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2968 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2969 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2970 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2971 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2973 N, DI(ImplicitOps, icebp), N, N,
2974 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2975 G(ByteOp, group3), G(0, group3),
2977 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2978 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2981 static struct opcode twobyte_table[256] = {
2983 G(0, group6), GD(0, &group7), N, N,
2984 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2985 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2986 N, D(ImplicitOps | ModRM), N, N,
2988 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2990 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2991 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2992 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2993 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
2995 N, N, N, N, N, N, N, N,
2997 DI(ImplicitOps | Priv, wrmsr),
2998 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
2999 DI(ImplicitOps | Priv, rdmsr),
3000 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3001 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3003 N, N, N, N, N, N, N, N,
3005 X16(D(DstReg | SrcMem | ModRM | Mov)),
3007 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3012 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3017 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3021 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3023 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3024 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3025 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3026 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3028 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3029 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3030 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3031 D(DstMem | SrcReg | Src2CL | ModRM),
3032 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3034 D2bv(DstMem | SrcReg | ModRM | Lock),
3035 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3036 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3037 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3040 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3041 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3042 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3044 D2bv(DstMem | SrcReg | ModRM | Lock),
3045 N, D(DstMem | SrcReg | ModRM | Mov),
3046 N, N, N, GD(0, &group9),
3047 N, N, N, N, N, N, N, N,
3049 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3051 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3053 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3069 static unsigned imm_size(struct decode_cache *c)
3073 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3079 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3080 unsigned size, bool sign_extension)
3082 struct decode_cache *c = &ctxt->decode;
3083 struct x86_emulate_ops *ops = ctxt->ops;
3084 int rc = X86EMUL_CONTINUE;
3088 op->addr.mem.ea = c->eip;
3089 /* NB. Immediates are sign-extended as necessary. */
3090 switch (op->bytes) {
3092 op->val = insn_fetch(s8, 1, c->eip);
3095 op->val = insn_fetch(s16, 2, c->eip);
3098 op->val = insn_fetch(s32, 4, c->eip);
3101 if (!sign_extension) {
3102 switch (op->bytes) {
3110 op->val &= 0xffffffff;
3119 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3121 struct x86_emulate_ops *ops = ctxt->ops;
3122 struct decode_cache *c = &ctxt->decode;
3123 int rc = X86EMUL_CONTINUE;
3124 int mode = ctxt->mode;
3125 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3126 bool op_prefix = false;
3127 struct opcode opcode, *g_mod012, *g_mod3;
3128 struct operand memop = { .type = OP_NONE };
3131 c->fetch.start = c->eip;
3132 c->fetch.end = c->fetch.start + insn_len;
3134 memcpy(c->fetch.data, insn, insn_len);
3135 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
3138 case X86EMUL_MODE_REAL:
3139 case X86EMUL_MODE_VM86:
3140 case X86EMUL_MODE_PROT16:
3141 def_op_bytes = def_ad_bytes = 2;
3143 case X86EMUL_MODE_PROT32:
3144 def_op_bytes = def_ad_bytes = 4;
3146 #ifdef CONFIG_X86_64
3147 case X86EMUL_MODE_PROT64:
3156 c->op_bytes = def_op_bytes;
3157 c->ad_bytes = def_ad_bytes;
3159 /* Legacy prefixes. */
3161 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3162 case 0x66: /* operand-size override */
3164 /* switch between 2/4 bytes */
3165 c->op_bytes = def_op_bytes ^ 6;
3167 case 0x67: /* address-size override */
3168 if (mode == X86EMUL_MODE_PROT64)
3169 /* switch between 4/8 bytes */
3170 c->ad_bytes = def_ad_bytes ^ 12;
3172 /* switch between 2/4 bytes */
3173 c->ad_bytes = def_ad_bytes ^ 6;
3175 case 0x26: /* ES override */
3176 case 0x2e: /* CS override */
3177 case 0x36: /* SS override */
3178 case 0x3e: /* DS override */
3179 set_seg_override(c, (c->b >> 3) & 3);
3181 case 0x64: /* FS override */
3182 case 0x65: /* GS override */
3183 set_seg_override(c, c->b & 7);
3185 case 0x40 ... 0x4f: /* REX */
3186 if (mode != X86EMUL_MODE_PROT64)
3188 c->rex_prefix = c->b;
3190 case 0xf0: /* LOCK */
3193 case 0xf2: /* REPNE/REPNZ */
3194 case 0xf3: /* REP/REPE/REPZ */
3195 c->rep_prefix = c->b;
3201 /* Any legacy prefix after a REX prefix nullifies its effect. */
3209 if (c->rex_prefix & 8)
3210 c->op_bytes = 8; /* REX.W */
3212 /* Opcode byte(s). */
3213 opcode = opcode_table[c->b];
3214 /* Two-byte opcode? */
3217 c->b = insn_fetch(u8, 1, c->eip);
3218 opcode = twobyte_table[c->b];
3220 c->d = opcode.flags;
3223 dual = c->d & GroupDual;
3224 c->modrm = insn_fetch(u8, 1, c->eip);
3227 if (c->d & GroupDual) {
3228 g_mod012 = opcode.u.gdual->mod012;
3229 g_mod3 = opcode.u.gdual->mod3;
3231 g_mod012 = g_mod3 = opcode.u.group;
3233 c->d &= ~(Group | GroupDual);
3235 goffset = (c->modrm >> 3) & 7;
3237 if ((c->modrm >> 6) == 3)
3238 opcode = g_mod3[goffset];
3240 opcode = g_mod012[goffset];
3242 if (opcode.flags & RMExt) {
3243 goffset = c->modrm & 7;
3244 opcode = opcode.u.group[goffset];
3247 c->d |= opcode.flags;
3250 if (c->d & Prefix) {
3251 if (c->rep_prefix && op_prefix)
3252 return X86EMUL_UNHANDLEABLE;
3253 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3254 switch (simd_prefix) {
3255 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3256 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3257 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3258 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3260 c->d |= opcode.flags;
3263 c->execute = opcode.u.execute;
3264 c->check_perm = opcode.check_perm;
3265 c->intercept = opcode.intercept;
3268 if (c->d == 0 || (c->d & Undefined))
3271 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3274 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3277 if (c->d & Op3264) {
3278 if (mode == X86EMUL_MODE_PROT64)
3287 /* ModRM and SIB bytes. */
3289 rc = decode_modrm(ctxt, ops, &memop);
3290 if (!c->has_seg_override)
3291 set_seg_override(c, c->modrm_seg);
3292 } else if (c->d & MemAbs)
3293 rc = decode_abs(ctxt, ops, &memop);
3294 if (rc != X86EMUL_CONTINUE)
3297 if (!c->has_seg_override)
3298 set_seg_override(c, VCPU_SREG_DS);
3300 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3302 if (memop.type == OP_MEM && c->ad_bytes != 8)
3303 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3305 if (memop.type == OP_MEM && c->rip_relative)
3306 memop.addr.mem.ea += c->eip;
3309 * Decode and fetch the source operand: register, memory
3312 switch (c->d & SrcMask) {
3316 decode_register_operand(ctxt, &c->src, c, 0);
3325 memop.bytes = (c->d & ByteOp) ? 1 :
3331 rc = decode_imm(ctxt, &c->src, 2, false);
3334 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3337 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3340 rc = decode_imm(ctxt, &c->src, 1, true);
3343 rc = decode_imm(ctxt, &c->src, 1, false);
3346 c->src.type = OP_REG;
3347 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3348 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3349 fetch_register_operand(&c->src);
3356 c->src.type = OP_MEM;
3357 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3358 c->src.addr.mem.ea =
3359 register_address(c, c->regs[VCPU_REGS_RSI]);
3360 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3364 c->src.type = OP_IMM;
3365 c->src.addr.mem.ea = c->eip;
3366 c->src.bytes = c->op_bytes + 2;
3367 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3370 memop.bytes = c->op_bytes + 2;
3375 if (rc != X86EMUL_CONTINUE)
3379 * Decode and fetch the second source operand: register, memory
3382 switch (c->d & Src2Mask) {
3387 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3390 rc = decode_imm(ctxt, &c->src2, 1, true);
3397 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3401 if (rc != X86EMUL_CONTINUE)
3404 /* Decode and fetch the destination operand: register or memory. */
3405 switch (c->d & DstMask) {
3407 decode_register_operand(ctxt, &c->dst, c,
3408 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3411 c->dst.type = OP_IMM;
3412 c->dst.addr.mem.ea = c->eip;
3414 c->dst.val = insn_fetch(u8, 1, c->eip);
3419 if ((c->d & DstMask) == DstMem64)
3422 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3424 fetch_bit_operand(c);
3425 c->dst.orig_val = c->dst.val;
3428 c->dst.type = OP_REG;
3429 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3430 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3431 fetch_register_operand(&c->dst);
3432 c->dst.orig_val = c->dst.val;
3435 c->dst.type = OP_MEM;
3436 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3437 c->dst.addr.mem.ea =
3438 register_address(c, c->regs[VCPU_REGS_RDI]);
3439 c->dst.addr.mem.seg = VCPU_SREG_ES;
3443 /* Special instructions do their own operand decoding. */
3445 c->dst.type = OP_NONE; /* Disable writeback. */
3450 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3453 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3455 struct decode_cache *c = &ctxt->decode;
3457 /* The second termination condition only applies for REPE
3458 * and REPNE. Test if the repeat string operation prefix is
3459 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3460 * corresponding termination condition according to:
3461 * - if REPE/REPZ and ZF = 0 then done
3462 * - if REPNE/REPNZ and ZF = 1 then done
3464 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3465 (c->b == 0xae) || (c->b == 0xaf))
3466 && (((c->rep_prefix == REPE_PREFIX) &&
3467 ((ctxt->eflags & EFLG_ZF) == 0))
3468 || ((c->rep_prefix == REPNE_PREFIX) &&
3469 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3476 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3478 struct x86_emulate_ops *ops = ctxt->ops;
3480 struct decode_cache *c = &ctxt->decode;
3481 int rc = X86EMUL_CONTINUE;
3482 int saved_dst_type = c->dst.type;
3483 int irq; /* Used for int 3, int, and into */
3485 ctxt->decode.mem_read.pos = 0;
3487 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3488 rc = emulate_ud(ctxt);
3492 /* LOCK prefix is allowed only with some instructions */
3493 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3494 rc = emulate_ud(ctxt);
3498 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3499 rc = emulate_ud(ctxt);
3504 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3505 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3506 rc = emulate_ud(ctxt);
3510 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3511 rc = emulate_nm(ctxt);
3515 if (unlikely(ctxt->guest_mode) && c->intercept) {
3516 rc = emulator_check_intercept(ctxt, c->intercept,
3517 X86_ICPT_PRE_EXCEPT);
3518 if (rc != X86EMUL_CONTINUE)
3522 /* Privileged instruction can be executed only in CPL=0 */
3523 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3524 rc = emulate_gp(ctxt, 0);
3528 /* Instruction can only be executed in protected mode */
3529 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3530 rc = emulate_ud(ctxt);
3534 /* Do instruction specific permission checks */
3535 if (c->check_perm) {
3536 rc = c->check_perm(ctxt);
3537 if (rc != X86EMUL_CONTINUE)
3541 if (unlikely(ctxt->guest_mode) && c->intercept) {
3542 rc = emulator_check_intercept(ctxt, c->intercept,
3543 X86_ICPT_POST_EXCEPT);
3544 if (rc != X86EMUL_CONTINUE)
3548 if (c->rep_prefix && (c->d & String)) {
3549 /* All REP prefixes have the same first termination condition */
3550 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3556 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3557 rc = segmented_read(ctxt, c->src.addr.mem,
3558 c->src.valptr, c->src.bytes);
3559 if (rc != X86EMUL_CONTINUE)
3561 c->src.orig_val64 = c->src.val64;
3564 if (c->src2.type == OP_MEM) {
3565 rc = segmented_read(ctxt, c->src2.addr.mem,
3566 &c->src2.val, c->src2.bytes);
3567 if (rc != X86EMUL_CONTINUE)
3571 if ((c->d & DstMask) == ImplicitOps)
3575 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3576 /* optimisation - avoid slow emulated read if Mov */
3577 rc = segmented_read(ctxt, c->dst.addr.mem,
3578 &c->dst.val, c->dst.bytes);
3579 if (rc != X86EMUL_CONTINUE)
3582 c->dst.orig_val = c->dst.val;
3586 if (unlikely(ctxt->guest_mode) && c->intercept) {
3587 rc = emulator_check_intercept(ctxt, c->intercept,
3588 X86_ICPT_POST_MEMACCESS);
3589 if (rc != X86EMUL_CONTINUE)
3594 rc = c->execute(ctxt);
3595 if (rc != X86EMUL_CONTINUE)
3606 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3608 case 0x06: /* push es */
3609 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3611 case 0x07: /* pop es */
3612 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3616 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3618 case 0x0e: /* push cs */
3619 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3623 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3625 case 0x16: /* push ss */
3626 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3628 case 0x17: /* pop ss */
3629 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3633 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3635 case 0x1e: /* push ds */
3636 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3638 case 0x1f: /* pop ds */
3639 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3643 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3647 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3651 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3655 c->dst.type = OP_NONE; /* Disable writeback. */
3656 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3658 case 0x40 ... 0x47: /* inc r16/r32 */
3659 emulate_1op("inc", c->dst, ctxt->eflags);
3661 case 0x48 ... 0x4f: /* dec r16/r32 */
3662 emulate_1op("dec", c->dst, ctxt->eflags);
3664 case 0x58 ... 0x5f: /* pop reg */
3666 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3668 case 0x60: /* pusha */
3669 rc = emulate_pusha(ctxt, ops);
3671 case 0x61: /* popa */
3672 rc = emulate_popa(ctxt, ops);
3674 case 0x63: /* movsxd */
3675 if (ctxt->mode != X86EMUL_MODE_PROT64)
3676 goto cannot_emulate;
3677 c->dst.val = (s32) c->src.val;
3679 case 0x6c: /* insb */
3680 case 0x6d: /* insw/insd */
3681 c->src.val = c->regs[VCPU_REGS_RDX];
3683 case 0x6e: /* outsb */
3684 case 0x6f: /* outsw/outsd */
3685 c->dst.val = c->regs[VCPU_REGS_RDX];
3688 case 0x70 ... 0x7f: /* jcc (short) */
3689 if (test_cc(c->b, ctxt->eflags))
3690 jmp_rel(c, c->src.val);
3692 case 0x80 ... 0x83: /* Grp1 */
3693 switch (c->modrm_reg) {
3714 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3716 case 0x86 ... 0x87: /* xchg */
3718 /* Write back the register source. */
3719 c->src.val = c->dst.val;
3720 write_register_operand(&c->src);
3722 * Write back the memory destination with implicit LOCK
3725 c->dst.val = c->src.orig_val;
3728 case 0x8c: /* mov r/m, sreg */
3729 if (c->modrm_reg > VCPU_SREG_GS) {
3730 rc = emulate_ud(ctxt);
3733 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3735 case 0x8d: /* lea r16/r32, m */
3736 c->dst.val = c->src.addr.mem.ea;
3738 case 0x8e: { /* mov seg, r/m16 */
3743 if (c->modrm_reg == VCPU_SREG_CS ||
3744 c->modrm_reg > VCPU_SREG_GS) {
3745 rc = emulate_ud(ctxt);
3749 if (c->modrm_reg == VCPU_SREG_SS)
3750 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3752 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3754 c->dst.type = OP_NONE; /* Disable writeback. */
3757 case 0x8f: /* pop (sole member of Grp1a) */
3758 rc = emulate_grp1a(ctxt, ops);
3760 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3761 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3764 case 0x98: /* cbw/cwde/cdqe */
3765 switch (c->op_bytes) {
3766 case 2: c->dst.val = (s8)c->dst.val; break;
3767 case 4: c->dst.val = (s16)c->dst.val; break;
3768 case 8: c->dst.val = (s32)c->dst.val; break;
3771 case 0x9c: /* pushf */
3772 c->src.val = (unsigned long) ctxt->eflags;
3773 rc = emulate_push(ctxt, ops);
3775 case 0x9d: /* popf */
3776 c->dst.type = OP_REG;
3777 c->dst.addr.reg = &ctxt->eflags;
3778 c->dst.bytes = c->op_bytes;
3779 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3781 case 0xa6 ... 0xa7: /* cmps */
3783 case 0xa8 ... 0xa9: /* test ax, imm */
3785 case 0xae ... 0xaf: /* scas */
3790 case 0xc3: /* ret */
3791 c->dst.type = OP_REG;
3792 c->dst.addr.reg = &c->eip;
3793 c->dst.bytes = c->op_bytes;
3794 goto pop_instruction;
3795 case 0xc4: /* les */
3796 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3798 case 0xc5: /* lds */
3799 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3801 case 0xcb: /* ret far */
3802 rc = emulate_ret_far(ctxt, ops);
3804 case 0xcc: /* int3 */
3807 case 0xcd: /* int n */
3810 rc = emulate_int(ctxt, ops, irq);
3812 case 0xce: /* into */
3813 if (ctxt->eflags & EFLG_OF) {
3818 case 0xcf: /* iret */
3819 rc = emulate_iret(ctxt, ops);
3821 case 0xd0 ... 0xd1: /* Grp2 */
3824 case 0xd2 ... 0xd3: /* Grp2 */
3825 c->src.val = c->regs[VCPU_REGS_RCX];
3828 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3829 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3830 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3831 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3832 jmp_rel(c, c->src.val);
3834 case 0xe3: /* jcxz/jecxz/jrcxz */
3835 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3836 jmp_rel(c, c->src.val);
3838 case 0xe4: /* inb */
3841 case 0xe6: /* outb */
3842 case 0xe7: /* out */
3844 case 0xe8: /* call (near) */ {
3845 long int rel = c->src.val;
3846 c->src.val = (unsigned long) c->eip;
3848 rc = emulate_push(ctxt, ops);
3851 case 0xe9: /* jmp rel */
3853 case 0xea: { /* jmp far */
3856 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3858 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3862 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3866 jmp: /* jmp rel short */
3867 jmp_rel(c, c->src.val);
3868 c->dst.type = OP_NONE; /* Disable writeback. */
3870 case 0xec: /* in al,dx */
3871 case 0xed: /* in (e/r)ax,dx */
3872 c->src.val = c->regs[VCPU_REGS_RDX];
3874 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3876 goto done; /* IO is needed */
3878 case 0xee: /* out dx,al */
3879 case 0xef: /* out dx,(e/r)ax */
3880 c->dst.val = c->regs[VCPU_REGS_RDX];
3882 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3883 &c->src.val, 1, ctxt->vcpu);
3884 c->dst.type = OP_NONE; /* Disable writeback. */
3886 case 0xf4: /* hlt */
3887 ctxt->vcpu->arch.halt_request = 1;
3889 case 0xf5: /* cmc */
3890 /* complement carry flag from eflags reg */
3891 ctxt->eflags ^= EFLG_CF;
3893 case 0xf6 ... 0xf7: /* Grp3 */
3894 rc = emulate_grp3(ctxt, ops);
3896 case 0xf8: /* clc */
3897 ctxt->eflags &= ~EFLG_CF;
3899 case 0xf9: /* stc */
3900 ctxt->eflags |= EFLG_CF;
3902 case 0xfa: /* cli */
3903 if (emulator_bad_iopl(ctxt, ops)) {
3904 rc = emulate_gp(ctxt, 0);
3907 ctxt->eflags &= ~X86_EFLAGS_IF;
3909 case 0xfb: /* sti */
3910 if (emulator_bad_iopl(ctxt, ops)) {
3911 rc = emulate_gp(ctxt, 0);
3914 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3915 ctxt->eflags |= X86_EFLAGS_IF;
3918 case 0xfc: /* cld */
3919 ctxt->eflags &= ~EFLG_DF;
3921 case 0xfd: /* std */
3922 ctxt->eflags |= EFLG_DF;
3924 case 0xfe: /* Grp4 */
3926 rc = emulate_grp45(ctxt, ops);
3928 case 0xff: /* Grp5 */
3929 if (c->modrm_reg == 5)
3933 goto cannot_emulate;
3936 if (rc != X86EMUL_CONTINUE)
3940 rc = writeback(ctxt, ops);
3941 if (rc != X86EMUL_CONTINUE)
3945 * restore dst type in case the decoding will be reused
3946 * (happens for string instruction )
3948 c->dst.type = saved_dst_type;
3950 if ((c->d & SrcMask) == SrcSI)
3951 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3952 VCPU_REGS_RSI, &c->src);
3954 if ((c->d & DstMask) == DstDI)
3955 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3958 if (c->rep_prefix && (c->d & String)) {
3959 struct read_cache *r = &ctxt->decode.io_read;
3960 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3962 if (!string_insn_completed(ctxt)) {
3964 * Re-enter guest when pio read ahead buffer is empty
3965 * or, if it is not used, after each 1024 iteration.
3967 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3968 (r->end == 0 || r->end != r->pos)) {
3970 * Reset read cache. Usually happens before
3971 * decode, but since instruction is restarted
3972 * we have to do it here.
3974 ctxt->decode.mem_read.end = 0;
3975 return EMULATION_RESTART;
3977 goto done; /* skip rip writeback */
3984 if (rc == X86EMUL_PROPAGATE_FAULT)
3985 ctxt->have_exception = true;
3986 if (rc == X86EMUL_INTERCEPTED)
3987 return EMULATION_INTERCEPTED;
3989 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3993 case 0x01: /* lgdt, lidt, lmsw */
3994 switch (c->modrm_reg) {
3996 unsigned long address;
3998 case 0: /* vmcall */
3999 if (c->modrm_mod != 3 || c->modrm_rm != 1)
4000 goto cannot_emulate;
4002 rc = kvm_fix_hypercall(ctxt->vcpu);
4003 if (rc != X86EMUL_CONTINUE)
4006 /* Let the processor re-execute the fixed hypercall */
4008 /* Disable writeback. */
4009 c->dst.type = OP_NONE;
4012 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4013 &size, &address, c->op_bytes);
4014 if (rc != X86EMUL_CONTINUE)
4016 realmode_lgdt(ctxt->vcpu, size, address);
4017 /* Disable writeback. */
4018 c->dst.type = OP_NONE;
4020 case 3: /* lidt/vmmcall */
4021 if (c->modrm_mod == 3) {
4022 switch (c->modrm_rm) {
4024 rc = kvm_fix_hypercall(ctxt->vcpu);
4027 goto cannot_emulate;
4030 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4033 if (rc != X86EMUL_CONTINUE)
4035 realmode_lidt(ctxt->vcpu, size, address);
4037 /* Disable writeback. */
4038 c->dst.type = OP_NONE;
4042 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4045 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4046 (c->src.val & 0x0f), ctxt->vcpu);
4047 c->dst.type = OP_NONE;
4049 case 5: /* not defined */
4051 rc = X86EMUL_PROPAGATE_FAULT;
4054 rc = em_invlpg(ctxt);
4057 goto cannot_emulate;
4060 case 0x05: /* syscall */
4061 rc = emulate_syscall(ctxt, ops);
4064 emulate_clts(ctxt->vcpu);
4066 case 0x09: /* wbinvd */
4067 kvm_emulate_wbinvd(ctxt->vcpu);
4069 case 0x08: /* invd */
4070 case 0x0d: /* GrpP (prefetch) */
4071 case 0x18: /* Grp16 (prefetch/nop) */
4073 case 0x20: /* mov cr, reg */
4074 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4076 case 0x21: /* mov from dr to reg */
4077 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4079 case 0x22: /* mov reg, cr */
4080 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4081 emulate_gp(ctxt, 0);
4082 rc = X86EMUL_PROPAGATE_FAULT;
4085 c->dst.type = OP_NONE;
4087 case 0x23: /* mov from reg to dr */
4088 if (ops->set_dr(c->modrm_reg, c->src.val &
4089 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4090 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4091 /* #UD condition is already handled by the code above */
4092 emulate_gp(ctxt, 0);
4093 rc = X86EMUL_PROPAGATE_FAULT;
4097 c->dst.type = OP_NONE; /* no writeback */
4101 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4102 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4103 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4104 emulate_gp(ctxt, 0);
4105 rc = X86EMUL_PROPAGATE_FAULT;
4108 rc = X86EMUL_CONTINUE;
4112 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4113 emulate_gp(ctxt, 0);
4114 rc = X86EMUL_PROPAGATE_FAULT;
4117 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4118 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4120 rc = X86EMUL_CONTINUE;
4122 case 0x34: /* sysenter */
4123 rc = emulate_sysenter(ctxt, ops);
4125 case 0x35: /* sysexit */
4126 rc = emulate_sysexit(ctxt, ops);
4128 case 0x40 ... 0x4f: /* cmov */
4129 c->dst.val = c->dst.orig_val = c->src.val;
4130 if (!test_cc(c->b, ctxt->eflags))
4131 c->dst.type = OP_NONE; /* no writeback */
4133 case 0x80 ... 0x8f: /* jnz rel, etc*/
4134 if (test_cc(c->b, ctxt->eflags))
4135 jmp_rel(c, c->src.val);
4137 case 0x90 ... 0x9f: /* setcc r/m8 */
4138 c->dst.val = test_cc(c->b, ctxt->eflags);
4140 case 0xa0: /* push fs */
4141 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4143 case 0xa1: /* pop fs */
4144 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4148 c->dst.type = OP_NONE;
4149 /* only subword offset */
4150 c->src.val &= (c->dst.bytes << 3) - 1;
4151 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4153 case 0xa4: /* shld imm8, r, r/m */
4154 case 0xa5: /* shld cl, r, r/m */
4155 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4157 case 0xa8: /* push gs */
4158 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4160 case 0xa9: /* pop gs */
4161 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4165 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4167 case 0xac: /* shrd imm8, r, r/m */
4168 case 0xad: /* shrd cl, r, r/m */
4169 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4171 case 0xae: /* clflush */
4173 case 0xb0 ... 0xb1: /* cmpxchg */
4175 * Save real source value, then compare EAX against
4178 c->src.orig_val = c->src.val;
4179 c->src.val = c->regs[VCPU_REGS_RAX];
4180 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4181 if (ctxt->eflags & EFLG_ZF) {
4182 /* Success: write back to memory. */
4183 c->dst.val = c->src.orig_val;
4185 /* Failure: write the value we saw to EAX. */
4186 c->dst.type = OP_REG;
4187 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4190 case 0xb2: /* lss */
4191 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4195 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4197 case 0xb4: /* lfs */
4198 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4200 case 0xb5: /* lgs */
4201 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4203 case 0xb6 ... 0xb7: /* movzx */
4204 c->dst.bytes = c->op_bytes;
4205 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4208 case 0xba: /* Grp8 */
4209 switch (c->modrm_reg & 3) {
4222 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4224 case 0xbc: { /* bsf */
4226 __asm__ ("bsf %2, %0; setz %1"
4227 : "=r"(c->dst.val), "=q"(zf)
4229 ctxt->eflags &= ~X86_EFLAGS_ZF;
4231 ctxt->eflags |= X86_EFLAGS_ZF;
4232 c->dst.type = OP_NONE; /* Disable writeback. */
4236 case 0xbd: { /* bsr */
4238 __asm__ ("bsr %2, %0; setz %1"
4239 : "=r"(c->dst.val), "=q"(zf)
4241 ctxt->eflags &= ~X86_EFLAGS_ZF;
4243 ctxt->eflags |= X86_EFLAGS_ZF;
4244 c->dst.type = OP_NONE; /* Disable writeback. */
4248 case 0xbe ... 0xbf: /* movsx */
4249 c->dst.bytes = c->op_bytes;
4250 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4253 case 0xc0 ... 0xc1: /* xadd */
4254 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4255 /* Write back the register source. */
4256 c->src.val = c->dst.orig_val;
4257 write_register_operand(&c->src);
4259 case 0xc3: /* movnti */
4260 c->dst.bytes = c->op_bytes;
4261 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4264 case 0xc7: /* Grp9 (cmpxchg8b) */
4265 rc = emulate_grp9(ctxt, ops);
4268 goto cannot_emulate;
4271 if (rc != X86EMUL_CONTINUE)
4277 return EMULATION_FAILED;