1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static void set_seg_override(struct decode_cache *c, int seg)
469 c->has_seg_override = true;
470 c->seg_override = seg;
473 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
474 struct x86_emulate_ops *ops, int seg)
476 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
479 return ops->get_cached_segment_base(seg, ctxt->vcpu);
482 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
483 struct x86_emulate_ops *ops,
484 struct decode_cache *c)
486 if (!c->has_seg_override)
489 return c->seg_override;
492 static ulong linear(struct x86_emulate_ctxt *ctxt,
493 struct segmented_address addr)
495 struct decode_cache *c = &ctxt->decode;
498 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
499 if (c->ad_bytes != 8)
504 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
505 u32 error, bool valid)
507 ctxt->exception.vector = vec;
508 ctxt->exception.error_code = error;
509 ctxt->exception.error_code_valid = valid;
510 return X86EMUL_PROPAGATE_FAULT;
513 static int emulate_db(struct x86_emulate_ctxt *ctxt)
515 return emulate_exception(ctxt, DB_VECTOR, 0, false);
518 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, GP_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
548 return ctxt->ops->read_std(linear(ctxt, addr), data, size, ctxt->vcpu,
552 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
553 struct x86_emulate_ops *ops,
554 unsigned long eip, u8 *dest)
556 struct fetch_cache *fc = &ctxt->decode.fetch;
560 if (eip == fc->end) {
561 cur_size = fc->end - fc->start;
562 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
563 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
564 size, ctxt->vcpu, &ctxt->exception);
565 if (rc != X86EMUL_CONTINUE)
569 *dest = fc->data[eip - fc->start];
570 return X86EMUL_CONTINUE;
573 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
574 struct x86_emulate_ops *ops,
575 unsigned long eip, void *dest, unsigned size)
579 /* x86 instructions are limited to 15 bytes. */
580 if (eip + size - ctxt->eip > 15)
581 return X86EMUL_UNHANDLEABLE;
583 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
584 if (rc != X86EMUL_CONTINUE)
587 return X86EMUL_CONTINUE;
591 * Given the 'reg' portion of a ModRM byte, and a register block, return a
592 * pointer into the block that addresses the relevant register.
593 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
595 static void *decode_register(u8 modrm_reg, unsigned long *regs,
600 p = ®s[modrm_reg];
601 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
602 p = (unsigned char *)®s[modrm_reg & 3] + 1;
606 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
607 struct x86_emulate_ops *ops,
608 struct segmented_address addr,
609 u16 *size, unsigned long *address, int op_bytes)
616 rc = segmented_read_std(ctxt, addr, size, 2);
617 if (rc != X86EMUL_CONTINUE)
620 rc = segmented_read_std(ctxt, addr, address, op_bytes);
624 static int test_cc(unsigned int condition, unsigned int flags)
628 switch ((condition & 15) >> 1) {
630 rc |= (flags & EFLG_OF);
632 case 1: /* b/c/nae */
633 rc |= (flags & EFLG_CF);
636 rc |= (flags & EFLG_ZF);
639 rc |= (flags & (EFLG_CF|EFLG_ZF));
642 rc |= (flags & EFLG_SF);
645 rc |= (flags & EFLG_PF);
648 rc |= (flags & EFLG_ZF);
651 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
655 /* Odd condition identifiers (lsb == 1) have inverted sense. */
656 return (!!rc ^ (condition & 1));
659 static void fetch_register_operand(struct operand *op)
663 op->val = *(u8 *)op->addr.reg;
666 op->val = *(u16 *)op->addr.reg;
669 op->val = *(u32 *)op->addr.reg;
672 op->val = *(u64 *)op->addr.reg;
677 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
679 ctxt->ops->get_fpu(ctxt);
681 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
682 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
683 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
684 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
685 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
686 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
687 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
688 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
690 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
691 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
692 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
693 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
694 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
695 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
696 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
697 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
701 ctxt->ops->put_fpu(ctxt);
704 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
707 ctxt->ops->get_fpu(ctxt);
709 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
710 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
711 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
712 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
713 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
714 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
715 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
716 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
718 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
719 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
720 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
721 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
722 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
723 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
724 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
725 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
729 ctxt->ops->put_fpu(ctxt);
732 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
734 struct decode_cache *c,
737 unsigned reg = c->modrm_reg;
738 int highbyte_regs = c->rex_prefix == 0;
741 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
747 read_sse_reg(ctxt, &op->vec_val, reg);
752 if ((c->d & ByteOp) && !inhibit_bytereg) {
753 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
756 op->addr.reg = decode_register(reg, c->regs, 0);
757 op->bytes = c->op_bytes;
759 fetch_register_operand(op);
760 op->orig_val = op->val;
763 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
764 struct x86_emulate_ops *ops,
767 struct decode_cache *c = &ctxt->decode;
769 int index_reg = 0, base_reg = 0, scale;
770 int rc = X86EMUL_CONTINUE;
774 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
775 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
776 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
779 c->modrm = insn_fetch(u8, 1, c->eip);
780 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
781 c->modrm_reg |= (c->modrm & 0x38) >> 3;
782 c->modrm_rm |= (c->modrm & 0x07);
783 c->modrm_seg = VCPU_SREG_DS;
785 if (c->modrm_mod == 3) {
787 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
788 op->addr.reg = decode_register(c->modrm_rm,
789 c->regs, c->d & ByteOp);
793 op->addr.xmm = c->modrm_rm;
794 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
797 fetch_register_operand(op);
803 if (c->ad_bytes == 2) {
804 unsigned bx = c->regs[VCPU_REGS_RBX];
805 unsigned bp = c->regs[VCPU_REGS_RBP];
806 unsigned si = c->regs[VCPU_REGS_RSI];
807 unsigned di = c->regs[VCPU_REGS_RDI];
809 /* 16-bit ModR/M decode. */
810 switch (c->modrm_mod) {
812 if (c->modrm_rm == 6)
813 modrm_ea += insn_fetch(u16, 2, c->eip);
816 modrm_ea += insn_fetch(s8, 1, c->eip);
819 modrm_ea += insn_fetch(u16, 2, c->eip);
822 switch (c->modrm_rm) {
842 if (c->modrm_mod != 0)
849 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
850 (c->modrm_rm == 6 && c->modrm_mod != 0))
851 c->modrm_seg = VCPU_SREG_SS;
852 modrm_ea = (u16)modrm_ea;
854 /* 32/64-bit ModR/M decode. */
855 if ((c->modrm_rm & 7) == 4) {
856 sib = insn_fetch(u8, 1, c->eip);
857 index_reg |= (sib >> 3) & 7;
861 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
862 modrm_ea += insn_fetch(s32, 4, c->eip);
864 modrm_ea += c->regs[base_reg];
866 modrm_ea += c->regs[index_reg] << scale;
867 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
868 if (ctxt->mode == X86EMUL_MODE_PROT64)
871 modrm_ea += c->regs[c->modrm_rm];
872 switch (c->modrm_mod) {
874 if (c->modrm_rm == 5)
875 modrm_ea += insn_fetch(s32, 4, c->eip);
878 modrm_ea += insn_fetch(s8, 1, c->eip);
881 modrm_ea += insn_fetch(s32, 4, c->eip);
885 op->addr.mem.ea = modrm_ea;
890 static int decode_abs(struct x86_emulate_ctxt *ctxt,
891 struct x86_emulate_ops *ops,
894 struct decode_cache *c = &ctxt->decode;
895 int rc = X86EMUL_CONTINUE;
898 switch (c->ad_bytes) {
900 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
903 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
906 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
913 static void fetch_bit_operand(struct decode_cache *c)
917 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
918 mask = ~(c->dst.bytes * 8 - 1);
920 if (c->src.bytes == 2)
921 sv = (s16)c->src.val & (s16)mask;
922 else if (c->src.bytes == 4)
923 sv = (s32)c->src.val & (s32)mask;
925 c->dst.addr.mem.ea += (sv >> 3);
928 /* only subword offset */
929 c->src.val &= (c->dst.bytes << 3) - 1;
932 static int read_emulated(struct x86_emulate_ctxt *ctxt,
933 struct x86_emulate_ops *ops,
934 unsigned long addr, void *dest, unsigned size)
937 struct read_cache *mc = &ctxt->decode.mem_read;
940 int n = min(size, 8u);
942 if (mc->pos < mc->end)
945 rc = ops->read_emulated(addr, mc->data + mc->end, n,
946 &ctxt->exception, ctxt->vcpu);
947 if (rc != X86EMUL_CONTINUE)
952 memcpy(dest, mc->data + mc->pos, n);
957 return X86EMUL_CONTINUE;
960 static int segmented_read(struct x86_emulate_ctxt *ctxt,
961 struct segmented_address addr,
965 return read_emulated(ctxt, ctxt->ops, linear(ctxt, addr), data, size);
968 static int segmented_write(struct x86_emulate_ctxt *ctxt,
969 struct segmented_address addr,
973 return ctxt->ops->write_emulated(linear(ctxt, addr), data, size,
974 &ctxt->exception, ctxt->vcpu);
977 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
978 struct segmented_address addr,
979 const void *orig_data, const void *data,
982 return ctxt->ops->cmpxchg_emulated(linear(ctxt, addr), orig_data, data,
983 size, &ctxt->exception, ctxt->vcpu);
986 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
987 struct x86_emulate_ops *ops,
988 unsigned int size, unsigned short port,
991 struct read_cache *rc = &ctxt->decode.io_read;
993 if (rc->pos == rc->end) { /* refill pio read ahead */
994 struct decode_cache *c = &ctxt->decode;
995 unsigned int in_page, n;
996 unsigned int count = c->rep_prefix ?
997 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
998 in_page = (ctxt->eflags & EFLG_DF) ?
999 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1000 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1001 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1005 rc->pos = rc->end = 0;
1006 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1011 memcpy(dest, rc->data + rc->pos, size);
1016 static u32 desc_limit_scaled(struct desc_struct *desc)
1018 u32 limit = get_desc_limit(desc);
1020 return desc->g ? (limit << 12) | 0xfff : limit;
1023 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1024 struct x86_emulate_ops *ops,
1025 u16 selector, struct desc_ptr *dt)
1027 if (selector & 1 << 2) {
1028 struct desc_struct desc;
1029 memset (dt, 0, sizeof *dt);
1030 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1034 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1035 dt->address = get_desc_base(&desc);
1037 ops->get_gdt(dt, ctxt->vcpu);
1040 /* allowed just for 8 bytes segments */
1041 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1042 struct x86_emulate_ops *ops,
1043 u16 selector, struct desc_struct *desc)
1046 u16 index = selector >> 3;
1050 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1052 if (dt.size < index * 8 + 7)
1053 return emulate_gp(ctxt, selector & 0xfffc);
1054 addr = dt.address + index * 8;
1055 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1061 /* allowed just for 8 bytes segments */
1062 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1063 struct x86_emulate_ops *ops,
1064 u16 selector, struct desc_struct *desc)
1067 u16 index = selector >> 3;
1071 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1073 if (dt.size < index * 8 + 7)
1074 return emulate_gp(ctxt, selector & 0xfffc);
1076 addr = dt.address + index * 8;
1077 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1083 /* Does not support long mode */
1084 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1085 struct x86_emulate_ops *ops,
1086 u16 selector, int seg)
1088 struct desc_struct seg_desc;
1090 unsigned err_vec = GP_VECTOR;
1092 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1095 memset(&seg_desc, 0, sizeof seg_desc);
1097 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1098 || ctxt->mode == X86EMUL_MODE_REAL) {
1099 /* set real mode segment descriptor */
1100 set_desc_base(&seg_desc, selector << 4);
1101 set_desc_limit(&seg_desc, 0xffff);
1108 /* NULL selector is not valid for TR, CS and SS */
1109 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1113 /* TR should be in GDT only */
1114 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1117 if (null_selector) /* for NULL selector skip all following checks */
1120 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1121 if (ret != X86EMUL_CONTINUE)
1124 err_code = selector & 0xfffc;
1125 err_vec = GP_VECTOR;
1127 /* can't load system descriptor into segment selecor */
1128 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1132 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1138 cpl = ops->cpl(ctxt->vcpu);
1143 * segment is not a writable data segment or segment
1144 * selector's RPL != CPL or segment selector's RPL != CPL
1146 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1150 if (!(seg_desc.type & 8))
1153 if (seg_desc.type & 4) {
1159 if (rpl > cpl || dpl != cpl)
1162 /* CS(RPL) <- CPL */
1163 selector = (selector & 0xfffc) | cpl;
1166 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1169 case VCPU_SREG_LDTR:
1170 if (seg_desc.s || seg_desc.type != 2)
1173 default: /* DS, ES, FS, or GS */
1175 * segment is not a data or readable code segment or
1176 * ((segment is a data or nonconforming code segment)
1177 * and (both RPL and CPL > DPL))
1179 if ((seg_desc.type & 0xa) == 0x8 ||
1180 (((seg_desc.type & 0xc) != 0xc) &&
1181 (rpl > dpl && cpl > dpl)))
1187 /* mark segment as accessed */
1189 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1190 if (ret != X86EMUL_CONTINUE)
1194 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1195 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1196 return X86EMUL_CONTINUE;
1198 emulate_exception(ctxt, err_vec, err_code, true);
1199 return X86EMUL_PROPAGATE_FAULT;
1202 static void write_register_operand(struct operand *op)
1204 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1205 switch (op->bytes) {
1207 *(u8 *)op->addr.reg = (u8)op->val;
1210 *(u16 *)op->addr.reg = (u16)op->val;
1213 *op->addr.reg = (u32)op->val;
1214 break; /* 64b: zero-extend */
1216 *op->addr.reg = op->val;
1221 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1222 struct x86_emulate_ops *ops)
1225 struct decode_cache *c = &ctxt->decode;
1227 switch (c->dst.type) {
1229 write_register_operand(&c->dst);
1233 rc = segmented_cmpxchg(ctxt,
1239 rc = segmented_write(ctxt,
1243 if (rc != X86EMUL_CONTINUE)
1247 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1255 return X86EMUL_CONTINUE;
1258 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1259 struct x86_emulate_ops *ops)
1261 struct decode_cache *c = &ctxt->decode;
1263 c->dst.type = OP_MEM;
1264 c->dst.bytes = c->op_bytes;
1265 c->dst.val = c->src.val;
1266 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1267 c->dst.addr.mem.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1268 c->dst.addr.mem.seg = VCPU_SREG_SS;
1271 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1272 struct x86_emulate_ops *ops,
1273 void *dest, int len)
1275 struct decode_cache *c = &ctxt->decode;
1277 struct segmented_address addr;
1279 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1280 addr.seg = VCPU_SREG_SS;
1281 rc = segmented_read(ctxt, addr, dest, len);
1282 if (rc != X86EMUL_CONTINUE)
1285 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1289 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1290 struct x86_emulate_ops *ops,
1291 void *dest, int len)
1294 unsigned long val, change_mask;
1295 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1296 int cpl = ops->cpl(ctxt->vcpu);
1298 rc = emulate_pop(ctxt, ops, &val, len);
1299 if (rc != X86EMUL_CONTINUE)
1302 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1303 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1305 switch(ctxt->mode) {
1306 case X86EMUL_MODE_PROT64:
1307 case X86EMUL_MODE_PROT32:
1308 case X86EMUL_MODE_PROT16:
1310 change_mask |= EFLG_IOPL;
1312 change_mask |= EFLG_IF;
1314 case X86EMUL_MODE_VM86:
1316 return emulate_gp(ctxt, 0);
1317 change_mask |= EFLG_IF;
1319 default: /* real mode */
1320 change_mask |= (EFLG_IOPL | EFLG_IF);
1324 *(unsigned long *)dest =
1325 (ctxt->eflags & ~change_mask) | (val & change_mask);
1330 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1331 struct x86_emulate_ops *ops, int seg)
1333 struct decode_cache *c = &ctxt->decode;
1335 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1337 emulate_push(ctxt, ops);
1340 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1341 struct x86_emulate_ops *ops, int seg)
1343 struct decode_cache *c = &ctxt->decode;
1344 unsigned long selector;
1347 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1348 if (rc != X86EMUL_CONTINUE)
1351 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1355 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1356 struct x86_emulate_ops *ops)
1358 struct decode_cache *c = &ctxt->decode;
1359 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1360 int rc = X86EMUL_CONTINUE;
1361 int reg = VCPU_REGS_RAX;
1363 while (reg <= VCPU_REGS_RDI) {
1364 (reg == VCPU_REGS_RSP) ?
1365 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1367 emulate_push(ctxt, ops);
1369 rc = writeback(ctxt, ops);
1370 if (rc != X86EMUL_CONTINUE)
1376 /* Disable writeback. */
1377 c->dst.type = OP_NONE;
1382 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1383 struct x86_emulate_ops *ops)
1385 struct decode_cache *c = &ctxt->decode;
1386 int rc = X86EMUL_CONTINUE;
1387 int reg = VCPU_REGS_RDI;
1389 while (reg >= VCPU_REGS_RAX) {
1390 if (reg == VCPU_REGS_RSP) {
1391 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1396 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1397 if (rc != X86EMUL_CONTINUE)
1404 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1405 struct x86_emulate_ops *ops, int irq)
1407 struct decode_cache *c = &ctxt->decode;
1414 /* TODO: Add limit checks */
1415 c->src.val = ctxt->eflags;
1416 emulate_push(ctxt, ops);
1417 rc = writeback(ctxt, ops);
1418 if (rc != X86EMUL_CONTINUE)
1421 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1423 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1424 emulate_push(ctxt, ops);
1425 rc = writeback(ctxt, ops);
1426 if (rc != X86EMUL_CONTINUE)
1429 c->src.val = c->eip;
1430 emulate_push(ctxt, ops);
1431 rc = writeback(ctxt, ops);
1432 if (rc != X86EMUL_CONTINUE)
1435 c->dst.type = OP_NONE;
1437 ops->get_idt(&dt, ctxt->vcpu);
1439 eip_addr = dt.address + (irq << 2);
1440 cs_addr = dt.address + (irq << 2) + 2;
1442 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1443 if (rc != X86EMUL_CONTINUE)
1446 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1447 if (rc != X86EMUL_CONTINUE)
1450 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1451 if (rc != X86EMUL_CONTINUE)
1459 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1460 struct x86_emulate_ops *ops, int irq)
1462 switch(ctxt->mode) {
1463 case X86EMUL_MODE_REAL:
1464 return emulate_int_real(ctxt, ops, irq);
1465 case X86EMUL_MODE_VM86:
1466 case X86EMUL_MODE_PROT16:
1467 case X86EMUL_MODE_PROT32:
1468 case X86EMUL_MODE_PROT64:
1470 /* Protected mode interrupts unimplemented yet */
1471 return X86EMUL_UNHANDLEABLE;
1475 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1476 struct x86_emulate_ops *ops)
1478 struct decode_cache *c = &ctxt->decode;
1479 int rc = X86EMUL_CONTINUE;
1480 unsigned long temp_eip = 0;
1481 unsigned long temp_eflags = 0;
1482 unsigned long cs = 0;
1483 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1484 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1485 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1486 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1488 /* TODO: Add stack limit check */
1490 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1492 if (rc != X86EMUL_CONTINUE)
1495 if (temp_eip & ~0xffff)
1496 return emulate_gp(ctxt, 0);
1498 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1500 if (rc != X86EMUL_CONTINUE)
1503 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1505 if (rc != X86EMUL_CONTINUE)
1508 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1510 if (rc != X86EMUL_CONTINUE)
1516 if (c->op_bytes == 4)
1517 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1518 else if (c->op_bytes == 2) {
1519 ctxt->eflags &= ~0xffff;
1520 ctxt->eflags |= temp_eflags;
1523 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1524 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1529 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1530 struct x86_emulate_ops* ops)
1532 switch(ctxt->mode) {
1533 case X86EMUL_MODE_REAL:
1534 return emulate_iret_real(ctxt, ops);
1535 case X86EMUL_MODE_VM86:
1536 case X86EMUL_MODE_PROT16:
1537 case X86EMUL_MODE_PROT32:
1538 case X86EMUL_MODE_PROT64:
1540 /* iret from protected mode unimplemented yet */
1541 return X86EMUL_UNHANDLEABLE;
1545 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1546 struct x86_emulate_ops *ops)
1548 struct decode_cache *c = &ctxt->decode;
1550 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1553 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1555 struct decode_cache *c = &ctxt->decode;
1556 switch (c->modrm_reg) {
1558 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1561 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1564 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1567 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1569 case 4: /* sal/shl */
1570 case 6: /* sal/shl */
1571 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1574 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1577 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1582 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1583 struct x86_emulate_ops *ops)
1585 struct decode_cache *c = &ctxt->decode;
1586 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1587 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1590 switch (c->modrm_reg) {
1591 case 0 ... 1: /* test */
1592 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1595 c->dst.val = ~c->dst.val;
1598 emulate_1op("neg", c->dst, ctxt->eflags);
1601 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1604 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1607 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1611 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1615 return X86EMUL_UNHANDLEABLE;
1618 return emulate_de(ctxt);
1619 return X86EMUL_CONTINUE;
1622 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1623 struct x86_emulate_ops *ops)
1625 struct decode_cache *c = &ctxt->decode;
1627 switch (c->modrm_reg) {
1629 emulate_1op("inc", c->dst, ctxt->eflags);
1632 emulate_1op("dec", c->dst, ctxt->eflags);
1634 case 2: /* call near abs */ {
1637 c->eip = c->src.val;
1638 c->src.val = old_eip;
1639 emulate_push(ctxt, ops);
1642 case 4: /* jmp abs */
1643 c->eip = c->src.val;
1646 emulate_push(ctxt, ops);
1649 return X86EMUL_CONTINUE;
1652 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1653 struct x86_emulate_ops *ops)
1655 struct decode_cache *c = &ctxt->decode;
1656 u64 old = c->dst.orig_val64;
1658 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1659 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1660 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1661 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1662 ctxt->eflags &= ~EFLG_ZF;
1664 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1665 (u32) c->regs[VCPU_REGS_RBX];
1667 ctxt->eflags |= EFLG_ZF;
1669 return X86EMUL_CONTINUE;
1672 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1673 struct x86_emulate_ops *ops)
1675 struct decode_cache *c = &ctxt->decode;
1679 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1680 if (rc != X86EMUL_CONTINUE)
1682 if (c->op_bytes == 4)
1683 c->eip = (u32)c->eip;
1684 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1685 if (rc != X86EMUL_CONTINUE)
1687 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1691 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1692 struct x86_emulate_ops *ops, int seg)
1694 struct decode_cache *c = &ctxt->decode;
1698 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1700 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1701 if (rc != X86EMUL_CONTINUE)
1704 c->dst.val = c->src.val;
1709 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1710 struct x86_emulate_ops *ops, struct desc_struct *cs,
1711 struct desc_struct *ss)
1713 memset(cs, 0, sizeof(struct desc_struct));
1714 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1715 memset(ss, 0, sizeof(struct desc_struct));
1717 cs->l = 0; /* will be adjusted later */
1718 set_desc_base(cs, 0); /* flat segment */
1719 cs->g = 1; /* 4kb granularity */
1720 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1721 cs->type = 0x0b; /* Read, Execute, Accessed */
1723 cs->dpl = 0; /* will be adjusted later */
1727 set_desc_base(ss, 0); /* flat segment */
1728 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1729 ss->g = 1; /* 4kb granularity */
1731 ss->type = 0x03; /* Read/Write, Accessed */
1732 ss->d = 1; /* 32bit stack segment */
1738 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1740 struct decode_cache *c = &ctxt->decode;
1741 struct desc_struct cs, ss;
1745 /* syscall is not available in real mode */
1746 if (ctxt->mode == X86EMUL_MODE_REAL ||
1747 ctxt->mode == X86EMUL_MODE_VM86)
1748 return emulate_ud(ctxt);
1750 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1752 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1754 cs_sel = (u16)(msr_data & 0xfffc);
1755 ss_sel = (u16)(msr_data + 8);
1757 if (is_long_mode(ctxt->vcpu)) {
1761 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1762 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1763 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1764 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1766 c->regs[VCPU_REGS_RCX] = c->eip;
1767 if (is_long_mode(ctxt->vcpu)) {
1768 #ifdef CONFIG_X86_64
1769 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1771 ops->get_msr(ctxt->vcpu,
1772 ctxt->mode == X86EMUL_MODE_PROT64 ?
1773 MSR_LSTAR : MSR_CSTAR, &msr_data);
1776 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1777 ctxt->eflags &= ~(msr_data | EFLG_RF);
1781 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1782 c->eip = (u32)msr_data;
1784 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1787 return X86EMUL_CONTINUE;
1791 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1793 struct decode_cache *c = &ctxt->decode;
1794 struct desc_struct cs, ss;
1798 /* inject #GP if in real mode */
1799 if (ctxt->mode == X86EMUL_MODE_REAL)
1800 return emulate_gp(ctxt, 0);
1802 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1803 * Therefore, we inject an #UD.
1805 if (ctxt->mode == X86EMUL_MODE_PROT64)
1806 return emulate_ud(ctxt);
1808 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1810 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1811 switch (ctxt->mode) {
1812 case X86EMUL_MODE_PROT32:
1813 if ((msr_data & 0xfffc) == 0x0)
1814 return emulate_gp(ctxt, 0);
1816 case X86EMUL_MODE_PROT64:
1817 if (msr_data == 0x0)
1818 return emulate_gp(ctxt, 0);
1822 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1823 cs_sel = (u16)msr_data;
1824 cs_sel &= ~SELECTOR_RPL_MASK;
1825 ss_sel = cs_sel + 8;
1826 ss_sel &= ~SELECTOR_RPL_MASK;
1827 if (ctxt->mode == X86EMUL_MODE_PROT64
1828 || is_long_mode(ctxt->vcpu)) {
1833 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1834 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1835 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1836 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1838 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1841 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1842 c->regs[VCPU_REGS_RSP] = msr_data;
1844 return X86EMUL_CONTINUE;
1848 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1850 struct decode_cache *c = &ctxt->decode;
1851 struct desc_struct cs, ss;
1856 /* inject #GP if in real mode or Virtual 8086 mode */
1857 if (ctxt->mode == X86EMUL_MODE_REAL ||
1858 ctxt->mode == X86EMUL_MODE_VM86)
1859 return emulate_gp(ctxt, 0);
1861 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1863 if ((c->rex_prefix & 0x8) != 0x0)
1864 usermode = X86EMUL_MODE_PROT64;
1866 usermode = X86EMUL_MODE_PROT32;
1870 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1872 case X86EMUL_MODE_PROT32:
1873 cs_sel = (u16)(msr_data + 16);
1874 if ((msr_data & 0xfffc) == 0x0)
1875 return emulate_gp(ctxt, 0);
1876 ss_sel = (u16)(msr_data + 24);
1878 case X86EMUL_MODE_PROT64:
1879 cs_sel = (u16)(msr_data + 32);
1880 if (msr_data == 0x0)
1881 return emulate_gp(ctxt, 0);
1882 ss_sel = cs_sel + 8;
1887 cs_sel |= SELECTOR_RPL_MASK;
1888 ss_sel |= SELECTOR_RPL_MASK;
1890 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1891 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1892 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1893 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1895 c->eip = c->regs[VCPU_REGS_RDX];
1896 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1898 return X86EMUL_CONTINUE;
1901 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1902 struct x86_emulate_ops *ops)
1905 if (ctxt->mode == X86EMUL_MODE_REAL)
1907 if (ctxt->mode == X86EMUL_MODE_VM86)
1909 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1910 return ops->cpl(ctxt->vcpu) > iopl;
1913 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1914 struct x86_emulate_ops *ops,
1917 struct desc_struct tr_seg;
1920 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
1921 unsigned mask = (1 << len) - 1;
1924 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
1927 if (desc_limit_scaled(&tr_seg) < 103)
1929 base = get_desc_base(&tr_seg);
1930 #ifdef CONFIG_X86_64
1931 base |= ((u64)base3) << 32;
1933 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
1934 if (r != X86EMUL_CONTINUE)
1936 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
1938 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
1940 if (r != X86EMUL_CONTINUE)
1942 if ((perm >> bit_idx) & mask)
1947 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1948 struct x86_emulate_ops *ops,
1954 if (emulator_bad_iopl(ctxt, ops))
1955 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1958 ctxt->perm_ok = true;
1963 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
1964 struct x86_emulate_ops *ops,
1965 struct tss_segment_16 *tss)
1967 struct decode_cache *c = &ctxt->decode;
1970 tss->flag = ctxt->eflags;
1971 tss->ax = c->regs[VCPU_REGS_RAX];
1972 tss->cx = c->regs[VCPU_REGS_RCX];
1973 tss->dx = c->regs[VCPU_REGS_RDX];
1974 tss->bx = c->regs[VCPU_REGS_RBX];
1975 tss->sp = c->regs[VCPU_REGS_RSP];
1976 tss->bp = c->regs[VCPU_REGS_RBP];
1977 tss->si = c->regs[VCPU_REGS_RSI];
1978 tss->di = c->regs[VCPU_REGS_RDI];
1980 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
1981 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1982 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
1983 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
1984 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
1987 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
1988 struct x86_emulate_ops *ops,
1989 struct tss_segment_16 *tss)
1991 struct decode_cache *c = &ctxt->decode;
1995 ctxt->eflags = tss->flag | 2;
1996 c->regs[VCPU_REGS_RAX] = tss->ax;
1997 c->regs[VCPU_REGS_RCX] = tss->cx;
1998 c->regs[VCPU_REGS_RDX] = tss->dx;
1999 c->regs[VCPU_REGS_RBX] = tss->bx;
2000 c->regs[VCPU_REGS_RSP] = tss->sp;
2001 c->regs[VCPU_REGS_RBP] = tss->bp;
2002 c->regs[VCPU_REGS_RSI] = tss->si;
2003 c->regs[VCPU_REGS_RDI] = tss->di;
2006 * SDM says that segment selectors are loaded before segment
2009 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2010 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2011 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2012 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2013 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2016 * Now load segment descriptors. If fault happenes at this stage
2017 * it is handled in a context of new task
2019 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2020 if (ret != X86EMUL_CONTINUE)
2022 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2023 if (ret != X86EMUL_CONTINUE)
2025 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2026 if (ret != X86EMUL_CONTINUE)
2028 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2029 if (ret != X86EMUL_CONTINUE)
2031 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2032 if (ret != X86EMUL_CONTINUE)
2035 return X86EMUL_CONTINUE;
2038 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2039 struct x86_emulate_ops *ops,
2040 u16 tss_selector, u16 old_tss_sel,
2041 ulong old_tss_base, struct desc_struct *new_desc)
2043 struct tss_segment_16 tss_seg;
2045 u32 new_tss_base = get_desc_base(new_desc);
2047 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2049 if (ret != X86EMUL_CONTINUE)
2050 /* FIXME: need to provide precise fault address */
2053 save_state_to_tss16(ctxt, ops, &tss_seg);
2055 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2057 if (ret != X86EMUL_CONTINUE)
2058 /* FIXME: need to provide precise fault address */
2061 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2063 if (ret != X86EMUL_CONTINUE)
2064 /* FIXME: need to provide precise fault address */
2067 if (old_tss_sel != 0xffff) {
2068 tss_seg.prev_task_link = old_tss_sel;
2070 ret = ops->write_std(new_tss_base,
2071 &tss_seg.prev_task_link,
2072 sizeof tss_seg.prev_task_link,
2073 ctxt->vcpu, &ctxt->exception);
2074 if (ret != X86EMUL_CONTINUE)
2075 /* FIXME: need to provide precise fault address */
2079 return load_state_from_tss16(ctxt, ops, &tss_seg);
2082 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2083 struct x86_emulate_ops *ops,
2084 struct tss_segment_32 *tss)
2086 struct decode_cache *c = &ctxt->decode;
2088 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2090 tss->eflags = ctxt->eflags;
2091 tss->eax = c->regs[VCPU_REGS_RAX];
2092 tss->ecx = c->regs[VCPU_REGS_RCX];
2093 tss->edx = c->regs[VCPU_REGS_RDX];
2094 tss->ebx = c->regs[VCPU_REGS_RBX];
2095 tss->esp = c->regs[VCPU_REGS_RSP];
2096 tss->ebp = c->regs[VCPU_REGS_RBP];
2097 tss->esi = c->regs[VCPU_REGS_RSI];
2098 tss->edi = c->regs[VCPU_REGS_RDI];
2100 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2101 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2102 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2103 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2104 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2105 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2106 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2109 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2110 struct x86_emulate_ops *ops,
2111 struct tss_segment_32 *tss)
2113 struct decode_cache *c = &ctxt->decode;
2116 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2117 return emulate_gp(ctxt, 0);
2119 ctxt->eflags = tss->eflags | 2;
2120 c->regs[VCPU_REGS_RAX] = tss->eax;
2121 c->regs[VCPU_REGS_RCX] = tss->ecx;
2122 c->regs[VCPU_REGS_RDX] = tss->edx;
2123 c->regs[VCPU_REGS_RBX] = tss->ebx;
2124 c->regs[VCPU_REGS_RSP] = tss->esp;
2125 c->regs[VCPU_REGS_RBP] = tss->ebp;
2126 c->regs[VCPU_REGS_RSI] = tss->esi;
2127 c->regs[VCPU_REGS_RDI] = tss->edi;
2130 * SDM says that segment selectors are loaded before segment
2133 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2134 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2135 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2136 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2137 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2138 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2139 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2142 * Now load segment descriptors. If fault happenes at this stage
2143 * it is handled in a context of new task
2145 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2146 if (ret != X86EMUL_CONTINUE)
2148 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2149 if (ret != X86EMUL_CONTINUE)
2151 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2152 if (ret != X86EMUL_CONTINUE)
2154 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2155 if (ret != X86EMUL_CONTINUE)
2157 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2158 if (ret != X86EMUL_CONTINUE)
2160 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2161 if (ret != X86EMUL_CONTINUE)
2163 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2164 if (ret != X86EMUL_CONTINUE)
2167 return X86EMUL_CONTINUE;
2170 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2171 struct x86_emulate_ops *ops,
2172 u16 tss_selector, u16 old_tss_sel,
2173 ulong old_tss_base, struct desc_struct *new_desc)
2175 struct tss_segment_32 tss_seg;
2177 u32 new_tss_base = get_desc_base(new_desc);
2179 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2181 if (ret != X86EMUL_CONTINUE)
2182 /* FIXME: need to provide precise fault address */
2185 save_state_to_tss32(ctxt, ops, &tss_seg);
2187 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2189 if (ret != X86EMUL_CONTINUE)
2190 /* FIXME: need to provide precise fault address */
2193 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2195 if (ret != X86EMUL_CONTINUE)
2196 /* FIXME: need to provide precise fault address */
2199 if (old_tss_sel != 0xffff) {
2200 tss_seg.prev_task_link = old_tss_sel;
2202 ret = ops->write_std(new_tss_base,
2203 &tss_seg.prev_task_link,
2204 sizeof tss_seg.prev_task_link,
2205 ctxt->vcpu, &ctxt->exception);
2206 if (ret != X86EMUL_CONTINUE)
2207 /* FIXME: need to provide precise fault address */
2211 return load_state_from_tss32(ctxt, ops, &tss_seg);
2214 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2215 struct x86_emulate_ops *ops,
2216 u16 tss_selector, int reason,
2217 bool has_error_code, u32 error_code)
2219 struct desc_struct curr_tss_desc, next_tss_desc;
2221 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2222 ulong old_tss_base =
2223 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2226 /* FIXME: old_tss_base == ~0 ? */
2228 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2229 if (ret != X86EMUL_CONTINUE)
2231 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2232 if (ret != X86EMUL_CONTINUE)
2235 /* FIXME: check that next_tss_desc is tss */
2237 if (reason != TASK_SWITCH_IRET) {
2238 if ((tss_selector & 3) > next_tss_desc.dpl ||
2239 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2240 return emulate_gp(ctxt, 0);
2243 desc_limit = desc_limit_scaled(&next_tss_desc);
2244 if (!next_tss_desc.p ||
2245 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2246 desc_limit < 0x2b)) {
2247 emulate_ts(ctxt, tss_selector & 0xfffc);
2248 return X86EMUL_PROPAGATE_FAULT;
2251 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2252 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2253 write_segment_descriptor(ctxt, ops, old_tss_sel,
2257 if (reason == TASK_SWITCH_IRET)
2258 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2260 /* set back link to prev task only if NT bit is set in eflags
2261 note that old_tss_sel is not used afetr this point */
2262 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2263 old_tss_sel = 0xffff;
2265 if (next_tss_desc.type & 8)
2266 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2267 old_tss_base, &next_tss_desc);
2269 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2270 old_tss_base, &next_tss_desc);
2271 if (ret != X86EMUL_CONTINUE)
2274 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2275 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2277 if (reason != TASK_SWITCH_IRET) {
2278 next_tss_desc.type |= (1 << 1); /* set busy flag */
2279 write_segment_descriptor(ctxt, ops, tss_selector,
2283 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2284 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2285 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2287 if (has_error_code) {
2288 struct decode_cache *c = &ctxt->decode;
2290 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2292 c->src.val = (unsigned long) error_code;
2293 emulate_push(ctxt, ops);
2299 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2300 u16 tss_selector, int reason,
2301 bool has_error_code, u32 error_code)
2303 struct x86_emulate_ops *ops = ctxt->ops;
2304 struct decode_cache *c = &ctxt->decode;
2308 c->dst.type = OP_NONE;
2310 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2311 has_error_code, error_code);
2313 if (rc == X86EMUL_CONTINUE) {
2314 rc = writeback(ctxt, ops);
2315 if (rc == X86EMUL_CONTINUE)
2319 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2322 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2323 int reg, struct operand *op)
2325 struct decode_cache *c = &ctxt->decode;
2326 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2328 register_address_increment(c, &c->regs[reg], df * op->bytes);
2329 op->addr.mem.ea = register_address(c, c->regs[reg]);
2330 op->addr.mem.seg = seg;
2333 static int em_push(struct x86_emulate_ctxt *ctxt)
2335 emulate_push(ctxt, ctxt->ops);
2336 return X86EMUL_CONTINUE;
2339 static int em_das(struct x86_emulate_ctxt *ctxt)
2341 struct decode_cache *c = &ctxt->decode;
2343 bool af, cf, old_cf;
2345 cf = ctxt->eflags & X86_EFLAGS_CF;
2351 af = ctxt->eflags & X86_EFLAGS_AF;
2352 if ((al & 0x0f) > 9 || af) {
2354 cf = old_cf | (al >= 250);
2359 if (old_al > 0x99 || old_cf) {
2365 /* Set PF, ZF, SF */
2366 c->src.type = OP_IMM;
2369 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2370 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2372 ctxt->eflags |= X86_EFLAGS_CF;
2374 ctxt->eflags |= X86_EFLAGS_AF;
2375 return X86EMUL_CONTINUE;
2378 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2380 struct decode_cache *c = &ctxt->decode;
2385 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2388 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2389 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2390 return X86EMUL_CONTINUE;
2393 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2395 c->src.val = old_cs;
2396 emulate_push(ctxt, ctxt->ops);
2397 rc = writeback(ctxt, ctxt->ops);
2398 if (rc != X86EMUL_CONTINUE)
2401 c->src.val = old_eip;
2402 emulate_push(ctxt, ctxt->ops);
2403 rc = writeback(ctxt, ctxt->ops);
2404 if (rc != X86EMUL_CONTINUE)
2407 c->dst.type = OP_NONE;
2409 return X86EMUL_CONTINUE;
2412 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2414 struct decode_cache *c = &ctxt->decode;
2417 c->dst.type = OP_REG;
2418 c->dst.addr.reg = &c->eip;
2419 c->dst.bytes = c->op_bytes;
2420 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2421 if (rc != X86EMUL_CONTINUE)
2423 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2424 return X86EMUL_CONTINUE;
2427 static int em_imul(struct x86_emulate_ctxt *ctxt)
2429 struct decode_cache *c = &ctxt->decode;
2431 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2432 return X86EMUL_CONTINUE;
2435 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2437 struct decode_cache *c = &ctxt->decode;
2439 c->dst.val = c->src2.val;
2440 return em_imul(ctxt);
2443 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2445 struct decode_cache *c = &ctxt->decode;
2447 c->dst.type = OP_REG;
2448 c->dst.bytes = c->src.bytes;
2449 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2450 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2452 return X86EMUL_CONTINUE;
2455 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2457 struct decode_cache *c = &ctxt->decode;
2460 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2461 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2462 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2463 return X86EMUL_CONTINUE;
2466 static int em_mov(struct x86_emulate_ctxt *ctxt)
2468 struct decode_cache *c = &ctxt->decode;
2469 c->dst.val = c->src.val;
2470 return X86EMUL_CONTINUE;
2473 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2475 struct decode_cache *c = &ctxt->decode;
2476 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2477 return X86EMUL_CONTINUE;
2480 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2482 struct decode_cache *c = &ctxt->decode;
2483 emulate_invlpg(ctxt->vcpu, linear(ctxt, c->src.addr.mem));
2484 /* Disable writeback. */
2485 c->dst.type = OP_NONE;
2486 return X86EMUL_CONTINUE;
2489 static bool valid_cr(int nr)
2501 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2503 struct decode_cache *c = &ctxt->decode;
2505 if (!valid_cr(c->modrm_reg))
2506 return emulate_ud(ctxt);
2508 return X86EMUL_CONTINUE;
2511 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2513 struct decode_cache *c = &ctxt->decode;
2514 u64 new_val = c->src.val64;
2515 int cr = c->modrm_reg;
2517 static u64 cr_reserved_bits[] = {
2518 0xffffffff00000000ULL,
2519 0, 0, 0, /* CR3 checked later */
2526 return emulate_ud(ctxt);
2528 if (new_val & cr_reserved_bits[cr])
2529 return emulate_gp(ctxt, 0);
2534 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2535 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2536 return emulate_gp(ctxt, 0);
2538 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2539 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2541 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2542 !(cr4 & X86_CR4_PAE))
2543 return emulate_gp(ctxt, 0);
2550 if (is_long_mode(ctxt->vcpu))
2551 rsvd = CR3_L_MODE_RESERVED_BITS;
2552 else if (is_pae(ctxt->vcpu))
2553 rsvd = CR3_PAE_RESERVED_BITS;
2554 else if (is_paging(ctxt->vcpu))
2555 rsvd = CR3_NONPAE_RESERVED_BITS;
2558 return emulate_gp(ctxt, 0);
2565 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2566 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2568 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2569 return emulate_gp(ctxt, 0);
2575 return X86EMUL_CONTINUE;
2578 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2582 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2584 /* Check if DR7.Global_Enable is set */
2585 return dr7 & (1 << 13);
2588 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2590 struct decode_cache *c = &ctxt->decode;
2591 int dr = c->modrm_reg;
2595 return emulate_ud(ctxt);
2597 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2598 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2599 return emulate_ud(ctxt);
2601 if (check_dr7_gd(ctxt))
2602 return emulate_db(ctxt);
2604 return X86EMUL_CONTINUE;
2607 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2609 struct decode_cache *c = &ctxt->decode;
2610 u64 new_val = c->src.val64;
2611 int dr = c->modrm_reg;
2613 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2614 return emulate_gp(ctxt, 0);
2616 return check_dr_read(ctxt);
2619 static int check_svme(struct x86_emulate_ctxt *ctxt)
2623 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2625 if (!(efer & EFER_SVME))
2626 return emulate_ud(ctxt);
2628 return X86EMUL_CONTINUE;
2631 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2633 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2635 /* Valid physical address? */
2636 if (rax & 0xffff000000000000)
2637 return emulate_gp(ctxt, 0);
2639 return check_svme(ctxt);
2642 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2644 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2646 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2647 return emulate_ud(ctxt);
2649 return X86EMUL_CONTINUE;
2652 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2654 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2655 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2657 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2659 return emulate_gp(ctxt, 0);
2661 return X86EMUL_CONTINUE;
2664 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2666 struct decode_cache *c = &ctxt->decode;
2668 c->dst.bytes = min(c->dst.bytes, 4u);
2669 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2670 return emulate_gp(ctxt, 0);
2672 return X86EMUL_CONTINUE;
2675 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2677 struct decode_cache *c = &ctxt->decode;
2679 c->src.bytes = min(c->src.bytes, 4u);
2680 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2681 return emulate_gp(ctxt, 0);
2683 return X86EMUL_CONTINUE;
2686 #define D(_y) { .flags = (_y) }
2687 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2688 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2689 .check_perm = (_p) }
2691 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2692 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2693 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2694 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2695 #define II(_f, _e, _i) \
2696 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2697 #define IIP(_f, _e, _i, _p) \
2698 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2699 .check_perm = (_p) }
2700 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2702 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2703 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2704 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2706 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2707 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2708 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2710 static struct opcode group7_rm1[] = {
2711 DI(SrcNone | ModRM | Priv, monitor),
2712 DI(SrcNone | ModRM | Priv, mwait),
2716 static struct opcode group7_rm3[] = {
2717 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2718 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2719 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2720 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2721 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2722 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2723 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2724 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2727 static struct opcode group7_rm7[] = {
2729 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2732 static struct opcode group1[] = {
2736 static struct opcode group1A[] = {
2737 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2740 static struct opcode group3[] = {
2741 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2742 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2743 X4(D(SrcMem | ModRM)),
2746 static struct opcode group4[] = {
2747 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2751 static struct opcode group5[] = {
2752 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2753 D(SrcMem | ModRM | Stack),
2754 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2755 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2756 D(SrcMem | ModRM | Stack), N,
2759 static struct opcode group6[] = {
2760 DI(ModRM | Prot, sldt),
2761 DI(ModRM | Prot, str),
2762 DI(ModRM | Prot | Priv, lldt),
2763 DI(ModRM | Prot | Priv, ltr),
2767 static struct group_dual group7 = { {
2768 DI(ModRM | Mov | DstMem | Priv, sgdt),
2769 DI(ModRM | Mov | DstMem | Priv, sidt),
2770 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2771 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2772 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2773 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2775 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2776 N, EXT(0, group7_rm3),
2777 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2778 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2781 static struct opcode group8[] = {
2783 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2784 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2787 static struct group_dual group9 = { {
2788 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2790 N, N, N, N, N, N, N, N,
2793 static struct opcode group11[] = {
2794 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2797 static struct gprefix pfx_0f_6f_0f_7f = {
2798 N, N, N, I(Sse, em_movdqu),
2801 static struct opcode opcode_table[256] = {
2804 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2807 D(ImplicitOps | Stack | No64), N,
2810 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2813 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2817 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2825 X8(I(SrcReg | Stack, em_push)),
2827 X8(D(DstReg | Stack)),
2829 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2830 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2833 I(SrcImm | Mov | Stack, em_push),
2834 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2835 I(SrcImmByte | Mov | Stack, em_push),
2836 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2837 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2838 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2842 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2843 G(DstMem | SrcImm | ModRM | Group, group1),
2844 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2845 G(DstMem | SrcImmByte | ModRM | Group, group1),
2846 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2848 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2849 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2850 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2851 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2853 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2855 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2856 I(SrcImmFAddr | No64, em_call_far), N,
2857 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2859 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2860 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2861 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2862 D2bv(SrcSI | DstDI | String),
2864 D2bv(DstAcc | SrcImm),
2865 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2866 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2867 D2bv(SrcAcc | DstDI | String),
2869 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2871 X8(I(DstReg | SrcImm | Mov, em_mov)),
2873 D2bv(DstMem | SrcImmByte | ModRM),
2874 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2875 D(ImplicitOps | Stack),
2876 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2877 G(ByteOp, group11), G(0, group11),
2879 N, N, N, D(ImplicitOps | Stack),
2880 D(ImplicitOps), DI(SrcImmByte, intn),
2881 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2883 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2886 N, N, N, N, N, N, N, N,
2889 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2890 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2892 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2893 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2894 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2895 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2897 N, DI(ImplicitOps, icebp), N, N,
2898 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2899 G(ByteOp, group3), G(0, group3),
2901 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2902 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2905 static struct opcode twobyte_table[256] = {
2907 G(0, group6), GD(0, &group7), N, N,
2908 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2909 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2910 N, D(ImplicitOps | ModRM), N, N,
2912 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2914 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2915 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2916 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2917 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
2919 N, N, N, N, N, N, N, N,
2921 DI(ImplicitOps | Priv, wrmsr),
2922 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
2923 DI(ImplicitOps | Priv, rdmsr),
2924 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
2925 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2927 N, N, N, N, N, N, N, N,
2929 X16(D(DstReg | SrcMem | ModRM | Mov)),
2931 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2936 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
2941 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
2945 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
2947 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2948 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
2949 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2950 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
2952 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
2953 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2954 D(DstMem | SrcReg | Src2ImmByte | ModRM),
2955 D(DstMem | SrcReg | Src2CL | ModRM),
2956 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
2958 D2bv(DstMem | SrcReg | ModRM | Lock),
2959 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2960 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
2961 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2964 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
2965 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
2966 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
2968 D2bv(DstMem | SrcReg | ModRM | Lock),
2969 N, D(DstMem | SrcReg | ModRM | Mov),
2970 N, N, N, GD(0, &group9),
2971 N, N, N, N, N, N, N, N,
2973 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2975 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
2977 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
2993 static unsigned imm_size(struct decode_cache *c)
2997 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3003 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3004 unsigned size, bool sign_extension)
3006 struct decode_cache *c = &ctxt->decode;
3007 struct x86_emulate_ops *ops = ctxt->ops;
3008 int rc = X86EMUL_CONTINUE;
3012 op->addr.mem.ea = c->eip;
3013 /* NB. Immediates are sign-extended as necessary. */
3014 switch (op->bytes) {
3016 op->val = insn_fetch(s8, 1, c->eip);
3019 op->val = insn_fetch(s16, 2, c->eip);
3022 op->val = insn_fetch(s32, 4, c->eip);
3025 if (!sign_extension) {
3026 switch (op->bytes) {
3034 op->val &= 0xffffffff;
3043 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3045 struct x86_emulate_ops *ops = ctxt->ops;
3046 struct decode_cache *c = &ctxt->decode;
3047 int rc = X86EMUL_CONTINUE;
3048 int mode = ctxt->mode;
3049 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3050 bool op_prefix = false;
3051 struct opcode opcode, *g_mod012, *g_mod3;
3052 struct operand memop = { .type = OP_NONE };
3055 c->fetch.start = c->eip;
3056 c->fetch.end = c->fetch.start + insn_len;
3058 memcpy(c->fetch.data, insn, insn_len);
3059 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
3062 case X86EMUL_MODE_REAL:
3063 case X86EMUL_MODE_VM86:
3064 case X86EMUL_MODE_PROT16:
3065 def_op_bytes = def_ad_bytes = 2;
3067 case X86EMUL_MODE_PROT32:
3068 def_op_bytes = def_ad_bytes = 4;
3070 #ifdef CONFIG_X86_64
3071 case X86EMUL_MODE_PROT64:
3080 c->op_bytes = def_op_bytes;
3081 c->ad_bytes = def_ad_bytes;
3083 /* Legacy prefixes. */
3085 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3086 case 0x66: /* operand-size override */
3088 /* switch between 2/4 bytes */
3089 c->op_bytes = def_op_bytes ^ 6;
3091 case 0x67: /* address-size override */
3092 if (mode == X86EMUL_MODE_PROT64)
3093 /* switch between 4/8 bytes */
3094 c->ad_bytes = def_ad_bytes ^ 12;
3096 /* switch between 2/4 bytes */
3097 c->ad_bytes = def_ad_bytes ^ 6;
3099 case 0x26: /* ES override */
3100 case 0x2e: /* CS override */
3101 case 0x36: /* SS override */
3102 case 0x3e: /* DS override */
3103 set_seg_override(c, (c->b >> 3) & 3);
3105 case 0x64: /* FS override */
3106 case 0x65: /* GS override */
3107 set_seg_override(c, c->b & 7);
3109 case 0x40 ... 0x4f: /* REX */
3110 if (mode != X86EMUL_MODE_PROT64)
3112 c->rex_prefix = c->b;
3114 case 0xf0: /* LOCK */
3117 case 0xf2: /* REPNE/REPNZ */
3118 case 0xf3: /* REP/REPE/REPZ */
3119 c->rep_prefix = c->b;
3125 /* Any legacy prefix after a REX prefix nullifies its effect. */
3133 if (c->rex_prefix & 8)
3134 c->op_bytes = 8; /* REX.W */
3136 /* Opcode byte(s). */
3137 opcode = opcode_table[c->b];
3138 /* Two-byte opcode? */
3141 c->b = insn_fetch(u8, 1, c->eip);
3142 opcode = twobyte_table[c->b];
3144 c->d = opcode.flags;
3147 dual = c->d & GroupDual;
3148 c->modrm = insn_fetch(u8, 1, c->eip);
3151 if (c->d & GroupDual) {
3152 g_mod012 = opcode.u.gdual->mod012;
3153 g_mod3 = opcode.u.gdual->mod3;
3155 g_mod012 = g_mod3 = opcode.u.group;
3157 c->d &= ~(Group | GroupDual);
3159 goffset = (c->modrm >> 3) & 7;
3161 if ((c->modrm >> 6) == 3)
3162 opcode = g_mod3[goffset];
3164 opcode = g_mod012[goffset];
3166 if (opcode.flags & RMExt) {
3167 goffset = c->modrm & 7;
3168 opcode = opcode.u.group[goffset];
3171 c->d |= opcode.flags;
3174 if (c->d & Prefix) {
3175 if (c->rep_prefix && op_prefix)
3176 return X86EMUL_UNHANDLEABLE;
3177 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3178 switch (simd_prefix) {
3179 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3180 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3181 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3182 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3184 c->d |= opcode.flags;
3187 c->execute = opcode.u.execute;
3188 c->check_perm = opcode.check_perm;
3189 c->intercept = opcode.intercept;
3192 if (c->d == 0 || (c->d & Undefined))
3195 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3198 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3201 if (c->d & Op3264) {
3202 if (mode == X86EMUL_MODE_PROT64)
3211 /* ModRM and SIB bytes. */
3213 rc = decode_modrm(ctxt, ops, &memop);
3214 if (!c->has_seg_override)
3215 set_seg_override(c, c->modrm_seg);
3216 } else if (c->d & MemAbs)
3217 rc = decode_abs(ctxt, ops, &memop);
3218 if (rc != X86EMUL_CONTINUE)
3221 if (!c->has_seg_override)
3222 set_seg_override(c, VCPU_SREG_DS);
3224 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3226 if (memop.type == OP_MEM && c->ad_bytes != 8)
3227 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3229 if (memop.type == OP_MEM && c->rip_relative)
3230 memop.addr.mem.ea += c->eip;
3233 * Decode and fetch the source operand: register, memory
3236 switch (c->d & SrcMask) {
3240 decode_register_operand(ctxt, &c->src, c, 0);
3249 memop.bytes = (c->d & ByteOp) ? 1 :
3255 rc = decode_imm(ctxt, &c->src, 2, false);
3258 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3261 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3264 rc = decode_imm(ctxt, &c->src, 1, true);
3267 rc = decode_imm(ctxt, &c->src, 1, false);
3270 c->src.type = OP_REG;
3271 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3272 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3273 fetch_register_operand(&c->src);
3280 c->src.type = OP_MEM;
3281 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3282 c->src.addr.mem.ea =
3283 register_address(c, c->regs[VCPU_REGS_RSI]);
3284 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3288 c->src.type = OP_IMM;
3289 c->src.addr.mem.ea = c->eip;
3290 c->src.bytes = c->op_bytes + 2;
3291 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3294 memop.bytes = c->op_bytes + 2;
3299 if (rc != X86EMUL_CONTINUE)
3303 * Decode and fetch the second source operand: register, memory
3306 switch (c->d & Src2Mask) {
3311 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3314 rc = decode_imm(ctxt, &c->src2, 1, true);
3321 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3325 if (rc != X86EMUL_CONTINUE)
3328 /* Decode and fetch the destination operand: register or memory. */
3329 switch (c->d & DstMask) {
3331 decode_register_operand(ctxt, &c->dst, c,
3332 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3335 c->dst.type = OP_IMM;
3336 c->dst.addr.mem.ea = c->eip;
3338 c->dst.val = insn_fetch(u8, 1, c->eip);
3343 if ((c->d & DstMask) == DstMem64)
3346 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3348 fetch_bit_operand(c);
3349 c->dst.orig_val = c->dst.val;
3352 c->dst.type = OP_REG;
3353 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3354 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3355 fetch_register_operand(&c->dst);
3356 c->dst.orig_val = c->dst.val;
3359 c->dst.type = OP_MEM;
3360 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3361 c->dst.addr.mem.ea =
3362 register_address(c, c->regs[VCPU_REGS_RDI]);
3363 c->dst.addr.mem.seg = VCPU_SREG_ES;
3367 /* Special instructions do their own operand decoding. */
3369 c->dst.type = OP_NONE; /* Disable writeback. */
3374 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3377 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3379 struct decode_cache *c = &ctxt->decode;
3381 /* The second termination condition only applies for REPE
3382 * and REPNE. Test if the repeat string operation prefix is
3383 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3384 * corresponding termination condition according to:
3385 * - if REPE/REPZ and ZF = 0 then done
3386 * - if REPNE/REPNZ and ZF = 1 then done
3388 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3389 (c->b == 0xae) || (c->b == 0xaf))
3390 && (((c->rep_prefix == REPE_PREFIX) &&
3391 ((ctxt->eflags & EFLG_ZF) == 0))
3392 || ((c->rep_prefix == REPNE_PREFIX) &&
3393 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3400 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3402 struct x86_emulate_ops *ops = ctxt->ops;
3404 struct decode_cache *c = &ctxt->decode;
3405 int rc = X86EMUL_CONTINUE;
3406 int saved_dst_type = c->dst.type;
3407 int irq; /* Used for int 3, int, and into */
3409 ctxt->decode.mem_read.pos = 0;
3411 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3412 rc = emulate_ud(ctxt);
3416 /* LOCK prefix is allowed only with some instructions */
3417 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3418 rc = emulate_ud(ctxt);
3422 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3423 rc = emulate_ud(ctxt);
3428 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3429 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3430 rc = emulate_ud(ctxt);
3434 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3435 rc = emulate_nm(ctxt);
3439 if (unlikely(ctxt->guest_mode) && c->intercept) {
3440 rc = emulator_check_intercept(ctxt, c->intercept,
3441 X86_ICPT_PRE_EXCEPT);
3442 if (rc != X86EMUL_CONTINUE)
3446 /* Privileged instruction can be executed only in CPL=0 */
3447 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3448 rc = emulate_gp(ctxt, 0);
3452 /* Instruction can only be executed in protected mode */
3453 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3454 rc = emulate_ud(ctxt);
3458 /* Do instruction specific permission checks */
3459 if (c->check_perm) {
3460 rc = c->check_perm(ctxt);
3461 if (rc != X86EMUL_CONTINUE)
3465 if (unlikely(ctxt->guest_mode) && c->intercept) {
3466 rc = emulator_check_intercept(ctxt, c->intercept,
3467 X86_ICPT_POST_EXCEPT);
3468 if (rc != X86EMUL_CONTINUE)
3472 if (c->rep_prefix && (c->d & String)) {
3473 /* All REP prefixes have the same first termination condition */
3474 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3480 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3481 rc = segmented_read(ctxt, c->src.addr.mem,
3482 c->src.valptr, c->src.bytes);
3483 if (rc != X86EMUL_CONTINUE)
3485 c->src.orig_val64 = c->src.val64;
3488 if (c->src2.type == OP_MEM) {
3489 rc = segmented_read(ctxt, c->src2.addr.mem,
3490 &c->src2.val, c->src2.bytes);
3491 if (rc != X86EMUL_CONTINUE)
3495 if ((c->d & DstMask) == ImplicitOps)
3499 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3500 /* optimisation - avoid slow emulated read if Mov */
3501 rc = segmented_read(ctxt, c->dst.addr.mem,
3502 &c->dst.val, c->dst.bytes);
3503 if (rc != X86EMUL_CONTINUE)
3506 c->dst.orig_val = c->dst.val;
3510 if (unlikely(ctxt->guest_mode) && c->intercept) {
3511 rc = emulator_check_intercept(ctxt, c->intercept,
3512 X86_ICPT_POST_MEMACCESS);
3513 if (rc != X86EMUL_CONTINUE)
3518 rc = c->execute(ctxt);
3519 if (rc != X86EMUL_CONTINUE)
3530 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3532 case 0x06: /* push es */
3533 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3535 case 0x07: /* pop es */
3536 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3540 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3542 case 0x0e: /* push cs */
3543 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3547 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3549 case 0x16: /* push ss */
3550 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3552 case 0x17: /* pop ss */
3553 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3557 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3559 case 0x1e: /* push ds */
3560 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3562 case 0x1f: /* pop ds */
3563 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3567 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3571 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3575 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3579 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3581 case 0x40 ... 0x47: /* inc r16/r32 */
3582 emulate_1op("inc", c->dst, ctxt->eflags);
3584 case 0x48 ... 0x4f: /* dec r16/r32 */
3585 emulate_1op("dec", c->dst, ctxt->eflags);
3587 case 0x58 ... 0x5f: /* pop reg */
3589 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3591 case 0x60: /* pusha */
3592 rc = emulate_pusha(ctxt, ops);
3594 case 0x61: /* popa */
3595 rc = emulate_popa(ctxt, ops);
3597 case 0x63: /* movsxd */
3598 if (ctxt->mode != X86EMUL_MODE_PROT64)
3599 goto cannot_emulate;
3600 c->dst.val = (s32) c->src.val;
3602 case 0x6c: /* insb */
3603 case 0x6d: /* insw/insd */
3604 c->src.val = c->regs[VCPU_REGS_RDX];
3606 case 0x6e: /* outsb */
3607 case 0x6f: /* outsw/outsd */
3608 c->dst.val = c->regs[VCPU_REGS_RDX];
3611 case 0x70 ... 0x7f: /* jcc (short) */
3612 if (test_cc(c->b, ctxt->eflags))
3613 jmp_rel(c, c->src.val);
3615 case 0x80 ... 0x83: /* Grp1 */
3616 switch (c->modrm_reg) {
3637 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3639 case 0x86 ... 0x87: /* xchg */
3641 /* Write back the register source. */
3642 c->src.val = c->dst.val;
3643 write_register_operand(&c->src);
3645 * Write back the memory destination with implicit LOCK
3648 c->dst.val = c->src.orig_val;
3651 case 0x8c: /* mov r/m, sreg */
3652 if (c->modrm_reg > VCPU_SREG_GS) {
3653 rc = emulate_ud(ctxt);
3656 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3658 case 0x8d: /* lea r16/r32, m */
3659 c->dst.val = c->src.addr.mem.ea;
3661 case 0x8e: { /* mov seg, r/m16 */
3666 if (c->modrm_reg == VCPU_SREG_CS ||
3667 c->modrm_reg > VCPU_SREG_GS) {
3668 rc = emulate_ud(ctxt);
3672 if (c->modrm_reg == VCPU_SREG_SS)
3673 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3675 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3677 c->dst.type = OP_NONE; /* Disable writeback. */
3680 case 0x8f: /* pop (sole member of Grp1a) */
3681 rc = emulate_grp1a(ctxt, ops);
3683 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3684 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3687 case 0x98: /* cbw/cwde/cdqe */
3688 switch (c->op_bytes) {
3689 case 2: c->dst.val = (s8)c->dst.val; break;
3690 case 4: c->dst.val = (s16)c->dst.val; break;
3691 case 8: c->dst.val = (s32)c->dst.val; break;
3694 case 0x9c: /* pushf */
3695 c->src.val = (unsigned long) ctxt->eflags;
3696 emulate_push(ctxt, ops);
3698 case 0x9d: /* popf */
3699 c->dst.type = OP_REG;
3700 c->dst.addr.reg = &ctxt->eflags;
3701 c->dst.bytes = c->op_bytes;
3702 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3704 case 0xa6 ... 0xa7: /* cmps */
3705 c->dst.type = OP_NONE; /* Disable writeback. */
3707 case 0xa8 ... 0xa9: /* test ax, imm */
3709 case 0xae ... 0xaf: /* scas */
3714 case 0xc3: /* ret */
3715 c->dst.type = OP_REG;
3716 c->dst.addr.reg = &c->eip;
3717 c->dst.bytes = c->op_bytes;
3718 goto pop_instruction;
3719 case 0xc4: /* les */
3720 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3722 case 0xc5: /* lds */
3723 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3725 case 0xcb: /* ret far */
3726 rc = emulate_ret_far(ctxt, ops);
3728 case 0xcc: /* int3 */
3731 case 0xcd: /* int n */
3734 rc = emulate_int(ctxt, ops, irq);
3736 case 0xce: /* into */
3737 if (ctxt->eflags & EFLG_OF) {
3742 case 0xcf: /* iret */
3743 rc = emulate_iret(ctxt, ops);
3745 case 0xd0 ... 0xd1: /* Grp2 */
3748 case 0xd2 ... 0xd3: /* Grp2 */
3749 c->src.val = c->regs[VCPU_REGS_RCX];
3752 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3753 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3754 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3755 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3756 jmp_rel(c, c->src.val);
3758 case 0xe3: /* jcxz/jecxz/jrcxz */
3759 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3760 jmp_rel(c, c->src.val);
3762 case 0xe4: /* inb */
3765 case 0xe6: /* outb */
3766 case 0xe7: /* out */
3768 case 0xe8: /* call (near) */ {
3769 long int rel = c->src.val;
3770 c->src.val = (unsigned long) c->eip;
3772 emulate_push(ctxt, ops);
3775 case 0xe9: /* jmp rel */
3777 case 0xea: { /* jmp far */
3780 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3782 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3786 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3790 jmp: /* jmp rel short */
3791 jmp_rel(c, c->src.val);
3792 c->dst.type = OP_NONE; /* Disable writeback. */
3794 case 0xec: /* in al,dx */
3795 case 0xed: /* in (e/r)ax,dx */
3796 c->src.val = c->regs[VCPU_REGS_RDX];
3798 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3800 goto done; /* IO is needed */
3802 case 0xee: /* out dx,al */
3803 case 0xef: /* out dx,(e/r)ax */
3804 c->dst.val = c->regs[VCPU_REGS_RDX];
3806 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3807 &c->src.val, 1, ctxt->vcpu);
3808 c->dst.type = OP_NONE; /* Disable writeback. */
3810 case 0xf4: /* hlt */
3811 ctxt->vcpu->arch.halt_request = 1;
3813 case 0xf5: /* cmc */
3814 /* complement carry flag from eflags reg */
3815 ctxt->eflags ^= EFLG_CF;
3817 case 0xf6 ... 0xf7: /* Grp3 */
3818 rc = emulate_grp3(ctxt, ops);
3820 case 0xf8: /* clc */
3821 ctxt->eflags &= ~EFLG_CF;
3823 case 0xf9: /* stc */
3824 ctxt->eflags |= EFLG_CF;
3826 case 0xfa: /* cli */
3827 if (emulator_bad_iopl(ctxt, ops)) {
3828 rc = emulate_gp(ctxt, 0);
3831 ctxt->eflags &= ~X86_EFLAGS_IF;
3833 case 0xfb: /* sti */
3834 if (emulator_bad_iopl(ctxt, ops)) {
3835 rc = emulate_gp(ctxt, 0);
3838 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3839 ctxt->eflags |= X86_EFLAGS_IF;
3842 case 0xfc: /* cld */
3843 ctxt->eflags &= ~EFLG_DF;
3845 case 0xfd: /* std */
3846 ctxt->eflags |= EFLG_DF;
3848 case 0xfe: /* Grp4 */
3850 rc = emulate_grp45(ctxt, ops);
3852 case 0xff: /* Grp5 */
3853 if (c->modrm_reg == 5)
3857 goto cannot_emulate;
3860 if (rc != X86EMUL_CONTINUE)
3864 rc = writeback(ctxt, ops);
3865 if (rc != X86EMUL_CONTINUE)
3869 * restore dst type in case the decoding will be reused
3870 * (happens for string instruction )
3872 c->dst.type = saved_dst_type;
3874 if ((c->d & SrcMask) == SrcSI)
3875 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3876 VCPU_REGS_RSI, &c->src);
3878 if ((c->d & DstMask) == DstDI)
3879 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3882 if (c->rep_prefix && (c->d & String)) {
3883 struct read_cache *r = &ctxt->decode.io_read;
3884 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3886 if (!string_insn_completed(ctxt)) {
3888 * Re-enter guest when pio read ahead buffer is empty
3889 * or, if it is not used, after each 1024 iteration.
3891 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3892 (r->end == 0 || r->end != r->pos)) {
3894 * Reset read cache. Usually happens before
3895 * decode, but since instruction is restarted
3896 * we have to do it here.
3898 ctxt->decode.mem_read.end = 0;
3899 return EMULATION_RESTART;
3901 goto done; /* skip rip writeback */
3908 if (rc == X86EMUL_PROPAGATE_FAULT)
3909 ctxt->have_exception = true;
3910 if (rc == X86EMUL_INTERCEPTED)
3911 return EMULATION_INTERCEPTED;
3913 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3917 case 0x01: /* lgdt, lidt, lmsw */
3918 switch (c->modrm_reg) {
3920 unsigned long address;
3922 case 0: /* vmcall */
3923 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3924 goto cannot_emulate;
3926 rc = kvm_fix_hypercall(ctxt->vcpu);
3927 if (rc != X86EMUL_CONTINUE)
3930 /* Let the processor re-execute the fixed hypercall */
3932 /* Disable writeback. */
3933 c->dst.type = OP_NONE;
3936 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3937 &size, &address, c->op_bytes);
3938 if (rc != X86EMUL_CONTINUE)
3940 realmode_lgdt(ctxt->vcpu, size, address);
3941 /* Disable writeback. */
3942 c->dst.type = OP_NONE;
3944 case 3: /* lidt/vmmcall */
3945 if (c->modrm_mod == 3) {
3946 switch (c->modrm_rm) {
3948 rc = kvm_fix_hypercall(ctxt->vcpu);
3951 goto cannot_emulate;
3954 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
3957 if (rc != X86EMUL_CONTINUE)
3959 realmode_lidt(ctxt->vcpu, size, address);
3961 /* Disable writeback. */
3962 c->dst.type = OP_NONE;
3966 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3969 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
3970 (c->src.val & 0x0f), ctxt->vcpu);
3971 c->dst.type = OP_NONE;
3973 case 5: /* not defined */
3975 rc = X86EMUL_PROPAGATE_FAULT;
3978 rc = em_invlpg(ctxt);
3981 goto cannot_emulate;
3984 case 0x05: /* syscall */
3985 rc = emulate_syscall(ctxt, ops);
3988 emulate_clts(ctxt->vcpu);
3990 case 0x09: /* wbinvd */
3991 kvm_emulate_wbinvd(ctxt->vcpu);
3993 case 0x08: /* invd */
3994 case 0x0d: /* GrpP (prefetch) */
3995 case 0x18: /* Grp16 (prefetch/nop) */
3997 case 0x20: /* mov cr, reg */
3998 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4000 case 0x21: /* mov from dr to reg */
4001 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4003 case 0x22: /* mov reg, cr */
4004 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4005 emulate_gp(ctxt, 0);
4006 rc = X86EMUL_PROPAGATE_FAULT;
4009 c->dst.type = OP_NONE;
4011 case 0x23: /* mov from reg to dr */
4012 if (ops->set_dr(c->modrm_reg, c->src.val &
4013 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4014 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4015 /* #UD condition is already handled by the code above */
4016 emulate_gp(ctxt, 0);
4017 rc = X86EMUL_PROPAGATE_FAULT;
4021 c->dst.type = OP_NONE; /* no writeback */
4025 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4026 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4027 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4028 emulate_gp(ctxt, 0);
4029 rc = X86EMUL_PROPAGATE_FAULT;
4032 rc = X86EMUL_CONTINUE;
4036 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4037 emulate_gp(ctxt, 0);
4038 rc = X86EMUL_PROPAGATE_FAULT;
4041 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4042 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4044 rc = X86EMUL_CONTINUE;
4046 case 0x34: /* sysenter */
4047 rc = emulate_sysenter(ctxt, ops);
4049 case 0x35: /* sysexit */
4050 rc = emulate_sysexit(ctxt, ops);
4052 case 0x40 ... 0x4f: /* cmov */
4053 c->dst.val = c->dst.orig_val = c->src.val;
4054 if (!test_cc(c->b, ctxt->eflags))
4055 c->dst.type = OP_NONE; /* no writeback */
4057 case 0x80 ... 0x8f: /* jnz rel, etc*/
4058 if (test_cc(c->b, ctxt->eflags))
4059 jmp_rel(c, c->src.val);
4061 case 0x90 ... 0x9f: /* setcc r/m8 */
4062 c->dst.val = test_cc(c->b, ctxt->eflags);
4064 case 0xa0: /* push fs */
4065 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4067 case 0xa1: /* pop fs */
4068 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4072 c->dst.type = OP_NONE;
4073 /* only subword offset */
4074 c->src.val &= (c->dst.bytes << 3) - 1;
4075 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4077 case 0xa4: /* shld imm8, r, r/m */
4078 case 0xa5: /* shld cl, r, r/m */
4079 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4081 case 0xa8: /* push gs */
4082 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4084 case 0xa9: /* pop gs */
4085 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4089 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4091 case 0xac: /* shrd imm8, r, r/m */
4092 case 0xad: /* shrd cl, r, r/m */
4093 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4095 case 0xae: /* clflush */
4097 case 0xb0 ... 0xb1: /* cmpxchg */
4099 * Save real source value, then compare EAX against
4102 c->src.orig_val = c->src.val;
4103 c->src.val = c->regs[VCPU_REGS_RAX];
4104 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4105 if (ctxt->eflags & EFLG_ZF) {
4106 /* Success: write back to memory. */
4107 c->dst.val = c->src.orig_val;
4109 /* Failure: write the value we saw to EAX. */
4110 c->dst.type = OP_REG;
4111 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4114 case 0xb2: /* lss */
4115 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4119 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4121 case 0xb4: /* lfs */
4122 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4124 case 0xb5: /* lgs */
4125 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4127 case 0xb6 ... 0xb7: /* movzx */
4128 c->dst.bytes = c->op_bytes;
4129 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4132 case 0xba: /* Grp8 */
4133 switch (c->modrm_reg & 3) {
4146 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4148 case 0xbc: { /* bsf */
4150 __asm__ ("bsf %2, %0; setz %1"
4151 : "=r"(c->dst.val), "=q"(zf)
4153 ctxt->eflags &= ~X86_EFLAGS_ZF;
4155 ctxt->eflags |= X86_EFLAGS_ZF;
4156 c->dst.type = OP_NONE; /* Disable writeback. */
4160 case 0xbd: { /* bsr */
4162 __asm__ ("bsr %2, %0; setz %1"
4163 : "=r"(c->dst.val), "=q"(zf)
4165 ctxt->eflags &= ~X86_EFLAGS_ZF;
4167 ctxt->eflags |= X86_EFLAGS_ZF;
4168 c->dst.type = OP_NONE; /* Disable writeback. */
4172 case 0xbe ... 0xbf: /* movsx */
4173 c->dst.bytes = c->op_bytes;
4174 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4177 case 0xc0 ... 0xc1: /* xadd */
4178 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4179 /* Write back the register source. */
4180 c->src.val = c->dst.orig_val;
4181 write_register_operand(&c->src);
4183 case 0xc3: /* movnti */
4184 c->dst.bytes = c->op_bytes;
4185 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4188 case 0xc7: /* Grp9 (cmpxchg8b) */
4189 rc = emulate_grp9(ctxt, ops);
4192 goto cannot_emulate;
4195 if (rc != X86EMUL_CONTINUE)
4201 return EMULATION_FAILED;