1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 struct opcode *group;
165 struct group_dual *gdual;
166 struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
206 * Instruction emulation:
207 * Most instructions are emulated directly via a fragment of inline assembly
208 * code. This allows us to save/restore EFLAGS and thus very easily pick up
209 * any modified flags.
212 #if defined(CONFIG_X86_64)
213 #define _LO32 "k" /* force 32-bit operand */
214 #define _STK "%%rsp" /* stack pointer */
215 #elif defined(__i386__)
216 #define _LO32 "" /* force 32-bit operand */
217 #define _STK "%%esp" /* stack pointer */
221 * These EFLAGS bits are restored from saved value during emulation, and
222 * any changes are written back to the saved value after emulation.
224 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
226 /* Before executing instruction: restore necessary bits in EFLAGS. */
227 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
228 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
229 "movl %"_sav",%"_LO32 _tmp"; " \
232 "movl %"_msk",%"_LO32 _tmp"; " \
233 "andl %"_LO32 _tmp",("_STK"); " \
235 "notl %"_LO32 _tmp"; " \
236 "andl %"_LO32 _tmp",("_STK"); " \
237 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
239 "orl %"_LO32 _tmp",("_STK"); " \
243 /* After executing instruction: write-back necessary bits in EFLAGS. */
244 #define _POST_EFLAGS(_sav, _msk, _tmp) \
245 /* _sav |= EFLAGS & _msk; */ \
248 "andl %"_msk",%"_LO32 _tmp"; " \
249 "orl %"_LO32 _tmp",%"_sav"; "
257 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
259 __asm__ __volatile__ ( \
260 _PRE_EFLAGS("0", "4", "2") \
261 _op _suffix " %"_x"3,%1; " \
262 _POST_EFLAGS("0", "4", "2") \
263 : "=m" ((ctxt)->eflags), \
264 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
266 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
270 /* Raw emulation: instruction has two explicit operands. */
271 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
273 unsigned long _tmp; \
275 switch ((ctxt)->dst.bytes) { \
277 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
280 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
283 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
288 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
290 unsigned long _tmp; \
291 switch ((ctxt)->dst.bytes) { \
293 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
296 __emulate_2op_nobyte(ctxt, _op, \
297 _wx, _wy, _lx, _ly, _qx, _qy); \
302 /* Source operand is byte-sized and may be restricted to just %cl. */
303 #define emulate_2op_SrcB(ctxt, _op) \
304 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
306 /* Source operand is byte, word, long or quad sized. */
307 #define emulate_2op_SrcV(ctxt, _op) \
308 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
310 /* Source operand is word, long or quad sized. */
311 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
312 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
314 /* Instruction has three operands and one operand is stored in ECX register */
315 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
317 unsigned long _tmp; \
318 _type _clv = (ctxt)->src2.val; \
319 _type _srcv = (ctxt)->src.val; \
320 _type _dstv = (ctxt)->dst.val; \
322 __asm__ __volatile__ ( \
323 _PRE_EFLAGS("0", "5", "2") \
324 _op _suffix " %4,%1 \n" \
325 _POST_EFLAGS("0", "5", "2") \
326 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
327 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
330 (ctxt)->src2.val = (unsigned long) _clv; \
331 (ctxt)->src2.val = (unsigned long) _srcv; \
332 (ctxt)->dst.val = (unsigned long) _dstv; \
335 #define emulate_2op_cl(ctxt, _op) \
337 switch ((ctxt)->dst.bytes) { \
339 __emulate_2op_cl(ctxt, _op, "w", u16); \
342 __emulate_2op_cl(ctxt, _op, "l", u32); \
345 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
350 #define __emulate_1op(ctxt, _op, _suffix) \
352 unsigned long _tmp; \
354 __asm__ __volatile__ ( \
355 _PRE_EFLAGS("0", "3", "2") \
356 _op _suffix " %1; " \
357 _POST_EFLAGS("0", "3", "2") \
358 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
360 : "i" (EFLAGS_MASK)); \
363 /* Instruction has only one explicit operand (no source operand). */
364 #define emulate_1op(ctxt, _op) \
366 switch ((ctxt)->dst.bytes) { \
367 case 1: __emulate_1op(ctxt, _op, "b"); break; \
368 case 2: __emulate_1op(ctxt, _op, "w"); break; \
369 case 4: __emulate_1op(ctxt, _op, "l"); break; \
370 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
374 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
376 unsigned long _tmp; \
377 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
378 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
380 __asm__ __volatile__ ( \
381 _PRE_EFLAGS("0", "5", "1") \
383 _op _suffix " %6; " \
385 _POST_EFLAGS("0", "5", "1") \
386 ".pushsection .fixup,\"ax\" \n\t" \
387 "3: movb $1, %4 \n\t" \
390 _ASM_EXTABLE(1b, 3b) \
391 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
392 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
393 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
394 "a" (*rax), "d" (*rdx)); \
397 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
398 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
400 switch((ctxt)->src.bytes) { \
402 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
408 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
411 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
416 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
417 enum x86_intercept intercept,
418 enum x86_intercept_stage stage)
420 struct x86_instruction_info info = {
421 .intercept = intercept,
422 .rep_prefix = ctxt->rep_prefix,
423 .modrm_mod = ctxt->modrm_mod,
424 .modrm_reg = ctxt->modrm_reg,
425 .modrm_rm = ctxt->modrm_rm,
426 .src_val = ctxt->src.val64,
427 .src_bytes = ctxt->src.bytes,
428 .dst_bytes = ctxt->dst.bytes,
429 .ad_bytes = ctxt->ad_bytes,
430 .next_rip = ctxt->eip,
433 return ctxt->ops->intercept(ctxt, &info, stage);
436 static void assign_masked(ulong *dest, ulong src, ulong mask)
438 *dest = (*dest & ~mask) | (src & mask);
441 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
443 return (1UL << (ctxt->ad_bytes << 3)) - 1;
446 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
449 struct desc_struct ss;
451 if (ctxt->mode == X86EMUL_MODE_PROT64)
453 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
454 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
457 static int stack_size(struct x86_emulate_ctxt *ctxt)
459 return (__fls(stack_mask(ctxt)) + 1) >> 3;
462 /* Access/update address held in a register, based on addressing mode. */
463 static inline unsigned long
464 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
466 if (ctxt->ad_bytes == sizeof(unsigned long))
469 return reg & ad_mask(ctxt);
472 static inline unsigned long
473 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
475 return address_mask(ctxt, reg);
479 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
481 if (ctxt->ad_bytes == sizeof(unsigned long))
484 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
487 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
489 register_address_increment(ctxt, &ctxt->_eip, rel);
492 static u32 desc_limit_scaled(struct desc_struct *desc)
494 u32 limit = get_desc_limit(desc);
496 return desc->g ? (limit << 12) | 0xfff : limit;
499 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
501 ctxt->has_seg_override = true;
502 ctxt->seg_override = seg;
505 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
507 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
510 return ctxt->ops->get_cached_segment_base(ctxt, seg);
513 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
515 if (!ctxt->has_seg_override)
518 return ctxt->seg_override;
521 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
522 u32 error, bool valid)
524 ctxt->exception.vector = vec;
525 ctxt->exception.error_code = error;
526 ctxt->exception.error_code_valid = valid;
527 return X86EMUL_PROPAGATE_FAULT;
530 static int emulate_db(struct x86_emulate_ctxt *ctxt)
532 return emulate_exception(ctxt, DB_VECTOR, 0, false);
535 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
537 return emulate_exception(ctxt, GP_VECTOR, err, true);
540 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, SS_VECTOR, err, true);
545 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, UD_VECTOR, 0, false);
550 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
552 return emulate_exception(ctxt, TS_VECTOR, err, true);
555 static int emulate_de(struct x86_emulate_ctxt *ctxt)
557 return emulate_exception(ctxt, DE_VECTOR, 0, false);
560 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
562 return emulate_exception(ctxt, NM_VECTOR, 0, false);
565 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
568 struct desc_struct desc;
570 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
574 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
579 struct desc_struct desc;
581 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
582 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
586 * x86 defines three classes of vector instructions: explicitly
587 * aligned, explicitly unaligned, and the rest, which change behaviour
588 * depending on whether they're AVX encoded or not.
590 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
591 * subject to the same check.
593 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
595 if (likely(size < 16))
598 if (ctxt->d & Aligned)
600 else if (ctxt->d & Unaligned)
602 else if (ctxt->d & Avx)
608 static int __linearize(struct x86_emulate_ctxt *ctxt,
609 struct segmented_address addr,
610 unsigned size, bool write, bool fetch,
613 struct desc_struct desc;
620 la = seg_base(ctxt, addr.seg) + addr.ea;
621 switch (ctxt->mode) {
622 case X86EMUL_MODE_REAL:
624 case X86EMUL_MODE_PROT64:
625 if (((signed long)la << 16) >> 16 != la)
626 return emulate_gp(ctxt, 0);
629 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
633 /* code segment or read-only data segment */
634 if (((desc.type & 8) || !(desc.type & 2)) && write)
636 /* unreadable code segment */
637 if (!fetch && (desc.type & 8) && !(desc.type & 2))
639 lim = desc_limit_scaled(&desc);
640 if ((desc.type & 8) || !(desc.type & 4)) {
641 /* expand-up segment */
642 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
645 /* exapand-down segment */
646 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
648 lim = desc.d ? 0xffffffff : 0xffff;
649 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
652 cpl = ctxt->ops->cpl(ctxt);
655 if (!(desc.type & 8)) {
659 } else if ((desc.type & 8) && !(desc.type & 4)) {
660 /* nonconforming code segment */
663 } else if ((desc.type & 8) && (desc.type & 4)) {
664 /* conforming code segment */
670 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
672 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
673 return emulate_gp(ctxt, 0);
675 return X86EMUL_CONTINUE;
677 if (addr.seg == VCPU_SREG_SS)
678 return emulate_ss(ctxt, addr.seg);
680 return emulate_gp(ctxt, addr.seg);
683 static int linearize(struct x86_emulate_ctxt *ctxt,
684 struct segmented_address addr,
685 unsigned size, bool write,
688 return __linearize(ctxt, addr, size, write, false, linear);
692 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
693 struct segmented_address addr,
700 rc = linearize(ctxt, addr, size, false, &linear);
701 if (rc != X86EMUL_CONTINUE)
703 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
707 * Fetch the next byte of the instruction being emulated which is pointed to
708 * by ctxt->_eip, then increment ctxt->_eip.
710 * Also prefetch the remaining bytes of the instruction without crossing page
711 * boundary if they are not in fetch_cache yet.
713 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
715 struct fetch_cache *fc = &ctxt->fetch;
719 if (ctxt->_eip == fc->end) {
720 unsigned long linear;
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
723 cur_size = fc->end - fc->start;
724 size = min(15UL - cur_size,
725 PAGE_SIZE - offset_in_page(ctxt->_eip));
726 rc = __linearize(ctxt, addr, size, false, true, &linear);
727 if (unlikely(rc != X86EMUL_CONTINUE))
729 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
730 size, &ctxt->exception);
731 if (unlikely(rc != X86EMUL_CONTINUE))
735 *dest = fc->data[ctxt->_eip - fc->start];
737 return X86EMUL_CONTINUE;
740 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
741 void *dest, unsigned size)
745 /* x86 instructions are limited to 15 bytes. */
746 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
747 return X86EMUL_UNHANDLEABLE;
749 rc = do_insn_fetch_byte(ctxt, dest++);
750 if (rc != X86EMUL_CONTINUE)
753 return X86EMUL_CONTINUE;
756 /* Fetch next part of the instruction being emulated. */
757 #define insn_fetch(_type, _ctxt) \
758 ({ unsigned long _x; \
759 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
760 if (rc != X86EMUL_CONTINUE) \
765 #define insn_fetch_arr(_arr, _size, _ctxt) \
766 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
767 if (rc != X86EMUL_CONTINUE) \
772 * Given the 'reg' portion of a ModRM byte, and a register block, return a
773 * pointer into the block that addresses the relevant register.
774 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
776 static void *decode_register(u8 modrm_reg, unsigned long *regs,
781 p = ®s[modrm_reg];
782 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
783 p = (unsigned char *)®s[modrm_reg & 3] + 1;
787 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 u16 *size, unsigned long *address, int op_bytes)
796 rc = segmented_read_std(ctxt, addr, size, 2);
797 if (rc != X86EMUL_CONTINUE)
800 rc = segmented_read_std(ctxt, addr, address, op_bytes);
804 static int test_cc(unsigned int condition, unsigned int flags)
808 switch ((condition & 15) >> 1) {
810 rc |= (flags & EFLG_OF);
812 case 1: /* b/c/nae */
813 rc |= (flags & EFLG_CF);
816 rc |= (flags & EFLG_ZF);
819 rc |= (flags & (EFLG_CF|EFLG_ZF));
822 rc |= (flags & EFLG_SF);
825 rc |= (flags & EFLG_PF);
828 rc |= (flags & EFLG_ZF);
831 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
835 /* Odd condition identifiers (lsb == 1) have inverted sense. */
836 return (!!rc ^ (condition & 1));
839 static void fetch_register_operand(struct operand *op)
843 op->val = *(u8 *)op->addr.reg;
846 op->val = *(u16 *)op->addr.reg;
849 op->val = *(u32 *)op->addr.reg;
852 op->val = *(u64 *)op->addr.reg;
857 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
859 ctxt->ops->get_fpu(ctxt);
861 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
862 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
863 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
864 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
865 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
866 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
867 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
868 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
870 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
871 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
872 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
873 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
874 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
875 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
876 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
877 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
881 ctxt->ops->put_fpu(ctxt);
884 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
887 ctxt->ops->get_fpu(ctxt);
889 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
890 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
891 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
892 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
893 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
894 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
895 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
896 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
898 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
899 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
900 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
901 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
902 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
903 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
904 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
905 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
909 ctxt->ops->put_fpu(ctxt);
912 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
914 ctxt->ops->get_fpu(ctxt);
916 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
917 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
918 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
919 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
920 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
921 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
922 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
923 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
926 ctxt->ops->put_fpu(ctxt);
929 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
931 ctxt->ops->get_fpu(ctxt);
933 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
934 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
935 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
936 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
937 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
938 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
939 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
940 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
943 ctxt->ops->put_fpu(ctxt);
946 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
949 unsigned reg = ctxt->modrm_reg;
950 int highbyte_regs = ctxt->rex_prefix == 0;
952 if (!(ctxt->d & ModRM))
953 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
959 read_sse_reg(ctxt, &op->vec_val, reg);
971 if (ctxt->d & ByteOp) {
972 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
975 op->addr.reg = decode_register(reg, ctxt->regs, 0);
976 op->bytes = ctxt->op_bytes;
978 fetch_register_operand(op);
979 op->orig_val = op->val;
982 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
984 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
985 ctxt->modrm_seg = VCPU_SREG_SS;
988 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
992 int index_reg = 0, base_reg = 0, scale;
993 int rc = X86EMUL_CONTINUE;
996 if (ctxt->rex_prefix) {
997 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
998 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
999 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1002 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1003 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1004 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1005 ctxt->modrm_seg = VCPU_SREG_DS;
1007 if (ctxt->modrm_mod == 3) {
1009 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1010 op->addr.reg = decode_register(ctxt->modrm_rm,
1011 ctxt->regs, ctxt->d & ByteOp);
1012 if (ctxt->d & Sse) {
1015 op->addr.xmm = ctxt->modrm_rm;
1016 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1019 if (ctxt->d & Mmx) {
1022 op->addr.xmm = ctxt->modrm_rm & 7;
1025 fetch_register_operand(op);
1031 if (ctxt->ad_bytes == 2) {
1032 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
1033 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
1034 unsigned si = ctxt->regs[VCPU_REGS_RSI];
1035 unsigned di = ctxt->regs[VCPU_REGS_RDI];
1037 /* 16-bit ModR/M decode. */
1038 switch (ctxt->modrm_mod) {
1040 if (ctxt->modrm_rm == 6)
1041 modrm_ea += insn_fetch(u16, ctxt);
1044 modrm_ea += insn_fetch(s8, ctxt);
1047 modrm_ea += insn_fetch(u16, ctxt);
1050 switch (ctxt->modrm_rm) {
1052 modrm_ea += bx + si;
1055 modrm_ea += bx + di;
1058 modrm_ea += bp + si;
1061 modrm_ea += bp + di;
1070 if (ctxt->modrm_mod != 0)
1077 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1078 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1079 ctxt->modrm_seg = VCPU_SREG_SS;
1080 modrm_ea = (u16)modrm_ea;
1082 /* 32/64-bit ModR/M decode. */
1083 if ((ctxt->modrm_rm & 7) == 4) {
1084 sib = insn_fetch(u8, ctxt);
1085 index_reg |= (sib >> 3) & 7;
1086 base_reg |= sib & 7;
1089 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1090 modrm_ea += insn_fetch(s32, ctxt);
1092 modrm_ea += ctxt->regs[base_reg];
1093 adjust_modrm_seg(ctxt, base_reg);
1096 modrm_ea += ctxt->regs[index_reg] << scale;
1097 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1098 if (ctxt->mode == X86EMUL_MODE_PROT64)
1099 ctxt->rip_relative = 1;
1101 base_reg = ctxt->modrm_rm;
1102 modrm_ea += ctxt->regs[base_reg];
1103 adjust_modrm_seg(ctxt, base_reg);
1105 switch (ctxt->modrm_mod) {
1107 if (ctxt->modrm_rm == 5)
1108 modrm_ea += insn_fetch(s32, ctxt);
1111 modrm_ea += insn_fetch(s8, ctxt);
1114 modrm_ea += insn_fetch(s32, ctxt);
1118 op->addr.mem.ea = modrm_ea;
1123 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1126 int rc = X86EMUL_CONTINUE;
1129 switch (ctxt->ad_bytes) {
1131 op->addr.mem.ea = insn_fetch(u16, ctxt);
1134 op->addr.mem.ea = insn_fetch(u32, ctxt);
1137 op->addr.mem.ea = insn_fetch(u64, ctxt);
1144 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1148 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1149 mask = ~(ctxt->dst.bytes * 8 - 1);
1151 if (ctxt->src.bytes == 2)
1152 sv = (s16)ctxt->src.val & (s16)mask;
1153 else if (ctxt->src.bytes == 4)
1154 sv = (s32)ctxt->src.val & (s32)mask;
1156 ctxt->dst.addr.mem.ea += (sv >> 3);
1159 /* only subword offset */
1160 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1163 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1164 unsigned long addr, void *dest, unsigned size)
1167 struct read_cache *mc = &ctxt->mem_read;
1170 int n = min(size, 8u);
1172 if (mc->pos < mc->end)
1175 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1177 if (rc != X86EMUL_CONTINUE)
1182 memcpy(dest, mc->data + mc->pos, n);
1187 return X86EMUL_CONTINUE;
1190 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1191 struct segmented_address addr,
1198 rc = linearize(ctxt, addr, size, false, &linear);
1199 if (rc != X86EMUL_CONTINUE)
1201 return read_emulated(ctxt, linear, data, size);
1204 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1205 struct segmented_address addr,
1212 rc = linearize(ctxt, addr, size, true, &linear);
1213 if (rc != X86EMUL_CONTINUE)
1215 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1219 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1220 struct segmented_address addr,
1221 const void *orig_data, const void *data,
1227 rc = linearize(ctxt, addr, size, true, &linear);
1228 if (rc != X86EMUL_CONTINUE)
1230 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1231 size, &ctxt->exception);
1234 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1235 unsigned int size, unsigned short port,
1238 struct read_cache *rc = &ctxt->io_read;
1240 if (rc->pos == rc->end) { /* refill pio read ahead */
1241 unsigned int in_page, n;
1242 unsigned int count = ctxt->rep_prefix ?
1243 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1244 in_page = (ctxt->eflags & EFLG_DF) ?
1245 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1246 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1247 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1251 rc->pos = rc->end = 0;
1252 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1257 memcpy(dest, rc->data + rc->pos, size);
1262 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1263 u16 index, struct desc_struct *desc)
1268 ctxt->ops->get_idt(ctxt, &dt);
1270 if (dt.size < index * 8 + 7)
1271 return emulate_gp(ctxt, index << 3 | 0x2);
1273 addr = dt.address + index * 8;
1274 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1278 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1279 u16 selector, struct desc_ptr *dt)
1281 struct x86_emulate_ops *ops = ctxt->ops;
1283 if (selector & 1 << 2) {
1284 struct desc_struct desc;
1287 memset (dt, 0, sizeof *dt);
1288 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1291 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1292 dt->address = get_desc_base(&desc);
1294 ops->get_gdt(ctxt, dt);
1297 /* allowed just for 8 bytes segments */
1298 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1299 u16 selector, struct desc_struct *desc,
1303 u16 index = selector >> 3;
1306 get_descriptor_table_ptr(ctxt, selector, &dt);
1308 if (dt.size < index * 8 + 7)
1309 return emulate_gp(ctxt, selector & 0xfffc);
1311 *desc_addr_p = addr = dt.address + index * 8;
1312 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1316 /* allowed just for 8 bytes segments */
1317 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1318 u16 selector, struct desc_struct *desc)
1321 u16 index = selector >> 3;
1324 get_descriptor_table_ptr(ctxt, selector, &dt);
1326 if (dt.size < index * 8 + 7)
1327 return emulate_gp(ctxt, selector & 0xfffc);
1329 addr = dt.address + index * 8;
1330 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1334 /* Does not support long mode */
1335 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1336 u16 selector, int seg)
1338 struct desc_struct seg_desc, old_desc;
1340 unsigned err_vec = GP_VECTOR;
1342 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1346 memset(&seg_desc, 0, sizeof seg_desc);
1348 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1349 || ctxt->mode == X86EMUL_MODE_REAL) {
1350 /* set real mode segment descriptor */
1351 set_desc_base(&seg_desc, selector << 4);
1352 set_desc_limit(&seg_desc, 0xffff);
1356 if (ctxt->mode == X86EMUL_MODE_VM86)
1362 cpl = ctxt->ops->cpl(ctxt);
1364 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1365 if ((seg == VCPU_SREG_CS
1366 || (seg == VCPU_SREG_SS
1367 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1368 || seg == VCPU_SREG_TR)
1372 /* TR should be in GDT only */
1373 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1376 if (null_selector) /* for NULL selector skip all following checks */
1379 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1380 if (ret != X86EMUL_CONTINUE)
1383 err_code = selector & 0xfffc;
1384 err_vec = GP_VECTOR;
1386 /* can't load system descriptor into segment selecor */
1387 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1391 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1400 * segment is not a writable data segment or segment
1401 * selector's RPL != CPL or segment selector's RPL != CPL
1403 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1407 if (!(seg_desc.type & 8))
1410 if (seg_desc.type & 4) {
1416 if (rpl > cpl || dpl != cpl)
1419 /* CS(RPL) <- CPL */
1420 selector = (selector & 0xfffc) | cpl;
1423 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1425 old_desc = seg_desc;
1426 seg_desc.type |= 2; /* busy */
1427 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1428 sizeof(seg_desc), &ctxt->exception);
1429 if (ret != X86EMUL_CONTINUE)
1432 case VCPU_SREG_LDTR:
1433 if (seg_desc.s || seg_desc.type != 2)
1436 default: /* DS, ES, FS, or GS */
1438 * segment is not a data or readable code segment or
1439 * ((segment is a data or nonconforming code segment)
1440 * and (both RPL and CPL > DPL))
1442 if ((seg_desc.type & 0xa) == 0x8 ||
1443 (((seg_desc.type & 0xc) != 0xc) &&
1444 (rpl > dpl && cpl > dpl)))
1450 /* mark segment as accessed */
1452 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1453 if (ret != X86EMUL_CONTINUE)
1457 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1458 return X86EMUL_CONTINUE;
1460 emulate_exception(ctxt, err_vec, err_code, true);
1461 return X86EMUL_PROPAGATE_FAULT;
1464 static void write_register_operand(struct operand *op)
1466 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1467 switch (op->bytes) {
1469 *(u8 *)op->addr.reg = (u8)op->val;
1472 *(u16 *)op->addr.reg = (u16)op->val;
1475 *op->addr.reg = (u32)op->val;
1476 break; /* 64b: zero-extend */
1478 *op->addr.reg = op->val;
1483 static int writeback(struct x86_emulate_ctxt *ctxt)
1487 switch (ctxt->dst.type) {
1489 write_register_operand(&ctxt->dst);
1492 if (ctxt->lock_prefix)
1493 rc = segmented_cmpxchg(ctxt,
1495 &ctxt->dst.orig_val,
1499 rc = segmented_write(ctxt,
1503 if (rc != X86EMUL_CONTINUE)
1507 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1510 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1518 return X86EMUL_CONTINUE;
1521 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1523 struct segmented_address addr;
1525 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -bytes);
1526 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1527 addr.seg = VCPU_SREG_SS;
1529 return segmented_write(ctxt, addr, data, bytes);
1532 static int em_push(struct x86_emulate_ctxt *ctxt)
1534 /* Disable writeback. */
1535 ctxt->dst.type = OP_NONE;
1536 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1539 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1540 void *dest, int len)
1543 struct segmented_address addr;
1545 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1546 addr.seg = VCPU_SREG_SS;
1547 rc = segmented_read(ctxt, addr, dest, len);
1548 if (rc != X86EMUL_CONTINUE)
1551 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1555 static int em_pop(struct x86_emulate_ctxt *ctxt)
1557 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1560 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1561 void *dest, int len)
1564 unsigned long val, change_mask;
1565 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1566 int cpl = ctxt->ops->cpl(ctxt);
1568 rc = emulate_pop(ctxt, &val, len);
1569 if (rc != X86EMUL_CONTINUE)
1572 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1573 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1575 switch(ctxt->mode) {
1576 case X86EMUL_MODE_PROT64:
1577 case X86EMUL_MODE_PROT32:
1578 case X86EMUL_MODE_PROT16:
1580 change_mask |= EFLG_IOPL;
1582 change_mask |= EFLG_IF;
1584 case X86EMUL_MODE_VM86:
1586 return emulate_gp(ctxt, 0);
1587 change_mask |= EFLG_IF;
1589 default: /* real mode */
1590 change_mask |= (EFLG_IOPL | EFLG_IF);
1594 *(unsigned long *)dest =
1595 (ctxt->eflags & ~change_mask) | (val & change_mask);
1600 static int em_popf(struct x86_emulate_ctxt *ctxt)
1602 ctxt->dst.type = OP_REG;
1603 ctxt->dst.addr.reg = &ctxt->eflags;
1604 ctxt->dst.bytes = ctxt->op_bytes;
1605 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1608 static int em_enter(struct x86_emulate_ctxt *ctxt)
1611 unsigned frame_size = ctxt->src.val;
1612 unsigned nesting_level = ctxt->src2.val & 31;
1615 return X86EMUL_UNHANDLEABLE;
1617 rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt));
1618 if (rc != X86EMUL_CONTINUE)
1620 assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP],
1622 assign_masked(&ctxt->regs[VCPU_REGS_RSP],
1623 ctxt->regs[VCPU_REGS_RSP] - frame_size,
1625 return X86EMUL_CONTINUE;
1628 static int em_leave(struct x86_emulate_ctxt *ctxt)
1630 assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP],
1632 return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes);
1635 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1637 int seg = ctxt->src2.val;
1639 ctxt->src.val = get_segment_selector(ctxt, seg);
1641 return em_push(ctxt);
1644 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1646 int seg = ctxt->src2.val;
1647 unsigned long selector;
1650 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1651 if (rc != X86EMUL_CONTINUE)
1654 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1658 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1660 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1661 int rc = X86EMUL_CONTINUE;
1662 int reg = VCPU_REGS_RAX;
1664 while (reg <= VCPU_REGS_RDI) {
1665 (reg == VCPU_REGS_RSP) ?
1666 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1669 if (rc != X86EMUL_CONTINUE)
1678 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1680 ctxt->src.val = (unsigned long)ctxt->eflags;
1681 return em_push(ctxt);
1684 static int em_popa(struct x86_emulate_ctxt *ctxt)
1686 int rc = X86EMUL_CONTINUE;
1687 int reg = VCPU_REGS_RDI;
1689 while (reg >= VCPU_REGS_RAX) {
1690 if (reg == VCPU_REGS_RSP) {
1691 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1696 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1697 if (rc != X86EMUL_CONTINUE)
1704 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1706 struct x86_emulate_ops *ops = ctxt->ops;
1713 /* TODO: Add limit checks */
1714 ctxt->src.val = ctxt->eflags;
1716 if (rc != X86EMUL_CONTINUE)
1719 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1721 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1723 if (rc != X86EMUL_CONTINUE)
1726 ctxt->src.val = ctxt->_eip;
1728 if (rc != X86EMUL_CONTINUE)
1731 ops->get_idt(ctxt, &dt);
1733 eip_addr = dt.address + (irq << 2);
1734 cs_addr = dt.address + (irq << 2) + 2;
1736 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1737 if (rc != X86EMUL_CONTINUE)
1740 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1741 if (rc != X86EMUL_CONTINUE)
1744 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1745 if (rc != X86EMUL_CONTINUE)
1753 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1755 switch(ctxt->mode) {
1756 case X86EMUL_MODE_REAL:
1757 return emulate_int_real(ctxt, irq);
1758 case X86EMUL_MODE_VM86:
1759 case X86EMUL_MODE_PROT16:
1760 case X86EMUL_MODE_PROT32:
1761 case X86EMUL_MODE_PROT64:
1763 /* Protected mode interrupts unimplemented yet */
1764 return X86EMUL_UNHANDLEABLE;
1768 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1770 int rc = X86EMUL_CONTINUE;
1771 unsigned long temp_eip = 0;
1772 unsigned long temp_eflags = 0;
1773 unsigned long cs = 0;
1774 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1775 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1776 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1777 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1779 /* TODO: Add stack limit check */
1781 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1783 if (rc != X86EMUL_CONTINUE)
1786 if (temp_eip & ~0xffff)
1787 return emulate_gp(ctxt, 0);
1789 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1791 if (rc != X86EMUL_CONTINUE)
1794 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1796 if (rc != X86EMUL_CONTINUE)
1799 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1801 if (rc != X86EMUL_CONTINUE)
1804 ctxt->_eip = temp_eip;
1807 if (ctxt->op_bytes == 4)
1808 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1809 else if (ctxt->op_bytes == 2) {
1810 ctxt->eflags &= ~0xffff;
1811 ctxt->eflags |= temp_eflags;
1814 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1815 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1820 static int em_iret(struct x86_emulate_ctxt *ctxt)
1822 switch(ctxt->mode) {
1823 case X86EMUL_MODE_REAL:
1824 return emulate_iret_real(ctxt);
1825 case X86EMUL_MODE_VM86:
1826 case X86EMUL_MODE_PROT16:
1827 case X86EMUL_MODE_PROT32:
1828 case X86EMUL_MODE_PROT64:
1830 /* iret from protected mode unimplemented yet */
1831 return X86EMUL_UNHANDLEABLE;
1835 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1840 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1842 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1843 if (rc != X86EMUL_CONTINUE)
1847 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1848 return X86EMUL_CONTINUE;
1851 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1853 switch (ctxt->modrm_reg) {
1855 emulate_2op_SrcB(ctxt, "rol");
1858 emulate_2op_SrcB(ctxt, "ror");
1861 emulate_2op_SrcB(ctxt, "rcl");
1864 emulate_2op_SrcB(ctxt, "rcr");
1866 case 4: /* sal/shl */
1867 case 6: /* sal/shl */
1868 emulate_2op_SrcB(ctxt, "sal");
1871 emulate_2op_SrcB(ctxt, "shr");
1874 emulate_2op_SrcB(ctxt, "sar");
1877 return X86EMUL_CONTINUE;
1880 static int em_not(struct x86_emulate_ctxt *ctxt)
1882 ctxt->dst.val = ~ctxt->dst.val;
1883 return X86EMUL_CONTINUE;
1886 static int em_neg(struct x86_emulate_ctxt *ctxt)
1888 emulate_1op(ctxt, "neg");
1889 return X86EMUL_CONTINUE;
1892 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1896 emulate_1op_rax_rdx(ctxt, "mul", ex);
1897 return X86EMUL_CONTINUE;
1900 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1904 emulate_1op_rax_rdx(ctxt, "imul", ex);
1905 return X86EMUL_CONTINUE;
1908 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1912 emulate_1op_rax_rdx(ctxt, "div", de);
1914 return emulate_de(ctxt);
1915 return X86EMUL_CONTINUE;
1918 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1922 emulate_1op_rax_rdx(ctxt, "idiv", de);
1924 return emulate_de(ctxt);
1925 return X86EMUL_CONTINUE;
1928 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1930 int rc = X86EMUL_CONTINUE;
1932 switch (ctxt->modrm_reg) {
1934 emulate_1op(ctxt, "inc");
1937 emulate_1op(ctxt, "dec");
1939 case 2: /* call near abs */ {
1941 old_eip = ctxt->_eip;
1942 ctxt->_eip = ctxt->src.val;
1943 ctxt->src.val = old_eip;
1947 case 4: /* jmp abs */
1948 ctxt->_eip = ctxt->src.val;
1950 case 5: /* jmp far */
1951 rc = em_jmp_far(ctxt);
1960 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1962 u64 old = ctxt->dst.orig_val64;
1964 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1965 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1966 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1967 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1968 ctxt->eflags &= ~EFLG_ZF;
1970 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1971 (u32) ctxt->regs[VCPU_REGS_RBX];
1973 ctxt->eflags |= EFLG_ZF;
1975 return X86EMUL_CONTINUE;
1978 static int em_ret(struct x86_emulate_ctxt *ctxt)
1980 ctxt->dst.type = OP_REG;
1981 ctxt->dst.addr.reg = &ctxt->_eip;
1982 ctxt->dst.bytes = ctxt->op_bytes;
1983 return em_pop(ctxt);
1986 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1991 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1992 if (rc != X86EMUL_CONTINUE)
1994 if (ctxt->op_bytes == 4)
1995 ctxt->_eip = (u32)ctxt->_eip;
1996 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1997 if (rc != X86EMUL_CONTINUE)
1999 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2003 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2005 /* Save real source value, then compare EAX against destination. */
2006 ctxt->src.orig_val = ctxt->src.val;
2007 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
2008 emulate_2op_SrcV(ctxt, "cmp");
2010 if (ctxt->eflags & EFLG_ZF) {
2011 /* Success: write back to memory. */
2012 ctxt->dst.val = ctxt->src.orig_val;
2014 /* Failure: write the value we saw to EAX. */
2015 ctxt->dst.type = OP_REG;
2016 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
2018 return X86EMUL_CONTINUE;
2021 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2023 int seg = ctxt->src2.val;
2027 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2029 rc = load_segment_descriptor(ctxt, sel, seg);
2030 if (rc != X86EMUL_CONTINUE)
2033 ctxt->dst.val = ctxt->src.val;
2038 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2039 struct desc_struct *cs, struct desc_struct *ss)
2043 memset(cs, 0, sizeof(struct desc_struct));
2044 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
2045 memset(ss, 0, sizeof(struct desc_struct));
2047 cs->l = 0; /* will be adjusted later */
2048 set_desc_base(cs, 0); /* flat segment */
2049 cs->g = 1; /* 4kb granularity */
2050 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2051 cs->type = 0x0b; /* Read, Execute, Accessed */
2053 cs->dpl = 0; /* will be adjusted later */
2057 set_desc_base(ss, 0); /* flat segment */
2058 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2059 ss->g = 1; /* 4kb granularity */
2061 ss->type = 0x03; /* Read/Write, Accessed */
2062 ss->d = 1; /* 32bit stack segment */
2067 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2069 u32 eax, ebx, ecx, edx;
2072 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2073 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2074 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2075 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2078 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2080 struct x86_emulate_ops *ops = ctxt->ops;
2081 u32 eax, ebx, ecx, edx;
2084 * syscall should always be enabled in longmode - so only become
2085 * vendor specific (cpuid) if other modes are active...
2087 if (ctxt->mode == X86EMUL_MODE_PROT64)
2092 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2094 * Intel ("GenuineIntel")
2095 * remark: Intel CPUs only support "syscall" in 64bit
2096 * longmode. Also an 64bit guest with a
2097 * 32bit compat-app running will #UD !! While this
2098 * behaviour can be fixed (by emulating) into AMD
2099 * response - CPUs of AMD can't behave like Intel.
2101 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2102 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2103 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2106 /* AMD ("AuthenticAMD") */
2107 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2108 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2109 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2112 /* AMD ("AMDisbetter!") */
2113 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2114 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2115 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2118 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2122 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2124 struct x86_emulate_ops *ops = ctxt->ops;
2125 struct desc_struct cs, ss;
2130 /* syscall is not available in real mode */
2131 if (ctxt->mode == X86EMUL_MODE_REAL ||
2132 ctxt->mode == X86EMUL_MODE_VM86)
2133 return emulate_ud(ctxt);
2135 if (!(em_syscall_is_enabled(ctxt)))
2136 return emulate_ud(ctxt);
2138 ops->get_msr(ctxt, MSR_EFER, &efer);
2139 setup_syscalls_segments(ctxt, &cs, &ss);
2141 if (!(efer & EFER_SCE))
2142 return emulate_ud(ctxt);
2144 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2146 cs_sel = (u16)(msr_data & 0xfffc);
2147 ss_sel = (u16)(msr_data + 8);
2149 if (efer & EFER_LMA) {
2153 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2154 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2156 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
2157 if (efer & EFER_LMA) {
2158 #ifdef CONFIG_X86_64
2159 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2162 ctxt->mode == X86EMUL_MODE_PROT64 ?
2163 MSR_LSTAR : MSR_CSTAR, &msr_data);
2164 ctxt->_eip = msr_data;
2166 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2167 ctxt->eflags &= ~(msr_data | EFLG_RF);
2171 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2172 ctxt->_eip = (u32)msr_data;
2174 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2177 return X86EMUL_CONTINUE;
2180 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2182 struct x86_emulate_ops *ops = ctxt->ops;
2183 struct desc_struct cs, ss;
2188 ops->get_msr(ctxt, MSR_EFER, &efer);
2189 /* inject #GP if in real mode */
2190 if (ctxt->mode == X86EMUL_MODE_REAL)
2191 return emulate_gp(ctxt, 0);
2194 * Not recognized on AMD in compat mode (but is recognized in legacy
2197 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2198 && !vendor_intel(ctxt))
2199 return emulate_ud(ctxt);
2201 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2202 * Therefore, we inject an #UD.
2204 if (ctxt->mode == X86EMUL_MODE_PROT64)
2205 return emulate_ud(ctxt);
2207 setup_syscalls_segments(ctxt, &cs, &ss);
2209 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2210 switch (ctxt->mode) {
2211 case X86EMUL_MODE_PROT32:
2212 if ((msr_data & 0xfffc) == 0x0)
2213 return emulate_gp(ctxt, 0);
2215 case X86EMUL_MODE_PROT64:
2216 if (msr_data == 0x0)
2217 return emulate_gp(ctxt, 0);
2221 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2222 cs_sel = (u16)msr_data;
2223 cs_sel &= ~SELECTOR_RPL_MASK;
2224 ss_sel = cs_sel + 8;
2225 ss_sel &= ~SELECTOR_RPL_MASK;
2226 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2231 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2232 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2234 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2235 ctxt->_eip = msr_data;
2237 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2238 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2240 return X86EMUL_CONTINUE;
2243 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2245 struct x86_emulate_ops *ops = ctxt->ops;
2246 struct desc_struct cs, ss;
2249 u16 cs_sel = 0, ss_sel = 0;
2251 /* inject #GP if in real mode or Virtual 8086 mode */
2252 if (ctxt->mode == X86EMUL_MODE_REAL ||
2253 ctxt->mode == X86EMUL_MODE_VM86)
2254 return emulate_gp(ctxt, 0);
2256 setup_syscalls_segments(ctxt, &cs, &ss);
2258 if ((ctxt->rex_prefix & 0x8) != 0x0)
2259 usermode = X86EMUL_MODE_PROT64;
2261 usermode = X86EMUL_MODE_PROT32;
2265 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2267 case X86EMUL_MODE_PROT32:
2268 cs_sel = (u16)(msr_data + 16);
2269 if ((msr_data & 0xfffc) == 0x0)
2270 return emulate_gp(ctxt, 0);
2271 ss_sel = (u16)(msr_data + 24);
2273 case X86EMUL_MODE_PROT64:
2274 cs_sel = (u16)(msr_data + 32);
2275 if (msr_data == 0x0)
2276 return emulate_gp(ctxt, 0);
2277 ss_sel = cs_sel + 8;
2282 cs_sel |= SELECTOR_RPL_MASK;
2283 ss_sel |= SELECTOR_RPL_MASK;
2285 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2286 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2288 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2289 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2291 return X86EMUL_CONTINUE;
2294 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2297 if (ctxt->mode == X86EMUL_MODE_REAL)
2299 if (ctxt->mode == X86EMUL_MODE_VM86)
2301 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2302 return ctxt->ops->cpl(ctxt) > iopl;
2305 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2308 struct x86_emulate_ops *ops = ctxt->ops;
2309 struct desc_struct tr_seg;
2312 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2313 unsigned mask = (1 << len) - 1;
2316 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2319 if (desc_limit_scaled(&tr_seg) < 103)
2321 base = get_desc_base(&tr_seg);
2322 #ifdef CONFIG_X86_64
2323 base |= ((u64)base3) << 32;
2325 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2326 if (r != X86EMUL_CONTINUE)
2328 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2330 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2331 if (r != X86EMUL_CONTINUE)
2333 if ((perm >> bit_idx) & mask)
2338 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2344 if (emulator_bad_iopl(ctxt))
2345 if (!emulator_io_port_access_allowed(ctxt, port, len))
2348 ctxt->perm_ok = true;
2353 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2354 struct tss_segment_16 *tss)
2356 tss->ip = ctxt->_eip;
2357 tss->flag = ctxt->eflags;
2358 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2359 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2360 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2361 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2362 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2363 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2364 tss->si = ctxt->regs[VCPU_REGS_RSI];
2365 tss->di = ctxt->regs[VCPU_REGS_RDI];
2367 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2368 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2369 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2370 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2371 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2374 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2375 struct tss_segment_16 *tss)
2379 ctxt->_eip = tss->ip;
2380 ctxt->eflags = tss->flag | 2;
2381 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2382 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2383 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2384 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2385 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2386 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2387 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2388 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2391 * SDM says that segment selectors are loaded before segment
2394 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2395 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2396 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2397 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2398 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2401 * Now load segment descriptors. If fault happenes at this stage
2402 * it is handled in a context of new task
2404 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2405 if (ret != X86EMUL_CONTINUE)
2407 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2408 if (ret != X86EMUL_CONTINUE)
2410 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2411 if (ret != X86EMUL_CONTINUE)
2413 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2414 if (ret != X86EMUL_CONTINUE)
2416 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2417 if (ret != X86EMUL_CONTINUE)
2420 return X86EMUL_CONTINUE;
2423 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2424 u16 tss_selector, u16 old_tss_sel,
2425 ulong old_tss_base, struct desc_struct *new_desc)
2427 struct x86_emulate_ops *ops = ctxt->ops;
2428 struct tss_segment_16 tss_seg;
2430 u32 new_tss_base = get_desc_base(new_desc);
2432 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2434 if (ret != X86EMUL_CONTINUE)
2435 /* FIXME: need to provide precise fault address */
2438 save_state_to_tss16(ctxt, &tss_seg);
2440 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2442 if (ret != X86EMUL_CONTINUE)
2443 /* FIXME: need to provide precise fault address */
2446 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2448 if (ret != X86EMUL_CONTINUE)
2449 /* FIXME: need to provide precise fault address */
2452 if (old_tss_sel != 0xffff) {
2453 tss_seg.prev_task_link = old_tss_sel;
2455 ret = ops->write_std(ctxt, new_tss_base,
2456 &tss_seg.prev_task_link,
2457 sizeof tss_seg.prev_task_link,
2459 if (ret != X86EMUL_CONTINUE)
2460 /* FIXME: need to provide precise fault address */
2464 return load_state_from_tss16(ctxt, &tss_seg);
2467 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2468 struct tss_segment_32 *tss)
2470 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2471 tss->eip = ctxt->_eip;
2472 tss->eflags = ctxt->eflags;
2473 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2474 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2475 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2476 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2477 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2478 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2479 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2480 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2482 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2483 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2484 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2485 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2486 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2487 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2488 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2491 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2492 struct tss_segment_32 *tss)
2496 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2497 return emulate_gp(ctxt, 0);
2498 ctxt->_eip = tss->eip;
2499 ctxt->eflags = tss->eflags | 2;
2501 /* General purpose registers */
2502 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2503 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2504 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2505 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2506 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2507 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2508 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2509 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2512 * SDM says that segment selectors are loaded before segment
2515 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2516 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2517 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2518 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2519 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2520 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2521 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2524 * If we're switching between Protected Mode and VM86, we need to make
2525 * sure to update the mode before loading the segment descriptors so
2526 * that the selectors are interpreted correctly.
2528 * Need to get rflags to the vcpu struct immediately because it
2529 * influences the CPL which is checked at least when loading the segment
2530 * descriptors and when pushing an error code to the new kernel stack.
2532 * TODO Introduce a separate ctxt->ops->set_cpl callback
2534 if (ctxt->eflags & X86_EFLAGS_VM)
2535 ctxt->mode = X86EMUL_MODE_VM86;
2537 ctxt->mode = X86EMUL_MODE_PROT32;
2539 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2542 * Now load segment descriptors. If fault happenes at this stage
2543 * it is handled in a context of new task
2545 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2546 if (ret != X86EMUL_CONTINUE)
2548 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2549 if (ret != X86EMUL_CONTINUE)
2551 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2552 if (ret != X86EMUL_CONTINUE)
2554 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2555 if (ret != X86EMUL_CONTINUE)
2557 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2558 if (ret != X86EMUL_CONTINUE)
2560 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2561 if (ret != X86EMUL_CONTINUE)
2563 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2564 if (ret != X86EMUL_CONTINUE)
2567 return X86EMUL_CONTINUE;
2570 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2571 u16 tss_selector, u16 old_tss_sel,
2572 ulong old_tss_base, struct desc_struct *new_desc)
2574 struct x86_emulate_ops *ops = ctxt->ops;
2575 struct tss_segment_32 tss_seg;
2577 u32 new_tss_base = get_desc_base(new_desc);
2579 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2581 if (ret != X86EMUL_CONTINUE)
2582 /* FIXME: need to provide precise fault address */
2585 save_state_to_tss32(ctxt, &tss_seg);
2587 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2589 if (ret != X86EMUL_CONTINUE)
2590 /* FIXME: need to provide precise fault address */
2593 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2595 if (ret != X86EMUL_CONTINUE)
2596 /* FIXME: need to provide precise fault address */
2599 if (old_tss_sel != 0xffff) {
2600 tss_seg.prev_task_link = old_tss_sel;
2602 ret = ops->write_std(ctxt, new_tss_base,
2603 &tss_seg.prev_task_link,
2604 sizeof tss_seg.prev_task_link,
2606 if (ret != X86EMUL_CONTINUE)
2607 /* FIXME: need to provide precise fault address */
2611 return load_state_from_tss32(ctxt, &tss_seg);
2614 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2615 u16 tss_selector, int idt_index, int reason,
2616 bool has_error_code, u32 error_code)
2618 struct x86_emulate_ops *ops = ctxt->ops;
2619 struct desc_struct curr_tss_desc, next_tss_desc;
2621 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2622 ulong old_tss_base =
2623 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2627 /* FIXME: old_tss_base == ~0 ? */
2629 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2630 if (ret != X86EMUL_CONTINUE)
2632 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2633 if (ret != X86EMUL_CONTINUE)
2636 /* FIXME: check that next_tss_desc is tss */
2639 * Check privileges. The three cases are task switch caused by...
2641 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2642 * 2. Exception/IRQ/iret: No check is performed
2643 * 3. jmp/call to TSS: Check agains DPL of the TSS
2645 if (reason == TASK_SWITCH_GATE) {
2646 if (idt_index != -1) {
2647 /* Software interrupts */
2648 struct desc_struct task_gate_desc;
2651 ret = read_interrupt_descriptor(ctxt, idt_index,
2653 if (ret != X86EMUL_CONTINUE)
2656 dpl = task_gate_desc.dpl;
2657 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2658 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2660 } else if (reason != TASK_SWITCH_IRET) {
2661 int dpl = next_tss_desc.dpl;
2662 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2663 return emulate_gp(ctxt, tss_selector);
2667 desc_limit = desc_limit_scaled(&next_tss_desc);
2668 if (!next_tss_desc.p ||
2669 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2670 desc_limit < 0x2b)) {
2671 emulate_ts(ctxt, tss_selector & 0xfffc);
2672 return X86EMUL_PROPAGATE_FAULT;
2675 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2676 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2677 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2680 if (reason == TASK_SWITCH_IRET)
2681 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2683 /* set back link to prev task only if NT bit is set in eflags
2684 note that old_tss_sel is not used afetr this point */
2685 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2686 old_tss_sel = 0xffff;
2688 if (next_tss_desc.type & 8)
2689 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2690 old_tss_base, &next_tss_desc);
2692 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2693 old_tss_base, &next_tss_desc);
2694 if (ret != X86EMUL_CONTINUE)
2697 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2698 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2700 if (reason != TASK_SWITCH_IRET) {
2701 next_tss_desc.type |= (1 << 1); /* set busy flag */
2702 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2705 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2706 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2708 if (has_error_code) {
2709 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2710 ctxt->lock_prefix = 0;
2711 ctxt->src.val = (unsigned long) error_code;
2712 ret = em_push(ctxt);
2718 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2719 u16 tss_selector, int idt_index, int reason,
2720 bool has_error_code, u32 error_code)
2724 ctxt->_eip = ctxt->eip;
2725 ctxt->dst.type = OP_NONE;
2727 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2728 has_error_code, error_code);
2730 if (rc == X86EMUL_CONTINUE)
2731 ctxt->eip = ctxt->_eip;
2733 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2736 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2737 int reg, struct operand *op)
2739 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2741 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2742 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2743 op->addr.mem.seg = seg;
2746 static int em_das(struct x86_emulate_ctxt *ctxt)
2749 bool af, cf, old_cf;
2751 cf = ctxt->eflags & X86_EFLAGS_CF;
2757 af = ctxt->eflags & X86_EFLAGS_AF;
2758 if ((al & 0x0f) > 9 || af) {
2760 cf = old_cf | (al >= 250);
2765 if (old_al > 0x99 || old_cf) {
2771 /* Set PF, ZF, SF */
2772 ctxt->src.type = OP_IMM;
2774 ctxt->src.bytes = 1;
2775 emulate_2op_SrcV(ctxt, "or");
2776 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2778 ctxt->eflags |= X86_EFLAGS_CF;
2780 ctxt->eflags |= X86_EFLAGS_AF;
2781 return X86EMUL_CONTINUE;
2784 static int em_call(struct x86_emulate_ctxt *ctxt)
2786 long rel = ctxt->src.val;
2788 ctxt->src.val = (unsigned long)ctxt->_eip;
2790 return em_push(ctxt);
2793 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2799 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2800 old_eip = ctxt->_eip;
2802 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2803 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2804 return X86EMUL_CONTINUE;
2807 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2809 ctxt->src.val = old_cs;
2811 if (rc != X86EMUL_CONTINUE)
2814 ctxt->src.val = old_eip;
2815 return em_push(ctxt);
2818 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2822 ctxt->dst.type = OP_REG;
2823 ctxt->dst.addr.reg = &ctxt->_eip;
2824 ctxt->dst.bytes = ctxt->op_bytes;
2825 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2826 if (rc != X86EMUL_CONTINUE)
2828 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2829 return X86EMUL_CONTINUE;
2832 static int em_add(struct x86_emulate_ctxt *ctxt)
2834 emulate_2op_SrcV(ctxt, "add");
2835 return X86EMUL_CONTINUE;
2838 static int em_or(struct x86_emulate_ctxt *ctxt)
2840 emulate_2op_SrcV(ctxt, "or");
2841 return X86EMUL_CONTINUE;
2844 static int em_adc(struct x86_emulate_ctxt *ctxt)
2846 emulate_2op_SrcV(ctxt, "adc");
2847 return X86EMUL_CONTINUE;
2850 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2852 emulate_2op_SrcV(ctxt, "sbb");
2853 return X86EMUL_CONTINUE;
2856 static int em_and(struct x86_emulate_ctxt *ctxt)
2858 emulate_2op_SrcV(ctxt, "and");
2859 return X86EMUL_CONTINUE;
2862 static int em_sub(struct x86_emulate_ctxt *ctxt)
2864 emulate_2op_SrcV(ctxt, "sub");
2865 return X86EMUL_CONTINUE;
2868 static int em_xor(struct x86_emulate_ctxt *ctxt)
2870 emulate_2op_SrcV(ctxt, "xor");
2871 return X86EMUL_CONTINUE;
2874 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2876 emulate_2op_SrcV(ctxt, "cmp");
2877 /* Disable writeback. */
2878 ctxt->dst.type = OP_NONE;
2879 return X86EMUL_CONTINUE;
2882 static int em_test(struct x86_emulate_ctxt *ctxt)
2884 emulate_2op_SrcV(ctxt, "test");
2885 /* Disable writeback. */
2886 ctxt->dst.type = OP_NONE;
2887 return X86EMUL_CONTINUE;
2890 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2892 /* Write back the register source. */
2893 ctxt->src.val = ctxt->dst.val;
2894 write_register_operand(&ctxt->src);
2896 /* Write back the memory destination with implicit LOCK prefix. */
2897 ctxt->dst.val = ctxt->src.orig_val;
2898 ctxt->lock_prefix = 1;
2899 return X86EMUL_CONTINUE;
2902 static int em_imul(struct x86_emulate_ctxt *ctxt)
2904 emulate_2op_SrcV_nobyte(ctxt, "imul");
2905 return X86EMUL_CONTINUE;
2908 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2910 ctxt->dst.val = ctxt->src2.val;
2911 return em_imul(ctxt);
2914 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2916 ctxt->dst.type = OP_REG;
2917 ctxt->dst.bytes = ctxt->src.bytes;
2918 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2919 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2921 return X86EMUL_CONTINUE;
2924 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2928 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2929 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2930 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2931 return X86EMUL_CONTINUE;
2934 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2938 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2939 return emulate_gp(ctxt, 0);
2940 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2941 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2942 return X86EMUL_CONTINUE;
2945 static int em_mov(struct x86_emulate_ctxt *ctxt)
2947 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2948 return X86EMUL_CONTINUE;
2951 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2953 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2954 return emulate_gp(ctxt, 0);
2956 /* Disable writeback. */
2957 ctxt->dst.type = OP_NONE;
2958 return X86EMUL_CONTINUE;
2961 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2965 if (ctxt->mode == X86EMUL_MODE_PROT64)
2966 val = ctxt->src.val & ~0ULL;
2968 val = ctxt->src.val & ~0U;
2970 /* #UD condition is already handled. */
2971 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2972 return emulate_gp(ctxt, 0);
2974 /* Disable writeback. */
2975 ctxt->dst.type = OP_NONE;
2976 return X86EMUL_CONTINUE;
2979 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2983 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2984 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2985 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2986 return emulate_gp(ctxt, 0);
2988 return X86EMUL_CONTINUE;
2991 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2995 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2996 return emulate_gp(ctxt, 0);
2998 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2999 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
3000 return X86EMUL_CONTINUE;
3003 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3005 if (ctxt->modrm_reg > VCPU_SREG_GS)
3006 return emulate_ud(ctxt);
3008 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3009 return X86EMUL_CONTINUE;
3012 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3014 u16 sel = ctxt->src.val;
3016 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3017 return emulate_ud(ctxt);
3019 if (ctxt->modrm_reg == VCPU_SREG_SS)
3020 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3022 /* Disable writeback. */
3023 ctxt->dst.type = OP_NONE;
3024 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3027 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3029 u16 sel = ctxt->src.val;
3031 /* Disable writeback. */
3032 ctxt->dst.type = OP_NONE;
3033 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3036 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3038 u16 sel = ctxt->src.val;
3040 /* Disable writeback. */
3041 ctxt->dst.type = OP_NONE;
3042 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3045 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3050 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3051 if (rc == X86EMUL_CONTINUE)
3052 ctxt->ops->invlpg(ctxt, linear);
3053 /* Disable writeback. */
3054 ctxt->dst.type = OP_NONE;
3055 return X86EMUL_CONTINUE;
3058 static int em_clts(struct x86_emulate_ctxt *ctxt)
3062 cr0 = ctxt->ops->get_cr(ctxt, 0);
3064 ctxt->ops->set_cr(ctxt, 0, cr0);
3065 return X86EMUL_CONTINUE;
3068 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3072 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3073 return X86EMUL_UNHANDLEABLE;
3075 rc = ctxt->ops->fix_hypercall(ctxt);
3076 if (rc != X86EMUL_CONTINUE)
3079 /* Let the processor re-execute the fixed hypercall */
3080 ctxt->_eip = ctxt->eip;
3081 /* Disable writeback. */
3082 ctxt->dst.type = OP_NONE;
3083 return X86EMUL_CONTINUE;
3086 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3087 void (*get)(struct x86_emulate_ctxt *ctxt,
3088 struct desc_ptr *ptr))
3090 struct desc_ptr desc_ptr;
3092 if (ctxt->mode == X86EMUL_MODE_PROT64)
3094 get(ctxt, &desc_ptr);
3095 if (ctxt->op_bytes == 2) {
3097 desc_ptr.address &= 0x00ffffff;
3099 /* Disable writeback. */
3100 ctxt->dst.type = OP_NONE;
3101 return segmented_write(ctxt, ctxt->dst.addr.mem,
3102 &desc_ptr, 2 + ctxt->op_bytes);
3105 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3107 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3110 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3112 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3115 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3117 struct desc_ptr desc_ptr;
3120 if (ctxt->mode == X86EMUL_MODE_PROT64)
3122 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3123 &desc_ptr.size, &desc_ptr.address,
3125 if (rc != X86EMUL_CONTINUE)
3127 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3128 /* Disable writeback. */
3129 ctxt->dst.type = OP_NONE;
3130 return X86EMUL_CONTINUE;
3133 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3137 rc = ctxt->ops->fix_hypercall(ctxt);
3139 /* Disable writeback. */
3140 ctxt->dst.type = OP_NONE;
3144 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3146 struct desc_ptr desc_ptr;
3149 if (ctxt->mode == X86EMUL_MODE_PROT64)
3151 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3152 &desc_ptr.size, &desc_ptr.address,
3154 if (rc != X86EMUL_CONTINUE)
3156 ctxt->ops->set_idt(ctxt, &desc_ptr);
3157 /* Disable writeback. */
3158 ctxt->dst.type = OP_NONE;
3159 return X86EMUL_CONTINUE;
3162 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3164 ctxt->dst.bytes = 2;
3165 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3166 return X86EMUL_CONTINUE;
3169 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3171 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3172 | (ctxt->src.val & 0x0f));
3173 ctxt->dst.type = OP_NONE;
3174 return X86EMUL_CONTINUE;
3177 static int em_loop(struct x86_emulate_ctxt *ctxt)
3179 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3180 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
3181 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3182 jmp_rel(ctxt, ctxt->src.val);
3184 return X86EMUL_CONTINUE;
3187 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3189 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
3190 jmp_rel(ctxt, ctxt->src.val);
3192 return X86EMUL_CONTINUE;
3195 static int em_in(struct x86_emulate_ctxt *ctxt)
3197 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3199 return X86EMUL_IO_NEEDED;
3201 return X86EMUL_CONTINUE;
3204 static int em_out(struct x86_emulate_ctxt *ctxt)
3206 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3208 /* Disable writeback. */
3209 ctxt->dst.type = OP_NONE;
3210 return X86EMUL_CONTINUE;
3213 static int em_cli(struct x86_emulate_ctxt *ctxt)
3215 if (emulator_bad_iopl(ctxt))
3216 return emulate_gp(ctxt, 0);
3218 ctxt->eflags &= ~X86_EFLAGS_IF;
3219 return X86EMUL_CONTINUE;
3222 static int em_sti(struct x86_emulate_ctxt *ctxt)
3224 if (emulator_bad_iopl(ctxt))
3225 return emulate_gp(ctxt, 0);
3227 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3228 ctxt->eflags |= X86_EFLAGS_IF;
3229 return X86EMUL_CONTINUE;
3232 static int em_bt(struct x86_emulate_ctxt *ctxt)
3234 /* Disable writeback. */
3235 ctxt->dst.type = OP_NONE;
3236 /* only subword offset */
3237 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3239 emulate_2op_SrcV_nobyte(ctxt, "bt");
3240 return X86EMUL_CONTINUE;
3243 static int em_bts(struct x86_emulate_ctxt *ctxt)
3245 emulate_2op_SrcV_nobyte(ctxt, "bts");
3246 return X86EMUL_CONTINUE;
3249 static int em_btr(struct x86_emulate_ctxt *ctxt)
3251 emulate_2op_SrcV_nobyte(ctxt, "btr");
3252 return X86EMUL_CONTINUE;
3255 static int em_btc(struct x86_emulate_ctxt *ctxt)
3257 emulate_2op_SrcV_nobyte(ctxt, "btc");
3258 return X86EMUL_CONTINUE;
3261 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3263 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3264 return X86EMUL_CONTINUE;
3267 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3269 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3270 return X86EMUL_CONTINUE;
3273 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3275 u32 eax, ebx, ecx, edx;
3277 eax = ctxt->regs[VCPU_REGS_RAX];
3278 ecx = ctxt->regs[VCPU_REGS_RCX];
3279 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3280 ctxt->regs[VCPU_REGS_RAX] = eax;
3281 ctxt->regs[VCPU_REGS_RBX] = ebx;
3282 ctxt->regs[VCPU_REGS_RCX] = ecx;
3283 ctxt->regs[VCPU_REGS_RDX] = edx;
3284 return X86EMUL_CONTINUE;
3287 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3289 ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL;
3290 ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8;
3291 return X86EMUL_CONTINUE;
3294 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3296 switch (ctxt->op_bytes) {
3297 #ifdef CONFIG_X86_64
3299 asm("bswap %0" : "+r"(ctxt->dst.val));
3303 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3306 return X86EMUL_CONTINUE;
3309 static bool valid_cr(int nr)
3321 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3323 if (!valid_cr(ctxt->modrm_reg))
3324 return emulate_ud(ctxt);
3326 return X86EMUL_CONTINUE;
3329 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3331 u64 new_val = ctxt->src.val64;
3332 int cr = ctxt->modrm_reg;
3335 static u64 cr_reserved_bits[] = {
3336 0xffffffff00000000ULL,
3337 0, 0, 0, /* CR3 checked later */
3344 return emulate_ud(ctxt);
3346 if (new_val & cr_reserved_bits[cr])
3347 return emulate_gp(ctxt, 0);
3352 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3353 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3354 return emulate_gp(ctxt, 0);
3356 cr4 = ctxt->ops->get_cr(ctxt, 4);
3357 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3359 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3360 !(cr4 & X86_CR4_PAE))
3361 return emulate_gp(ctxt, 0);
3368 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3369 if (efer & EFER_LMA)
3370 rsvd = CR3_L_MODE_RESERVED_BITS;
3371 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3372 rsvd = CR3_PAE_RESERVED_BITS;
3373 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3374 rsvd = CR3_NONPAE_RESERVED_BITS;
3377 return emulate_gp(ctxt, 0);
3382 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3384 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3385 return emulate_gp(ctxt, 0);
3391 return X86EMUL_CONTINUE;
3394 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3398 ctxt->ops->get_dr(ctxt, 7, &dr7);
3400 /* Check if DR7.Global_Enable is set */
3401 return dr7 & (1 << 13);
3404 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3406 int dr = ctxt->modrm_reg;
3410 return emulate_ud(ctxt);
3412 cr4 = ctxt->ops->get_cr(ctxt, 4);
3413 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3414 return emulate_ud(ctxt);
3416 if (check_dr7_gd(ctxt))
3417 return emulate_db(ctxt);
3419 return X86EMUL_CONTINUE;
3422 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3424 u64 new_val = ctxt->src.val64;
3425 int dr = ctxt->modrm_reg;
3427 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3428 return emulate_gp(ctxt, 0);
3430 return check_dr_read(ctxt);
3433 static int check_svme(struct x86_emulate_ctxt *ctxt)
3437 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3439 if (!(efer & EFER_SVME))
3440 return emulate_ud(ctxt);
3442 return X86EMUL_CONTINUE;
3445 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3447 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3449 /* Valid physical address? */
3450 if (rax & 0xffff000000000000ULL)
3451 return emulate_gp(ctxt, 0);
3453 return check_svme(ctxt);
3456 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3458 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3460 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3461 return emulate_ud(ctxt);
3463 return X86EMUL_CONTINUE;
3466 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3468 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3469 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3471 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3473 return emulate_gp(ctxt, 0);
3475 return X86EMUL_CONTINUE;
3478 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3480 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3481 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3482 return emulate_gp(ctxt, 0);
3484 return X86EMUL_CONTINUE;
3487 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3489 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3490 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3491 return emulate_gp(ctxt, 0);
3493 return X86EMUL_CONTINUE;
3496 #define D(_y) { .flags = (_y) }
3497 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3498 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3499 .check_perm = (_p) }
3501 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3502 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3503 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3504 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3505 #define II(_f, _e, _i) \
3506 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3507 #define IIP(_f, _e, _i, _p) \
3508 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3509 .check_perm = (_p) }
3510 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3512 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3513 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3514 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3515 #define I2bvIP(_f, _e, _i, _p) \
3516 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3518 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3519 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3520 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3522 static struct opcode group7_rm1[] = {
3523 DI(SrcNone | Priv, monitor),
3524 DI(SrcNone | Priv, mwait),
3528 static struct opcode group7_rm3[] = {
3529 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3530 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3531 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3532 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3533 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3534 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3535 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3536 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3539 static struct opcode group7_rm7[] = {
3541 DIP(SrcNone, rdtscp, check_rdtsc),
3545 static struct opcode group1[] = {
3547 I(Lock | PageTable, em_or),
3550 I(Lock | PageTable, em_and),
3556 static struct opcode group1A[] = {
3557 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3560 static struct opcode group3[] = {
3561 I(DstMem | SrcImm, em_test),
3562 I(DstMem | SrcImm, em_test),
3563 I(DstMem | SrcNone | Lock, em_not),
3564 I(DstMem | SrcNone | Lock, em_neg),
3565 I(SrcMem, em_mul_ex),
3566 I(SrcMem, em_imul_ex),
3567 I(SrcMem, em_div_ex),
3568 I(SrcMem, em_idiv_ex),
3571 static struct opcode group4[] = {
3572 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3573 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3577 static struct opcode group5[] = {
3578 I(DstMem | SrcNone | Lock, em_grp45),
3579 I(DstMem | SrcNone | Lock, em_grp45),
3580 I(SrcMem | Stack, em_grp45),
3581 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3582 I(SrcMem | Stack, em_grp45),
3583 I(SrcMemFAddr | ImplicitOps, em_grp45),
3584 I(SrcMem | Stack, em_grp45), N,
3587 static struct opcode group6[] = {
3590 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3591 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3595 static struct group_dual group7 = { {
3596 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3597 II(Mov | DstMem | Priv, em_sidt, sidt),
3598 II(SrcMem | Priv, em_lgdt, lgdt),
3599 II(SrcMem | Priv, em_lidt, lidt),
3600 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3601 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3602 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3604 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3606 N, EXT(0, group7_rm3),
3607 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3608 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3612 static struct opcode group8[] = {
3614 I(DstMem | SrcImmByte, em_bt),
3615 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3616 I(DstMem | SrcImmByte | Lock, em_btr),
3617 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3620 static struct group_dual group9 = { {
3621 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3623 N, N, N, N, N, N, N, N,
3626 static struct opcode group11[] = {
3627 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3631 static struct gprefix pfx_0f_6f_0f_7f = {
3632 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3635 static struct gprefix pfx_vmovntpx = {
3636 I(0, em_mov), N, N, N,
3639 static struct opcode opcode_table[256] = {
3641 I6ALU(Lock, em_add),
3642 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3643 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3645 I6ALU(Lock | PageTable, em_or),
3646 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3649 I6ALU(Lock, em_adc),
3650 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3651 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3653 I6ALU(Lock, em_sbb),
3654 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3655 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3657 I6ALU(Lock | PageTable, em_and), N, N,
3659 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3661 I6ALU(Lock, em_xor), N, N,
3663 I6ALU(0, em_cmp), N, N,
3667 X8(I(SrcReg | Stack, em_push)),
3669 X8(I(DstReg | Stack, em_pop)),
3671 I(ImplicitOps | Stack | No64, em_pusha),
3672 I(ImplicitOps | Stack | No64, em_popa),
3673 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3676 I(SrcImm | Mov | Stack, em_push),
3677 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3678 I(SrcImmByte | Mov | Stack, em_push),
3679 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3680 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3681 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3685 G(ByteOp | DstMem | SrcImm, group1),
3686 G(DstMem | SrcImm, group1),
3687 G(ByteOp | DstMem | SrcImm | No64, group1),
3688 G(DstMem | SrcImmByte, group1),
3689 I2bv(DstMem | SrcReg | ModRM, em_test),
3690 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3692 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3693 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3694 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3695 D(ModRM | SrcMem | NoAccess | DstReg),
3696 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3699 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3701 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3702 I(SrcImmFAddr | No64, em_call_far), N,
3703 II(ImplicitOps | Stack, em_pushf, pushf),
3704 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3706 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3707 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3708 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3709 I2bv(SrcSI | DstDI | String, em_cmp),
3711 I2bv(DstAcc | SrcImm, em_test),
3712 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3713 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3714 I2bv(SrcAcc | DstDI | String, em_cmp),
3716 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3718 X8(I(DstReg | SrcImm | Mov, em_mov)),
3720 D2bv(DstMem | SrcImmByte | ModRM),
3721 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3722 I(ImplicitOps | Stack, em_ret),
3723 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3724 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3725 G(ByteOp, group11), G(0, group11),
3727 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3728 N, I(ImplicitOps | Stack, em_ret_far),
3729 D(ImplicitOps), DI(SrcImmByte, intn),
3730 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3732 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3735 N, N, N, N, N, N, N, N,
3737 X3(I(SrcImmByte, em_loop)),
3738 I(SrcImmByte, em_jcxz),
3739 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3740 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3742 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3743 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3744 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3745 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3747 N, DI(ImplicitOps, icebp), N, N,
3748 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3749 G(ByteOp, group3), G(0, group3),
3751 D(ImplicitOps), D(ImplicitOps),
3752 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3753 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3756 static struct opcode twobyte_table[256] = {
3758 G(0, group6), GD(0, &group7), N, N,
3759 N, I(ImplicitOps | VendorSpecific, em_syscall),
3760 II(ImplicitOps | Priv, em_clts, clts), N,
3761 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3762 N, D(ImplicitOps | ModRM), N, N,
3764 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3766 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3767 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3768 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3769 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3771 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3774 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3775 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3776 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3777 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3778 I(ImplicitOps | VendorSpecific, em_sysenter),
3779 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3781 N, N, N, N, N, N, N, N,
3783 X16(D(DstReg | SrcMem | ModRM | Mov)),
3785 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3790 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3795 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3799 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3801 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3802 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3803 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3804 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3806 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3807 DI(ImplicitOps, rsm),
3808 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3809 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3810 D(DstMem | SrcReg | Src2CL | ModRM),
3811 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3813 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3814 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3815 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3816 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3817 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3818 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3822 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3823 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3824 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3826 D2bv(DstMem | SrcReg | ModRM | Lock),
3827 N, D(DstMem | SrcReg | ModRM | Mov),
3828 N, N, N, GD(0, &group9),
3830 X8(I(DstReg, em_bswap)),
3832 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3834 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3836 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3853 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3857 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3863 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3864 unsigned size, bool sign_extension)
3866 int rc = X86EMUL_CONTINUE;
3870 op->addr.mem.ea = ctxt->_eip;
3871 /* NB. Immediates are sign-extended as necessary. */
3872 switch (op->bytes) {
3874 op->val = insn_fetch(s8, ctxt);
3877 op->val = insn_fetch(s16, ctxt);
3880 op->val = insn_fetch(s32, ctxt);
3883 if (!sign_extension) {
3884 switch (op->bytes) {
3892 op->val &= 0xffffffff;
3900 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3903 int rc = X86EMUL_CONTINUE;
3907 decode_register_operand(ctxt, op);
3910 rc = decode_imm(ctxt, op, 1, false);
3913 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3917 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3918 fetch_bit_operand(ctxt);
3919 op->orig_val = op->val;
3922 ctxt->memop.bytes = 8;
3926 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3927 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3928 fetch_register_operand(op);
3929 op->orig_val = op->val;
3933 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3935 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3936 op->addr.mem.seg = VCPU_SREG_ES;
3942 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3943 fetch_register_operand(op);
3947 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3950 rc = decode_imm(ctxt, op, 1, true);
3957 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3960 ctxt->memop.bytes = 1;
3963 ctxt->memop.bytes = 2;
3966 ctxt->memop.bytes = 4;
3969 rc = decode_imm(ctxt, op, 2, false);
3972 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3976 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3978 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3979 op->addr.mem.seg = seg_override(ctxt);
3984 op->addr.mem.ea = ctxt->_eip;
3985 op->bytes = ctxt->op_bytes + 2;
3986 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3989 ctxt->memop.bytes = ctxt->op_bytes + 2;
3992 op->val = VCPU_SREG_ES;
3995 op->val = VCPU_SREG_CS;
3998 op->val = VCPU_SREG_SS;
4001 op->val = VCPU_SREG_DS;
4004 op->val = VCPU_SREG_FS;
4007 op->val = VCPU_SREG_GS;
4010 /* Special instructions do their own operand decoding. */
4012 op->type = OP_NONE; /* Disable writeback. */
4020 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4022 int rc = X86EMUL_CONTINUE;
4023 int mode = ctxt->mode;
4024 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4025 bool op_prefix = false;
4026 struct opcode opcode;
4028 ctxt->memop.type = OP_NONE;
4029 ctxt->memopp = NULL;
4030 ctxt->_eip = ctxt->eip;
4031 ctxt->fetch.start = ctxt->_eip;
4032 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4034 memcpy(ctxt->fetch.data, insn, insn_len);
4037 case X86EMUL_MODE_REAL:
4038 case X86EMUL_MODE_VM86:
4039 case X86EMUL_MODE_PROT16:
4040 def_op_bytes = def_ad_bytes = 2;
4042 case X86EMUL_MODE_PROT32:
4043 def_op_bytes = def_ad_bytes = 4;
4045 #ifdef CONFIG_X86_64
4046 case X86EMUL_MODE_PROT64:
4052 return EMULATION_FAILED;
4055 ctxt->op_bytes = def_op_bytes;
4056 ctxt->ad_bytes = def_ad_bytes;
4058 /* Legacy prefixes. */
4060 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4061 case 0x66: /* operand-size override */
4063 /* switch between 2/4 bytes */
4064 ctxt->op_bytes = def_op_bytes ^ 6;
4066 case 0x67: /* address-size override */
4067 if (mode == X86EMUL_MODE_PROT64)
4068 /* switch between 4/8 bytes */
4069 ctxt->ad_bytes = def_ad_bytes ^ 12;
4071 /* switch between 2/4 bytes */
4072 ctxt->ad_bytes = def_ad_bytes ^ 6;
4074 case 0x26: /* ES override */
4075 case 0x2e: /* CS override */
4076 case 0x36: /* SS override */
4077 case 0x3e: /* DS override */
4078 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4080 case 0x64: /* FS override */
4081 case 0x65: /* GS override */
4082 set_seg_override(ctxt, ctxt->b & 7);
4084 case 0x40 ... 0x4f: /* REX */
4085 if (mode != X86EMUL_MODE_PROT64)
4087 ctxt->rex_prefix = ctxt->b;
4089 case 0xf0: /* LOCK */
4090 ctxt->lock_prefix = 1;
4092 case 0xf2: /* REPNE/REPNZ */
4093 case 0xf3: /* REP/REPE/REPZ */
4094 ctxt->rep_prefix = ctxt->b;
4100 /* Any legacy prefix after a REX prefix nullifies its effect. */
4102 ctxt->rex_prefix = 0;
4108 if (ctxt->rex_prefix & 8)
4109 ctxt->op_bytes = 8; /* REX.W */
4111 /* Opcode byte(s). */
4112 opcode = opcode_table[ctxt->b];
4113 /* Two-byte opcode? */
4114 if (ctxt->b == 0x0f) {
4116 ctxt->b = insn_fetch(u8, ctxt);
4117 opcode = twobyte_table[ctxt->b];
4119 ctxt->d = opcode.flags;
4121 if (ctxt->d & ModRM)
4122 ctxt->modrm = insn_fetch(u8, ctxt);
4124 while (ctxt->d & GroupMask) {
4125 switch (ctxt->d & GroupMask) {
4127 goffset = (ctxt->modrm >> 3) & 7;
4128 opcode = opcode.u.group[goffset];
4131 goffset = (ctxt->modrm >> 3) & 7;
4132 if ((ctxt->modrm >> 6) == 3)
4133 opcode = opcode.u.gdual->mod3[goffset];
4135 opcode = opcode.u.gdual->mod012[goffset];
4138 goffset = ctxt->modrm & 7;
4139 opcode = opcode.u.group[goffset];
4142 if (ctxt->rep_prefix && op_prefix)
4143 return EMULATION_FAILED;
4144 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4145 switch (simd_prefix) {
4146 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4147 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4148 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4149 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4153 return EMULATION_FAILED;
4156 ctxt->d &= ~(u64)GroupMask;
4157 ctxt->d |= opcode.flags;
4160 ctxt->execute = opcode.u.execute;
4161 ctxt->check_perm = opcode.check_perm;
4162 ctxt->intercept = opcode.intercept;
4165 if (ctxt->d == 0 || (ctxt->d & Undefined))
4166 return EMULATION_FAILED;
4168 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4169 return EMULATION_FAILED;
4171 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4174 if (ctxt->d & Op3264) {
4175 if (mode == X86EMUL_MODE_PROT64)
4182 ctxt->op_bytes = 16;
4183 else if (ctxt->d & Mmx)
4186 /* ModRM and SIB bytes. */
4187 if (ctxt->d & ModRM) {
4188 rc = decode_modrm(ctxt, &ctxt->memop);
4189 if (!ctxt->has_seg_override)
4190 set_seg_override(ctxt, ctxt->modrm_seg);
4191 } else if (ctxt->d & MemAbs)
4192 rc = decode_abs(ctxt, &ctxt->memop);
4193 if (rc != X86EMUL_CONTINUE)
4196 if (!ctxt->has_seg_override)
4197 set_seg_override(ctxt, VCPU_SREG_DS);
4199 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4201 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4202 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4205 * Decode and fetch the source operand: register, memory
4208 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4209 if (rc != X86EMUL_CONTINUE)
4213 * Decode and fetch the second source operand: register, memory
4216 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4217 if (rc != X86EMUL_CONTINUE)
4220 /* Decode and fetch the destination operand: register or memory. */
4221 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4224 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4225 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4227 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4230 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4232 return ctxt->d & PageTable;
4235 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4237 /* The second termination condition only applies for REPE
4238 * and REPNE. Test if the repeat string operation prefix is
4239 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4240 * corresponding termination condition according to:
4241 * - if REPE/REPZ and ZF = 0 then done
4242 * - if REPNE/REPNZ and ZF = 1 then done
4244 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4245 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4246 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4247 ((ctxt->eflags & EFLG_ZF) == 0))
4248 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4249 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4255 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4259 ctxt->ops->get_fpu(ctxt);
4260 asm volatile("1: fwait \n\t"
4262 ".pushsection .fixup,\"ax\" \n\t"
4264 "movb $1, %[fault] \n\t"
4267 _ASM_EXTABLE(1b, 3b)
4268 : [fault]"+qm"(fault));
4269 ctxt->ops->put_fpu(ctxt);
4271 if (unlikely(fault))
4272 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4274 return X86EMUL_CONTINUE;
4277 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4280 if (op->type == OP_MM)
4281 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4284 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4286 struct x86_emulate_ops *ops = ctxt->ops;
4287 int rc = X86EMUL_CONTINUE;
4288 int saved_dst_type = ctxt->dst.type;
4290 ctxt->mem_read.pos = 0;
4292 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4293 rc = emulate_ud(ctxt);
4297 /* LOCK prefix is allowed only with some instructions */
4298 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4299 rc = emulate_ud(ctxt);
4303 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4304 rc = emulate_ud(ctxt);
4308 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4309 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4310 rc = emulate_ud(ctxt);
4314 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4315 rc = emulate_nm(ctxt);
4319 if (ctxt->d & Mmx) {
4320 rc = flush_pending_x87_faults(ctxt);
4321 if (rc != X86EMUL_CONTINUE)
4324 * Now that we know the fpu is exception safe, we can fetch
4327 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4328 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4329 if (!(ctxt->d & Mov))
4330 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4333 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4334 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4335 X86_ICPT_PRE_EXCEPT);
4336 if (rc != X86EMUL_CONTINUE)
4340 /* Privileged instruction can be executed only in CPL=0 */
4341 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4342 rc = emulate_gp(ctxt, 0);
4346 /* Instruction can only be executed in protected mode */
4347 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4348 rc = emulate_ud(ctxt);
4352 /* Do instruction specific permission checks */
4353 if (ctxt->check_perm) {
4354 rc = ctxt->check_perm(ctxt);
4355 if (rc != X86EMUL_CONTINUE)
4359 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4360 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4361 X86_ICPT_POST_EXCEPT);
4362 if (rc != X86EMUL_CONTINUE)
4366 if (ctxt->rep_prefix && (ctxt->d & String)) {
4367 /* All REP prefixes have the same first termination condition */
4368 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4369 ctxt->eip = ctxt->_eip;
4374 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4375 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4376 ctxt->src.valptr, ctxt->src.bytes);
4377 if (rc != X86EMUL_CONTINUE)
4379 ctxt->src.orig_val64 = ctxt->src.val64;
4382 if (ctxt->src2.type == OP_MEM) {
4383 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4384 &ctxt->src2.val, ctxt->src2.bytes);
4385 if (rc != X86EMUL_CONTINUE)
4389 if ((ctxt->d & DstMask) == ImplicitOps)
4393 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4394 /* optimisation - avoid slow emulated read if Mov */
4395 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4396 &ctxt->dst.val, ctxt->dst.bytes);
4397 if (rc != X86EMUL_CONTINUE)
4400 ctxt->dst.orig_val = ctxt->dst.val;
4404 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4405 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4406 X86_ICPT_POST_MEMACCESS);
4407 if (rc != X86EMUL_CONTINUE)
4411 if (ctxt->execute) {
4412 rc = ctxt->execute(ctxt);
4413 if (rc != X86EMUL_CONTINUE)
4422 case 0x40 ... 0x47: /* inc r16/r32 */
4423 emulate_1op(ctxt, "inc");
4425 case 0x48 ... 0x4f: /* dec r16/r32 */
4426 emulate_1op(ctxt, "dec");
4428 case 0x63: /* movsxd */
4429 if (ctxt->mode != X86EMUL_MODE_PROT64)
4430 goto cannot_emulate;
4431 ctxt->dst.val = (s32) ctxt->src.val;
4433 case 0x70 ... 0x7f: /* jcc (short) */
4434 if (test_cc(ctxt->b, ctxt->eflags))
4435 jmp_rel(ctxt, ctxt->src.val);
4437 case 0x8d: /* lea r16/r32, m */
4438 ctxt->dst.val = ctxt->src.addr.mem.ea;
4440 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4441 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4445 case 0x98: /* cbw/cwde/cdqe */
4446 switch (ctxt->op_bytes) {
4447 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4448 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4449 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4455 case 0xcc: /* int3 */
4456 rc = emulate_int(ctxt, 3);
4458 case 0xcd: /* int n */
4459 rc = emulate_int(ctxt, ctxt->src.val);
4461 case 0xce: /* into */
4462 if (ctxt->eflags & EFLG_OF)
4463 rc = emulate_int(ctxt, 4);
4465 case 0xd0 ... 0xd1: /* Grp2 */
4468 case 0xd2 ... 0xd3: /* Grp2 */
4469 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4472 case 0xe9: /* jmp rel */
4473 case 0xeb: /* jmp rel short */
4474 jmp_rel(ctxt, ctxt->src.val);
4475 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4477 case 0xf4: /* hlt */
4478 ctxt->ops->halt(ctxt);
4480 case 0xf5: /* cmc */
4481 /* complement carry flag from eflags reg */
4482 ctxt->eflags ^= EFLG_CF;
4484 case 0xf8: /* clc */
4485 ctxt->eflags &= ~EFLG_CF;
4487 case 0xf9: /* stc */
4488 ctxt->eflags |= EFLG_CF;
4490 case 0xfc: /* cld */
4491 ctxt->eflags &= ~EFLG_DF;
4493 case 0xfd: /* std */
4494 ctxt->eflags |= EFLG_DF;
4497 goto cannot_emulate;
4500 if (rc != X86EMUL_CONTINUE)
4504 rc = writeback(ctxt);
4505 if (rc != X86EMUL_CONTINUE)
4509 * restore dst type in case the decoding will be reused
4510 * (happens for string instruction )
4512 ctxt->dst.type = saved_dst_type;
4514 if ((ctxt->d & SrcMask) == SrcSI)
4515 string_addr_inc(ctxt, seg_override(ctxt),
4516 VCPU_REGS_RSI, &ctxt->src);
4518 if ((ctxt->d & DstMask) == DstDI)
4519 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4522 if (ctxt->rep_prefix && (ctxt->d & String)) {
4523 struct read_cache *r = &ctxt->io_read;
4524 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4526 if (!string_insn_completed(ctxt)) {
4528 * Re-enter guest when pio read ahead buffer is empty
4529 * or, if it is not used, after each 1024 iteration.
4531 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4532 (r->end == 0 || r->end != r->pos)) {
4534 * Reset read cache. Usually happens before
4535 * decode, but since instruction is restarted
4536 * we have to do it here.
4538 ctxt->mem_read.end = 0;
4539 return EMULATION_RESTART;
4541 goto done; /* skip rip writeback */
4545 ctxt->eip = ctxt->_eip;
4548 if (rc == X86EMUL_PROPAGATE_FAULT)
4549 ctxt->have_exception = true;
4550 if (rc == X86EMUL_INTERCEPTED)
4551 return EMULATION_INTERCEPTED;
4553 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4557 case 0x09: /* wbinvd */
4558 (ctxt->ops->wbinvd)(ctxt);
4560 case 0x08: /* invd */
4561 case 0x0d: /* GrpP (prefetch) */
4562 case 0x18: /* Grp16 (prefetch/nop) */
4564 case 0x20: /* mov cr, reg */
4565 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4567 case 0x21: /* mov from dr to reg */
4568 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4570 case 0x40 ... 0x4f: /* cmov */
4571 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4572 if (!test_cc(ctxt->b, ctxt->eflags))
4573 ctxt->dst.type = OP_NONE; /* no writeback */
4575 case 0x80 ... 0x8f: /* jnz rel, etc*/
4576 if (test_cc(ctxt->b, ctxt->eflags))
4577 jmp_rel(ctxt, ctxt->src.val);
4579 case 0x90 ... 0x9f: /* setcc r/m8 */
4580 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4582 case 0xa4: /* shld imm8, r, r/m */
4583 case 0xa5: /* shld cl, r, r/m */
4584 emulate_2op_cl(ctxt, "shld");
4586 case 0xac: /* shrd imm8, r, r/m */
4587 case 0xad: /* shrd cl, r, r/m */
4588 emulate_2op_cl(ctxt, "shrd");
4590 case 0xae: /* clflush */
4592 case 0xb6 ... 0xb7: /* movzx */
4593 ctxt->dst.bytes = ctxt->op_bytes;
4594 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4595 : (u16) ctxt->src.val;
4597 case 0xbe ... 0xbf: /* movsx */
4598 ctxt->dst.bytes = ctxt->op_bytes;
4599 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4600 (s16) ctxt->src.val;
4602 case 0xc0 ... 0xc1: /* xadd */
4603 emulate_2op_SrcV(ctxt, "add");
4604 /* Write back the register source. */
4605 ctxt->src.val = ctxt->dst.orig_val;
4606 write_register_operand(&ctxt->src);
4608 case 0xc3: /* movnti */
4609 ctxt->dst.bytes = ctxt->op_bytes;
4610 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4611 (u64) ctxt->src.val;
4614 goto cannot_emulate;
4617 if (rc != X86EMUL_CONTINUE)
4623 return EMULATION_FAILED;