1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
167 #define NoBigReal ((u64)1 << 50) /* No big real mode */
168 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
170 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
172 #define X2(x...) x, x
173 #define X3(x...) X2(x), x
174 #define X4(x...) X2(x), X2(x)
175 #define X5(x...) X4(x), x
176 #define X6(x...) X4(x), X2(x)
177 #define X7(x...) X4(x), X3(x)
178 #define X8(x...) X4(x), X4(x)
179 #define X16(x...) X8(x), X8(x)
181 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
182 #define FASTOP_SIZE 8
185 * fastop functions have a special calling convention:
190 * flags: rflags (in/out)
191 * ex: rsi (in:fastop pointer, out:zero if exception)
193 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
194 * different operand sizes can be reached by calculation, rather than a jump
195 * table (which would be bigger than the code).
197 * fastop functions are declared as taking a never-defined fastop parameter,
198 * so they can't be called from C directly.
207 int (*execute)(struct x86_emulate_ctxt *ctxt);
208 const struct opcode *group;
209 const struct group_dual *gdual;
210 const struct gprefix *gprefix;
211 const struct escape *esc;
212 void (*fastop)(struct fastop *fake);
214 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
218 struct opcode mod012[8];
219 struct opcode mod3[8];
223 struct opcode pfx_no;
224 struct opcode pfx_66;
225 struct opcode pfx_f2;
226 struct opcode pfx_f3;
231 struct opcode high[64];
234 /* EFLAGS bit definitions. */
235 #define EFLG_ID (1<<21)
236 #define EFLG_VIP (1<<20)
237 #define EFLG_VIF (1<<19)
238 #define EFLG_AC (1<<18)
239 #define EFLG_VM (1<<17)
240 #define EFLG_RF (1<<16)
241 #define EFLG_IOPL (3<<12)
242 #define EFLG_NT (1<<14)
243 #define EFLG_OF (1<<11)
244 #define EFLG_DF (1<<10)
245 #define EFLG_IF (1<<9)
246 #define EFLG_TF (1<<8)
247 #define EFLG_SF (1<<7)
248 #define EFLG_ZF (1<<6)
249 #define EFLG_AF (1<<4)
250 #define EFLG_PF (1<<2)
251 #define EFLG_CF (1<<0)
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 #define EFLG_RESERVED_ONE_MASK 2
256 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 if (!(ctxt->regs_valid & (1 << nr))) {
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
262 return ctxt->_regs[nr];
265 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->regs_dirty |= 1 << nr;
269 return &ctxt->_regs[nr];
272 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 return reg_write(ctxt, nr);
278 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
282 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
283 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
286 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
288 ctxt->regs_dirty = 0;
289 ctxt->regs_valid = 0;
293 * These EFLAGS bits are restored from saved value during emulation, and
294 * any changes are written back to the saved value after emulation.
296 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
304 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
306 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
307 #define FOP_RET "ret \n\t"
309 #define FOP_START(op) \
310 extern void em_##op(struct fastop *fake); \
311 asm(".pushsection .text, \"ax\" \n\t" \
312 ".global em_" #op " \n\t" \
319 #define FOPNOP() FOP_ALIGN FOP_RET
321 #define FOP1E(op, dst) \
322 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
324 #define FOP1EEX(op, dst) \
325 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
327 #define FASTOP1(op) \
332 ON64(FOP1E(op##q, rax)) \
335 /* 1-operand, using src2 (for MUL/DIV r/m) */
336 #define FASTOP1SRC2(op, name) \
341 ON64(FOP1E(op, rcx)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
345 #define FASTOP1SRC2EX(op, name) \
350 ON64(FOP1EEX(op, rcx)) \
353 #define FOP2E(op, dst, src) \
354 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
356 #define FASTOP2(op) \
358 FOP2E(op##b, al, dl) \
359 FOP2E(op##w, ax, dx) \
360 FOP2E(op##l, eax, edx) \
361 ON64(FOP2E(op##q, rax, rdx)) \
364 /* 2 operand, word only */
365 #define FASTOP2W(op) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, src is CL */
374 #define FASTOP2CL(op) \
376 FOP2E(op##b, al, cl) \
377 FOP2E(op##w, ax, cl) \
378 FOP2E(op##l, eax, cl) \
379 ON64(FOP2E(op##q, rax, cl)) \
382 #define FOP3E(op, dst, src, src2) \
383 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
385 /* 3-operand, word-only, src2=cl */
386 #define FASTOP3WCL(op) \
389 FOP3E(op##w, ax, dx, cl) \
390 FOP3E(op##l, eax, edx, cl) \
391 ON64(FOP3E(op##q, rax, rdx, cl)) \
394 /* Special case for SETcc - 1 instruction per cc */
395 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
397 asm(".global kvm_fastop_exception \n"
398 "kvm_fastop_exception: xor %esi, %esi; ret");
419 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
422 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
423 enum x86_intercept intercept,
424 enum x86_intercept_stage stage)
426 struct x86_instruction_info info = {
427 .intercept = intercept,
428 .rep_prefix = ctxt->rep_prefix,
429 .modrm_mod = ctxt->modrm_mod,
430 .modrm_reg = ctxt->modrm_reg,
431 .modrm_rm = ctxt->modrm_rm,
432 .src_val = ctxt->src.val64,
433 .dst_val = ctxt->dst.val64,
434 .src_bytes = ctxt->src.bytes,
435 .dst_bytes = ctxt->dst.bytes,
436 .ad_bytes = ctxt->ad_bytes,
437 .next_rip = ctxt->eip,
440 return ctxt->ops->intercept(ctxt, &info, stage);
443 static void assign_masked(ulong *dest, ulong src, ulong mask)
445 *dest = (*dest & ~mask) | (src & mask);
448 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
450 return (1UL << (ctxt->ad_bytes << 3)) - 1;
453 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
456 struct desc_struct ss;
458 if (ctxt->mode == X86EMUL_MODE_PROT64)
460 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
461 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
464 static int stack_size(struct x86_emulate_ctxt *ctxt)
466 return (__fls(stack_mask(ctxt)) + 1) >> 3;
469 /* Access/update address held in a register, based on addressing mode. */
470 static inline unsigned long
471 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
473 if (ctxt->ad_bytes == sizeof(unsigned long))
476 return reg & ad_mask(ctxt);
479 static inline unsigned long
480 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
482 return address_mask(ctxt, reg);
485 static void masked_increment(ulong *reg, ulong mask, int inc)
487 assign_masked(reg, *reg + inc, mask);
491 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
495 if (ctxt->ad_bytes == sizeof(unsigned long))
498 mask = ad_mask(ctxt);
499 masked_increment(reg, mask, inc);
502 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
504 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
507 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
509 register_address_increment(ctxt, &ctxt->_eip, rel);
512 static u32 desc_limit_scaled(struct desc_struct *desc)
514 u32 limit = get_desc_limit(desc);
516 return desc->g ? (limit << 12) | 0xfff : limit;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
528 u32 error, bool valid)
531 ctxt->exception.vector = vec;
532 ctxt->exception.error_code = error;
533 ctxt->exception.error_code_valid = valid;
534 return X86EMUL_PROPAGATE_FAULT;
537 static int emulate_db(struct x86_emulate_ctxt *ctxt)
539 return emulate_exception(ctxt, DB_VECTOR, 0, false);
542 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, GP_VECTOR, err, true);
547 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
549 return emulate_exception(ctxt, SS_VECTOR, err, true);
552 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
554 return emulate_exception(ctxt, UD_VECTOR, 0, false);
557 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
559 return emulate_exception(ctxt, TS_VECTOR, err, true);
562 static int emulate_de(struct x86_emulate_ctxt *ctxt)
564 return emulate_exception(ctxt, DE_VECTOR, 0, false);
567 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
569 return emulate_exception(ctxt, NM_VECTOR, 0, false);
572 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
575 struct desc_struct desc;
577 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
581 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
586 struct desc_struct desc;
588 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
589 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
593 * x86 defines three classes of vector instructions: explicitly
594 * aligned, explicitly unaligned, and the rest, which change behaviour
595 * depending on whether they're AVX encoded or not.
597 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
598 * subject to the same check.
600 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
602 if (likely(size < 16))
605 if (ctxt->d & Aligned)
607 else if (ctxt->d & Unaligned)
609 else if (ctxt->d & Avx)
615 static int __linearize(struct x86_emulate_ctxt *ctxt,
616 struct segmented_address addr,
617 unsigned size, bool write, bool fetch,
620 struct desc_struct desc;
627 la = seg_base(ctxt, addr.seg) + addr.ea;
628 switch (ctxt->mode) {
629 case X86EMUL_MODE_PROT64:
630 if (((signed long)la << 16) >> 16 != la)
631 return emulate_gp(ctxt, 0);
634 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
638 /* code segment in protected mode or read-only data segment */
639 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
640 || !(desc.type & 2)) && write)
642 /* unreadable code segment */
643 if (!fetch && (desc.type & 8) && !(desc.type & 2))
645 lim = desc_limit_scaled(&desc);
646 if ((ctxt->mode == X86EMUL_MODE_REAL) && !fetch &&
647 (ctxt->d & NoBigReal)) {
648 /* la is between zero and 0xffff */
649 if (la > 0xffff || (u32)(la + size - 1) > 0xffff)
651 } else if ((desc.type & 8) || !(desc.type & 4)) {
652 /* expand-up segment */
653 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
656 /* expand-down segment */
657 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
659 lim = desc.d ? 0xffffffff : 0xffff;
660 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
663 cpl = ctxt->ops->cpl(ctxt);
664 if (!(desc.type & 8)) {
668 } else if ((desc.type & 8) && !(desc.type & 4)) {
669 /* nonconforming code segment */
672 } else if ((desc.type & 8) && (desc.type & 4)) {
673 /* conforming code segment */
679 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
681 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
682 return emulate_gp(ctxt, 0);
684 return X86EMUL_CONTINUE;
686 if (addr.seg == VCPU_SREG_SS)
687 return emulate_ss(ctxt, sel);
689 return emulate_gp(ctxt, sel);
692 static int linearize(struct x86_emulate_ctxt *ctxt,
693 struct segmented_address addr,
694 unsigned size, bool write,
697 return __linearize(ctxt, addr, size, write, false, linear);
701 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
702 struct segmented_address addr,
709 rc = linearize(ctxt, addr, size, false, &linear);
710 if (rc != X86EMUL_CONTINUE)
712 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
716 * Prefetch the remaining bytes of the instruction without crossing page
717 * boundary if they are not in fetch_cache yet.
719 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
723 unsigned long linear;
724 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
725 struct segmented_address addr = { .seg = VCPU_SREG_CS,
726 .ea = ctxt->eip + cur_size };
728 size = 15UL ^ cur_size;
729 rc = __linearize(ctxt, addr, size, false, true, &linear);
730 if (unlikely(rc != X86EMUL_CONTINUE))
733 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
736 * One instruction can only straddle two pages,
737 * and one has been loaded at the beginning of
738 * x86_decode_insn. So, if not enough bytes
739 * still, we must have hit the 15-byte boundary.
741 if (unlikely(size < op_size))
742 return X86EMUL_UNHANDLEABLE;
743 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
744 size, &ctxt->exception);
745 if (unlikely(rc != X86EMUL_CONTINUE))
747 ctxt->fetch.end += size;
748 return X86EMUL_CONTINUE;
751 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
754 if (unlikely(ctxt->fetch.end - ctxt->fetch.ptr < size))
755 return __do_insn_fetch_bytes(ctxt, size);
757 return X86EMUL_CONTINUE;
760 /* Fetch next part of the instruction being emulated. */
761 #define insn_fetch(_type, _ctxt) \
764 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
765 if (rc != X86EMUL_CONTINUE) \
767 ctxt->_eip += sizeof(_type); \
768 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
769 ctxt->fetch.ptr += sizeof(_type); \
773 #define insn_fetch_arr(_arr, _size, _ctxt) \
775 rc = do_insn_fetch_bytes(_ctxt, _size); \
776 if (rc != X86EMUL_CONTINUE) \
778 ctxt->_eip += (_size); \
779 memcpy(_arr, ctxt->fetch.ptr, _size); \
780 ctxt->fetch.ptr += (_size); \
784 * Given the 'reg' portion of a ModRM byte, and a register block, return a
785 * pointer into the block that addresses the relevant register.
786 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
788 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
794 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
795 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
797 p = reg_rmw(ctxt, modrm_reg);
801 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
802 struct segmented_address addr,
803 u16 *size, unsigned long *address, int op_bytes)
810 rc = segmented_read_std(ctxt, addr, size, 2);
811 if (rc != X86EMUL_CONTINUE)
814 rc = segmented_read_std(ctxt, addr, address, op_bytes);
828 FASTOP1SRC2(mul, mul_ex);
829 FASTOP1SRC2(imul, imul_ex);
830 FASTOP1SRC2EX(div, div_ex);
831 FASTOP1SRC2EX(idiv, idiv_ex);
860 static u8 test_cc(unsigned int condition, unsigned long flags)
863 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
865 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
866 asm("push %[flags]; popf; call *%[fastop]"
867 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
871 static void fetch_register_operand(struct operand *op)
875 op->val = *(u8 *)op->addr.reg;
878 op->val = *(u16 *)op->addr.reg;
881 op->val = *(u32 *)op->addr.reg;
884 op->val = *(u64 *)op->addr.reg;
889 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
891 ctxt->ops->get_fpu(ctxt);
893 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
894 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
895 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
896 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
897 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
898 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
899 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
900 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
902 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
903 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
904 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
905 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
906 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
907 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
908 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
909 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
913 ctxt->ops->put_fpu(ctxt);
916 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
919 ctxt->ops->get_fpu(ctxt);
921 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
922 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
923 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
924 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
925 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
926 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
927 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
928 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
930 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
931 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
932 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
933 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
934 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
935 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
936 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
937 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
941 ctxt->ops->put_fpu(ctxt);
944 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
946 ctxt->ops->get_fpu(ctxt);
948 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
949 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
950 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
951 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
952 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
953 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
954 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
955 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
958 ctxt->ops->put_fpu(ctxt);
961 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
963 ctxt->ops->get_fpu(ctxt);
965 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
966 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
967 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
968 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
969 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
970 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
971 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
972 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
975 ctxt->ops->put_fpu(ctxt);
978 static int em_fninit(struct x86_emulate_ctxt *ctxt)
980 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
981 return emulate_nm(ctxt);
983 ctxt->ops->get_fpu(ctxt);
984 asm volatile("fninit");
985 ctxt->ops->put_fpu(ctxt);
986 return X86EMUL_CONTINUE;
989 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
993 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
994 return emulate_nm(ctxt);
996 ctxt->ops->get_fpu(ctxt);
997 asm volatile("fnstcw %0": "+m"(fcw));
998 ctxt->ops->put_fpu(ctxt);
1000 /* force 2 byte destination */
1001 ctxt->dst.bytes = 2;
1002 ctxt->dst.val = fcw;
1004 return X86EMUL_CONTINUE;
1007 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1011 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1012 return emulate_nm(ctxt);
1014 ctxt->ops->get_fpu(ctxt);
1015 asm volatile("fnstsw %0": "+m"(fsw));
1016 ctxt->ops->put_fpu(ctxt);
1018 /* force 2 byte destination */
1019 ctxt->dst.bytes = 2;
1020 ctxt->dst.val = fsw;
1022 return X86EMUL_CONTINUE;
1025 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1028 unsigned reg = ctxt->modrm_reg;
1030 if (!(ctxt->d & ModRM))
1031 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1033 if (ctxt->d & Sse) {
1037 read_sse_reg(ctxt, &op->vec_val, reg);
1040 if (ctxt->d & Mmx) {
1049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1050 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1052 fetch_register_operand(op);
1053 op->orig_val = op->val;
1056 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1058 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1059 ctxt->modrm_seg = VCPU_SREG_SS;
1062 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066 int index_reg, base_reg, scale;
1067 int rc = X86EMUL_CONTINUE;
1070 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1071 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1072 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1074 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1075 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1076 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1077 ctxt->modrm_seg = VCPU_SREG_DS;
1079 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1081 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1082 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1084 if (ctxt->d & Sse) {
1087 op->addr.xmm = ctxt->modrm_rm;
1088 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1091 if (ctxt->d & Mmx) {
1094 op->addr.mm = ctxt->modrm_rm & 7;
1097 fetch_register_operand(op);
1103 if (ctxt->ad_bytes == 2) {
1104 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1105 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1106 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1107 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1109 /* 16-bit ModR/M decode. */
1110 switch (ctxt->modrm_mod) {
1112 if (ctxt->modrm_rm == 6)
1113 modrm_ea += insn_fetch(u16, ctxt);
1116 modrm_ea += insn_fetch(s8, ctxt);
1119 modrm_ea += insn_fetch(u16, ctxt);
1122 switch (ctxt->modrm_rm) {
1124 modrm_ea += bx + si;
1127 modrm_ea += bx + di;
1130 modrm_ea += bp + si;
1133 modrm_ea += bp + di;
1142 if (ctxt->modrm_mod != 0)
1149 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1150 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1151 ctxt->modrm_seg = VCPU_SREG_SS;
1152 modrm_ea = (u16)modrm_ea;
1154 /* 32/64-bit ModR/M decode. */
1155 if ((ctxt->modrm_rm & 7) == 4) {
1156 sib = insn_fetch(u8, ctxt);
1157 index_reg |= (sib >> 3) & 7;
1158 base_reg |= sib & 7;
1161 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1162 modrm_ea += insn_fetch(s32, ctxt);
1164 modrm_ea += reg_read(ctxt, base_reg);
1165 adjust_modrm_seg(ctxt, base_reg);
1168 modrm_ea += reg_read(ctxt, index_reg) << scale;
1169 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1170 if (ctxt->mode == X86EMUL_MODE_PROT64)
1171 ctxt->rip_relative = 1;
1173 base_reg = ctxt->modrm_rm;
1174 modrm_ea += reg_read(ctxt, base_reg);
1175 adjust_modrm_seg(ctxt, base_reg);
1177 switch (ctxt->modrm_mod) {
1179 if (ctxt->modrm_rm == 5)
1180 modrm_ea += insn_fetch(s32, ctxt);
1183 modrm_ea += insn_fetch(s8, ctxt);
1186 modrm_ea += insn_fetch(s32, ctxt);
1190 op->addr.mem.ea = modrm_ea;
1191 if (ctxt->ad_bytes != 8)
1192 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1198 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1201 int rc = X86EMUL_CONTINUE;
1204 switch (ctxt->ad_bytes) {
1206 op->addr.mem.ea = insn_fetch(u16, ctxt);
1209 op->addr.mem.ea = insn_fetch(u32, ctxt);
1212 op->addr.mem.ea = insn_fetch(u64, ctxt);
1219 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1223 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1224 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1226 if (ctxt->src.bytes == 2)
1227 sv = (s16)ctxt->src.val & (s16)mask;
1228 else if (ctxt->src.bytes == 4)
1229 sv = (s32)ctxt->src.val & (s32)mask;
1231 sv = (s64)ctxt->src.val & (s64)mask;
1233 ctxt->dst.addr.mem.ea += (sv >> 3);
1236 /* only subword offset */
1237 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1240 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1241 unsigned long addr, void *dest, unsigned size)
1244 struct read_cache *mc = &ctxt->mem_read;
1246 if (mc->pos < mc->end)
1249 WARN_ON((mc->end + size) >= sizeof(mc->data));
1251 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1253 if (rc != X86EMUL_CONTINUE)
1259 memcpy(dest, mc->data + mc->pos, size);
1261 return X86EMUL_CONTINUE;
1264 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1265 struct segmented_address addr,
1272 rc = linearize(ctxt, addr, size, false, &linear);
1273 if (rc != X86EMUL_CONTINUE)
1275 return read_emulated(ctxt, linear, data, size);
1278 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1279 struct segmented_address addr,
1286 rc = linearize(ctxt, addr, size, true, &linear);
1287 if (rc != X86EMUL_CONTINUE)
1289 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1293 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1294 struct segmented_address addr,
1295 const void *orig_data, const void *data,
1301 rc = linearize(ctxt, addr, size, true, &linear);
1302 if (rc != X86EMUL_CONTINUE)
1304 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1305 size, &ctxt->exception);
1308 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1309 unsigned int size, unsigned short port,
1312 struct read_cache *rc = &ctxt->io_read;
1314 if (rc->pos == rc->end) { /* refill pio read ahead */
1315 unsigned int in_page, n;
1316 unsigned int count = ctxt->rep_prefix ?
1317 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1318 in_page = (ctxt->eflags & EFLG_DF) ?
1319 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1320 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1321 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1324 rc->pos = rc->end = 0;
1325 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1330 if (ctxt->rep_prefix && (ctxt->d & String) &&
1331 !(ctxt->eflags & EFLG_DF)) {
1332 ctxt->dst.data = rc->data + rc->pos;
1333 ctxt->dst.type = OP_MEM_STR;
1334 ctxt->dst.count = (rc->end - rc->pos) / size;
1337 memcpy(dest, rc->data + rc->pos, size);
1343 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1344 u16 index, struct desc_struct *desc)
1349 ctxt->ops->get_idt(ctxt, &dt);
1351 if (dt.size < index * 8 + 7)
1352 return emulate_gp(ctxt, index << 3 | 0x2);
1354 addr = dt.address + index * 8;
1355 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1359 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1360 u16 selector, struct desc_ptr *dt)
1362 const struct x86_emulate_ops *ops = ctxt->ops;
1365 if (selector & 1 << 2) {
1366 struct desc_struct desc;
1369 memset (dt, 0, sizeof *dt);
1370 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1374 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1375 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1377 ops->get_gdt(ctxt, dt);
1380 /* allowed just for 8 bytes segments */
1381 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1382 u16 selector, struct desc_struct *desc,
1386 u16 index = selector >> 3;
1389 get_descriptor_table_ptr(ctxt, selector, &dt);
1391 if (dt.size < index * 8 + 7)
1392 return emulate_gp(ctxt, selector & 0xfffc);
1394 *desc_addr_p = addr = dt.address + index * 8;
1395 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1399 /* allowed just for 8 bytes segments */
1400 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1401 u16 selector, struct desc_struct *desc)
1404 u16 index = selector >> 3;
1407 get_descriptor_table_ptr(ctxt, selector, &dt);
1409 if (dt.size < index * 8 + 7)
1410 return emulate_gp(ctxt, selector & 0xfffc);
1412 addr = dt.address + index * 8;
1413 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1417 /* Does not support long mode */
1418 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1419 u16 selector, int seg, u8 cpl, bool in_task_switch)
1421 struct desc_struct seg_desc, old_desc;
1423 unsigned err_vec = GP_VECTOR;
1425 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1431 memset(&seg_desc, 0, sizeof seg_desc);
1433 if (ctxt->mode == X86EMUL_MODE_REAL) {
1434 /* set real mode segment descriptor (keep limit etc. for
1436 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1437 set_desc_base(&seg_desc, selector << 4);
1439 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1440 /* VM86 needs a clean new segment descriptor */
1441 set_desc_base(&seg_desc, selector << 4);
1442 set_desc_limit(&seg_desc, 0xffff);
1452 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1453 if ((seg == VCPU_SREG_CS
1454 || (seg == VCPU_SREG_SS
1455 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1456 || seg == VCPU_SREG_TR)
1460 /* TR should be in GDT only */
1461 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1464 if (null_selector) /* for NULL selector skip all following checks */
1467 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1468 if (ret != X86EMUL_CONTINUE)
1471 err_code = selector & 0xfffc;
1472 err_vec = in_task_switch ? TS_VECTOR : GP_VECTOR;
1474 /* can't load system descriptor into segment selector */
1475 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1479 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1488 * segment is not a writable data segment or segment
1489 * selector's RPL != CPL or segment selector's RPL != CPL
1491 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1495 if (!(seg_desc.type & 8))
1498 if (seg_desc.type & 4) {
1504 if (rpl > cpl || dpl != cpl)
1507 /* in long-mode d/b must be clear if l is set */
1508 if (seg_desc.d && seg_desc.l) {
1511 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1512 if (efer & EFER_LMA)
1516 /* CS(RPL) <- CPL */
1517 selector = (selector & 0xfffc) | cpl;
1520 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1522 old_desc = seg_desc;
1523 seg_desc.type |= 2; /* busy */
1524 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1525 sizeof(seg_desc), &ctxt->exception);
1526 if (ret != X86EMUL_CONTINUE)
1529 case VCPU_SREG_LDTR:
1530 if (seg_desc.s || seg_desc.type != 2)
1533 default: /* DS, ES, FS, or GS */
1535 * segment is not a data or readable code segment or
1536 * ((segment is a data or nonconforming code segment)
1537 * and (both RPL and CPL > DPL))
1539 if ((seg_desc.type & 0xa) == 0x8 ||
1540 (((seg_desc.type & 0xc) != 0xc) &&
1541 (rpl > dpl && cpl > dpl)))
1547 /* mark segment as accessed */
1549 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1550 if (ret != X86EMUL_CONTINUE)
1552 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1553 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1554 sizeof(base3), &ctxt->exception);
1555 if (ret != X86EMUL_CONTINUE)
1559 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1560 return X86EMUL_CONTINUE;
1562 return emulate_exception(ctxt, err_vec, err_code, true);
1565 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1566 u16 selector, int seg)
1568 u8 cpl = ctxt->ops->cpl(ctxt);
1569 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1572 static void write_register_operand(struct operand *op)
1574 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1575 switch (op->bytes) {
1577 *(u8 *)op->addr.reg = (u8)op->val;
1580 *(u16 *)op->addr.reg = (u16)op->val;
1583 *op->addr.reg = (u32)op->val;
1584 break; /* 64b: zero-extend */
1586 *op->addr.reg = op->val;
1591 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1595 write_register_operand(op);
1598 if (ctxt->lock_prefix)
1599 return segmented_cmpxchg(ctxt,
1605 return segmented_write(ctxt,
1611 return segmented_write(ctxt,
1614 op->bytes * op->count);
1617 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1620 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1628 return X86EMUL_CONTINUE;
1631 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1633 struct segmented_address addr;
1635 rsp_increment(ctxt, -bytes);
1636 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1637 addr.seg = VCPU_SREG_SS;
1639 return segmented_write(ctxt, addr, data, bytes);
1642 static int em_push(struct x86_emulate_ctxt *ctxt)
1644 /* Disable writeback. */
1645 ctxt->dst.type = OP_NONE;
1646 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1649 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1650 void *dest, int len)
1653 struct segmented_address addr;
1655 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1656 addr.seg = VCPU_SREG_SS;
1657 rc = segmented_read(ctxt, addr, dest, len);
1658 if (rc != X86EMUL_CONTINUE)
1661 rsp_increment(ctxt, len);
1665 static int em_pop(struct x86_emulate_ctxt *ctxt)
1667 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1670 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1671 void *dest, int len)
1674 unsigned long val, change_mask;
1675 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1676 int cpl = ctxt->ops->cpl(ctxt);
1678 rc = emulate_pop(ctxt, &val, len);
1679 if (rc != X86EMUL_CONTINUE)
1682 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1683 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1685 switch(ctxt->mode) {
1686 case X86EMUL_MODE_PROT64:
1687 case X86EMUL_MODE_PROT32:
1688 case X86EMUL_MODE_PROT16:
1690 change_mask |= EFLG_IOPL;
1692 change_mask |= EFLG_IF;
1694 case X86EMUL_MODE_VM86:
1696 return emulate_gp(ctxt, 0);
1697 change_mask |= EFLG_IF;
1699 default: /* real mode */
1700 change_mask |= (EFLG_IOPL | EFLG_IF);
1704 *(unsigned long *)dest =
1705 (ctxt->eflags & ~change_mask) | (val & change_mask);
1710 static int em_popf(struct x86_emulate_ctxt *ctxt)
1712 ctxt->dst.type = OP_REG;
1713 ctxt->dst.addr.reg = &ctxt->eflags;
1714 ctxt->dst.bytes = ctxt->op_bytes;
1715 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1718 static int em_enter(struct x86_emulate_ctxt *ctxt)
1721 unsigned frame_size = ctxt->src.val;
1722 unsigned nesting_level = ctxt->src2.val & 31;
1726 return X86EMUL_UNHANDLEABLE;
1728 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1729 rc = push(ctxt, &rbp, stack_size(ctxt));
1730 if (rc != X86EMUL_CONTINUE)
1732 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1734 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1735 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1737 return X86EMUL_CONTINUE;
1740 static int em_leave(struct x86_emulate_ctxt *ctxt)
1742 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1744 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1747 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1749 int seg = ctxt->src2.val;
1751 ctxt->src.val = get_segment_selector(ctxt, seg);
1753 return em_push(ctxt);
1756 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1758 int seg = ctxt->src2.val;
1759 unsigned long selector;
1762 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1763 if (rc != X86EMUL_CONTINUE)
1766 if (ctxt->modrm_reg == VCPU_SREG_SS)
1767 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1769 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1773 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1775 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1776 int rc = X86EMUL_CONTINUE;
1777 int reg = VCPU_REGS_RAX;
1779 while (reg <= VCPU_REGS_RDI) {
1780 (reg == VCPU_REGS_RSP) ?
1781 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1784 if (rc != X86EMUL_CONTINUE)
1793 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1795 ctxt->src.val = (unsigned long)ctxt->eflags;
1796 return em_push(ctxt);
1799 static int em_popa(struct x86_emulate_ctxt *ctxt)
1801 int rc = X86EMUL_CONTINUE;
1802 int reg = VCPU_REGS_RDI;
1804 while (reg >= VCPU_REGS_RAX) {
1805 if (reg == VCPU_REGS_RSP) {
1806 rsp_increment(ctxt, ctxt->op_bytes);
1810 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1811 if (rc != X86EMUL_CONTINUE)
1818 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1820 const struct x86_emulate_ops *ops = ctxt->ops;
1827 /* TODO: Add limit checks */
1828 ctxt->src.val = ctxt->eflags;
1830 if (rc != X86EMUL_CONTINUE)
1833 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1835 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1837 if (rc != X86EMUL_CONTINUE)
1840 ctxt->src.val = ctxt->_eip;
1842 if (rc != X86EMUL_CONTINUE)
1845 ops->get_idt(ctxt, &dt);
1847 eip_addr = dt.address + (irq << 2);
1848 cs_addr = dt.address + (irq << 2) + 2;
1850 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1851 if (rc != X86EMUL_CONTINUE)
1854 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1855 if (rc != X86EMUL_CONTINUE)
1858 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1859 if (rc != X86EMUL_CONTINUE)
1867 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1871 invalidate_registers(ctxt);
1872 rc = __emulate_int_real(ctxt, irq);
1873 if (rc == X86EMUL_CONTINUE)
1874 writeback_registers(ctxt);
1878 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1880 switch(ctxt->mode) {
1881 case X86EMUL_MODE_REAL:
1882 return __emulate_int_real(ctxt, irq);
1883 case X86EMUL_MODE_VM86:
1884 case X86EMUL_MODE_PROT16:
1885 case X86EMUL_MODE_PROT32:
1886 case X86EMUL_MODE_PROT64:
1888 /* Protected mode interrupts unimplemented yet */
1889 return X86EMUL_UNHANDLEABLE;
1893 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1895 int rc = X86EMUL_CONTINUE;
1896 unsigned long temp_eip = 0;
1897 unsigned long temp_eflags = 0;
1898 unsigned long cs = 0;
1899 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1900 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1901 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1902 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1904 /* TODO: Add stack limit check */
1906 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1908 if (rc != X86EMUL_CONTINUE)
1911 if (temp_eip & ~0xffff)
1912 return emulate_gp(ctxt, 0);
1914 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1916 if (rc != X86EMUL_CONTINUE)
1919 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1921 if (rc != X86EMUL_CONTINUE)
1924 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1926 if (rc != X86EMUL_CONTINUE)
1929 ctxt->_eip = temp_eip;
1932 if (ctxt->op_bytes == 4)
1933 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1934 else if (ctxt->op_bytes == 2) {
1935 ctxt->eflags &= ~0xffff;
1936 ctxt->eflags |= temp_eflags;
1939 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1940 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1945 static int em_iret(struct x86_emulate_ctxt *ctxt)
1947 switch(ctxt->mode) {
1948 case X86EMUL_MODE_REAL:
1949 return emulate_iret_real(ctxt);
1950 case X86EMUL_MODE_VM86:
1951 case X86EMUL_MODE_PROT16:
1952 case X86EMUL_MODE_PROT32:
1953 case X86EMUL_MODE_PROT64:
1955 /* iret from protected mode unimplemented yet */
1956 return X86EMUL_UNHANDLEABLE;
1960 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1965 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1967 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1968 if (rc != X86EMUL_CONTINUE)
1972 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1973 return X86EMUL_CONTINUE;
1976 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1978 int rc = X86EMUL_CONTINUE;
1980 switch (ctxt->modrm_reg) {
1981 case 2: /* call near abs */ {
1983 old_eip = ctxt->_eip;
1984 ctxt->_eip = ctxt->src.val;
1985 ctxt->src.val = old_eip;
1989 case 4: /* jmp abs */
1990 ctxt->_eip = ctxt->src.val;
1992 case 5: /* jmp far */
1993 rc = em_jmp_far(ctxt);
2002 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2004 u64 old = ctxt->dst.orig_val64;
2006 if (ctxt->dst.bytes == 16)
2007 return X86EMUL_UNHANDLEABLE;
2009 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2010 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2011 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2012 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2013 ctxt->eflags &= ~EFLG_ZF;
2015 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2016 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2018 ctxt->eflags |= EFLG_ZF;
2020 return X86EMUL_CONTINUE;
2023 static int em_ret(struct x86_emulate_ctxt *ctxt)
2025 ctxt->dst.type = OP_REG;
2026 ctxt->dst.addr.reg = &ctxt->_eip;
2027 ctxt->dst.bytes = ctxt->op_bytes;
2028 return em_pop(ctxt);
2031 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2035 int cpl = ctxt->ops->cpl(ctxt);
2037 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2038 if (rc != X86EMUL_CONTINUE)
2040 if (ctxt->op_bytes == 4)
2041 ctxt->_eip = (u32)ctxt->_eip;
2042 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2043 if (rc != X86EMUL_CONTINUE)
2045 /* Outer-privilege level return is not implemented */
2046 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2047 return X86EMUL_UNHANDLEABLE;
2048 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2052 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2056 rc = em_ret_far(ctxt);
2057 if (rc != X86EMUL_CONTINUE)
2059 rsp_increment(ctxt, ctxt->src.val);
2060 return X86EMUL_CONTINUE;
2063 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2065 /* Save real source value, then compare EAX against destination. */
2066 ctxt->dst.orig_val = ctxt->dst.val;
2067 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2068 ctxt->src.orig_val = ctxt->src.val;
2069 ctxt->src.val = ctxt->dst.orig_val;
2070 fastop(ctxt, em_cmp);
2072 if (ctxt->eflags & EFLG_ZF) {
2073 /* Success: write back to memory. */
2074 ctxt->dst.val = ctxt->src.orig_val;
2076 /* Failure: write the value we saw to EAX. */
2077 ctxt->dst.type = OP_REG;
2078 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2079 ctxt->dst.val = ctxt->dst.orig_val;
2081 return X86EMUL_CONTINUE;
2084 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2086 int seg = ctxt->src2.val;
2090 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2092 rc = load_segment_descriptor(ctxt, sel, seg);
2093 if (rc != X86EMUL_CONTINUE)
2096 ctxt->dst.val = ctxt->src.val;
2101 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2102 struct desc_struct *cs, struct desc_struct *ss)
2104 cs->l = 0; /* will be adjusted later */
2105 set_desc_base(cs, 0); /* flat segment */
2106 cs->g = 1; /* 4kb granularity */
2107 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2108 cs->type = 0x0b; /* Read, Execute, Accessed */
2110 cs->dpl = 0; /* will be adjusted later */
2115 set_desc_base(ss, 0); /* flat segment */
2116 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2117 ss->g = 1; /* 4kb granularity */
2119 ss->type = 0x03; /* Read/Write, Accessed */
2120 ss->d = 1; /* 32bit stack segment */
2127 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2129 u32 eax, ebx, ecx, edx;
2132 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2133 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2134 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2135 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2138 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2140 const struct x86_emulate_ops *ops = ctxt->ops;
2141 u32 eax, ebx, ecx, edx;
2144 * syscall should always be enabled in longmode - so only become
2145 * vendor specific (cpuid) if other modes are active...
2147 if (ctxt->mode == X86EMUL_MODE_PROT64)
2152 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2154 * Intel ("GenuineIntel")
2155 * remark: Intel CPUs only support "syscall" in 64bit
2156 * longmode. Also an 64bit guest with a
2157 * 32bit compat-app running will #UD !! While this
2158 * behaviour can be fixed (by emulating) into AMD
2159 * response - CPUs of AMD can't behave like Intel.
2161 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2162 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2163 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2166 /* AMD ("AuthenticAMD") */
2167 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2168 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2169 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2172 /* AMD ("AMDisbetter!") */
2173 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2174 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2175 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2178 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2182 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2184 const struct x86_emulate_ops *ops = ctxt->ops;
2185 struct desc_struct cs, ss;
2190 /* syscall is not available in real mode */
2191 if (ctxt->mode == X86EMUL_MODE_REAL ||
2192 ctxt->mode == X86EMUL_MODE_VM86)
2193 return emulate_ud(ctxt);
2195 if (!(em_syscall_is_enabled(ctxt)))
2196 return emulate_ud(ctxt);
2198 ops->get_msr(ctxt, MSR_EFER, &efer);
2199 setup_syscalls_segments(ctxt, &cs, &ss);
2201 if (!(efer & EFER_SCE))
2202 return emulate_ud(ctxt);
2204 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2206 cs_sel = (u16)(msr_data & 0xfffc);
2207 ss_sel = (u16)(msr_data + 8);
2209 if (efer & EFER_LMA) {
2213 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2214 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2216 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2217 if (efer & EFER_LMA) {
2218 #ifdef CONFIG_X86_64
2219 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2222 ctxt->mode == X86EMUL_MODE_PROT64 ?
2223 MSR_LSTAR : MSR_CSTAR, &msr_data);
2224 ctxt->_eip = msr_data;
2226 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2227 ctxt->eflags &= ~msr_data;
2231 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2232 ctxt->_eip = (u32)msr_data;
2234 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2237 return X86EMUL_CONTINUE;
2240 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2242 const struct x86_emulate_ops *ops = ctxt->ops;
2243 struct desc_struct cs, ss;
2248 ops->get_msr(ctxt, MSR_EFER, &efer);
2249 /* inject #GP if in real mode */
2250 if (ctxt->mode == X86EMUL_MODE_REAL)
2251 return emulate_gp(ctxt, 0);
2254 * Not recognized on AMD in compat mode (but is recognized in legacy
2257 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2258 && !vendor_intel(ctxt))
2259 return emulate_ud(ctxt);
2261 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2262 * Therefore, we inject an #UD.
2264 if (ctxt->mode == X86EMUL_MODE_PROT64)
2265 return emulate_ud(ctxt);
2267 setup_syscalls_segments(ctxt, &cs, &ss);
2269 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2270 switch (ctxt->mode) {
2271 case X86EMUL_MODE_PROT32:
2272 if ((msr_data & 0xfffc) == 0x0)
2273 return emulate_gp(ctxt, 0);
2275 case X86EMUL_MODE_PROT64:
2276 if (msr_data == 0x0)
2277 return emulate_gp(ctxt, 0);
2283 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2284 cs_sel = (u16)msr_data;
2285 cs_sel &= ~SELECTOR_RPL_MASK;
2286 ss_sel = cs_sel + 8;
2287 ss_sel &= ~SELECTOR_RPL_MASK;
2288 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2293 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2294 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2296 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2297 ctxt->_eip = msr_data;
2299 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2300 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2302 return X86EMUL_CONTINUE;
2305 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2307 const struct x86_emulate_ops *ops = ctxt->ops;
2308 struct desc_struct cs, ss;
2311 u16 cs_sel = 0, ss_sel = 0;
2313 /* inject #GP if in real mode or Virtual 8086 mode */
2314 if (ctxt->mode == X86EMUL_MODE_REAL ||
2315 ctxt->mode == X86EMUL_MODE_VM86)
2316 return emulate_gp(ctxt, 0);
2318 setup_syscalls_segments(ctxt, &cs, &ss);
2320 if ((ctxt->rex_prefix & 0x8) != 0x0)
2321 usermode = X86EMUL_MODE_PROT64;
2323 usermode = X86EMUL_MODE_PROT32;
2327 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2329 case X86EMUL_MODE_PROT32:
2330 cs_sel = (u16)(msr_data + 16);
2331 if ((msr_data & 0xfffc) == 0x0)
2332 return emulate_gp(ctxt, 0);
2333 ss_sel = (u16)(msr_data + 24);
2335 case X86EMUL_MODE_PROT64:
2336 cs_sel = (u16)(msr_data + 32);
2337 if (msr_data == 0x0)
2338 return emulate_gp(ctxt, 0);
2339 ss_sel = cs_sel + 8;
2344 cs_sel |= SELECTOR_RPL_MASK;
2345 ss_sel |= SELECTOR_RPL_MASK;
2347 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2348 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2350 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2351 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2353 return X86EMUL_CONTINUE;
2356 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2359 if (ctxt->mode == X86EMUL_MODE_REAL)
2361 if (ctxt->mode == X86EMUL_MODE_VM86)
2363 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2364 return ctxt->ops->cpl(ctxt) > iopl;
2367 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2370 const struct x86_emulate_ops *ops = ctxt->ops;
2371 struct desc_struct tr_seg;
2374 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2375 unsigned mask = (1 << len) - 1;
2378 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2381 if (desc_limit_scaled(&tr_seg) < 103)
2383 base = get_desc_base(&tr_seg);
2384 #ifdef CONFIG_X86_64
2385 base |= ((u64)base3) << 32;
2387 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2388 if (r != X86EMUL_CONTINUE)
2390 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2392 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2393 if (r != X86EMUL_CONTINUE)
2395 if ((perm >> bit_idx) & mask)
2400 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2406 if (emulator_bad_iopl(ctxt))
2407 if (!emulator_io_port_access_allowed(ctxt, port, len))
2410 ctxt->perm_ok = true;
2415 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2416 struct tss_segment_16 *tss)
2418 tss->ip = ctxt->_eip;
2419 tss->flag = ctxt->eflags;
2420 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2421 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2422 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2423 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2424 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2425 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2426 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2427 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2429 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2430 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2431 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2432 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2433 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2436 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2437 struct tss_segment_16 *tss)
2442 ctxt->_eip = tss->ip;
2443 ctxt->eflags = tss->flag | 2;
2444 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2445 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2446 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2447 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2448 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2449 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2450 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2451 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2454 * SDM says that segment selectors are loaded before segment
2457 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2458 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2459 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2460 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2461 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2466 * Now load segment descriptors. If fault happens at this stage
2467 * it is handled in a context of new task
2469 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2470 if (ret != X86EMUL_CONTINUE)
2472 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2473 if (ret != X86EMUL_CONTINUE)
2475 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2476 if (ret != X86EMUL_CONTINUE)
2478 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2479 if (ret != X86EMUL_CONTINUE)
2481 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2482 if (ret != X86EMUL_CONTINUE)
2485 return X86EMUL_CONTINUE;
2488 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2489 u16 tss_selector, u16 old_tss_sel,
2490 ulong old_tss_base, struct desc_struct *new_desc)
2492 const struct x86_emulate_ops *ops = ctxt->ops;
2493 struct tss_segment_16 tss_seg;
2495 u32 new_tss_base = get_desc_base(new_desc);
2497 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2499 if (ret != X86EMUL_CONTINUE)
2500 /* FIXME: need to provide precise fault address */
2503 save_state_to_tss16(ctxt, &tss_seg);
2505 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2507 if (ret != X86EMUL_CONTINUE)
2508 /* FIXME: need to provide precise fault address */
2511 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2513 if (ret != X86EMUL_CONTINUE)
2514 /* FIXME: need to provide precise fault address */
2517 if (old_tss_sel != 0xffff) {
2518 tss_seg.prev_task_link = old_tss_sel;
2520 ret = ops->write_std(ctxt, new_tss_base,
2521 &tss_seg.prev_task_link,
2522 sizeof tss_seg.prev_task_link,
2524 if (ret != X86EMUL_CONTINUE)
2525 /* FIXME: need to provide precise fault address */
2529 return load_state_from_tss16(ctxt, &tss_seg);
2532 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2533 struct tss_segment_32 *tss)
2535 /* CR3 and ldt selector are not saved intentionally */
2536 tss->eip = ctxt->_eip;
2537 tss->eflags = ctxt->eflags;
2538 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2539 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2540 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2541 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2542 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2543 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2544 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2545 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2547 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2548 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2549 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2550 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2551 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2552 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2555 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2556 struct tss_segment_32 *tss)
2561 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2562 return emulate_gp(ctxt, 0);
2563 ctxt->_eip = tss->eip;
2564 ctxt->eflags = tss->eflags | 2;
2566 /* General purpose registers */
2567 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2568 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2569 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2570 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2571 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2572 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2573 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2574 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2577 * SDM says that segment selectors are loaded before segment
2578 * descriptors. This is important because CPL checks will
2581 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2582 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2583 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2584 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2585 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2586 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2587 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2590 * If we're switching between Protected Mode and VM86, we need to make
2591 * sure to update the mode before loading the segment descriptors so
2592 * that the selectors are interpreted correctly.
2594 if (ctxt->eflags & X86_EFLAGS_VM) {
2595 ctxt->mode = X86EMUL_MODE_VM86;
2598 ctxt->mode = X86EMUL_MODE_PROT32;
2603 * Now load segment descriptors. If fault happenes at this stage
2604 * it is handled in a context of new task
2606 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2607 if (ret != X86EMUL_CONTINUE)
2609 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2610 if (ret != X86EMUL_CONTINUE)
2612 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2613 if (ret != X86EMUL_CONTINUE)
2615 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2616 if (ret != X86EMUL_CONTINUE)
2618 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2619 if (ret != X86EMUL_CONTINUE)
2621 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2622 if (ret != X86EMUL_CONTINUE)
2624 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2625 if (ret != X86EMUL_CONTINUE)
2628 return X86EMUL_CONTINUE;
2631 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2632 u16 tss_selector, u16 old_tss_sel,
2633 ulong old_tss_base, struct desc_struct *new_desc)
2635 const struct x86_emulate_ops *ops = ctxt->ops;
2636 struct tss_segment_32 tss_seg;
2638 u32 new_tss_base = get_desc_base(new_desc);
2639 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2640 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2642 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2644 if (ret != X86EMUL_CONTINUE)
2645 /* FIXME: need to provide precise fault address */
2648 save_state_to_tss32(ctxt, &tss_seg);
2650 /* Only GP registers and segment selectors are saved */
2651 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2652 ldt_sel_offset - eip_offset, &ctxt->exception);
2653 if (ret != X86EMUL_CONTINUE)
2654 /* FIXME: need to provide precise fault address */
2657 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2659 if (ret != X86EMUL_CONTINUE)
2660 /* FIXME: need to provide precise fault address */
2663 if (old_tss_sel != 0xffff) {
2664 tss_seg.prev_task_link = old_tss_sel;
2666 ret = ops->write_std(ctxt, new_tss_base,
2667 &tss_seg.prev_task_link,
2668 sizeof tss_seg.prev_task_link,
2670 if (ret != X86EMUL_CONTINUE)
2671 /* FIXME: need to provide precise fault address */
2675 return load_state_from_tss32(ctxt, &tss_seg);
2678 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2679 u16 tss_selector, int idt_index, int reason,
2680 bool has_error_code, u32 error_code)
2682 const struct x86_emulate_ops *ops = ctxt->ops;
2683 struct desc_struct curr_tss_desc, next_tss_desc;
2685 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2686 ulong old_tss_base =
2687 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2691 /* FIXME: old_tss_base == ~0 ? */
2693 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2694 if (ret != X86EMUL_CONTINUE)
2696 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2697 if (ret != X86EMUL_CONTINUE)
2700 /* FIXME: check that next_tss_desc is tss */
2703 * Check privileges. The three cases are task switch caused by...
2705 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2706 * 2. Exception/IRQ/iret: No check is performed
2707 * 3. jmp/call to TSS: Check against DPL of the TSS
2709 if (reason == TASK_SWITCH_GATE) {
2710 if (idt_index != -1) {
2711 /* Software interrupts */
2712 struct desc_struct task_gate_desc;
2715 ret = read_interrupt_descriptor(ctxt, idt_index,
2717 if (ret != X86EMUL_CONTINUE)
2720 dpl = task_gate_desc.dpl;
2721 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2722 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2724 } else if (reason != TASK_SWITCH_IRET) {
2725 int dpl = next_tss_desc.dpl;
2726 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2727 return emulate_gp(ctxt, tss_selector);
2731 desc_limit = desc_limit_scaled(&next_tss_desc);
2732 if (!next_tss_desc.p ||
2733 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2734 desc_limit < 0x2b)) {
2735 return emulate_ts(ctxt, tss_selector & 0xfffc);
2738 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2739 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2740 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2743 if (reason == TASK_SWITCH_IRET)
2744 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2746 /* set back link to prev task only if NT bit is set in eflags
2747 note that old_tss_sel is not used after this point */
2748 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2749 old_tss_sel = 0xffff;
2751 if (next_tss_desc.type & 8)
2752 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2753 old_tss_base, &next_tss_desc);
2755 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2756 old_tss_base, &next_tss_desc);
2757 if (ret != X86EMUL_CONTINUE)
2760 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2761 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2763 if (reason != TASK_SWITCH_IRET) {
2764 next_tss_desc.type |= (1 << 1); /* set busy flag */
2765 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2768 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2769 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2771 if (has_error_code) {
2772 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2773 ctxt->lock_prefix = 0;
2774 ctxt->src.val = (unsigned long) error_code;
2775 ret = em_push(ctxt);
2781 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2782 u16 tss_selector, int idt_index, int reason,
2783 bool has_error_code, u32 error_code)
2787 invalidate_registers(ctxt);
2788 ctxt->_eip = ctxt->eip;
2789 ctxt->dst.type = OP_NONE;
2791 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2792 has_error_code, error_code);
2794 if (rc == X86EMUL_CONTINUE) {
2795 ctxt->eip = ctxt->_eip;
2796 writeback_registers(ctxt);
2799 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2802 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2805 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2807 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2808 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2811 static int em_das(struct x86_emulate_ctxt *ctxt)
2814 bool af, cf, old_cf;
2816 cf = ctxt->eflags & X86_EFLAGS_CF;
2822 af = ctxt->eflags & X86_EFLAGS_AF;
2823 if ((al & 0x0f) > 9 || af) {
2825 cf = old_cf | (al >= 250);
2830 if (old_al > 0x99 || old_cf) {
2836 /* Set PF, ZF, SF */
2837 ctxt->src.type = OP_IMM;
2839 ctxt->src.bytes = 1;
2840 fastop(ctxt, em_or);
2841 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2843 ctxt->eflags |= X86_EFLAGS_CF;
2845 ctxt->eflags |= X86_EFLAGS_AF;
2846 return X86EMUL_CONTINUE;
2849 static int em_aam(struct x86_emulate_ctxt *ctxt)
2853 if (ctxt->src.val == 0)
2854 return emulate_de(ctxt);
2856 al = ctxt->dst.val & 0xff;
2857 ah = al / ctxt->src.val;
2858 al %= ctxt->src.val;
2860 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2862 /* Set PF, ZF, SF */
2863 ctxt->src.type = OP_IMM;
2865 ctxt->src.bytes = 1;
2866 fastop(ctxt, em_or);
2868 return X86EMUL_CONTINUE;
2871 static int em_aad(struct x86_emulate_ctxt *ctxt)
2873 u8 al = ctxt->dst.val & 0xff;
2874 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2876 al = (al + (ah * ctxt->src.val)) & 0xff;
2878 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2880 /* Set PF, ZF, SF */
2881 ctxt->src.type = OP_IMM;
2883 ctxt->src.bytes = 1;
2884 fastop(ctxt, em_or);
2886 return X86EMUL_CONTINUE;
2889 static int em_call(struct x86_emulate_ctxt *ctxt)
2891 long rel = ctxt->src.val;
2893 ctxt->src.val = (unsigned long)ctxt->_eip;
2895 return em_push(ctxt);
2898 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2904 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2905 old_eip = ctxt->_eip;
2907 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2908 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2909 return X86EMUL_CONTINUE;
2912 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2914 ctxt->src.val = old_cs;
2916 if (rc != X86EMUL_CONTINUE)
2919 ctxt->src.val = old_eip;
2920 return em_push(ctxt);
2923 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2927 ctxt->dst.type = OP_REG;
2928 ctxt->dst.addr.reg = &ctxt->_eip;
2929 ctxt->dst.bytes = ctxt->op_bytes;
2930 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2931 if (rc != X86EMUL_CONTINUE)
2933 rsp_increment(ctxt, ctxt->src.val);
2934 return X86EMUL_CONTINUE;
2937 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2939 /* Write back the register source. */
2940 ctxt->src.val = ctxt->dst.val;
2941 write_register_operand(&ctxt->src);
2943 /* Write back the memory destination with implicit LOCK prefix. */
2944 ctxt->dst.val = ctxt->src.orig_val;
2945 ctxt->lock_prefix = 1;
2946 return X86EMUL_CONTINUE;
2949 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2951 ctxt->dst.val = ctxt->src2.val;
2952 return fastop(ctxt, em_imul);
2955 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2957 ctxt->dst.type = OP_REG;
2958 ctxt->dst.bytes = ctxt->src.bytes;
2959 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2960 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2962 return X86EMUL_CONTINUE;
2965 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2969 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2970 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2971 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2972 return X86EMUL_CONTINUE;
2975 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2979 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2980 return emulate_gp(ctxt, 0);
2981 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2982 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2983 return X86EMUL_CONTINUE;
2986 static int em_mov(struct x86_emulate_ctxt *ctxt)
2988 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
2989 return X86EMUL_CONTINUE;
2992 #define FFL(x) bit(X86_FEATURE_##x)
2994 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2996 u32 ebx, ecx, edx, eax = 1;
3000 * Check MOVBE is set in the guest-visible CPUID leaf.
3002 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3003 if (!(ecx & FFL(MOVBE)))
3004 return emulate_ud(ctxt);
3006 switch (ctxt->op_bytes) {
3009 * From MOVBE definition: "...When the operand size is 16 bits,
3010 * the upper word of the destination register remains unchanged
3013 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3014 * rules so we have to do the operation almost per hand.
3016 tmp = (u16)ctxt->src.val;
3017 ctxt->dst.val &= ~0xffffUL;
3018 ctxt->dst.val |= (unsigned long)swab16(tmp);
3021 ctxt->dst.val = swab32((u32)ctxt->src.val);
3024 ctxt->dst.val = swab64(ctxt->src.val);
3029 return X86EMUL_CONTINUE;
3032 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3034 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3035 return emulate_gp(ctxt, 0);
3037 /* Disable writeback. */
3038 ctxt->dst.type = OP_NONE;
3039 return X86EMUL_CONTINUE;
3042 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3046 if (ctxt->mode == X86EMUL_MODE_PROT64)
3047 val = ctxt->src.val & ~0ULL;
3049 val = ctxt->src.val & ~0U;
3051 /* #UD condition is already handled. */
3052 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3053 return emulate_gp(ctxt, 0);
3055 /* Disable writeback. */
3056 ctxt->dst.type = OP_NONE;
3057 return X86EMUL_CONTINUE;
3060 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3064 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3065 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3066 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3067 return emulate_gp(ctxt, 0);
3069 return X86EMUL_CONTINUE;
3072 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3076 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3077 return emulate_gp(ctxt, 0);
3079 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3080 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3081 return X86EMUL_CONTINUE;
3084 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3086 if (ctxt->modrm_reg > VCPU_SREG_GS)
3087 return emulate_ud(ctxt);
3089 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3090 return X86EMUL_CONTINUE;
3093 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3095 u16 sel = ctxt->src.val;
3097 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3098 return emulate_ud(ctxt);
3100 if (ctxt->modrm_reg == VCPU_SREG_SS)
3101 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3103 /* Disable writeback. */
3104 ctxt->dst.type = OP_NONE;
3105 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3108 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3110 u16 sel = ctxt->src.val;
3112 /* Disable writeback. */
3113 ctxt->dst.type = OP_NONE;
3114 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3117 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3119 u16 sel = ctxt->src.val;
3121 /* Disable writeback. */
3122 ctxt->dst.type = OP_NONE;
3123 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3126 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3131 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3132 if (rc == X86EMUL_CONTINUE)
3133 ctxt->ops->invlpg(ctxt, linear);
3134 /* Disable writeback. */
3135 ctxt->dst.type = OP_NONE;
3136 return X86EMUL_CONTINUE;
3139 static int em_clts(struct x86_emulate_ctxt *ctxt)
3143 cr0 = ctxt->ops->get_cr(ctxt, 0);
3145 ctxt->ops->set_cr(ctxt, 0, cr0);
3146 return X86EMUL_CONTINUE;
3149 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3151 int rc = ctxt->ops->fix_hypercall(ctxt);
3153 if (rc != X86EMUL_CONTINUE)
3156 /* Let the processor re-execute the fixed hypercall */
3157 ctxt->_eip = ctxt->eip;
3158 /* Disable writeback. */
3159 ctxt->dst.type = OP_NONE;
3160 return X86EMUL_CONTINUE;
3163 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3164 void (*get)(struct x86_emulate_ctxt *ctxt,
3165 struct desc_ptr *ptr))
3167 struct desc_ptr desc_ptr;
3169 if (ctxt->mode == X86EMUL_MODE_PROT64)
3171 get(ctxt, &desc_ptr);
3172 if (ctxt->op_bytes == 2) {
3174 desc_ptr.address &= 0x00ffffff;
3176 /* Disable writeback. */
3177 ctxt->dst.type = OP_NONE;
3178 return segmented_write(ctxt, ctxt->dst.addr.mem,
3179 &desc_ptr, 2 + ctxt->op_bytes);
3182 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3184 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3187 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3189 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3192 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3194 struct desc_ptr desc_ptr;
3197 if (ctxt->mode == X86EMUL_MODE_PROT64)
3199 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3200 &desc_ptr.size, &desc_ptr.address,
3202 if (rc != X86EMUL_CONTINUE)
3204 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3205 /* Disable writeback. */
3206 ctxt->dst.type = OP_NONE;
3207 return X86EMUL_CONTINUE;
3210 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3214 rc = ctxt->ops->fix_hypercall(ctxt);
3216 /* Disable writeback. */
3217 ctxt->dst.type = OP_NONE;
3221 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3223 struct desc_ptr desc_ptr;
3226 if (ctxt->mode == X86EMUL_MODE_PROT64)
3228 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3229 &desc_ptr.size, &desc_ptr.address,
3231 if (rc != X86EMUL_CONTINUE)
3233 ctxt->ops->set_idt(ctxt, &desc_ptr);
3234 /* Disable writeback. */
3235 ctxt->dst.type = OP_NONE;
3236 return X86EMUL_CONTINUE;
3239 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3241 if (ctxt->dst.type == OP_MEM)
3242 ctxt->dst.bytes = 2;
3243 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3244 return X86EMUL_CONTINUE;
3247 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3249 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3250 | (ctxt->src.val & 0x0f));
3251 ctxt->dst.type = OP_NONE;
3252 return X86EMUL_CONTINUE;
3255 static int em_loop(struct x86_emulate_ctxt *ctxt)
3257 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3258 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3259 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3260 jmp_rel(ctxt, ctxt->src.val);
3262 return X86EMUL_CONTINUE;
3265 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3267 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3268 jmp_rel(ctxt, ctxt->src.val);
3270 return X86EMUL_CONTINUE;
3273 static int em_in(struct x86_emulate_ctxt *ctxt)
3275 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3277 return X86EMUL_IO_NEEDED;
3279 return X86EMUL_CONTINUE;
3282 static int em_out(struct x86_emulate_ctxt *ctxt)
3284 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3286 /* Disable writeback. */
3287 ctxt->dst.type = OP_NONE;
3288 return X86EMUL_CONTINUE;
3291 static int em_cli(struct x86_emulate_ctxt *ctxt)
3293 if (emulator_bad_iopl(ctxt))
3294 return emulate_gp(ctxt, 0);
3296 ctxt->eflags &= ~X86_EFLAGS_IF;
3297 return X86EMUL_CONTINUE;
3300 static int em_sti(struct x86_emulate_ctxt *ctxt)
3302 if (emulator_bad_iopl(ctxt))
3303 return emulate_gp(ctxt, 0);
3305 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3306 ctxt->eflags |= X86_EFLAGS_IF;
3307 return X86EMUL_CONTINUE;
3310 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3312 u32 eax, ebx, ecx, edx;
3314 eax = reg_read(ctxt, VCPU_REGS_RAX);
3315 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3316 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3317 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3318 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3319 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3320 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3321 return X86EMUL_CONTINUE;
3324 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3328 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3329 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3331 ctxt->eflags &= ~0xffUL;
3332 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3333 return X86EMUL_CONTINUE;
3336 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3338 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3339 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3340 return X86EMUL_CONTINUE;
3343 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3345 switch (ctxt->op_bytes) {
3346 #ifdef CONFIG_X86_64
3348 asm("bswap %0" : "+r"(ctxt->dst.val));
3352 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3355 return X86EMUL_CONTINUE;
3358 static bool valid_cr(int nr)
3370 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3372 if (!valid_cr(ctxt->modrm_reg))
3373 return emulate_ud(ctxt);
3375 return X86EMUL_CONTINUE;
3378 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3380 u64 new_val = ctxt->src.val64;
3381 int cr = ctxt->modrm_reg;
3384 static u64 cr_reserved_bits[] = {
3385 0xffffffff00000000ULL,
3386 0, 0, 0, /* CR3 checked later */
3393 return emulate_ud(ctxt);
3395 if (new_val & cr_reserved_bits[cr])
3396 return emulate_gp(ctxt, 0);
3401 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3402 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3403 return emulate_gp(ctxt, 0);
3405 cr4 = ctxt->ops->get_cr(ctxt, 4);
3406 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3408 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3409 !(cr4 & X86_CR4_PAE))
3410 return emulate_gp(ctxt, 0);
3417 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3418 if (efer & EFER_LMA)
3419 rsvd = CR3_L_MODE_RESERVED_BITS;
3422 return emulate_gp(ctxt, 0);
3427 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3429 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3430 return emulate_gp(ctxt, 0);
3436 return X86EMUL_CONTINUE;
3439 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3443 ctxt->ops->get_dr(ctxt, 7, &dr7);
3445 /* Check if DR7.Global_Enable is set */
3446 return dr7 & (1 << 13);
3449 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3451 int dr = ctxt->modrm_reg;
3455 return emulate_ud(ctxt);
3457 cr4 = ctxt->ops->get_cr(ctxt, 4);
3458 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3459 return emulate_ud(ctxt);
3461 if (check_dr7_gd(ctxt))
3462 return emulate_db(ctxt);
3464 return X86EMUL_CONTINUE;
3467 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3469 u64 new_val = ctxt->src.val64;
3470 int dr = ctxt->modrm_reg;
3472 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3473 return emulate_gp(ctxt, 0);
3475 return check_dr_read(ctxt);
3478 static int check_svme(struct x86_emulate_ctxt *ctxt)
3482 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3484 if (!(efer & EFER_SVME))
3485 return emulate_ud(ctxt);
3487 return X86EMUL_CONTINUE;
3490 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3492 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3494 /* Valid physical address? */
3495 if (rax & 0xffff000000000000ULL)
3496 return emulate_gp(ctxt, 0);
3498 return check_svme(ctxt);
3501 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3503 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3505 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3506 return emulate_ud(ctxt);
3508 return X86EMUL_CONTINUE;
3511 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3513 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3514 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3516 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3517 ctxt->ops->check_pmc(ctxt, rcx))
3518 return emulate_gp(ctxt, 0);
3520 return X86EMUL_CONTINUE;
3523 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3525 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3526 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3527 return emulate_gp(ctxt, 0);
3529 return X86EMUL_CONTINUE;
3532 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3534 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3535 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3536 return emulate_gp(ctxt, 0);
3538 return X86EMUL_CONTINUE;
3541 #define D(_y) { .flags = (_y) }
3542 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3543 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3544 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3545 #define N D(NotImpl)
3546 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3547 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3548 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3549 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3550 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3551 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3552 #define II(_f, _e, _i) \
3553 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3554 #define IIP(_f, _e, _i, _p) \
3555 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3556 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3557 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3559 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3560 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3561 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3562 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3563 #define I2bvIP(_f, _e, _i, _p) \
3564 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3566 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3567 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3568 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3570 static const struct opcode group7_rm0[] = {
3572 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3576 static const struct opcode group7_rm1[] = {
3577 DI(SrcNone | Priv, monitor),
3578 DI(SrcNone | Priv, mwait),
3582 static const struct opcode group7_rm3[] = {
3583 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3584 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3585 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3586 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3587 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3588 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3589 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3590 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3593 static const struct opcode group7_rm7[] = {
3595 DIP(SrcNone, rdtscp, check_rdtsc),
3599 static const struct opcode group1[] = {
3601 F(Lock | PageTable, em_or),
3604 F(Lock | PageTable, em_and),
3610 static const struct opcode group1A[] = {
3611 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3614 static const struct opcode group2[] = {
3615 F(DstMem | ModRM, em_rol),
3616 F(DstMem | ModRM, em_ror),
3617 F(DstMem | ModRM, em_rcl),
3618 F(DstMem | ModRM, em_rcr),
3619 F(DstMem | ModRM, em_shl),
3620 F(DstMem | ModRM, em_shr),
3621 F(DstMem | ModRM, em_shl),
3622 F(DstMem | ModRM, em_sar),
3625 static const struct opcode group3[] = {
3626 F(DstMem | SrcImm | NoWrite, em_test),
3627 F(DstMem | SrcImm | NoWrite, em_test),
3628 F(DstMem | SrcNone | Lock, em_not),
3629 F(DstMem | SrcNone | Lock, em_neg),
3630 F(DstXacc | Src2Mem, em_mul_ex),
3631 F(DstXacc | Src2Mem, em_imul_ex),
3632 F(DstXacc | Src2Mem, em_div_ex),
3633 F(DstXacc | Src2Mem, em_idiv_ex),
3636 static const struct opcode group4[] = {
3637 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3638 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3642 static const struct opcode group5[] = {
3643 F(DstMem | SrcNone | Lock, em_inc),
3644 F(DstMem | SrcNone | Lock, em_dec),
3645 I(SrcMem | Stack, em_grp45),
3646 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3647 I(SrcMem | Stack, em_grp45),
3648 I(SrcMemFAddr | ImplicitOps, em_grp45),
3649 I(SrcMem | Stack, em_grp45), D(Undefined),
3652 static const struct opcode group6[] = {
3655 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3656 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3660 static const struct group_dual group7 = { {
3661 II(Mov | DstMem, em_sgdt, sgdt),
3662 II(Mov | DstMem, em_sidt, sidt),
3663 II(SrcMem | Priv, em_lgdt, lgdt),
3664 II(SrcMem | Priv, em_lidt, lidt),
3665 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3666 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3667 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3671 N, EXT(0, group7_rm3),
3672 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3673 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3677 static const struct opcode group8[] = {
3679 F(DstMem | SrcImmByte | NoWrite, em_bt),
3680 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3681 F(DstMem | SrcImmByte | Lock, em_btr),
3682 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3685 static const struct group_dual group9 = { {
3686 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3688 N, N, N, N, N, N, N, N,
3691 static const struct opcode group11[] = {
3692 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3696 static const struct gprefix pfx_0f_6f_0f_7f = {
3697 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3700 static const struct gprefix pfx_0f_2b = {
3701 I(0, em_mov), I(0, em_mov), N, N,
3704 static const struct gprefix pfx_0f_28_0f_29 = {
3705 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3708 static const struct gprefix pfx_0f_e7 = {
3709 N, I(Sse, em_mov), N, N,
3712 static const struct escape escape_d9 = { {
3713 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3716 N, N, N, N, N, N, N, N,
3718 N, N, N, N, N, N, N, N,
3720 N, N, N, N, N, N, N, N,
3722 N, N, N, N, N, N, N, N,
3724 N, N, N, N, N, N, N, N,
3726 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3730 N, N, N, N, N, N, N, N,
3733 static const struct escape escape_db = { {
3734 N, N, N, N, N, N, N, N,
3737 N, N, N, N, N, N, N, N,
3739 N, N, N, N, N, N, N, N,
3741 N, N, N, N, N, N, N, N,
3743 N, N, N, N, N, N, N, N,
3745 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3747 N, N, N, N, N, N, N, N,
3749 N, N, N, N, N, N, N, N,
3751 N, N, N, N, N, N, N, N,
3754 static const struct escape escape_dd = { {
3755 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3758 N, N, N, N, N, N, N, N,
3760 N, N, N, N, N, N, N, N,
3762 N, N, N, N, N, N, N, N,
3764 N, N, N, N, N, N, N, N,
3766 N, N, N, N, N, N, N, N,
3768 N, N, N, N, N, N, N, N,
3770 N, N, N, N, N, N, N, N,
3772 N, N, N, N, N, N, N, N,
3775 static const struct opcode opcode_table[256] = {
3777 F6ALU(Lock, em_add),
3778 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3779 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3781 F6ALU(Lock | PageTable, em_or),
3782 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3785 F6ALU(Lock, em_adc),
3786 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3787 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3789 F6ALU(Lock, em_sbb),
3790 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3791 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3793 F6ALU(Lock | PageTable, em_and), N, N,
3795 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3797 F6ALU(Lock, em_xor), N, N,
3799 F6ALU(NoWrite, em_cmp), N, N,
3801 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3803 X8(I(SrcReg | Stack, em_push)),
3805 X8(I(DstReg | Stack, em_pop)),
3807 I(ImplicitOps | Stack | No64, em_pusha),
3808 I(ImplicitOps | Stack | No64, em_popa),
3809 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3812 I(SrcImm | Mov | Stack, em_push),
3813 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3814 I(SrcImmByte | Mov | Stack, em_push),
3815 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3816 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3817 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3821 G(ByteOp | DstMem | SrcImm, group1),
3822 G(DstMem | SrcImm, group1),
3823 G(ByteOp | DstMem | SrcImm | No64, group1),
3824 G(DstMem | SrcImmByte, group1),
3825 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3826 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3828 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3829 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3830 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3831 D(ModRM | SrcMem | NoAccess | DstReg),
3832 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3835 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3837 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3838 I(SrcImmFAddr | No64, em_call_far), N,
3839 II(ImplicitOps | Stack, em_pushf, pushf),
3840 II(ImplicitOps | Stack, em_popf, popf),
3841 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3843 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3844 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3845 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3846 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3848 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3849 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3850 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3851 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3853 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3855 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3857 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3858 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3859 I(ImplicitOps | Stack, em_ret),
3860 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3861 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3862 G(ByteOp, group11), G(0, group11),
3864 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3865 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3866 I(ImplicitOps | Stack, em_ret_far),
3867 D(ImplicitOps), DI(SrcImmByte, intn),
3868 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3870 G(Src2One | ByteOp, group2), G(Src2One, group2),
3871 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3872 I(DstAcc | SrcImmUByte | No64, em_aam),
3873 I(DstAcc | SrcImmUByte | No64, em_aad),
3874 F(DstAcc | ByteOp | No64, em_salc),
3875 I(DstAcc | SrcXLat | ByteOp, em_mov),
3877 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3879 X3(I(SrcImmByte, em_loop)),
3880 I(SrcImmByte, em_jcxz),
3881 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3882 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3884 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3885 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3886 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3887 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3889 N, DI(ImplicitOps, icebp), N, N,
3890 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3891 G(ByteOp, group3), G(0, group3),
3893 D(ImplicitOps), D(ImplicitOps),
3894 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3895 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3898 static const struct opcode twobyte_table[256] = {
3900 G(0, group6), GD(0, &group7), N, N,
3901 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3902 II(ImplicitOps | Priv, em_clts, clts), N,
3903 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3904 N, D(ImplicitOps | ModRM), N, N,
3906 N, N, N, N, N, N, N, N,
3907 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3909 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3910 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3911 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3913 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3916 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3917 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3918 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
3921 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3922 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3923 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3924 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3925 I(ImplicitOps | EmulateOnUD, em_sysenter),
3926 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3928 N, N, N, N, N, N, N, N,
3930 X16(D(DstReg | SrcMem | ModRM)),
3932 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3937 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3942 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3946 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3948 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3949 II(ImplicitOps, em_cpuid, cpuid),
3950 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3951 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3952 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3954 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3955 DI(ImplicitOps, rsm),
3956 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3957 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3958 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3959 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3961 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3962 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3963 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3964 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3965 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3966 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3970 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3971 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3972 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3974 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3975 N, D(DstMem | SrcReg | ModRM | Mov),
3976 N, N, N, GD(0, &group9),
3978 X8(I(DstReg, em_bswap)),
3980 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3982 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
3983 N, N, N, N, N, N, N, N,
3985 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3988 static const struct gprefix three_byte_0f_38_f0 = {
3989 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3992 static const struct gprefix three_byte_0f_38_f1 = {
3993 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3997 * Insns below are selected by the prefix which indexed by the third opcode
4000 static const struct opcode opcode_map_0f_38[256] = {
4002 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4004 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4006 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
4007 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4026 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4030 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4036 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4037 unsigned size, bool sign_extension)
4039 int rc = X86EMUL_CONTINUE;
4043 op->addr.mem.ea = ctxt->_eip;
4044 /* NB. Immediates are sign-extended as necessary. */
4045 switch (op->bytes) {
4047 op->val = insn_fetch(s8, ctxt);
4050 op->val = insn_fetch(s16, ctxt);
4053 op->val = insn_fetch(s32, ctxt);
4056 op->val = insn_fetch(s64, ctxt);
4059 if (!sign_extension) {
4060 switch (op->bytes) {
4068 op->val &= 0xffffffff;
4076 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4079 int rc = X86EMUL_CONTINUE;
4083 decode_register_operand(ctxt, op);
4086 rc = decode_imm(ctxt, op, 1, false);
4089 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4093 if (ctxt->d & BitOp)
4094 fetch_bit_operand(ctxt);
4095 op->orig_val = op->val;
4098 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4102 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4103 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4104 fetch_register_operand(op);
4105 op->orig_val = op->val;
4109 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4110 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4111 fetch_register_operand(op);
4112 op->orig_val = op->val;
4115 if (ctxt->d & ByteOp) {
4120 op->bytes = ctxt->op_bytes;
4121 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4122 fetch_register_operand(op);
4123 op->orig_val = op->val;
4127 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4129 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4130 op->addr.mem.seg = VCPU_SREG_ES;
4137 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4138 fetch_register_operand(op);
4142 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4145 rc = decode_imm(ctxt, op, 1, true);
4152 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4155 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4158 ctxt->memop.bytes = 1;
4159 if (ctxt->memop.type == OP_REG) {
4160 ctxt->memop.addr.reg = decode_register(ctxt,
4161 ctxt->modrm_rm, true);
4162 fetch_register_operand(&ctxt->memop);
4166 ctxt->memop.bytes = 2;
4169 ctxt->memop.bytes = 4;
4172 rc = decode_imm(ctxt, op, 2, false);
4175 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4179 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4181 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4182 op->addr.mem.seg = ctxt->seg_override;
4188 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4190 register_address(ctxt,
4191 reg_read(ctxt, VCPU_REGS_RBX) +
4192 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4193 op->addr.mem.seg = ctxt->seg_override;
4198 op->addr.mem.ea = ctxt->_eip;
4199 op->bytes = ctxt->op_bytes + 2;
4200 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4203 ctxt->memop.bytes = ctxt->op_bytes + 2;
4206 op->val = VCPU_SREG_ES;
4209 op->val = VCPU_SREG_CS;
4212 op->val = VCPU_SREG_SS;
4215 op->val = VCPU_SREG_DS;
4218 op->val = VCPU_SREG_FS;
4221 op->val = VCPU_SREG_GS;
4224 /* Special instructions do their own operand decoding. */
4226 op->type = OP_NONE; /* Disable writeback. */
4234 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4236 int rc = X86EMUL_CONTINUE;
4237 int mode = ctxt->mode;
4238 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4239 bool op_prefix = false;
4240 bool has_seg_override = false;
4241 struct opcode opcode;
4243 ctxt->memop.type = OP_NONE;
4244 ctxt->memopp = NULL;
4245 ctxt->_eip = ctxt->eip;
4246 ctxt->fetch.ptr = ctxt->fetch.data;
4247 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4248 ctxt->opcode_len = 1;
4250 memcpy(ctxt->fetch.data, insn, insn_len);
4252 rc = __do_insn_fetch_bytes(ctxt, 1);
4253 if (rc != X86EMUL_CONTINUE)
4258 case X86EMUL_MODE_REAL:
4259 case X86EMUL_MODE_VM86:
4260 case X86EMUL_MODE_PROT16:
4261 def_op_bytes = def_ad_bytes = 2;
4263 case X86EMUL_MODE_PROT32:
4264 def_op_bytes = def_ad_bytes = 4;
4266 #ifdef CONFIG_X86_64
4267 case X86EMUL_MODE_PROT64:
4273 return EMULATION_FAILED;
4276 ctxt->op_bytes = def_op_bytes;
4277 ctxt->ad_bytes = def_ad_bytes;
4279 /* Legacy prefixes. */
4281 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4282 case 0x66: /* operand-size override */
4284 /* switch between 2/4 bytes */
4285 ctxt->op_bytes = def_op_bytes ^ 6;
4287 case 0x67: /* address-size override */
4288 if (mode == X86EMUL_MODE_PROT64)
4289 /* switch between 4/8 bytes */
4290 ctxt->ad_bytes = def_ad_bytes ^ 12;
4292 /* switch between 2/4 bytes */
4293 ctxt->ad_bytes = def_ad_bytes ^ 6;
4295 case 0x26: /* ES override */
4296 case 0x2e: /* CS override */
4297 case 0x36: /* SS override */
4298 case 0x3e: /* DS override */
4299 has_seg_override = true;
4300 ctxt->seg_override = (ctxt->b >> 3) & 3;
4302 case 0x64: /* FS override */
4303 case 0x65: /* GS override */
4304 has_seg_override = true;
4305 ctxt->seg_override = ctxt->b & 7;
4307 case 0x40 ... 0x4f: /* REX */
4308 if (mode != X86EMUL_MODE_PROT64)
4310 ctxt->rex_prefix = ctxt->b;
4312 case 0xf0: /* LOCK */
4313 ctxt->lock_prefix = 1;
4315 case 0xf2: /* REPNE/REPNZ */
4316 case 0xf3: /* REP/REPE/REPZ */
4317 ctxt->rep_prefix = ctxt->b;
4323 /* Any legacy prefix after a REX prefix nullifies its effect. */
4325 ctxt->rex_prefix = 0;
4331 if (ctxt->rex_prefix & 8)
4332 ctxt->op_bytes = 8; /* REX.W */
4334 /* Opcode byte(s). */
4335 opcode = opcode_table[ctxt->b];
4336 /* Two-byte opcode? */
4337 if (ctxt->b == 0x0f) {
4338 ctxt->opcode_len = 2;
4339 ctxt->b = insn_fetch(u8, ctxt);
4340 opcode = twobyte_table[ctxt->b];
4342 /* 0F_38 opcode map */
4343 if (ctxt->b == 0x38) {
4344 ctxt->opcode_len = 3;
4345 ctxt->b = insn_fetch(u8, ctxt);
4346 opcode = opcode_map_0f_38[ctxt->b];
4349 ctxt->d = opcode.flags;
4351 if (ctxt->d & ModRM)
4352 ctxt->modrm = insn_fetch(u8, ctxt);
4354 /* vex-prefix instructions are not implemented */
4355 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4356 (mode == X86EMUL_MODE_PROT64 ||
4357 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4361 while (ctxt->d & GroupMask) {
4362 switch (ctxt->d & GroupMask) {
4364 goffset = (ctxt->modrm >> 3) & 7;
4365 opcode = opcode.u.group[goffset];
4368 goffset = (ctxt->modrm >> 3) & 7;
4369 if ((ctxt->modrm >> 6) == 3)
4370 opcode = opcode.u.gdual->mod3[goffset];
4372 opcode = opcode.u.gdual->mod012[goffset];
4375 goffset = ctxt->modrm & 7;
4376 opcode = opcode.u.group[goffset];
4379 if (ctxt->rep_prefix && op_prefix)
4380 return EMULATION_FAILED;
4381 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4382 switch (simd_prefix) {
4383 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4384 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4385 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4386 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4390 if (ctxt->modrm > 0xbf)
4391 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4393 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4396 return EMULATION_FAILED;
4399 ctxt->d &= ~(u64)GroupMask;
4400 ctxt->d |= opcode.flags;
4405 return EMULATION_FAILED;
4407 ctxt->execute = opcode.u.execute;
4409 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4410 return EMULATION_FAILED;
4412 if (unlikely(ctxt->d &
4413 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4415 * These are copied unconditionally here, and checked unconditionally
4416 * in x86_emulate_insn.
4418 ctxt->check_perm = opcode.check_perm;
4419 ctxt->intercept = opcode.intercept;
4421 if (ctxt->d & NotImpl)
4422 return EMULATION_FAILED;
4424 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4427 if (ctxt->d & Op3264) {
4428 if (mode == X86EMUL_MODE_PROT64)
4435 ctxt->op_bytes = 16;
4436 else if (ctxt->d & Mmx)
4440 /* ModRM and SIB bytes. */
4441 if (ctxt->d & ModRM) {
4442 rc = decode_modrm(ctxt, &ctxt->memop);
4443 if (!has_seg_override) {
4444 has_seg_override = true;
4445 ctxt->seg_override = ctxt->modrm_seg;
4447 } else if (ctxt->d & MemAbs)
4448 rc = decode_abs(ctxt, &ctxt->memop);
4449 if (rc != X86EMUL_CONTINUE)
4452 if (!has_seg_override)
4453 ctxt->seg_override = VCPU_SREG_DS;
4455 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4458 * Decode and fetch the source operand: register, memory
4461 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4462 if (rc != X86EMUL_CONTINUE)
4466 * Decode and fetch the second source operand: register, memory
4469 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4470 if (rc != X86EMUL_CONTINUE)
4473 /* Decode and fetch the destination operand: register or memory. */
4474 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4477 if (ctxt->rip_relative)
4478 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4480 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4483 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4485 return ctxt->d & PageTable;
4488 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4490 /* The second termination condition only applies for REPE
4491 * and REPNE. Test if the repeat string operation prefix is
4492 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4493 * corresponding termination condition according to:
4494 * - if REPE/REPZ and ZF = 0 then done
4495 * - if REPNE/REPNZ and ZF = 1 then done
4497 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4498 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4499 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4500 ((ctxt->eflags & EFLG_ZF) == 0))
4501 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4502 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4508 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4512 ctxt->ops->get_fpu(ctxt);
4513 asm volatile("1: fwait \n\t"
4515 ".pushsection .fixup,\"ax\" \n\t"
4517 "movb $1, %[fault] \n\t"
4520 _ASM_EXTABLE(1b, 3b)
4521 : [fault]"+qm"(fault));
4522 ctxt->ops->put_fpu(ctxt);
4524 if (unlikely(fault))
4525 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4527 return X86EMUL_CONTINUE;
4530 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4533 if (op->type == OP_MM)
4534 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4537 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4539 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4540 if (!(ctxt->d & ByteOp))
4541 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4542 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4543 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4545 : "c"(ctxt->src2.val));
4546 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4547 if (!fop) /* exception is returned in fop variable */
4548 return emulate_de(ctxt);
4549 return X86EMUL_CONTINUE;
4552 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4554 memset(&ctxt->rip_relative, 0,
4555 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4557 ctxt->io_read.pos = 0;
4558 ctxt->io_read.end = 0;
4559 ctxt->mem_read.end = 0;
4562 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4564 const struct x86_emulate_ops *ops = ctxt->ops;
4565 int rc = X86EMUL_CONTINUE;
4566 int saved_dst_type = ctxt->dst.type;
4568 ctxt->mem_read.pos = 0;
4570 /* LOCK prefix is allowed only with some instructions */
4571 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4572 rc = emulate_ud(ctxt);
4576 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4577 rc = emulate_ud(ctxt);
4581 if (unlikely(ctxt->d &
4582 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4583 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4584 (ctxt->d & Undefined)) {
4585 rc = emulate_ud(ctxt);
4589 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4590 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4591 rc = emulate_ud(ctxt);
4595 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4596 rc = emulate_nm(ctxt);
4600 if (ctxt->d & Mmx) {
4601 rc = flush_pending_x87_faults(ctxt);
4602 if (rc != X86EMUL_CONTINUE)
4605 * Now that we know the fpu is exception safe, we can fetch
4608 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4609 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4610 if (!(ctxt->d & Mov))
4611 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4614 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4615 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4616 X86_ICPT_PRE_EXCEPT);
4617 if (rc != X86EMUL_CONTINUE)
4621 /* Privileged instruction can be executed only in CPL=0 */
4622 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4623 if (ctxt->d & PrivUD)
4624 rc = emulate_ud(ctxt);
4626 rc = emulate_gp(ctxt, 0);
4630 /* Instruction can only be executed in protected mode */
4631 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4632 rc = emulate_ud(ctxt);
4636 /* Do instruction specific permission checks */
4637 if (ctxt->d & CheckPerm) {
4638 rc = ctxt->check_perm(ctxt);
4639 if (rc != X86EMUL_CONTINUE)
4643 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4644 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4645 X86_ICPT_POST_EXCEPT);
4646 if (rc != X86EMUL_CONTINUE)
4650 if (ctxt->rep_prefix && (ctxt->d & String)) {
4651 /* All REP prefixes have the same first termination condition */
4652 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4653 ctxt->eip = ctxt->_eip;
4654 ctxt->eflags &= ~EFLG_RF;
4660 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4661 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4662 ctxt->src.valptr, ctxt->src.bytes);
4663 if (rc != X86EMUL_CONTINUE)
4665 ctxt->src.orig_val64 = ctxt->src.val64;
4668 if (ctxt->src2.type == OP_MEM) {
4669 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4670 &ctxt->src2.val, ctxt->src2.bytes);
4671 if (rc != X86EMUL_CONTINUE)
4675 if ((ctxt->d & DstMask) == ImplicitOps)
4679 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4680 /* optimisation - avoid slow emulated read if Mov */
4681 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4682 &ctxt->dst.val, ctxt->dst.bytes);
4683 if (rc != X86EMUL_CONTINUE)
4686 ctxt->dst.orig_val = ctxt->dst.val;
4690 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4691 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4692 X86_ICPT_POST_MEMACCESS);
4693 if (rc != X86EMUL_CONTINUE)
4697 if (ctxt->rep_prefix && (ctxt->d & String))
4698 ctxt->eflags |= EFLG_RF;
4700 ctxt->eflags &= ~EFLG_RF;
4702 if (ctxt->execute) {
4703 if (ctxt->d & Fastop) {
4704 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4705 rc = fastop(ctxt, fop);
4706 if (rc != X86EMUL_CONTINUE)
4710 rc = ctxt->execute(ctxt);
4711 if (rc != X86EMUL_CONTINUE)
4716 if (ctxt->opcode_len == 2)
4718 else if (ctxt->opcode_len == 3)
4719 goto threebyte_insn;
4722 case 0x63: /* movsxd */
4723 if (ctxt->mode != X86EMUL_MODE_PROT64)
4724 goto cannot_emulate;
4725 ctxt->dst.val = (s32) ctxt->src.val;
4727 case 0x70 ... 0x7f: /* jcc (short) */
4728 if (test_cc(ctxt->b, ctxt->eflags))
4729 jmp_rel(ctxt, ctxt->src.val);
4731 case 0x8d: /* lea r16/r32, m */
4732 ctxt->dst.val = ctxt->src.addr.mem.ea;
4734 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4735 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4736 ctxt->dst.type = OP_NONE;
4740 case 0x98: /* cbw/cwde/cdqe */
4741 switch (ctxt->op_bytes) {
4742 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4743 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4744 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4747 case 0xcc: /* int3 */
4748 rc = emulate_int(ctxt, 3);
4750 case 0xcd: /* int n */
4751 rc = emulate_int(ctxt, ctxt->src.val);
4753 case 0xce: /* into */
4754 if (ctxt->eflags & EFLG_OF)
4755 rc = emulate_int(ctxt, 4);
4757 case 0xe9: /* jmp rel */
4758 case 0xeb: /* jmp rel short */
4759 jmp_rel(ctxt, ctxt->src.val);
4760 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4762 case 0xf4: /* hlt */
4763 ctxt->ops->halt(ctxt);
4765 case 0xf5: /* cmc */
4766 /* complement carry flag from eflags reg */
4767 ctxt->eflags ^= EFLG_CF;
4769 case 0xf8: /* clc */
4770 ctxt->eflags &= ~EFLG_CF;
4772 case 0xf9: /* stc */
4773 ctxt->eflags |= EFLG_CF;
4775 case 0xfc: /* cld */
4776 ctxt->eflags &= ~EFLG_DF;
4778 case 0xfd: /* std */
4779 ctxt->eflags |= EFLG_DF;
4782 goto cannot_emulate;
4785 if (rc != X86EMUL_CONTINUE)
4789 if (ctxt->d & SrcWrite) {
4790 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4791 rc = writeback(ctxt, &ctxt->src);
4792 if (rc != X86EMUL_CONTINUE)
4795 if (!(ctxt->d & NoWrite)) {
4796 rc = writeback(ctxt, &ctxt->dst);
4797 if (rc != X86EMUL_CONTINUE)
4802 * restore dst type in case the decoding will be reused
4803 * (happens for string instruction )
4805 ctxt->dst.type = saved_dst_type;
4807 if ((ctxt->d & SrcMask) == SrcSI)
4808 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4810 if ((ctxt->d & DstMask) == DstDI)
4811 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4813 if (ctxt->rep_prefix && (ctxt->d & String)) {
4815 struct read_cache *r = &ctxt->io_read;
4816 if ((ctxt->d & SrcMask) == SrcSI)
4817 count = ctxt->src.count;
4819 count = ctxt->dst.count;
4820 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4823 if (!string_insn_completed(ctxt)) {
4825 * Re-enter guest when pio read ahead buffer is empty
4826 * or, if it is not used, after each 1024 iteration.
4828 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4829 (r->end == 0 || r->end != r->pos)) {
4831 * Reset read cache. Usually happens before
4832 * decode, but since instruction is restarted
4833 * we have to do it here.
4835 ctxt->mem_read.end = 0;
4836 writeback_registers(ctxt);
4837 return EMULATION_RESTART;
4839 goto done; /* skip rip writeback */
4841 ctxt->eflags &= ~EFLG_RF;
4844 ctxt->eip = ctxt->_eip;
4847 if (rc == X86EMUL_PROPAGATE_FAULT) {
4848 WARN_ON(ctxt->exception.vector > 0x1f);
4849 ctxt->have_exception = true;
4851 if (rc == X86EMUL_INTERCEPTED)
4852 return EMULATION_INTERCEPTED;
4854 if (rc == X86EMUL_CONTINUE)
4855 writeback_registers(ctxt);
4857 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4861 case 0x09: /* wbinvd */
4862 (ctxt->ops->wbinvd)(ctxt);
4864 case 0x08: /* invd */
4865 case 0x0d: /* GrpP (prefetch) */
4866 case 0x18: /* Grp16 (prefetch/nop) */
4867 case 0x1f: /* nop */
4869 case 0x20: /* mov cr, reg */
4870 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4872 case 0x21: /* mov from dr to reg */
4873 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4875 case 0x40 ... 0x4f: /* cmov */
4876 if (test_cc(ctxt->b, ctxt->eflags))
4877 ctxt->dst.val = ctxt->src.val;
4878 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4879 ctxt->op_bytes != 4)
4880 ctxt->dst.type = OP_NONE; /* no writeback */
4882 case 0x80 ... 0x8f: /* jnz rel, etc*/
4883 if (test_cc(ctxt->b, ctxt->eflags))
4884 jmp_rel(ctxt, ctxt->src.val);
4886 case 0x90 ... 0x9f: /* setcc r/m8 */
4887 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4889 case 0xae: /* clflush */
4891 case 0xb6 ... 0xb7: /* movzx */
4892 ctxt->dst.bytes = ctxt->op_bytes;
4893 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4894 : (u16) ctxt->src.val;
4896 case 0xbe ... 0xbf: /* movsx */
4897 ctxt->dst.bytes = ctxt->op_bytes;
4898 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4899 (s16) ctxt->src.val;
4901 case 0xc3: /* movnti */
4902 ctxt->dst.bytes = ctxt->op_bytes;
4903 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4904 (u32) ctxt->src.val;
4907 goto cannot_emulate;
4912 if (rc != X86EMUL_CONTINUE)
4918 return EMULATION_FAILED;
4921 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4923 invalidate_registers(ctxt);
4926 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4928 writeback_registers(ctxt);