1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
165 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
167 #define X2(x...) x, x
168 #define X3(x...) X2(x), x
169 #define X4(x...) X2(x), X2(x)
170 #define X5(x...) X4(x), x
171 #define X6(x...) X4(x), X2(x)
172 #define X7(x...) X4(x), X3(x)
173 #define X8(x...) X4(x), X4(x)
174 #define X16(x...) X8(x), X8(x)
176 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177 #define FASTOP_SIZE 8
180 * fastop functions have a special calling convention:
185 * flags: rflags (in/out)
186 * ex: rsi (in:fastop pointer, out:zero if exception)
188 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
189 * different operand sizes can be reached by calculation, rather than a jump
190 * table (which would be bigger than the code).
192 * fastop functions are declared as taking a never-defined fastop parameter,
193 * so they can't be called from C directly.
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
229 /* EFLAGS bit definitions. */
230 #define EFLG_ID (1<<21)
231 #define EFLG_VIP (1<<20)
232 #define EFLG_VIF (1<<19)
233 #define EFLG_AC (1<<18)
234 #define EFLG_VM (1<<17)
235 #define EFLG_RF (1<<16)
236 #define EFLG_IOPL (3<<12)
237 #define EFLG_NT (1<<14)
238 #define EFLG_OF (1<<11)
239 #define EFLG_DF (1<<10)
240 #define EFLG_IF (1<<9)
241 #define EFLG_TF (1<<8)
242 #define EFLG_SF (1<<7)
243 #define EFLG_ZF (1<<6)
244 #define EFLG_AF (1<<4)
245 #define EFLG_PF (1<<2)
246 #define EFLG_CF (1<<0)
248 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249 #define EFLG_RESERVED_ONE_MASK 2
251 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
257 return ctxt->_regs[nr];
260 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
267 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
270 return reg_write(ctxt, nr);
273 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
281 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
288 * These EFLAGS bits are restored from saved value during emulation, and
289 * any changes are written back to the saved value after emulation.
291 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
301 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302 #define FOP_RET "ret \n\t"
304 #define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
314 #define FOPNOP() FOP_ALIGN FOP_RET
316 #define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
319 #define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
322 #define FASTOP1(op) \
327 ON64(FOP1E(op##q, rax)) \
330 /* 1-operand, using src2 (for MUL/DIV r/m) */
331 #define FASTOP1SRC2(op, name) \
336 ON64(FOP1E(op, rcx)) \
339 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
340 #define FASTOP1SRC2EX(op, name) \
345 ON64(FOP1EEX(op, rcx)) \
348 #define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
351 #define FASTOP2(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
359 /* 2 operand, word only */
360 #define FASTOP2W(op) \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
368 /* 2 operand, src is CL */
369 #define FASTOP2CL(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
377 #define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
380 /* 3-operand, word-only, src2=cl */
381 #define FASTOP3WCL(op) \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
389 /* Special case for SETcc - 1 instruction per cc */
390 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
392 asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
414 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
417 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
434 return ctxt->ops->intercept(ctxt, &info, stage);
437 static void assign_masked(ulong *dest, ulong src, ulong mask)
439 *dest = (*dest & ~mask) | (src & mask);
442 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
447 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
450 struct desc_struct ss;
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
458 static int stack_size(struct x86_emulate_ctxt *ctxt)
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
463 /* Access/update address held in a register, based on addressing mode. */
464 static inline unsigned long
465 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
467 if (ctxt->ad_bytes == sizeof(unsigned long))
470 return reg & ad_mask(ctxt);
473 static inline unsigned long
474 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
476 return address_mask(ctxt, reg);
479 static void masked_increment(ulong *reg, ulong mask, int inc)
481 assign_masked(reg, *reg + inc, mask);
485 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
489 if (ctxt->ad_bytes == sizeof(unsigned long))
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
496 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
501 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
503 register_address_increment(ctxt, &ctxt->_eip, rel);
506 static u32 desc_limit_scaled(struct desc_struct *desc)
508 u32 limit = get_desc_limit(desc);
510 return desc->g ? (limit << 12) | 0xfff : limit;
513 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
515 ctxt->has_seg_override = true;
516 ctxt->seg_override = seg;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
529 if (!ctxt->has_seg_override)
532 return ctxt->seg_override;
535 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
536 u32 error, bool valid)
538 ctxt->exception.vector = vec;
539 ctxt->exception.error_code = error;
540 ctxt->exception.error_code_valid = valid;
541 return X86EMUL_PROPAGATE_FAULT;
544 static int emulate_db(struct x86_emulate_ctxt *ctxt)
546 return emulate_exception(ctxt, DB_VECTOR, 0, false);
549 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
551 return emulate_exception(ctxt, GP_VECTOR, err, true);
554 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
556 return emulate_exception(ctxt, SS_VECTOR, err, true);
559 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, UD_VECTOR, 0, false);
564 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
566 return emulate_exception(ctxt, TS_VECTOR, err, true);
569 static int emulate_de(struct x86_emulate_ctxt *ctxt)
571 return emulate_exception(ctxt, DE_VECTOR, 0, false);
574 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
576 return emulate_exception(ctxt, NM_VECTOR, 0, false);
579 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
582 struct desc_struct desc;
584 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
588 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
593 struct desc_struct desc;
595 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
596 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
600 * x86 defines three classes of vector instructions: explicitly
601 * aligned, explicitly unaligned, and the rest, which change behaviour
602 * depending on whether they're AVX encoded or not.
604 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
605 * subject to the same check.
607 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
609 if (likely(size < 16))
612 if (ctxt->d & Aligned)
614 else if (ctxt->d & Unaligned)
616 else if (ctxt->d & Avx)
622 static int __linearize(struct x86_emulate_ctxt *ctxt,
623 struct segmented_address addr,
624 unsigned size, bool write, bool fetch,
627 struct desc_struct desc;
634 la = seg_base(ctxt, addr.seg) + addr.ea;
635 switch (ctxt->mode) {
636 case X86EMUL_MODE_PROT64:
637 if (((signed long)la << 16) >> 16 != la)
638 return emulate_gp(ctxt, 0);
641 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
645 /* code segment in protected mode or read-only data segment */
646 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
647 || !(desc.type & 2)) && write)
649 /* unreadable code segment */
650 if (!fetch && (desc.type & 8) && !(desc.type & 2))
652 lim = desc_limit_scaled(&desc);
653 if ((desc.type & 8) || !(desc.type & 4)) {
654 /* expand-up segment */
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
658 /* expand-down segment */
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
661 lim = desc.d ? 0xffffffff : 0xffff;
662 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
665 cpl = ctxt->ops->cpl(ctxt);
666 if (!(desc.type & 8)) {
670 } else if ((desc.type & 8) && !(desc.type & 4)) {
671 /* nonconforming code segment */
674 } else if ((desc.type & 8) && (desc.type & 4)) {
675 /* conforming code segment */
681 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
683 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
684 return emulate_gp(ctxt, 0);
686 return X86EMUL_CONTINUE;
688 if (addr.seg == VCPU_SREG_SS)
689 return emulate_ss(ctxt, sel);
691 return emulate_gp(ctxt, sel);
694 static int linearize(struct x86_emulate_ctxt *ctxt,
695 struct segmented_address addr,
696 unsigned size, bool write,
699 return __linearize(ctxt, addr, size, write, false, linear);
703 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
704 struct segmented_address addr,
711 rc = linearize(ctxt, addr, size, false, &linear);
712 if (rc != X86EMUL_CONTINUE)
714 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
718 * Fetch the next byte of the instruction being emulated which is pointed to
719 * by ctxt->_eip, then increment ctxt->_eip.
721 * Also prefetch the remaining bytes of the instruction without crossing page
722 * boundary if they are not in fetch_cache yet.
724 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
726 struct fetch_cache *fc = &ctxt->fetch;
730 if (ctxt->_eip == fc->end) {
731 unsigned long linear;
732 struct segmented_address addr = { .seg = VCPU_SREG_CS,
734 cur_size = fc->end - fc->start;
735 size = min(15UL - cur_size,
736 PAGE_SIZE - offset_in_page(ctxt->_eip));
737 rc = __linearize(ctxt, addr, size, false, true, &linear);
738 if (unlikely(rc != X86EMUL_CONTINUE))
740 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
741 size, &ctxt->exception);
742 if (unlikely(rc != X86EMUL_CONTINUE))
746 *dest = fc->data[ctxt->_eip - fc->start];
748 return X86EMUL_CONTINUE;
751 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752 void *dest, unsigned size)
756 /* x86 instructions are limited to 15 bytes. */
757 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758 return X86EMUL_UNHANDLEABLE;
760 rc = do_insn_fetch_byte(ctxt, dest++);
761 if (rc != X86EMUL_CONTINUE)
764 return X86EMUL_CONTINUE;
767 /* Fetch next part of the instruction being emulated. */
768 #define insn_fetch(_type, _ctxt) \
769 ({ unsigned long _x; \
770 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
771 if (rc != X86EMUL_CONTINUE) \
776 #define insn_fetch_arr(_arr, _size, _ctxt) \
777 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
778 if (rc != X86EMUL_CONTINUE) \
783 * Given the 'reg' portion of a ModRM byte, and a register block, return a
784 * pointer into the block that addresses the relevant register.
785 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
787 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
791 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
793 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
794 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
796 p = reg_rmw(ctxt, modrm_reg);
800 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
801 struct segmented_address addr,
802 u16 *size, unsigned long *address, int op_bytes)
809 rc = segmented_read_std(ctxt, addr, size, 2);
810 if (rc != X86EMUL_CONTINUE)
813 rc = segmented_read_std(ctxt, addr, address, op_bytes);
827 FASTOP1SRC2(mul, mul_ex);
828 FASTOP1SRC2(imul, imul_ex);
829 FASTOP1SRC2EX(div, div_ex);
830 FASTOP1SRC2EX(idiv, idiv_ex);
859 static u8 test_cc(unsigned int condition, unsigned long flags)
862 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
864 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
865 asm("push %[flags]; popf; call *%[fastop]"
866 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
870 static void fetch_register_operand(struct operand *op)
874 op->val = *(u8 *)op->addr.reg;
877 op->val = *(u16 *)op->addr.reg;
880 op->val = *(u32 *)op->addr.reg;
883 op->val = *(u64 *)op->addr.reg;
888 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
890 ctxt->ops->get_fpu(ctxt);
892 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
893 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
894 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
895 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
896 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
897 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
898 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
899 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
901 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
902 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
903 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
904 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
905 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
906 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
907 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
908 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
912 ctxt->ops->put_fpu(ctxt);
915 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
918 ctxt->ops->get_fpu(ctxt);
920 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
921 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
922 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
923 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
924 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
925 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
926 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
927 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
929 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
930 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
931 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
932 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
933 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
934 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
935 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
936 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
940 ctxt->ops->put_fpu(ctxt);
943 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
945 ctxt->ops->get_fpu(ctxt);
947 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
948 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
949 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
950 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
951 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
952 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
953 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
954 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
957 ctxt->ops->put_fpu(ctxt);
960 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
962 ctxt->ops->get_fpu(ctxt);
964 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
965 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
966 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
967 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
968 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
969 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
970 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
971 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
974 ctxt->ops->put_fpu(ctxt);
977 static int em_fninit(struct x86_emulate_ctxt *ctxt)
979 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
980 return emulate_nm(ctxt);
982 ctxt->ops->get_fpu(ctxt);
983 asm volatile("fninit");
984 ctxt->ops->put_fpu(ctxt);
985 return X86EMUL_CONTINUE;
988 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
992 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
993 return emulate_nm(ctxt);
995 ctxt->ops->get_fpu(ctxt);
996 asm volatile("fnstcw %0": "+m"(fcw));
997 ctxt->ops->put_fpu(ctxt);
999 /* force 2 byte destination */
1000 ctxt->dst.bytes = 2;
1001 ctxt->dst.val = fcw;
1003 return X86EMUL_CONTINUE;
1006 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1010 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1011 return emulate_nm(ctxt);
1013 ctxt->ops->get_fpu(ctxt);
1014 asm volatile("fnstsw %0": "+m"(fsw));
1015 ctxt->ops->put_fpu(ctxt);
1017 /* force 2 byte destination */
1018 ctxt->dst.bytes = 2;
1019 ctxt->dst.val = fsw;
1021 return X86EMUL_CONTINUE;
1024 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1027 unsigned reg = ctxt->modrm_reg;
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1032 if (ctxt->d & Sse) {
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1039 if (ctxt->d & Mmx) {
1048 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1049 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1051 fetch_register_operand(op);
1052 op->orig_val = op->val;
1055 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1057 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1058 ctxt->modrm_seg = VCPU_SREG_SS;
1061 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1065 int index_reg = 0, base_reg = 0, scale;
1066 int rc = X86EMUL_CONTINUE;
1069 if (ctxt->rex_prefix) {
1070 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1071 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1072 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1075 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1076 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1077 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1078 ctxt->modrm_seg = VCPU_SREG_DS;
1080 if (ctxt->modrm_mod == 3) {
1082 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1083 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1085 if (ctxt->d & Sse) {
1088 op->addr.xmm = ctxt->modrm_rm;
1089 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1092 if (ctxt->d & Mmx) {
1095 op->addr.xmm = ctxt->modrm_rm & 7;
1098 fetch_register_operand(op);
1104 if (ctxt->ad_bytes == 2) {
1105 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1106 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1107 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1108 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1110 /* 16-bit ModR/M decode. */
1111 switch (ctxt->modrm_mod) {
1113 if (ctxt->modrm_rm == 6)
1114 modrm_ea += insn_fetch(u16, ctxt);
1117 modrm_ea += insn_fetch(s8, ctxt);
1120 modrm_ea += insn_fetch(u16, ctxt);
1123 switch (ctxt->modrm_rm) {
1125 modrm_ea += bx + si;
1128 modrm_ea += bx + di;
1131 modrm_ea += bp + si;
1134 modrm_ea += bp + di;
1143 if (ctxt->modrm_mod != 0)
1150 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1151 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1152 ctxt->modrm_seg = VCPU_SREG_SS;
1153 modrm_ea = (u16)modrm_ea;
1155 /* 32/64-bit ModR/M decode. */
1156 if ((ctxt->modrm_rm & 7) == 4) {
1157 sib = insn_fetch(u8, ctxt);
1158 index_reg |= (sib >> 3) & 7;
1159 base_reg |= sib & 7;
1162 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1163 modrm_ea += insn_fetch(s32, ctxt);
1165 modrm_ea += reg_read(ctxt, base_reg);
1166 adjust_modrm_seg(ctxt, base_reg);
1169 modrm_ea += reg_read(ctxt, index_reg) << scale;
1170 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1171 if (ctxt->mode == X86EMUL_MODE_PROT64)
1172 ctxt->rip_relative = 1;
1174 base_reg = ctxt->modrm_rm;
1175 modrm_ea += reg_read(ctxt, base_reg);
1176 adjust_modrm_seg(ctxt, base_reg);
1178 switch (ctxt->modrm_mod) {
1180 if (ctxt->modrm_rm == 5)
1181 modrm_ea += insn_fetch(s32, ctxt);
1184 modrm_ea += insn_fetch(s8, ctxt);
1187 modrm_ea += insn_fetch(s32, ctxt);
1191 op->addr.mem.ea = modrm_ea;
1196 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1199 int rc = X86EMUL_CONTINUE;
1202 switch (ctxt->ad_bytes) {
1204 op->addr.mem.ea = insn_fetch(u16, ctxt);
1207 op->addr.mem.ea = insn_fetch(u32, ctxt);
1210 op->addr.mem.ea = insn_fetch(u64, ctxt);
1217 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1221 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1222 mask = ~(ctxt->dst.bytes * 8 - 1);
1224 if (ctxt->src.bytes == 2)
1225 sv = (s16)ctxt->src.val & (s16)mask;
1226 else if (ctxt->src.bytes == 4)
1227 sv = (s32)ctxt->src.val & (s32)mask;
1229 ctxt->dst.addr.mem.ea += (sv >> 3);
1232 /* only subword offset */
1233 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1236 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1237 unsigned long addr, void *dest, unsigned size)
1240 struct read_cache *mc = &ctxt->mem_read;
1242 if (mc->pos < mc->end)
1245 WARN_ON((mc->end + size) >= sizeof(mc->data));
1247 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1249 if (rc != X86EMUL_CONTINUE)
1255 memcpy(dest, mc->data + mc->pos, size);
1257 return X86EMUL_CONTINUE;
1260 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1261 struct segmented_address addr,
1268 rc = linearize(ctxt, addr, size, false, &linear);
1269 if (rc != X86EMUL_CONTINUE)
1271 return read_emulated(ctxt, linear, data, size);
1274 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1275 struct segmented_address addr,
1282 rc = linearize(ctxt, addr, size, true, &linear);
1283 if (rc != X86EMUL_CONTINUE)
1285 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1289 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1290 struct segmented_address addr,
1291 const void *orig_data, const void *data,
1297 rc = linearize(ctxt, addr, size, true, &linear);
1298 if (rc != X86EMUL_CONTINUE)
1300 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1301 size, &ctxt->exception);
1304 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1305 unsigned int size, unsigned short port,
1308 struct read_cache *rc = &ctxt->io_read;
1310 if (rc->pos == rc->end) { /* refill pio read ahead */
1311 unsigned int in_page, n;
1312 unsigned int count = ctxt->rep_prefix ?
1313 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1314 in_page = (ctxt->eflags & EFLG_DF) ?
1315 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1316 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1317 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1321 rc->pos = rc->end = 0;
1322 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1327 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1328 ctxt->dst.data = rc->data + rc->pos;
1329 ctxt->dst.type = OP_MEM_STR;
1330 ctxt->dst.count = (rc->end - rc->pos) / size;
1333 memcpy(dest, rc->data + rc->pos, size);
1339 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1340 u16 index, struct desc_struct *desc)
1345 ctxt->ops->get_idt(ctxt, &dt);
1347 if (dt.size < index * 8 + 7)
1348 return emulate_gp(ctxt, index << 3 | 0x2);
1350 addr = dt.address + index * 8;
1351 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1355 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1356 u16 selector, struct desc_ptr *dt)
1358 const struct x86_emulate_ops *ops = ctxt->ops;
1360 if (selector & 1 << 2) {
1361 struct desc_struct desc;
1364 memset (dt, 0, sizeof *dt);
1365 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1368 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1369 dt->address = get_desc_base(&desc);
1371 ops->get_gdt(ctxt, dt);
1374 /* allowed just for 8 bytes segments */
1375 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1376 u16 selector, struct desc_struct *desc,
1380 u16 index = selector >> 3;
1383 get_descriptor_table_ptr(ctxt, selector, &dt);
1385 if (dt.size < index * 8 + 7)
1386 return emulate_gp(ctxt, selector & 0xfffc);
1388 *desc_addr_p = addr = dt.address + index * 8;
1389 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1393 /* allowed just for 8 bytes segments */
1394 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1395 u16 selector, struct desc_struct *desc)
1398 u16 index = selector >> 3;
1401 get_descriptor_table_ptr(ctxt, selector, &dt);
1403 if (dt.size < index * 8 + 7)
1404 return emulate_gp(ctxt, selector & 0xfffc);
1406 addr = dt.address + index * 8;
1407 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1411 /* Does not support long mode */
1412 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1413 u16 selector, int seg)
1415 struct desc_struct seg_desc, old_desc;
1417 unsigned err_vec = GP_VECTOR;
1419 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1424 memset(&seg_desc, 0, sizeof seg_desc);
1426 if (ctxt->mode == X86EMUL_MODE_REAL) {
1427 /* set real mode segment descriptor (keep limit etc. for
1429 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1430 set_desc_base(&seg_desc, selector << 4);
1432 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1433 /* VM86 needs a clean new segment descriptor */
1434 set_desc_base(&seg_desc, selector << 4);
1435 set_desc_limit(&seg_desc, 0xffff);
1444 cpl = ctxt->ops->cpl(ctxt);
1446 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1447 if ((seg == VCPU_SREG_CS
1448 || (seg == VCPU_SREG_SS
1449 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1450 || seg == VCPU_SREG_TR)
1454 /* TR should be in GDT only */
1455 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1458 if (null_selector) /* for NULL selector skip all following checks */
1461 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1462 if (ret != X86EMUL_CONTINUE)
1465 err_code = selector & 0xfffc;
1466 err_vec = GP_VECTOR;
1468 /* can't load system descriptor into segment selector */
1469 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1473 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1482 * segment is not a writable data segment or segment
1483 * selector's RPL != CPL or segment selector's RPL != CPL
1485 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1489 if (!(seg_desc.type & 8))
1492 if (seg_desc.type & 4) {
1498 if (rpl > cpl || dpl != cpl)
1501 /* CS(RPL) <- CPL */
1502 selector = (selector & 0xfffc) | cpl;
1505 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1507 old_desc = seg_desc;
1508 seg_desc.type |= 2; /* busy */
1509 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1510 sizeof(seg_desc), &ctxt->exception);
1511 if (ret != X86EMUL_CONTINUE)
1514 case VCPU_SREG_LDTR:
1515 if (seg_desc.s || seg_desc.type != 2)
1518 default: /* DS, ES, FS, or GS */
1520 * segment is not a data or readable code segment or
1521 * ((segment is a data or nonconforming code segment)
1522 * and (both RPL and CPL > DPL))
1524 if ((seg_desc.type & 0xa) == 0x8 ||
1525 (((seg_desc.type & 0xc) != 0xc) &&
1526 (rpl > dpl && cpl > dpl)))
1532 /* mark segment as accessed */
1534 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1535 if (ret != X86EMUL_CONTINUE)
1539 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1540 return X86EMUL_CONTINUE;
1542 emulate_exception(ctxt, err_vec, err_code, true);
1543 return X86EMUL_PROPAGATE_FAULT;
1546 static void write_register_operand(struct operand *op)
1548 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1549 switch (op->bytes) {
1551 *(u8 *)op->addr.reg = (u8)op->val;
1554 *(u16 *)op->addr.reg = (u16)op->val;
1557 *op->addr.reg = (u32)op->val;
1558 break; /* 64b: zero-extend */
1560 *op->addr.reg = op->val;
1565 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1571 write_register_operand(op);
1574 if (ctxt->lock_prefix)
1575 rc = segmented_cmpxchg(ctxt,
1581 rc = segmented_write(ctxt,
1585 if (rc != X86EMUL_CONTINUE)
1589 rc = segmented_write(ctxt,
1592 op->bytes * op->count);
1593 if (rc != X86EMUL_CONTINUE)
1597 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1600 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1608 return X86EMUL_CONTINUE;
1611 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1613 struct segmented_address addr;
1615 rsp_increment(ctxt, -bytes);
1616 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1617 addr.seg = VCPU_SREG_SS;
1619 return segmented_write(ctxt, addr, data, bytes);
1622 static int em_push(struct x86_emulate_ctxt *ctxt)
1624 /* Disable writeback. */
1625 ctxt->dst.type = OP_NONE;
1626 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1629 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1630 void *dest, int len)
1633 struct segmented_address addr;
1635 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1636 addr.seg = VCPU_SREG_SS;
1637 rc = segmented_read(ctxt, addr, dest, len);
1638 if (rc != X86EMUL_CONTINUE)
1641 rsp_increment(ctxt, len);
1645 static int em_pop(struct x86_emulate_ctxt *ctxt)
1647 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1650 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1651 void *dest, int len)
1654 unsigned long val, change_mask;
1655 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1656 int cpl = ctxt->ops->cpl(ctxt);
1658 rc = emulate_pop(ctxt, &val, len);
1659 if (rc != X86EMUL_CONTINUE)
1662 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1663 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1665 switch(ctxt->mode) {
1666 case X86EMUL_MODE_PROT64:
1667 case X86EMUL_MODE_PROT32:
1668 case X86EMUL_MODE_PROT16:
1670 change_mask |= EFLG_IOPL;
1672 change_mask |= EFLG_IF;
1674 case X86EMUL_MODE_VM86:
1676 return emulate_gp(ctxt, 0);
1677 change_mask |= EFLG_IF;
1679 default: /* real mode */
1680 change_mask |= (EFLG_IOPL | EFLG_IF);
1684 *(unsigned long *)dest =
1685 (ctxt->eflags & ~change_mask) | (val & change_mask);
1690 static int em_popf(struct x86_emulate_ctxt *ctxt)
1692 ctxt->dst.type = OP_REG;
1693 ctxt->dst.addr.reg = &ctxt->eflags;
1694 ctxt->dst.bytes = ctxt->op_bytes;
1695 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1698 static int em_enter(struct x86_emulate_ctxt *ctxt)
1701 unsigned frame_size = ctxt->src.val;
1702 unsigned nesting_level = ctxt->src2.val & 31;
1706 return X86EMUL_UNHANDLEABLE;
1708 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1709 rc = push(ctxt, &rbp, stack_size(ctxt));
1710 if (rc != X86EMUL_CONTINUE)
1712 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1714 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1715 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1717 return X86EMUL_CONTINUE;
1720 static int em_leave(struct x86_emulate_ctxt *ctxt)
1722 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1724 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1727 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1729 int seg = ctxt->src2.val;
1731 ctxt->src.val = get_segment_selector(ctxt, seg);
1733 return em_push(ctxt);
1736 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1738 int seg = ctxt->src2.val;
1739 unsigned long selector;
1742 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1743 if (rc != X86EMUL_CONTINUE)
1746 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1750 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1752 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1753 int rc = X86EMUL_CONTINUE;
1754 int reg = VCPU_REGS_RAX;
1756 while (reg <= VCPU_REGS_RDI) {
1757 (reg == VCPU_REGS_RSP) ?
1758 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1761 if (rc != X86EMUL_CONTINUE)
1770 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1772 ctxt->src.val = (unsigned long)ctxt->eflags;
1773 return em_push(ctxt);
1776 static int em_popa(struct x86_emulate_ctxt *ctxt)
1778 int rc = X86EMUL_CONTINUE;
1779 int reg = VCPU_REGS_RDI;
1781 while (reg >= VCPU_REGS_RAX) {
1782 if (reg == VCPU_REGS_RSP) {
1783 rsp_increment(ctxt, ctxt->op_bytes);
1787 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1788 if (rc != X86EMUL_CONTINUE)
1795 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1797 const struct x86_emulate_ops *ops = ctxt->ops;
1804 /* TODO: Add limit checks */
1805 ctxt->src.val = ctxt->eflags;
1807 if (rc != X86EMUL_CONTINUE)
1810 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1812 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1814 if (rc != X86EMUL_CONTINUE)
1817 ctxt->src.val = ctxt->_eip;
1819 if (rc != X86EMUL_CONTINUE)
1822 ops->get_idt(ctxt, &dt);
1824 eip_addr = dt.address + (irq << 2);
1825 cs_addr = dt.address + (irq << 2) + 2;
1827 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1828 if (rc != X86EMUL_CONTINUE)
1831 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1832 if (rc != X86EMUL_CONTINUE)
1835 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1836 if (rc != X86EMUL_CONTINUE)
1844 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1848 invalidate_registers(ctxt);
1849 rc = __emulate_int_real(ctxt, irq);
1850 if (rc == X86EMUL_CONTINUE)
1851 writeback_registers(ctxt);
1855 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1857 switch(ctxt->mode) {
1858 case X86EMUL_MODE_REAL:
1859 return __emulate_int_real(ctxt, irq);
1860 case X86EMUL_MODE_VM86:
1861 case X86EMUL_MODE_PROT16:
1862 case X86EMUL_MODE_PROT32:
1863 case X86EMUL_MODE_PROT64:
1865 /* Protected mode interrupts unimplemented yet */
1866 return X86EMUL_UNHANDLEABLE;
1870 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1872 int rc = X86EMUL_CONTINUE;
1873 unsigned long temp_eip = 0;
1874 unsigned long temp_eflags = 0;
1875 unsigned long cs = 0;
1876 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1877 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1878 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1879 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1881 /* TODO: Add stack limit check */
1883 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1885 if (rc != X86EMUL_CONTINUE)
1888 if (temp_eip & ~0xffff)
1889 return emulate_gp(ctxt, 0);
1891 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1893 if (rc != X86EMUL_CONTINUE)
1896 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1898 if (rc != X86EMUL_CONTINUE)
1901 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1903 if (rc != X86EMUL_CONTINUE)
1906 ctxt->_eip = temp_eip;
1909 if (ctxt->op_bytes == 4)
1910 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1911 else if (ctxt->op_bytes == 2) {
1912 ctxt->eflags &= ~0xffff;
1913 ctxt->eflags |= temp_eflags;
1916 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1917 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1922 static int em_iret(struct x86_emulate_ctxt *ctxt)
1924 switch(ctxt->mode) {
1925 case X86EMUL_MODE_REAL:
1926 return emulate_iret_real(ctxt);
1927 case X86EMUL_MODE_VM86:
1928 case X86EMUL_MODE_PROT16:
1929 case X86EMUL_MODE_PROT32:
1930 case X86EMUL_MODE_PROT64:
1932 /* iret from protected mode unimplemented yet */
1933 return X86EMUL_UNHANDLEABLE;
1937 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1942 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1944 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1945 if (rc != X86EMUL_CONTINUE)
1949 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1950 return X86EMUL_CONTINUE;
1953 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1955 int rc = X86EMUL_CONTINUE;
1957 switch (ctxt->modrm_reg) {
1958 case 2: /* call near abs */ {
1960 old_eip = ctxt->_eip;
1961 ctxt->_eip = ctxt->src.val;
1962 ctxt->src.val = old_eip;
1966 case 4: /* jmp abs */
1967 ctxt->_eip = ctxt->src.val;
1969 case 5: /* jmp far */
1970 rc = em_jmp_far(ctxt);
1979 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1981 u64 old = ctxt->dst.orig_val64;
1983 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1984 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1985 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1986 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1987 ctxt->eflags &= ~EFLG_ZF;
1989 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1990 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1992 ctxt->eflags |= EFLG_ZF;
1994 return X86EMUL_CONTINUE;
1997 static int em_ret(struct x86_emulate_ctxt *ctxt)
1999 ctxt->dst.type = OP_REG;
2000 ctxt->dst.addr.reg = &ctxt->_eip;
2001 ctxt->dst.bytes = ctxt->op_bytes;
2002 return em_pop(ctxt);
2005 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2010 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2011 if (rc != X86EMUL_CONTINUE)
2013 if (ctxt->op_bytes == 4)
2014 ctxt->_eip = (u32)ctxt->_eip;
2015 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2016 if (rc != X86EMUL_CONTINUE)
2018 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2022 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2026 rc = em_ret_far(ctxt);
2027 if (rc != X86EMUL_CONTINUE)
2029 rsp_increment(ctxt, ctxt->src.val);
2030 return X86EMUL_CONTINUE;
2033 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2035 /* Save real source value, then compare EAX against destination. */
2036 ctxt->src.orig_val = ctxt->src.val;
2037 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2038 fastop(ctxt, em_cmp);
2040 if (ctxt->eflags & EFLG_ZF) {
2041 /* Success: write back to memory. */
2042 ctxt->dst.val = ctxt->src.orig_val;
2044 /* Failure: write the value we saw to EAX. */
2045 ctxt->dst.type = OP_REG;
2046 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2048 return X86EMUL_CONTINUE;
2051 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2053 int seg = ctxt->src2.val;
2057 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2059 rc = load_segment_descriptor(ctxt, sel, seg);
2060 if (rc != X86EMUL_CONTINUE)
2063 ctxt->dst.val = ctxt->src.val;
2068 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2069 struct desc_struct *cs, struct desc_struct *ss)
2071 cs->l = 0; /* will be adjusted later */
2072 set_desc_base(cs, 0); /* flat segment */
2073 cs->g = 1; /* 4kb granularity */
2074 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2075 cs->type = 0x0b; /* Read, Execute, Accessed */
2077 cs->dpl = 0; /* will be adjusted later */
2082 set_desc_base(ss, 0); /* flat segment */
2083 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2084 ss->g = 1; /* 4kb granularity */
2086 ss->type = 0x03; /* Read/Write, Accessed */
2087 ss->d = 1; /* 32bit stack segment */
2094 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2096 u32 eax, ebx, ecx, edx;
2099 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2100 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2101 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2102 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2105 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2107 const struct x86_emulate_ops *ops = ctxt->ops;
2108 u32 eax, ebx, ecx, edx;
2111 * syscall should always be enabled in longmode - so only become
2112 * vendor specific (cpuid) if other modes are active...
2114 if (ctxt->mode == X86EMUL_MODE_PROT64)
2119 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2121 * Intel ("GenuineIntel")
2122 * remark: Intel CPUs only support "syscall" in 64bit
2123 * longmode. Also an 64bit guest with a
2124 * 32bit compat-app running will #UD !! While this
2125 * behaviour can be fixed (by emulating) into AMD
2126 * response - CPUs of AMD can't behave like Intel.
2128 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2129 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2130 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2133 /* AMD ("AuthenticAMD") */
2134 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2135 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2136 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2139 /* AMD ("AMDisbetter!") */
2140 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2141 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2142 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2145 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2149 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2151 const struct x86_emulate_ops *ops = ctxt->ops;
2152 struct desc_struct cs, ss;
2157 /* syscall is not available in real mode */
2158 if (ctxt->mode == X86EMUL_MODE_REAL ||
2159 ctxt->mode == X86EMUL_MODE_VM86)
2160 return emulate_ud(ctxt);
2162 if (!(em_syscall_is_enabled(ctxt)))
2163 return emulate_ud(ctxt);
2165 ops->get_msr(ctxt, MSR_EFER, &efer);
2166 setup_syscalls_segments(ctxt, &cs, &ss);
2168 if (!(efer & EFER_SCE))
2169 return emulate_ud(ctxt);
2171 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2173 cs_sel = (u16)(msr_data & 0xfffc);
2174 ss_sel = (u16)(msr_data + 8);
2176 if (efer & EFER_LMA) {
2180 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2181 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2183 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2184 if (efer & EFER_LMA) {
2185 #ifdef CONFIG_X86_64
2186 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2189 ctxt->mode == X86EMUL_MODE_PROT64 ?
2190 MSR_LSTAR : MSR_CSTAR, &msr_data);
2191 ctxt->_eip = msr_data;
2193 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2194 ctxt->eflags &= ~(msr_data | EFLG_RF);
2198 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2199 ctxt->_eip = (u32)msr_data;
2201 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2204 return X86EMUL_CONTINUE;
2207 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2209 const struct x86_emulate_ops *ops = ctxt->ops;
2210 struct desc_struct cs, ss;
2215 ops->get_msr(ctxt, MSR_EFER, &efer);
2216 /* inject #GP if in real mode */
2217 if (ctxt->mode == X86EMUL_MODE_REAL)
2218 return emulate_gp(ctxt, 0);
2221 * Not recognized on AMD in compat mode (but is recognized in legacy
2224 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2225 && !vendor_intel(ctxt))
2226 return emulate_ud(ctxt);
2228 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2229 * Therefore, we inject an #UD.
2231 if (ctxt->mode == X86EMUL_MODE_PROT64)
2232 return emulate_ud(ctxt);
2234 setup_syscalls_segments(ctxt, &cs, &ss);
2236 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2237 switch (ctxt->mode) {
2238 case X86EMUL_MODE_PROT32:
2239 if ((msr_data & 0xfffc) == 0x0)
2240 return emulate_gp(ctxt, 0);
2242 case X86EMUL_MODE_PROT64:
2243 if (msr_data == 0x0)
2244 return emulate_gp(ctxt, 0);
2250 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2251 cs_sel = (u16)msr_data;
2252 cs_sel &= ~SELECTOR_RPL_MASK;
2253 ss_sel = cs_sel + 8;
2254 ss_sel &= ~SELECTOR_RPL_MASK;
2255 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2260 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2261 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2263 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2264 ctxt->_eip = msr_data;
2266 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2267 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2269 return X86EMUL_CONTINUE;
2272 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2274 const struct x86_emulate_ops *ops = ctxt->ops;
2275 struct desc_struct cs, ss;
2278 u16 cs_sel = 0, ss_sel = 0;
2280 /* inject #GP if in real mode or Virtual 8086 mode */
2281 if (ctxt->mode == X86EMUL_MODE_REAL ||
2282 ctxt->mode == X86EMUL_MODE_VM86)
2283 return emulate_gp(ctxt, 0);
2285 setup_syscalls_segments(ctxt, &cs, &ss);
2287 if ((ctxt->rex_prefix & 0x8) != 0x0)
2288 usermode = X86EMUL_MODE_PROT64;
2290 usermode = X86EMUL_MODE_PROT32;
2294 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2296 case X86EMUL_MODE_PROT32:
2297 cs_sel = (u16)(msr_data + 16);
2298 if ((msr_data & 0xfffc) == 0x0)
2299 return emulate_gp(ctxt, 0);
2300 ss_sel = (u16)(msr_data + 24);
2302 case X86EMUL_MODE_PROT64:
2303 cs_sel = (u16)(msr_data + 32);
2304 if (msr_data == 0x0)
2305 return emulate_gp(ctxt, 0);
2306 ss_sel = cs_sel + 8;
2311 cs_sel |= SELECTOR_RPL_MASK;
2312 ss_sel |= SELECTOR_RPL_MASK;
2314 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2315 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2317 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2318 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2320 return X86EMUL_CONTINUE;
2323 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2326 if (ctxt->mode == X86EMUL_MODE_REAL)
2328 if (ctxt->mode == X86EMUL_MODE_VM86)
2330 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2331 return ctxt->ops->cpl(ctxt) > iopl;
2334 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2337 const struct x86_emulate_ops *ops = ctxt->ops;
2338 struct desc_struct tr_seg;
2341 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2342 unsigned mask = (1 << len) - 1;
2345 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2348 if (desc_limit_scaled(&tr_seg) < 103)
2350 base = get_desc_base(&tr_seg);
2351 #ifdef CONFIG_X86_64
2352 base |= ((u64)base3) << 32;
2354 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2355 if (r != X86EMUL_CONTINUE)
2357 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2359 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2360 if (r != X86EMUL_CONTINUE)
2362 if ((perm >> bit_idx) & mask)
2367 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2373 if (emulator_bad_iopl(ctxt))
2374 if (!emulator_io_port_access_allowed(ctxt, port, len))
2377 ctxt->perm_ok = true;
2382 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2383 struct tss_segment_16 *tss)
2385 tss->ip = ctxt->_eip;
2386 tss->flag = ctxt->eflags;
2387 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2388 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2389 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2390 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2391 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2392 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2393 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2394 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2396 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2397 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2398 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2399 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2400 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2403 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2404 struct tss_segment_16 *tss)
2408 ctxt->_eip = tss->ip;
2409 ctxt->eflags = tss->flag | 2;
2410 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2411 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2412 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2413 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2414 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2415 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2416 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2417 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2420 * SDM says that segment selectors are loaded before segment
2423 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2424 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2425 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2426 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2427 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2430 * Now load segment descriptors. If fault happens at this stage
2431 * it is handled in a context of new task
2433 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2434 if (ret != X86EMUL_CONTINUE)
2436 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2437 if (ret != X86EMUL_CONTINUE)
2439 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2440 if (ret != X86EMUL_CONTINUE)
2442 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2443 if (ret != X86EMUL_CONTINUE)
2445 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2446 if (ret != X86EMUL_CONTINUE)
2449 return X86EMUL_CONTINUE;
2452 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2453 u16 tss_selector, u16 old_tss_sel,
2454 ulong old_tss_base, struct desc_struct *new_desc)
2456 const struct x86_emulate_ops *ops = ctxt->ops;
2457 struct tss_segment_16 tss_seg;
2459 u32 new_tss_base = get_desc_base(new_desc);
2461 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2463 if (ret != X86EMUL_CONTINUE)
2464 /* FIXME: need to provide precise fault address */
2467 save_state_to_tss16(ctxt, &tss_seg);
2469 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2471 if (ret != X86EMUL_CONTINUE)
2472 /* FIXME: need to provide precise fault address */
2475 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2477 if (ret != X86EMUL_CONTINUE)
2478 /* FIXME: need to provide precise fault address */
2481 if (old_tss_sel != 0xffff) {
2482 tss_seg.prev_task_link = old_tss_sel;
2484 ret = ops->write_std(ctxt, new_tss_base,
2485 &tss_seg.prev_task_link,
2486 sizeof tss_seg.prev_task_link,
2488 if (ret != X86EMUL_CONTINUE)
2489 /* FIXME: need to provide precise fault address */
2493 return load_state_from_tss16(ctxt, &tss_seg);
2496 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2497 struct tss_segment_32 *tss)
2499 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2500 tss->eip = ctxt->_eip;
2501 tss->eflags = ctxt->eflags;
2502 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2503 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2504 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2505 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2506 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2507 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2508 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2509 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2511 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2512 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2513 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2514 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2515 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2516 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2517 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2520 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2521 struct tss_segment_32 *tss)
2525 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2526 return emulate_gp(ctxt, 0);
2527 ctxt->_eip = tss->eip;
2528 ctxt->eflags = tss->eflags | 2;
2530 /* General purpose registers */
2531 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2532 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2533 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2534 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2535 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2536 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2537 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2538 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2541 * SDM says that segment selectors are loaded before segment
2544 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2545 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2546 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2547 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2548 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2549 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2550 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2553 * If we're switching between Protected Mode and VM86, we need to make
2554 * sure to update the mode before loading the segment descriptors so
2555 * that the selectors are interpreted correctly.
2557 * Need to get rflags to the vcpu struct immediately because it
2558 * influences the CPL which is checked at least when loading the segment
2559 * descriptors and when pushing an error code to the new kernel stack.
2561 * TODO Introduce a separate ctxt->ops->set_cpl callback
2563 if (ctxt->eflags & X86_EFLAGS_VM)
2564 ctxt->mode = X86EMUL_MODE_VM86;
2566 ctxt->mode = X86EMUL_MODE_PROT32;
2568 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2571 * Now load segment descriptors. If fault happenes at this stage
2572 * it is handled in a context of new task
2574 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2575 if (ret != X86EMUL_CONTINUE)
2577 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2578 if (ret != X86EMUL_CONTINUE)
2580 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2581 if (ret != X86EMUL_CONTINUE)
2583 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2584 if (ret != X86EMUL_CONTINUE)
2586 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2587 if (ret != X86EMUL_CONTINUE)
2589 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2590 if (ret != X86EMUL_CONTINUE)
2592 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2593 if (ret != X86EMUL_CONTINUE)
2596 return X86EMUL_CONTINUE;
2599 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2600 u16 tss_selector, u16 old_tss_sel,
2601 ulong old_tss_base, struct desc_struct *new_desc)
2603 const struct x86_emulate_ops *ops = ctxt->ops;
2604 struct tss_segment_32 tss_seg;
2606 u32 new_tss_base = get_desc_base(new_desc);
2608 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2610 if (ret != X86EMUL_CONTINUE)
2611 /* FIXME: need to provide precise fault address */
2614 save_state_to_tss32(ctxt, &tss_seg);
2616 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2618 if (ret != X86EMUL_CONTINUE)
2619 /* FIXME: need to provide precise fault address */
2622 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2624 if (ret != X86EMUL_CONTINUE)
2625 /* FIXME: need to provide precise fault address */
2628 if (old_tss_sel != 0xffff) {
2629 tss_seg.prev_task_link = old_tss_sel;
2631 ret = ops->write_std(ctxt, new_tss_base,
2632 &tss_seg.prev_task_link,
2633 sizeof tss_seg.prev_task_link,
2635 if (ret != X86EMUL_CONTINUE)
2636 /* FIXME: need to provide precise fault address */
2640 return load_state_from_tss32(ctxt, &tss_seg);
2643 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2644 u16 tss_selector, int idt_index, int reason,
2645 bool has_error_code, u32 error_code)
2647 const struct x86_emulate_ops *ops = ctxt->ops;
2648 struct desc_struct curr_tss_desc, next_tss_desc;
2650 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2651 ulong old_tss_base =
2652 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2656 /* FIXME: old_tss_base == ~0 ? */
2658 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2659 if (ret != X86EMUL_CONTINUE)
2661 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2662 if (ret != X86EMUL_CONTINUE)
2665 /* FIXME: check that next_tss_desc is tss */
2668 * Check privileges. The three cases are task switch caused by...
2670 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2671 * 2. Exception/IRQ/iret: No check is performed
2672 * 3. jmp/call to TSS: Check against DPL of the TSS
2674 if (reason == TASK_SWITCH_GATE) {
2675 if (idt_index != -1) {
2676 /* Software interrupts */
2677 struct desc_struct task_gate_desc;
2680 ret = read_interrupt_descriptor(ctxt, idt_index,
2682 if (ret != X86EMUL_CONTINUE)
2685 dpl = task_gate_desc.dpl;
2686 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2687 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2689 } else if (reason != TASK_SWITCH_IRET) {
2690 int dpl = next_tss_desc.dpl;
2691 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2692 return emulate_gp(ctxt, tss_selector);
2696 desc_limit = desc_limit_scaled(&next_tss_desc);
2697 if (!next_tss_desc.p ||
2698 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2699 desc_limit < 0x2b)) {
2700 emulate_ts(ctxt, tss_selector & 0xfffc);
2701 return X86EMUL_PROPAGATE_FAULT;
2704 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2705 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2706 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2709 if (reason == TASK_SWITCH_IRET)
2710 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2712 /* set back link to prev task only if NT bit is set in eflags
2713 note that old_tss_sel is not used after this point */
2714 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2715 old_tss_sel = 0xffff;
2717 if (next_tss_desc.type & 8)
2718 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2719 old_tss_base, &next_tss_desc);
2721 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2722 old_tss_base, &next_tss_desc);
2723 if (ret != X86EMUL_CONTINUE)
2726 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2727 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2729 if (reason != TASK_SWITCH_IRET) {
2730 next_tss_desc.type |= (1 << 1); /* set busy flag */
2731 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2734 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2735 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2737 if (has_error_code) {
2738 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2739 ctxt->lock_prefix = 0;
2740 ctxt->src.val = (unsigned long) error_code;
2741 ret = em_push(ctxt);
2747 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2748 u16 tss_selector, int idt_index, int reason,
2749 bool has_error_code, u32 error_code)
2753 invalidate_registers(ctxt);
2754 ctxt->_eip = ctxt->eip;
2755 ctxt->dst.type = OP_NONE;
2757 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2758 has_error_code, error_code);
2760 if (rc == X86EMUL_CONTINUE) {
2761 ctxt->eip = ctxt->_eip;
2762 writeback_registers(ctxt);
2765 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2768 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2771 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2773 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2774 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2777 static int em_das(struct x86_emulate_ctxt *ctxt)
2780 bool af, cf, old_cf;
2782 cf = ctxt->eflags & X86_EFLAGS_CF;
2788 af = ctxt->eflags & X86_EFLAGS_AF;
2789 if ((al & 0x0f) > 9 || af) {
2791 cf = old_cf | (al >= 250);
2796 if (old_al > 0x99 || old_cf) {
2802 /* Set PF, ZF, SF */
2803 ctxt->src.type = OP_IMM;
2805 ctxt->src.bytes = 1;
2806 fastop(ctxt, em_or);
2807 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2809 ctxt->eflags |= X86_EFLAGS_CF;
2811 ctxt->eflags |= X86_EFLAGS_AF;
2812 return X86EMUL_CONTINUE;
2815 static int em_aam(struct x86_emulate_ctxt *ctxt)
2819 if (ctxt->src.val == 0)
2820 return emulate_de(ctxt);
2822 al = ctxt->dst.val & 0xff;
2823 ah = al / ctxt->src.val;
2824 al %= ctxt->src.val;
2826 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2828 /* Set PF, ZF, SF */
2829 ctxt->src.type = OP_IMM;
2831 ctxt->src.bytes = 1;
2832 fastop(ctxt, em_or);
2834 return X86EMUL_CONTINUE;
2837 static int em_aad(struct x86_emulate_ctxt *ctxt)
2839 u8 al = ctxt->dst.val & 0xff;
2840 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2842 al = (al + (ah * ctxt->src.val)) & 0xff;
2844 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2846 /* Set PF, ZF, SF */
2847 ctxt->src.type = OP_IMM;
2849 ctxt->src.bytes = 1;
2850 fastop(ctxt, em_or);
2852 return X86EMUL_CONTINUE;
2855 static int em_call(struct x86_emulate_ctxt *ctxt)
2857 long rel = ctxt->src.val;
2859 ctxt->src.val = (unsigned long)ctxt->_eip;
2861 return em_push(ctxt);
2864 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2870 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2871 old_eip = ctxt->_eip;
2873 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2874 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2875 return X86EMUL_CONTINUE;
2878 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2880 ctxt->src.val = old_cs;
2882 if (rc != X86EMUL_CONTINUE)
2885 ctxt->src.val = old_eip;
2886 return em_push(ctxt);
2889 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2893 ctxt->dst.type = OP_REG;
2894 ctxt->dst.addr.reg = &ctxt->_eip;
2895 ctxt->dst.bytes = ctxt->op_bytes;
2896 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2897 if (rc != X86EMUL_CONTINUE)
2899 rsp_increment(ctxt, ctxt->src.val);
2900 return X86EMUL_CONTINUE;
2903 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2905 /* Write back the register source. */
2906 ctxt->src.val = ctxt->dst.val;
2907 write_register_operand(&ctxt->src);
2909 /* Write back the memory destination with implicit LOCK prefix. */
2910 ctxt->dst.val = ctxt->src.orig_val;
2911 ctxt->lock_prefix = 1;
2912 return X86EMUL_CONTINUE;
2915 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2917 ctxt->dst.val = ctxt->src2.val;
2918 return fastop(ctxt, em_imul);
2921 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2923 ctxt->dst.type = OP_REG;
2924 ctxt->dst.bytes = ctxt->src.bytes;
2925 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2926 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2928 return X86EMUL_CONTINUE;
2931 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2935 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2936 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2937 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2938 return X86EMUL_CONTINUE;
2941 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2945 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2946 return emulate_gp(ctxt, 0);
2947 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2948 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2949 return X86EMUL_CONTINUE;
2952 static int em_mov(struct x86_emulate_ctxt *ctxt)
2954 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2955 return X86EMUL_CONTINUE;
2958 #define FFL(x) bit(X86_FEATURE_##x)
2960 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2962 u32 ebx, ecx, edx, eax = 1;
2966 * Check MOVBE is set in the guest-visible CPUID leaf.
2968 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2969 if (!(ecx & FFL(MOVBE)))
2970 return emulate_ud(ctxt);
2972 switch (ctxt->op_bytes) {
2975 * From MOVBE definition: "...When the operand size is 16 bits,
2976 * the upper word of the destination register remains unchanged
2979 * Both casting ->valptr and ->val to u16 breaks strict aliasing
2980 * rules so we have to do the operation almost per hand.
2982 tmp = (u16)ctxt->src.val;
2983 ctxt->dst.val &= ~0xffffUL;
2984 ctxt->dst.val |= (unsigned long)swab16(tmp);
2987 ctxt->dst.val = swab32((u32)ctxt->src.val);
2990 ctxt->dst.val = swab64(ctxt->src.val);
2993 return X86EMUL_PROPAGATE_FAULT;
2995 return X86EMUL_CONTINUE;
2998 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3000 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3001 return emulate_gp(ctxt, 0);
3003 /* Disable writeback. */
3004 ctxt->dst.type = OP_NONE;
3005 return X86EMUL_CONTINUE;
3008 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3012 if (ctxt->mode == X86EMUL_MODE_PROT64)
3013 val = ctxt->src.val & ~0ULL;
3015 val = ctxt->src.val & ~0U;
3017 /* #UD condition is already handled. */
3018 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3019 return emulate_gp(ctxt, 0);
3021 /* Disable writeback. */
3022 ctxt->dst.type = OP_NONE;
3023 return X86EMUL_CONTINUE;
3026 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3030 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3031 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3032 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3033 return emulate_gp(ctxt, 0);
3035 return X86EMUL_CONTINUE;
3038 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3042 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3043 return emulate_gp(ctxt, 0);
3045 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3046 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3047 return X86EMUL_CONTINUE;
3050 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3052 if (ctxt->modrm_reg > VCPU_SREG_GS)
3053 return emulate_ud(ctxt);
3055 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3056 return X86EMUL_CONTINUE;
3059 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3061 u16 sel = ctxt->src.val;
3063 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3064 return emulate_ud(ctxt);
3066 if (ctxt->modrm_reg == VCPU_SREG_SS)
3067 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3069 /* Disable writeback. */
3070 ctxt->dst.type = OP_NONE;
3071 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3074 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3076 u16 sel = ctxt->src.val;
3078 /* Disable writeback. */
3079 ctxt->dst.type = OP_NONE;
3080 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3083 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3085 u16 sel = ctxt->src.val;
3087 /* Disable writeback. */
3088 ctxt->dst.type = OP_NONE;
3089 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3092 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3097 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3098 if (rc == X86EMUL_CONTINUE)
3099 ctxt->ops->invlpg(ctxt, linear);
3100 /* Disable writeback. */
3101 ctxt->dst.type = OP_NONE;
3102 return X86EMUL_CONTINUE;
3105 static int em_clts(struct x86_emulate_ctxt *ctxt)
3109 cr0 = ctxt->ops->get_cr(ctxt, 0);
3111 ctxt->ops->set_cr(ctxt, 0, cr0);
3112 return X86EMUL_CONTINUE;
3115 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3119 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3120 return X86EMUL_UNHANDLEABLE;
3122 rc = ctxt->ops->fix_hypercall(ctxt);
3123 if (rc != X86EMUL_CONTINUE)
3126 /* Let the processor re-execute the fixed hypercall */
3127 ctxt->_eip = ctxt->eip;
3128 /* Disable writeback. */
3129 ctxt->dst.type = OP_NONE;
3130 return X86EMUL_CONTINUE;
3133 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3134 void (*get)(struct x86_emulate_ctxt *ctxt,
3135 struct desc_ptr *ptr))
3137 struct desc_ptr desc_ptr;
3139 if (ctxt->mode == X86EMUL_MODE_PROT64)
3141 get(ctxt, &desc_ptr);
3142 if (ctxt->op_bytes == 2) {
3144 desc_ptr.address &= 0x00ffffff;
3146 /* Disable writeback. */
3147 ctxt->dst.type = OP_NONE;
3148 return segmented_write(ctxt, ctxt->dst.addr.mem,
3149 &desc_ptr, 2 + ctxt->op_bytes);
3152 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3154 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3157 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3159 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3162 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3164 struct desc_ptr desc_ptr;
3167 if (ctxt->mode == X86EMUL_MODE_PROT64)
3169 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3170 &desc_ptr.size, &desc_ptr.address,
3172 if (rc != X86EMUL_CONTINUE)
3174 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3175 /* Disable writeback. */
3176 ctxt->dst.type = OP_NONE;
3177 return X86EMUL_CONTINUE;
3180 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3184 rc = ctxt->ops->fix_hypercall(ctxt);
3186 /* Disable writeback. */
3187 ctxt->dst.type = OP_NONE;
3191 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3193 struct desc_ptr desc_ptr;
3196 if (ctxt->mode == X86EMUL_MODE_PROT64)
3198 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3199 &desc_ptr.size, &desc_ptr.address,
3201 if (rc != X86EMUL_CONTINUE)
3203 ctxt->ops->set_idt(ctxt, &desc_ptr);
3204 /* Disable writeback. */
3205 ctxt->dst.type = OP_NONE;
3206 return X86EMUL_CONTINUE;
3209 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3211 ctxt->dst.bytes = 2;
3212 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3213 return X86EMUL_CONTINUE;
3216 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3218 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3219 | (ctxt->src.val & 0x0f));
3220 ctxt->dst.type = OP_NONE;
3221 return X86EMUL_CONTINUE;
3224 static int em_loop(struct x86_emulate_ctxt *ctxt)
3226 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3227 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3228 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3229 jmp_rel(ctxt, ctxt->src.val);
3231 return X86EMUL_CONTINUE;
3234 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3236 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3237 jmp_rel(ctxt, ctxt->src.val);
3239 return X86EMUL_CONTINUE;
3242 static int em_in(struct x86_emulate_ctxt *ctxt)
3244 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3246 return X86EMUL_IO_NEEDED;
3248 return X86EMUL_CONTINUE;
3251 static int em_out(struct x86_emulate_ctxt *ctxt)
3253 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3255 /* Disable writeback. */
3256 ctxt->dst.type = OP_NONE;
3257 return X86EMUL_CONTINUE;
3260 static int em_cli(struct x86_emulate_ctxt *ctxt)
3262 if (emulator_bad_iopl(ctxt))
3263 return emulate_gp(ctxt, 0);
3265 ctxt->eflags &= ~X86_EFLAGS_IF;
3266 return X86EMUL_CONTINUE;
3269 static int em_sti(struct x86_emulate_ctxt *ctxt)
3271 if (emulator_bad_iopl(ctxt))
3272 return emulate_gp(ctxt, 0);
3274 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3275 ctxt->eflags |= X86_EFLAGS_IF;
3276 return X86EMUL_CONTINUE;
3279 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3281 u32 eax, ebx, ecx, edx;
3283 eax = reg_read(ctxt, VCPU_REGS_RAX);
3284 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3285 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3286 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3287 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3288 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3289 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3290 return X86EMUL_CONTINUE;
3293 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3297 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3298 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3300 ctxt->eflags &= ~0xffUL;
3301 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3302 return X86EMUL_CONTINUE;
3305 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3307 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3308 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3309 return X86EMUL_CONTINUE;
3312 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3314 switch (ctxt->op_bytes) {
3315 #ifdef CONFIG_X86_64
3317 asm("bswap %0" : "+r"(ctxt->dst.val));
3321 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3324 return X86EMUL_CONTINUE;
3327 static bool valid_cr(int nr)
3339 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3341 if (!valid_cr(ctxt->modrm_reg))
3342 return emulate_ud(ctxt);
3344 return X86EMUL_CONTINUE;
3347 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3349 u64 new_val = ctxt->src.val64;
3350 int cr = ctxt->modrm_reg;
3353 static u64 cr_reserved_bits[] = {
3354 0xffffffff00000000ULL,
3355 0, 0, 0, /* CR3 checked later */
3362 return emulate_ud(ctxt);
3364 if (new_val & cr_reserved_bits[cr])
3365 return emulate_gp(ctxt, 0);
3370 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3371 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3372 return emulate_gp(ctxt, 0);
3374 cr4 = ctxt->ops->get_cr(ctxt, 4);
3375 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3377 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3378 !(cr4 & X86_CR4_PAE))
3379 return emulate_gp(ctxt, 0);
3386 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3387 if (efer & EFER_LMA)
3388 rsvd = CR3_L_MODE_RESERVED_BITS;
3389 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3390 rsvd = CR3_PAE_RESERVED_BITS;
3391 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3392 rsvd = CR3_NONPAE_RESERVED_BITS;
3395 return emulate_gp(ctxt, 0);
3400 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3402 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3403 return emulate_gp(ctxt, 0);
3409 return X86EMUL_CONTINUE;
3412 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3416 ctxt->ops->get_dr(ctxt, 7, &dr7);
3418 /* Check if DR7.Global_Enable is set */
3419 return dr7 & (1 << 13);
3422 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3424 int dr = ctxt->modrm_reg;
3428 return emulate_ud(ctxt);
3430 cr4 = ctxt->ops->get_cr(ctxt, 4);
3431 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3432 return emulate_ud(ctxt);
3434 if (check_dr7_gd(ctxt))
3435 return emulate_db(ctxt);
3437 return X86EMUL_CONTINUE;
3440 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3442 u64 new_val = ctxt->src.val64;
3443 int dr = ctxt->modrm_reg;
3445 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3446 return emulate_gp(ctxt, 0);
3448 return check_dr_read(ctxt);
3451 static int check_svme(struct x86_emulate_ctxt *ctxt)
3455 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3457 if (!(efer & EFER_SVME))
3458 return emulate_ud(ctxt);
3460 return X86EMUL_CONTINUE;
3463 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3465 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3467 /* Valid physical address? */
3468 if (rax & 0xffff000000000000ULL)
3469 return emulate_gp(ctxt, 0);
3471 return check_svme(ctxt);
3474 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3476 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3478 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3479 return emulate_ud(ctxt);
3481 return X86EMUL_CONTINUE;
3484 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3486 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3487 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3489 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3491 return emulate_gp(ctxt, 0);
3493 return X86EMUL_CONTINUE;
3496 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3498 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3499 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3500 return emulate_gp(ctxt, 0);
3502 return X86EMUL_CONTINUE;
3505 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3507 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3508 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3509 return emulate_gp(ctxt, 0);
3511 return X86EMUL_CONTINUE;
3514 #define D(_y) { .flags = (_y) }
3515 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3516 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3517 .check_perm = (_p) }
3518 #define N D(NotImpl)
3519 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3520 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3521 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3522 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3523 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3524 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3525 #define II(_f, _e, _i) \
3526 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3527 #define IIP(_f, _e, _i, _p) \
3528 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3529 .check_perm = (_p) }
3530 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3532 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3533 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3534 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3535 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3536 #define I2bvIP(_f, _e, _i, _p) \
3537 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3539 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3540 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3541 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3543 static const struct opcode group7_rm1[] = {
3544 DI(SrcNone | Priv, monitor),
3545 DI(SrcNone | Priv, mwait),
3549 static const struct opcode group7_rm3[] = {
3550 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3551 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3552 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3553 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3554 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3555 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3556 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3557 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3560 static const struct opcode group7_rm7[] = {
3562 DIP(SrcNone, rdtscp, check_rdtsc),
3566 static const struct opcode group1[] = {
3568 F(Lock | PageTable, em_or),
3571 F(Lock | PageTable, em_and),
3577 static const struct opcode group1A[] = {
3578 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3581 static const struct opcode group2[] = {
3582 F(DstMem | ModRM, em_rol),
3583 F(DstMem | ModRM, em_ror),
3584 F(DstMem | ModRM, em_rcl),
3585 F(DstMem | ModRM, em_rcr),
3586 F(DstMem | ModRM, em_shl),
3587 F(DstMem | ModRM, em_shr),
3588 F(DstMem | ModRM, em_shl),
3589 F(DstMem | ModRM, em_sar),
3592 static const struct opcode group3[] = {
3593 F(DstMem | SrcImm | NoWrite, em_test),
3594 F(DstMem | SrcImm | NoWrite, em_test),
3595 F(DstMem | SrcNone | Lock, em_not),
3596 F(DstMem | SrcNone | Lock, em_neg),
3597 F(DstXacc | Src2Mem, em_mul_ex),
3598 F(DstXacc | Src2Mem, em_imul_ex),
3599 F(DstXacc | Src2Mem, em_div_ex),
3600 F(DstXacc | Src2Mem, em_idiv_ex),
3603 static const struct opcode group4[] = {
3604 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3605 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3609 static const struct opcode group5[] = {
3610 F(DstMem | SrcNone | Lock, em_inc),
3611 F(DstMem | SrcNone | Lock, em_dec),
3612 I(SrcMem | Stack, em_grp45),
3613 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3614 I(SrcMem | Stack, em_grp45),
3615 I(SrcMemFAddr | ImplicitOps, em_grp45),
3616 I(SrcMem | Stack, em_grp45), D(Undefined),
3619 static const struct opcode group6[] = {
3622 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3623 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3627 static const struct group_dual group7 = { {
3628 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3629 II(Mov | DstMem | Priv, em_sidt, sidt),
3630 II(SrcMem | Priv, em_lgdt, lgdt),
3631 II(SrcMem | Priv, em_lidt, lidt),
3632 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3633 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3634 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3636 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3638 N, EXT(0, group7_rm3),
3639 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3640 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3644 static const struct opcode group8[] = {
3646 F(DstMem | SrcImmByte | NoWrite, em_bt),
3647 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3648 F(DstMem | SrcImmByte | Lock, em_btr),
3649 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3652 static const struct group_dual group9 = { {
3653 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3655 N, N, N, N, N, N, N, N,
3658 static const struct opcode group11[] = {
3659 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3663 static const struct gprefix pfx_0f_6f_0f_7f = {
3664 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3667 static const struct gprefix pfx_vmovntpx = {
3668 I(0, em_mov), N, N, N,
3671 static const struct gprefix pfx_0f_28_0f_29 = {
3672 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3675 static const struct escape escape_d9 = { {
3676 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3679 N, N, N, N, N, N, N, N,
3681 N, N, N, N, N, N, N, N,
3683 N, N, N, N, N, N, N, N,
3685 N, N, N, N, N, N, N, N,
3687 N, N, N, N, N, N, N, N,
3689 N, N, N, N, N, N, N, N,
3691 N, N, N, N, N, N, N, N,
3693 N, N, N, N, N, N, N, N,
3696 static const struct escape escape_db = { {
3697 N, N, N, N, N, N, N, N,
3700 N, N, N, N, N, N, N, N,
3702 N, N, N, N, N, N, N, N,
3704 N, N, N, N, N, N, N, N,
3706 N, N, N, N, N, N, N, N,
3708 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3710 N, N, N, N, N, N, N, N,
3712 N, N, N, N, N, N, N, N,
3714 N, N, N, N, N, N, N, N,
3717 static const struct escape escape_dd = { {
3718 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3721 N, N, N, N, N, N, N, N,
3723 N, N, N, N, N, N, N, N,
3725 N, N, N, N, N, N, N, N,
3727 N, N, N, N, N, N, N, N,
3729 N, N, N, N, N, N, N, N,
3731 N, N, N, N, N, N, N, N,
3733 N, N, N, N, N, N, N, N,
3735 N, N, N, N, N, N, N, N,
3738 static const struct opcode opcode_table[256] = {
3740 F6ALU(Lock, em_add),
3741 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3742 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3744 F6ALU(Lock | PageTable, em_or),
3745 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3748 F6ALU(Lock, em_adc),
3749 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3750 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3752 F6ALU(Lock, em_sbb),
3753 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3754 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3756 F6ALU(Lock | PageTable, em_and), N, N,
3758 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3760 F6ALU(Lock, em_xor), N, N,
3762 F6ALU(NoWrite, em_cmp), N, N,
3764 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3766 X8(I(SrcReg | Stack, em_push)),
3768 X8(I(DstReg | Stack, em_pop)),
3770 I(ImplicitOps | Stack | No64, em_pusha),
3771 I(ImplicitOps | Stack | No64, em_popa),
3772 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3775 I(SrcImm | Mov | Stack, em_push),
3776 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3777 I(SrcImmByte | Mov | Stack, em_push),
3778 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3779 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3780 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3784 G(ByteOp | DstMem | SrcImm, group1),
3785 G(DstMem | SrcImm, group1),
3786 G(ByteOp | DstMem | SrcImm | No64, group1),
3787 G(DstMem | SrcImmByte, group1),
3788 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3789 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3791 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3792 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3793 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3794 D(ModRM | SrcMem | NoAccess | DstReg),
3795 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3798 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3800 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3801 I(SrcImmFAddr | No64, em_call_far), N,
3802 II(ImplicitOps | Stack, em_pushf, pushf),
3803 II(ImplicitOps | Stack, em_popf, popf),
3804 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3806 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3807 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3808 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3809 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3811 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3812 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3813 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3814 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3816 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3818 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3820 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3821 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3822 I(ImplicitOps | Stack, em_ret),
3823 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3824 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3825 G(ByteOp, group11), G(0, group11),
3827 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3828 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3829 I(ImplicitOps | Stack, em_ret_far),
3830 D(ImplicitOps), DI(SrcImmByte, intn),
3831 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3833 G(Src2One | ByteOp, group2), G(Src2One, group2),
3834 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3835 I(DstAcc | SrcImmUByte | No64, em_aam),
3836 I(DstAcc | SrcImmUByte | No64, em_aad),
3837 F(DstAcc | ByteOp | No64, em_salc),
3838 I(DstAcc | SrcXLat | ByteOp, em_mov),
3840 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3842 X3(I(SrcImmByte, em_loop)),
3843 I(SrcImmByte, em_jcxz),
3844 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3845 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3847 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3848 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3849 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3850 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3852 N, DI(ImplicitOps, icebp), N, N,
3853 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3854 G(ByteOp, group3), G(0, group3),
3856 D(ImplicitOps), D(ImplicitOps),
3857 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3858 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3861 static const struct opcode twobyte_table[256] = {
3863 G(0, group6), GD(0, &group7), N, N,
3864 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3865 II(ImplicitOps | Priv, em_clts, clts), N,
3866 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3867 N, D(ImplicitOps | ModRM), N, N,
3869 N, N, N, N, N, N, N, N,
3870 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3872 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3873 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3874 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3875 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3877 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3878 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3879 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3882 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3883 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3884 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3885 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3886 I(ImplicitOps | EmulateOnUD, em_sysenter),
3887 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3889 N, N, N, N, N, N, N, N,
3891 X16(D(DstReg | SrcMem | ModRM | Mov)),
3893 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3898 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3903 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3907 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3909 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3910 II(ImplicitOps, em_cpuid, cpuid),
3911 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3912 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3913 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3915 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3916 DI(ImplicitOps, rsm),
3917 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3918 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3919 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3920 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3922 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3923 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3924 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3925 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3926 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3927 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3931 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3932 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3933 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3935 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3936 N, D(DstMem | SrcReg | ModRM | Mov),
3937 N, N, N, GD(0, &group9),
3939 X8(I(DstReg, em_bswap)),
3941 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3943 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3945 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3948 static const struct gprefix three_byte_0f_38_f0 = {
3949 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3952 static const struct gprefix three_byte_0f_38_f1 = {
3953 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3957 * Insns below are selected by the prefix which indexed by the third opcode
3960 static const struct opcode opcode_map_0f_38[256] = {
3962 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3964 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3966 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3967 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
3986 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3990 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3996 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3997 unsigned size, bool sign_extension)
3999 int rc = X86EMUL_CONTINUE;
4003 op->addr.mem.ea = ctxt->_eip;
4004 /* NB. Immediates are sign-extended as necessary. */
4005 switch (op->bytes) {
4007 op->val = insn_fetch(s8, ctxt);
4010 op->val = insn_fetch(s16, ctxt);
4013 op->val = insn_fetch(s32, ctxt);
4016 op->val = insn_fetch(s64, ctxt);
4019 if (!sign_extension) {
4020 switch (op->bytes) {
4028 op->val &= 0xffffffff;
4036 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4039 int rc = X86EMUL_CONTINUE;
4043 decode_register_operand(ctxt, op);
4046 rc = decode_imm(ctxt, op, 1, false);
4049 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4053 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4054 fetch_bit_operand(ctxt);
4055 op->orig_val = op->val;
4058 ctxt->memop.bytes = 8;
4062 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4063 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4064 fetch_register_operand(op);
4065 op->orig_val = op->val;
4069 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4070 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4071 fetch_register_operand(op);
4072 op->orig_val = op->val;
4075 if (ctxt->d & ByteOp) {
4080 op->bytes = ctxt->op_bytes;
4081 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4082 fetch_register_operand(op);
4083 op->orig_val = op->val;
4087 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4089 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4090 op->addr.mem.seg = VCPU_SREG_ES;
4097 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4098 fetch_register_operand(op);
4102 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4105 rc = decode_imm(ctxt, op, 1, true);
4112 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4115 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4118 ctxt->memop.bytes = 1;
4119 if (ctxt->memop.type == OP_REG) {
4120 ctxt->memop.addr.reg = decode_register(ctxt,
4121 ctxt->modrm_rm, true);
4122 fetch_register_operand(&ctxt->memop);
4126 ctxt->memop.bytes = 2;
4129 ctxt->memop.bytes = 4;
4132 rc = decode_imm(ctxt, op, 2, false);
4135 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4139 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4141 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4142 op->addr.mem.seg = seg_override(ctxt);
4148 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4150 register_address(ctxt,
4151 reg_read(ctxt, VCPU_REGS_RBX) +
4152 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4153 op->addr.mem.seg = seg_override(ctxt);
4158 op->addr.mem.ea = ctxt->_eip;
4159 op->bytes = ctxt->op_bytes + 2;
4160 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4163 ctxt->memop.bytes = ctxt->op_bytes + 2;
4166 op->val = VCPU_SREG_ES;
4169 op->val = VCPU_SREG_CS;
4172 op->val = VCPU_SREG_SS;
4175 op->val = VCPU_SREG_DS;
4178 op->val = VCPU_SREG_FS;
4181 op->val = VCPU_SREG_GS;
4184 /* Special instructions do their own operand decoding. */
4186 op->type = OP_NONE; /* Disable writeback. */
4194 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4196 int rc = X86EMUL_CONTINUE;
4197 int mode = ctxt->mode;
4198 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4199 bool op_prefix = false;
4200 struct opcode opcode;
4202 ctxt->memop.type = OP_NONE;
4203 ctxt->memopp = NULL;
4204 ctxt->_eip = ctxt->eip;
4205 ctxt->fetch.start = ctxt->_eip;
4206 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4207 ctxt->opcode_len = 1;
4209 memcpy(ctxt->fetch.data, insn, insn_len);
4212 case X86EMUL_MODE_REAL:
4213 case X86EMUL_MODE_VM86:
4214 case X86EMUL_MODE_PROT16:
4215 def_op_bytes = def_ad_bytes = 2;
4217 case X86EMUL_MODE_PROT32:
4218 def_op_bytes = def_ad_bytes = 4;
4220 #ifdef CONFIG_X86_64
4221 case X86EMUL_MODE_PROT64:
4227 return EMULATION_FAILED;
4230 ctxt->op_bytes = def_op_bytes;
4231 ctxt->ad_bytes = def_ad_bytes;
4233 /* Legacy prefixes. */
4235 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4236 case 0x66: /* operand-size override */
4238 /* switch between 2/4 bytes */
4239 ctxt->op_bytes = def_op_bytes ^ 6;
4241 case 0x67: /* address-size override */
4242 if (mode == X86EMUL_MODE_PROT64)
4243 /* switch between 4/8 bytes */
4244 ctxt->ad_bytes = def_ad_bytes ^ 12;
4246 /* switch between 2/4 bytes */
4247 ctxt->ad_bytes = def_ad_bytes ^ 6;
4249 case 0x26: /* ES override */
4250 case 0x2e: /* CS override */
4251 case 0x36: /* SS override */
4252 case 0x3e: /* DS override */
4253 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4255 case 0x64: /* FS override */
4256 case 0x65: /* GS override */
4257 set_seg_override(ctxt, ctxt->b & 7);
4259 case 0x40 ... 0x4f: /* REX */
4260 if (mode != X86EMUL_MODE_PROT64)
4262 ctxt->rex_prefix = ctxt->b;
4264 case 0xf0: /* LOCK */
4265 ctxt->lock_prefix = 1;
4267 case 0xf2: /* REPNE/REPNZ */
4268 case 0xf3: /* REP/REPE/REPZ */
4269 ctxt->rep_prefix = ctxt->b;
4275 /* Any legacy prefix after a REX prefix nullifies its effect. */
4277 ctxt->rex_prefix = 0;
4283 if (ctxt->rex_prefix & 8)
4284 ctxt->op_bytes = 8; /* REX.W */
4286 /* Opcode byte(s). */
4287 opcode = opcode_table[ctxt->b];
4288 /* Two-byte opcode? */
4289 if (ctxt->b == 0x0f) {
4290 ctxt->opcode_len = 2;
4291 ctxt->b = insn_fetch(u8, ctxt);
4292 opcode = twobyte_table[ctxt->b];
4294 /* 0F_38 opcode map */
4295 if (ctxt->b == 0x38) {
4296 ctxt->opcode_len = 3;
4297 ctxt->b = insn_fetch(u8, ctxt);
4298 opcode = opcode_map_0f_38[ctxt->b];
4301 ctxt->d = opcode.flags;
4303 if (ctxt->d & ModRM)
4304 ctxt->modrm = insn_fetch(u8, ctxt);
4306 while (ctxt->d & GroupMask) {
4307 switch (ctxt->d & GroupMask) {
4309 goffset = (ctxt->modrm >> 3) & 7;
4310 opcode = opcode.u.group[goffset];
4313 goffset = (ctxt->modrm >> 3) & 7;
4314 if ((ctxt->modrm >> 6) == 3)
4315 opcode = opcode.u.gdual->mod3[goffset];
4317 opcode = opcode.u.gdual->mod012[goffset];
4320 goffset = ctxt->modrm & 7;
4321 opcode = opcode.u.group[goffset];
4324 if (ctxt->rep_prefix && op_prefix)
4325 return EMULATION_FAILED;
4326 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4327 switch (simd_prefix) {
4328 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4329 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4330 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4331 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4335 if (ctxt->modrm > 0xbf)
4336 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4338 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4341 return EMULATION_FAILED;
4344 ctxt->d &= ~(u64)GroupMask;
4345 ctxt->d |= opcode.flags;
4348 ctxt->execute = opcode.u.execute;
4349 ctxt->check_perm = opcode.check_perm;
4350 ctxt->intercept = opcode.intercept;
4353 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4354 return EMULATION_FAILED;
4356 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4357 return EMULATION_FAILED;
4359 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4362 if (ctxt->d & Op3264) {
4363 if (mode == X86EMUL_MODE_PROT64)
4370 ctxt->op_bytes = 16;
4371 else if (ctxt->d & Mmx)
4374 /* ModRM and SIB bytes. */
4375 if (ctxt->d & ModRM) {
4376 rc = decode_modrm(ctxt, &ctxt->memop);
4377 if (!ctxt->has_seg_override)
4378 set_seg_override(ctxt, ctxt->modrm_seg);
4379 } else if (ctxt->d & MemAbs)
4380 rc = decode_abs(ctxt, &ctxt->memop);
4381 if (rc != X86EMUL_CONTINUE)
4384 if (!ctxt->has_seg_override)
4385 set_seg_override(ctxt, VCPU_SREG_DS);
4387 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4389 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4390 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4393 * Decode and fetch the source operand: register, memory
4396 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4397 if (rc != X86EMUL_CONTINUE)
4401 * Decode and fetch the second source operand: register, memory
4404 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4405 if (rc != X86EMUL_CONTINUE)
4408 /* Decode and fetch the destination operand: register or memory. */
4409 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4412 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4413 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4415 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4418 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4420 return ctxt->d & PageTable;
4423 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4425 /* The second termination condition only applies for REPE
4426 * and REPNE. Test if the repeat string operation prefix is
4427 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4428 * corresponding termination condition according to:
4429 * - if REPE/REPZ and ZF = 0 then done
4430 * - if REPNE/REPNZ and ZF = 1 then done
4432 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4433 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4434 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4435 ((ctxt->eflags & EFLG_ZF) == 0))
4436 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4437 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4443 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4447 ctxt->ops->get_fpu(ctxt);
4448 asm volatile("1: fwait \n\t"
4450 ".pushsection .fixup,\"ax\" \n\t"
4452 "movb $1, %[fault] \n\t"
4455 _ASM_EXTABLE(1b, 3b)
4456 : [fault]"+qm"(fault));
4457 ctxt->ops->put_fpu(ctxt);
4459 if (unlikely(fault))
4460 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4462 return X86EMUL_CONTINUE;
4465 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4468 if (op->type == OP_MM)
4469 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4472 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4474 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4475 if (!(ctxt->d & ByteOp))
4476 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4477 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4478 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4480 : "c"(ctxt->src2.val));
4481 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4482 if (!fop) /* exception is returned in fop variable */
4483 return emulate_de(ctxt);
4484 return X86EMUL_CONTINUE;
4487 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4489 const struct x86_emulate_ops *ops = ctxt->ops;
4490 int rc = X86EMUL_CONTINUE;
4491 int saved_dst_type = ctxt->dst.type;
4493 ctxt->mem_read.pos = 0;
4495 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4496 (ctxt->d & Undefined)) {
4497 rc = emulate_ud(ctxt);
4501 /* LOCK prefix is allowed only with some instructions */
4502 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4503 rc = emulate_ud(ctxt);
4507 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4508 rc = emulate_ud(ctxt);
4512 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4513 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4514 rc = emulate_ud(ctxt);
4518 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4519 rc = emulate_nm(ctxt);
4523 if (ctxt->d & Mmx) {
4524 rc = flush_pending_x87_faults(ctxt);
4525 if (rc != X86EMUL_CONTINUE)
4528 * Now that we know the fpu is exception safe, we can fetch
4531 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4532 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4533 if (!(ctxt->d & Mov))
4534 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4537 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4538 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4539 X86_ICPT_PRE_EXCEPT);
4540 if (rc != X86EMUL_CONTINUE)
4544 /* Privileged instruction can be executed only in CPL=0 */
4545 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4546 rc = emulate_gp(ctxt, 0);
4550 /* Instruction can only be executed in protected mode */
4551 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4552 rc = emulate_ud(ctxt);
4556 /* Do instruction specific permission checks */
4557 if (ctxt->check_perm) {
4558 rc = ctxt->check_perm(ctxt);
4559 if (rc != X86EMUL_CONTINUE)
4563 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4564 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4565 X86_ICPT_POST_EXCEPT);
4566 if (rc != X86EMUL_CONTINUE)
4570 if (ctxt->rep_prefix && (ctxt->d & String)) {
4571 /* All REP prefixes have the same first termination condition */
4572 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4573 ctxt->eip = ctxt->_eip;
4578 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4579 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4580 ctxt->src.valptr, ctxt->src.bytes);
4581 if (rc != X86EMUL_CONTINUE)
4583 ctxt->src.orig_val64 = ctxt->src.val64;
4586 if (ctxt->src2.type == OP_MEM) {
4587 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4588 &ctxt->src2.val, ctxt->src2.bytes);
4589 if (rc != X86EMUL_CONTINUE)
4593 if ((ctxt->d & DstMask) == ImplicitOps)
4597 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4598 /* optimisation - avoid slow emulated read if Mov */
4599 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4600 &ctxt->dst.val, ctxt->dst.bytes);
4601 if (rc != X86EMUL_CONTINUE)
4604 ctxt->dst.orig_val = ctxt->dst.val;
4608 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4609 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4610 X86_ICPT_POST_MEMACCESS);
4611 if (rc != X86EMUL_CONTINUE)
4615 if (ctxt->execute) {
4616 if (ctxt->d & Fastop) {
4617 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4618 rc = fastop(ctxt, fop);
4619 if (rc != X86EMUL_CONTINUE)
4623 rc = ctxt->execute(ctxt);
4624 if (rc != X86EMUL_CONTINUE)
4629 if (ctxt->opcode_len == 2)
4631 else if (ctxt->opcode_len == 3)
4632 goto threebyte_insn;
4635 case 0x63: /* movsxd */
4636 if (ctxt->mode != X86EMUL_MODE_PROT64)
4637 goto cannot_emulate;
4638 ctxt->dst.val = (s32) ctxt->src.val;
4640 case 0x70 ... 0x7f: /* jcc (short) */
4641 if (test_cc(ctxt->b, ctxt->eflags))
4642 jmp_rel(ctxt, ctxt->src.val);
4644 case 0x8d: /* lea r16/r32, m */
4645 ctxt->dst.val = ctxt->src.addr.mem.ea;
4647 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4648 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4652 case 0x98: /* cbw/cwde/cdqe */
4653 switch (ctxt->op_bytes) {
4654 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4655 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4656 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4659 case 0xcc: /* int3 */
4660 rc = emulate_int(ctxt, 3);
4662 case 0xcd: /* int n */
4663 rc = emulate_int(ctxt, ctxt->src.val);
4665 case 0xce: /* into */
4666 if (ctxt->eflags & EFLG_OF)
4667 rc = emulate_int(ctxt, 4);
4669 case 0xe9: /* jmp rel */
4670 case 0xeb: /* jmp rel short */
4671 jmp_rel(ctxt, ctxt->src.val);
4672 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4674 case 0xf4: /* hlt */
4675 ctxt->ops->halt(ctxt);
4677 case 0xf5: /* cmc */
4678 /* complement carry flag from eflags reg */
4679 ctxt->eflags ^= EFLG_CF;
4681 case 0xf8: /* clc */
4682 ctxt->eflags &= ~EFLG_CF;
4684 case 0xf9: /* stc */
4685 ctxt->eflags |= EFLG_CF;
4687 case 0xfc: /* cld */
4688 ctxt->eflags &= ~EFLG_DF;
4690 case 0xfd: /* std */
4691 ctxt->eflags |= EFLG_DF;
4694 goto cannot_emulate;
4697 if (rc != X86EMUL_CONTINUE)
4701 if (!(ctxt->d & NoWrite)) {
4702 rc = writeback(ctxt, &ctxt->dst);
4703 if (rc != X86EMUL_CONTINUE)
4706 if (ctxt->d & SrcWrite) {
4707 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4708 rc = writeback(ctxt, &ctxt->src);
4709 if (rc != X86EMUL_CONTINUE)
4714 * restore dst type in case the decoding will be reused
4715 * (happens for string instruction )
4717 ctxt->dst.type = saved_dst_type;
4719 if ((ctxt->d & SrcMask) == SrcSI)
4720 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4722 if ((ctxt->d & DstMask) == DstDI)
4723 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4725 if (ctxt->rep_prefix && (ctxt->d & String)) {
4727 struct read_cache *r = &ctxt->io_read;
4728 if ((ctxt->d & SrcMask) == SrcSI)
4729 count = ctxt->src.count;
4731 count = ctxt->dst.count;
4732 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4735 if (!string_insn_completed(ctxt)) {
4737 * Re-enter guest when pio read ahead buffer is empty
4738 * or, if it is not used, after each 1024 iteration.
4740 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4741 (r->end == 0 || r->end != r->pos)) {
4743 * Reset read cache. Usually happens before
4744 * decode, but since instruction is restarted
4745 * we have to do it here.
4747 ctxt->mem_read.end = 0;
4748 writeback_registers(ctxt);
4749 return EMULATION_RESTART;
4751 goto done; /* skip rip writeback */
4755 ctxt->eip = ctxt->_eip;
4758 if (rc == X86EMUL_PROPAGATE_FAULT)
4759 ctxt->have_exception = true;
4760 if (rc == X86EMUL_INTERCEPTED)
4761 return EMULATION_INTERCEPTED;
4763 if (rc == X86EMUL_CONTINUE)
4764 writeback_registers(ctxt);
4766 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4770 case 0x09: /* wbinvd */
4771 (ctxt->ops->wbinvd)(ctxt);
4773 case 0x08: /* invd */
4774 case 0x0d: /* GrpP (prefetch) */
4775 case 0x18: /* Grp16 (prefetch/nop) */
4776 case 0x1f: /* nop */
4778 case 0x20: /* mov cr, reg */
4779 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4781 case 0x21: /* mov from dr to reg */
4782 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4784 case 0x40 ... 0x4f: /* cmov */
4785 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4786 if (!test_cc(ctxt->b, ctxt->eflags))
4787 ctxt->dst.type = OP_NONE; /* no writeback */
4789 case 0x80 ... 0x8f: /* jnz rel, etc*/
4790 if (test_cc(ctxt->b, ctxt->eflags))
4791 jmp_rel(ctxt, ctxt->src.val);
4793 case 0x90 ... 0x9f: /* setcc r/m8 */
4794 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4796 case 0xae: /* clflush */
4798 case 0xb6 ... 0xb7: /* movzx */
4799 ctxt->dst.bytes = ctxt->op_bytes;
4800 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4801 : (u16) ctxt->src.val;
4803 case 0xbe ... 0xbf: /* movsx */
4804 ctxt->dst.bytes = ctxt->op_bytes;
4805 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4806 (s16) ctxt->src.val;
4808 case 0xc3: /* movnti */
4809 ctxt->dst.bytes = ctxt->op_bytes;
4810 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4811 (u64) ctxt->src.val;
4814 goto cannot_emulate;
4819 if (rc != X86EMUL_CONTINUE)
4825 return EMULATION_FAILED;
4828 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4830 invalidate_registers(ctxt);
4833 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4835 writeback_registers(ctxt);