1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
61 #define OpBits 5 /* Width of operand field */
62 #define OpMask ((1ull << OpBits) - 1)
65 * Opcode effective-address decode tables.
66 * Note that we only emulate instructions that have at least one memory
67 * operand (excluding implicit stack references). We assume that stack
68 * references and instruction fetches will never occur in special memory
69 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
73 /* Operand sizes: 8-bit operands or specified/overridden size. */
74 #define ByteOp (1<<0) /* 8-bit operands. */
75 /* Destination operand type. */
77 #define ImplicitOps (OpImplicit << DstShift)
78 #define DstReg (OpReg << DstShift)
79 #define DstMem (OpMem << DstShift)
80 #define DstAcc (OpAcc << DstShift)
81 #define DstDI (OpDI << DstShift)
82 #define DstMem64 (OpMem64 << DstShift)
83 #define DstImmUByte (OpImmUByte << DstShift)
84 #define DstDX (OpDX << DstShift)
85 #define DstMask (OpMask << DstShift)
86 /* Source operand type. */
88 #define SrcNone (OpNone << SrcShift)
89 #define SrcReg (OpReg << SrcShift)
90 #define SrcMem (OpMem << SrcShift)
91 #define SrcMem16 (OpMem16 << SrcShift)
92 #define SrcMem32 (OpMem32 << SrcShift)
93 #define SrcImm (OpImm << SrcShift)
94 #define SrcImmByte (OpImmByte << SrcShift)
95 #define SrcOne (OpOne << SrcShift)
96 #define SrcImmUByte (OpImmUByte << SrcShift)
97 #define SrcImmU (OpImmU << SrcShift)
98 #define SrcSI (OpSI << SrcShift)
99 #define SrcImmFAddr (OpImmFAddr << SrcShift)
100 #define SrcMemFAddr (OpMemFAddr << SrcShift)
101 #define SrcAcc (OpAcc << SrcShift)
102 #define SrcImmU16 (OpImmU16 << SrcShift)
103 #define SrcDX (OpDX << SrcShift)
104 #define SrcMask (OpMask << SrcShift)
105 #define BitOp (1<<11)
106 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
107 #define String (1<<13) /* String instruction (rep capable) */
108 #define Stack (1<<14) /* Stack instruction (push/pop) */
109 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
110 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
111 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
112 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
113 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
114 #define Sse (1<<18) /* SSE Vector instruction */
115 /* Generic ModRM decode. */
116 #define ModRM (1<<19)
117 /* Destination is only written; never read. */
120 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
121 #define VendorSpecific (1<<22) /* Vendor specific instruction */
122 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
123 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
124 #define Undefined (1<<25) /* No Such Instruction */
125 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
126 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
128 #define PageTable (1 << 29) /* instruction used to write page table */
129 /* Source 2 operand type */
130 #define Src2Shift (30)
131 #define Src2None (OpNone << Src2Shift)
132 #define Src2CL (OpCL << Src2Shift)
133 #define Src2ImmByte (OpImmByte << Src2Shift)
134 #define Src2One (OpOne << Src2Shift)
135 #define Src2Imm (OpImm << Src2Shift)
136 #define Src2ES (OpES << Src2Shift)
137 #define Src2CS (OpCS << Src2Shift)
138 #define Src2SS (OpSS << Src2Shift)
139 #define Src2DS (OpDS << Src2Shift)
140 #define Src2FS (OpFS << Src2Shift)
141 #define Src2GS (OpGS << Src2Shift)
142 #define Src2Mask (OpMask << Src2Shift)
144 #define X2(x...) x, x
145 #define X3(x...) X2(x), x
146 #define X4(x...) X2(x), X2(x)
147 #define X5(x...) X4(x), x
148 #define X6(x...) X4(x), X2(x)
149 #define X7(x...) X4(x), X3(x)
150 #define X8(x...) X4(x), X4(x)
151 #define X16(x...) X8(x), X8(x)
157 int (*execute)(struct x86_emulate_ctxt *ctxt);
158 struct opcode *group;
159 struct group_dual *gdual;
160 struct gprefix *gprefix;
162 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
166 struct opcode mod012[8];
167 struct opcode mod3[8];
171 struct opcode pfx_no;
172 struct opcode pfx_66;
173 struct opcode pfx_f2;
174 struct opcode pfx_f3;
177 /* EFLAGS bit definitions. */
178 #define EFLG_ID (1<<21)
179 #define EFLG_VIP (1<<20)
180 #define EFLG_VIF (1<<19)
181 #define EFLG_AC (1<<18)
182 #define EFLG_VM (1<<17)
183 #define EFLG_RF (1<<16)
184 #define EFLG_IOPL (3<<12)
185 #define EFLG_NT (1<<14)
186 #define EFLG_OF (1<<11)
187 #define EFLG_DF (1<<10)
188 #define EFLG_IF (1<<9)
189 #define EFLG_TF (1<<8)
190 #define EFLG_SF (1<<7)
191 #define EFLG_ZF (1<<6)
192 #define EFLG_AF (1<<4)
193 #define EFLG_PF (1<<2)
194 #define EFLG_CF (1<<0)
196 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
197 #define EFLG_RESERVED_ONE_MASK 2
200 * Instruction emulation:
201 * Most instructions are emulated directly via a fragment of inline assembly
202 * code. This allows us to save/restore EFLAGS and thus very easily pick up
203 * any modified flags.
206 #if defined(CONFIG_X86_64)
207 #define _LO32 "k" /* force 32-bit operand */
208 #define _STK "%%rsp" /* stack pointer */
209 #elif defined(__i386__)
210 #define _LO32 "" /* force 32-bit operand */
211 #define _STK "%%esp" /* stack pointer */
215 * These EFLAGS bits are restored from saved value during emulation, and
216 * any changes are written back to the saved value after emulation.
218 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
220 /* Before executing instruction: restore necessary bits in EFLAGS. */
221 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
222 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
223 "movl %"_sav",%"_LO32 _tmp"; " \
226 "movl %"_msk",%"_LO32 _tmp"; " \
227 "andl %"_LO32 _tmp",("_STK"); " \
229 "notl %"_LO32 _tmp"; " \
230 "andl %"_LO32 _tmp",("_STK"); " \
231 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
233 "orl %"_LO32 _tmp",("_STK"); " \
237 /* After executing instruction: write-back necessary bits in EFLAGS. */
238 #define _POST_EFLAGS(_sav, _msk, _tmp) \
239 /* _sav |= EFLAGS & _msk; */ \
242 "andl %"_msk",%"_LO32 _tmp"; " \
243 "orl %"_LO32 _tmp",%"_sav"; "
251 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
253 __asm__ __volatile__ ( \
254 _PRE_EFLAGS("0", "4", "2") \
255 _op _suffix " %"_x"3,%1; " \
256 _POST_EFLAGS("0", "4", "2") \
257 : "=m" ((ctxt)->eflags), \
258 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
260 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
264 /* Raw emulation: instruction has two explicit operands. */
265 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
267 unsigned long _tmp; \
269 switch ((ctxt)->dst.bytes) { \
271 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
274 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
277 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
282 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
284 unsigned long _tmp; \
285 switch ((ctxt)->dst.bytes) { \
287 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
290 __emulate_2op_nobyte(ctxt, _op, \
291 _wx, _wy, _lx, _ly, _qx, _qy); \
296 /* Source operand is byte-sized and may be restricted to just %cl. */
297 #define emulate_2op_SrcB(ctxt, _op) \
298 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
300 /* Source operand is byte, word, long or quad sized. */
301 #define emulate_2op_SrcV(ctxt, _op) \
302 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
304 /* Source operand is word, long or quad sized. */
305 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
306 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
308 /* Instruction has three operands and one operand is stored in ECX register */
309 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
311 unsigned long _tmp; \
312 _type _clv = (ctxt)->src2.val; \
313 _type _srcv = (ctxt)->src.val; \
314 _type _dstv = (ctxt)->dst.val; \
316 __asm__ __volatile__ ( \
317 _PRE_EFLAGS("0", "5", "2") \
318 _op _suffix " %4,%1 \n" \
319 _POST_EFLAGS("0", "5", "2") \
320 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
321 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
324 (ctxt)->src2.val = (unsigned long) _clv; \
325 (ctxt)->src2.val = (unsigned long) _srcv; \
326 (ctxt)->dst.val = (unsigned long) _dstv; \
329 #define emulate_2op_cl(ctxt, _op) \
331 switch ((ctxt)->dst.bytes) { \
333 __emulate_2op_cl(ctxt, _op, "w", u16); \
336 __emulate_2op_cl(ctxt, _op, "l", u32); \
339 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
344 #define __emulate_1op(ctxt, _op, _suffix) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "3", "2") \
350 _op _suffix " %1; " \
351 _POST_EFLAGS("0", "3", "2") \
352 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
354 : "i" (EFLAGS_MASK)); \
357 /* Instruction has only one explicit operand (no source operand). */
358 #define emulate_1op(ctxt, _op) \
360 switch ((ctxt)->dst.bytes) { \
361 case 1: __emulate_1op(ctxt, _op, "b"); break; \
362 case 2: __emulate_1op(ctxt, _op, "w"); break; \
363 case 4: __emulate_1op(ctxt, _op, "l"); break; \
364 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
368 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
370 unsigned long _tmp; \
371 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
372 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
374 __asm__ __volatile__ ( \
375 _PRE_EFLAGS("0", "5", "1") \
377 _op _suffix " %6; " \
379 _POST_EFLAGS("0", "5", "1") \
380 ".pushsection .fixup,\"ax\" \n\t" \
381 "3: movb $1, %4 \n\t" \
384 _ASM_EXTABLE(1b, 3b) \
385 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
386 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
387 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
388 "a" (*rax), "d" (*rdx)); \
391 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
392 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
394 switch((ctxt)->src.bytes) { \
396 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
399 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
402 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
405 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->rep_prefix,
417 .modrm_mod = ctxt->modrm_mod,
418 .modrm_reg = ctxt->modrm_reg,
419 .modrm_rm = ctxt->modrm_rm,
420 .src_val = ctxt->src.val64,
421 .src_bytes = ctxt->src.bytes,
422 .dst_bytes = ctxt->dst.bytes,
423 .ad_bytes = ctxt->ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
432 return (1UL << (ctxt->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
439 if (ctxt->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(ctxt);
445 static inline unsigned long
446 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
448 return address_mask(ctxt, reg);
452 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
454 if (ctxt->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
460 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
462 register_address_increment(ctxt, &ctxt->_eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
474 ctxt->has_seg_override = true;
475 ctxt->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
488 if (!ctxt->has_seg_override)
491 return ctxt->seg_override;
494 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
495 u32 error, bool valid)
497 ctxt->exception.vector = vec;
498 ctxt->exception.error_code = error;
499 ctxt->exception.error_code_valid = valid;
500 return X86EMUL_PROPAGATE_FAULT;
503 static int emulate_db(struct x86_emulate_ctxt *ctxt)
505 return emulate_exception(ctxt, DB_VECTOR, 0, false);
508 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
510 return emulate_exception(ctxt, GP_VECTOR, err, true);
513 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, SS_VECTOR, err, true);
518 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
520 return emulate_exception(ctxt, UD_VECTOR, 0, false);
523 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
525 return emulate_exception(ctxt, TS_VECTOR, err, true);
528 static int emulate_de(struct x86_emulate_ctxt *ctxt)
530 return emulate_exception(ctxt, DE_VECTOR, 0, false);
533 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, NM_VECTOR, 0, false);
538 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
541 struct desc_struct desc;
543 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
547 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
552 struct desc_struct desc;
554 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
558 static int __linearize(struct x86_emulate_ctxt *ctxt,
559 struct segmented_address addr,
560 unsigned size, bool write, bool fetch,
563 struct desc_struct desc;
570 la = seg_base(ctxt, addr.seg) + addr.ea;
571 switch (ctxt->mode) {
572 case X86EMUL_MODE_REAL:
574 case X86EMUL_MODE_PROT64:
575 if (((signed long)la << 16) >> 16 != la)
576 return emulate_gp(ctxt, 0);
579 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
583 /* code segment or read-only data segment */
584 if (((desc.type & 8) || !(desc.type & 2)) && write)
586 /* unreadable code segment */
587 if (!fetch && (desc.type & 8) && !(desc.type & 2))
589 lim = desc_limit_scaled(&desc);
590 if ((desc.type & 8) || !(desc.type & 4)) {
591 /* expand-up segment */
592 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
595 /* exapand-down segment */
596 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
598 lim = desc.d ? 0xffffffff : 0xffff;
599 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
602 cpl = ctxt->ops->cpl(ctxt);
605 if (!(desc.type & 8)) {
609 } else if ((desc.type & 8) && !(desc.type & 4)) {
610 /* nonconforming code segment */
613 } else if ((desc.type & 8) && (desc.type & 4)) {
614 /* conforming code segment */
620 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
623 return X86EMUL_CONTINUE;
625 if (addr.seg == VCPU_SREG_SS)
626 return emulate_ss(ctxt, addr.seg);
628 return emulate_gp(ctxt, addr.seg);
631 static int linearize(struct x86_emulate_ctxt *ctxt,
632 struct segmented_address addr,
633 unsigned size, bool write,
636 return __linearize(ctxt, addr, size, write, false, linear);
640 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
648 rc = linearize(ctxt, addr, size, false, &linear);
649 if (rc != X86EMUL_CONTINUE)
651 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
655 * Fetch the next byte of the instruction being emulated which is pointed to
656 * by ctxt->_eip, then increment ctxt->_eip.
658 * Also prefetch the remaining bytes of the instruction without crossing page
659 * boundary if they are not in fetch_cache yet.
661 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
663 struct fetch_cache *fc = &ctxt->fetch;
667 if (ctxt->_eip == fc->end) {
668 unsigned long linear;
669 struct segmented_address addr = { .seg = VCPU_SREG_CS,
671 cur_size = fc->end - fc->start;
672 size = min(15UL - cur_size,
673 PAGE_SIZE - offset_in_page(ctxt->_eip));
674 rc = __linearize(ctxt, addr, size, false, true, &linear);
675 if (unlikely(rc != X86EMUL_CONTINUE))
677 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
678 size, &ctxt->exception);
679 if (unlikely(rc != X86EMUL_CONTINUE))
683 *dest = fc->data[ctxt->_eip - fc->start];
685 return X86EMUL_CONTINUE;
688 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
689 void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_insn_fetch_byte(ctxt, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
704 /* Fetch next part of the instruction being emulated. */
705 #define insn_fetch(_type, _ctxt) \
706 ({ unsigned long _x; \
707 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
708 if (rc != X86EMUL_CONTINUE) \
713 #define insn_fetch_arr(_arr, _size, _ctxt) \
714 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
715 if (rc != X86EMUL_CONTINUE) \
720 * Given the 'reg' portion of a ModRM byte, and a register block, return a
721 * pointer into the block that addresses the relevant register.
722 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
724 static void *decode_register(u8 modrm_reg, unsigned long *regs,
729 p = ®s[modrm_reg];
730 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
731 p = (unsigned char *)®s[modrm_reg & 3] + 1;
735 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 u16 *size, unsigned long *address, int op_bytes)
744 rc = segmented_read_std(ctxt, addr, size, 2);
745 if (rc != X86EMUL_CONTINUE)
748 rc = segmented_read_std(ctxt, addr, address, op_bytes);
752 static int test_cc(unsigned int condition, unsigned int flags)
756 switch ((condition & 15) >> 1) {
758 rc |= (flags & EFLG_OF);
760 case 1: /* b/c/nae */
761 rc |= (flags & EFLG_CF);
764 rc |= (flags & EFLG_ZF);
767 rc |= (flags & (EFLG_CF|EFLG_ZF));
770 rc |= (flags & EFLG_SF);
773 rc |= (flags & EFLG_PF);
776 rc |= (flags & EFLG_ZF);
779 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
783 /* Odd condition identifiers (lsb == 1) have inverted sense. */
784 return (!!rc ^ (condition & 1));
787 static void fetch_register_operand(struct operand *op)
791 op->val = *(u8 *)op->addr.reg;
794 op->val = *(u16 *)op->addr.reg;
797 op->val = *(u32 *)op->addr.reg;
800 op->val = *(u64 *)op->addr.reg;
805 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
807 ctxt->ops->get_fpu(ctxt);
809 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
810 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
811 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
812 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
813 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
814 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
815 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
816 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
818 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
819 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
820 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
821 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
822 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
823 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
824 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
825 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
829 ctxt->ops->put_fpu(ctxt);
832 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
835 ctxt->ops->get_fpu(ctxt);
837 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
838 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
839 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
840 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
841 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
842 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
843 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
844 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
846 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
847 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
848 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
849 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
850 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
851 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
852 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
853 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
857 ctxt->ops->put_fpu(ctxt);
860 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
864 unsigned reg = ctxt->modrm_reg;
865 int highbyte_regs = ctxt->rex_prefix == 0;
867 if (!(ctxt->d & ModRM))
868 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
874 read_sse_reg(ctxt, &op->vec_val, reg);
879 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
880 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
883 op->addr.reg = decode_register(reg, ctxt->regs, 0);
884 op->bytes = ctxt->op_bytes;
886 fetch_register_operand(op);
887 op->orig_val = op->val;
890 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
894 int index_reg = 0, base_reg = 0, scale;
895 int rc = X86EMUL_CONTINUE;
898 if (ctxt->rex_prefix) {
899 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
900 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
901 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
904 ctxt->modrm = insn_fetch(u8, ctxt);
905 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
906 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
907 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
908 ctxt->modrm_seg = VCPU_SREG_DS;
910 if (ctxt->modrm_mod == 3) {
912 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
913 op->addr.reg = decode_register(ctxt->modrm_rm,
914 ctxt->regs, ctxt->d & ByteOp);
918 op->addr.xmm = ctxt->modrm_rm;
919 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
922 fetch_register_operand(op);
928 if (ctxt->ad_bytes == 2) {
929 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
930 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
931 unsigned si = ctxt->regs[VCPU_REGS_RSI];
932 unsigned di = ctxt->regs[VCPU_REGS_RDI];
934 /* 16-bit ModR/M decode. */
935 switch (ctxt->modrm_mod) {
937 if (ctxt->modrm_rm == 6)
938 modrm_ea += insn_fetch(u16, ctxt);
941 modrm_ea += insn_fetch(s8, ctxt);
944 modrm_ea += insn_fetch(u16, ctxt);
947 switch (ctxt->modrm_rm) {
967 if (ctxt->modrm_mod != 0)
974 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
975 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
976 ctxt->modrm_seg = VCPU_SREG_SS;
977 modrm_ea = (u16)modrm_ea;
979 /* 32/64-bit ModR/M decode. */
980 if ((ctxt->modrm_rm & 7) == 4) {
981 sib = insn_fetch(u8, ctxt);
982 index_reg |= (sib >> 3) & 7;
986 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
987 modrm_ea += insn_fetch(s32, ctxt);
989 modrm_ea += ctxt->regs[base_reg];
991 modrm_ea += ctxt->regs[index_reg] << scale;
992 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
993 if (ctxt->mode == X86EMUL_MODE_PROT64)
994 ctxt->rip_relative = 1;
996 modrm_ea += ctxt->regs[ctxt->modrm_rm];
997 switch (ctxt->modrm_mod) {
999 if (ctxt->modrm_rm == 5)
1000 modrm_ea += insn_fetch(s32, ctxt);
1003 modrm_ea += insn_fetch(s8, ctxt);
1006 modrm_ea += insn_fetch(s32, ctxt);
1010 op->addr.mem.ea = modrm_ea;
1015 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1018 int rc = X86EMUL_CONTINUE;
1021 switch (ctxt->ad_bytes) {
1023 op->addr.mem.ea = insn_fetch(u16, ctxt);
1026 op->addr.mem.ea = insn_fetch(u32, ctxt);
1029 op->addr.mem.ea = insn_fetch(u64, ctxt);
1036 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1041 mask = ~(ctxt->dst.bytes * 8 - 1);
1043 if (ctxt->src.bytes == 2)
1044 sv = (s16)ctxt->src.val & (s16)mask;
1045 else if (ctxt->src.bytes == 4)
1046 sv = (s32)ctxt->src.val & (s32)mask;
1048 ctxt->dst.addr.mem.ea += (sv >> 3);
1051 /* only subword offset */
1052 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1055 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1056 unsigned long addr, void *dest, unsigned size)
1059 struct read_cache *mc = &ctxt->mem_read;
1062 int n = min(size, 8u);
1064 if (mc->pos < mc->end)
1067 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1069 if (rc != X86EMUL_CONTINUE)
1074 memcpy(dest, mc->data + mc->pos, n);
1079 return X86EMUL_CONTINUE;
1082 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1083 struct segmented_address addr,
1090 rc = linearize(ctxt, addr, size, false, &linear);
1091 if (rc != X86EMUL_CONTINUE)
1093 return read_emulated(ctxt, linear, data, size);
1096 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1097 struct segmented_address addr,
1104 rc = linearize(ctxt, addr, size, true, &linear);
1105 if (rc != X86EMUL_CONTINUE)
1107 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1111 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1112 struct segmented_address addr,
1113 const void *orig_data, const void *data,
1119 rc = linearize(ctxt, addr, size, true, &linear);
1120 if (rc != X86EMUL_CONTINUE)
1122 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1123 size, &ctxt->exception);
1126 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1127 unsigned int size, unsigned short port,
1130 struct read_cache *rc = &ctxt->io_read;
1132 if (rc->pos == rc->end) { /* refill pio read ahead */
1133 unsigned int in_page, n;
1134 unsigned int count = ctxt->rep_prefix ?
1135 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1136 in_page = (ctxt->eflags & EFLG_DF) ?
1137 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1138 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1139 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1143 rc->pos = rc->end = 0;
1144 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1149 memcpy(dest, rc->data + rc->pos, size);
1154 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1155 u16 selector, struct desc_ptr *dt)
1157 struct x86_emulate_ops *ops = ctxt->ops;
1159 if (selector & 1 << 2) {
1160 struct desc_struct desc;
1163 memset (dt, 0, sizeof *dt);
1164 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1167 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1168 dt->address = get_desc_base(&desc);
1170 ops->get_gdt(ctxt, dt);
1173 /* allowed just for 8 bytes segments */
1174 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 u16 selector, struct desc_struct *desc)
1178 u16 index = selector >> 3;
1181 get_descriptor_table_ptr(ctxt, selector, &dt);
1183 if (dt.size < index * 8 + 7)
1184 return emulate_gp(ctxt, selector & 0xfffc);
1186 addr = dt.address + index * 8;
1187 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1191 /* allowed just for 8 bytes segments */
1192 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1193 u16 selector, struct desc_struct *desc)
1196 u16 index = selector >> 3;
1199 get_descriptor_table_ptr(ctxt, selector, &dt);
1201 if (dt.size < index * 8 + 7)
1202 return emulate_gp(ctxt, selector & 0xfffc);
1204 addr = dt.address + index * 8;
1205 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1209 /* Does not support long mode */
1210 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1211 u16 selector, int seg)
1213 struct desc_struct seg_desc;
1215 unsigned err_vec = GP_VECTOR;
1217 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1220 memset(&seg_desc, 0, sizeof seg_desc);
1222 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1223 || ctxt->mode == X86EMUL_MODE_REAL) {
1224 /* set real mode segment descriptor */
1225 set_desc_base(&seg_desc, selector << 4);
1226 set_desc_limit(&seg_desc, 0xffff);
1233 /* NULL selector is not valid for TR, CS and SS */
1234 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1238 /* TR should be in GDT only */
1239 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1242 if (null_selector) /* for NULL selector skip all following checks */
1245 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1246 if (ret != X86EMUL_CONTINUE)
1249 err_code = selector & 0xfffc;
1250 err_vec = GP_VECTOR;
1252 /* can't load system descriptor into segment selecor */
1253 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1257 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1263 cpl = ctxt->ops->cpl(ctxt);
1268 * segment is not a writable data segment or segment
1269 * selector's RPL != CPL or segment selector's RPL != CPL
1271 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1275 if (!(seg_desc.type & 8))
1278 if (seg_desc.type & 4) {
1284 if (rpl > cpl || dpl != cpl)
1287 /* CS(RPL) <- CPL */
1288 selector = (selector & 0xfffc) | cpl;
1291 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1294 case VCPU_SREG_LDTR:
1295 if (seg_desc.s || seg_desc.type != 2)
1298 default: /* DS, ES, FS, or GS */
1300 * segment is not a data or readable code segment or
1301 * ((segment is a data or nonconforming code segment)
1302 * and (both RPL and CPL > DPL))
1304 if ((seg_desc.type & 0xa) == 0x8 ||
1305 (((seg_desc.type & 0xc) != 0xc) &&
1306 (rpl > dpl && cpl > dpl)))
1312 /* mark segment as accessed */
1314 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1315 if (ret != X86EMUL_CONTINUE)
1319 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1320 return X86EMUL_CONTINUE;
1322 emulate_exception(ctxt, err_vec, err_code, true);
1323 return X86EMUL_PROPAGATE_FAULT;
1326 static void write_register_operand(struct operand *op)
1328 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1329 switch (op->bytes) {
1331 *(u8 *)op->addr.reg = (u8)op->val;
1334 *(u16 *)op->addr.reg = (u16)op->val;
1337 *op->addr.reg = (u32)op->val;
1338 break; /* 64b: zero-extend */
1340 *op->addr.reg = op->val;
1345 static int writeback(struct x86_emulate_ctxt *ctxt)
1349 switch (ctxt->dst.type) {
1351 write_register_operand(&ctxt->dst);
1354 if (ctxt->lock_prefix)
1355 rc = segmented_cmpxchg(ctxt,
1357 &ctxt->dst.orig_val,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct segmented_address addr;
1384 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1385 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1386 addr.seg = VCPU_SREG_SS;
1388 /* Disable writeback. */
1389 ctxt->dst.type = OP_NONE;
1390 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1393 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1394 void *dest, int len)
1397 struct segmented_address addr;
1399 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1400 addr.seg = VCPU_SREG_SS;
1401 rc = segmented_read(ctxt, addr, dest, len);
1402 if (rc != X86EMUL_CONTINUE)
1405 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1409 static int em_pop(struct x86_emulate_ctxt *ctxt)
1411 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1414 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1415 void *dest, int len)
1418 unsigned long val, change_mask;
1419 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1420 int cpl = ctxt->ops->cpl(ctxt);
1422 rc = emulate_pop(ctxt, &val, len);
1423 if (rc != X86EMUL_CONTINUE)
1426 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1427 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1429 switch(ctxt->mode) {
1430 case X86EMUL_MODE_PROT64:
1431 case X86EMUL_MODE_PROT32:
1432 case X86EMUL_MODE_PROT16:
1434 change_mask |= EFLG_IOPL;
1436 change_mask |= EFLG_IF;
1438 case X86EMUL_MODE_VM86:
1440 return emulate_gp(ctxt, 0);
1441 change_mask |= EFLG_IF;
1443 default: /* real mode */
1444 change_mask |= (EFLG_IOPL | EFLG_IF);
1448 *(unsigned long *)dest =
1449 (ctxt->eflags & ~change_mask) | (val & change_mask);
1454 static int em_popf(struct x86_emulate_ctxt *ctxt)
1456 ctxt->dst.type = OP_REG;
1457 ctxt->dst.addr.reg = &ctxt->eflags;
1458 ctxt->dst.bytes = ctxt->op_bytes;
1459 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1462 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1464 int seg = ctxt->src2.val;
1466 ctxt->src.val = get_segment_selector(ctxt, seg);
1468 return em_push(ctxt);
1471 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1473 int seg = ctxt->src2.val;
1474 unsigned long selector;
1477 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1481 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1485 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1487 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1488 int rc = X86EMUL_CONTINUE;
1489 int reg = VCPU_REGS_RAX;
1491 while (reg <= VCPU_REGS_RDI) {
1492 (reg == VCPU_REGS_RSP) ?
1493 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1496 if (rc != X86EMUL_CONTINUE)
1505 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1507 ctxt->src.val = (unsigned long)ctxt->eflags;
1508 return em_push(ctxt);
1511 static int em_popa(struct x86_emulate_ctxt *ctxt)
1513 int rc = X86EMUL_CONTINUE;
1514 int reg = VCPU_REGS_RDI;
1516 while (reg >= VCPU_REGS_RAX) {
1517 if (reg == VCPU_REGS_RSP) {
1518 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1523 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1524 if (rc != X86EMUL_CONTINUE)
1531 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1533 struct x86_emulate_ops *ops = ctxt->ops;
1540 /* TODO: Add limit checks */
1541 ctxt->src.val = ctxt->eflags;
1543 if (rc != X86EMUL_CONTINUE)
1546 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1548 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1550 if (rc != X86EMUL_CONTINUE)
1553 ctxt->src.val = ctxt->_eip;
1555 if (rc != X86EMUL_CONTINUE)
1558 ops->get_idt(ctxt, &dt);
1560 eip_addr = dt.address + (irq << 2);
1561 cs_addr = dt.address + (irq << 2) + 2;
1563 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1564 if (rc != X86EMUL_CONTINUE)
1567 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1568 if (rc != X86EMUL_CONTINUE)
1571 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1572 if (rc != X86EMUL_CONTINUE)
1580 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1582 switch(ctxt->mode) {
1583 case X86EMUL_MODE_REAL:
1584 return emulate_int_real(ctxt, irq);
1585 case X86EMUL_MODE_VM86:
1586 case X86EMUL_MODE_PROT16:
1587 case X86EMUL_MODE_PROT32:
1588 case X86EMUL_MODE_PROT64:
1590 /* Protected mode interrupts unimplemented yet */
1591 return X86EMUL_UNHANDLEABLE;
1595 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1597 int rc = X86EMUL_CONTINUE;
1598 unsigned long temp_eip = 0;
1599 unsigned long temp_eflags = 0;
1600 unsigned long cs = 0;
1601 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1602 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1603 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1604 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1606 /* TODO: Add stack limit check */
1608 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1610 if (rc != X86EMUL_CONTINUE)
1613 if (temp_eip & ~0xffff)
1614 return emulate_gp(ctxt, 0);
1616 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1618 if (rc != X86EMUL_CONTINUE)
1621 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1623 if (rc != X86EMUL_CONTINUE)
1626 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1628 if (rc != X86EMUL_CONTINUE)
1631 ctxt->_eip = temp_eip;
1634 if (ctxt->op_bytes == 4)
1635 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1636 else if (ctxt->op_bytes == 2) {
1637 ctxt->eflags &= ~0xffff;
1638 ctxt->eflags |= temp_eflags;
1641 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1642 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1647 static int em_iret(struct x86_emulate_ctxt *ctxt)
1649 switch(ctxt->mode) {
1650 case X86EMUL_MODE_REAL:
1651 return emulate_iret_real(ctxt);
1652 case X86EMUL_MODE_VM86:
1653 case X86EMUL_MODE_PROT16:
1654 case X86EMUL_MODE_PROT32:
1655 case X86EMUL_MODE_PROT64:
1657 /* iret from protected mode unimplemented yet */
1658 return X86EMUL_UNHANDLEABLE;
1662 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1667 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1669 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1670 if (rc != X86EMUL_CONTINUE)
1674 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1675 return X86EMUL_CONTINUE;
1678 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1680 switch (ctxt->modrm_reg) {
1682 emulate_2op_SrcB(ctxt, "rol");
1685 emulate_2op_SrcB(ctxt, "ror");
1688 emulate_2op_SrcB(ctxt, "rcl");
1691 emulate_2op_SrcB(ctxt, "rcr");
1693 case 4: /* sal/shl */
1694 case 6: /* sal/shl */
1695 emulate_2op_SrcB(ctxt, "sal");
1698 emulate_2op_SrcB(ctxt, "shr");
1701 emulate_2op_SrcB(ctxt, "sar");
1704 return X86EMUL_CONTINUE;
1707 static int em_not(struct x86_emulate_ctxt *ctxt)
1709 ctxt->dst.val = ~ctxt->dst.val;
1710 return X86EMUL_CONTINUE;
1713 static int em_neg(struct x86_emulate_ctxt *ctxt)
1715 emulate_1op(ctxt, "neg");
1716 return X86EMUL_CONTINUE;
1719 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1723 emulate_1op_rax_rdx(ctxt, "mul", ex);
1724 return X86EMUL_CONTINUE;
1727 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1731 emulate_1op_rax_rdx(ctxt, "imul", ex);
1732 return X86EMUL_CONTINUE;
1735 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1739 emulate_1op_rax_rdx(ctxt, "div", de);
1741 return emulate_de(ctxt);
1742 return X86EMUL_CONTINUE;
1745 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1749 emulate_1op_rax_rdx(ctxt, "idiv", de);
1751 return emulate_de(ctxt);
1752 return X86EMUL_CONTINUE;
1755 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1757 int rc = X86EMUL_CONTINUE;
1759 switch (ctxt->modrm_reg) {
1761 emulate_1op(ctxt, "inc");
1764 emulate_1op(ctxt, "dec");
1766 case 2: /* call near abs */ {
1768 old_eip = ctxt->_eip;
1769 ctxt->_eip = ctxt->src.val;
1770 ctxt->src.val = old_eip;
1774 case 4: /* jmp abs */
1775 ctxt->_eip = ctxt->src.val;
1777 case 5: /* jmp far */
1778 rc = em_jmp_far(ctxt);
1787 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1789 u64 old = ctxt->dst.orig_val64;
1791 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1792 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1793 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1794 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1795 ctxt->eflags &= ~EFLG_ZF;
1797 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1798 (u32) ctxt->regs[VCPU_REGS_RBX];
1800 ctxt->eflags |= EFLG_ZF;
1802 return X86EMUL_CONTINUE;
1805 static int em_ret(struct x86_emulate_ctxt *ctxt)
1807 ctxt->dst.type = OP_REG;
1808 ctxt->dst.addr.reg = &ctxt->_eip;
1809 ctxt->dst.bytes = ctxt->op_bytes;
1810 return em_pop(ctxt);
1813 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1818 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1819 if (rc != X86EMUL_CONTINUE)
1821 if (ctxt->op_bytes == 4)
1822 ctxt->_eip = (u32)ctxt->_eip;
1823 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1824 if (rc != X86EMUL_CONTINUE)
1826 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1830 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
1832 /* Save real source value, then compare EAX against destination. */
1833 ctxt->src.orig_val = ctxt->src.val;
1834 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
1835 emulate_2op_SrcV(ctxt, "cmp");
1837 if (ctxt->eflags & EFLG_ZF) {
1838 /* Success: write back to memory. */
1839 ctxt->dst.val = ctxt->src.orig_val;
1841 /* Failure: write the value we saw to EAX. */
1842 ctxt->dst.type = OP_REG;
1843 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
1845 return X86EMUL_CONTINUE;
1848 static int em_lseg(struct x86_emulate_ctxt *ctxt)
1850 int seg = ctxt->src2.val;
1854 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1856 rc = load_segment_descriptor(ctxt, sel, seg);
1857 if (rc != X86EMUL_CONTINUE)
1860 ctxt->dst.val = ctxt->src.val;
1865 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1866 struct desc_struct *cs, struct desc_struct *ss)
1870 memset(cs, 0, sizeof(struct desc_struct));
1871 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1872 memset(ss, 0, sizeof(struct desc_struct));
1874 cs->l = 0; /* will be adjusted later */
1875 set_desc_base(cs, 0); /* flat segment */
1876 cs->g = 1; /* 4kb granularity */
1877 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1878 cs->type = 0x0b; /* Read, Execute, Accessed */
1880 cs->dpl = 0; /* will be adjusted later */
1884 set_desc_base(ss, 0); /* flat segment */
1885 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1886 ss->g = 1; /* 4kb granularity */
1888 ss->type = 0x03; /* Read/Write, Accessed */
1889 ss->d = 1; /* 32bit stack segment */
1894 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1896 struct x86_emulate_ops *ops = ctxt->ops;
1897 u32 eax, ebx, ecx, edx;
1900 * syscall should always be enabled in longmode - so only become
1901 * vendor specific (cpuid) if other modes are active...
1903 if (ctxt->mode == X86EMUL_MODE_PROT64)
1908 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1910 * Intel ("GenuineIntel")
1911 * remark: Intel CPUs only support "syscall" in 64bit
1912 * longmode. Also an 64bit guest with a
1913 * 32bit compat-app running will #UD !! While this
1914 * behaviour can be fixed (by emulating) into AMD
1915 * response - CPUs of AMD can't behave like Intel.
1917 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1918 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1919 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1922 /* AMD ("AuthenticAMD") */
1923 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1924 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1925 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1928 /* AMD ("AMDisbetter!") */
1929 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1930 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1931 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1935 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1939 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1941 struct x86_emulate_ops *ops = ctxt->ops;
1942 struct desc_struct cs, ss;
1947 /* syscall is not available in real mode */
1948 if (ctxt->mode == X86EMUL_MODE_REAL ||
1949 ctxt->mode == X86EMUL_MODE_VM86)
1950 return emulate_ud(ctxt);
1952 if (!(em_syscall_is_enabled(ctxt)))
1953 return emulate_ud(ctxt);
1955 ops->get_msr(ctxt, MSR_EFER, &efer);
1956 setup_syscalls_segments(ctxt, &cs, &ss);
1958 if (!(efer & EFER_SCE))
1959 return emulate_ud(ctxt);
1961 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1963 cs_sel = (u16)(msr_data & 0xfffc);
1964 ss_sel = (u16)(msr_data + 8);
1966 if (efer & EFER_LMA) {
1970 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1971 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1973 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1974 if (efer & EFER_LMA) {
1975 #ifdef CONFIG_X86_64
1976 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1979 ctxt->mode == X86EMUL_MODE_PROT64 ?
1980 MSR_LSTAR : MSR_CSTAR, &msr_data);
1981 ctxt->_eip = msr_data;
1983 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1984 ctxt->eflags &= ~(msr_data | EFLG_RF);
1988 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1989 ctxt->_eip = (u32)msr_data;
1991 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1994 return X86EMUL_CONTINUE;
1997 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1999 struct x86_emulate_ops *ops = ctxt->ops;
2000 struct desc_struct cs, ss;
2005 ops->get_msr(ctxt, MSR_EFER, &efer);
2006 /* inject #GP if in real mode */
2007 if (ctxt->mode == X86EMUL_MODE_REAL)
2008 return emulate_gp(ctxt, 0);
2010 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2011 * Therefore, we inject an #UD.
2013 if (ctxt->mode == X86EMUL_MODE_PROT64)
2014 return emulate_ud(ctxt);
2016 setup_syscalls_segments(ctxt, &cs, &ss);
2018 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2019 switch (ctxt->mode) {
2020 case X86EMUL_MODE_PROT32:
2021 if ((msr_data & 0xfffc) == 0x0)
2022 return emulate_gp(ctxt, 0);
2024 case X86EMUL_MODE_PROT64:
2025 if (msr_data == 0x0)
2026 return emulate_gp(ctxt, 0);
2030 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2031 cs_sel = (u16)msr_data;
2032 cs_sel &= ~SELECTOR_RPL_MASK;
2033 ss_sel = cs_sel + 8;
2034 ss_sel &= ~SELECTOR_RPL_MASK;
2035 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2040 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2041 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2043 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2044 ctxt->_eip = msr_data;
2046 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2047 ctxt->regs[VCPU_REGS_RSP] = msr_data;
2049 return X86EMUL_CONTINUE;
2052 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2054 struct x86_emulate_ops *ops = ctxt->ops;
2055 struct desc_struct cs, ss;
2058 u16 cs_sel = 0, ss_sel = 0;
2060 /* inject #GP if in real mode or Virtual 8086 mode */
2061 if (ctxt->mode == X86EMUL_MODE_REAL ||
2062 ctxt->mode == X86EMUL_MODE_VM86)
2063 return emulate_gp(ctxt, 0);
2065 setup_syscalls_segments(ctxt, &cs, &ss);
2067 if ((ctxt->rex_prefix & 0x8) != 0x0)
2068 usermode = X86EMUL_MODE_PROT64;
2070 usermode = X86EMUL_MODE_PROT32;
2074 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2076 case X86EMUL_MODE_PROT32:
2077 cs_sel = (u16)(msr_data + 16);
2078 if ((msr_data & 0xfffc) == 0x0)
2079 return emulate_gp(ctxt, 0);
2080 ss_sel = (u16)(msr_data + 24);
2082 case X86EMUL_MODE_PROT64:
2083 cs_sel = (u16)(msr_data + 32);
2084 if (msr_data == 0x0)
2085 return emulate_gp(ctxt, 0);
2086 ss_sel = cs_sel + 8;
2091 cs_sel |= SELECTOR_RPL_MASK;
2092 ss_sel |= SELECTOR_RPL_MASK;
2094 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2095 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2097 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
2098 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
2100 return X86EMUL_CONTINUE;
2103 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2106 if (ctxt->mode == X86EMUL_MODE_REAL)
2108 if (ctxt->mode == X86EMUL_MODE_VM86)
2110 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2111 return ctxt->ops->cpl(ctxt) > iopl;
2114 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2117 struct x86_emulate_ops *ops = ctxt->ops;
2118 struct desc_struct tr_seg;
2121 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2122 unsigned mask = (1 << len) - 1;
2125 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2128 if (desc_limit_scaled(&tr_seg) < 103)
2130 base = get_desc_base(&tr_seg);
2131 #ifdef CONFIG_X86_64
2132 base |= ((u64)base3) << 32;
2134 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2135 if (r != X86EMUL_CONTINUE)
2137 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2139 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2140 if (r != X86EMUL_CONTINUE)
2142 if ((perm >> bit_idx) & mask)
2147 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2153 if (emulator_bad_iopl(ctxt))
2154 if (!emulator_io_port_access_allowed(ctxt, port, len))
2157 ctxt->perm_ok = true;
2162 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2163 struct tss_segment_16 *tss)
2165 tss->ip = ctxt->_eip;
2166 tss->flag = ctxt->eflags;
2167 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2168 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2169 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2170 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2171 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2172 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2173 tss->si = ctxt->regs[VCPU_REGS_RSI];
2174 tss->di = ctxt->regs[VCPU_REGS_RDI];
2176 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2177 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2178 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2179 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2180 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2183 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2184 struct tss_segment_16 *tss)
2188 ctxt->_eip = tss->ip;
2189 ctxt->eflags = tss->flag | 2;
2190 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2191 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2192 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2193 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2194 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2195 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2196 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2197 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2200 * SDM says that segment selectors are loaded before segment
2203 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2204 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2205 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2206 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2207 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2210 * Now load segment descriptors. If fault happenes at this stage
2211 * it is handled in a context of new task
2213 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2214 if (ret != X86EMUL_CONTINUE)
2216 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2217 if (ret != X86EMUL_CONTINUE)
2219 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2220 if (ret != X86EMUL_CONTINUE)
2222 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2223 if (ret != X86EMUL_CONTINUE)
2225 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2226 if (ret != X86EMUL_CONTINUE)
2229 return X86EMUL_CONTINUE;
2232 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2233 u16 tss_selector, u16 old_tss_sel,
2234 ulong old_tss_base, struct desc_struct *new_desc)
2236 struct x86_emulate_ops *ops = ctxt->ops;
2237 struct tss_segment_16 tss_seg;
2239 u32 new_tss_base = get_desc_base(new_desc);
2241 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2243 if (ret != X86EMUL_CONTINUE)
2244 /* FIXME: need to provide precise fault address */
2247 save_state_to_tss16(ctxt, &tss_seg);
2249 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2251 if (ret != X86EMUL_CONTINUE)
2252 /* FIXME: need to provide precise fault address */
2255 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2257 if (ret != X86EMUL_CONTINUE)
2258 /* FIXME: need to provide precise fault address */
2261 if (old_tss_sel != 0xffff) {
2262 tss_seg.prev_task_link = old_tss_sel;
2264 ret = ops->write_std(ctxt, new_tss_base,
2265 &tss_seg.prev_task_link,
2266 sizeof tss_seg.prev_task_link,
2268 if (ret != X86EMUL_CONTINUE)
2269 /* FIXME: need to provide precise fault address */
2273 return load_state_from_tss16(ctxt, &tss_seg);
2276 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2277 struct tss_segment_32 *tss)
2279 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2280 tss->eip = ctxt->_eip;
2281 tss->eflags = ctxt->eflags;
2282 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2283 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2284 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2285 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2286 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2287 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2288 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2289 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2291 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2292 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2293 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2294 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2295 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2296 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2297 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2300 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2301 struct tss_segment_32 *tss)
2305 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2306 return emulate_gp(ctxt, 0);
2307 ctxt->_eip = tss->eip;
2308 ctxt->eflags = tss->eflags | 2;
2309 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2310 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2311 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2312 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2313 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2314 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2315 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2316 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2319 * SDM says that segment selectors are loaded before segment
2322 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2323 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2324 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2325 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2326 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2327 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2328 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2331 * Now load segment descriptors. If fault happenes at this stage
2332 * it is handled in a context of new task
2334 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2335 if (ret != X86EMUL_CONTINUE)
2337 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2338 if (ret != X86EMUL_CONTINUE)
2340 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2341 if (ret != X86EMUL_CONTINUE)
2343 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2344 if (ret != X86EMUL_CONTINUE)
2346 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2347 if (ret != X86EMUL_CONTINUE)
2349 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2350 if (ret != X86EMUL_CONTINUE)
2352 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2353 if (ret != X86EMUL_CONTINUE)
2356 return X86EMUL_CONTINUE;
2359 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2360 u16 tss_selector, u16 old_tss_sel,
2361 ulong old_tss_base, struct desc_struct *new_desc)
2363 struct x86_emulate_ops *ops = ctxt->ops;
2364 struct tss_segment_32 tss_seg;
2366 u32 new_tss_base = get_desc_base(new_desc);
2368 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2370 if (ret != X86EMUL_CONTINUE)
2371 /* FIXME: need to provide precise fault address */
2374 save_state_to_tss32(ctxt, &tss_seg);
2376 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2378 if (ret != X86EMUL_CONTINUE)
2379 /* FIXME: need to provide precise fault address */
2382 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2384 if (ret != X86EMUL_CONTINUE)
2385 /* FIXME: need to provide precise fault address */
2388 if (old_tss_sel != 0xffff) {
2389 tss_seg.prev_task_link = old_tss_sel;
2391 ret = ops->write_std(ctxt, new_tss_base,
2392 &tss_seg.prev_task_link,
2393 sizeof tss_seg.prev_task_link,
2395 if (ret != X86EMUL_CONTINUE)
2396 /* FIXME: need to provide precise fault address */
2400 return load_state_from_tss32(ctxt, &tss_seg);
2403 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2404 u16 tss_selector, int reason,
2405 bool has_error_code, u32 error_code)
2407 struct x86_emulate_ops *ops = ctxt->ops;
2408 struct desc_struct curr_tss_desc, next_tss_desc;
2410 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2411 ulong old_tss_base =
2412 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2415 /* FIXME: old_tss_base == ~0 ? */
2417 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2418 if (ret != X86EMUL_CONTINUE)
2420 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2421 if (ret != X86EMUL_CONTINUE)
2424 /* FIXME: check that next_tss_desc is tss */
2426 if (reason != TASK_SWITCH_IRET) {
2427 if ((tss_selector & 3) > next_tss_desc.dpl ||
2428 ops->cpl(ctxt) > next_tss_desc.dpl)
2429 return emulate_gp(ctxt, 0);
2432 desc_limit = desc_limit_scaled(&next_tss_desc);
2433 if (!next_tss_desc.p ||
2434 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2435 desc_limit < 0x2b)) {
2436 emulate_ts(ctxt, tss_selector & 0xfffc);
2437 return X86EMUL_PROPAGATE_FAULT;
2440 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2441 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2442 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2445 if (reason == TASK_SWITCH_IRET)
2446 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2448 /* set back link to prev task only if NT bit is set in eflags
2449 note that old_tss_sel is not used afetr this point */
2450 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2451 old_tss_sel = 0xffff;
2453 if (next_tss_desc.type & 8)
2454 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2455 old_tss_base, &next_tss_desc);
2457 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2458 old_tss_base, &next_tss_desc);
2459 if (ret != X86EMUL_CONTINUE)
2462 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2463 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2465 if (reason != TASK_SWITCH_IRET) {
2466 next_tss_desc.type |= (1 << 1); /* set busy flag */
2467 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2470 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2471 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2473 if (has_error_code) {
2474 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2475 ctxt->lock_prefix = 0;
2476 ctxt->src.val = (unsigned long) error_code;
2477 ret = em_push(ctxt);
2483 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2484 u16 tss_selector, int reason,
2485 bool has_error_code, u32 error_code)
2489 ctxt->_eip = ctxt->eip;
2490 ctxt->dst.type = OP_NONE;
2492 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2493 has_error_code, error_code);
2495 if (rc == X86EMUL_CONTINUE)
2496 ctxt->eip = ctxt->_eip;
2498 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2501 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2502 int reg, struct operand *op)
2504 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2506 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2507 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2508 op->addr.mem.seg = seg;
2511 static int em_das(struct x86_emulate_ctxt *ctxt)
2514 bool af, cf, old_cf;
2516 cf = ctxt->eflags & X86_EFLAGS_CF;
2522 af = ctxt->eflags & X86_EFLAGS_AF;
2523 if ((al & 0x0f) > 9 || af) {
2525 cf = old_cf | (al >= 250);
2530 if (old_al > 0x99 || old_cf) {
2536 /* Set PF, ZF, SF */
2537 ctxt->src.type = OP_IMM;
2539 ctxt->src.bytes = 1;
2540 emulate_2op_SrcV(ctxt, "or");
2541 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2543 ctxt->eflags |= X86_EFLAGS_CF;
2545 ctxt->eflags |= X86_EFLAGS_AF;
2546 return X86EMUL_CONTINUE;
2549 static int em_call(struct x86_emulate_ctxt *ctxt)
2551 long rel = ctxt->src.val;
2553 ctxt->src.val = (unsigned long)ctxt->_eip;
2555 return em_push(ctxt);
2558 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2564 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2565 old_eip = ctxt->_eip;
2567 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2568 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2569 return X86EMUL_CONTINUE;
2572 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2574 ctxt->src.val = old_cs;
2576 if (rc != X86EMUL_CONTINUE)
2579 ctxt->src.val = old_eip;
2580 return em_push(ctxt);
2583 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2587 ctxt->dst.type = OP_REG;
2588 ctxt->dst.addr.reg = &ctxt->_eip;
2589 ctxt->dst.bytes = ctxt->op_bytes;
2590 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2591 if (rc != X86EMUL_CONTINUE)
2593 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2594 return X86EMUL_CONTINUE;
2597 static int em_add(struct x86_emulate_ctxt *ctxt)
2599 emulate_2op_SrcV(ctxt, "add");
2600 return X86EMUL_CONTINUE;
2603 static int em_or(struct x86_emulate_ctxt *ctxt)
2605 emulate_2op_SrcV(ctxt, "or");
2606 return X86EMUL_CONTINUE;
2609 static int em_adc(struct x86_emulate_ctxt *ctxt)
2611 emulate_2op_SrcV(ctxt, "adc");
2612 return X86EMUL_CONTINUE;
2615 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2617 emulate_2op_SrcV(ctxt, "sbb");
2618 return X86EMUL_CONTINUE;
2621 static int em_and(struct x86_emulate_ctxt *ctxt)
2623 emulate_2op_SrcV(ctxt, "and");
2624 return X86EMUL_CONTINUE;
2627 static int em_sub(struct x86_emulate_ctxt *ctxt)
2629 emulate_2op_SrcV(ctxt, "sub");
2630 return X86EMUL_CONTINUE;
2633 static int em_xor(struct x86_emulate_ctxt *ctxt)
2635 emulate_2op_SrcV(ctxt, "xor");
2636 return X86EMUL_CONTINUE;
2639 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2641 emulate_2op_SrcV(ctxt, "cmp");
2642 /* Disable writeback. */
2643 ctxt->dst.type = OP_NONE;
2644 return X86EMUL_CONTINUE;
2647 static int em_test(struct x86_emulate_ctxt *ctxt)
2649 emulate_2op_SrcV(ctxt, "test");
2650 /* Disable writeback. */
2651 ctxt->dst.type = OP_NONE;
2652 return X86EMUL_CONTINUE;
2655 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2657 /* Write back the register source. */
2658 ctxt->src.val = ctxt->dst.val;
2659 write_register_operand(&ctxt->src);
2661 /* Write back the memory destination with implicit LOCK prefix. */
2662 ctxt->dst.val = ctxt->src.orig_val;
2663 ctxt->lock_prefix = 1;
2664 return X86EMUL_CONTINUE;
2667 static int em_imul(struct x86_emulate_ctxt *ctxt)
2669 emulate_2op_SrcV_nobyte(ctxt, "imul");
2670 return X86EMUL_CONTINUE;
2673 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2675 ctxt->dst.val = ctxt->src2.val;
2676 return em_imul(ctxt);
2679 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2681 ctxt->dst.type = OP_REG;
2682 ctxt->dst.bytes = ctxt->src.bytes;
2683 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2684 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2686 return X86EMUL_CONTINUE;
2689 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2693 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2694 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2695 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2696 return X86EMUL_CONTINUE;
2699 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2703 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
2704 return emulate_gp(ctxt, 0);
2705 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
2706 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
2707 return X86EMUL_CONTINUE;
2710 static int em_mov(struct x86_emulate_ctxt *ctxt)
2712 ctxt->dst.val = ctxt->src.val;
2713 return X86EMUL_CONTINUE;
2716 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2718 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2719 return emulate_gp(ctxt, 0);
2721 /* Disable writeback. */
2722 ctxt->dst.type = OP_NONE;
2723 return X86EMUL_CONTINUE;
2726 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2730 if (ctxt->mode == X86EMUL_MODE_PROT64)
2731 val = ctxt->src.val & ~0ULL;
2733 val = ctxt->src.val & ~0U;
2735 /* #UD condition is already handled. */
2736 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2737 return emulate_gp(ctxt, 0);
2739 /* Disable writeback. */
2740 ctxt->dst.type = OP_NONE;
2741 return X86EMUL_CONTINUE;
2744 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2748 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
2749 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
2750 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
2751 return emulate_gp(ctxt, 0);
2753 return X86EMUL_CONTINUE;
2756 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2760 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
2761 return emulate_gp(ctxt, 0);
2763 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
2764 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
2765 return X86EMUL_CONTINUE;
2768 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2770 if (ctxt->modrm_reg > VCPU_SREG_GS)
2771 return emulate_ud(ctxt);
2773 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2774 return X86EMUL_CONTINUE;
2777 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2779 u16 sel = ctxt->src.val;
2781 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2782 return emulate_ud(ctxt);
2784 if (ctxt->modrm_reg == VCPU_SREG_SS)
2785 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2787 /* Disable writeback. */
2788 ctxt->dst.type = OP_NONE;
2789 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2792 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2794 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2795 return X86EMUL_CONTINUE;
2798 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2803 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2804 if (rc == X86EMUL_CONTINUE)
2805 ctxt->ops->invlpg(ctxt, linear);
2806 /* Disable writeback. */
2807 ctxt->dst.type = OP_NONE;
2808 return X86EMUL_CONTINUE;
2811 static int em_clts(struct x86_emulate_ctxt *ctxt)
2815 cr0 = ctxt->ops->get_cr(ctxt, 0);
2817 ctxt->ops->set_cr(ctxt, 0, cr0);
2818 return X86EMUL_CONTINUE;
2821 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2825 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2826 return X86EMUL_UNHANDLEABLE;
2828 rc = ctxt->ops->fix_hypercall(ctxt);
2829 if (rc != X86EMUL_CONTINUE)
2832 /* Let the processor re-execute the fixed hypercall */
2833 ctxt->_eip = ctxt->eip;
2834 /* Disable writeback. */
2835 ctxt->dst.type = OP_NONE;
2836 return X86EMUL_CONTINUE;
2839 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2841 struct desc_ptr desc_ptr;
2844 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2845 &desc_ptr.size, &desc_ptr.address,
2847 if (rc != X86EMUL_CONTINUE)
2849 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2850 /* Disable writeback. */
2851 ctxt->dst.type = OP_NONE;
2852 return X86EMUL_CONTINUE;
2855 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2859 rc = ctxt->ops->fix_hypercall(ctxt);
2861 /* Disable writeback. */
2862 ctxt->dst.type = OP_NONE;
2866 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2868 struct desc_ptr desc_ptr;
2871 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2872 &desc_ptr.size, &desc_ptr.address,
2874 if (rc != X86EMUL_CONTINUE)
2876 ctxt->ops->set_idt(ctxt, &desc_ptr);
2877 /* Disable writeback. */
2878 ctxt->dst.type = OP_NONE;
2879 return X86EMUL_CONTINUE;
2882 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2884 ctxt->dst.bytes = 2;
2885 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2886 return X86EMUL_CONTINUE;
2889 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2891 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2892 | (ctxt->src.val & 0x0f));
2893 ctxt->dst.type = OP_NONE;
2894 return X86EMUL_CONTINUE;
2897 static int em_loop(struct x86_emulate_ctxt *ctxt)
2899 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2900 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2901 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2902 jmp_rel(ctxt, ctxt->src.val);
2904 return X86EMUL_CONTINUE;
2907 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2909 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2910 jmp_rel(ctxt, ctxt->src.val);
2912 return X86EMUL_CONTINUE;
2915 static int em_in(struct x86_emulate_ctxt *ctxt)
2917 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
2919 return X86EMUL_IO_NEEDED;
2921 return X86EMUL_CONTINUE;
2924 static int em_out(struct x86_emulate_ctxt *ctxt)
2926 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
2928 /* Disable writeback. */
2929 ctxt->dst.type = OP_NONE;
2930 return X86EMUL_CONTINUE;
2933 static int em_cli(struct x86_emulate_ctxt *ctxt)
2935 if (emulator_bad_iopl(ctxt))
2936 return emulate_gp(ctxt, 0);
2938 ctxt->eflags &= ~X86_EFLAGS_IF;
2939 return X86EMUL_CONTINUE;
2942 static int em_sti(struct x86_emulate_ctxt *ctxt)
2944 if (emulator_bad_iopl(ctxt))
2945 return emulate_gp(ctxt, 0);
2947 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2948 ctxt->eflags |= X86_EFLAGS_IF;
2949 return X86EMUL_CONTINUE;
2952 static int em_bt(struct x86_emulate_ctxt *ctxt)
2954 /* Disable writeback. */
2955 ctxt->dst.type = OP_NONE;
2956 /* only subword offset */
2957 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
2959 emulate_2op_SrcV_nobyte(ctxt, "bt");
2960 return X86EMUL_CONTINUE;
2963 static int em_bts(struct x86_emulate_ctxt *ctxt)
2965 emulate_2op_SrcV_nobyte(ctxt, "bts");
2966 return X86EMUL_CONTINUE;
2969 static int em_btr(struct x86_emulate_ctxt *ctxt)
2971 emulate_2op_SrcV_nobyte(ctxt, "btr");
2972 return X86EMUL_CONTINUE;
2975 static int em_btc(struct x86_emulate_ctxt *ctxt)
2977 emulate_2op_SrcV_nobyte(ctxt, "btc");
2978 return X86EMUL_CONTINUE;
2981 static int em_bsf(struct x86_emulate_ctxt *ctxt)
2985 __asm__ ("bsf %2, %0; setz %1"
2986 : "=r"(ctxt->dst.val), "=q"(zf)
2987 : "r"(ctxt->src.val));
2989 ctxt->eflags &= ~X86_EFLAGS_ZF;
2991 ctxt->eflags |= X86_EFLAGS_ZF;
2992 /* Disable writeback. */
2993 ctxt->dst.type = OP_NONE;
2995 return X86EMUL_CONTINUE;
2998 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3002 __asm__ ("bsr %2, %0; setz %1"
3003 : "=r"(ctxt->dst.val), "=q"(zf)
3004 : "r"(ctxt->src.val));
3006 ctxt->eflags &= ~X86_EFLAGS_ZF;
3008 ctxt->eflags |= X86_EFLAGS_ZF;
3009 /* Disable writeback. */
3010 ctxt->dst.type = OP_NONE;
3012 return X86EMUL_CONTINUE;
3015 static bool valid_cr(int nr)
3027 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3029 if (!valid_cr(ctxt->modrm_reg))
3030 return emulate_ud(ctxt);
3032 return X86EMUL_CONTINUE;
3035 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3037 u64 new_val = ctxt->src.val64;
3038 int cr = ctxt->modrm_reg;
3041 static u64 cr_reserved_bits[] = {
3042 0xffffffff00000000ULL,
3043 0, 0, 0, /* CR3 checked later */
3050 return emulate_ud(ctxt);
3052 if (new_val & cr_reserved_bits[cr])
3053 return emulate_gp(ctxt, 0);
3058 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3059 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3060 return emulate_gp(ctxt, 0);
3062 cr4 = ctxt->ops->get_cr(ctxt, 4);
3063 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3065 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3066 !(cr4 & X86_CR4_PAE))
3067 return emulate_gp(ctxt, 0);
3074 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3075 if (efer & EFER_LMA)
3076 rsvd = CR3_L_MODE_RESERVED_BITS;
3077 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3078 rsvd = CR3_PAE_RESERVED_BITS;
3079 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3080 rsvd = CR3_NONPAE_RESERVED_BITS;
3083 return emulate_gp(ctxt, 0);
3088 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3090 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3091 return emulate_gp(ctxt, 0);
3097 return X86EMUL_CONTINUE;
3100 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3104 ctxt->ops->get_dr(ctxt, 7, &dr7);
3106 /* Check if DR7.Global_Enable is set */
3107 return dr7 & (1 << 13);
3110 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3112 int dr = ctxt->modrm_reg;
3116 return emulate_ud(ctxt);
3118 cr4 = ctxt->ops->get_cr(ctxt, 4);
3119 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3120 return emulate_ud(ctxt);
3122 if (check_dr7_gd(ctxt))
3123 return emulate_db(ctxt);
3125 return X86EMUL_CONTINUE;
3128 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3130 u64 new_val = ctxt->src.val64;
3131 int dr = ctxt->modrm_reg;
3133 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3134 return emulate_gp(ctxt, 0);
3136 return check_dr_read(ctxt);
3139 static int check_svme(struct x86_emulate_ctxt *ctxt)
3143 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3145 if (!(efer & EFER_SVME))
3146 return emulate_ud(ctxt);
3148 return X86EMUL_CONTINUE;
3151 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3153 u64 rax = ctxt->regs[VCPU_REGS_RAX];
3155 /* Valid physical address? */
3156 if (rax & 0xffff000000000000ULL)
3157 return emulate_gp(ctxt, 0);
3159 return check_svme(ctxt);
3162 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3164 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3166 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3167 return emulate_ud(ctxt);
3169 return X86EMUL_CONTINUE;
3172 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3174 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3175 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
3177 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3179 return emulate_gp(ctxt, 0);
3181 return X86EMUL_CONTINUE;
3184 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3186 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3187 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3188 return emulate_gp(ctxt, 0);
3190 return X86EMUL_CONTINUE;
3193 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3195 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3196 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3197 return emulate_gp(ctxt, 0);
3199 return X86EMUL_CONTINUE;
3202 #define D(_y) { .flags = (_y) }
3203 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3204 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3205 .check_perm = (_p) }
3207 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3208 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3209 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3210 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3211 #define II(_f, _e, _i) \
3212 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3213 #define IIP(_f, _e, _i, _p) \
3214 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3215 .check_perm = (_p) }
3216 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3218 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3219 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3220 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3221 #define I2bvIP(_f, _e, _i, _p) \
3222 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3224 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3225 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3226 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3228 static struct opcode group7_rm1[] = {
3229 DI(SrcNone | ModRM | Priv, monitor),
3230 DI(SrcNone | ModRM | Priv, mwait),
3234 static struct opcode group7_rm3[] = {
3235 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3236 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3237 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3238 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3239 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3240 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3241 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3242 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3245 static struct opcode group7_rm7[] = {
3247 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3251 static struct opcode group1[] = {
3253 I(Lock | PageTable, em_or),
3256 I(Lock | PageTable, em_and),
3262 static struct opcode group1A[] = {
3263 I(DstMem | SrcNone | ModRM | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3266 static struct opcode group3[] = {
3267 I(DstMem | SrcImm | ModRM, em_test),
3268 I(DstMem | SrcImm | ModRM, em_test),
3269 I(DstMem | SrcNone | ModRM | Lock, em_not),
3270 I(DstMem | SrcNone | ModRM | Lock, em_neg),
3271 I(SrcMem | ModRM, em_mul_ex),
3272 I(SrcMem | ModRM, em_imul_ex),
3273 I(SrcMem | ModRM, em_div_ex),
3274 I(SrcMem | ModRM, em_idiv_ex),
3277 static struct opcode group4[] = {
3278 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3279 I(ByteOp | DstMem | SrcNone | ModRM | Lock, em_grp45),
3283 static struct opcode group5[] = {
3284 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3285 I(DstMem | SrcNone | ModRM | Lock, em_grp45),
3286 I(SrcMem | ModRM | Stack, em_grp45),
3287 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3288 I(SrcMem | ModRM | Stack, em_grp45),
3289 I(SrcMemFAddr | ModRM | ImplicitOps, em_grp45),
3290 I(SrcMem | ModRM | Stack, em_grp45), N,
3293 static struct opcode group6[] = {
3294 DI(ModRM | Prot, sldt),
3295 DI(ModRM | Prot, str),
3296 DI(ModRM | Prot | Priv, lldt),
3297 DI(ModRM | Prot | Priv, ltr),
3301 static struct group_dual group7 = { {
3302 DI(ModRM | Mov | DstMem | Priv, sgdt),
3303 DI(ModRM | Mov | DstMem | Priv, sidt),
3304 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3305 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3306 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3307 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3308 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3310 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3312 N, EXT(0, group7_rm3),
3313 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3314 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3317 static struct opcode group8[] = {
3319 I(DstMem | SrcImmByte | ModRM, em_bt),
3320 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_bts),
3321 I(DstMem | SrcImmByte | ModRM | Lock, em_btr),
3322 I(DstMem | SrcImmByte | ModRM | Lock | PageTable, em_btc),
3325 static struct group_dual group9 = { {
3326 N, I(DstMem64 | ModRM | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3328 N, N, N, N, N, N, N, N,
3331 static struct opcode group11[] = {
3332 I(DstMem | SrcImm | ModRM | Mov | PageTable, em_mov),
3336 static struct gprefix pfx_0f_6f_0f_7f = {
3337 N, N, N, I(Sse, em_movdqu),
3340 static struct opcode opcode_table[256] = {
3342 I6ALU(Lock, em_add),
3343 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3344 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3346 I6ALU(Lock | PageTable, em_or),
3347 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3350 I6ALU(Lock, em_adc),
3351 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3352 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3354 I6ALU(Lock, em_sbb),
3355 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3356 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3358 I6ALU(Lock | PageTable, em_and), N, N,
3360 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3362 I6ALU(Lock, em_xor), N, N,
3364 I6ALU(0, em_cmp), N, N,
3368 X8(I(SrcReg | Stack, em_push)),
3370 X8(I(DstReg | Stack, em_pop)),
3372 I(ImplicitOps | Stack | No64, em_pusha),
3373 I(ImplicitOps | Stack | No64, em_popa),
3374 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3377 I(SrcImm | Mov | Stack, em_push),
3378 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3379 I(SrcImmByte | Mov | Stack, em_push),
3380 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3381 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
3382 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3386 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3387 G(DstMem | SrcImm | ModRM | Group, group1),
3388 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3389 G(DstMem | SrcImmByte | ModRM | Group, group1),
3390 I2bv(DstMem | SrcReg | ModRM, em_test),
3391 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3393 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3394 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3395 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3396 D(ModRM | SrcMem | NoAccess | DstReg),
3397 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3400 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3402 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3403 I(SrcImmFAddr | No64, em_call_far), N,
3404 II(ImplicitOps | Stack, em_pushf, pushf),
3405 II(ImplicitOps | Stack, em_popf, popf), N, N,
3407 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3408 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3409 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3410 I2bv(SrcSI | DstDI | String, em_cmp),
3412 I2bv(DstAcc | SrcImm, em_test),
3413 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3414 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3415 I2bv(SrcAcc | DstDI | String, em_cmp),
3417 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3419 X8(I(DstReg | SrcImm | Mov, em_mov)),
3421 D2bv(DstMem | SrcImmByte | ModRM),
3422 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3423 I(ImplicitOps | Stack, em_ret),
3424 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3425 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3426 G(ByteOp, group11), G(0, group11),
3428 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3429 D(ImplicitOps), DI(SrcImmByte, intn),
3430 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3432 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3435 N, N, N, N, N, N, N, N,
3437 X3(I(SrcImmByte, em_loop)),
3438 I(SrcImmByte, em_jcxz),
3439 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3440 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3442 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3443 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3444 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3445 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3447 N, DI(ImplicitOps, icebp), N, N,
3448 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3449 G(ByteOp, group3), G(0, group3),
3451 D(ImplicitOps), D(ImplicitOps),
3452 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3453 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3456 static struct opcode twobyte_table[256] = {
3458 G(0, group6), GD(0, &group7), N, N,
3459 N, I(ImplicitOps | VendorSpecific, em_syscall),
3460 II(ImplicitOps | Priv, em_clts, clts), N,
3461 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3462 N, D(ImplicitOps | ModRM), N, N,
3464 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3466 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3467 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3468 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3469 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3471 N, N, N, N, N, N, N, N,
3473 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3474 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3475 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3476 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3477 I(ImplicitOps | VendorSpecific, em_sysenter),
3478 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3480 N, N, N, N, N, N, N, N,
3482 X16(D(DstReg | SrcMem | ModRM | Mov)),
3484 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3489 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3494 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3498 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3500 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3501 DI(ImplicitOps, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3502 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3503 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3505 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3506 DI(ImplicitOps, rsm),
3507 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3508 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3509 D(DstMem | SrcReg | Src2CL | ModRM),
3510 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3512 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3513 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3514 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3515 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3516 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3517 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3521 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3522 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3523 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3525 D2bv(DstMem | SrcReg | ModRM | Lock),
3526 N, D(DstMem | SrcReg | ModRM | Mov),
3527 N, N, N, GD(0, &group9),
3528 N, N, N, N, N, N, N, N,
3530 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3532 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3534 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3551 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3555 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3561 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3562 unsigned size, bool sign_extension)
3564 int rc = X86EMUL_CONTINUE;
3568 op->addr.mem.ea = ctxt->_eip;
3569 /* NB. Immediates are sign-extended as necessary. */
3570 switch (op->bytes) {
3572 op->val = insn_fetch(s8, ctxt);
3575 op->val = insn_fetch(s16, ctxt);
3578 op->val = insn_fetch(s32, ctxt);
3581 if (!sign_extension) {
3582 switch (op->bytes) {
3590 op->val &= 0xffffffff;
3598 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3601 int rc = X86EMUL_CONTINUE;
3605 decode_register_operand(ctxt, op,
3607 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3610 rc = decode_imm(ctxt, op, 1, false);
3613 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3617 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3618 fetch_bit_operand(ctxt);
3619 op->orig_val = op->val;
3622 ctxt->memop.bytes = 8;
3626 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3627 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3628 fetch_register_operand(op);
3629 op->orig_val = op->val;
3633 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3635 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3636 op->addr.mem.seg = VCPU_SREG_ES;
3642 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3643 fetch_register_operand(op);
3647 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3650 rc = decode_imm(ctxt, op, 1, true);
3657 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
3660 ctxt->memop.bytes = 2;
3663 ctxt->memop.bytes = 4;
3666 rc = decode_imm(ctxt, op, 2, false);
3669 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
3673 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3675 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3676 op->addr.mem.seg = seg_override(ctxt);
3681 op->addr.mem.ea = ctxt->_eip;
3682 op->bytes = ctxt->op_bytes + 2;
3683 insn_fetch_arr(op->valptr, op->bytes, ctxt);
3686 ctxt->memop.bytes = ctxt->op_bytes + 2;
3689 op->val = VCPU_SREG_ES;
3692 op->val = VCPU_SREG_CS;
3695 op->val = VCPU_SREG_SS;
3698 op->val = VCPU_SREG_DS;
3701 op->val = VCPU_SREG_FS;
3704 op->val = VCPU_SREG_GS;
3707 /* Special instructions do their own operand decoding. */
3709 op->type = OP_NONE; /* Disable writeback. */
3717 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3719 int rc = X86EMUL_CONTINUE;
3720 int mode = ctxt->mode;
3721 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3722 bool op_prefix = false;
3723 struct opcode opcode;
3725 ctxt->memop.type = OP_NONE;
3726 ctxt->memopp = NULL;
3727 ctxt->_eip = ctxt->eip;
3728 ctxt->fetch.start = ctxt->_eip;
3729 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3731 memcpy(ctxt->fetch.data, insn, insn_len);
3734 case X86EMUL_MODE_REAL:
3735 case X86EMUL_MODE_VM86:
3736 case X86EMUL_MODE_PROT16:
3737 def_op_bytes = def_ad_bytes = 2;
3739 case X86EMUL_MODE_PROT32:
3740 def_op_bytes = def_ad_bytes = 4;
3742 #ifdef CONFIG_X86_64
3743 case X86EMUL_MODE_PROT64:
3749 return EMULATION_FAILED;
3752 ctxt->op_bytes = def_op_bytes;
3753 ctxt->ad_bytes = def_ad_bytes;
3755 /* Legacy prefixes. */
3757 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3758 case 0x66: /* operand-size override */
3760 /* switch between 2/4 bytes */
3761 ctxt->op_bytes = def_op_bytes ^ 6;
3763 case 0x67: /* address-size override */
3764 if (mode == X86EMUL_MODE_PROT64)
3765 /* switch between 4/8 bytes */
3766 ctxt->ad_bytes = def_ad_bytes ^ 12;
3768 /* switch between 2/4 bytes */
3769 ctxt->ad_bytes = def_ad_bytes ^ 6;
3771 case 0x26: /* ES override */
3772 case 0x2e: /* CS override */
3773 case 0x36: /* SS override */
3774 case 0x3e: /* DS override */
3775 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3777 case 0x64: /* FS override */
3778 case 0x65: /* GS override */
3779 set_seg_override(ctxt, ctxt->b & 7);
3781 case 0x40 ... 0x4f: /* REX */
3782 if (mode != X86EMUL_MODE_PROT64)
3784 ctxt->rex_prefix = ctxt->b;
3786 case 0xf0: /* LOCK */
3787 ctxt->lock_prefix = 1;
3789 case 0xf2: /* REPNE/REPNZ */
3790 case 0xf3: /* REP/REPE/REPZ */
3791 ctxt->rep_prefix = ctxt->b;
3797 /* Any legacy prefix after a REX prefix nullifies its effect. */
3799 ctxt->rex_prefix = 0;
3805 if (ctxt->rex_prefix & 8)
3806 ctxt->op_bytes = 8; /* REX.W */
3808 /* Opcode byte(s). */
3809 opcode = opcode_table[ctxt->b];
3810 /* Two-byte opcode? */
3811 if (ctxt->b == 0x0f) {
3813 ctxt->b = insn_fetch(u8, ctxt);
3814 opcode = twobyte_table[ctxt->b];
3816 ctxt->d = opcode.flags;
3818 while (ctxt->d & GroupMask) {
3819 switch (ctxt->d & GroupMask) {
3821 ctxt->modrm = insn_fetch(u8, ctxt);
3823 goffset = (ctxt->modrm >> 3) & 7;
3824 opcode = opcode.u.group[goffset];
3827 ctxt->modrm = insn_fetch(u8, ctxt);
3829 goffset = (ctxt->modrm >> 3) & 7;
3830 if ((ctxt->modrm >> 6) == 3)
3831 opcode = opcode.u.gdual->mod3[goffset];
3833 opcode = opcode.u.gdual->mod012[goffset];
3836 goffset = ctxt->modrm & 7;
3837 opcode = opcode.u.group[goffset];
3840 if (ctxt->rep_prefix && op_prefix)
3841 return EMULATION_FAILED;
3842 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3843 switch (simd_prefix) {
3844 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3845 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3846 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3847 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3851 return EMULATION_FAILED;
3854 ctxt->d &= ~(u64)GroupMask;
3855 ctxt->d |= opcode.flags;
3858 ctxt->execute = opcode.u.execute;
3859 ctxt->check_perm = opcode.check_perm;
3860 ctxt->intercept = opcode.intercept;
3863 if (ctxt->d == 0 || (ctxt->d & Undefined))
3864 return EMULATION_FAILED;
3866 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3867 return EMULATION_FAILED;
3869 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3872 if (ctxt->d & Op3264) {
3873 if (mode == X86EMUL_MODE_PROT64)
3880 ctxt->op_bytes = 16;
3882 /* ModRM and SIB bytes. */
3883 if (ctxt->d & ModRM) {
3884 rc = decode_modrm(ctxt, &ctxt->memop);
3885 if (!ctxt->has_seg_override)
3886 set_seg_override(ctxt, ctxt->modrm_seg);
3887 } else if (ctxt->d & MemAbs)
3888 rc = decode_abs(ctxt, &ctxt->memop);
3889 if (rc != X86EMUL_CONTINUE)
3892 if (!ctxt->has_seg_override)
3893 set_seg_override(ctxt, VCPU_SREG_DS);
3895 ctxt->memop.addr.mem.seg = seg_override(ctxt);
3897 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
3898 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
3901 * Decode and fetch the source operand: register, memory
3904 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
3905 if (rc != X86EMUL_CONTINUE)
3909 * Decode and fetch the second source operand: register, memory
3912 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
3913 if (rc != X86EMUL_CONTINUE)
3916 /* Decode and fetch the destination operand: register or memory. */
3917 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
3920 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
3921 ctxt->memopp->addr.mem.ea += ctxt->_eip;
3923 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3926 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
3928 return ctxt->d & PageTable;
3931 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3933 /* The second termination condition only applies for REPE
3934 * and REPNE. Test if the repeat string operation prefix is
3935 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3936 * corresponding termination condition according to:
3937 * - if REPE/REPZ and ZF = 0 then done
3938 * - if REPNE/REPNZ and ZF = 1 then done
3940 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3941 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3942 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3943 ((ctxt->eflags & EFLG_ZF) == 0))
3944 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3945 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3951 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3953 struct x86_emulate_ops *ops = ctxt->ops;
3954 int rc = X86EMUL_CONTINUE;
3955 int saved_dst_type = ctxt->dst.type;
3957 ctxt->mem_read.pos = 0;
3959 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3960 rc = emulate_ud(ctxt);
3964 /* LOCK prefix is allowed only with some instructions */
3965 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3966 rc = emulate_ud(ctxt);
3970 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3971 rc = emulate_ud(ctxt);
3976 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3977 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3978 rc = emulate_ud(ctxt);
3982 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3983 rc = emulate_nm(ctxt);
3987 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3988 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3989 X86_ICPT_PRE_EXCEPT);
3990 if (rc != X86EMUL_CONTINUE)
3994 /* Privileged instruction can be executed only in CPL=0 */
3995 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3996 rc = emulate_gp(ctxt, 0);
4000 /* Instruction can only be executed in protected mode */
4001 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
4002 rc = emulate_ud(ctxt);
4006 /* Do instruction specific permission checks */
4007 if (ctxt->check_perm) {
4008 rc = ctxt->check_perm(ctxt);
4009 if (rc != X86EMUL_CONTINUE)
4013 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4014 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4015 X86_ICPT_POST_EXCEPT);
4016 if (rc != X86EMUL_CONTINUE)
4020 if (ctxt->rep_prefix && (ctxt->d & String)) {
4021 /* All REP prefixes have the same first termination condition */
4022 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
4023 ctxt->eip = ctxt->_eip;
4028 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4029 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4030 ctxt->src.valptr, ctxt->src.bytes);
4031 if (rc != X86EMUL_CONTINUE)
4033 ctxt->src.orig_val64 = ctxt->src.val64;
4036 if (ctxt->src2.type == OP_MEM) {
4037 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4038 &ctxt->src2.val, ctxt->src2.bytes);
4039 if (rc != X86EMUL_CONTINUE)
4043 if ((ctxt->d & DstMask) == ImplicitOps)
4047 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4048 /* optimisation - avoid slow emulated read if Mov */
4049 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4050 &ctxt->dst.val, ctxt->dst.bytes);
4051 if (rc != X86EMUL_CONTINUE)
4054 ctxt->dst.orig_val = ctxt->dst.val;
4058 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4059 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4060 X86_ICPT_POST_MEMACCESS);
4061 if (rc != X86EMUL_CONTINUE)
4065 if (ctxt->execute) {
4066 rc = ctxt->execute(ctxt);
4067 if (rc != X86EMUL_CONTINUE)
4076 case 0x40 ... 0x47: /* inc r16/r32 */
4077 emulate_1op(ctxt, "inc");
4079 case 0x48 ... 0x4f: /* dec r16/r32 */
4080 emulate_1op(ctxt, "dec");
4082 case 0x63: /* movsxd */
4083 if (ctxt->mode != X86EMUL_MODE_PROT64)
4084 goto cannot_emulate;
4085 ctxt->dst.val = (s32) ctxt->src.val;
4087 case 0x70 ... 0x7f: /* jcc (short) */
4088 if (test_cc(ctxt->b, ctxt->eflags))
4089 jmp_rel(ctxt, ctxt->src.val);
4091 case 0x8d: /* lea r16/r32, m */
4092 ctxt->dst.val = ctxt->src.addr.mem.ea;
4094 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4095 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
4099 case 0x98: /* cbw/cwde/cdqe */
4100 switch (ctxt->op_bytes) {
4101 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4102 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4103 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4109 case 0xcc: /* int3 */
4110 rc = emulate_int(ctxt, 3);
4112 case 0xcd: /* int n */
4113 rc = emulate_int(ctxt, ctxt->src.val);
4115 case 0xce: /* into */
4116 if (ctxt->eflags & EFLG_OF)
4117 rc = emulate_int(ctxt, 4);
4119 case 0xd0 ... 0xd1: /* Grp2 */
4122 case 0xd2 ... 0xd3: /* Grp2 */
4123 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
4126 case 0xe9: /* jmp rel */
4127 case 0xeb: /* jmp rel short */
4128 jmp_rel(ctxt, ctxt->src.val);
4129 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4131 case 0xf4: /* hlt */
4132 ctxt->ops->halt(ctxt);
4134 case 0xf5: /* cmc */
4135 /* complement carry flag from eflags reg */
4136 ctxt->eflags ^= EFLG_CF;
4138 case 0xf8: /* clc */
4139 ctxt->eflags &= ~EFLG_CF;
4141 case 0xf9: /* stc */
4142 ctxt->eflags |= EFLG_CF;
4144 case 0xfc: /* cld */
4145 ctxt->eflags &= ~EFLG_DF;
4147 case 0xfd: /* std */
4148 ctxt->eflags |= EFLG_DF;
4151 goto cannot_emulate;
4154 if (rc != X86EMUL_CONTINUE)
4158 rc = writeback(ctxt);
4159 if (rc != X86EMUL_CONTINUE)
4163 * restore dst type in case the decoding will be reused
4164 * (happens for string instruction )
4166 ctxt->dst.type = saved_dst_type;
4168 if ((ctxt->d & SrcMask) == SrcSI)
4169 string_addr_inc(ctxt, seg_override(ctxt),
4170 VCPU_REGS_RSI, &ctxt->src);
4172 if ((ctxt->d & DstMask) == DstDI)
4173 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4176 if (ctxt->rep_prefix && (ctxt->d & String)) {
4177 struct read_cache *r = &ctxt->io_read;
4178 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
4180 if (!string_insn_completed(ctxt)) {
4182 * Re-enter guest when pio read ahead buffer is empty
4183 * or, if it is not used, after each 1024 iteration.
4185 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
4186 (r->end == 0 || r->end != r->pos)) {
4188 * Reset read cache. Usually happens before
4189 * decode, but since instruction is restarted
4190 * we have to do it here.
4192 ctxt->mem_read.end = 0;
4193 return EMULATION_RESTART;
4195 goto done; /* skip rip writeback */
4199 ctxt->eip = ctxt->_eip;
4202 if (rc == X86EMUL_PROPAGATE_FAULT)
4203 ctxt->have_exception = true;
4204 if (rc == X86EMUL_INTERCEPTED)
4205 return EMULATION_INTERCEPTED;
4207 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4211 case 0x09: /* wbinvd */
4212 (ctxt->ops->wbinvd)(ctxt);
4214 case 0x08: /* invd */
4215 case 0x0d: /* GrpP (prefetch) */
4216 case 0x18: /* Grp16 (prefetch/nop) */
4218 case 0x20: /* mov cr, reg */
4219 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4221 case 0x21: /* mov from dr to reg */
4222 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4224 case 0x40 ... 0x4f: /* cmov */
4225 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4226 if (!test_cc(ctxt->b, ctxt->eflags))
4227 ctxt->dst.type = OP_NONE; /* no writeback */
4229 case 0x80 ... 0x8f: /* jnz rel, etc*/
4230 if (test_cc(ctxt->b, ctxt->eflags))
4231 jmp_rel(ctxt, ctxt->src.val);
4233 case 0x90 ... 0x9f: /* setcc r/m8 */
4234 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4236 case 0xa4: /* shld imm8, r, r/m */
4237 case 0xa5: /* shld cl, r, r/m */
4238 emulate_2op_cl(ctxt, "shld");
4240 case 0xac: /* shrd imm8, r, r/m */
4241 case 0xad: /* shrd cl, r, r/m */
4242 emulate_2op_cl(ctxt, "shrd");
4244 case 0xae: /* clflush */
4246 case 0xb6 ... 0xb7: /* movzx */
4247 ctxt->dst.bytes = ctxt->op_bytes;
4248 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4249 : (u16) ctxt->src.val;
4251 case 0xbe ... 0xbf: /* movsx */
4252 ctxt->dst.bytes = ctxt->op_bytes;
4253 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4254 (s16) ctxt->src.val;
4256 case 0xc0 ... 0xc1: /* xadd */
4257 emulate_2op_SrcV(ctxt, "add");
4258 /* Write back the register source. */
4259 ctxt->src.val = ctxt->dst.orig_val;
4260 write_register_operand(&ctxt->src);
4262 case 0xc3: /* movnti */
4263 ctxt->dst.bytes = ctxt->op_bytes;
4264 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4265 (u64) ctxt->src.val;
4268 goto cannot_emulate;
4271 if (rc != X86EMUL_CONTINUE)
4277 return EMULATION_FAILED;