1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<16) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<17) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<17) /* Register operand. */
53 #define DstMem (3<<17) /* Memory operand. */
54 #define DstAcc (4<<17) /* Destination Accumulator */
55 #define DstDI (5<<17) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<17) /* 64bit memory operand */
57 #define DstMask (7<<17)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0x0f /* Group number stored in bits 0:3 */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x) X2(x), x
100 #define X4(x) X2(x), X2(x)
101 #define X5(x) X4(x), x
102 #define X6(x) X4(x), X2(x)
103 #define X7(x) X4(x), X3(x)
104 #define X8(x) X4(x), X4(x)
105 #define X16(x) X8(x), X8(x)
108 Group1, Group1A, Group3, Group4, Group5, Group7, Group8, Group9,
115 #define D(_y) { .flags = (_y) }
118 static struct opcode opcode_table[256] = {
120 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
121 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
122 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
123 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
125 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
126 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
127 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
128 D(ImplicitOps | Stack | No64), N,
130 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
131 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
132 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
133 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
135 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
136 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
137 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
138 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
140 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
141 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
142 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
144 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
145 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
146 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
148 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
149 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
150 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
152 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
153 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
154 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
159 X8(D(SrcReg | Stack)),
161 X8(D(DstReg | Stack)),
163 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
164 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
167 D(SrcImm | Mov | Stack), N, D(SrcImmByte | Mov | Stack), N,
168 D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
169 D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
173 D(ByteOp | DstMem | SrcImm | ModRM | Group | Group1),
174 D(DstMem | SrcImm | ModRM | Group | Group1),
175 D(ByteOp | DstMem | SrcImm | ModRM | No64 | Group | Group1),
176 D(DstMem | SrcImmByte | ModRM | Group | Group1),
177 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
178 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
180 D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
181 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
182 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | DstReg),
183 D(ImplicitOps | SrcMem16 | ModRM), D(Group | Group1A),
185 D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg),
187 N, N, D(SrcImmFAddr | No64), N,
188 D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
190 D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
191 D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
192 D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
193 D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
195 D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm), D(ByteOp | DstDI | Mov | String), D(DstDI | Mov | String),
196 D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
197 D(ByteOp | DstDI | String), D(DstDI | String),
199 X8(D(ByteOp | DstReg | SrcImm | Mov)),
201 X8(D(DstReg | SrcImm | Mov)),
203 D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
204 N, D(ImplicitOps | Stack), N, N,
205 D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
207 N, N, N, D(ImplicitOps | Stack),
208 D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
210 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
211 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
214 N, N, N, N, N, N, N, N,
217 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
218 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
220 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
221 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
222 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
223 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
226 D(ImplicitOps | Priv), D(ImplicitOps), D(ByteOp | Group | Group3), D(Group | Group3),
228 D(ImplicitOps), N, D(ImplicitOps), D(ImplicitOps),
229 D(ImplicitOps), D(ImplicitOps), D(Group | Group4), D(Group | Group5),
232 static struct opcode twobyte_table[256] = {
234 N, D(Group | GroupDual | Group7), N, N,
235 N, D(ImplicitOps), D(ImplicitOps | Priv), N,
236 D(ImplicitOps | Priv), D(ImplicitOps | Priv), N, N,
237 N, D(ImplicitOps | ModRM), N, N,
239 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
241 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
242 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
244 N, N, N, N, N, N, N, N,
246 D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
247 D(ImplicitOps), D(ImplicitOps | Priv), N, N,
248 N, N, N, N, N, N, N, N,
250 X16(D(DstReg | SrcMem | ModRM | Mov)),
252 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
254 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
256 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
260 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
262 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
263 N, D(DstMem | SrcReg | ModRM | BitOp),
264 D(DstMem | SrcReg | Src2ImmByte | ModRM),
265 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
267 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
268 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
269 D(DstMem | SrcReg | Src2ImmByte | ModRM),
270 D(DstMem | SrcReg | Src2CL | ModRM),
273 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
274 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
275 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
276 D(DstReg | SrcMem16 | ModRM | Mov),
279 D(Group | Group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
280 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
281 D(DstReg | SrcMem16 | ModRM | Mov),
283 N, N, N, D(DstMem | SrcReg | ModRM | Mov),
284 N, N, N, D(Group | GroupDual | Group9),
285 N, N, N, N, N, N, N, N,
287 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
289 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
291 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
294 static struct opcode group_table[] = {
298 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
300 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
301 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
304 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
307 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
308 D(SrcMem | ModRM | Stack), N,
309 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
310 D(SrcMem | ModRM | Stack), N,
312 N, N, D(ModRM | SrcMem | Priv), D(ModRM | SrcMem | Priv),
313 D(SrcNone | ModRM | DstMem | Mov), N,
314 D(SrcMem16 | ModRM | Mov | Priv), D(SrcMem | ModRM | ByteOp | Priv),
317 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
318 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
320 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
323 static struct opcode group2_table[] = {
325 D(SrcNone | ModRM | Priv), N, N, D(SrcNone | ModRM | Priv),
326 D(SrcNone | ModRM | DstMem | Mov), N,
327 D(SrcMem16 | ModRM | Mov | Priv), N,
329 N, N, N, N, N, N, N, N,
335 /* EFLAGS bit definitions. */
336 #define EFLG_ID (1<<21)
337 #define EFLG_VIP (1<<20)
338 #define EFLG_VIF (1<<19)
339 #define EFLG_AC (1<<18)
340 #define EFLG_VM (1<<17)
341 #define EFLG_RF (1<<16)
342 #define EFLG_IOPL (3<<12)
343 #define EFLG_NT (1<<14)
344 #define EFLG_OF (1<<11)
345 #define EFLG_DF (1<<10)
346 #define EFLG_IF (1<<9)
347 #define EFLG_TF (1<<8)
348 #define EFLG_SF (1<<7)
349 #define EFLG_ZF (1<<6)
350 #define EFLG_AF (1<<4)
351 #define EFLG_PF (1<<2)
352 #define EFLG_CF (1<<0)
354 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
355 #define EFLG_RESERVED_ONE_MASK 2
358 * Instruction emulation:
359 * Most instructions are emulated directly via a fragment of inline assembly
360 * code. This allows us to save/restore EFLAGS and thus very easily pick up
361 * any modified flags.
364 #if defined(CONFIG_X86_64)
365 #define _LO32 "k" /* force 32-bit operand */
366 #define _STK "%%rsp" /* stack pointer */
367 #elif defined(__i386__)
368 #define _LO32 "" /* force 32-bit operand */
369 #define _STK "%%esp" /* stack pointer */
373 * These EFLAGS bits are restored from saved value during emulation, and
374 * any changes are written back to the saved value after emulation.
376 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
378 /* Before executing instruction: restore necessary bits in EFLAGS. */
379 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
380 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
381 "movl %"_sav",%"_LO32 _tmp"; " \
384 "movl %"_msk",%"_LO32 _tmp"; " \
385 "andl %"_LO32 _tmp",("_STK"); " \
387 "notl %"_LO32 _tmp"; " \
388 "andl %"_LO32 _tmp",("_STK"); " \
389 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
391 "orl %"_LO32 _tmp",("_STK"); " \
395 /* After executing instruction: write-back necessary bits in EFLAGS. */
396 #define _POST_EFLAGS(_sav, _msk, _tmp) \
397 /* _sav |= EFLAGS & _msk; */ \
400 "andl %"_msk",%"_LO32 _tmp"; " \
401 "orl %"_LO32 _tmp",%"_sav"; "
409 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
411 __asm__ __volatile__ ( \
412 _PRE_EFLAGS("0", "4", "2") \
413 _op _suffix " %"_x"3,%1; " \
414 _POST_EFLAGS("0", "4", "2") \
415 : "=m" (_eflags), "=m" ((_dst).val), \
417 : _y ((_src).val), "i" (EFLAGS_MASK)); \
421 /* Raw emulation: instruction has two explicit operands. */
422 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
424 unsigned long _tmp; \
426 switch ((_dst).bytes) { \
428 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
431 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
434 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
439 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
441 unsigned long _tmp; \
442 switch ((_dst).bytes) { \
444 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
447 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
448 _wx, _wy, _lx, _ly, _qx, _qy); \
453 /* Source operand is byte-sized and may be restricted to just %cl. */
454 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
455 __emulate_2op(_op, _src, _dst, _eflags, \
456 "b", "c", "b", "c", "b", "c", "b", "c")
458 /* Source operand is byte, word, long or quad sized. */
459 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
460 __emulate_2op(_op, _src, _dst, _eflags, \
461 "b", "q", "w", "r", _LO32, "r", "", "r")
463 /* Source operand is word, long or quad sized. */
464 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
465 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
466 "w", "r", _LO32, "r", "", "r")
468 /* Instruction has three operands and one operand is stored in ECX register */
469 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
471 unsigned long _tmp; \
472 _type _clv = (_cl).val; \
473 _type _srcv = (_src).val; \
474 _type _dstv = (_dst).val; \
476 __asm__ __volatile__ ( \
477 _PRE_EFLAGS("0", "5", "2") \
478 _op _suffix " %4,%1 \n" \
479 _POST_EFLAGS("0", "5", "2") \
480 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
481 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
484 (_cl).val = (unsigned long) _clv; \
485 (_src).val = (unsigned long) _srcv; \
486 (_dst).val = (unsigned long) _dstv; \
489 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
491 switch ((_dst).bytes) { \
493 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
494 "w", unsigned short); \
497 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
498 "l", unsigned int); \
501 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
502 "q", unsigned long)); \
507 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
509 unsigned long _tmp; \
511 __asm__ __volatile__ ( \
512 _PRE_EFLAGS("0", "3", "2") \
513 _op _suffix " %1; " \
514 _POST_EFLAGS("0", "3", "2") \
515 : "=m" (_eflags), "+m" ((_dst).val), \
517 : "i" (EFLAGS_MASK)); \
520 /* Instruction has only one explicit operand (no source operand). */
521 #define emulate_1op(_op, _dst, _eflags) \
523 switch ((_dst).bytes) { \
524 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
525 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
526 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
527 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
531 /* Fetch next part of the instruction being emulated. */
532 #define insn_fetch(_type, _size, _eip) \
533 ({ unsigned long _x; \
534 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
535 if (rc != X86EMUL_CONTINUE) \
541 #define insn_fetch_arr(_arr, _size, _eip) \
542 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
543 if (rc != X86EMUL_CONTINUE) \
548 static inline unsigned long ad_mask(struct decode_cache *c)
550 return (1UL << (c->ad_bytes << 3)) - 1;
553 /* Access/update address held in a register, based on addressing mode. */
554 static inline unsigned long
555 address_mask(struct decode_cache *c, unsigned long reg)
557 if (c->ad_bytes == sizeof(unsigned long))
560 return reg & ad_mask(c);
563 static inline unsigned long
564 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
566 return base + address_mask(c, reg);
570 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
572 if (c->ad_bytes == sizeof(unsigned long))
575 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
578 static inline void jmp_rel(struct decode_cache *c, int rel)
580 register_address_increment(c, &c->eip, rel);
583 static void set_seg_override(struct decode_cache *c, int seg)
585 c->has_seg_override = true;
586 c->seg_override = seg;
589 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
590 struct x86_emulate_ops *ops, int seg)
592 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
595 return ops->get_cached_segment_base(seg, ctxt->vcpu);
598 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
599 struct x86_emulate_ops *ops,
600 struct decode_cache *c)
602 if (!c->has_seg_override)
605 return seg_base(ctxt, ops, c->seg_override);
608 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
609 struct x86_emulate_ops *ops)
611 return seg_base(ctxt, ops, VCPU_SREG_ES);
614 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
615 struct x86_emulate_ops *ops)
617 return seg_base(ctxt, ops, VCPU_SREG_SS);
620 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
621 u32 error, bool valid)
623 ctxt->exception = vec;
624 ctxt->error_code = error;
625 ctxt->error_code_valid = valid;
626 ctxt->restart = false;
629 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
631 emulate_exception(ctxt, GP_VECTOR, err, true);
634 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
638 emulate_exception(ctxt, PF_VECTOR, err, true);
641 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
643 emulate_exception(ctxt, UD_VECTOR, 0, false);
646 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
648 emulate_exception(ctxt, TS_VECTOR, err, true);
651 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
652 struct x86_emulate_ops *ops,
653 unsigned long eip, u8 *dest)
655 struct fetch_cache *fc = &ctxt->decode.fetch;
659 if (eip == fc->end) {
660 cur_size = fc->end - fc->start;
661 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
662 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
663 size, ctxt->vcpu, NULL);
664 if (rc != X86EMUL_CONTINUE)
668 *dest = fc->data[eip - fc->start];
669 return X86EMUL_CONTINUE;
672 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
673 struct x86_emulate_ops *ops,
674 unsigned long eip, void *dest, unsigned size)
678 /* x86 instructions are limited to 15 bytes. */
679 if (eip + size - ctxt->eip > 15)
680 return X86EMUL_UNHANDLEABLE;
682 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
683 if (rc != X86EMUL_CONTINUE)
686 return X86EMUL_CONTINUE;
690 * Given the 'reg' portion of a ModRM byte, and a register block, return a
691 * pointer into the block that addresses the relevant register.
692 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
694 static void *decode_register(u8 modrm_reg, unsigned long *regs,
699 p = ®s[modrm_reg];
700 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
701 p = (unsigned char *)®s[modrm_reg & 3] + 1;
705 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
706 struct x86_emulate_ops *ops,
708 u16 *size, unsigned long *address, int op_bytes)
715 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
717 if (rc != X86EMUL_CONTINUE)
719 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
724 static int test_cc(unsigned int condition, unsigned int flags)
728 switch ((condition & 15) >> 1) {
730 rc |= (flags & EFLG_OF);
732 case 1: /* b/c/nae */
733 rc |= (flags & EFLG_CF);
736 rc |= (flags & EFLG_ZF);
739 rc |= (flags & (EFLG_CF|EFLG_ZF));
742 rc |= (flags & EFLG_SF);
745 rc |= (flags & EFLG_PF);
748 rc |= (flags & EFLG_ZF);
751 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
755 /* Odd condition identifiers (lsb == 1) have inverted sense. */
756 return (!!rc ^ (condition & 1));
759 static void decode_register_operand(struct operand *op,
760 struct decode_cache *c,
763 unsigned reg = c->modrm_reg;
764 int highbyte_regs = c->rex_prefix == 0;
767 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
769 if ((c->d & ByteOp) && !inhibit_bytereg) {
770 op->ptr = decode_register(reg, c->regs, highbyte_regs);
771 op->val = *(u8 *)op->ptr;
774 op->ptr = decode_register(reg, c->regs, 0);
775 op->bytes = c->op_bytes;
778 op->val = *(u16 *)op->ptr;
781 op->val = *(u32 *)op->ptr;
784 op->val = *(u64 *) op->ptr;
788 op->orig_val = op->val;
791 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
792 struct x86_emulate_ops *ops)
794 struct decode_cache *c = &ctxt->decode;
796 int index_reg = 0, base_reg = 0, scale;
797 int rc = X86EMUL_CONTINUE;
800 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
801 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
802 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
805 c->modrm = insn_fetch(u8, 1, c->eip);
806 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
807 c->modrm_reg |= (c->modrm & 0x38) >> 3;
808 c->modrm_rm |= (c->modrm & 0x07);
812 if (c->modrm_mod == 3) {
813 c->modrm_ptr = decode_register(c->modrm_rm,
814 c->regs, c->d & ByteOp);
815 c->modrm_val = *(unsigned long *)c->modrm_ptr;
819 if (c->ad_bytes == 2) {
820 unsigned bx = c->regs[VCPU_REGS_RBX];
821 unsigned bp = c->regs[VCPU_REGS_RBP];
822 unsigned si = c->regs[VCPU_REGS_RSI];
823 unsigned di = c->regs[VCPU_REGS_RDI];
825 /* 16-bit ModR/M decode. */
826 switch (c->modrm_mod) {
828 if (c->modrm_rm == 6)
829 c->modrm_ea += insn_fetch(u16, 2, c->eip);
832 c->modrm_ea += insn_fetch(s8, 1, c->eip);
835 c->modrm_ea += insn_fetch(u16, 2, c->eip);
838 switch (c->modrm_rm) {
840 c->modrm_ea += bx + si;
843 c->modrm_ea += bx + di;
846 c->modrm_ea += bp + si;
849 c->modrm_ea += bp + di;
858 if (c->modrm_mod != 0)
865 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
866 (c->modrm_rm == 6 && c->modrm_mod != 0))
867 if (!c->has_seg_override)
868 set_seg_override(c, VCPU_SREG_SS);
869 c->modrm_ea = (u16)c->modrm_ea;
871 /* 32/64-bit ModR/M decode. */
872 if ((c->modrm_rm & 7) == 4) {
873 sib = insn_fetch(u8, 1, c->eip);
874 index_reg |= (sib >> 3) & 7;
878 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
879 c->modrm_ea += insn_fetch(s32, 4, c->eip);
881 c->modrm_ea += c->regs[base_reg];
883 c->modrm_ea += c->regs[index_reg] << scale;
884 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
885 if (ctxt->mode == X86EMUL_MODE_PROT64)
888 c->modrm_ea += c->regs[c->modrm_rm];
889 switch (c->modrm_mod) {
891 if (c->modrm_rm == 5)
892 c->modrm_ea += insn_fetch(s32, 4, c->eip);
895 c->modrm_ea += insn_fetch(s8, 1, c->eip);
898 c->modrm_ea += insn_fetch(s32, 4, c->eip);
906 static int decode_abs(struct x86_emulate_ctxt *ctxt,
907 struct x86_emulate_ops *ops)
909 struct decode_cache *c = &ctxt->decode;
910 int rc = X86EMUL_CONTINUE;
912 switch (c->ad_bytes) {
914 c->modrm_ea = insn_fetch(u16, 2, c->eip);
917 c->modrm_ea = insn_fetch(u32, 4, c->eip);
920 c->modrm_ea = insn_fetch(u64, 8, c->eip);
928 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
930 struct decode_cache *c = &ctxt->decode;
931 int rc = X86EMUL_CONTINUE;
932 int mode = ctxt->mode;
933 int def_op_bytes, def_ad_bytes, group, dual;
936 /* we cannot decode insn before we complete previous rep insn */
937 WARN_ON(ctxt->restart);
940 c->fetch.start = c->fetch.end = c->eip;
941 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
944 case X86EMUL_MODE_REAL:
945 case X86EMUL_MODE_VM86:
946 case X86EMUL_MODE_PROT16:
947 def_op_bytes = def_ad_bytes = 2;
949 case X86EMUL_MODE_PROT32:
950 def_op_bytes = def_ad_bytes = 4;
953 case X86EMUL_MODE_PROT64:
962 c->op_bytes = def_op_bytes;
963 c->ad_bytes = def_ad_bytes;
965 /* Legacy prefixes. */
967 switch (c->b = insn_fetch(u8, 1, c->eip)) {
968 case 0x66: /* operand-size override */
969 /* switch between 2/4 bytes */
970 c->op_bytes = def_op_bytes ^ 6;
972 case 0x67: /* address-size override */
973 if (mode == X86EMUL_MODE_PROT64)
974 /* switch between 4/8 bytes */
975 c->ad_bytes = def_ad_bytes ^ 12;
977 /* switch between 2/4 bytes */
978 c->ad_bytes = def_ad_bytes ^ 6;
980 case 0x26: /* ES override */
981 case 0x2e: /* CS override */
982 case 0x36: /* SS override */
983 case 0x3e: /* DS override */
984 set_seg_override(c, (c->b >> 3) & 3);
986 case 0x64: /* FS override */
987 case 0x65: /* GS override */
988 set_seg_override(c, c->b & 7);
990 case 0x40 ... 0x4f: /* REX */
991 if (mode != X86EMUL_MODE_PROT64)
993 c->rex_prefix = c->b;
995 case 0xf0: /* LOCK */
998 case 0xf2: /* REPNE/REPNZ */
999 c->rep_prefix = REPNE_PREFIX;
1001 case 0xf3: /* REP/REPE/REPZ */
1002 c->rep_prefix = REPE_PREFIX;
1008 /* Any legacy prefix after a REX prefix nullifies its effect. */
1017 if (c->rex_prefix & 8)
1018 c->op_bytes = 8; /* REX.W */
1020 /* Opcode byte(s). */
1021 c->d = opcode_table[c->b].flags;
1023 /* Two-byte opcode? */
1026 c->b = insn_fetch(u8, 1, c->eip);
1027 c->d = twobyte_table[c->b].flags;
1032 group = c->d & GroupMask;
1033 dual = c->d & GroupDual;
1034 c->modrm = insn_fetch(u8, 1, c->eip);
1037 group = (group << 3) + ((c->modrm >> 3) & 7);
1038 c->d &= ~(Group | GroupDual | GroupMask);
1039 if (dual && (c->modrm >> 6) == 3)
1040 c->d |= group2_table[group].flags;
1042 c->d |= group_table[group].flags;
1046 if (c->d == 0 || (c->d & Undefined)) {
1047 DPRINTF("Cannot emulate %02x\n", c->b);
1051 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1054 /* ModRM and SIB bytes. */
1056 rc = decode_modrm(ctxt, ops);
1057 else if (c->d & MemAbs)
1058 rc = decode_abs(ctxt, ops);
1059 if (rc != X86EMUL_CONTINUE)
1062 if (!c->has_seg_override)
1063 set_seg_override(c, VCPU_SREG_DS);
1065 if (!(!c->twobyte && c->b == 0x8d))
1066 c->modrm_ea += seg_override_base(ctxt, ops, c);
1068 if (c->ad_bytes != 8)
1069 c->modrm_ea = (u32)c->modrm_ea;
1071 if (c->rip_relative)
1072 c->modrm_ea += c->eip;
1075 * Decode and fetch the source operand: register, memory
1078 switch (c->d & SrcMask) {
1082 decode_register_operand(&c->src, c, 0);
1091 c->src.bytes = (c->d & ByteOp) ? 1 :
1093 /* Don't fetch the address for invlpg: it could be unmapped. */
1094 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1098 * For instructions with a ModR/M byte, switch to register
1099 * access if Mod = 3.
1101 if ((c->d & ModRM) && c->modrm_mod == 3) {
1102 c->src.type = OP_REG;
1103 c->src.val = c->modrm_val;
1104 c->src.ptr = c->modrm_ptr;
1107 c->src.type = OP_MEM;
1108 c->src.ptr = (unsigned long *)c->modrm_ea;
1113 c->src.type = OP_IMM;
1114 c->src.ptr = (unsigned long *)c->eip;
1115 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1116 if (c->src.bytes == 8)
1118 /* NB. Immediates are sign-extended as necessary. */
1119 switch (c->src.bytes) {
1121 c->src.val = insn_fetch(s8, 1, c->eip);
1124 c->src.val = insn_fetch(s16, 2, c->eip);
1127 c->src.val = insn_fetch(s32, 4, c->eip);
1130 if ((c->d & SrcMask) == SrcImmU) {
1131 switch (c->src.bytes) {
1136 c->src.val &= 0xffff;
1139 c->src.val &= 0xffffffff;
1146 c->src.type = OP_IMM;
1147 c->src.ptr = (unsigned long *)c->eip;
1149 if ((c->d & SrcMask) == SrcImmByte)
1150 c->src.val = insn_fetch(s8, 1, c->eip);
1152 c->src.val = insn_fetch(u8, 1, c->eip);
1155 c->src.type = OP_REG;
1156 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1157 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1158 switch (c->src.bytes) {
1160 c->src.val = *(u8 *)c->src.ptr;
1163 c->src.val = *(u16 *)c->src.ptr;
1166 c->src.val = *(u32 *)c->src.ptr;
1169 c->src.val = *(u64 *)c->src.ptr;
1178 c->src.type = OP_MEM;
1179 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1180 c->src.ptr = (unsigned long *)
1181 register_address(c, seg_override_base(ctxt, ops, c),
1182 c->regs[VCPU_REGS_RSI]);
1186 c->src.type = OP_IMM;
1187 c->src.ptr = (unsigned long *)c->eip;
1188 c->src.bytes = c->op_bytes + 2;
1189 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1192 c->src.type = OP_MEM;
1193 c->src.ptr = (unsigned long *)c->modrm_ea;
1194 c->src.bytes = c->op_bytes + 2;
1199 * Decode and fetch the second source operand: register, memory
1202 switch (c->d & Src2Mask) {
1207 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1210 c->src2.type = OP_IMM;
1211 c->src2.ptr = (unsigned long *)c->eip;
1213 c->src2.val = insn_fetch(u8, 1, c->eip);
1221 /* Decode and fetch the destination operand: register or memory. */
1222 switch (c->d & DstMask) {
1224 /* Special instructions do their own operand decoding. */
1227 decode_register_operand(&c->dst, c,
1228 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1232 if ((c->d & ModRM) && c->modrm_mod == 3) {
1233 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1234 c->dst.type = OP_REG;
1235 c->dst.val = c->dst.orig_val = c->modrm_val;
1236 c->dst.ptr = c->modrm_ptr;
1239 c->dst.type = OP_MEM;
1240 c->dst.ptr = (unsigned long *)c->modrm_ea;
1241 if ((c->d & DstMask) == DstMem64)
1244 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1247 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1249 c->dst.ptr = (void *)c->dst.ptr +
1250 (c->src.val & mask) / 8;
1254 c->dst.type = OP_REG;
1255 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1256 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1257 switch (c->dst.bytes) {
1259 c->dst.val = *(u8 *)c->dst.ptr;
1262 c->dst.val = *(u16 *)c->dst.ptr;
1265 c->dst.val = *(u32 *)c->dst.ptr;
1268 c->dst.val = *(u64 *)c->dst.ptr;
1271 c->dst.orig_val = c->dst.val;
1274 c->dst.type = OP_MEM;
1275 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1276 c->dst.ptr = (unsigned long *)
1277 register_address(c, es_base(ctxt, ops),
1278 c->regs[VCPU_REGS_RDI]);
1284 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1287 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1288 struct x86_emulate_ops *ops,
1289 unsigned long addr, void *dest, unsigned size)
1292 struct read_cache *mc = &ctxt->decode.mem_read;
1296 int n = min(size, 8u);
1298 if (mc->pos < mc->end)
1301 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1303 if (rc == X86EMUL_PROPAGATE_FAULT)
1304 emulate_pf(ctxt, addr, err);
1305 if (rc != X86EMUL_CONTINUE)
1310 memcpy(dest, mc->data + mc->pos, n);
1315 return X86EMUL_CONTINUE;
1318 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1319 struct x86_emulate_ops *ops,
1320 unsigned int size, unsigned short port,
1323 struct read_cache *rc = &ctxt->decode.io_read;
1325 if (rc->pos == rc->end) { /* refill pio read ahead */
1326 struct decode_cache *c = &ctxt->decode;
1327 unsigned int in_page, n;
1328 unsigned int count = c->rep_prefix ?
1329 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1330 in_page = (ctxt->eflags & EFLG_DF) ?
1331 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1332 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1333 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1337 rc->pos = rc->end = 0;
1338 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1343 memcpy(dest, rc->data + rc->pos, size);
1348 static u32 desc_limit_scaled(struct desc_struct *desc)
1350 u32 limit = get_desc_limit(desc);
1352 return desc->g ? (limit << 12) | 0xfff : limit;
1355 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1356 struct x86_emulate_ops *ops,
1357 u16 selector, struct desc_ptr *dt)
1359 if (selector & 1 << 2) {
1360 struct desc_struct desc;
1361 memset (dt, 0, sizeof *dt);
1362 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1365 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1366 dt->address = get_desc_base(&desc);
1368 ops->get_gdt(dt, ctxt->vcpu);
1371 /* allowed just for 8 bytes segments */
1372 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1373 struct x86_emulate_ops *ops,
1374 u16 selector, struct desc_struct *desc)
1377 u16 index = selector >> 3;
1382 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1384 if (dt.size < index * 8 + 7) {
1385 emulate_gp(ctxt, selector & 0xfffc);
1386 return X86EMUL_PROPAGATE_FAULT;
1388 addr = dt.address + index * 8;
1389 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1390 if (ret == X86EMUL_PROPAGATE_FAULT)
1391 emulate_pf(ctxt, addr, err);
1396 /* allowed just for 8 bytes segments */
1397 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1398 struct x86_emulate_ops *ops,
1399 u16 selector, struct desc_struct *desc)
1402 u16 index = selector >> 3;
1407 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1409 if (dt.size < index * 8 + 7) {
1410 emulate_gp(ctxt, selector & 0xfffc);
1411 return X86EMUL_PROPAGATE_FAULT;
1414 addr = dt.address + index * 8;
1415 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1416 if (ret == X86EMUL_PROPAGATE_FAULT)
1417 emulate_pf(ctxt, addr, err);
1422 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1423 struct x86_emulate_ops *ops,
1424 u16 selector, int seg)
1426 struct desc_struct seg_desc;
1428 unsigned err_vec = GP_VECTOR;
1430 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1433 memset(&seg_desc, 0, sizeof seg_desc);
1435 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1436 || ctxt->mode == X86EMUL_MODE_REAL) {
1437 /* set real mode segment descriptor */
1438 set_desc_base(&seg_desc, selector << 4);
1439 set_desc_limit(&seg_desc, 0xffff);
1446 /* NULL selector is not valid for TR, CS and SS */
1447 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1451 /* TR should be in GDT only */
1452 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1455 if (null_selector) /* for NULL selector skip all following checks */
1458 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1459 if (ret != X86EMUL_CONTINUE)
1462 err_code = selector & 0xfffc;
1463 err_vec = GP_VECTOR;
1465 /* can't load system descriptor into segment selecor */
1466 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1470 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1476 cpl = ops->cpl(ctxt->vcpu);
1481 * segment is not a writable data segment or segment
1482 * selector's RPL != CPL or segment selector's RPL != CPL
1484 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1488 if (!(seg_desc.type & 8))
1491 if (seg_desc.type & 4) {
1497 if (rpl > cpl || dpl != cpl)
1500 /* CS(RPL) <- CPL */
1501 selector = (selector & 0xfffc) | cpl;
1504 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1507 case VCPU_SREG_LDTR:
1508 if (seg_desc.s || seg_desc.type != 2)
1511 default: /* DS, ES, FS, or GS */
1513 * segment is not a data or readable code segment or
1514 * ((segment is a data or nonconforming code segment)
1515 * and (both RPL and CPL > DPL))
1517 if ((seg_desc.type & 0xa) == 0x8 ||
1518 (((seg_desc.type & 0xc) != 0xc) &&
1519 (rpl > dpl && cpl > dpl)))
1525 /* mark segment as accessed */
1527 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1528 if (ret != X86EMUL_CONTINUE)
1532 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1533 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1534 return X86EMUL_CONTINUE;
1536 emulate_exception(ctxt, err_vec, err_code, true);
1537 return X86EMUL_PROPAGATE_FAULT;
1540 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1541 struct x86_emulate_ops *ops)
1544 struct decode_cache *c = &ctxt->decode;
1547 switch (c->dst.type) {
1549 /* The 4-byte case *is* correct:
1550 * in 64-bit mode we zero-extend.
1552 switch (c->dst.bytes) {
1554 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1557 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1560 *c->dst.ptr = (u32)c->dst.val;
1561 break; /* 64b: zero-ext */
1563 *c->dst.ptr = c->dst.val;
1569 rc = ops->cmpxchg_emulated(
1570 (unsigned long)c->dst.ptr,
1577 rc = ops->write_emulated(
1578 (unsigned long)c->dst.ptr,
1583 if (rc == X86EMUL_PROPAGATE_FAULT)
1585 (unsigned long)c->dst.ptr, err);
1586 if (rc != X86EMUL_CONTINUE)
1595 return X86EMUL_CONTINUE;
1598 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1599 struct x86_emulate_ops *ops)
1601 struct decode_cache *c = &ctxt->decode;
1603 c->dst.type = OP_MEM;
1604 c->dst.bytes = c->op_bytes;
1605 c->dst.val = c->src.val;
1606 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1607 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1608 c->regs[VCPU_REGS_RSP]);
1611 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1612 struct x86_emulate_ops *ops,
1613 void *dest, int len)
1615 struct decode_cache *c = &ctxt->decode;
1618 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1619 c->regs[VCPU_REGS_RSP]),
1621 if (rc != X86EMUL_CONTINUE)
1624 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1628 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1629 struct x86_emulate_ops *ops,
1630 void *dest, int len)
1633 unsigned long val, change_mask;
1634 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1635 int cpl = ops->cpl(ctxt->vcpu);
1637 rc = emulate_pop(ctxt, ops, &val, len);
1638 if (rc != X86EMUL_CONTINUE)
1641 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1642 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1644 switch(ctxt->mode) {
1645 case X86EMUL_MODE_PROT64:
1646 case X86EMUL_MODE_PROT32:
1647 case X86EMUL_MODE_PROT16:
1649 change_mask |= EFLG_IOPL;
1651 change_mask |= EFLG_IF;
1653 case X86EMUL_MODE_VM86:
1655 emulate_gp(ctxt, 0);
1656 return X86EMUL_PROPAGATE_FAULT;
1658 change_mask |= EFLG_IF;
1660 default: /* real mode */
1661 change_mask |= (EFLG_IOPL | EFLG_IF);
1665 *(unsigned long *)dest =
1666 (ctxt->eflags & ~change_mask) | (val & change_mask);
1671 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1672 struct x86_emulate_ops *ops, int seg)
1674 struct decode_cache *c = &ctxt->decode;
1676 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1678 emulate_push(ctxt, ops);
1681 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1682 struct x86_emulate_ops *ops, int seg)
1684 struct decode_cache *c = &ctxt->decode;
1685 unsigned long selector;
1688 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1689 if (rc != X86EMUL_CONTINUE)
1692 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1696 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1697 struct x86_emulate_ops *ops)
1699 struct decode_cache *c = &ctxt->decode;
1700 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1701 int rc = X86EMUL_CONTINUE;
1702 int reg = VCPU_REGS_RAX;
1704 while (reg <= VCPU_REGS_RDI) {
1705 (reg == VCPU_REGS_RSP) ?
1706 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1708 emulate_push(ctxt, ops);
1710 rc = writeback(ctxt, ops);
1711 if (rc != X86EMUL_CONTINUE)
1717 /* Disable writeback. */
1718 c->dst.type = OP_NONE;
1723 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1724 struct x86_emulate_ops *ops)
1726 struct decode_cache *c = &ctxt->decode;
1727 int rc = X86EMUL_CONTINUE;
1728 int reg = VCPU_REGS_RDI;
1730 while (reg >= VCPU_REGS_RAX) {
1731 if (reg == VCPU_REGS_RSP) {
1732 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1737 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1738 if (rc != X86EMUL_CONTINUE)
1745 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1746 struct x86_emulate_ops *ops)
1748 struct decode_cache *c = &ctxt->decode;
1749 int rc = X86EMUL_CONTINUE;
1750 unsigned long temp_eip = 0;
1751 unsigned long temp_eflags = 0;
1752 unsigned long cs = 0;
1753 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1754 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1755 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1756 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1758 /* TODO: Add stack limit check */
1760 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1762 if (rc != X86EMUL_CONTINUE)
1765 if (temp_eip & ~0xffff) {
1766 emulate_gp(ctxt, 0);
1767 return X86EMUL_PROPAGATE_FAULT;
1770 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1772 if (rc != X86EMUL_CONTINUE)
1775 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1777 if (rc != X86EMUL_CONTINUE)
1780 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1782 if (rc != X86EMUL_CONTINUE)
1788 if (c->op_bytes == 4)
1789 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1790 else if (c->op_bytes == 2) {
1791 ctxt->eflags &= ~0xffff;
1792 ctxt->eflags |= temp_eflags;
1795 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1796 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1801 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1802 struct x86_emulate_ops* ops)
1804 switch(ctxt->mode) {
1805 case X86EMUL_MODE_REAL:
1806 return emulate_iret_real(ctxt, ops);
1807 case X86EMUL_MODE_VM86:
1808 case X86EMUL_MODE_PROT16:
1809 case X86EMUL_MODE_PROT32:
1810 case X86EMUL_MODE_PROT64:
1812 /* iret from protected mode unimplemented yet */
1813 return X86EMUL_UNHANDLEABLE;
1817 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1818 struct x86_emulate_ops *ops)
1820 struct decode_cache *c = &ctxt->decode;
1822 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1825 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1827 struct decode_cache *c = &ctxt->decode;
1828 switch (c->modrm_reg) {
1830 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1833 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1836 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1839 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1841 case 4: /* sal/shl */
1842 case 6: /* sal/shl */
1843 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1846 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1849 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1854 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1855 struct x86_emulate_ops *ops)
1857 struct decode_cache *c = &ctxt->decode;
1859 switch (c->modrm_reg) {
1860 case 0 ... 1: /* test */
1861 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1864 c->dst.val = ~c->dst.val;
1867 emulate_1op("neg", c->dst, ctxt->eflags);
1875 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1876 struct x86_emulate_ops *ops)
1878 struct decode_cache *c = &ctxt->decode;
1880 switch (c->modrm_reg) {
1882 emulate_1op("inc", c->dst, ctxt->eflags);
1885 emulate_1op("dec", c->dst, ctxt->eflags);
1887 case 2: /* call near abs */ {
1890 c->eip = c->src.val;
1891 c->src.val = old_eip;
1892 emulate_push(ctxt, ops);
1895 case 4: /* jmp abs */
1896 c->eip = c->src.val;
1899 emulate_push(ctxt, ops);
1902 return X86EMUL_CONTINUE;
1905 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1906 struct x86_emulate_ops *ops)
1908 struct decode_cache *c = &ctxt->decode;
1909 u64 old = c->dst.orig_val64;
1911 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1912 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1913 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1914 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1915 ctxt->eflags &= ~EFLG_ZF;
1917 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1918 (u32) c->regs[VCPU_REGS_RBX];
1920 ctxt->eflags |= EFLG_ZF;
1922 return X86EMUL_CONTINUE;
1925 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1926 struct x86_emulate_ops *ops)
1928 struct decode_cache *c = &ctxt->decode;
1932 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1933 if (rc != X86EMUL_CONTINUE)
1935 if (c->op_bytes == 4)
1936 c->eip = (u32)c->eip;
1937 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1938 if (rc != X86EMUL_CONTINUE)
1940 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1945 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1946 struct x86_emulate_ops *ops, struct desc_struct *cs,
1947 struct desc_struct *ss)
1949 memset(cs, 0, sizeof(struct desc_struct));
1950 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1951 memset(ss, 0, sizeof(struct desc_struct));
1953 cs->l = 0; /* will be adjusted later */
1954 set_desc_base(cs, 0); /* flat segment */
1955 cs->g = 1; /* 4kb granularity */
1956 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1957 cs->type = 0x0b; /* Read, Execute, Accessed */
1959 cs->dpl = 0; /* will be adjusted later */
1963 set_desc_base(ss, 0); /* flat segment */
1964 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1965 ss->g = 1; /* 4kb granularity */
1967 ss->type = 0x03; /* Read/Write, Accessed */
1968 ss->d = 1; /* 32bit stack segment */
1974 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1976 struct decode_cache *c = &ctxt->decode;
1977 struct desc_struct cs, ss;
1981 /* syscall is not available in real mode */
1982 if (ctxt->mode == X86EMUL_MODE_REAL ||
1983 ctxt->mode == X86EMUL_MODE_VM86) {
1985 return X86EMUL_PROPAGATE_FAULT;
1988 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1990 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1992 cs_sel = (u16)(msr_data & 0xfffc);
1993 ss_sel = (u16)(msr_data + 8);
1995 if (is_long_mode(ctxt->vcpu)) {
1999 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2000 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2001 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2002 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2004 c->regs[VCPU_REGS_RCX] = c->eip;
2005 if (is_long_mode(ctxt->vcpu)) {
2006 #ifdef CONFIG_X86_64
2007 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2009 ops->get_msr(ctxt->vcpu,
2010 ctxt->mode == X86EMUL_MODE_PROT64 ?
2011 MSR_LSTAR : MSR_CSTAR, &msr_data);
2014 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2015 ctxt->eflags &= ~(msr_data | EFLG_RF);
2019 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2020 c->eip = (u32)msr_data;
2022 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2025 return X86EMUL_CONTINUE;
2029 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2031 struct decode_cache *c = &ctxt->decode;
2032 struct desc_struct cs, ss;
2036 /* inject #GP if in real mode */
2037 if (ctxt->mode == X86EMUL_MODE_REAL) {
2038 emulate_gp(ctxt, 0);
2039 return X86EMUL_PROPAGATE_FAULT;
2042 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2043 * Therefore, we inject an #UD.
2045 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2047 return X86EMUL_PROPAGATE_FAULT;
2050 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2052 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2053 switch (ctxt->mode) {
2054 case X86EMUL_MODE_PROT32:
2055 if ((msr_data & 0xfffc) == 0x0) {
2056 emulate_gp(ctxt, 0);
2057 return X86EMUL_PROPAGATE_FAULT;
2060 case X86EMUL_MODE_PROT64:
2061 if (msr_data == 0x0) {
2062 emulate_gp(ctxt, 0);
2063 return X86EMUL_PROPAGATE_FAULT;
2068 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2069 cs_sel = (u16)msr_data;
2070 cs_sel &= ~SELECTOR_RPL_MASK;
2071 ss_sel = cs_sel + 8;
2072 ss_sel &= ~SELECTOR_RPL_MASK;
2073 if (ctxt->mode == X86EMUL_MODE_PROT64
2074 || is_long_mode(ctxt->vcpu)) {
2079 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2080 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2081 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2082 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2084 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2087 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2088 c->regs[VCPU_REGS_RSP] = msr_data;
2090 return X86EMUL_CONTINUE;
2094 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2096 struct decode_cache *c = &ctxt->decode;
2097 struct desc_struct cs, ss;
2102 /* inject #GP if in real mode or Virtual 8086 mode */
2103 if (ctxt->mode == X86EMUL_MODE_REAL ||
2104 ctxt->mode == X86EMUL_MODE_VM86) {
2105 emulate_gp(ctxt, 0);
2106 return X86EMUL_PROPAGATE_FAULT;
2109 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2111 if ((c->rex_prefix & 0x8) != 0x0)
2112 usermode = X86EMUL_MODE_PROT64;
2114 usermode = X86EMUL_MODE_PROT32;
2118 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2120 case X86EMUL_MODE_PROT32:
2121 cs_sel = (u16)(msr_data + 16);
2122 if ((msr_data & 0xfffc) == 0x0) {
2123 emulate_gp(ctxt, 0);
2124 return X86EMUL_PROPAGATE_FAULT;
2126 ss_sel = (u16)(msr_data + 24);
2128 case X86EMUL_MODE_PROT64:
2129 cs_sel = (u16)(msr_data + 32);
2130 if (msr_data == 0x0) {
2131 emulate_gp(ctxt, 0);
2132 return X86EMUL_PROPAGATE_FAULT;
2134 ss_sel = cs_sel + 8;
2139 cs_sel |= SELECTOR_RPL_MASK;
2140 ss_sel |= SELECTOR_RPL_MASK;
2142 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2143 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2144 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2145 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2147 c->eip = c->regs[VCPU_REGS_RDX];
2148 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2150 return X86EMUL_CONTINUE;
2153 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2154 struct x86_emulate_ops *ops)
2157 if (ctxt->mode == X86EMUL_MODE_REAL)
2159 if (ctxt->mode == X86EMUL_MODE_VM86)
2161 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2162 return ops->cpl(ctxt->vcpu) > iopl;
2165 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2166 struct x86_emulate_ops *ops,
2169 struct desc_struct tr_seg;
2172 u8 perm, bit_idx = port & 0x7;
2173 unsigned mask = (1 << len) - 1;
2175 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2178 if (desc_limit_scaled(&tr_seg) < 103)
2180 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2182 if (r != X86EMUL_CONTINUE)
2184 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2186 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2187 &perm, 1, ctxt->vcpu, NULL);
2188 if (r != X86EMUL_CONTINUE)
2190 if ((perm >> bit_idx) & mask)
2195 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2196 struct x86_emulate_ops *ops,
2199 if (emulator_bad_iopl(ctxt, ops))
2200 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2205 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2206 struct x86_emulate_ops *ops,
2207 struct tss_segment_16 *tss)
2209 struct decode_cache *c = &ctxt->decode;
2212 tss->flag = ctxt->eflags;
2213 tss->ax = c->regs[VCPU_REGS_RAX];
2214 tss->cx = c->regs[VCPU_REGS_RCX];
2215 tss->dx = c->regs[VCPU_REGS_RDX];
2216 tss->bx = c->regs[VCPU_REGS_RBX];
2217 tss->sp = c->regs[VCPU_REGS_RSP];
2218 tss->bp = c->regs[VCPU_REGS_RBP];
2219 tss->si = c->regs[VCPU_REGS_RSI];
2220 tss->di = c->regs[VCPU_REGS_RDI];
2222 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2223 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2224 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2225 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2226 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2229 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2230 struct x86_emulate_ops *ops,
2231 struct tss_segment_16 *tss)
2233 struct decode_cache *c = &ctxt->decode;
2237 ctxt->eflags = tss->flag | 2;
2238 c->regs[VCPU_REGS_RAX] = tss->ax;
2239 c->regs[VCPU_REGS_RCX] = tss->cx;
2240 c->regs[VCPU_REGS_RDX] = tss->dx;
2241 c->regs[VCPU_REGS_RBX] = tss->bx;
2242 c->regs[VCPU_REGS_RSP] = tss->sp;
2243 c->regs[VCPU_REGS_RBP] = tss->bp;
2244 c->regs[VCPU_REGS_RSI] = tss->si;
2245 c->regs[VCPU_REGS_RDI] = tss->di;
2248 * SDM says that segment selectors are loaded before segment
2251 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2252 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2253 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2254 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2255 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2258 * Now load segment descriptors. If fault happenes at this stage
2259 * it is handled in a context of new task
2261 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2262 if (ret != X86EMUL_CONTINUE)
2264 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2265 if (ret != X86EMUL_CONTINUE)
2267 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2268 if (ret != X86EMUL_CONTINUE)
2270 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2271 if (ret != X86EMUL_CONTINUE)
2273 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2274 if (ret != X86EMUL_CONTINUE)
2277 return X86EMUL_CONTINUE;
2280 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2281 struct x86_emulate_ops *ops,
2282 u16 tss_selector, u16 old_tss_sel,
2283 ulong old_tss_base, struct desc_struct *new_desc)
2285 struct tss_segment_16 tss_seg;
2287 u32 err, new_tss_base = get_desc_base(new_desc);
2289 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2291 if (ret == X86EMUL_PROPAGATE_FAULT) {
2292 /* FIXME: need to provide precise fault address */
2293 emulate_pf(ctxt, old_tss_base, err);
2297 save_state_to_tss16(ctxt, ops, &tss_seg);
2299 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2301 if (ret == X86EMUL_PROPAGATE_FAULT) {
2302 /* FIXME: need to provide precise fault address */
2303 emulate_pf(ctxt, old_tss_base, err);
2307 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2309 if (ret == X86EMUL_PROPAGATE_FAULT) {
2310 /* FIXME: need to provide precise fault address */
2311 emulate_pf(ctxt, new_tss_base, err);
2315 if (old_tss_sel != 0xffff) {
2316 tss_seg.prev_task_link = old_tss_sel;
2318 ret = ops->write_std(new_tss_base,
2319 &tss_seg.prev_task_link,
2320 sizeof tss_seg.prev_task_link,
2322 if (ret == X86EMUL_PROPAGATE_FAULT) {
2323 /* FIXME: need to provide precise fault address */
2324 emulate_pf(ctxt, new_tss_base, err);
2329 return load_state_from_tss16(ctxt, ops, &tss_seg);
2332 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2333 struct x86_emulate_ops *ops,
2334 struct tss_segment_32 *tss)
2336 struct decode_cache *c = &ctxt->decode;
2338 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2340 tss->eflags = ctxt->eflags;
2341 tss->eax = c->regs[VCPU_REGS_RAX];
2342 tss->ecx = c->regs[VCPU_REGS_RCX];
2343 tss->edx = c->regs[VCPU_REGS_RDX];
2344 tss->ebx = c->regs[VCPU_REGS_RBX];
2345 tss->esp = c->regs[VCPU_REGS_RSP];
2346 tss->ebp = c->regs[VCPU_REGS_RBP];
2347 tss->esi = c->regs[VCPU_REGS_RSI];
2348 tss->edi = c->regs[VCPU_REGS_RDI];
2350 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2351 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2352 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2353 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2354 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2355 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2356 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2359 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2360 struct x86_emulate_ops *ops,
2361 struct tss_segment_32 *tss)
2363 struct decode_cache *c = &ctxt->decode;
2366 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2367 emulate_gp(ctxt, 0);
2368 return X86EMUL_PROPAGATE_FAULT;
2371 ctxt->eflags = tss->eflags | 2;
2372 c->regs[VCPU_REGS_RAX] = tss->eax;
2373 c->regs[VCPU_REGS_RCX] = tss->ecx;
2374 c->regs[VCPU_REGS_RDX] = tss->edx;
2375 c->regs[VCPU_REGS_RBX] = tss->ebx;
2376 c->regs[VCPU_REGS_RSP] = tss->esp;
2377 c->regs[VCPU_REGS_RBP] = tss->ebp;
2378 c->regs[VCPU_REGS_RSI] = tss->esi;
2379 c->regs[VCPU_REGS_RDI] = tss->edi;
2382 * SDM says that segment selectors are loaded before segment
2385 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2386 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2387 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2388 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2389 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2390 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2391 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2394 * Now load segment descriptors. If fault happenes at this stage
2395 * it is handled in a context of new task
2397 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2398 if (ret != X86EMUL_CONTINUE)
2400 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2401 if (ret != X86EMUL_CONTINUE)
2403 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2404 if (ret != X86EMUL_CONTINUE)
2406 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2407 if (ret != X86EMUL_CONTINUE)
2409 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2410 if (ret != X86EMUL_CONTINUE)
2412 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2413 if (ret != X86EMUL_CONTINUE)
2415 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2416 if (ret != X86EMUL_CONTINUE)
2419 return X86EMUL_CONTINUE;
2422 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2423 struct x86_emulate_ops *ops,
2424 u16 tss_selector, u16 old_tss_sel,
2425 ulong old_tss_base, struct desc_struct *new_desc)
2427 struct tss_segment_32 tss_seg;
2429 u32 err, new_tss_base = get_desc_base(new_desc);
2431 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2433 if (ret == X86EMUL_PROPAGATE_FAULT) {
2434 /* FIXME: need to provide precise fault address */
2435 emulate_pf(ctxt, old_tss_base, err);
2439 save_state_to_tss32(ctxt, ops, &tss_seg);
2441 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2443 if (ret == X86EMUL_PROPAGATE_FAULT) {
2444 /* FIXME: need to provide precise fault address */
2445 emulate_pf(ctxt, old_tss_base, err);
2449 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2451 if (ret == X86EMUL_PROPAGATE_FAULT) {
2452 /* FIXME: need to provide precise fault address */
2453 emulate_pf(ctxt, new_tss_base, err);
2457 if (old_tss_sel != 0xffff) {
2458 tss_seg.prev_task_link = old_tss_sel;
2460 ret = ops->write_std(new_tss_base,
2461 &tss_seg.prev_task_link,
2462 sizeof tss_seg.prev_task_link,
2464 if (ret == X86EMUL_PROPAGATE_FAULT) {
2465 /* FIXME: need to provide precise fault address */
2466 emulate_pf(ctxt, new_tss_base, err);
2471 return load_state_from_tss32(ctxt, ops, &tss_seg);
2474 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2475 struct x86_emulate_ops *ops,
2476 u16 tss_selector, int reason,
2477 bool has_error_code, u32 error_code)
2479 struct desc_struct curr_tss_desc, next_tss_desc;
2481 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2482 ulong old_tss_base =
2483 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2486 /* FIXME: old_tss_base == ~0 ? */
2488 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2489 if (ret != X86EMUL_CONTINUE)
2491 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2492 if (ret != X86EMUL_CONTINUE)
2495 /* FIXME: check that next_tss_desc is tss */
2497 if (reason != TASK_SWITCH_IRET) {
2498 if ((tss_selector & 3) > next_tss_desc.dpl ||
2499 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2500 emulate_gp(ctxt, 0);
2501 return X86EMUL_PROPAGATE_FAULT;
2505 desc_limit = desc_limit_scaled(&next_tss_desc);
2506 if (!next_tss_desc.p ||
2507 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2508 desc_limit < 0x2b)) {
2509 emulate_ts(ctxt, tss_selector & 0xfffc);
2510 return X86EMUL_PROPAGATE_FAULT;
2513 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2514 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2515 write_segment_descriptor(ctxt, ops, old_tss_sel,
2519 if (reason == TASK_SWITCH_IRET)
2520 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2522 /* set back link to prev task only if NT bit is set in eflags
2523 note that old_tss_sel is not used afetr this point */
2524 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2525 old_tss_sel = 0xffff;
2527 if (next_tss_desc.type & 8)
2528 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2529 old_tss_base, &next_tss_desc);
2531 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2532 old_tss_base, &next_tss_desc);
2533 if (ret != X86EMUL_CONTINUE)
2536 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2537 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2539 if (reason != TASK_SWITCH_IRET) {
2540 next_tss_desc.type |= (1 << 1); /* set busy flag */
2541 write_segment_descriptor(ctxt, ops, tss_selector,
2545 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2546 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2547 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2549 if (has_error_code) {
2550 struct decode_cache *c = &ctxt->decode;
2552 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2554 c->src.val = (unsigned long) error_code;
2555 emulate_push(ctxt, ops);
2561 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2562 struct x86_emulate_ops *ops,
2563 u16 tss_selector, int reason,
2564 bool has_error_code, u32 error_code)
2566 struct decode_cache *c = &ctxt->decode;
2570 c->dst.type = OP_NONE;
2572 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2573 has_error_code, error_code);
2575 if (rc == X86EMUL_CONTINUE) {
2576 rc = writeback(ctxt, ops);
2577 if (rc == X86EMUL_CONTINUE)
2581 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2584 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2585 int reg, struct operand *op)
2587 struct decode_cache *c = &ctxt->decode;
2588 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2590 register_address_increment(c, &c->regs[reg], df * op->bytes);
2591 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2595 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2598 struct decode_cache *c = &ctxt->decode;
2599 int rc = X86EMUL_CONTINUE;
2600 int saved_dst_type = c->dst.type;
2602 ctxt->decode.mem_read.pos = 0;
2604 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2609 /* LOCK prefix is allowed only with some instructions */
2610 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2615 /* Privileged instruction can be executed only in CPL=0 */
2616 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2617 emulate_gp(ctxt, 0);
2621 if (c->rep_prefix && (c->d & String)) {
2622 ctxt->restart = true;
2623 /* All REP prefixes have the same first termination condition */
2624 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2626 ctxt->restart = false;
2630 /* The second termination condition only applies for REPE
2631 * and REPNE. Test if the repeat string operation prefix is
2632 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2633 * corresponding termination condition according to:
2634 * - if REPE/REPZ and ZF = 0 then done
2635 * - if REPNE/REPNZ and ZF = 1 then done
2637 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2638 (c->b == 0xae) || (c->b == 0xaf)) {
2639 if ((c->rep_prefix == REPE_PREFIX) &&
2640 ((ctxt->eflags & EFLG_ZF) == 0))
2642 if ((c->rep_prefix == REPNE_PREFIX) &&
2643 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2649 if (c->src.type == OP_MEM) {
2650 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2651 c->src.valptr, c->src.bytes);
2652 if (rc != X86EMUL_CONTINUE)
2654 c->src.orig_val64 = c->src.val64;
2657 if (c->src2.type == OP_MEM) {
2658 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2659 &c->src2.val, c->src2.bytes);
2660 if (rc != X86EMUL_CONTINUE)
2664 if ((c->d & DstMask) == ImplicitOps)
2668 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2669 /* optimisation - avoid slow emulated read if Mov */
2670 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2671 &c->dst.val, c->dst.bytes);
2672 if (rc != X86EMUL_CONTINUE)
2675 c->dst.orig_val = c->dst.val;
2685 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2687 case 0x06: /* push es */
2688 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2690 case 0x07: /* pop es */
2691 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2692 if (rc != X86EMUL_CONTINUE)
2697 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2699 case 0x0e: /* push cs */
2700 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2704 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2706 case 0x16: /* push ss */
2707 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2709 case 0x17: /* pop ss */
2710 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2711 if (rc != X86EMUL_CONTINUE)
2716 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2718 case 0x1e: /* push ds */
2719 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2721 case 0x1f: /* pop ds */
2722 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2723 if (rc != X86EMUL_CONTINUE)
2728 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2732 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2736 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2740 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2742 case 0x40 ... 0x47: /* inc r16/r32 */
2743 emulate_1op("inc", c->dst, ctxt->eflags);
2745 case 0x48 ... 0x4f: /* dec r16/r32 */
2746 emulate_1op("dec", c->dst, ctxt->eflags);
2748 case 0x50 ... 0x57: /* push reg */
2749 emulate_push(ctxt, ops);
2751 case 0x58 ... 0x5f: /* pop reg */
2753 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2754 if (rc != X86EMUL_CONTINUE)
2757 case 0x60: /* pusha */
2758 rc = emulate_pusha(ctxt, ops);
2759 if (rc != X86EMUL_CONTINUE)
2762 case 0x61: /* popa */
2763 rc = emulate_popa(ctxt, ops);
2764 if (rc != X86EMUL_CONTINUE)
2767 case 0x63: /* movsxd */
2768 if (ctxt->mode != X86EMUL_MODE_PROT64)
2769 goto cannot_emulate;
2770 c->dst.val = (s32) c->src.val;
2772 case 0x68: /* push imm */
2773 case 0x6a: /* push imm8 */
2774 emulate_push(ctxt, ops);
2776 case 0x6c: /* insb */
2777 case 0x6d: /* insw/insd */
2778 c->dst.bytes = min(c->dst.bytes, 4u);
2779 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2781 emulate_gp(ctxt, 0);
2784 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2785 c->regs[VCPU_REGS_RDX], &c->dst.val))
2786 goto done; /* IO is needed, skip writeback */
2788 case 0x6e: /* outsb */
2789 case 0x6f: /* outsw/outsd */
2790 c->src.bytes = min(c->src.bytes, 4u);
2791 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2793 emulate_gp(ctxt, 0);
2796 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2797 &c->src.val, 1, ctxt->vcpu);
2799 c->dst.type = OP_NONE; /* nothing to writeback */
2801 case 0x70 ... 0x7f: /* jcc (short) */
2802 if (test_cc(c->b, ctxt->eflags))
2803 jmp_rel(c, c->src.val);
2805 case 0x80 ... 0x83: /* Grp1 */
2806 switch (c->modrm_reg) {
2827 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2829 case 0x86 ... 0x87: /* xchg */
2831 /* Write back the register source. */
2832 switch (c->dst.bytes) {
2834 *(u8 *) c->src.ptr = (u8) c->dst.val;
2837 *(u16 *) c->src.ptr = (u16) c->dst.val;
2840 *c->src.ptr = (u32) c->dst.val;
2841 break; /* 64b reg: zero-extend */
2843 *c->src.ptr = c->dst.val;
2847 * Write back the memory destination with implicit LOCK
2850 c->dst.val = c->src.val;
2853 case 0x88 ... 0x8b: /* mov */
2855 case 0x8c: /* mov r/m, sreg */
2856 if (c->modrm_reg > VCPU_SREG_GS) {
2860 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2862 case 0x8d: /* lea r16/r32, m */
2863 c->dst.val = c->modrm_ea;
2865 case 0x8e: { /* mov seg, r/m16 */
2870 if (c->modrm_reg == VCPU_SREG_CS ||
2871 c->modrm_reg > VCPU_SREG_GS) {
2876 if (c->modrm_reg == VCPU_SREG_SS)
2877 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2879 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2881 c->dst.type = OP_NONE; /* Disable writeback. */
2884 case 0x8f: /* pop (sole member of Grp1a) */
2885 rc = emulate_grp1a(ctxt, ops);
2886 if (rc != X86EMUL_CONTINUE)
2889 case 0x90: /* nop / xchg r8,rax */
2890 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2891 c->dst.type = OP_NONE; /* nop */
2894 case 0x91 ... 0x97: /* xchg reg,rax */
2895 c->src.type = OP_REG;
2896 c->src.bytes = c->op_bytes;
2897 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2898 c->src.val = *(c->src.ptr);
2900 case 0x9c: /* pushf */
2901 c->src.val = (unsigned long) ctxt->eflags;
2902 emulate_push(ctxt, ops);
2904 case 0x9d: /* popf */
2905 c->dst.type = OP_REG;
2906 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2907 c->dst.bytes = c->op_bytes;
2908 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2909 if (rc != X86EMUL_CONTINUE)
2912 case 0xa0 ... 0xa3: /* mov */
2913 case 0xa4 ... 0xa5: /* movs */
2915 case 0xa6 ... 0xa7: /* cmps */
2916 c->dst.type = OP_NONE; /* Disable writeback. */
2917 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2919 case 0xa8 ... 0xa9: /* test ax, imm */
2921 case 0xaa ... 0xab: /* stos */
2922 c->dst.val = c->regs[VCPU_REGS_RAX];
2924 case 0xac ... 0xad: /* lods */
2926 case 0xae ... 0xaf: /* scas */
2927 DPRINTF("Urk! I don't handle SCAS.\n");
2928 goto cannot_emulate;
2929 case 0xb0 ... 0xbf: /* mov r, imm */
2934 case 0xc3: /* ret */
2935 c->dst.type = OP_REG;
2936 c->dst.ptr = &c->eip;
2937 c->dst.bytes = c->op_bytes;
2938 goto pop_instruction;
2939 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2941 c->dst.val = c->src.val;
2943 case 0xcb: /* ret far */
2944 rc = emulate_ret_far(ctxt, ops);
2945 if (rc != X86EMUL_CONTINUE)
2948 case 0xcf: /* iret */
2949 rc = emulate_iret(ctxt, ops);
2951 if (rc != X86EMUL_CONTINUE)
2954 case 0xd0 ... 0xd1: /* Grp2 */
2958 case 0xd2 ... 0xd3: /* Grp2 */
2959 c->src.val = c->regs[VCPU_REGS_RCX];
2962 case 0xe4: /* inb */
2965 case 0xe6: /* outb */
2966 case 0xe7: /* out */
2968 case 0xe8: /* call (near) */ {
2969 long int rel = c->src.val;
2970 c->src.val = (unsigned long) c->eip;
2972 emulate_push(ctxt, ops);
2975 case 0xe9: /* jmp rel */
2977 case 0xea: { /* jmp far */
2980 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2982 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
2986 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2990 jmp: /* jmp rel short */
2991 jmp_rel(c, c->src.val);
2992 c->dst.type = OP_NONE; /* Disable writeback. */
2994 case 0xec: /* in al,dx */
2995 case 0xed: /* in (e/r)ax,dx */
2996 c->src.val = c->regs[VCPU_REGS_RDX];
2998 c->dst.bytes = min(c->dst.bytes, 4u);
2999 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3000 emulate_gp(ctxt, 0);
3003 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3005 goto done; /* IO is needed */
3007 case 0xee: /* out dx,al */
3008 case 0xef: /* out dx,(e/r)ax */
3009 c->src.val = c->regs[VCPU_REGS_RDX];
3011 c->dst.bytes = min(c->dst.bytes, 4u);
3012 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3013 emulate_gp(ctxt, 0);
3016 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3018 c->dst.type = OP_NONE; /* Disable writeback. */
3020 case 0xf4: /* hlt */
3021 ctxt->vcpu->arch.halt_request = 1;
3023 case 0xf5: /* cmc */
3024 /* complement carry flag from eflags reg */
3025 ctxt->eflags ^= EFLG_CF;
3026 c->dst.type = OP_NONE; /* Disable writeback. */
3028 case 0xf6 ... 0xf7: /* Grp3 */
3029 if (!emulate_grp3(ctxt, ops))
3030 goto cannot_emulate;
3032 case 0xf8: /* clc */
3033 ctxt->eflags &= ~EFLG_CF;
3034 c->dst.type = OP_NONE; /* Disable writeback. */
3036 case 0xfa: /* cli */
3037 if (emulator_bad_iopl(ctxt, ops)) {
3038 emulate_gp(ctxt, 0);
3041 ctxt->eflags &= ~X86_EFLAGS_IF;
3042 c->dst.type = OP_NONE; /* Disable writeback. */
3045 case 0xfb: /* sti */
3046 if (emulator_bad_iopl(ctxt, ops)) {
3047 emulate_gp(ctxt, 0);
3050 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3051 ctxt->eflags |= X86_EFLAGS_IF;
3052 c->dst.type = OP_NONE; /* Disable writeback. */
3055 case 0xfc: /* cld */
3056 ctxt->eflags &= ~EFLG_DF;
3057 c->dst.type = OP_NONE; /* Disable writeback. */
3059 case 0xfd: /* std */
3060 ctxt->eflags |= EFLG_DF;
3061 c->dst.type = OP_NONE; /* Disable writeback. */
3063 case 0xfe: /* Grp4 */
3065 rc = emulate_grp45(ctxt, ops);
3066 if (rc != X86EMUL_CONTINUE)
3069 case 0xff: /* Grp5 */
3070 if (c->modrm_reg == 5)
3074 goto cannot_emulate;
3078 rc = writeback(ctxt, ops);
3079 if (rc != X86EMUL_CONTINUE)
3083 * restore dst type in case the decoding will be reused
3084 * (happens for string instruction )
3086 c->dst.type = saved_dst_type;
3088 if ((c->d & SrcMask) == SrcSI)
3089 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3090 VCPU_REGS_RSI, &c->src);
3092 if ((c->d & DstMask) == DstDI)
3093 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3096 if (c->rep_prefix && (c->d & String)) {
3097 struct read_cache *rc = &ctxt->decode.io_read;
3098 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3100 * Re-enter guest when pio read ahead buffer is empty or,
3101 * if it is not used, after each 1024 iteration.
3103 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3104 (rc->end != 0 && rc->end == rc->pos))
3105 ctxt->restart = false;
3108 * reset read cache here in case string instruction is restared
3111 ctxt->decode.mem_read.end = 0;
3115 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3119 case 0x01: /* lgdt, lidt, lmsw */
3120 switch (c->modrm_reg) {
3122 unsigned long address;
3124 case 0: /* vmcall */
3125 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3126 goto cannot_emulate;
3128 rc = kvm_fix_hypercall(ctxt->vcpu);
3129 if (rc != X86EMUL_CONTINUE)
3132 /* Let the processor re-execute the fixed hypercall */
3134 /* Disable writeback. */
3135 c->dst.type = OP_NONE;
3138 rc = read_descriptor(ctxt, ops, c->src.ptr,
3139 &size, &address, c->op_bytes);
3140 if (rc != X86EMUL_CONTINUE)
3142 realmode_lgdt(ctxt->vcpu, size, address);
3143 /* Disable writeback. */
3144 c->dst.type = OP_NONE;
3146 case 3: /* lidt/vmmcall */
3147 if (c->modrm_mod == 3) {
3148 switch (c->modrm_rm) {
3150 rc = kvm_fix_hypercall(ctxt->vcpu);
3151 if (rc != X86EMUL_CONTINUE)
3155 goto cannot_emulate;
3158 rc = read_descriptor(ctxt, ops, c->src.ptr,
3161 if (rc != X86EMUL_CONTINUE)
3163 realmode_lidt(ctxt->vcpu, size, address);
3165 /* Disable writeback. */
3166 c->dst.type = OP_NONE;
3170 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3173 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3174 (c->src.val & 0x0f), ctxt->vcpu);
3175 c->dst.type = OP_NONE;
3177 case 5: /* not defined */
3181 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3182 /* Disable writeback. */
3183 c->dst.type = OP_NONE;
3186 goto cannot_emulate;
3189 case 0x05: /* syscall */
3190 rc = emulate_syscall(ctxt, ops);
3191 if (rc != X86EMUL_CONTINUE)
3197 emulate_clts(ctxt->vcpu);
3198 c->dst.type = OP_NONE;
3200 case 0x09: /* wbinvd */
3201 kvm_emulate_wbinvd(ctxt->vcpu);
3202 c->dst.type = OP_NONE;
3204 case 0x08: /* invd */
3205 case 0x0d: /* GrpP (prefetch) */
3206 case 0x18: /* Grp16 (prefetch/nop) */
3207 c->dst.type = OP_NONE;
3209 case 0x20: /* mov cr, reg */
3210 switch (c->modrm_reg) {
3217 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3218 c->dst.type = OP_NONE; /* no writeback */
3220 case 0x21: /* mov from dr to reg */
3221 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3222 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3226 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3227 c->dst.type = OP_NONE; /* no writeback */
3229 case 0x22: /* mov reg, cr */
3230 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3231 emulate_gp(ctxt, 0);
3234 c->dst.type = OP_NONE;
3236 case 0x23: /* mov from reg to dr */
3237 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3238 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3243 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3244 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3245 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3246 /* #UD condition is already handled by the code above */
3247 emulate_gp(ctxt, 0);
3251 c->dst.type = OP_NONE; /* no writeback */
3255 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3256 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3257 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3258 emulate_gp(ctxt, 0);
3261 rc = X86EMUL_CONTINUE;
3262 c->dst.type = OP_NONE;
3266 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3267 emulate_gp(ctxt, 0);
3270 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3271 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3273 rc = X86EMUL_CONTINUE;
3274 c->dst.type = OP_NONE;
3276 case 0x34: /* sysenter */
3277 rc = emulate_sysenter(ctxt, ops);
3278 if (rc != X86EMUL_CONTINUE)
3283 case 0x35: /* sysexit */
3284 rc = emulate_sysexit(ctxt, ops);
3285 if (rc != X86EMUL_CONTINUE)
3290 case 0x40 ... 0x4f: /* cmov */
3291 c->dst.val = c->dst.orig_val = c->src.val;
3292 if (!test_cc(c->b, ctxt->eflags))
3293 c->dst.type = OP_NONE; /* no writeback */
3295 case 0x80 ... 0x8f: /* jnz rel, etc*/
3296 if (test_cc(c->b, ctxt->eflags))
3297 jmp_rel(c, c->src.val);
3298 c->dst.type = OP_NONE;
3300 case 0xa0: /* push fs */
3301 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3303 case 0xa1: /* pop fs */
3304 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3305 if (rc != X86EMUL_CONTINUE)
3310 c->dst.type = OP_NONE;
3311 /* only subword offset */
3312 c->src.val &= (c->dst.bytes << 3) - 1;
3313 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3315 case 0xa4: /* shld imm8, r, r/m */
3316 case 0xa5: /* shld cl, r, r/m */
3317 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3319 case 0xa8: /* push gs */
3320 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3322 case 0xa9: /* pop gs */
3323 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3324 if (rc != X86EMUL_CONTINUE)
3329 /* only subword offset */
3330 c->src.val &= (c->dst.bytes << 3) - 1;
3331 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3333 case 0xac: /* shrd imm8, r, r/m */
3334 case 0xad: /* shrd cl, r, r/m */
3335 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3337 case 0xae: /* clflush */
3339 case 0xb0 ... 0xb1: /* cmpxchg */
3341 * Save real source value, then compare EAX against
3344 c->src.orig_val = c->src.val;
3345 c->src.val = c->regs[VCPU_REGS_RAX];
3346 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3347 if (ctxt->eflags & EFLG_ZF) {
3348 /* Success: write back to memory. */
3349 c->dst.val = c->src.orig_val;
3351 /* Failure: write the value we saw to EAX. */
3352 c->dst.type = OP_REG;
3353 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3358 /* only subword offset */
3359 c->src.val &= (c->dst.bytes << 3) - 1;
3360 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3362 case 0xb6 ... 0xb7: /* movzx */
3363 c->dst.bytes = c->op_bytes;
3364 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3367 case 0xba: /* Grp8 */
3368 switch (c->modrm_reg & 3) {
3381 /* only subword offset */
3382 c->src.val &= (c->dst.bytes << 3) - 1;
3383 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3385 case 0xbe ... 0xbf: /* movsx */
3386 c->dst.bytes = c->op_bytes;
3387 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3390 case 0xc3: /* movnti */
3391 c->dst.bytes = c->op_bytes;
3392 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3395 case 0xc7: /* Grp9 (cmpxchg8b) */
3396 rc = emulate_grp9(ctxt, ops);
3397 if (rc != X86EMUL_CONTINUE)
3401 goto cannot_emulate;
3406 DPRINTF("Cannot emulate %02x\n", c->b);