1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<16) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<17) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<17) /* Register operand. */
53 #define DstMem (3<<17) /* Memory operand. */
54 #define DstAcc (4<<17) /* Destination Accumulator */
55 #define DstDI (5<<17) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<17) /* 64bit memory operand */
57 #define DstMask (7<<17)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0x0f /* Group number stored in bits 0:3 */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Mask (7<<29)
98 #define X2(x) (x), (x)
99 #define X3(x) X2(x), (x)
100 #define X4(x) X2(x), X2(x)
101 #define X5(x) X4(x), (x)
102 #define X6(x) X4(x), X2(x)
103 #define X7(x) X4(x), X3(x)
104 #define X8(x) X4(x), X4(x)
105 #define X16(x) X8(x), X8(x)
108 Group1_80, Group1_81, Group1_82, Group1_83,
109 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
113 static u32 opcode_table[256] = {
115 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
116 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
117 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
118 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
120 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
121 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
122 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
123 ImplicitOps | Stack | No64, 0,
125 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
126 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
127 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
128 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
130 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
131 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
132 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
133 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
135 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
136 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
137 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
139 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
140 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
141 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
143 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
144 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
145 ByteOp | DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
147 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
148 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
149 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
158 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
159 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
162 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
163 DstDI | ByteOp | Mov | String, DstDI | Mov | String, /* insb, insw/insd */
164 SrcSI | ByteOp | ImplicitOps | String, SrcSI | ImplicitOps | String, /* outsb, outsw/outsd */
168 Group | Group1_80, Group | Group1_81,
169 Group | Group1_82, Group | Group1_83,
170 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
171 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
173 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
174 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
175 DstMem | SrcNone | ModRM | Mov, ModRM | DstReg,
176 ImplicitOps | SrcMem16 | ModRM, Group | Group1A,
178 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
180 0, 0, SrcImmFAddr | No64, 0,
181 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
183 ByteOp | DstAcc | SrcMem | Mov | MemAbs, DstAcc | SrcMem | Mov | MemAbs,
184 ByteOp | DstMem | SrcAcc | Mov | MemAbs, DstMem | SrcAcc | Mov | MemAbs,
185 ByteOp | SrcSI | DstDI | Mov | String, SrcSI | DstDI | Mov | String,
186 ByteOp | SrcSI | DstDI | String, SrcSI | DstDI | String,
188 DstAcc | SrcImmByte | ByteOp, DstAcc | SrcImm, ByteOp | DstDI | Mov | String, DstDI | Mov | String,
189 ByteOp | SrcSI | DstAcc | Mov | String, SrcSI | DstAcc | Mov | String,
190 ByteOp | DstDI | String, DstDI | String,
192 X8(ByteOp | DstReg | SrcImm | Mov),
194 X8(DstReg | SrcImm | Mov),
196 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
197 0, ImplicitOps | Stack, 0, 0,
198 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
200 0, 0, 0, ImplicitOps | Stack,
201 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
203 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
204 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
207 0, 0, 0, 0, 0, 0, 0, 0,
210 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
211 ByteOp | SrcImmUByte | DstAcc, SrcImmUByte | DstAcc,
213 SrcImm | Stack, SrcImm | ImplicitOps,
214 SrcImmFAddr | No64, SrcImmByte | ImplicitOps,
215 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
216 SrcNone | ByteOp | DstAcc, SrcNone | DstAcc,
219 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
221 ImplicitOps, 0, ImplicitOps, ImplicitOps,
222 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
225 static u32 twobyte_table[256] = {
227 0, Group | GroupDual | Group7, 0, 0,
228 0, ImplicitOps, ImplicitOps | Priv, 0,
229 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
230 0, ImplicitOps | ModRM, 0, 0,
232 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
234 ModRM | ImplicitOps | Priv, ModRM | Priv,
235 ModRM | ImplicitOps | Priv, ModRM | Priv,
237 0, 0, 0, 0, 0, 0, 0, 0,
239 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
240 ImplicitOps, ImplicitOps | Priv, 0, 0,
241 0, 0, 0, 0, 0, 0, 0, 0,
243 X16(DstReg | SrcMem | ModRM | Mov),
245 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
247 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
249 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
253 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
255 ImplicitOps | Stack, ImplicitOps | Stack,
256 0, DstMem | SrcReg | ModRM | BitOp,
257 DstMem | SrcReg | Src2ImmByte | ModRM,
258 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
260 ImplicitOps | Stack, ImplicitOps | Stack,
261 0, DstMem | SrcReg | ModRM | BitOp | Lock,
262 DstMem | SrcReg | Src2ImmByte | ModRM,
263 DstMem | SrcReg | Src2CL | ModRM,
266 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
267 0, DstMem | SrcReg | ModRM | BitOp | Lock,
268 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
269 DstReg | SrcMem16 | ModRM | Mov,
272 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
273 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
274 DstReg | SrcMem16 | ModRM | Mov,
276 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
277 0, 0, 0, Group | GroupDual | Group9,
278 0, 0, 0, 0, 0, 0, 0, 0,
280 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
282 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
284 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
287 static u32 group_table[] = {
289 ByteOp | DstMem | SrcImm | ModRM | Lock,
290 ByteOp | DstMem | SrcImm | ModRM | Lock,
291 ByteOp | DstMem | SrcImm | ModRM | Lock,
292 ByteOp | DstMem | SrcImm | ModRM | Lock,
293 ByteOp | DstMem | SrcImm | ModRM | Lock,
294 ByteOp | DstMem | SrcImm | ModRM | Lock,
295 ByteOp | DstMem | SrcImm | ModRM | Lock,
296 ByteOp | DstMem | SrcImm | ModRM,
298 DstMem | SrcImm | ModRM | Lock,
299 DstMem | SrcImm | ModRM | Lock,
300 DstMem | SrcImm | ModRM | Lock,
301 DstMem | SrcImm | ModRM | Lock,
302 DstMem | SrcImm | ModRM | Lock,
303 DstMem | SrcImm | ModRM | Lock,
304 DstMem | SrcImm | ModRM | Lock,
305 DstMem | SrcImm | ModRM,
307 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
308 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
309 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
310 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
311 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
312 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
313 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
314 ByteOp | DstMem | SrcImm | ModRM | No64,
316 DstMem | SrcImmByte | ModRM | Lock,
317 DstMem | SrcImmByte | ModRM | Lock,
318 DstMem | SrcImmByte | ModRM | Lock,
319 DstMem | SrcImmByte | ModRM | Lock,
320 DstMem | SrcImmByte | ModRM | Lock,
321 DstMem | SrcImmByte | ModRM | Lock,
322 DstMem | SrcImmByte | ModRM | Lock,
323 DstMem | SrcImmByte | ModRM,
325 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
327 ByteOp | SrcImm | DstMem | ModRM, ByteOp | SrcImm | DstMem | ModRM,
328 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
331 DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM,
332 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
335 ByteOp | DstMem | SrcNone | ModRM | Lock, ByteOp | DstMem | SrcNone | ModRM | Lock,
338 DstMem | SrcNone | ModRM | Lock, DstMem | SrcNone | ModRM | Lock,
339 SrcMem | ModRM | Stack, 0,
340 SrcMem | ModRM | Stack, SrcMemFAddr | ModRM | ImplicitOps,
341 SrcMem | ModRM | Stack, 0,
343 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
344 SrcNone | ModRM | DstMem | Mov, 0,
345 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
348 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
349 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
351 0, DstMem64 | ModRM | Lock, 0, 0, 0, 0, 0, 0,
354 static u32 group2_table[] = {
356 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM | Priv,
357 SrcNone | ModRM | DstMem | Mov, 0,
358 SrcMem16 | ModRM | Mov | Priv, 0,
360 0, 0, 0, 0, 0, 0, 0, 0,
363 /* EFLAGS bit definitions. */
364 #define EFLG_ID (1<<21)
365 #define EFLG_VIP (1<<20)
366 #define EFLG_VIF (1<<19)
367 #define EFLG_AC (1<<18)
368 #define EFLG_VM (1<<17)
369 #define EFLG_RF (1<<16)
370 #define EFLG_IOPL (3<<12)
371 #define EFLG_NT (1<<14)
372 #define EFLG_OF (1<<11)
373 #define EFLG_DF (1<<10)
374 #define EFLG_IF (1<<9)
375 #define EFLG_TF (1<<8)
376 #define EFLG_SF (1<<7)
377 #define EFLG_ZF (1<<6)
378 #define EFLG_AF (1<<4)
379 #define EFLG_PF (1<<2)
380 #define EFLG_CF (1<<0)
383 * Instruction emulation:
384 * Most instructions are emulated directly via a fragment of inline assembly
385 * code. This allows us to save/restore EFLAGS and thus very easily pick up
386 * any modified flags.
389 #if defined(CONFIG_X86_64)
390 #define _LO32 "k" /* force 32-bit operand */
391 #define _STK "%%rsp" /* stack pointer */
392 #elif defined(__i386__)
393 #define _LO32 "" /* force 32-bit operand */
394 #define _STK "%%esp" /* stack pointer */
398 * These EFLAGS bits are restored from saved value during emulation, and
399 * any changes are written back to the saved value after emulation.
401 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
403 /* Before executing instruction: restore necessary bits in EFLAGS. */
404 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
405 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
406 "movl %"_sav",%"_LO32 _tmp"; " \
409 "movl %"_msk",%"_LO32 _tmp"; " \
410 "andl %"_LO32 _tmp",("_STK"); " \
412 "notl %"_LO32 _tmp"; " \
413 "andl %"_LO32 _tmp",("_STK"); " \
414 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
416 "orl %"_LO32 _tmp",("_STK"); " \
420 /* After executing instruction: write-back necessary bits in EFLAGS. */
421 #define _POST_EFLAGS(_sav, _msk, _tmp) \
422 /* _sav |= EFLAGS & _msk; */ \
425 "andl %"_msk",%"_LO32 _tmp"; " \
426 "orl %"_LO32 _tmp",%"_sav"; "
434 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
436 __asm__ __volatile__ ( \
437 _PRE_EFLAGS("0", "4", "2") \
438 _op _suffix " %"_x"3,%1; " \
439 _POST_EFLAGS("0", "4", "2") \
440 : "=m" (_eflags), "=m" ((_dst).val), \
442 : _y ((_src).val), "i" (EFLAGS_MASK)); \
446 /* Raw emulation: instruction has two explicit operands. */
447 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
449 unsigned long _tmp; \
451 switch ((_dst).bytes) { \
453 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
456 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
459 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
464 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
466 unsigned long _tmp; \
467 switch ((_dst).bytes) { \
469 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
472 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
473 _wx, _wy, _lx, _ly, _qx, _qy); \
478 /* Source operand is byte-sized and may be restricted to just %cl. */
479 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
480 __emulate_2op(_op, _src, _dst, _eflags, \
481 "b", "c", "b", "c", "b", "c", "b", "c")
483 /* Source operand is byte, word, long or quad sized. */
484 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
485 __emulate_2op(_op, _src, _dst, _eflags, \
486 "b", "q", "w", "r", _LO32, "r", "", "r")
488 /* Source operand is word, long or quad sized. */
489 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
490 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
491 "w", "r", _LO32, "r", "", "r")
493 /* Instruction has three operands and one operand is stored in ECX register */
494 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
496 unsigned long _tmp; \
497 _type _clv = (_cl).val; \
498 _type _srcv = (_src).val; \
499 _type _dstv = (_dst).val; \
501 __asm__ __volatile__ ( \
502 _PRE_EFLAGS("0", "5", "2") \
503 _op _suffix " %4,%1 \n" \
504 _POST_EFLAGS("0", "5", "2") \
505 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
506 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
509 (_cl).val = (unsigned long) _clv; \
510 (_src).val = (unsigned long) _srcv; \
511 (_dst).val = (unsigned long) _dstv; \
514 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
516 switch ((_dst).bytes) { \
518 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
519 "w", unsigned short); \
522 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
523 "l", unsigned int); \
526 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
527 "q", unsigned long)); \
532 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
534 unsigned long _tmp; \
536 __asm__ __volatile__ ( \
537 _PRE_EFLAGS("0", "3", "2") \
538 _op _suffix " %1; " \
539 _POST_EFLAGS("0", "3", "2") \
540 : "=m" (_eflags), "+m" ((_dst).val), \
542 : "i" (EFLAGS_MASK)); \
545 /* Instruction has only one explicit operand (no source operand). */
546 #define emulate_1op(_op, _dst, _eflags) \
548 switch ((_dst).bytes) { \
549 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
550 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
551 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
552 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
556 /* Fetch next part of the instruction being emulated. */
557 #define insn_fetch(_type, _size, _eip) \
558 ({ unsigned long _x; \
559 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
560 if (rc != X86EMUL_CONTINUE) \
566 #define insn_fetch_arr(_arr, _size, _eip) \
567 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
568 if (rc != X86EMUL_CONTINUE) \
573 static inline unsigned long ad_mask(struct decode_cache *c)
575 return (1UL << (c->ad_bytes << 3)) - 1;
578 /* Access/update address held in a register, based on addressing mode. */
579 static inline unsigned long
580 address_mask(struct decode_cache *c, unsigned long reg)
582 if (c->ad_bytes == sizeof(unsigned long))
585 return reg & ad_mask(c);
588 static inline unsigned long
589 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
591 return base + address_mask(c, reg);
595 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
597 if (c->ad_bytes == sizeof(unsigned long))
600 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
603 static inline void jmp_rel(struct decode_cache *c, int rel)
605 register_address_increment(c, &c->eip, rel);
608 static void set_seg_override(struct decode_cache *c, int seg)
610 c->has_seg_override = true;
611 c->seg_override = seg;
614 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
615 struct x86_emulate_ops *ops, int seg)
617 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
620 return ops->get_cached_segment_base(seg, ctxt->vcpu);
623 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
624 struct x86_emulate_ops *ops,
625 struct decode_cache *c)
627 if (!c->has_seg_override)
630 return seg_base(ctxt, ops, c->seg_override);
633 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
634 struct x86_emulate_ops *ops)
636 return seg_base(ctxt, ops, VCPU_SREG_ES);
639 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
640 struct x86_emulate_ops *ops)
642 return seg_base(ctxt, ops, VCPU_SREG_SS);
645 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
646 u32 error, bool valid)
648 ctxt->exception = vec;
649 ctxt->error_code = error;
650 ctxt->error_code_valid = valid;
651 ctxt->restart = false;
654 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
656 emulate_exception(ctxt, GP_VECTOR, err, true);
659 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
663 emulate_exception(ctxt, PF_VECTOR, err, true);
666 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
668 emulate_exception(ctxt, UD_VECTOR, 0, false);
671 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
673 emulate_exception(ctxt, TS_VECTOR, err, true);
676 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
677 struct x86_emulate_ops *ops,
678 unsigned long eip, u8 *dest)
680 struct fetch_cache *fc = &ctxt->decode.fetch;
684 if (eip == fc->end) {
685 cur_size = fc->end - fc->start;
686 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
687 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
688 size, ctxt->vcpu, NULL);
689 if (rc != X86EMUL_CONTINUE)
693 *dest = fc->data[eip - fc->start];
694 return X86EMUL_CONTINUE;
697 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
698 struct x86_emulate_ops *ops,
699 unsigned long eip, void *dest, unsigned size)
703 /* x86 instructions are limited to 15 bytes. */
704 if (eip + size - ctxt->eip > 15)
705 return X86EMUL_UNHANDLEABLE;
707 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
708 if (rc != X86EMUL_CONTINUE)
711 return X86EMUL_CONTINUE;
715 * Given the 'reg' portion of a ModRM byte, and a register block, return a
716 * pointer into the block that addresses the relevant register.
717 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
719 static void *decode_register(u8 modrm_reg, unsigned long *regs,
724 p = ®s[modrm_reg];
725 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
726 p = (unsigned char *)®s[modrm_reg & 3] + 1;
730 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
731 struct x86_emulate_ops *ops,
733 u16 *size, unsigned long *address, int op_bytes)
740 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
742 if (rc != X86EMUL_CONTINUE)
744 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
749 static int test_cc(unsigned int condition, unsigned int flags)
753 switch ((condition & 15) >> 1) {
755 rc |= (flags & EFLG_OF);
757 case 1: /* b/c/nae */
758 rc |= (flags & EFLG_CF);
761 rc |= (flags & EFLG_ZF);
764 rc |= (flags & (EFLG_CF|EFLG_ZF));
767 rc |= (flags & EFLG_SF);
770 rc |= (flags & EFLG_PF);
773 rc |= (flags & EFLG_ZF);
776 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
780 /* Odd condition identifiers (lsb == 1) have inverted sense. */
781 return (!!rc ^ (condition & 1));
784 static void decode_register_operand(struct operand *op,
785 struct decode_cache *c,
788 unsigned reg = c->modrm_reg;
789 int highbyte_regs = c->rex_prefix == 0;
792 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
794 if ((c->d & ByteOp) && !inhibit_bytereg) {
795 op->ptr = decode_register(reg, c->regs, highbyte_regs);
796 op->val = *(u8 *)op->ptr;
799 op->ptr = decode_register(reg, c->regs, 0);
800 op->bytes = c->op_bytes;
803 op->val = *(u16 *)op->ptr;
806 op->val = *(u32 *)op->ptr;
809 op->val = *(u64 *) op->ptr;
813 op->orig_val = op->val;
816 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
817 struct x86_emulate_ops *ops)
819 struct decode_cache *c = &ctxt->decode;
821 int index_reg = 0, base_reg = 0, scale;
822 int rc = X86EMUL_CONTINUE;
825 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
826 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
827 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
830 c->modrm = insn_fetch(u8, 1, c->eip);
831 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
832 c->modrm_reg |= (c->modrm & 0x38) >> 3;
833 c->modrm_rm |= (c->modrm & 0x07);
837 if (c->modrm_mod == 3) {
838 c->modrm_ptr = decode_register(c->modrm_rm,
839 c->regs, c->d & ByteOp);
840 c->modrm_val = *(unsigned long *)c->modrm_ptr;
844 if (c->ad_bytes == 2) {
845 unsigned bx = c->regs[VCPU_REGS_RBX];
846 unsigned bp = c->regs[VCPU_REGS_RBP];
847 unsigned si = c->regs[VCPU_REGS_RSI];
848 unsigned di = c->regs[VCPU_REGS_RDI];
850 /* 16-bit ModR/M decode. */
851 switch (c->modrm_mod) {
853 if (c->modrm_rm == 6)
854 c->modrm_ea += insn_fetch(u16, 2, c->eip);
857 c->modrm_ea += insn_fetch(s8, 1, c->eip);
860 c->modrm_ea += insn_fetch(u16, 2, c->eip);
863 switch (c->modrm_rm) {
865 c->modrm_ea += bx + si;
868 c->modrm_ea += bx + di;
871 c->modrm_ea += bp + si;
874 c->modrm_ea += bp + di;
883 if (c->modrm_mod != 0)
890 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
891 (c->modrm_rm == 6 && c->modrm_mod != 0))
892 if (!c->has_seg_override)
893 set_seg_override(c, VCPU_SREG_SS);
894 c->modrm_ea = (u16)c->modrm_ea;
896 /* 32/64-bit ModR/M decode. */
897 if ((c->modrm_rm & 7) == 4) {
898 sib = insn_fetch(u8, 1, c->eip);
899 index_reg |= (sib >> 3) & 7;
903 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
904 c->modrm_ea += insn_fetch(s32, 4, c->eip);
906 c->modrm_ea += c->regs[base_reg];
908 c->modrm_ea += c->regs[index_reg] << scale;
909 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
910 if (ctxt->mode == X86EMUL_MODE_PROT64)
913 c->modrm_ea += c->regs[c->modrm_rm];
914 switch (c->modrm_mod) {
916 if (c->modrm_rm == 5)
917 c->modrm_ea += insn_fetch(s32, 4, c->eip);
920 c->modrm_ea += insn_fetch(s8, 1, c->eip);
923 c->modrm_ea += insn_fetch(s32, 4, c->eip);
931 static int decode_abs(struct x86_emulate_ctxt *ctxt,
932 struct x86_emulate_ops *ops)
934 struct decode_cache *c = &ctxt->decode;
935 int rc = X86EMUL_CONTINUE;
937 switch (c->ad_bytes) {
939 c->modrm_ea = insn_fetch(u16, 2, c->eip);
942 c->modrm_ea = insn_fetch(u32, 4, c->eip);
945 c->modrm_ea = insn_fetch(u64, 8, c->eip);
953 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
955 struct decode_cache *c = &ctxt->decode;
956 int rc = X86EMUL_CONTINUE;
957 int mode = ctxt->mode;
958 int def_op_bytes, def_ad_bytes, group;
961 /* we cannot decode insn before we complete previous rep insn */
962 WARN_ON(ctxt->restart);
965 c->fetch.start = c->fetch.end = c->eip;
966 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
969 case X86EMUL_MODE_REAL:
970 case X86EMUL_MODE_VM86:
971 case X86EMUL_MODE_PROT16:
972 def_op_bytes = def_ad_bytes = 2;
974 case X86EMUL_MODE_PROT32:
975 def_op_bytes = def_ad_bytes = 4;
978 case X86EMUL_MODE_PROT64:
987 c->op_bytes = def_op_bytes;
988 c->ad_bytes = def_ad_bytes;
990 /* Legacy prefixes. */
992 switch (c->b = insn_fetch(u8, 1, c->eip)) {
993 case 0x66: /* operand-size override */
994 /* switch between 2/4 bytes */
995 c->op_bytes = def_op_bytes ^ 6;
997 case 0x67: /* address-size override */
998 if (mode == X86EMUL_MODE_PROT64)
999 /* switch between 4/8 bytes */
1000 c->ad_bytes = def_ad_bytes ^ 12;
1002 /* switch between 2/4 bytes */
1003 c->ad_bytes = def_ad_bytes ^ 6;
1005 case 0x26: /* ES override */
1006 case 0x2e: /* CS override */
1007 case 0x36: /* SS override */
1008 case 0x3e: /* DS override */
1009 set_seg_override(c, (c->b >> 3) & 3);
1011 case 0x64: /* FS override */
1012 case 0x65: /* GS override */
1013 set_seg_override(c, c->b & 7);
1015 case 0x40 ... 0x4f: /* REX */
1016 if (mode != X86EMUL_MODE_PROT64)
1018 c->rex_prefix = c->b;
1020 case 0xf0: /* LOCK */
1023 case 0xf2: /* REPNE/REPNZ */
1024 c->rep_prefix = REPNE_PREFIX;
1026 case 0xf3: /* REP/REPE/REPZ */
1027 c->rep_prefix = REPE_PREFIX;
1033 /* Any legacy prefix after a REX prefix nullifies its effect. */
1042 if (c->rex_prefix & 8)
1043 c->op_bytes = 8; /* REX.W */
1045 /* Opcode byte(s). */
1046 c->d = opcode_table[c->b];
1048 /* Two-byte opcode? */
1051 c->b = insn_fetch(u8, 1, c->eip);
1052 c->d = twobyte_table[c->b];
1057 group = c->d & GroupMask;
1058 c->modrm = insn_fetch(u8, 1, c->eip);
1061 group = (group << 3) + ((c->modrm >> 3) & 7);
1062 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1063 c->d = group2_table[group];
1065 c->d = group_table[group];
1069 if (c->d == 0 || (c->d & Undefined)) {
1070 DPRINTF("Cannot emulate %02x\n", c->b);
1074 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1077 /* ModRM and SIB bytes. */
1079 rc = decode_modrm(ctxt, ops);
1080 else if (c->d & MemAbs)
1081 rc = decode_abs(ctxt, ops);
1082 if (rc != X86EMUL_CONTINUE)
1085 if (!c->has_seg_override)
1086 set_seg_override(c, VCPU_SREG_DS);
1088 if (!(!c->twobyte && c->b == 0x8d))
1089 c->modrm_ea += seg_override_base(ctxt, ops, c);
1091 if (c->ad_bytes != 8)
1092 c->modrm_ea = (u32)c->modrm_ea;
1094 if (c->rip_relative)
1095 c->modrm_ea += c->eip;
1098 * Decode and fetch the source operand: register, memory
1101 switch (c->d & SrcMask) {
1105 decode_register_operand(&c->src, c, 0);
1114 c->src.bytes = (c->d & ByteOp) ? 1 :
1116 /* Don't fetch the address for invlpg: it could be unmapped. */
1117 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1121 * For instructions with a ModR/M byte, switch to register
1122 * access if Mod = 3.
1124 if ((c->d & ModRM) && c->modrm_mod == 3) {
1125 c->src.type = OP_REG;
1126 c->src.val = c->modrm_val;
1127 c->src.ptr = c->modrm_ptr;
1130 c->src.type = OP_MEM;
1131 c->src.ptr = (unsigned long *)c->modrm_ea;
1136 c->src.type = OP_IMM;
1137 c->src.ptr = (unsigned long *)c->eip;
1138 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1139 if (c->src.bytes == 8)
1141 /* NB. Immediates are sign-extended as necessary. */
1142 switch (c->src.bytes) {
1144 c->src.val = insn_fetch(s8, 1, c->eip);
1147 c->src.val = insn_fetch(s16, 2, c->eip);
1150 c->src.val = insn_fetch(s32, 4, c->eip);
1153 if ((c->d & SrcMask) == SrcImmU) {
1154 switch (c->src.bytes) {
1159 c->src.val &= 0xffff;
1162 c->src.val &= 0xffffffff;
1169 c->src.type = OP_IMM;
1170 c->src.ptr = (unsigned long *)c->eip;
1172 if ((c->d & SrcMask) == SrcImmByte)
1173 c->src.val = insn_fetch(s8, 1, c->eip);
1175 c->src.val = insn_fetch(u8, 1, c->eip);
1178 c->src.type = OP_REG;
1179 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1180 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1181 switch (c->src.bytes) {
1183 c->src.val = *(u8 *)c->src.ptr;
1186 c->src.val = *(u16 *)c->src.ptr;
1189 c->src.val = *(u32 *)c->src.ptr;
1192 c->src.val = *(u64 *)c->src.ptr;
1201 c->src.type = OP_MEM;
1202 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1203 c->src.ptr = (unsigned long *)
1204 register_address(c, seg_override_base(ctxt, ops, c),
1205 c->regs[VCPU_REGS_RSI]);
1209 c->src.type = OP_IMM;
1210 c->src.ptr = (unsigned long *)c->eip;
1211 c->src.bytes = c->op_bytes + 2;
1212 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1215 c->src.type = OP_MEM;
1216 c->src.ptr = (unsigned long *)c->modrm_ea;
1217 c->src.bytes = c->op_bytes + 2;
1222 * Decode and fetch the second source operand: register, memory
1225 switch (c->d & Src2Mask) {
1230 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1233 c->src2.type = OP_IMM;
1234 c->src2.ptr = (unsigned long *)c->eip;
1236 c->src2.val = insn_fetch(u8, 1, c->eip);
1244 /* Decode and fetch the destination operand: register or memory. */
1245 switch (c->d & DstMask) {
1247 /* Special instructions do their own operand decoding. */
1250 decode_register_operand(&c->dst, c,
1251 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1255 if ((c->d & ModRM) && c->modrm_mod == 3) {
1256 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1257 c->dst.type = OP_REG;
1258 c->dst.val = c->dst.orig_val = c->modrm_val;
1259 c->dst.ptr = c->modrm_ptr;
1262 c->dst.type = OP_MEM;
1263 c->dst.ptr = (unsigned long *)c->modrm_ea;
1264 if ((c->d & DstMask) == DstMem64)
1267 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1270 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1272 c->dst.ptr = (void *)c->dst.ptr +
1273 (c->src.val & mask) / 8;
1277 c->dst.type = OP_REG;
1278 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1279 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1280 switch (c->dst.bytes) {
1282 c->dst.val = *(u8 *)c->dst.ptr;
1285 c->dst.val = *(u16 *)c->dst.ptr;
1288 c->dst.val = *(u32 *)c->dst.ptr;
1291 c->dst.val = *(u64 *)c->dst.ptr;
1294 c->dst.orig_val = c->dst.val;
1297 c->dst.type = OP_MEM;
1298 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1299 c->dst.ptr = (unsigned long *)
1300 register_address(c, es_base(ctxt, ops),
1301 c->regs[VCPU_REGS_RDI]);
1307 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1310 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1311 struct x86_emulate_ops *ops,
1312 unsigned long addr, void *dest, unsigned size)
1315 struct read_cache *mc = &ctxt->decode.mem_read;
1319 int n = min(size, 8u);
1321 if (mc->pos < mc->end)
1324 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1326 if (rc == X86EMUL_PROPAGATE_FAULT)
1327 emulate_pf(ctxt, addr, err);
1328 if (rc != X86EMUL_CONTINUE)
1333 memcpy(dest, mc->data + mc->pos, n);
1338 return X86EMUL_CONTINUE;
1341 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1342 struct x86_emulate_ops *ops,
1343 unsigned int size, unsigned short port,
1346 struct read_cache *rc = &ctxt->decode.io_read;
1348 if (rc->pos == rc->end) { /* refill pio read ahead */
1349 struct decode_cache *c = &ctxt->decode;
1350 unsigned int in_page, n;
1351 unsigned int count = c->rep_prefix ?
1352 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1353 in_page = (ctxt->eflags & EFLG_DF) ?
1354 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1355 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1356 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1360 rc->pos = rc->end = 0;
1361 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1366 memcpy(dest, rc->data + rc->pos, size);
1371 static u32 desc_limit_scaled(struct desc_struct *desc)
1373 u32 limit = get_desc_limit(desc);
1375 return desc->g ? (limit << 12) | 0xfff : limit;
1378 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1379 struct x86_emulate_ops *ops,
1380 u16 selector, struct desc_ptr *dt)
1382 if (selector & 1 << 2) {
1383 struct desc_struct desc;
1384 memset (dt, 0, sizeof *dt);
1385 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1388 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1389 dt->address = get_desc_base(&desc);
1391 ops->get_gdt(dt, ctxt->vcpu);
1394 /* allowed just for 8 bytes segments */
1395 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1396 struct x86_emulate_ops *ops,
1397 u16 selector, struct desc_struct *desc)
1400 u16 index = selector >> 3;
1405 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1407 if (dt.size < index * 8 + 7) {
1408 emulate_gp(ctxt, selector & 0xfffc);
1409 return X86EMUL_PROPAGATE_FAULT;
1411 addr = dt.address + index * 8;
1412 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1413 if (ret == X86EMUL_PROPAGATE_FAULT)
1414 emulate_pf(ctxt, addr, err);
1419 /* allowed just for 8 bytes segments */
1420 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1421 struct x86_emulate_ops *ops,
1422 u16 selector, struct desc_struct *desc)
1425 u16 index = selector >> 3;
1430 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1432 if (dt.size < index * 8 + 7) {
1433 emulate_gp(ctxt, selector & 0xfffc);
1434 return X86EMUL_PROPAGATE_FAULT;
1437 addr = dt.address + index * 8;
1438 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1439 if (ret == X86EMUL_PROPAGATE_FAULT)
1440 emulate_pf(ctxt, addr, err);
1445 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1446 struct x86_emulate_ops *ops,
1447 u16 selector, int seg)
1449 struct desc_struct seg_desc;
1451 unsigned err_vec = GP_VECTOR;
1453 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1456 memset(&seg_desc, 0, sizeof seg_desc);
1458 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1459 || ctxt->mode == X86EMUL_MODE_REAL) {
1460 /* set real mode segment descriptor */
1461 set_desc_base(&seg_desc, selector << 4);
1462 set_desc_limit(&seg_desc, 0xffff);
1469 /* NULL selector is not valid for TR, CS and SS */
1470 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1474 /* TR should be in GDT only */
1475 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1478 if (null_selector) /* for NULL selector skip all following checks */
1481 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1482 if (ret != X86EMUL_CONTINUE)
1485 err_code = selector & 0xfffc;
1486 err_vec = GP_VECTOR;
1488 /* can't load system descriptor into segment selecor */
1489 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1493 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1499 cpl = ops->cpl(ctxt->vcpu);
1504 * segment is not a writable data segment or segment
1505 * selector's RPL != CPL or segment selector's RPL != CPL
1507 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1511 if (!(seg_desc.type & 8))
1514 if (seg_desc.type & 4) {
1520 if (rpl > cpl || dpl != cpl)
1523 /* CS(RPL) <- CPL */
1524 selector = (selector & 0xfffc) | cpl;
1527 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1530 case VCPU_SREG_LDTR:
1531 if (seg_desc.s || seg_desc.type != 2)
1534 default: /* DS, ES, FS, or GS */
1536 * segment is not a data or readable code segment or
1537 * ((segment is a data or nonconforming code segment)
1538 * and (both RPL and CPL > DPL))
1540 if ((seg_desc.type & 0xa) == 0x8 ||
1541 (((seg_desc.type & 0xc) != 0xc) &&
1542 (rpl > dpl && cpl > dpl)))
1548 /* mark segment as accessed */
1550 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1551 if (ret != X86EMUL_CONTINUE)
1555 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1556 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1557 return X86EMUL_CONTINUE;
1559 emulate_exception(ctxt, err_vec, err_code, true);
1560 return X86EMUL_PROPAGATE_FAULT;
1563 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1564 struct x86_emulate_ops *ops)
1567 struct decode_cache *c = &ctxt->decode;
1570 switch (c->dst.type) {
1572 /* The 4-byte case *is* correct:
1573 * in 64-bit mode we zero-extend.
1575 switch (c->dst.bytes) {
1577 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1580 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1583 *c->dst.ptr = (u32)c->dst.val;
1584 break; /* 64b: zero-ext */
1586 *c->dst.ptr = c->dst.val;
1592 rc = ops->cmpxchg_emulated(
1593 (unsigned long)c->dst.ptr,
1600 rc = ops->write_emulated(
1601 (unsigned long)c->dst.ptr,
1606 if (rc == X86EMUL_PROPAGATE_FAULT)
1608 (unsigned long)c->dst.ptr, err);
1609 if (rc != X86EMUL_CONTINUE)
1618 return X86EMUL_CONTINUE;
1621 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1622 struct x86_emulate_ops *ops)
1624 struct decode_cache *c = &ctxt->decode;
1626 c->dst.type = OP_MEM;
1627 c->dst.bytes = c->op_bytes;
1628 c->dst.val = c->src.val;
1629 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1630 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1631 c->regs[VCPU_REGS_RSP]);
1634 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1635 struct x86_emulate_ops *ops,
1636 void *dest, int len)
1638 struct decode_cache *c = &ctxt->decode;
1641 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1642 c->regs[VCPU_REGS_RSP]),
1644 if (rc != X86EMUL_CONTINUE)
1647 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1651 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1652 struct x86_emulate_ops *ops,
1653 void *dest, int len)
1656 unsigned long val, change_mask;
1657 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1658 int cpl = ops->cpl(ctxt->vcpu);
1660 rc = emulate_pop(ctxt, ops, &val, len);
1661 if (rc != X86EMUL_CONTINUE)
1664 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1665 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1667 switch(ctxt->mode) {
1668 case X86EMUL_MODE_PROT64:
1669 case X86EMUL_MODE_PROT32:
1670 case X86EMUL_MODE_PROT16:
1672 change_mask |= EFLG_IOPL;
1674 change_mask |= EFLG_IF;
1676 case X86EMUL_MODE_VM86:
1678 emulate_gp(ctxt, 0);
1679 return X86EMUL_PROPAGATE_FAULT;
1681 change_mask |= EFLG_IF;
1683 default: /* real mode */
1684 change_mask |= (EFLG_IOPL | EFLG_IF);
1688 *(unsigned long *)dest =
1689 (ctxt->eflags & ~change_mask) | (val & change_mask);
1694 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1695 struct x86_emulate_ops *ops, int seg)
1697 struct decode_cache *c = &ctxt->decode;
1699 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1701 emulate_push(ctxt, ops);
1704 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1705 struct x86_emulate_ops *ops, int seg)
1707 struct decode_cache *c = &ctxt->decode;
1708 unsigned long selector;
1711 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1712 if (rc != X86EMUL_CONTINUE)
1715 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1719 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1720 struct x86_emulate_ops *ops)
1722 struct decode_cache *c = &ctxt->decode;
1723 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1724 int rc = X86EMUL_CONTINUE;
1725 int reg = VCPU_REGS_RAX;
1727 while (reg <= VCPU_REGS_RDI) {
1728 (reg == VCPU_REGS_RSP) ?
1729 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1731 emulate_push(ctxt, ops);
1733 rc = writeback(ctxt, ops);
1734 if (rc != X86EMUL_CONTINUE)
1740 /* Disable writeback. */
1741 c->dst.type = OP_NONE;
1746 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1747 struct x86_emulate_ops *ops)
1749 struct decode_cache *c = &ctxt->decode;
1750 int rc = X86EMUL_CONTINUE;
1751 int reg = VCPU_REGS_RDI;
1753 while (reg >= VCPU_REGS_RAX) {
1754 if (reg == VCPU_REGS_RSP) {
1755 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1760 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1761 if (rc != X86EMUL_CONTINUE)
1768 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1769 struct x86_emulate_ops *ops)
1771 struct decode_cache *c = &ctxt->decode;
1773 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1776 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1778 struct decode_cache *c = &ctxt->decode;
1779 switch (c->modrm_reg) {
1781 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1784 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1787 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1790 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1792 case 4: /* sal/shl */
1793 case 6: /* sal/shl */
1794 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1797 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1800 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1805 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1806 struct x86_emulate_ops *ops)
1808 struct decode_cache *c = &ctxt->decode;
1810 switch (c->modrm_reg) {
1811 case 0 ... 1: /* test */
1812 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1815 c->dst.val = ~c->dst.val;
1818 emulate_1op("neg", c->dst, ctxt->eflags);
1826 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1827 struct x86_emulate_ops *ops)
1829 struct decode_cache *c = &ctxt->decode;
1831 switch (c->modrm_reg) {
1833 emulate_1op("inc", c->dst, ctxt->eflags);
1836 emulate_1op("dec", c->dst, ctxt->eflags);
1838 case 2: /* call near abs */ {
1841 c->eip = c->src.val;
1842 c->src.val = old_eip;
1843 emulate_push(ctxt, ops);
1846 case 4: /* jmp abs */
1847 c->eip = c->src.val;
1850 emulate_push(ctxt, ops);
1853 return X86EMUL_CONTINUE;
1856 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1857 struct x86_emulate_ops *ops)
1859 struct decode_cache *c = &ctxt->decode;
1860 u64 old = c->dst.orig_val64;
1862 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1863 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1864 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1865 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1866 ctxt->eflags &= ~EFLG_ZF;
1868 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1869 (u32) c->regs[VCPU_REGS_RBX];
1871 ctxt->eflags |= EFLG_ZF;
1873 return X86EMUL_CONTINUE;
1876 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1877 struct x86_emulate_ops *ops)
1879 struct decode_cache *c = &ctxt->decode;
1883 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1884 if (rc != X86EMUL_CONTINUE)
1886 if (c->op_bytes == 4)
1887 c->eip = (u32)c->eip;
1888 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1889 if (rc != X86EMUL_CONTINUE)
1891 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1896 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1897 struct x86_emulate_ops *ops, struct desc_struct *cs,
1898 struct desc_struct *ss)
1900 memset(cs, 0, sizeof(struct desc_struct));
1901 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1902 memset(ss, 0, sizeof(struct desc_struct));
1904 cs->l = 0; /* will be adjusted later */
1905 set_desc_base(cs, 0); /* flat segment */
1906 cs->g = 1; /* 4kb granularity */
1907 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1908 cs->type = 0x0b; /* Read, Execute, Accessed */
1910 cs->dpl = 0; /* will be adjusted later */
1914 set_desc_base(ss, 0); /* flat segment */
1915 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1916 ss->g = 1; /* 4kb granularity */
1918 ss->type = 0x03; /* Read/Write, Accessed */
1919 ss->d = 1; /* 32bit stack segment */
1925 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1927 struct decode_cache *c = &ctxt->decode;
1928 struct desc_struct cs, ss;
1932 /* syscall is not available in real mode */
1933 if (ctxt->mode == X86EMUL_MODE_REAL ||
1934 ctxt->mode == X86EMUL_MODE_VM86) {
1936 return X86EMUL_PROPAGATE_FAULT;
1939 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1941 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1943 cs_sel = (u16)(msr_data & 0xfffc);
1944 ss_sel = (u16)(msr_data + 8);
1946 if (is_long_mode(ctxt->vcpu)) {
1950 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
1951 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1952 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
1953 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1955 c->regs[VCPU_REGS_RCX] = c->eip;
1956 if (is_long_mode(ctxt->vcpu)) {
1957 #ifdef CONFIG_X86_64
1958 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1960 ops->get_msr(ctxt->vcpu,
1961 ctxt->mode == X86EMUL_MODE_PROT64 ?
1962 MSR_LSTAR : MSR_CSTAR, &msr_data);
1965 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1966 ctxt->eflags &= ~(msr_data | EFLG_RF);
1970 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1971 c->eip = (u32)msr_data;
1973 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1976 return X86EMUL_CONTINUE;
1980 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1982 struct decode_cache *c = &ctxt->decode;
1983 struct desc_struct cs, ss;
1987 /* inject #GP if in real mode */
1988 if (ctxt->mode == X86EMUL_MODE_REAL) {
1989 emulate_gp(ctxt, 0);
1990 return X86EMUL_PROPAGATE_FAULT;
1993 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1994 * Therefore, we inject an #UD.
1996 if (ctxt->mode == X86EMUL_MODE_PROT64) {
1998 return X86EMUL_PROPAGATE_FAULT;
2001 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2003 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2004 switch (ctxt->mode) {
2005 case X86EMUL_MODE_PROT32:
2006 if ((msr_data & 0xfffc) == 0x0) {
2007 emulate_gp(ctxt, 0);
2008 return X86EMUL_PROPAGATE_FAULT;
2011 case X86EMUL_MODE_PROT64:
2012 if (msr_data == 0x0) {
2013 emulate_gp(ctxt, 0);
2014 return X86EMUL_PROPAGATE_FAULT;
2019 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2020 cs_sel = (u16)msr_data;
2021 cs_sel &= ~SELECTOR_RPL_MASK;
2022 ss_sel = cs_sel + 8;
2023 ss_sel &= ~SELECTOR_RPL_MASK;
2024 if (ctxt->mode == X86EMUL_MODE_PROT64
2025 || is_long_mode(ctxt->vcpu)) {
2030 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2031 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2032 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2033 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2035 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2038 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2039 c->regs[VCPU_REGS_RSP] = msr_data;
2041 return X86EMUL_CONTINUE;
2045 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2047 struct decode_cache *c = &ctxt->decode;
2048 struct desc_struct cs, ss;
2053 /* inject #GP if in real mode or Virtual 8086 mode */
2054 if (ctxt->mode == X86EMUL_MODE_REAL ||
2055 ctxt->mode == X86EMUL_MODE_VM86) {
2056 emulate_gp(ctxt, 0);
2057 return X86EMUL_PROPAGATE_FAULT;
2060 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2062 if ((c->rex_prefix & 0x8) != 0x0)
2063 usermode = X86EMUL_MODE_PROT64;
2065 usermode = X86EMUL_MODE_PROT32;
2069 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2071 case X86EMUL_MODE_PROT32:
2072 cs_sel = (u16)(msr_data + 16);
2073 if ((msr_data & 0xfffc) == 0x0) {
2074 emulate_gp(ctxt, 0);
2075 return X86EMUL_PROPAGATE_FAULT;
2077 ss_sel = (u16)(msr_data + 24);
2079 case X86EMUL_MODE_PROT64:
2080 cs_sel = (u16)(msr_data + 32);
2081 if (msr_data == 0x0) {
2082 emulate_gp(ctxt, 0);
2083 return X86EMUL_PROPAGATE_FAULT;
2085 ss_sel = cs_sel + 8;
2090 cs_sel |= SELECTOR_RPL_MASK;
2091 ss_sel |= SELECTOR_RPL_MASK;
2093 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2094 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2095 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2096 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2098 c->eip = c->regs[VCPU_REGS_RDX];
2099 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2101 return X86EMUL_CONTINUE;
2104 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2105 struct x86_emulate_ops *ops)
2108 if (ctxt->mode == X86EMUL_MODE_REAL)
2110 if (ctxt->mode == X86EMUL_MODE_VM86)
2112 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2113 return ops->cpl(ctxt->vcpu) > iopl;
2116 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2117 struct x86_emulate_ops *ops,
2120 struct desc_struct tr_seg;
2123 u8 perm, bit_idx = port & 0x7;
2124 unsigned mask = (1 << len) - 1;
2126 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2129 if (desc_limit_scaled(&tr_seg) < 103)
2131 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2133 if (r != X86EMUL_CONTINUE)
2135 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2137 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2138 &perm, 1, ctxt->vcpu, NULL);
2139 if (r != X86EMUL_CONTINUE)
2141 if ((perm >> bit_idx) & mask)
2146 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2147 struct x86_emulate_ops *ops,
2150 if (emulator_bad_iopl(ctxt, ops))
2151 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2156 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2157 struct x86_emulate_ops *ops,
2158 struct tss_segment_16 *tss)
2160 struct decode_cache *c = &ctxt->decode;
2163 tss->flag = ctxt->eflags;
2164 tss->ax = c->regs[VCPU_REGS_RAX];
2165 tss->cx = c->regs[VCPU_REGS_RCX];
2166 tss->dx = c->regs[VCPU_REGS_RDX];
2167 tss->bx = c->regs[VCPU_REGS_RBX];
2168 tss->sp = c->regs[VCPU_REGS_RSP];
2169 tss->bp = c->regs[VCPU_REGS_RBP];
2170 tss->si = c->regs[VCPU_REGS_RSI];
2171 tss->di = c->regs[VCPU_REGS_RDI];
2173 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2174 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2175 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2176 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2177 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2180 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2181 struct x86_emulate_ops *ops,
2182 struct tss_segment_16 *tss)
2184 struct decode_cache *c = &ctxt->decode;
2188 ctxt->eflags = tss->flag | 2;
2189 c->regs[VCPU_REGS_RAX] = tss->ax;
2190 c->regs[VCPU_REGS_RCX] = tss->cx;
2191 c->regs[VCPU_REGS_RDX] = tss->dx;
2192 c->regs[VCPU_REGS_RBX] = tss->bx;
2193 c->regs[VCPU_REGS_RSP] = tss->sp;
2194 c->regs[VCPU_REGS_RBP] = tss->bp;
2195 c->regs[VCPU_REGS_RSI] = tss->si;
2196 c->regs[VCPU_REGS_RDI] = tss->di;
2199 * SDM says that segment selectors are loaded before segment
2202 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2203 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2204 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2205 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2206 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2209 * Now load segment descriptors. If fault happenes at this stage
2210 * it is handled in a context of new task
2212 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2213 if (ret != X86EMUL_CONTINUE)
2215 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2216 if (ret != X86EMUL_CONTINUE)
2218 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2219 if (ret != X86EMUL_CONTINUE)
2221 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2222 if (ret != X86EMUL_CONTINUE)
2224 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2225 if (ret != X86EMUL_CONTINUE)
2228 return X86EMUL_CONTINUE;
2231 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2232 struct x86_emulate_ops *ops,
2233 u16 tss_selector, u16 old_tss_sel,
2234 ulong old_tss_base, struct desc_struct *new_desc)
2236 struct tss_segment_16 tss_seg;
2238 u32 err, new_tss_base = get_desc_base(new_desc);
2240 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2242 if (ret == X86EMUL_PROPAGATE_FAULT) {
2243 /* FIXME: need to provide precise fault address */
2244 emulate_pf(ctxt, old_tss_base, err);
2248 save_state_to_tss16(ctxt, ops, &tss_seg);
2250 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2252 if (ret == X86EMUL_PROPAGATE_FAULT) {
2253 /* FIXME: need to provide precise fault address */
2254 emulate_pf(ctxt, old_tss_base, err);
2258 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2260 if (ret == X86EMUL_PROPAGATE_FAULT) {
2261 /* FIXME: need to provide precise fault address */
2262 emulate_pf(ctxt, new_tss_base, err);
2266 if (old_tss_sel != 0xffff) {
2267 tss_seg.prev_task_link = old_tss_sel;
2269 ret = ops->write_std(new_tss_base,
2270 &tss_seg.prev_task_link,
2271 sizeof tss_seg.prev_task_link,
2273 if (ret == X86EMUL_PROPAGATE_FAULT) {
2274 /* FIXME: need to provide precise fault address */
2275 emulate_pf(ctxt, new_tss_base, err);
2280 return load_state_from_tss16(ctxt, ops, &tss_seg);
2283 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2284 struct x86_emulate_ops *ops,
2285 struct tss_segment_32 *tss)
2287 struct decode_cache *c = &ctxt->decode;
2289 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2291 tss->eflags = ctxt->eflags;
2292 tss->eax = c->regs[VCPU_REGS_RAX];
2293 tss->ecx = c->regs[VCPU_REGS_RCX];
2294 tss->edx = c->regs[VCPU_REGS_RDX];
2295 tss->ebx = c->regs[VCPU_REGS_RBX];
2296 tss->esp = c->regs[VCPU_REGS_RSP];
2297 tss->ebp = c->regs[VCPU_REGS_RBP];
2298 tss->esi = c->regs[VCPU_REGS_RSI];
2299 tss->edi = c->regs[VCPU_REGS_RDI];
2301 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2302 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2303 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2304 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2305 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2306 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2307 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2310 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2311 struct x86_emulate_ops *ops,
2312 struct tss_segment_32 *tss)
2314 struct decode_cache *c = &ctxt->decode;
2317 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2318 emulate_gp(ctxt, 0);
2319 return X86EMUL_PROPAGATE_FAULT;
2322 ctxt->eflags = tss->eflags | 2;
2323 c->regs[VCPU_REGS_RAX] = tss->eax;
2324 c->regs[VCPU_REGS_RCX] = tss->ecx;
2325 c->regs[VCPU_REGS_RDX] = tss->edx;
2326 c->regs[VCPU_REGS_RBX] = tss->ebx;
2327 c->regs[VCPU_REGS_RSP] = tss->esp;
2328 c->regs[VCPU_REGS_RBP] = tss->ebp;
2329 c->regs[VCPU_REGS_RSI] = tss->esi;
2330 c->regs[VCPU_REGS_RDI] = tss->edi;
2333 * SDM says that segment selectors are loaded before segment
2336 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2337 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2338 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2339 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2340 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2341 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2342 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2345 * Now load segment descriptors. If fault happenes at this stage
2346 * it is handled in a context of new task
2348 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2349 if (ret != X86EMUL_CONTINUE)
2351 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2352 if (ret != X86EMUL_CONTINUE)
2354 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2355 if (ret != X86EMUL_CONTINUE)
2357 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2358 if (ret != X86EMUL_CONTINUE)
2360 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2361 if (ret != X86EMUL_CONTINUE)
2363 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2364 if (ret != X86EMUL_CONTINUE)
2366 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2367 if (ret != X86EMUL_CONTINUE)
2370 return X86EMUL_CONTINUE;
2373 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2374 struct x86_emulate_ops *ops,
2375 u16 tss_selector, u16 old_tss_sel,
2376 ulong old_tss_base, struct desc_struct *new_desc)
2378 struct tss_segment_32 tss_seg;
2380 u32 err, new_tss_base = get_desc_base(new_desc);
2382 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2384 if (ret == X86EMUL_PROPAGATE_FAULT) {
2385 /* FIXME: need to provide precise fault address */
2386 emulate_pf(ctxt, old_tss_base, err);
2390 save_state_to_tss32(ctxt, ops, &tss_seg);
2392 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2394 if (ret == X86EMUL_PROPAGATE_FAULT) {
2395 /* FIXME: need to provide precise fault address */
2396 emulate_pf(ctxt, old_tss_base, err);
2400 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2402 if (ret == X86EMUL_PROPAGATE_FAULT) {
2403 /* FIXME: need to provide precise fault address */
2404 emulate_pf(ctxt, new_tss_base, err);
2408 if (old_tss_sel != 0xffff) {
2409 tss_seg.prev_task_link = old_tss_sel;
2411 ret = ops->write_std(new_tss_base,
2412 &tss_seg.prev_task_link,
2413 sizeof tss_seg.prev_task_link,
2415 if (ret == X86EMUL_PROPAGATE_FAULT) {
2416 /* FIXME: need to provide precise fault address */
2417 emulate_pf(ctxt, new_tss_base, err);
2422 return load_state_from_tss32(ctxt, ops, &tss_seg);
2425 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2426 struct x86_emulate_ops *ops,
2427 u16 tss_selector, int reason,
2428 bool has_error_code, u32 error_code)
2430 struct desc_struct curr_tss_desc, next_tss_desc;
2432 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2433 ulong old_tss_base =
2434 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2437 /* FIXME: old_tss_base == ~0 ? */
2439 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2440 if (ret != X86EMUL_CONTINUE)
2442 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2443 if (ret != X86EMUL_CONTINUE)
2446 /* FIXME: check that next_tss_desc is tss */
2448 if (reason != TASK_SWITCH_IRET) {
2449 if ((tss_selector & 3) > next_tss_desc.dpl ||
2450 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2451 emulate_gp(ctxt, 0);
2452 return X86EMUL_PROPAGATE_FAULT;
2456 desc_limit = desc_limit_scaled(&next_tss_desc);
2457 if (!next_tss_desc.p ||
2458 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2459 desc_limit < 0x2b)) {
2460 emulate_ts(ctxt, tss_selector & 0xfffc);
2461 return X86EMUL_PROPAGATE_FAULT;
2464 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2465 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2466 write_segment_descriptor(ctxt, ops, old_tss_sel,
2470 if (reason == TASK_SWITCH_IRET)
2471 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2473 /* set back link to prev task only if NT bit is set in eflags
2474 note that old_tss_sel is not used afetr this point */
2475 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2476 old_tss_sel = 0xffff;
2478 if (next_tss_desc.type & 8)
2479 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2480 old_tss_base, &next_tss_desc);
2482 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2483 old_tss_base, &next_tss_desc);
2484 if (ret != X86EMUL_CONTINUE)
2487 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2488 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2490 if (reason != TASK_SWITCH_IRET) {
2491 next_tss_desc.type |= (1 << 1); /* set busy flag */
2492 write_segment_descriptor(ctxt, ops, tss_selector,
2496 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2497 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2498 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2500 if (has_error_code) {
2501 struct decode_cache *c = &ctxt->decode;
2503 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2505 c->src.val = (unsigned long) error_code;
2506 emulate_push(ctxt, ops);
2512 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2513 struct x86_emulate_ops *ops,
2514 u16 tss_selector, int reason,
2515 bool has_error_code, u32 error_code)
2517 struct decode_cache *c = &ctxt->decode;
2521 c->dst.type = OP_NONE;
2523 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2524 has_error_code, error_code);
2526 if (rc == X86EMUL_CONTINUE) {
2527 rc = writeback(ctxt, ops);
2528 if (rc == X86EMUL_CONTINUE)
2532 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2535 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2536 int reg, struct operand *op)
2538 struct decode_cache *c = &ctxt->decode;
2539 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2541 register_address_increment(c, &c->regs[reg], df * op->bytes);
2542 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2546 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2549 struct decode_cache *c = &ctxt->decode;
2550 int rc = X86EMUL_CONTINUE;
2551 int saved_dst_type = c->dst.type;
2553 ctxt->decode.mem_read.pos = 0;
2555 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2560 /* LOCK prefix is allowed only with some instructions */
2561 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2566 /* Privileged instruction can be executed only in CPL=0 */
2567 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2568 emulate_gp(ctxt, 0);
2572 if (c->rep_prefix && (c->d & String)) {
2573 ctxt->restart = true;
2574 /* All REP prefixes have the same first termination condition */
2575 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2577 ctxt->restart = false;
2581 /* The second termination condition only applies for REPE
2582 * and REPNE. Test if the repeat string operation prefix is
2583 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2584 * corresponding termination condition according to:
2585 * - if REPE/REPZ and ZF = 0 then done
2586 * - if REPNE/REPNZ and ZF = 1 then done
2588 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2589 (c->b == 0xae) || (c->b == 0xaf)) {
2590 if ((c->rep_prefix == REPE_PREFIX) &&
2591 ((ctxt->eflags & EFLG_ZF) == 0))
2593 if ((c->rep_prefix == REPNE_PREFIX) &&
2594 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2600 if (c->src.type == OP_MEM) {
2601 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2602 c->src.valptr, c->src.bytes);
2603 if (rc != X86EMUL_CONTINUE)
2605 c->src.orig_val64 = c->src.val64;
2608 if (c->src2.type == OP_MEM) {
2609 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2610 &c->src2.val, c->src2.bytes);
2611 if (rc != X86EMUL_CONTINUE)
2615 if ((c->d & DstMask) == ImplicitOps)
2619 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2620 /* optimisation - avoid slow emulated read if Mov */
2621 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2622 &c->dst.val, c->dst.bytes);
2623 if (rc != X86EMUL_CONTINUE)
2626 c->dst.orig_val = c->dst.val;
2636 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2638 case 0x06: /* push es */
2639 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2641 case 0x07: /* pop es */
2642 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2643 if (rc != X86EMUL_CONTINUE)
2648 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2650 case 0x0e: /* push cs */
2651 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2655 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2657 case 0x16: /* push ss */
2658 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2660 case 0x17: /* pop ss */
2661 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2662 if (rc != X86EMUL_CONTINUE)
2667 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2669 case 0x1e: /* push ds */
2670 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2672 case 0x1f: /* pop ds */
2673 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2674 if (rc != X86EMUL_CONTINUE)
2679 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2683 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2687 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2691 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2693 case 0x40 ... 0x47: /* inc r16/r32 */
2694 emulate_1op("inc", c->dst, ctxt->eflags);
2696 case 0x48 ... 0x4f: /* dec r16/r32 */
2697 emulate_1op("dec", c->dst, ctxt->eflags);
2699 case 0x50 ... 0x57: /* push reg */
2700 emulate_push(ctxt, ops);
2702 case 0x58 ... 0x5f: /* pop reg */
2704 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2705 if (rc != X86EMUL_CONTINUE)
2708 case 0x60: /* pusha */
2709 rc = emulate_pusha(ctxt, ops);
2710 if (rc != X86EMUL_CONTINUE)
2713 case 0x61: /* popa */
2714 rc = emulate_popa(ctxt, ops);
2715 if (rc != X86EMUL_CONTINUE)
2718 case 0x63: /* movsxd */
2719 if (ctxt->mode != X86EMUL_MODE_PROT64)
2720 goto cannot_emulate;
2721 c->dst.val = (s32) c->src.val;
2723 case 0x68: /* push imm */
2724 case 0x6a: /* push imm8 */
2725 emulate_push(ctxt, ops);
2727 case 0x6c: /* insb */
2728 case 0x6d: /* insw/insd */
2729 c->dst.bytes = min(c->dst.bytes, 4u);
2730 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2732 emulate_gp(ctxt, 0);
2735 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2736 c->regs[VCPU_REGS_RDX], &c->dst.val))
2737 goto done; /* IO is needed, skip writeback */
2739 case 0x6e: /* outsb */
2740 case 0x6f: /* outsw/outsd */
2741 c->src.bytes = min(c->src.bytes, 4u);
2742 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2744 emulate_gp(ctxt, 0);
2747 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2748 &c->src.val, 1, ctxt->vcpu);
2750 c->dst.type = OP_NONE; /* nothing to writeback */
2752 case 0x70 ... 0x7f: /* jcc (short) */
2753 if (test_cc(c->b, ctxt->eflags))
2754 jmp_rel(c, c->src.val);
2756 case 0x80 ... 0x83: /* Grp1 */
2757 switch (c->modrm_reg) {
2778 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2780 case 0x86 ... 0x87: /* xchg */
2782 /* Write back the register source. */
2783 switch (c->dst.bytes) {
2785 *(u8 *) c->src.ptr = (u8) c->dst.val;
2788 *(u16 *) c->src.ptr = (u16) c->dst.val;
2791 *c->src.ptr = (u32) c->dst.val;
2792 break; /* 64b reg: zero-extend */
2794 *c->src.ptr = c->dst.val;
2798 * Write back the memory destination with implicit LOCK
2801 c->dst.val = c->src.val;
2804 case 0x88 ... 0x8b: /* mov */
2806 case 0x8c: /* mov r/m, sreg */
2807 if (c->modrm_reg > VCPU_SREG_GS) {
2811 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2813 case 0x8d: /* lea r16/r32, m */
2814 c->dst.val = c->modrm_ea;
2816 case 0x8e: { /* mov seg, r/m16 */
2821 if (c->modrm_reg == VCPU_SREG_CS ||
2822 c->modrm_reg > VCPU_SREG_GS) {
2827 if (c->modrm_reg == VCPU_SREG_SS)
2828 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2830 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2832 c->dst.type = OP_NONE; /* Disable writeback. */
2835 case 0x8f: /* pop (sole member of Grp1a) */
2836 rc = emulate_grp1a(ctxt, ops);
2837 if (rc != X86EMUL_CONTINUE)
2840 case 0x90: /* nop / xchg r8,rax */
2841 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2842 c->dst.type = OP_NONE; /* nop */
2845 case 0x91 ... 0x97: /* xchg reg,rax */
2846 c->src.type = OP_REG;
2847 c->src.bytes = c->op_bytes;
2848 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2849 c->src.val = *(c->src.ptr);
2851 case 0x9c: /* pushf */
2852 c->src.val = (unsigned long) ctxt->eflags;
2853 emulate_push(ctxt, ops);
2855 case 0x9d: /* popf */
2856 c->dst.type = OP_REG;
2857 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2858 c->dst.bytes = c->op_bytes;
2859 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2860 if (rc != X86EMUL_CONTINUE)
2863 case 0xa0 ... 0xa3: /* mov */
2864 case 0xa4 ... 0xa5: /* movs */
2866 case 0xa6 ... 0xa7: /* cmps */
2867 c->dst.type = OP_NONE; /* Disable writeback. */
2868 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2870 case 0xa8 ... 0xa9: /* test ax, imm */
2872 case 0xaa ... 0xab: /* stos */
2873 c->dst.val = c->regs[VCPU_REGS_RAX];
2875 case 0xac ... 0xad: /* lods */
2877 case 0xae ... 0xaf: /* scas */
2878 DPRINTF("Urk! I don't handle SCAS.\n");
2879 goto cannot_emulate;
2880 case 0xb0 ... 0xbf: /* mov r, imm */
2885 case 0xc3: /* ret */
2886 c->dst.type = OP_REG;
2887 c->dst.ptr = &c->eip;
2888 c->dst.bytes = c->op_bytes;
2889 goto pop_instruction;
2890 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2892 c->dst.val = c->src.val;
2894 case 0xcb: /* ret far */
2895 rc = emulate_ret_far(ctxt, ops);
2896 if (rc != X86EMUL_CONTINUE)
2899 case 0xd0 ... 0xd1: /* Grp2 */
2903 case 0xd2 ... 0xd3: /* Grp2 */
2904 c->src.val = c->regs[VCPU_REGS_RCX];
2907 case 0xe4: /* inb */
2910 case 0xe6: /* outb */
2911 case 0xe7: /* out */
2913 case 0xe8: /* call (near) */ {
2914 long int rel = c->src.val;
2915 c->src.val = (unsigned long) c->eip;
2917 emulate_push(ctxt, ops);
2920 case 0xe9: /* jmp rel */
2922 case 0xea: { /* jmp far */
2925 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2927 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
2931 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2935 jmp: /* jmp rel short */
2936 jmp_rel(c, c->src.val);
2937 c->dst.type = OP_NONE; /* Disable writeback. */
2939 case 0xec: /* in al,dx */
2940 case 0xed: /* in (e/r)ax,dx */
2941 c->src.val = c->regs[VCPU_REGS_RDX];
2943 c->dst.bytes = min(c->dst.bytes, 4u);
2944 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2945 emulate_gp(ctxt, 0);
2948 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
2950 goto done; /* IO is needed */
2952 case 0xee: /* out dx,al */
2953 case 0xef: /* out dx,(e/r)ax */
2954 c->src.val = c->regs[VCPU_REGS_RDX];
2956 c->dst.bytes = min(c->dst.bytes, 4u);
2957 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
2958 emulate_gp(ctxt, 0);
2961 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
2963 c->dst.type = OP_NONE; /* Disable writeback. */
2965 case 0xf4: /* hlt */
2966 ctxt->vcpu->arch.halt_request = 1;
2968 case 0xf5: /* cmc */
2969 /* complement carry flag from eflags reg */
2970 ctxt->eflags ^= EFLG_CF;
2971 c->dst.type = OP_NONE; /* Disable writeback. */
2973 case 0xf6 ... 0xf7: /* Grp3 */
2974 if (!emulate_grp3(ctxt, ops))
2975 goto cannot_emulate;
2977 case 0xf8: /* clc */
2978 ctxt->eflags &= ~EFLG_CF;
2979 c->dst.type = OP_NONE; /* Disable writeback. */
2981 case 0xfa: /* cli */
2982 if (emulator_bad_iopl(ctxt, ops)) {
2983 emulate_gp(ctxt, 0);
2986 ctxt->eflags &= ~X86_EFLAGS_IF;
2987 c->dst.type = OP_NONE; /* Disable writeback. */
2990 case 0xfb: /* sti */
2991 if (emulator_bad_iopl(ctxt, ops)) {
2992 emulate_gp(ctxt, 0);
2995 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2996 ctxt->eflags |= X86_EFLAGS_IF;
2997 c->dst.type = OP_NONE; /* Disable writeback. */
3000 case 0xfc: /* cld */
3001 ctxt->eflags &= ~EFLG_DF;
3002 c->dst.type = OP_NONE; /* Disable writeback. */
3004 case 0xfd: /* std */
3005 ctxt->eflags |= EFLG_DF;
3006 c->dst.type = OP_NONE; /* Disable writeback. */
3008 case 0xfe: /* Grp4 */
3010 rc = emulate_grp45(ctxt, ops);
3011 if (rc != X86EMUL_CONTINUE)
3014 case 0xff: /* Grp5 */
3015 if (c->modrm_reg == 5)
3019 goto cannot_emulate;
3023 rc = writeback(ctxt, ops);
3024 if (rc != X86EMUL_CONTINUE)
3028 * restore dst type in case the decoding will be reused
3029 * (happens for string instruction )
3031 c->dst.type = saved_dst_type;
3033 if ((c->d & SrcMask) == SrcSI)
3034 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3035 VCPU_REGS_RSI, &c->src);
3037 if ((c->d & DstMask) == DstDI)
3038 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3041 if (c->rep_prefix && (c->d & String)) {
3042 struct read_cache *rc = &ctxt->decode.io_read;
3043 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3045 * Re-enter guest when pio read ahead buffer is empty or,
3046 * if it is not used, after each 1024 iteration.
3048 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3049 (rc->end != 0 && rc->end == rc->pos))
3050 ctxt->restart = false;
3053 * reset read cache here in case string instruction is restared
3056 ctxt->decode.mem_read.end = 0;
3060 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3064 case 0x01: /* lgdt, lidt, lmsw */
3065 switch (c->modrm_reg) {
3067 unsigned long address;
3069 case 0: /* vmcall */
3070 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3071 goto cannot_emulate;
3073 rc = kvm_fix_hypercall(ctxt->vcpu);
3074 if (rc != X86EMUL_CONTINUE)
3077 /* Let the processor re-execute the fixed hypercall */
3079 /* Disable writeback. */
3080 c->dst.type = OP_NONE;
3083 rc = read_descriptor(ctxt, ops, c->src.ptr,
3084 &size, &address, c->op_bytes);
3085 if (rc != X86EMUL_CONTINUE)
3087 realmode_lgdt(ctxt->vcpu, size, address);
3088 /* Disable writeback. */
3089 c->dst.type = OP_NONE;
3091 case 3: /* lidt/vmmcall */
3092 if (c->modrm_mod == 3) {
3093 switch (c->modrm_rm) {
3095 rc = kvm_fix_hypercall(ctxt->vcpu);
3096 if (rc != X86EMUL_CONTINUE)
3100 goto cannot_emulate;
3103 rc = read_descriptor(ctxt, ops, c->src.ptr,
3106 if (rc != X86EMUL_CONTINUE)
3108 realmode_lidt(ctxt->vcpu, size, address);
3110 /* Disable writeback. */
3111 c->dst.type = OP_NONE;
3115 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3118 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3119 (c->src.val & 0x0f), ctxt->vcpu);
3120 c->dst.type = OP_NONE;
3122 case 5: /* not defined */
3126 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3127 /* Disable writeback. */
3128 c->dst.type = OP_NONE;
3131 goto cannot_emulate;
3134 case 0x05: /* syscall */
3135 rc = emulate_syscall(ctxt, ops);
3136 if (rc != X86EMUL_CONTINUE)
3142 emulate_clts(ctxt->vcpu);
3143 c->dst.type = OP_NONE;
3145 case 0x09: /* wbinvd */
3146 kvm_emulate_wbinvd(ctxt->vcpu);
3147 c->dst.type = OP_NONE;
3149 case 0x08: /* invd */
3150 case 0x0d: /* GrpP (prefetch) */
3151 case 0x18: /* Grp16 (prefetch/nop) */
3152 c->dst.type = OP_NONE;
3154 case 0x20: /* mov cr, reg */
3155 switch (c->modrm_reg) {
3162 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3163 c->dst.type = OP_NONE; /* no writeback */
3165 case 0x21: /* mov from dr to reg */
3166 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3167 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3171 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3172 c->dst.type = OP_NONE; /* no writeback */
3174 case 0x22: /* mov reg, cr */
3175 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3176 emulate_gp(ctxt, 0);
3179 c->dst.type = OP_NONE;
3181 case 0x23: /* mov from reg to dr */
3182 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3183 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3188 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3189 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3190 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3191 /* #UD condition is already handled by the code above */
3192 emulate_gp(ctxt, 0);
3196 c->dst.type = OP_NONE; /* no writeback */
3200 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3201 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3202 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3203 emulate_gp(ctxt, 0);
3206 rc = X86EMUL_CONTINUE;
3207 c->dst.type = OP_NONE;
3211 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3212 emulate_gp(ctxt, 0);
3215 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3216 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3218 rc = X86EMUL_CONTINUE;
3219 c->dst.type = OP_NONE;
3221 case 0x34: /* sysenter */
3222 rc = emulate_sysenter(ctxt, ops);
3223 if (rc != X86EMUL_CONTINUE)
3228 case 0x35: /* sysexit */
3229 rc = emulate_sysexit(ctxt, ops);
3230 if (rc != X86EMUL_CONTINUE)
3235 case 0x40 ... 0x4f: /* cmov */
3236 c->dst.val = c->dst.orig_val = c->src.val;
3237 if (!test_cc(c->b, ctxt->eflags))
3238 c->dst.type = OP_NONE; /* no writeback */
3240 case 0x80 ... 0x8f: /* jnz rel, etc*/
3241 if (test_cc(c->b, ctxt->eflags))
3242 jmp_rel(c, c->src.val);
3243 c->dst.type = OP_NONE;
3245 case 0xa0: /* push fs */
3246 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3248 case 0xa1: /* pop fs */
3249 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3250 if (rc != X86EMUL_CONTINUE)
3255 c->dst.type = OP_NONE;
3256 /* only subword offset */
3257 c->src.val &= (c->dst.bytes << 3) - 1;
3258 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3260 case 0xa4: /* shld imm8, r, r/m */
3261 case 0xa5: /* shld cl, r, r/m */
3262 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3264 case 0xa8: /* push gs */
3265 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3267 case 0xa9: /* pop gs */
3268 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3269 if (rc != X86EMUL_CONTINUE)
3274 /* only subword offset */
3275 c->src.val &= (c->dst.bytes << 3) - 1;
3276 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3278 case 0xac: /* shrd imm8, r, r/m */
3279 case 0xad: /* shrd cl, r, r/m */
3280 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3282 case 0xae: /* clflush */
3284 case 0xb0 ... 0xb1: /* cmpxchg */
3286 * Save real source value, then compare EAX against
3289 c->src.orig_val = c->src.val;
3290 c->src.val = c->regs[VCPU_REGS_RAX];
3291 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3292 if (ctxt->eflags & EFLG_ZF) {
3293 /* Success: write back to memory. */
3294 c->dst.val = c->src.orig_val;
3296 /* Failure: write the value we saw to EAX. */
3297 c->dst.type = OP_REG;
3298 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3303 /* only subword offset */
3304 c->src.val &= (c->dst.bytes << 3) - 1;
3305 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3307 case 0xb6 ... 0xb7: /* movzx */
3308 c->dst.bytes = c->op_bytes;
3309 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3312 case 0xba: /* Grp8 */
3313 switch (c->modrm_reg & 3) {
3326 /* only subword offset */
3327 c->src.val &= (c->dst.bytes << 3) - 1;
3328 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3330 case 0xbe ... 0xbf: /* movsx */
3331 c->dst.bytes = c->op_bytes;
3332 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3335 case 0xc3: /* movnti */
3336 c->dst.bytes = c->op_bytes;
3337 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3340 case 0xc7: /* Grp9 (cmpxchg8b) */
3341 rc = emulate_grp9(ctxt, ops);
3342 if (rc != X86EMUL_CONTINUE)
3346 goto cannot_emulate;
3351 DPRINTF("Cannot emulate %02x\n", c->b);