1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
19 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
25 #include <public/xen.h>
26 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
28 #include <linux/kvm_host.h>
29 #include "kvm_cache_regs.h"
30 #define DPRINTF(x...) do {} while (0)
32 #include <linux/module.h>
33 #include <asm/kvm_emulate.h>
38 * Opcode effective-address decode tables.
39 * Note that we only emulate instructions that have at least one memory
40 * operand (excluding implicit stack references). We assume that stack
41 * references and instruction fetches will never occur in special memory
42 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
46 /* Operand sizes: 8-bit operands or specified/overridden size. */
47 #define ByteOp (1<<0) /* 8-bit operands. */
48 /* Destination operand type. */
49 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
50 #define DstReg (2<<1) /* Register operand. */
51 #define DstMem (3<<1) /* Memory operand. */
52 #define DstAcc (4<<1) /* Destination Accumulator */
53 #define DstMask (7<<1)
54 /* Source operand type. */
55 #define SrcNone (0<<4) /* No source operand. */
56 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
57 #define SrcReg (1<<4) /* Register operand. */
58 #define SrcMem (2<<4) /* Memory operand. */
59 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
60 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
61 #define SrcImm (5<<4) /* Immediate operand. */
62 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
63 #define SrcOne (7<<4) /* Implied '1' */
64 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
65 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
66 #define SrcMask (0xf<<4)
67 /* Generic ModRM decode. */
69 /* Destination is only written; never read. */
72 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
73 #define String (1<<12) /* String instruction (rep capable) */
74 #define Stack (1<<13) /* Stack instruction (push/pop) */
75 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
76 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
77 #define GroupMask 0xff /* Group number stored in bits 0:7 */
79 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
80 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
82 /* Source 2 operand type */
83 #define Src2None (0<<29)
84 #define Src2CL (1<<29)
85 #define Src2ImmByte (2<<29)
86 #define Src2One (3<<29)
87 #define Src2Imm16 (4<<29)
88 #define Src2Mask (7<<29)
91 Group1_80, Group1_81, Group1_82, Group1_83,
92 Group1A, Group3_Byte, Group3, Group4, Group5, Group7,
96 static u32 opcode_table[256] = {
98 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
99 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
100 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
101 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
103 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
104 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
105 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
106 ImplicitOps | Stack | No64, 0,
108 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
109 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
110 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
111 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
113 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
114 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
115 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
116 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
118 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
119 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
120 DstAcc | SrcImmByte, DstAcc | SrcImm, 0, 0,
122 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
123 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
126 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
127 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
130 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
131 ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM,
132 ByteOp | DstAcc | SrcImm, DstAcc | SrcImm,
135 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
137 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
139 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
140 SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack,
142 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
143 DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack,
145 ImplicitOps | Stack | No64, ImplicitOps | Stack | No64,
146 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ ,
149 SrcImm | Mov | Stack, 0, SrcImmByte | Mov | Stack, 0,
150 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */
151 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */
153 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
154 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
156 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
157 SrcImmByte, SrcImmByte, SrcImmByte, SrcImmByte,
159 Group | Group1_80, Group | Group1_81,
160 Group | Group1_82, Group | Group1_83,
161 ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM,
162 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
164 ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov,
165 ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
166 DstMem | SrcReg | ModRM | Mov, ModRM | DstReg,
167 DstReg | SrcMem | ModRM | Mov, Group | Group1A,
169 DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg,
171 0, 0, SrcImm | Src2Imm16 | No64, 0,
172 ImplicitOps | Stack, ImplicitOps | Stack, 0, 0,
174 ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs,
175 ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs,
176 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
177 ByteOp | ImplicitOps | String, ImplicitOps | String,
179 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
180 ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String,
181 ByteOp | ImplicitOps | String, ImplicitOps | String,
183 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
184 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
185 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
186 ByteOp | DstReg | SrcImm | Mov, ByteOp | DstReg | SrcImm | Mov,
188 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
189 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
190 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
191 DstReg | SrcImm | Mov, DstReg | SrcImm | Mov,
193 ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM,
194 0, ImplicitOps | Stack, 0, 0,
195 ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov,
197 0, 0, 0, ImplicitOps | Stack,
198 ImplicitOps, SrcImmByte, ImplicitOps | No64, ImplicitOps,
200 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
201 ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM,
204 0, 0, 0, 0, 0, 0, 0, 0,
207 ByteOp | SrcImmUByte, SrcImmUByte,
208 ByteOp | SrcImmUByte, SrcImmUByte,
210 SrcImm | Stack, SrcImm | ImplicitOps,
211 SrcImmU | Src2Imm16 | No64, SrcImmByte | ImplicitOps,
212 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
213 SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps,
216 ImplicitOps | Priv, ImplicitOps, Group | Group3_Byte, Group | Group3,
218 ImplicitOps, 0, ImplicitOps, ImplicitOps,
219 ImplicitOps, ImplicitOps, Group | Group4, Group | Group5,
222 static u32 twobyte_table[256] = {
224 0, Group | GroupDual | Group7, 0, 0,
225 0, ImplicitOps, ImplicitOps | Priv, 0,
226 ImplicitOps | Priv, ImplicitOps | Priv, 0, 0,
227 0, ImplicitOps | ModRM, 0, 0,
229 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0,
231 ModRM | ImplicitOps | Priv, ModRM | Priv,
232 ModRM | ImplicitOps | Priv, ModRM | Priv,
234 0, 0, 0, 0, 0, 0, 0, 0,
236 ImplicitOps | Priv, 0, ImplicitOps | Priv, 0,
237 ImplicitOps, ImplicitOps | Priv, 0, 0,
238 0, 0, 0, 0, 0, 0, 0, 0,
240 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
241 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
242 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
243 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
245 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
246 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
247 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
248 DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov,
250 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
252 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
254 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
256 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
257 SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm, SrcImm,
259 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
261 ImplicitOps | Stack, ImplicitOps | Stack,
262 0, DstMem | SrcReg | ModRM | BitOp,
263 DstMem | SrcReg | Src2ImmByte | ModRM,
264 DstMem | SrcReg | Src2CL | ModRM, 0, 0,
266 ImplicitOps | Stack, ImplicitOps | Stack,
267 0, DstMem | SrcReg | ModRM | BitOp | Lock,
268 DstMem | SrcReg | Src2ImmByte | ModRM,
269 DstMem | SrcReg | Src2CL | ModRM,
272 ByteOp | DstMem | SrcReg | ModRM | Lock, DstMem | SrcReg | ModRM | Lock,
273 0, DstMem | SrcReg | ModRM | BitOp | Lock,
274 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
275 DstReg | SrcMem16 | ModRM | Mov,
278 Group | Group8, DstMem | SrcReg | ModRM | BitOp | Lock,
279 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov,
280 DstReg | SrcMem16 | ModRM | Mov,
282 0, 0, 0, DstMem | SrcReg | ModRM | Mov,
283 0, 0, 0, Group | GroupDual | Group9,
284 0, 0, 0, 0, 0, 0, 0, 0,
286 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
288 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
290 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
293 static u32 group_table[] = {
295 ByteOp | DstMem | SrcImm | ModRM | Lock,
296 ByteOp | DstMem | SrcImm | ModRM | Lock,
297 ByteOp | DstMem | SrcImm | ModRM | Lock,
298 ByteOp | DstMem | SrcImm | ModRM | Lock,
299 ByteOp | DstMem | SrcImm | ModRM | Lock,
300 ByteOp | DstMem | SrcImm | ModRM | Lock,
301 ByteOp | DstMem | SrcImm | ModRM | Lock,
302 ByteOp | DstMem | SrcImm | ModRM,
304 DstMem | SrcImm | ModRM | Lock,
305 DstMem | SrcImm | ModRM | Lock,
306 DstMem | SrcImm | ModRM | Lock,
307 DstMem | SrcImm | ModRM | Lock,
308 DstMem | SrcImm | ModRM | Lock,
309 DstMem | SrcImm | ModRM | Lock,
310 DstMem | SrcImm | ModRM | Lock,
311 DstMem | SrcImm | ModRM,
313 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
314 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
315 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
316 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
317 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
318 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
319 ByteOp | DstMem | SrcImm | ModRM | No64 | Lock,
320 ByteOp | DstMem | SrcImm | ModRM | No64,
322 DstMem | SrcImmByte | ModRM | Lock,
323 DstMem | SrcImmByte | ModRM | Lock,
324 DstMem | SrcImmByte | ModRM | Lock,
325 DstMem | SrcImmByte | ModRM | Lock,
326 DstMem | SrcImmByte | ModRM | Lock,
327 DstMem | SrcImmByte | ModRM | Lock,
328 DstMem | SrcImmByte | ModRM | Lock,
329 DstMem | SrcImmByte | ModRM,
331 DstMem | SrcNone | ModRM | Mov | Stack, 0, 0, 0, 0, 0, 0, 0,
333 ByteOp | SrcImm | DstMem | ModRM, 0,
334 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
337 DstMem | SrcImm | ModRM, 0,
338 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
341 ByteOp | DstMem | SrcNone | ModRM, ByteOp | DstMem | SrcNone | ModRM,
344 DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM,
345 SrcMem | ModRM | Stack, 0,
346 SrcMem | ModRM | Stack, 0, SrcMem | ModRM | Stack, 0,
348 0, 0, ModRM | SrcMem | Priv, ModRM | SrcMem | Priv,
349 SrcNone | ModRM | DstMem | Mov, 0,
350 SrcMem16 | ModRM | Mov | Priv, SrcMem | ModRM | ByteOp | Priv,
353 DstMem | SrcImmByte | ModRM, DstMem | SrcImmByte | ModRM | Lock,
354 DstMem | SrcImmByte | ModRM | Lock, DstMem | SrcImmByte | ModRM | Lock,
356 0, ImplicitOps | ModRM | Lock, 0, 0, 0, 0, 0, 0,
359 static u32 group2_table[] = {
361 SrcNone | ModRM | Priv, 0, 0, SrcNone | ModRM,
362 SrcNone | ModRM | DstMem | Mov, 0,
363 SrcMem16 | ModRM | Mov, 0,
365 0, 0, 0, 0, 0, 0, 0, 0,
368 /* EFLAGS bit definitions. */
369 #define EFLG_ID (1<<21)
370 #define EFLG_VIP (1<<20)
371 #define EFLG_VIF (1<<19)
372 #define EFLG_AC (1<<18)
373 #define EFLG_VM (1<<17)
374 #define EFLG_RF (1<<16)
375 #define EFLG_IOPL (3<<12)
376 #define EFLG_NT (1<<14)
377 #define EFLG_OF (1<<11)
378 #define EFLG_DF (1<<10)
379 #define EFLG_IF (1<<9)
380 #define EFLG_TF (1<<8)
381 #define EFLG_SF (1<<7)
382 #define EFLG_ZF (1<<6)
383 #define EFLG_AF (1<<4)
384 #define EFLG_PF (1<<2)
385 #define EFLG_CF (1<<0)
388 * Instruction emulation:
389 * Most instructions are emulated directly via a fragment of inline assembly
390 * code. This allows us to save/restore EFLAGS and thus very easily pick up
391 * any modified flags.
394 #if defined(CONFIG_X86_64)
395 #define _LO32 "k" /* force 32-bit operand */
396 #define _STK "%%rsp" /* stack pointer */
397 #elif defined(__i386__)
398 #define _LO32 "" /* force 32-bit operand */
399 #define _STK "%%esp" /* stack pointer */
403 * These EFLAGS bits are restored from saved value during emulation, and
404 * any changes are written back to the saved value after emulation.
406 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
408 /* Before executing instruction: restore necessary bits in EFLAGS. */
409 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
410 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
411 "movl %"_sav",%"_LO32 _tmp"; " \
414 "movl %"_msk",%"_LO32 _tmp"; " \
415 "andl %"_LO32 _tmp",("_STK"); " \
417 "notl %"_LO32 _tmp"; " \
418 "andl %"_LO32 _tmp",("_STK"); " \
419 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
421 "orl %"_LO32 _tmp",("_STK"); " \
425 /* After executing instruction: write-back necessary bits in EFLAGS. */
426 #define _POST_EFLAGS(_sav, _msk, _tmp) \
427 /* _sav |= EFLAGS & _msk; */ \
430 "andl %"_msk",%"_LO32 _tmp"; " \
431 "orl %"_LO32 _tmp",%"_sav"; "
439 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
441 __asm__ __volatile__ ( \
442 _PRE_EFLAGS("0", "4", "2") \
443 _op _suffix " %"_x"3,%1; " \
444 _POST_EFLAGS("0", "4", "2") \
445 : "=m" (_eflags), "=m" ((_dst).val), \
447 : _y ((_src).val), "i" (EFLAGS_MASK)); \
451 /* Raw emulation: instruction has two explicit operands. */
452 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
454 unsigned long _tmp; \
456 switch ((_dst).bytes) { \
458 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
461 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
464 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
469 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
471 unsigned long _tmp; \
472 switch ((_dst).bytes) { \
474 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
477 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
478 _wx, _wy, _lx, _ly, _qx, _qy); \
483 /* Source operand is byte-sized and may be restricted to just %cl. */
484 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
485 __emulate_2op(_op, _src, _dst, _eflags, \
486 "b", "c", "b", "c", "b", "c", "b", "c")
488 /* Source operand is byte, word, long or quad sized. */
489 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
490 __emulate_2op(_op, _src, _dst, _eflags, \
491 "b", "q", "w", "r", _LO32, "r", "", "r")
493 /* Source operand is word, long or quad sized. */
494 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
495 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
496 "w", "r", _LO32, "r", "", "r")
498 /* Instruction has three operands and one operand is stored in ECX register */
499 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
501 unsigned long _tmp; \
502 _type _clv = (_cl).val; \
503 _type _srcv = (_src).val; \
504 _type _dstv = (_dst).val; \
506 __asm__ __volatile__ ( \
507 _PRE_EFLAGS("0", "5", "2") \
508 _op _suffix " %4,%1 \n" \
509 _POST_EFLAGS("0", "5", "2") \
510 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
511 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
514 (_cl).val = (unsigned long) _clv; \
515 (_src).val = (unsigned long) _srcv; \
516 (_dst).val = (unsigned long) _dstv; \
519 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
521 switch ((_dst).bytes) { \
523 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
524 "w", unsigned short); \
527 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
528 "l", unsigned int); \
531 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
532 "q", unsigned long)); \
537 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
539 unsigned long _tmp; \
541 __asm__ __volatile__ ( \
542 _PRE_EFLAGS("0", "3", "2") \
543 _op _suffix " %1; " \
544 _POST_EFLAGS("0", "3", "2") \
545 : "=m" (_eflags), "+m" ((_dst).val), \
547 : "i" (EFLAGS_MASK)); \
550 /* Instruction has only one explicit operand (no source operand). */
551 #define emulate_1op(_op, _dst, _eflags) \
553 switch ((_dst).bytes) { \
554 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
555 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
556 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
557 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
561 /* Fetch next part of the instruction being emulated. */
562 #define insn_fetch(_type, _size, _eip) \
563 ({ unsigned long _x; \
564 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
571 static inline unsigned long ad_mask(struct decode_cache *c)
573 return (1UL << (c->ad_bytes << 3)) - 1;
576 /* Access/update address held in a register, based on addressing mode. */
577 static inline unsigned long
578 address_mask(struct decode_cache *c, unsigned long reg)
580 if (c->ad_bytes == sizeof(unsigned long))
583 return reg & ad_mask(c);
586 static inline unsigned long
587 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
589 return base + address_mask(c, reg);
593 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
595 if (c->ad_bytes == sizeof(unsigned long))
598 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
601 static inline void jmp_rel(struct decode_cache *c, int rel)
603 register_address_increment(c, &c->eip, rel);
606 static void set_seg_override(struct decode_cache *c, int seg)
608 c->has_seg_override = true;
609 c->seg_override = seg;
612 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
614 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
617 return kvm_x86_ops->get_segment_base(ctxt->vcpu, seg);
620 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
621 struct decode_cache *c)
623 if (!c->has_seg_override)
626 return seg_base(ctxt, c->seg_override);
629 static unsigned long es_base(struct x86_emulate_ctxt *ctxt)
631 return seg_base(ctxt, VCPU_SREG_ES);
634 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt)
636 return seg_base(ctxt, VCPU_SREG_SS);
639 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
640 struct x86_emulate_ops *ops,
641 unsigned long linear, u8 *dest)
643 struct fetch_cache *fc = &ctxt->decode.fetch;
647 if (linear < fc->start || linear >= fc->end) {
648 size = min(15UL, PAGE_SIZE - offset_in_page(linear));
649 rc = ops->fetch(linear, fc->data, size, ctxt->vcpu, NULL);
653 fc->end = linear + size;
655 *dest = fc->data[linear - fc->start];
659 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
660 struct x86_emulate_ops *ops,
661 unsigned long eip, void *dest, unsigned size)
665 /* x86 instructions are limited to 15 bytes. */
666 if (eip + size - ctxt->decode.eip_orig > 15)
667 return X86EMUL_UNHANDLEABLE;
668 eip += ctxt->cs_base;
670 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
678 * Given the 'reg' portion of a ModRM byte, and a register block, return a
679 * pointer into the block that addresses the relevant register.
680 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
682 static void *decode_register(u8 modrm_reg, unsigned long *regs,
687 p = ®s[modrm_reg];
688 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
689 p = (unsigned char *)®s[modrm_reg & 3] + 1;
693 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
694 struct x86_emulate_ops *ops,
696 u16 *size, unsigned long *address, int op_bytes)
703 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
707 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
712 static int test_cc(unsigned int condition, unsigned int flags)
716 switch ((condition & 15) >> 1) {
718 rc |= (flags & EFLG_OF);
720 case 1: /* b/c/nae */
721 rc |= (flags & EFLG_CF);
724 rc |= (flags & EFLG_ZF);
727 rc |= (flags & (EFLG_CF|EFLG_ZF));
730 rc |= (flags & EFLG_SF);
733 rc |= (flags & EFLG_PF);
736 rc |= (flags & EFLG_ZF);
739 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
743 /* Odd condition identifiers (lsb == 1) have inverted sense. */
744 return (!!rc ^ (condition & 1));
747 static void decode_register_operand(struct operand *op,
748 struct decode_cache *c,
751 unsigned reg = c->modrm_reg;
752 int highbyte_regs = c->rex_prefix == 0;
755 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
757 if ((c->d & ByteOp) && !inhibit_bytereg) {
758 op->ptr = decode_register(reg, c->regs, highbyte_regs);
759 op->val = *(u8 *)op->ptr;
762 op->ptr = decode_register(reg, c->regs, 0);
763 op->bytes = c->op_bytes;
766 op->val = *(u16 *)op->ptr;
769 op->val = *(u32 *)op->ptr;
772 op->val = *(u64 *) op->ptr;
776 op->orig_val = op->val;
779 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
780 struct x86_emulate_ops *ops)
782 struct decode_cache *c = &ctxt->decode;
784 int index_reg = 0, base_reg = 0, scale;
788 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
789 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
790 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
793 c->modrm = insn_fetch(u8, 1, c->eip);
794 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
795 c->modrm_reg |= (c->modrm & 0x38) >> 3;
796 c->modrm_rm |= (c->modrm & 0x07);
800 if (c->modrm_mod == 3) {
801 c->modrm_ptr = decode_register(c->modrm_rm,
802 c->regs, c->d & ByteOp);
803 c->modrm_val = *(unsigned long *)c->modrm_ptr;
807 if (c->ad_bytes == 2) {
808 unsigned bx = c->regs[VCPU_REGS_RBX];
809 unsigned bp = c->regs[VCPU_REGS_RBP];
810 unsigned si = c->regs[VCPU_REGS_RSI];
811 unsigned di = c->regs[VCPU_REGS_RDI];
813 /* 16-bit ModR/M decode. */
814 switch (c->modrm_mod) {
816 if (c->modrm_rm == 6)
817 c->modrm_ea += insn_fetch(u16, 2, c->eip);
820 c->modrm_ea += insn_fetch(s8, 1, c->eip);
823 c->modrm_ea += insn_fetch(u16, 2, c->eip);
826 switch (c->modrm_rm) {
828 c->modrm_ea += bx + si;
831 c->modrm_ea += bx + di;
834 c->modrm_ea += bp + si;
837 c->modrm_ea += bp + di;
846 if (c->modrm_mod != 0)
853 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
854 (c->modrm_rm == 6 && c->modrm_mod != 0))
855 if (!c->has_seg_override)
856 set_seg_override(c, VCPU_SREG_SS);
857 c->modrm_ea = (u16)c->modrm_ea;
859 /* 32/64-bit ModR/M decode. */
860 if ((c->modrm_rm & 7) == 4) {
861 sib = insn_fetch(u8, 1, c->eip);
862 index_reg |= (sib >> 3) & 7;
866 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
867 c->modrm_ea += insn_fetch(s32, 4, c->eip);
869 c->modrm_ea += c->regs[base_reg];
871 c->modrm_ea += c->regs[index_reg] << scale;
872 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
873 if (ctxt->mode == X86EMUL_MODE_PROT64)
876 c->modrm_ea += c->regs[c->modrm_rm];
877 switch (c->modrm_mod) {
879 if (c->modrm_rm == 5)
880 c->modrm_ea += insn_fetch(s32, 4, c->eip);
883 c->modrm_ea += insn_fetch(s8, 1, c->eip);
886 c->modrm_ea += insn_fetch(s32, 4, c->eip);
894 static int decode_abs(struct x86_emulate_ctxt *ctxt,
895 struct x86_emulate_ops *ops)
897 struct decode_cache *c = &ctxt->decode;
900 switch (c->ad_bytes) {
902 c->modrm_ea = insn_fetch(u16, 2, c->eip);
905 c->modrm_ea = insn_fetch(u32, 4, c->eip);
908 c->modrm_ea = insn_fetch(u64, 8, c->eip);
916 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
918 struct decode_cache *c = &ctxt->decode;
920 int mode = ctxt->mode;
921 int def_op_bytes, def_ad_bytes, group;
923 /* Shadow copy of register state. Committed on successful emulation. */
925 memset(c, 0, sizeof(struct decode_cache));
926 c->eip = c->eip_orig = kvm_rip_read(ctxt->vcpu);
927 ctxt->cs_base = seg_base(ctxt, VCPU_SREG_CS);
928 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
931 case X86EMUL_MODE_REAL:
932 case X86EMUL_MODE_VM86:
933 case X86EMUL_MODE_PROT16:
934 def_op_bytes = def_ad_bytes = 2;
936 case X86EMUL_MODE_PROT32:
937 def_op_bytes = def_ad_bytes = 4;
940 case X86EMUL_MODE_PROT64:
949 c->op_bytes = def_op_bytes;
950 c->ad_bytes = def_ad_bytes;
952 /* Legacy prefixes. */
954 switch (c->b = insn_fetch(u8, 1, c->eip)) {
955 case 0x66: /* operand-size override */
956 /* switch between 2/4 bytes */
957 c->op_bytes = def_op_bytes ^ 6;
959 case 0x67: /* address-size override */
960 if (mode == X86EMUL_MODE_PROT64)
961 /* switch between 4/8 bytes */
962 c->ad_bytes = def_ad_bytes ^ 12;
964 /* switch between 2/4 bytes */
965 c->ad_bytes = def_ad_bytes ^ 6;
967 case 0x26: /* ES override */
968 case 0x2e: /* CS override */
969 case 0x36: /* SS override */
970 case 0x3e: /* DS override */
971 set_seg_override(c, (c->b >> 3) & 3);
973 case 0x64: /* FS override */
974 case 0x65: /* GS override */
975 set_seg_override(c, c->b & 7);
977 case 0x40 ... 0x4f: /* REX */
978 if (mode != X86EMUL_MODE_PROT64)
980 c->rex_prefix = c->b;
982 case 0xf0: /* LOCK */
985 case 0xf2: /* REPNE/REPNZ */
986 c->rep_prefix = REPNE_PREFIX;
988 case 0xf3: /* REP/REPE/REPZ */
989 c->rep_prefix = REPE_PREFIX;
995 /* Any legacy prefix after a REX prefix nullifies its effect. */
1004 if (c->rex_prefix & 8)
1005 c->op_bytes = 8; /* REX.W */
1007 /* Opcode byte(s). */
1008 c->d = opcode_table[c->b];
1010 /* Two-byte opcode? */
1013 c->b = insn_fetch(u8, 1, c->eip);
1014 c->d = twobyte_table[c->b];
1018 if (mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
1019 kvm_report_emulation_failure(ctxt->vcpu, "invalid x86/64 instruction");
1024 group = c->d & GroupMask;
1025 c->modrm = insn_fetch(u8, 1, c->eip);
1028 group = (group << 3) + ((c->modrm >> 3) & 7);
1029 if ((c->d & GroupDual) && (c->modrm >> 6) == 3)
1030 c->d = group2_table[group];
1032 c->d = group_table[group];
1037 DPRINTF("Cannot emulate %02x\n", c->b);
1041 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1044 /* ModRM and SIB bytes. */
1046 rc = decode_modrm(ctxt, ops);
1047 else if (c->d & MemAbs)
1048 rc = decode_abs(ctxt, ops);
1052 if (!c->has_seg_override)
1053 set_seg_override(c, VCPU_SREG_DS);
1055 if (!(!c->twobyte && c->b == 0x8d))
1056 c->modrm_ea += seg_override_base(ctxt, c);
1058 if (c->ad_bytes != 8)
1059 c->modrm_ea = (u32)c->modrm_ea;
1061 * Decode and fetch the source operand: register, memory
1064 switch (c->d & SrcMask) {
1068 decode_register_operand(&c->src, c, 0);
1077 c->src.bytes = (c->d & ByteOp) ? 1 :
1079 /* Don't fetch the address for invlpg: it could be unmapped. */
1080 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1084 * For instructions with a ModR/M byte, switch to register
1085 * access if Mod = 3.
1087 if ((c->d & ModRM) && c->modrm_mod == 3) {
1088 c->src.type = OP_REG;
1089 c->src.val = c->modrm_val;
1090 c->src.ptr = c->modrm_ptr;
1093 c->src.type = OP_MEM;
1097 c->src.type = OP_IMM;
1098 c->src.ptr = (unsigned long *)c->eip;
1099 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1100 if (c->src.bytes == 8)
1102 /* NB. Immediates are sign-extended as necessary. */
1103 switch (c->src.bytes) {
1105 c->src.val = insn_fetch(s8, 1, c->eip);
1108 c->src.val = insn_fetch(s16, 2, c->eip);
1111 c->src.val = insn_fetch(s32, 4, c->eip);
1114 if ((c->d & SrcMask) == SrcImmU) {
1115 switch (c->src.bytes) {
1120 c->src.val &= 0xffff;
1123 c->src.val &= 0xffffffff;
1130 c->src.type = OP_IMM;
1131 c->src.ptr = (unsigned long *)c->eip;
1133 if ((c->d & SrcMask) == SrcImmByte)
1134 c->src.val = insn_fetch(s8, 1, c->eip);
1136 c->src.val = insn_fetch(u8, 1, c->eip);
1145 * Decode and fetch the second source operand: register, memory
1148 switch (c->d & Src2Mask) {
1153 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1156 c->src2.type = OP_IMM;
1157 c->src2.ptr = (unsigned long *)c->eip;
1159 c->src2.val = insn_fetch(u8, 1, c->eip);
1162 c->src2.type = OP_IMM;
1163 c->src2.ptr = (unsigned long *)c->eip;
1165 c->src2.val = insn_fetch(u16, 2, c->eip);
1173 /* Decode and fetch the destination operand: register or memory. */
1174 switch (c->d & DstMask) {
1176 /* Special instructions do their own operand decoding. */
1179 decode_register_operand(&c->dst, c,
1180 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1183 if ((c->d & ModRM) && c->modrm_mod == 3) {
1184 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1185 c->dst.type = OP_REG;
1186 c->dst.val = c->dst.orig_val = c->modrm_val;
1187 c->dst.ptr = c->modrm_ptr;
1190 c->dst.type = OP_MEM;
1193 c->dst.type = OP_REG;
1194 c->dst.bytes = c->op_bytes;
1195 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1196 switch (c->op_bytes) {
1198 c->dst.val = *(u8 *)c->dst.ptr;
1201 c->dst.val = *(u16 *)c->dst.ptr;
1204 c->dst.val = *(u32 *)c->dst.ptr;
1207 c->dst.orig_val = c->dst.val;
1211 if (c->rip_relative)
1212 c->modrm_ea += c->eip;
1215 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1218 static inline void emulate_push(struct x86_emulate_ctxt *ctxt)
1220 struct decode_cache *c = &ctxt->decode;
1222 c->dst.type = OP_MEM;
1223 c->dst.bytes = c->op_bytes;
1224 c->dst.val = c->src.val;
1225 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1226 c->dst.ptr = (void *) register_address(c, ss_base(ctxt),
1227 c->regs[VCPU_REGS_RSP]);
1230 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1231 struct x86_emulate_ops *ops,
1232 void *dest, int len)
1234 struct decode_cache *c = &ctxt->decode;
1237 rc = ops->read_emulated(register_address(c, ss_base(ctxt),
1238 c->regs[VCPU_REGS_RSP]),
1239 dest, len, ctxt->vcpu);
1240 if (rc != X86EMUL_CONTINUE)
1243 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1247 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1248 struct x86_emulate_ops *ops,
1249 void *dest, int len)
1252 unsigned long val, change_mask;
1253 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1254 int cpl = kvm_x86_ops->get_cpl(ctxt->vcpu);
1256 rc = emulate_pop(ctxt, ops, &val, len);
1257 if (rc != X86EMUL_CONTINUE)
1260 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1261 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1263 switch(ctxt->mode) {
1264 case X86EMUL_MODE_PROT64:
1265 case X86EMUL_MODE_PROT32:
1266 case X86EMUL_MODE_PROT16:
1268 change_mask |= EFLG_IOPL;
1270 change_mask |= EFLG_IF;
1272 case X86EMUL_MODE_VM86:
1274 kvm_inject_gp(ctxt->vcpu, 0);
1275 return X86EMUL_PROPAGATE_FAULT;
1277 change_mask |= EFLG_IF;
1279 default: /* real mode */
1280 change_mask |= (EFLG_IOPL | EFLG_IF);
1284 *(unsigned long *)dest =
1285 (ctxt->eflags & ~change_mask) | (val & change_mask);
1290 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1292 struct decode_cache *c = &ctxt->decode;
1293 struct kvm_segment segment;
1295 kvm_x86_ops->get_segment(ctxt->vcpu, &segment, seg);
1297 c->src.val = segment.selector;
1301 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1302 struct x86_emulate_ops *ops, int seg)
1304 struct decode_cache *c = &ctxt->decode;
1305 unsigned long selector;
1308 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1312 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)selector, seg);
1316 static void emulate_pusha(struct x86_emulate_ctxt *ctxt)
1318 struct decode_cache *c = &ctxt->decode;
1319 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1320 int reg = VCPU_REGS_RAX;
1322 while (reg <= VCPU_REGS_RDI) {
1323 (reg == VCPU_REGS_RSP) ?
1324 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1331 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1332 struct x86_emulate_ops *ops)
1334 struct decode_cache *c = &ctxt->decode;
1336 int reg = VCPU_REGS_RDI;
1338 while (reg >= VCPU_REGS_RAX) {
1339 if (reg == VCPU_REGS_RSP) {
1340 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1345 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1353 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1354 struct x86_emulate_ops *ops)
1356 struct decode_cache *c = &ctxt->decode;
1359 rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1365 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1367 struct decode_cache *c = &ctxt->decode;
1368 switch (c->modrm_reg) {
1370 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1373 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1376 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1379 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1381 case 4: /* sal/shl */
1382 case 6: /* sal/shl */
1383 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1386 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1389 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1394 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1395 struct x86_emulate_ops *ops)
1397 struct decode_cache *c = &ctxt->decode;
1400 switch (c->modrm_reg) {
1401 case 0 ... 1: /* test */
1402 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1405 c->dst.val = ~c->dst.val;
1408 emulate_1op("neg", c->dst, ctxt->eflags);
1411 DPRINTF("Cannot emulate %02x\n", c->b);
1412 rc = X86EMUL_UNHANDLEABLE;
1418 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1419 struct x86_emulate_ops *ops)
1421 struct decode_cache *c = &ctxt->decode;
1423 switch (c->modrm_reg) {
1425 emulate_1op("inc", c->dst, ctxt->eflags);
1428 emulate_1op("dec", c->dst, ctxt->eflags);
1430 case 2: /* call near abs */ {
1433 c->eip = c->src.val;
1434 c->src.val = old_eip;
1438 case 4: /* jmp abs */
1439 c->eip = c->src.val;
1448 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1449 struct x86_emulate_ops *ops,
1450 unsigned long memop)
1452 struct decode_cache *c = &ctxt->decode;
1456 rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu);
1457 if (rc != X86EMUL_CONTINUE)
1460 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1461 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1463 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1464 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1465 ctxt->eflags &= ~EFLG_ZF;
1468 new = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1469 (u32) c->regs[VCPU_REGS_RBX];
1471 rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu);
1472 if (rc != X86EMUL_CONTINUE)
1474 ctxt->eflags |= EFLG_ZF;
1479 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1480 struct x86_emulate_ops *ops)
1482 struct decode_cache *c = &ctxt->decode;
1486 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1489 if (c->op_bytes == 4)
1490 c->eip = (u32)c->eip;
1491 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1494 rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, VCPU_SREG_CS);
1498 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1499 struct x86_emulate_ops *ops)
1502 struct decode_cache *c = &ctxt->decode;
1504 switch (c->dst.type) {
1506 /* The 4-byte case *is* correct:
1507 * in 64-bit mode we zero-extend.
1509 switch (c->dst.bytes) {
1511 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1514 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1517 *c->dst.ptr = (u32)c->dst.val;
1518 break; /* 64b: zero-ext */
1520 *c->dst.ptr = c->dst.val;
1526 rc = ops->cmpxchg_emulated(
1527 (unsigned long)c->dst.ptr,
1533 rc = ops->write_emulated(
1534 (unsigned long)c->dst.ptr,
1538 if (rc != X86EMUL_CONTINUE)
1550 static void toggle_interruptibility(struct x86_emulate_ctxt *ctxt, u32 mask)
1552 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(ctxt->vcpu, mask);
1554 * an sti; sti; sequence only disable interrupts for the first
1555 * instruction. So, if the last instruction, be it emulated or
1556 * not, left the system with the INT_STI flag enabled, it
1557 * means that the last instruction is an sti. We should not
1558 * leave the flag on in this case. The same goes for mov ss
1560 if (!(int_shadow & mask))
1561 ctxt->interruptibility = mask;
1565 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1566 struct kvm_segment *cs, struct kvm_segment *ss)
1568 memset(cs, 0, sizeof(struct kvm_segment));
1569 kvm_x86_ops->get_segment(ctxt->vcpu, cs, VCPU_SREG_CS);
1570 memset(ss, 0, sizeof(struct kvm_segment));
1572 cs->l = 0; /* will be adjusted later */
1573 cs->base = 0; /* flat segment */
1574 cs->g = 1; /* 4kb granularity */
1575 cs->limit = 0xffffffff; /* 4GB limit */
1576 cs->type = 0x0b; /* Read, Execute, Accessed */
1578 cs->dpl = 0; /* will be adjusted later */
1583 ss->base = 0; /* flat segment */
1584 ss->limit = 0xffffffff; /* 4GB limit */
1585 ss->g = 1; /* 4kb granularity */
1587 ss->type = 0x03; /* Read/Write, Accessed */
1588 ss->db = 1; /* 32bit stack segment */
1594 emulate_syscall(struct x86_emulate_ctxt *ctxt)
1596 struct decode_cache *c = &ctxt->decode;
1597 struct kvm_segment cs, ss;
1600 /* syscall is not available in real mode */
1601 if (ctxt->mode == X86EMUL_MODE_REAL || ctxt->mode == X86EMUL_MODE_VM86)
1602 return X86EMUL_UNHANDLEABLE;
1604 setup_syscalls_segments(ctxt, &cs, &ss);
1606 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1608 cs.selector = (u16)(msr_data & 0xfffc);
1609 ss.selector = (u16)(msr_data + 8);
1611 if (is_long_mode(ctxt->vcpu)) {
1615 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1616 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1618 c->regs[VCPU_REGS_RCX] = c->eip;
1619 if (is_long_mode(ctxt->vcpu)) {
1620 #ifdef CONFIG_X86_64
1621 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1623 kvm_x86_ops->get_msr(ctxt->vcpu,
1624 ctxt->mode == X86EMUL_MODE_PROT64 ?
1625 MSR_LSTAR : MSR_CSTAR, &msr_data);
1628 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1629 ctxt->eflags &= ~(msr_data | EFLG_RF);
1633 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1634 c->eip = (u32)msr_data;
1636 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1639 return X86EMUL_CONTINUE;
1643 emulate_sysenter(struct x86_emulate_ctxt *ctxt)
1645 struct decode_cache *c = &ctxt->decode;
1646 struct kvm_segment cs, ss;
1649 /* inject #GP if in real mode */
1650 if (ctxt->mode == X86EMUL_MODE_REAL) {
1651 kvm_inject_gp(ctxt->vcpu, 0);
1652 return X86EMUL_UNHANDLEABLE;
1655 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1656 * Therefore, we inject an #UD.
1658 if (ctxt->mode == X86EMUL_MODE_PROT64)
1659 return X86EMUL_UNHANDLEABLE;
1661 setup_syscalls_segments(ctxt, &cs, &ss);
1663 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1664 switch (ctxt->mode) {
1665 case X86EMUL_MODE_PROT32:
1666 if ((msr_data & 0xfffc) == 0x0) {
1667 kvm_inject_gp(ctxt->vcpu, 0);
1668 return X86EMUL_PROPAGATE_FAULT;
1671 case X86EMUL_MODE_PROT64:
1672 if (msr_data == 0x0) {
1673 kvm_inject_gp(ctxt->vcpu, 0);
1674 return X86EMUL_PROPAGATE_FAULT;
1679 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1680 cs.selector = (u16)msr_data;
1681 cs.selector &= ~SELECTOR_RPL_MASK;
1682 ss.selector = cs.selector + 8;
1683 ss.selector &= ~SELECTOR_RPL_MASK;
1684 if (ctxt->mode == X86EMUL_MODE_PROT64
1685 || is_long_mode(ctxt->vcpu)) {
1690 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1691 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1693 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1696 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1697 c->regs[VCPU_REGS_RSP] = msr_data;
1699 return X86EMUL_CONTINUE;
1703 emulate_sysexit(struct x86_emulate_ctxt *ctxt)
1705 struct decode_cache *c = &ctxt->decode;
1706 struct kvm_segment cs, ss;
1710 /* inject #GP if in real mode or Virtual 8086 mode */
1711 if (ctxt->mode == X86EMUL_MODE_REAL ||
1712 ctxt->mode == X86EMUL_MODE_VM86) {
1713 kvm_inject_gp(ctxt->vcpu, 0);
1714 return X86EMUL_UNHANDLEABLE;
1717 setup_syscalls_segments(ctxt, &cs, &ss);
1719 if ((c->rex_prefix & 0x8) != 0x0)
1720 usermode = X86EMUL_MODE_PROT64;
1722 usermode = X86EMUL_MODE_PROT32;
1726 kvm_x86_ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1728 case X86EMUL_MODE_PROT32:
1729 cs.selector = (u16)(msr_data + 16);
1730 if ((msr_data & 0xfffc) == 0x0) {
1731 kvm_inject_gp(ctxt->vcpu, 0);
1732 return X86EMUL_PROPAGATE_FAULT;
1734 ss.selector = (u16)(msr_data + 24);
1736 case X86EMUL_MODE_PROT64:
1737 cs.selector = (u16)(msr_data + 32);
1738 if (msr_data == 0x0) {
1739 kvm_inject_gp(ctxt->vcpu, 0);
1740 return X86EMUL_PROPAGATE_FAULT;
1742 ss.selector = cs.selector + 8;
1747 cs.selector |= SELECTOR_RPL_MASK;
1748 ss.selector |= SELECTOR_RPL_MASK;
1750 kvm_x86_ops->set_segment(ctxt->vcpu, &cs, VCPU_SREG_CS);
1751 kvm_x86_ops->set_segment(ctxt->vcpu, &ss, VCPU_SREG_SS);
1753 c->eip = ctxt->vcpu->arch.regs[VCPU_REGS_RDX];
1754 c->regs[VCPU_REGS_RSP] = ctxt->vcpu->arch.regs[VCPU_REGS_RCX];
1756 return X86EMUL_CONTINUE;
1759 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1762 if (ctxt->mode == X86EMUL_MODE_REAL)
1764 if (ctxt->mode == X86EMUL_MODE_VM86)
1766 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1767 return kvm_x86_ops->get_cpl(ctxt->vcpu) > iopl;
1770 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1771 struct x86_emulate_ops *ops,
1774 struct kvm_segment tr_seg;
1777 u8 perm, bit_idx = port & 0x7;
1778 unsigned mask = (1 << len) - 1;
1780 kvm_get_segment(ctxt->vcpu, &tr_seg, VCPU_SREG_TR);
1781 if (tr_seg.unusable)
1783 if (tr_seg.limit < 103)
1785 r = ops->read_std(tr_seg.base + 102, &io_bitmap_ptr, 2, ctxt->vcpu,
1787 if (r != X86EMUL_CONTINUE)
1789 if (io_bitmap_ptr + port/8 > tr_seg.limit)
1791 r = ops->read_std(tr_seg.base + io_bitmap_ptr + port/8, &perm, 1,
1793 if (r != X86EMUL_CONTINUE)
1795 if ((perm >> bit_idx) & mask)
1800 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
1801 struct x86_emulate_ops *ops,
1804 if (emulator_bad_iopl(ctxt))
1805 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
1811 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1813 unsigned long memop = 0;
1815 unsigned long saved_eip = 0;
1816 struct decode_cache *c = &ctxt->decode;
1821 ctxt->interruptibility = 0;
1823 /* Shadow copy of register state. Committed on successful emulation.
1824 * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't
1828 memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs);
1831 /* LOCK prefix is allowed only with some instructions */
1832 if (c->lock_prefix && !(c->d & Lock)) {
1833 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
1837 /* Privileged instruction can be executed only in CPL=0 */
1838 if ((c->d & Priv) && kvm_x86_ops->get_cpl(ctxt->vcpu)) {
1839 kvm_inject_gp(ctxt->vcpu, 0);
1843 if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs))
1844 memop = c->modrm_ea;
1846 if (c->rep_prefix && (c->d & String)) {
1847 /* All REP prefixes have the same first termination condition */
1848 if (c->regs[VCPU_REGS_RCX] == 0) {
1849 kvm_rip_write(ctxt->vcpu, c->eip);
1852 /* The second termination condition only applies for REPE
1853 * and REPNE. Test if the repeat string operation prefix is
1854 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
1855 * corresponding termination condition according to:
1856 * - if REPE/REPZ and ZF = 0 then done
1857 * - if REPNE/REPNZ and ZF = 1 then done
1859 if ((c->b == 0xa6) || (c->b == 0xa7) ||
1860 (c->b == 0xae) || (c->b == 0xaf)) {
1861 if ((c->rep_prefix == REPE_PREFIX) &&
1862 ((ctxt->eflags & EFLG_ZF) == 0)) {
1863 kvm_rip_write(ctxt->vcpu, c->eip);
1866 if ((c->rep_prefix == REPNE_PREFIX) &&
1867 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) {
1868 kvm_rip_write(ctxt->vcpu, c->eip);
1872 c->regs[VCPU_REGS_RCX]--;
1873 c->eip = kvm_rip_read(ctxt->vcpu);
1876 if (c->src.type == OP_MEM) {
1877 c->src.ptr = (unsigned long *)memop;
1879 rc = ops->read_emulated((unsigned long)c->src.ptr,
1883 if (rc != X86EMUL_CONTINUE)
1885 c->src.orig_val = c->src.val;
1888 if ((c->d & DstMask) == ImplicitOps)
1892 if (c->dst.type == OP_MEM) {
1893 c->dst.ptr = (unsigned long *)memop;
1894 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1897 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1899 c->dst.ptr = (void *)c->dst.ptr +
1900 (c->src.val & mask) / 8;
1902 if (!(c->d & Mov)) {
1903 /* optimisation - avoid slow emulated read */
1904 rc = ops->read_emulated((unsigned long)c->dst.ptr,
1908 if (rc != X86EMUL_CONTINUE)
1912 c->dst.orig_val = c->dst.val;
1922 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
1924 case 0x06: /* push es */
1925 emulate_push_sreg(ctxt, VCPU_SREG_ES);
1927 case 0x07: /* pop es */
1928 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
1934 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
1936 case 0x0e: /* push cs */
1937 emulate_push_sreg(ctxt, VCPU_SREG_CS);
1941 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
1943 case 0x16: /* push ss */
1944 emulate_push_sreg(ctxt, VCPU_SREG_SS);
1946 case 0x17: /* pop ss */
1947 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
1953 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
1955 case 0x1e: /* push ds */
1956 emulate_push_sreg(ctxt, VCPU_SREG_DS);
1958 case 0x1f: /* pop ds */
1959 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
1965 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
1969 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
1973 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
1977 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
1979 case 0x40 ... 0x47: /* inc r16/r32 */
1980 emulate_1op("inc", c->dst, ctxt->eflags);
1982 case 0x48 ... 0x4f: /* dec r16/r32 */
1983 emulate_1op("dec", c->dst, ctxt->eflags);
1985 case 0x50 ... 0x57: /* push reg */
1988 case 0x58 ... 0x5f: /* pop reg */
1990 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
1994 case 0x60: /* pusha */
1995 emulate_pusha(ctxt);
1997 case 0x61: /* popa */
1998 rc = emulate_popa(ctxt, ops);
2002 case 0x63: /* movsxd */
2003 if (ctxt->mode != X86EMUL_MODE_PROT64)
2004 goto cannot_emulate;
2005 c->dst.val = (s32) c->src.val;
2007 case 0x68: /* push imm */
2008 case 0x6a: /* push imm8 */
2011 case 0x6c: /* insb */
2012 case 0x6d: /* insw/insd */
2013 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2014 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2015 kvm_inject_gp(ctxt->vcpu, 0);
2018 if (kvm_emulate_pio_string(ctxt->vcpu,
2020 (c->d & ByteOp) ? 1 : c->op_bytes,
2022 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2023 (ctxt->eflags & EFLG_DF),
2024 register_address(c, es_base(ctxt),
2025 c->regs[VCPU_REGS_RDI]),
2027 c->regs[VCPU_REGS_RDX]) == 0) {
2032 case 0x6e: /* outsb */
2033 case 0x6f: /* outsw/outsd */
2034 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2035 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2036 kvm_inject_gp(ctxt->vcpu, 0);
2039 if (kvm_emulate_pio_string(ctxt->vcpu,
2041 (c->d & ByteOp) ? 1 : c->op_bytes,
2043 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1,
2044 (ctxt->eflags & EFLG_DF),
2046 seg_override_base(ctxt, c),
2047 c->regs[VCPU_REGS_RSI]),
2049 c->regs[VCPU_REGS_RDX]) == 0) {
2054 case 0x70 ... 0x7f: /* jcc (short) */
2055 if (test_cc(c->b, ctxt->eflags))
2056 jmp_rel(c, c->src.val);
2058 case 0x80 ... 0x83: /* Grp1 */
2059 switch (c->modrm_reg) {
2079 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2081 case 0x86 ... 0x87: /* xchg */
2083 /* Write back the register source. */
2084 switch (c->dst.bytes) {
2086 *(u8 *) c->src.ptr = (u8) c->dst.val;
2089 *(u16 *) c->src.ptr = (u16) c->dst.val;
2092 *c->src.ptr = (u32) c->dst.val;
2093 break; /* 64b reg: zero-extend */
2095 *c->src.ptr = c->dst.val;
2099 * Write back the memory destination with implicit LOCK
2102 c->dst.val = c->src.val;
2105 case 0x88 ... 0x8b: /* mov */
2107 case 0x8c: { /* mov r/m, sreg */
2108 struct kvm_segment segreg;
2110 if (c->modrm_reg <= 5)
2111 kvm_get_segment(ctxt->vcpu, &segreg, c->modrm_reg);
2113 printk(KERN_INFO "0x8c: Invalid segreg in modrm byte 0x%02x\n",
2115 goto cannot_emulate;
2117 c->dst.val = segreg.selector;
2120 case 0x8d: /* lea r16/r32, m */
2121 c->dst.val = c->modrm_ea;
2123 case 0x8e: { /* mov seg, r/m16 */
2128 if (c->modrm_reg == VCPU_SREG_CS ||
2129 c->modrm_reg > VCPU_SREG_GS) {
2130 kvm_queue_exception(ctxt->vcpu, UD_VECTOR);
2134 if (c->modrm_reg == VCPU_SREG_SS)
2135 toggle_interruptibility(ctxt, X86_SHADOW_INT_MOV_SS);
2137 rc = kvm_load_segment_descriptor(ctxt->vcpu, sel, c->modrm_reg);
2139 c->dst.type = OP_NONE; /* Disable writeback. */
2142 case 0x8f: /* pop (sole member of Grp1a) */
2143 rc = emulate_grp1a(ctxt, ops);
2147 case 0x90: /* nop / xchg r8,rax */
2148 if (!(c->rex_prefix & 1)) { /* nop */
2149 c->dst.type = OP_NONE;
2152 case 0x91 ... 0x97: /* xchg reg,rax */
2153 c->src.type = c->dst.type = OP_REG;
2154 c->src.bytes = c->dst.bytes = c->op_bytes;
2155 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2156 c->src.val = *(c->src.ptr);
2158 case 0x9c: /* pushf */
2159 c->src.val = (unsigned long) ctxt->eflags;
2162 case 0x9d: /* popf */
2163 c->dst.type = OP_REG;
2164 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2165 c->dst.bytes = c->op_bytes;
2166 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2167 if (rc != X86EMUL_CONTINUE)
2170 case 0xa0 ... 0xa1: /* mov */
2171 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2172 c->dst.val = c->src.val;
2174 case 0xa2 ... 0xa3: /* mov */
2175 c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX];
2177 case 0xa4 ... 0xa5: /* movs */
2178 c->dst.type = OP_MEM;
2179 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2180 c->dst.ptr = (unsigned long *)register_address(c,
2182 c->regs[VCPU_REGS_RDI]);
2183 rc = ops->read_emulated(register_address(c,
2184 seg_override_base(ctxt, c),
2185 c->regs[VCPU_REGS_RSI]),
2187 c->dst.bytes, ctxt->vcpu);
2188 if (rc != X86EMUL_CONTINUE)
2190 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2191 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2193 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2194 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2197 case 0xa6 ... 0xa7: /* cmps */
2198 c->src.type = OP_NONE; /* Disable writeback. */
2199 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2200 c->src.ptr = (unsigned long *)register_address(c,
2201 seg_override_base(ctxt, c),
2202 c->regs[VCPU_REGS_RSI]);
2203 rc = ops->read_emulated((unsigned long)c->src.ptr,
2207 if (rc != X86EMUL_CONTINUE)
2210 c->dst.type = OP_NONE; /* Disable writeback. */
2211 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2212 c->dst.ptr = (unsigned long *)register_address(c,
2214 c->regs[VCPU_REGS_RDI]);
2215 rc = ops->read_emulated((unsigned long)c->dst.ptr,
2219 if (rc != X86EMUL_CONTINUE)
2222 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2224 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2226 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2227 (ctxt->eflags & EFLG_DF) ? -c->src.bytes
2229 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2230 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2234 case 0xaa ... 0xab: /* stos */
2235 c->dst.type = OP_MEM;
2236 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2237 c->dst.ptr = (unsigned long *)register_address(c,
2239 c->regs[VCPU_REGS_RDI]);
2240 c->dst.val = c->regs[VCPU_REGS_RAX];
2241 register_address_increment(c, &c->regs[VCPU_REGS_RDI],
2242 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2245 case 0xac ... 0xad: /* lods */
2246 c->dst.type = OP_REG;
2247 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
2248 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2249 rc = ops->read_emulated(register_address(c,
2250 seg_override_base(ctxt, c),
2251 c->regs[VCPU_REGS_RSI]),
2255 if (rc != X86EMUL_CONTINUE)
2257 register_address_increment(c, &c->regs[VCPU_REGS_RSI],
2258 (ctxt->eflags & EFLG_DF) ? -c->dst.bytes
2261 case 0xae ... 0xaf: /* scas */
2262 DPRINTF("Urk! I don't handle SCAS.\n");
2263 goto cannot_emulate;
2264 case 0xb0 ... 0xbf: /* mov r, imm */
2269 case 0xc3: /* ret */
2270 c->dst.type = OP_REG;
2271 c->dst.ptr = &c->eip;
2272 c->dst.bytes = c->op_bytes;
2273 goto pop_instruction;
2274 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2276 c->dst.val = c->src.val;
2278 case 0xcb: /* ret far */
2279 rc = emulate_ret_far(ctxt, ops);
2283 case 0xd0 ... 0xd1: /* Grp2 */
2287 case 0xd2 ... 0xd3: /* Grp2 */
2288 c->src.val = c->regs[VCPU_REGS_RCX];
2291 case 0xe4: /* inb */
2296 case 0xe6: /* outb */
2297 case 0xe7: /* out */
2301 case 0xe8: /* call (near) */ {
2302 long int rel = c->src.val;
2303 c->src.val = (unsigned long) c->eip;
2308 case 0xe9: /* jmp rel */
2310 case 0xea: /* jmp far */
2311 if (kvm_load_segment_descriptor(ctxt->vcpu, c->src2.val,
2315 c->eip = c->src.val;
2318 jmp: /* jmp rel short */
2319 jmp_rel(c, c->src.val);
2320 c->dst.type = OP_NONE; /* Disable writeback. */
2322 case 0xec: /* in al,dx */
2323 case 0xed: /* in (e/r)ax,dx */
2324 port = c->regs[VCPU_REGS_RDX];
2327 case 0xee: /* out al,dx */
2328 case 0xef: /* out (e/r)ax,dx */
2329 port = c->regs[VCPU_REGS_RDX];
2332 if (!emulator_io_permited(ctxt, ops, port,
2333 (c->d & ByteOp) ? 1 : c->op_bytes)) {
2334 kvm_inject_gp(ctxt->vcpu, 0);
2337 if (kvm_emulate_pio(ctxt->vcpu, io_dir_in,
2338 (c->d & ByteOp) ? 1 : c->op_bytes,
2341 goto cannot_emulate;
2344 case 0xf4: /* hlt */
2345 ctxt->vcpu->arch.halt_request = 1;
2347 case 0xf5: /* cmc */
2348 /* complement carry flag from eflags reg */
2349 ctxt->eflags ^= EFLG_CF;
2350 c->dst.type = OP_NONE; /* Disable writeback. */
2352 case 0xf6 ... 0xf7: /* Grp3 */
2353 rc = emulate_grp3(ctxt, ops);
2357 case 0xf8: /* clc */
2358 ctxt->eflags &= ~EFLG_CF;
2359 c->dst.type = OP_NONE; /* Disable writeback. */
2361 case 0xfa: /* cli */
2362 if (emulator_bad_iopl(ctxt))
2363 kvm_inject_gp(ctxt->vcpu, 0);
2365 ctxt->eflags &= ~X86_EFLAGS_IF;
2366 c->dst.type = OP_NONE; /* Disable writeback. */
2369 case 0xfb: /* sti */
2370 if (emulator_bad_iopl(ctxt))
2371 kvm_inject_gp(ctxt->vcpu, 0);
2373 toggle_interruptibility(ctxt, X86_SHADOW_INT_STI);
2374 ctxt->eflags |= X86_EFLAGS_IF;
2375 c->dst.type = OP_NONE; /* Disable writeback. */
2378 case 0xfc: /* cld */
2379 ctxt->eflags &= ~EFLG_DF;
2380 c->dst.type = OP_NONE; /* Disable writeback. */
2382 case 0xfd: /* std */
2383 ctxt->eflags |= EFLG_DF;
2384 c->dst.type = OP_NONE; /* Disable writeback. */
2386 case 0xfe ... 0xff: /* Grp4/Grp5 */
2387 rc = emulate_grp45(ctxt, ops);
2394 rc = writeback(ctxt, ops);
2398 /* Commit shadow register state. */
2399 memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs);
2400 kvm_rip_write(ctxt->vcpu, c->eip);
2403 if (rc == X86EMUL_UNHANDLEABLE) {
2411 case 0x01: /* lgdt, lidt, lmsw */
2412 switch (c->modrm_reg) {
2414 unsigned long address;
2416 case 0: /* vmcall */
2417 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2418 goto cannot_emulate;
2420 rc = kvm_fix_hypercall(ctxt->vcpu);
2424 /* Let the processor re-execute the fixed hypercall */
2425 c->eip = kvm_rip_read(ctxt->vcpu);
2426 /* Disable writeback. */
2427 c->dst.type = OP_NONE;
2430 rc = read_descriptor(ctxt, ops, c->src.ptr,
2431 &size, &address, c->op_bytes);
2434 realmode_lgdt(ctxt->vcpu, size, address);
2435 /* Disable writeback. */
2436 c->dst.type = OP_NONE;
2438 case 3: /* lidt/vmmcall */
2439 if (c->modrm_mod == 3) {
2440 switch (c->modrm_rm) {
2442 rc = kvm_fix_hypercall(ctxt->vcpu);
2447 goto cannot_emulate;
2450 rc = read_descriptor(ctxt, ops, c->src.ptr,
2455 realmode_lidt(ctxt->vcpu, size, address);
2457 /* Disable writeback. */
2458 c->dst.type = OP_NONE;
2462 c->dst.val = realmode_get_cr(ctxt->vcpu, 0);
2465 realmode_lmsw(ctxt->vcpu, (u16)c->src.val,
2467 c->dst.type = OP_NONE;
2470 emulate_invlpg(ctxt->vcpu, memop);
2471 /* Disable writeback. */
2472 c->dst.type = OP_NONE;
2475 goto cannot_emulate;
2478 case 0x05: /* syscall */
2479 rc = emulate_syscall(ctxt);
2480 if (rc != X86EMUL_CONTINUE)
2486 emulate_clts(ctxt->vcpu);
2487 c->dst.type = OP_NONE;
2489 case 0x08: /* invd */
2490 case 0x09: /* wbinvd */
2491 case 0x0d: /* GrpP (prefetch) */
2492 case 0x18: /* Grp16 (prefetch/nop) */
2493 c->dst.type = OP_NONE;
2495 case 0x20: /* mov cr, reg */
2496 if (c->modrm_mod != 3)
2497 goto cannot_emulate;
2498 c->regs[c->modrm_rm] =
2499 realmode_get_cr(ctxt->vcpu, c->modrm_reg);
2500 c->dst.type = OP_NONE; /* no writeback */
2502 case 0x21: /* mov from dr to reg */
2503 if (c->modrm_mod != 3)
2504 goto cannot_emulate;
2505 rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]);
2507 goto cannot_emulate;
2508 c->dst.type = OP_NONE; /* no writeback */
2510 case 0x22: /* mov reg, cr */
2511 if (c->modrm_mod != 3)
2512 goto cannot_emulate;
2513 realmode_set_cr(ctxt->vcpu,
2514 c->modrm_reg, c->modrm_val, &ctxt->eflags);
2515 c->dst.type = OP_NONE;
2517 case 0x23: /* mov from reg to dr */
2518 if (c->modrm_mod != 3)
2519 goto cannot_emulate;
2520 rc = emulator_set_dr(ctxt, c->modrm_reg,
2521 c->regs[c->modrm_rm]);
2523 goto cannot_emulate;
2524 c->dst.type = OP_NONE; /* no writeback */
2528 msr_data = (u32)c->regs[VCPU_REGS_RAX]
2529 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
2530 rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data);
2532 kvm_inject_gp(ctxt->vcpu, 0);
2533 c->eip = kvm_rip_read(ctxt->vcpu);
2535 rc = X86EMUL_CONTINUE;
2536 c->dst.type = OP_NONE;
2540 rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data);
2542 kvm_inject_gp(ctxt->vcpu, 0);
2543 c->eip = kvm_rip_read(ctxt->vcpu);
2545 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
2546 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
2548 rc = X86EMUL_CONTINUE;
2549 c->dst.type = OP_NONE;
2551 case 0x34: /* sysenter */
2552 rc = emulate_sysenter(ctxt);
2553 if (rc != X86EMUL_CONTINUE)
2558 case 0x35: /* sysexit */
2559 rc = emulate_sysexit(ctxt);
2560 if (rc != X86EMUL_CONTINUE)
2565 case 0x40 ... 0x4f: /* cmov */
2566 c->dst.val = c->dst.orig_val = c->src.val;
2567 if (!test_cc(c->b, ctxt->eflags))
2568 c->dst.type = OP_NONE; /* no writeback */
2570 case 0x80 ... 0x8f: /* jnz rel, etc*/
2571 if (test_cc(c->b, ctxt->eflags))
2572 jmp_rel(c, c->src.val);
2573 c->dst.type = OP_NONE;
2575 case 0xa0: /* push fs */
2576 emulate_push_sreg(ctxt, VCPU_SREG_FS);
2578 case 0xa1: /* pop fs */
2579 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
2585 c->dst.type = OP_NONE;
2586 /* only subword offset */
2587 c->src.val &= (c->dst.bytes << 3) - 1;
2588 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
2590 case 0xa4: /* shld imm8, r, r/m */
2591 case 0xa5: /* shld cl, r, r/m */
2592 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
2594 case 0xa8: /* push gs */
2595 emulate_push_sreg(ctxt, VCPU_SREG_GS);
2597 case 0xa9: /* pop gs */
2598 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
2604 /* only subword offset */
2605 c->src.val &= (c->dst.bytes << 3) - 1;
2606 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
2608 case 0xac: /* shrd imm8, r, r/m */
2609 case 0xad: /* shrd cl, r, r/m */
2610 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
2612 case 0xae: /* clflush */
2614 case 0xb0 ... 0xb1: /* cmpxchg */
2616 * Save real source value, then compare EAX against
2619 c->src.orig_val = c->src.val;
2620 c->src.val = c->regs[VCPU_REGS_RAX];
2621 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2622 if (ctxt->eflags & EFLG_ZF) {
2623 /* Success: write back to memory. */
2624 c->dst.val = c->src.orig_val;
2626 /* Failure: write the value we saw to EAX. */
2627 c->dst.type = OP_REG;
2628 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
2633 /* only subword offset */
2634 c->src.val &= (c->dst.bytes << 3) - 1;
2635 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
2637 case 0xb6 ... 0xb7: /* movzx */
2638 c->dst.bytes = c->op_bytes;
2639 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
2642 case 0xba: /* Grp8 */
2643 switch (c->modrm_reg & 3) {
2656 /* only subword offset */
2657 c->src.val &= (c->dst.bytes << 3) - 1;
2658 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
2660 case 0xbe ... 0xbf: /* movsx */
2661 c->dst.bytes = c->op_bytes;
2662 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
2665 case 0xc3: /* movnti */
2666 c->dst.bytes = c->op_bytes;
2667 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
2670 case 0xc7: /* Grp9 (cmpxchg8b) */
2671 rc = emulate_grp9(ctxt, ops, memop);
2674 c->dst.type = OP_NONE;
2680 DPRINTF("Cannot emulate %02x\n", c->b);