1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
154 #define NoWrite ((u64)1 << 45) /* No writeback */
156 #define X2(x...) x, x
157 #define X3(x...) X2(x), x
158 #define X4(x...) X2(x), X2(x)
159 #define X5(x...) X4(x), x
160 #define X6(x...) X4(x), X2(x)
161 #define X7(x...) X4(x), X3(x)
162 #define X8(x...) X4(x), X4(x)
163 #define X16(x...) X8(x), X8(x)
165 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
166 #define FASTOP_SIZE 8
169 * fastop functions have a special calling convention:
171 * dst: [rdx]:rax (in/out)
174 * flags: rflags (in/out)
176 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
177 * different operand sizes can be reached by calculation, rather than a jump
178 * table (which would be bigger than the code).
180 * fastop functions are declared as taking a never-defined fastop parameter,
181 * so they can't be called from C directly.
190 int (*execute)(struct x86_emulate_ctxt *ctxt);
191 const struct opcode *group;
192 const struct group_dual *gdual;
193 const struct gprefix *gprefix;
194 const struct escape *esc;
195 void (*fastop)(struct fastop *fake);
197 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201 struct opcode mod012[8];
202 struct opcode mod3[8];
206 struct opcode pfx_no;
207 struct opcode pfx_66;
208 struct opcode pfx_f2;
209 struct opcode pfx_f3;
214 struct opcode high[64];
217 /* EFLAGS bit definitions. */
218 #define EFLG_ID (1<<21)
219 #define EFLG_VIP (1<<20)
220 #define EFLG_VIF (1<<19)
221 #define EFLG_AC (1<<18)
222 #define EFLG_VM (1<<17)
223 #define EFLG_RF (1<<16)
224 #define EFLG_IOPL (3<<12)
225 #define EFLG_NT (1<<14)
226 #define EFLG_OF (1<<11)
227 #define EFLG_DF (1<<10)
228 #define EFLG_IF (1<<9)
229 #define EFLG_TF (1<<8)
230 #define EFLG_SF (1<<7)
231 #define EFLG_ZF (1<<6)
232 #define EFLG_AF (1<<4)
233 #define EFLG_PF (1<<2)
234 #define EFLG_CF (1<<0)
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
237 #define EFLG_RESERVED_ONE_MASK 2
239 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
241 if (!(ctxt->regs_valid & (1 << nr))) {
242 ctxt->regs_valid |= 1 << nr;
243 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
245 return ctxt->_regs[nr];
248 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 ctxt->regs_valid |= 1 << nr;
251 ctxt->regs_dirty |= 1 << nr;
252 return &ctxt->_regs[nr];
255 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 return reg_write(ctxt, nr);
261 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
265 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
266 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
269 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
271 ctxt->regs_dirty = 0;
272 ctxt->regs_valid = 0;
276 * Instruction emulation:
277 * Most instructions are emulated directly via a fragment of inline assembly
278 * code. This allows us to save/restore EFLAGS and thus very easily pick up
279 * any modified flags.
282 #if defined(CONFIG_X86_64)
283 #define _LO32 "k" /* force 32-bit operand */
284 #define _STK "%%rsp" /* stack pointer */
285 #elif defined(__i386__)
286 #define _LO32 "" /* force 32-bit operand */
287 #define _STK "%%esp" /* stack pointer */
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
296 /* Before executing instruction: restore necessary bits in EFLAGS. */
297 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
298 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
299 "movl %"_sav",%"_LO32 _tmp"; " \
302 "movl %"_msk",%"_LO32 _tmp"; " \
303 "andl %"_LO32 _tmp",("_STK"); " \
305 "notl %"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
307 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
309 "orl %"_LO32 _tmp",("_STK"); " \
313 /* After executing instruction: write-back necessary bits in EFLAGS. */
314 #define _POST_EFLAGS(_sav, _msk, _tmp) \
315 /* _sav |= EFLAGS & _msk; */ \
318 "andl %"_msk",%"_LO32 _tmp"; " \
319 "orl %"_LO32 _tmp",%"_sav"; "
327 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
329 __asm__ __volatile__ ( \
330 _PRE_EFLAGS("0", "4", "2") \
331 _op _suffix " %"_x"3,%1; " \
332 _POST_EFLAGS("0", "4", "2") \
333 : "=m" ((ctxt)->eflags), \
334 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
336 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
340 /* Raw emulation: instruction has two explicit operands. */
341 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
343 unsigned long _tmp; \
345 switch ((ctxt)->dst.bytes) { \
347 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
350 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
353 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
360 unsigned long _tmp; \
361 switch ((ctxt)->dst.bytes) { \
363 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
366 __emulate_2op_nobyte(ctxt, _op, \
367 _wx, _wy, _lx, _ly, _qx, _qy); \
372 /* Source operand is byte-sized and may be restricted to just %cl. */
373 #define emulate_2op_SrcB(ctxt, _op) \
374 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
376 /* Source operand is byte, word, long or quad sized. */
377 #define emulate_2op_SrcV(ctxt, _op) \
378 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
380 /* Source operand is word, long or quad sized. */
381 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
382 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
384 /* Instruction has three operands and one operand is stored in ECX register */
385 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
387 unsigned long _tmp; \
388 _type _clv = (ctxt)->src2.val; \
389 _type _srcv = (ctxt)->src.val; \
390 _type _dstv = (ctxt)->dst.val; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "5", "2") \
394 _op _suffix " %4,%1 \n" \
395 _POST_EFLAGS("0", "5", "2") \
396 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
400 (ctxt)->src2.val = (unsigned long) _clv; \
401 (ctxt)->src2.val = (unsigned long) _srcv; \
402 (ctxt)->dst.val = (unsigned long) _dstv; \
405 #define emulate_2op_cl(ctxt, _op) \
407 switch ((ctxt)->dst.bytes) { \
409 __emulate_2op_cl(ctxt, _op, "w", u16); \
412 __emulate_2op_cl(ctxt, _op, "l", u32); \
415 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
420 #define __emulate_1op(ctxt, _op, _suffix) \
422 unsigned long _tmp; \
424 __asm__ __volatile__ ( \
425 _PRE_EFLAGS("0", "3", "2") \
426 _op _suffix " %1; " \
427 _POST_EFLAGS("0", "3", "2") \
428 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
430 : "i" (EFLAGS_MASK)); \
433 /* Instruction has only one explicit operand (no source operand). */
434 #define emulate_1op(ctxt, _op) \
436 switch ((ctxt)->dst.bytes) { \
437 case 1: __emulate_1op(ctxt, _op, "b"); break; \
438 case 2: __emulate_1op(ctxt, _op, "w"); break; \
439 case 4: __emulate_1op(ctxt, _op, "l"); break; \
440 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
444 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
445 #define FOP_RET "ret \n\t"
447 #define FOP_START(op) \
448 extern void em_##op(struct fastop *fake); \
449 asm(".pushsection .text, \"ax\" \n\t" \
450 ".global em_" #op " \n\t" \
457 #define FOPNOP() FOP_ALIGN FOP_RET
459 #define FOP1E(op, dst) \
460 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
462 #define FASTOP1(op) \
467 ON64(FOP1E(op##q, rax)) \
470 #define FOP2E(op, dst, src) \
471 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
473 #define FASTOP2(op) \
475 FOP2E(op##b, al, bl) \
476 FOP2E(op##w, ax, bx) \
477 FOP2E(op##l, eax, ebx) \
478 ON64(FOP2E(op##q, rax, rbx)) \
481 #define FOP3E(op, dst, src, src2) \
482 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
484 /* 3-operand, word-only, src2=cl */
485 #define FASTOP3WCL(op) \
488 FOP3E(op##w, ax, bx, cl) \
489 FOP3E(op##l, eax, ebx, cl) \
490 ON64(FOP3E(op##q, rax, rbx, cl)) \
493 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
495 unsigned long _tmp; \
496 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
497 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
499 __asm__ __volatile__ ( \
500 _PRE_EFLAGS("0", "5", "1") \
502 _op _suffix " %6; " \
504 _POST_EFLAGS("0", "5", "1") \
505 ".pushsection .fixup,\"ax\" \n\t" \
506 "3: movb $1, %4 \n\t" \
509 _ASM_EXTABLE(1b, 3b) \
510 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
511 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
512 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
515 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
516 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
518 switch((ctxt)->src.bytes) { \
520 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
523 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
526 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
529 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
534 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
535 enum x86_intercept intercept,
536 enum x86_intercept_stage stage)
538 struct x86_instruction_info info = {
539 .intercept = intercept,
540 .rep_prefix = ctxt->rep_prefix,
541 .modrm_mod = ctxt->modrm_mod,
542 .modrm_reg = ctxt->modrm_reg,
543 .modrm_rm = ctxt->modrm_rm,
544 .src_val = ctxt->src.val64,
545 .src_bytes = ctxt->src.bytes,
546 .dst_bytes = ctxt->dst.bytes,
547 .ad_bytes = ctxt->ad_bytes,
548 .next_rip = ctxt->eip,
551 return ctxt->ops->intercept(ctxt, &info, stage);
554 static void assign_masked(ulong *dest, ulong src, ulong mask)
556 *dest = (*dest & ~mask) | (src & mask);
559 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
561 return (1UL << (ctxt->ad_bytes << 3)) - 1;
564 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
567 struct desc_struct ss;
569 if (ctxt->mode == X86EMUL_MODE_PROT64)
571 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
572 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
575 static int stack_size(struct x86_emulate_ctxt *ctxt)
577 return (__fls(stack_mask(ctxt)) + 1) >> 3;
580 /* Access/update address held in a register, based on addressing mode. */
581 static inline unsigned long
582 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
584 if (ctxt->ad_bytes == sizeof(unsigned long))
587 return reg & ad_mask(ctxt);
590 static inline unsigned long
591 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
593 return address_mask(ctxt, reg);
596 static void masked_increment(ulong *reg, ulong mask, int inc)
598 assign_masked(reg, *reg + inc, mask);
602 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
606 if (ctxt->ad_bytes == sizeof(unsigned long))
609 mask = ad_mask(ctxt);
610 masked_increment(reg, mask, inc);
613 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
615 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
618 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
620 register_address_increment(ctxt, &ctxt->_eip, rel);
623 static u32 desc_limit_scaled(struct desc_struct *desc)
625 u32 limit = get_desc_limit(desc);
627 return desc->g ? (limit << 12) | 0xfff : limit;
630 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
632 ctxt->has_seg_override = true;
633 ctxt->seg_override = seg;
636 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
638 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
641 return ctxt->ops->get_cached_segment_base(ctxt, seg);
644 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
646 if (!ctxt->has_seg_override)
649 return ctxt->seg_override;
652 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
653 u32 error, bool valid)
655 ctxt->exception.vector = vec;
656 ctxt->exception.error_code = error;
657 ctxt->exception.error_code_valid = valid;
658 return X86EMUL_PROPAGATE_FAULT;
661 static int emulate_db(struct x86_emulate_ctxt *ctxt)
663 return emulate_exception(ctxt, DB_VECTOR, 0, false);
666 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
668 return emulate_exception(ctxt, GP_VECTOR, err, true);
671 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
673 return emulate_exception(ctxt, SS_VECTOR, err, true);
676 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
678 return emulate_exception(ctxt, UD_VECTOR, 0, false);
681 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
683 return emulate_exception(ctxt, TS_VECTOR, err, true);
686 static int emulate_de(struct x86_emulate_ctxt *ctxt)
688 return emulate_exception(ctxt, DE_VECTOR, 0, false);
691 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
693 return emulate_exception(ctxt, NM_VECTOR, 0, false);
696 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
699 struct desc_struct desc;
701 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
705 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
710 struct desc_struct desc;
712 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
713 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
717 * x86 defines three classes of vector instructions: explicitly
718 * aligned, explicitly unaligned, and the rest, which change behaviour
719 * depending on whether they're AVX encoded or not.
721 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
722 * subject to the same check.
724 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
726 if (likely(size < 16))
729 if (ctxt->d & Aligned)
731 else if (ctxt->d & Unaligned)
733 else if (ctxt->d & Avx)
739 static int __linearize(struct x86_emulate_ctxt *ctxt,
740 struct segmented_address addr,
741 unsigned size, bool write, bool fetch,
744 struct desc_struct desc;
751 la = seg_base(ctxt, addr.seg) + addr.ea;
752 switch (ctxt->mode) {
753 case X86EMUL_MODE_PROT64:
754 if (((signed long)la << 16) >> 16 != la)
755 return emulate_gp(ctxt, 0);
758 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
762 /* code segment in protected mode or read-only data segment */
763 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
764 || !(desc.type & 2)) && write)
766 /* unreadable code segment */
767 if (!fetch && (desc.type & 8) && !(desc.type & 2))
769 lim = desc_limit_scaled(&desc);
770 if ((desc.type & 8) || !(desc.type & 4)) {
771 /* expand-up segment */
772 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
775 /* expand-down segment */
776 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
778 lim = desc.d ? 0xffffffff : 0xffff;
779 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
782 cpl = ctxt->ops->cpl(ctxt);
783 if (!(desc.type & 8)) {
787 } else if ((desc.type & 8) && !(desc.type & 4)) {
788 /* nonconforming code segment */
791 } else if ((desc.type & 8) && (desc.type & 4)) {
792 /* conforming code segment */
798 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
800 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
801 return emulate_gp(ctxt, 0);
803 return X86EMUL_CONTINUE;
805 if (addr.seg == VCPU_SREG_SS)
806 return emulate_ss(ctxt, sel);
808 return emulate_gp(ctxt, sel);
811 static int linearize(struct x86_emulate_ctxt *ctxt,
812 struct segmented_address addr,
813 unsigned size, bool write,
816 return __linearize(ctxt, addr, size, write, false, linear);
820 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
821 struct segmented_address addr,
828 rc = linearize(ctxt, addr, size, false, &linear);
829 if (rc != X86EMUL_CONTINUE)
831 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
835 * Fetch the next byte of the instruction being emulated which is pointed to
836 * by ctxt->_eip, then increment ctxt->_eip.
838 * Also prefetch the remaining bytes of the instruction without crossing page
839 * boundary if they are not in fetch_cache yet.
841 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
843 struct fetch_cache *fc = &ctxt->fetch;
847 if (ctxt->_eip == fc->end) {
848 unsigned long linear;
849 struct segmented_address addr = { .seg = VCPU_SREG_CS,
851 cur_size = fc->end - fc->start;
852 size = min(15UL - cur_size,
853 PAGE_SIZE - offset_in_page(ctxt->_eip));
854 rc = __linearize(ctxt, addr, size, false, true, &linear);
855 if (unlikely(rc != X86EMUL_CONTINUE))
857 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
858 size, &ctxt->exception);
859 if (unlikely(rc != X86EMUL_CONTINUE))
863 *dest = fc->data[ctxt->_eip - fc->start];
865 return X86EMUL_CONTINUE;
868 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
869 void *dest, unsigned size)
873 /* x86 instructions are limited to 15 bytes. */
874 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
875 return X86EMUL_UNHANDLEABLE;
877 rc = do_insn_fetch_byte(ctxt, dest++);
878 if (rc != X86EMUL_CONTINUE)
881 return X86EMUL_CONTINUE;
884 /* Fetch next part of the instruction being emulated. */
885 #define insn_fetch(_type, _ctxt) \
886 ({ unsigned long _x; \
887 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
888 if (rc != X86EMUL_CONTINUE) \
893 #define insn_fetch_arr(_arr, _size, _ctxt) \
894 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
895 if (rc != X86EMUL_CONTINUE) \
900 * Given the 'reg' portion of a ModRM byte, and a register block, return a
901 * pointer into the block that addresses the relevant register.
902 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
904 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
909 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
910 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
912 p = reg_rmw(ctxt, modrm_reg);
916 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
917 struct segmented_address addr,
918 u16 *size, unsigned long *address, int op_bytes)
925 rc = segmented_read_std(ctxt, addr, size, 2);
926 if (rc != X86EMUL_CONTINUE)
929 rc = segmented_read_std(ctxt, addr, address, op_bytes);
933 static int test_cc(unsigned int condition, unsigned int flags)
937 switch ((condition & 15) >> 1) {
939 rc |= (flags & EFLG_OF);
941 case 1: /* b/c/nae */
942 rc |= (flags & EFLG_CF);
945 rc |= (flags & EFLG_ZF);
948 rc |= (flags & (EFLG_CF|EFLG_ZF));
951 rc |= (flags & EFLG_SF);
954 rc |= (flags & EFLG_PF);
957 rc |= (flags & EFLG_ZF);
960 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
964 /* Odd condition identifiers (lsb == 1) have inverted sense. */
965 return (!!rc ^ (condition & 1));
968 static void fetch_register_operand(struct operand *op)
972 op->val = *(u8 *)op->addr.reg;
975 op->val = *(u16 *)op->addr.reg;
978 op->val = *(u32 *)op->addr.reg;
981 op->val = *(u64 *)op->addr.reg;
986 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
988 ctxt->ops->get_fpu(ctxt);
990 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
991 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
992 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
993 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
994 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
995 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
996 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
997 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
999 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1000 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1001 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1002 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1003 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1004 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1005 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1006 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1010 ctxt->ops->put_fpu(ctxt);
1013 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1016 ctxt->ops->get_fpu(ctxt);
1018 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1019 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1020 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1021 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1022 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1023 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1024 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1025 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1026 #ifdef CONFIG_X86_64
1027 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1028 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1029 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1030 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1031 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1032 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1033 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1034 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1038 ctxt->ops->put_fpu(ctxt);
1041 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1043 ctxt->ops->get_fpu(ctxt);
1045 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1046 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1047 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1048 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1049 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1050 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1051 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1052 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1055 ctxt->ops->put_fpu(ctxt);
1058 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1060 ctxt->ops->get_fpu(ctxt);
1062 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1063 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1064 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1065 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1066 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1067 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1068 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1069 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1072 ctxt->ops->put_fpu(ctxt);
1075 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1077 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1078 return emulate_nm(ctxt);
1080 ctxt->ops->get_fpu(ctxt);
1081 asm volatile("fninit");
1082 ctxt->ops->put_fpu(ctxt);
1083 return X86EMUL_CONTINUE;
1086 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1090 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1091 return emulate_nm(ctxt);
1093 ctxt->ops->get_fpu(ctxt);
1094 asm volatile("fnstcw %0": "+m"(fcw));
1095 ctxt->ops->put_fpu(ctxt);
1097 /* force 2 byte destination */
1098 ctxt->dst.bytes = 2;
1099 ctxt->dst.val = fcw;
1101 return X86EMUL_CONTINUE;
1104 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1108 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1109 return emulate_nm(ctxt);
1111 ctxt->ops->get_fpu(ctxt);
1112 asm volatile("fnstsw %0": "+m"(fsw));
1113 ctxt->ops->put_fpu(ctxt);
1115 /* force 2 byte destination */
1116 ctxt->dst.bytes = 2;
1117 ctxt->dst.val = fsw;
1119 return X86EMUL_CONTINUE;
1122 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1125 unsigned reg = ctxt->modrm_reg;
1126 int highbyte_regs = ctxt->rex_prefix == 0;
1128 if (!(ctxt->d & ModRM))
1129 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1131 if (ctxt->d & Sse) {
1135 read_sse_reg(ctxt, &op->vec_val, reg);
1138 if (ctxt->d & Mmx) {
1147 if (ctxt->d & ByteOp) {
1148 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1151 op->addr.reg = decode_register(ctxt, reg, 0);
1152 op->bytes = ctxt->op_bytes;
1154 fetch_register_operand(op);
1155 op->orig_val = op->val;
1158 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1160 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1161 ctxt->modrm_seg = VCPU_SREG_SS;
1164 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1168 int index_reg = 0, base_reg = 0, scale;
1169 int rc = X86EMUL_CONTINUE;
1172 if (ctxt->rex_prefix) {
1173 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1174 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1175 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1178 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1179 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1180 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1181 ctxt->modrm_seg = VCPU_SREG_DS;
1183 if (ctxt->modrm_mod == 3) {
1185 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1186 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1187 if (ctxt->d & Sse) {
1190 op->addr.xmm = ctxt->modrm_rm;
1191 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1194 if (ctxt->d & Mmx) {
1197 op->addr.xmm = ctxt->modrm_rm & 7;
1200 fetch_register_operand(op);
1206 if (ctxt->ad_bytes == 2) {
1207 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1208 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1209 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1210 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1212 /* 16-bit ModR/M decode. */
1213 switch (ctxt->modrm_mod) {
1215 if (ctxt->modrm_rm == 6)
1216 modrm_ea += insn_fetch(u16, ctxt);
1219 modrm_ea += insn_fetch(s8, ctxt);
1222 modrm_ea += insn_fetch(u16, ctxt);
1225 switch (ctxt->modrm_rm) {
1227 modrm_ea += bx + si;
1230 modrm_ea += bx + di;
1233 modrm_ea += bp + si;
1236 modrm_ea += bp + di;
1245 if (ctxt->modrm_mod != 0)
1252 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1253 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1254 ctxt->modrm_seg = VCPU_SREG_SS;
1255 modrm_ea = (u16)modrm_ea;
1257 /* 32/64-bit ModR/M decode. */
1258 if ((ctxt->modrm_rm & 7) == 4) {
1259 sib = insn_fetch(u8, ctxt);
1260 index_reg |= (sib >> 3) & 7;
1261 base_reg |= sib & 7;
1264 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1265 modrm_ea += insn_fetch(s32, ctxt);
1267 modrm_ea += reg_read(ctxt, base_reg);
1268 adjust_modrm_seg(ctxt, base_reg);
1271 modrm_ea += reg_read(ctxt, index_reg) << scale;
1272 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1273 if (ctxt->mode == X86EMUL_MODE_PROT64)
1274 ctxt->rip_relative = 1;
1276 base_reg = ctxt->modrm_rm;
1277 modrm_ea += reg_read(ctxt, base_reg);
1278 adjust_modrm_seg(ctxt, base_reg);
1280 switch (ctxt->modrm_mod) {
1282 if (ctxt->modrm_rm == 5)
1283 modrm_ea += insn_fetch(s32, ctxt);
1286 modrm_ea += insn_fetch(s8, ctxt);
1289 modrm_ea += insn_fetch(s32, ctxt);
1293 op->addr.mem.ea = modrm_ea;
1298 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1301 int rc = X86EMUL_CONTINUE;
1304 switch (ctxt->ad_bytes) {
1306 op->addr.mem.ea = insn_fetch(u16, ctxt);
1309 op->addr.mem.ea = insn_fetch(u32, ctxt);
1312 op->addr.mem.ea = insn_fetch(u64, ctxt);
1319 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1323 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1324 mask = ~(ctxt->dst.bytes * 8 - 1);
1326 if (ctxt->src.bytes == 2)
1327 sv = (s16)ctxt->src.val & (s16)mask;
1328 else if (ctxt->src.bytes == 4)
1329 sv = (s32)ctxt->src.val & (s32)mask;
1331 ctxt->dst.addr.mem.ea += (sv >> 3);
1334 /* only subword offset */
1335 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1338 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1339 unsigned long addr, void *dest, unsigned size)
1342 struct read_cache *mc = &ctxt->mem_read;
1344 if (mc->pos < mc->end)
1347 WARN_ON((mc->end + size) >= sizeof(mc->data));
1349 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1351 if (rc != X86EMUL_CONTINUE)
1357 memcpy(dest, mc->data + mc->pos, size);
1359 return X86EMUL_CONTINUE;
1362 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1363 struct segmented_address addr,
1370 rc = linearize(ctxt, addr, size, false, &linear);
1371 if (rc != X86EMUL_CONTINUE)
1373 return read_emulated(ctxt, linear, data, size);
1376 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1377 struct segmented_address addr,
1384 rc = linearize(ctxt, addr, size, true, &linear);
1385 if (rc != X86EMUL_CONTINUE)
1387 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1391 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1392 struct segmented_address addr,
1393 const void *orig_data, const void *data,
1399 rc = linearize(ctxt, addr, size, true, &linear);
1400 if (rc != X86EMUL_CONTINUE)
1402 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1403 size, &ctxt->exception);
1406 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1407 unsigned int size, unsigned short port,
1410 struct read_cache *rc = &ctxt->io_read;
1412 if (rc->pos == rc->end) { /* refill pio read ahead */
1413 unsigned int in_page, n;
1414 unsigned int count = ctxt->rep_prefix ?
1415 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1416 in_page = (ctxt->eflags & EFLG_DF) ?
1417 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1418 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1419 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1423 rc->pos = rc->end = 0;
1424 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1429 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1430 ctxt->dst.data = rc->data + rc->pos;
1431 ctxt->dst.type = OP_MEM_STR;
1432 ctxt->dst.count = (rc->end - rc->pos) / size;
1435 memcpy(dest, rc->data + rc->pos, size);
1441 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1442 u16 index, struct desc_struct *desc)
1447 ctxt->ops->get_idt(ctxt, &dt);
1449 if (dt.size < index * 8 + 7)
1450 return emulate_gp(ctxt, index << 3 | 0x2);
1452 addr = dt.address + index * 8;
1453 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1457 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1458 u16 selector, struct desc_ptr *dt)
1460 const struct x86_emulate_ops *ops = ctxt->ops;
1462 if (selector & 1 << 2) {
1463 struct desc_struct desc;
1466 memset (dt, 0, sizeof *dt);
1467 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1470 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1471 dt->address = get_desc_base(&desc);
1473 ops->get_gdt(ctxt, dt);
1476 /* allowed just for 8 bytes segments */
1477 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1478 u16 selector, struct desc_struct *desc,
1482 u16 index = selector >> 3;
1485 get_descriptor_table_ptr(ctxt, selector, &dt);
1487 if (dt.size < index * 8 + 7)
1488 return emulate_gp(ctxt, selector & 0xfffc);
1490 *desc_addr_p = addr = dt.address + index * 8;
1491 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1495 /* allowed just for 8 bytes segments */
1496 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1497 u16 selector, struct desc_struct *desc)
1500 u16 index = selector >> 3;
1503 get_descriptor_table_ptr(ctxt, selector, &dt);
1505 if (dt.size < index * 8 + 7)
1506 return emulate_gp(ctxt, selector & 0xfffc);
1508 addr = dt.address + index * 8;
1509 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1513 /* Does not support long mode */
1514 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1515 u16 selector, int seg)
1517 struct desc_struct seg_desc, old_desc;
1519 unsigned err_vec = GP_VECTOR;
1521 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1526 memset(&seg_desc, 0, sizeof seg_desc);
1528 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1529 || ctxt->mode == X86EMUL_MODE_REAL) {
1530 /* set real mode segment descriptor */
1531 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1532 set_desc_base(&seg_desc, selector << 4);
1537 cpl = ctxt->ops->cpl(ctxt);
1539 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1540 if ((seg == VCPU_SREG_CS
1541 || (seg == VCPU_SREG_SS
1542 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1543 || seg == VCPU_SREG_TR)
1547 /* TR should be in GDT only */
1548 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1551 if (null_selector) /* for NULL selector skip all following checks */
1554 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1555 if (ret != X86EMUL_CONTINUE)
1558 err_code = selector & 0xfffc;
1559 err_vec = GP_VECTOR;
1561 /* can't load system descriptor into segment selector */
1562 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1566 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1575 * segment is not a writable data segment or segment
1576 * selector's RPL != CPL or segment selector's RPL != CPL
1578 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1582 if (!(seg_desc.type & 8))
1585 if (seg_desc.type & 4) {
1591 if (rpl > cpl || dpl != cpl)
1594 /* CS(RPL) <- CPL */
1595 selector = (selector & 0xfffc) | cpl;
1598 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1600 old_desc = seg_desc;
1601 seg_desc.type |= 2; /* busy */
1602 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1603 sizeof(seg_desc), &ctxt->exception);
1604 if (ret != X86EMUL_CONTINUE)
1607 case VCPU_SREG_LDTR:
1608 if (seg_desc.s || seg_desc.type != 2)
1611 default: /* DS, ES, FS, or GS */
1613 * segment is not a data or readable code segment or
1614 * ((segment is a data or nonconforming code segment)
1615 * and (both RPL and CPL > DPL))
1617 if ((seg_desc.type & 0xa) == 0x8 ||
1618 (((seg_desc.type & 0xc) != 0xc) &&
1619 (rpl > dpl && cpl > dpl)))
1625 /* mark segment as accessed */
1627 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1628 if (ret != X86EMUL_CONTINUE)
1632 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1633 return X86EMUL_CONTINUE;
1635 emulate_exception(ctxt, err_vec, err_code, true);
1636 return X86EMUL_PROPAGATE_FAULT;
1639 static void write_register_operand(struct operand *op)
1641 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1642 switch (op->bytes) {
1644 *(u8 *)op->addr.reg = (u8)op->val;
1647 *(u16 *)op->addr.reg = (u16)op->val;
1650 *op->addr.reg = (u32)op->val;
1651 break; /* 64b: zero-extend */
1653 *op->addr.reg = op->val;
1658 static int writeback(struct x86_emulate_ctxt *ctxt)
1662 if (ctxt->d & NoWrite)
1663 return X86EMUL_CONTINUE;
1665 switch (ctxt->dst.type) {
1667 write_register_operand(&ctxt->dst);
1670 if (ctxt->lock_prefix)
1671 rc = segmented_cmpxchg(ctxt,
1673 &ctxt->dst.orig_val,
1677 rc = segmented_write(ctxt,
1681 if (rc != X86EMUL_CONTINUE)
1685 rc = segmented_write(ctxt,
1688 ctxt->dst.bytes * ctxt->dst.count);
1689 if (rc != X86EMUL_CONTINUE)
1693 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1696 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1704 return X86EMUL_CONTINUE;
1707 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1709 struct segmented_address addr;
1711 rsp_increment(ctxt, -bytes);
1712 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1713 addr.seg = VCPU_SREG_SS;
1715 return segmented_write(ctxt, addr, data, bytes);
1718 static int em_push(struct x86_emulate_ctxt *ctxt)
1720 /* Disable writeback. */
1721 ctxt->dst.type = OP_NONE;
1722 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1725 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1726 void *dest, int len)
1729 struct segmented_address addr;
1731 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1732 addr.seg = VCPU_SREG_SS;
1733 rc = segmented_read(ctxt, addr, dest, len);
1734 if (rc != X86EMUL_CONTINUE)
1737 rsp_increment(ctxt, len);
1741 static int em_pop(struct x86_emulate_ctxt *ctxt)
1743 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1746 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1747 void *dest, int len)
1750 unsigned long val, change_mask;
1751 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1752 int cpl = ctxt->ops->cpl(ctxt);
1754 rc = emulate_pop(ctxt, &val, len);
1755 if (rc != X86EMUL_CONTINUE)
1758 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1759 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1761 switch(ctxt->mode) {
1762 case X86EMUL_MODE_PROT64:
1763 case X86EMUL_MODE_PROT32:
1764 case X86EMUL_MODE_PROT16:
1766 change_mask |= EFLG_IOPL;
1768 change_mask |= EFLG_IF;
1770 case X86EMUL_MODE_VM86:
1772 return emulate_gp(ctxt, 0);
1773 change_mask |= EFLG_IF;
1775 default: /* real mode */
1776 change_mask |= (EFLG_IOPL | EFLG_IF);
1780 *(unsigned long *)dest =
1781 (ctxt->eflags & ~change_mask) | (val & change_mask);
1786 static int em_popf(struct x86_emulate_ctxt *ctxt)
1788 ctxt->dst.type = OP_REG;
1789 ctxt->dst.addr.reg = &ctxt->eflags;
1790 ctxt->dst.bytes = ctxt->op_bytes;
1791 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1794 static int em_enter(struct x86_emulate_ctxt *ctxt)
1797 unsigned frame_size = ctxt->src.val;
1798 unsigned nesting_level = ctxt->src2.val & 31;
1802 return X86EMUL_UNHANDLEABLE;
1804 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1805 rc = push(ctxt, &rbp, stack_size(ctxt));
1806 if (rc != X86EMUL_CONTINUE)
1808 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1810 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1811 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1813 return X86EMUL_CONTINUE;
1816 static int em_leave(struct x86_emulate_ctxt *ctxt)
1818 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1820 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1823 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1825 int seg = ctxt->src2.val;
1827 ctxt->src.val = get_segment_selector(ctxt, seg);
1829 return em_push(ctxt);
1832 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1834 int seg = ctxt->src2.val;
1835 unsigned long selector;
1838 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1839 if (rc != X86EMUL_CONTINUE)
1842 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1846 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1848 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1849 int rc = X86EMUL_CONTINUE;
1850 int reg = VCPU_REGS_RAX;
1852 while (reg <= VCPU_REGS_RDI) {
1853 (reg == VCPU_REGS_RSP) ?
1854 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1857 if (rc != X86EMUL_CONTINUE)
1866 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1868 ctxt->src.val = (unsigned long)ctxt->eflags;
1869 return em_push(ctxt);
1872 static int em_popa(struct x86_emulate_ctxt *ctxt)
1874 int rc = X86EMUL_CONTINUE;
1875 int reg = VCPU_REGS_RDI;
1877 while (reg >= VCPU_REGS_RAX) {
1878 if (reg == VCPU_REGS_RSP) {
1879 rsp_increment(ctxt, ctxt->op_bytes);
1883 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1884 if (rc != X86EMUL_CONTINUE)
1891 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1893 const struct x86_emulate_ops *ops = ctxt->ops;
1900 /* TODO: Add limit checks */
1901 ctxt->src.val = ctxt->eflags;
1903 if (rc != X86EMUL_CONTINUE)
1906 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1908 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1910 if (rc != X86EMUL_CONTINUE)
1913 ctxt->src.val = ctxt->_eip;
1915 if (rc != X86EMUL_CONTINUE)
1918 ops->get_idt(ctxt, &dt);
1920 eip_addr = dt.address + (irq << 2);
1921 cs_addr = dt.address + (irq << 2) + 2;
1923 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1924 if (rc != X86EMUL_CONTINUE)
1927 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1928 if (rc != X86EMUL_CONTINUE)
1931 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1932 if (rc != X86EMUL_CONTINUE)
1940 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1944 invalidate_registers(ctxt);
1945 rc = __emulate_int_real(ctxt, irq);
1946 if (rc == X86EMUL_CONTINUE)
1947 writeback_registers(ctxt);
1951 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1953 switch(ctxt->mode) {
1954 case X86EMUL_MODE_REAL:
1955 return __emulate_int_real(ctxt, irq);
1956 case X86EMUL_MODE_VM86:
1957 case X86EMUL_MODE_PROT16:
1958 case X86EMUL_MODE_PROT32:
1959 case X86EMUL_MODE_PROT64:
1961 /* Protected mode interrupts unimplemented yet */
1962 return X86EMUL_UNHANDLEABLE;
1966 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1968 int rc = X86EMUL_CONTINUE;
1969 unsigned long temp_eip = 0;
1970 unsigned long temp_eflags = 0;
1971 unsigned long cs = 0;
1972 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1973 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1974 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1975 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1977 /* TODO: Add stack limit check */
1979 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1981 if (rc != X86EMUL_CONTINUE)
1984 if (temp_eip & ~0xffff)
1985 return emulate_gp(ctxt, 0);
1987 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1989 if (rc != X86EMUL_CONTINUE)
1992 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1994 if (rc != X86EMUL_CONTINUE)
1997 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1999 if (rc != X86EMUL_CONTINUE)
2002 ctxt->_eip = temp_eip;
2005 if (ctxt->op_bytes == 4)
2006 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2007 else if (ctxt->op_bytes == 2) {
2008 ctxt->eflags &= ~0xffff;
2009 ctxt->eflags |= temp_eflags;
2012 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2013 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2018 static int em_iret(struct x86_emulate_ctxt *ctxt)
2020 switch(ctxt->mode) {
2021 case X86EMUL_MODE_REAL:
2022 return emulate_iret_real(ctxt);
2023 case X86EMUL_MODE_VM86:
2024 case X86EMUL_MODE_PROT16:
2025 case X86EMUL_MODE_PROT32:
2026 case X86EMUL_MODE_PROT64:
2028 /* iret from protected mode unimplemented yet */
2029 return X86EMUL_UNHANDLEABLE;
2033 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2038 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2040 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2041 if (rc != X86EMUL_CONTINUE)
2045 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2046 return X86EMUL_CONTINUE;
2049 static int em_grp2(struct x86_emulate_ctxt *ctxt)
2051 switch (ctxt->modrm_reg) {
2053 emulate_2op_SrcB(ctxt, "rol");
2056 emulate_2op_SrcB(ctxt, "ror");
2059 emulate_2op_SrcB(ctxt, "rcl");
2062 emulate_2op_SrcB(ctxt, "rcr");
2064 case 4: /* sal/shl */
2065 case 6: /* sal/shl */
2066 emulate_2op_SrcB(ctxt, "sal");
2069 emulate_2op_SrcB(ctxt, "shr");
2072 emulate_2op_SrcB(ctxt, "sar");
2075 return X86EMUL_CONTINUE;
2081 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2085 emulate_1op_rax_rdx(ctxt, "mul", ex);
2086 return X86EMUL_CONTINUE;
2089 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2093 emulate_1op_rax_rdx(ctxt, "imul", ex);
2094 return X86EMUL_CONTINUE;
2097 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2101 emulate_1op_rax_rdx(ctxt, "div", de);
2103 return emulate_de(ctxt);
2104 return X86EMUL_CONTINUE;
2107 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2111 emulate_1op_rax_rdx(ctxt, "idiv", de);
2113 return emulate_de(ctxt);
2114 return X86EMUL_CONTINUE;
2117 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2119 int rc = X86EMUL_CONTINUE;
2121 switch (ctxt->modrm_reg) {
2123 emulate_1op(ctxt, "inc");
2126 emulate_1op(ctxt, "dec");
2128 case 2: /* call near abs */ {
2130 old_eip = ctxt->_eip;
2131 ctxt->_eip = ctxt->src.val;
2132 ctxt->src.val = old_eip;
2136 case 4: /* jmp abs */
2137 ctxt->_eip = ctxt->src.val;
2139 case 5: /* jmp far */
2140 rc = em_jmp_far(ctxt);
2149 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2151 u64 old = ctxt->dst.orig_val64;
2153 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2154 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2155 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2156 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2157 ctxt->eflags &= ~EFLG_ZF;
2159 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2160 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2162 ctxt->eflags |= EFLG_ZF;
2164 return X86EMUL_CONTINUE;
2167 static int em_ret(struct x86_emulate_ctxt *ctxt)
2169 ctxt->dst.type = OP_REG;
2170 ctxt->dst.addr.reg = &ctxt->_eip;
2171 ctxt->dst.bytes = ctxt->op_bytes;
2172 return em_pop(ctxt);
2175 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2180 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2181 if (rc != X86EMUL_CONTINUE)
2183 if (ctxt->op_bytes == 4)
2184 ctxt->_eip = (u32)ctxt->_eip;
2185 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2186 if (rc != X86EMUL_CONTINUE)
2188 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2192 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2194 /* Save real source value, then compare EAX against destination. */
2195 ctxt->src.orig_val = ctxt->src.val;
2196 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2197 emulate_2op_SrcV(ctxt, "cmp");
2199 if (ctxt->eflags & EFLG_ZF) {
2200 /* Success: write back to memory. */
2201 ctxt->dst.val = ctxt->src.orig_val;
2203 /* Failure: write the value we saw to EAX. */
2204 ctxt->dst.type = OP_REG;
2205 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2207 return X86EMUL_CONTINUE;
2210 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2212 int seg = ctxt->src2.val;
2216 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2218 rc = load_segment_descriptor(ctxt, sel, seg);
2219 if (rc != X86EMUL_CONTINUE)
2222 ctxt->dst.val = ctxt->src.val;
2227 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2228 struct desc_struct *cs, struct desc_struct *ss)
2230 cs->l = 0; /* will be adjusted later */
2231 set_desc_base(cs, 0); /* flat segment */
2232 cs->g = 1; /* 4kb granularity */
2233 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2234 cs->type = 0x0b; /* Read, Execute, Accessed */
2236 cs->dpl = 0; /* will be adjusted later */
2241 set_desc_base(ss, 0); /* flat segment */
2242 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2243 ss->g = 1; /* 4kb granularity */
2245 ss->type = 0x03; /* Read/Write, Accessed */
2246 ss->d = 1; /* 32bit stack segment */
2253 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2255 u32 eax, ebx, ecx, edx;
2258 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2259 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2260 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2261 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2264 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2266 const struct x86_emulate_ops *ops = ctxt->ops;
2267 u32 eax, ebx, ecx, edx;
2270 * syscall should always be enabled in longmode - so only become
2271 * vendor specific (cpuid) if other modes are active...
2273 if (ctxt->mode == X86EMUL_MODE_PROT64)
2278 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2280 * Intel ("GenuineIntel")
2281 * remark: Intel CPUs only support "syscall" in 64bit
2282 * longmode. Also an 64bit guest with a
2283 * 32bit compat-app running will #UD !! While this
2284 * behaviour can be fixed (by emulating) into AMD
2285 * response - CPUs of AMD can't behave like Intel.
2287 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2288 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2289 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2292 /* AMD ("AuthenticAMD") */
2293 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2294 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2295 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2298 /* AMD ("AMDisbetter!") */
2299 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2300 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2301 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2304 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2308 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2310 const struct x86_emulate_ops *ops = ctxt->ops;
2311 struct desc_struct cs, ss;
2316 /* syscall is not available in real mode */
2317 if (ctxt->mode == X86EMUL_MODE_REAL ||
2318 ctxt->mode == X86EMUL_MODE_VM86)
2319 return emulate_ud(ctxt);
2321 if (!(em_syscall_is_enabled(ctxt)))
2322 return emulate_ud(ctxt);
2324 ops->get_msr(ctxt, MSR_EFER, &efer);
2325 setup_syscalls_segments(ctxt, &cs, &ss);
2327 if (!(efer & EFER_SCE))
2328 return emulate_ud(ctxt);
2330 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2332 cs_sel = (u16)(msr_data & 0xfffc);
2333 ss_sel = (u16)(msr_data + 8);
2335 if (efer & EFER_LMA) {
2339 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2340 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2342 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2343 if (efer & EFER_LMA) {
2344 #ifdef CONFIG_X86_64
2345 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2348 ctxt->mode == X86EMUL_MODE_PROT64 ?
2349 MSR_LSTAR : MSR_CSTAR, &msr_data);
2350 ctxt->_eip = msr_data;
2352 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2353 ctxt->eflags &= ~(msr_data | EFLG_RF);
2357 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2358 ctxt->_eip = (u32)msr_data;
2360 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2363 return X86EMUL_CONTINUE;
2366 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2368 const struct x86_emulate_ops *ops = ctxt->ops;
2369 struct desc_struct cs, ss;
2374 ops->get_msr(ctxt, MSR_EFER, &efer);
2375 /* inject #GP if in real mode */
2376 if (ctxt->mode == X86EMUL_MODE_REAL)
2377 return emulate_gp(ctxt, 0);
2380 * Not recognized on AMD in compat mode (but is recognized in legacy
2383 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2384 && !vendor_intel(ctxt))
2385 return emulate_ud(ctxt);
2387 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2388 * Therefore, we inject an #UD.
2390 if (ctxt->mode == X86EMUL_MODE_PROT64)
2391 return emulate_ud(ctxt);
2393 setup_syscalls_segments(ctxt, &cs, &ss);
2395 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2396 switch (ctxt->mode) {
2397 case X86EMUL_MODE_PROT32:
2398 if ((msr_data & 0xfffc) == 0x0)
2399 return emulate_gp(ctxt, 0);
2401 case X86EMUL_MODE_PROT64:
2402 if (msr_data == 0x0)
2403 return emulate_gp(ctxt, 0);
2409 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2410 cs_sel = (u16)msr_data;
2411 cs_sel &= ~SELECTOR_RPL_MASK;
2412 ss_sel = cs_sel + 8;
2413 ss_sel &= ~SELECTOR_RPL_MASK;
2414 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2419 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2420 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2422 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2423 ctxt->_eip = msr_data;
2425 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2426 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2428 return X86EMUL_CONTINUE;
2431 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2433 const struct x86_emulate_ops *ops = ctxt->ops;
2434 struct desc_struct cs, ss;
2437 u16 cs_sel = 0, ss_sel = 0;
2439 /* inject #GP if in real mode or Virtual 8086 mode */
2440 if (ctxt->mode == X86EMUL_MODE_REAL ||
2441 ctxt->mode == X86EMUL_MODE_VM86)
2442 return emulate_gp(ctxt, 0);
2444 setup_syscalls_segments(ctxt, &cs, &ss);
2446 if ((ctxt->rex_prefix & 0x8) != 0x0)
2447 usermode = X86EMUL_MODE_PROT64;
2449 usermode = X86EMUL_MODE_PROT32;
2453 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2455 case X86EMUL_MODE_PROT32:
2456 cs_sel = (u16)(msr_data + 16);
2457 if ((msr_data & 0xfffc) == 0x0)
2458 return emulate_gp(ctxt, 0);
2459 ss_sel = (u16)(msr_data + 24);
2461 case X86EMUL_MODE_PROT64:
2462 cs_sel = (u16)(msr_data + 32);
2463 if (msr_data == 0x0)
2464 return emulate_gp(ctxt, 0);
2465 ss_sel = cs_sel + 8;
2470 cs_sel |= SELECTOR_RPL_MASK;
2471 ss_sel |= SELECTOR_RPL_MASK;
2473 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2474 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2476 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2477 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2479 return X86EMUL_CONTINUE;
2482 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2485 if (ctxt->mode == X86EMUL_MODE_REAL)
2487 if (ctxt->mode == X86EMUL_MODE_VM86)
2489 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2490 return ctxt->ops->cpl(ctxt) > iopl;
2493 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2496 const struct x86_emulate_ops *ops = ctxt->ops;
2497 struct desc_struct tr_seg;
2500 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2501 unsigned mask = (1 << len) - 1;
2504 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2507 if (desc_limit_scaled(&tr_seg) < 103)
2509 base = get_desc_base(&tr_seg);
2510 #ifdef CONFIG_X86_64
2511 base |= ((u64)base3) << 32;
2513 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2514 if (r != X86EMUL_CONTINUE)
2516 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2518 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2519 if (r != X86EMUL_CONTINUE)
2521 if ((perm >> bit_idx) & mask)
2526 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2532 if (emulator_bad_iopl(ctxt))
2533 if (!emulator_io_port_access_allowed(ctxt, port, len))
2536 ctxt->perm_ok = true;
2541 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2542 struct tss_segment_16 *tss)
2544 tss->ip = ctxt->_eip;
2545 tss->flag = ctxt->eflags;
2546 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2547 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2548 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2549 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2550 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2551 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2552 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2553 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2555 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2556 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2557 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2558 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2559 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2562 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2563 struct tss_segment_16 *tss)
2567 ctxt->_eip = tss->ip;
2568 ctxt->eflags = tss->flag | 2;
2569 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2570 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2571 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2572 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2573 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2574 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2575 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2576 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2579 * SDM says that segment selectors are loaded before segment
2582 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2583 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2584 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2585 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2586 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2589 * Now load segment descriptors. If fault happens at this stage
2590 * it is handled in a context of new task
2592 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2593 if (ret != X86EMUL_CONTINUE)
2595 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2596 if (ret != X86EMUL_CONTINUE)
2598 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2599 if (ret != X86EMUL_CONTINUE)
2601 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2602 if (ret != X86EMUL_CONTINUE)
2604 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2605 if (ret != X86EMUL_CONTINUE)
2608 return X86EMUL_CONTINUE;
2611 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2612 u16 tss_selector, u16 old_tss_sel,
2613 ulong old_tss_base, struct desc_struct *new_desc)
2615 const struct x86_emulate_ops *ops = ctxt->ops;
2616 struct tss_segment_16 tss_seg;
2618 u32 new_tss_base = get_desc_base(new_desc);
2620 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2622 if (ret != X86EMUL_CONTINUE)
2623 /* FIXME: need to provide precise fault address */
2626 save_state_to_tss16(ctxt, &tss_seg);
2628 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2630 if (ret != X86EMUL_CONTINUE)
2631 /* FIXME: need to provide precise fault address */
2634 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2636 if (ret != X86EMUL_CONTINUE)
2637 /* FIXME: need to provide precise fault address */
2640 if (old_tss_sel != 0xffff) {
2641 tss_seg.prev_task_link = old_tss_sel;
2643 ret = ops->write_std(ctxt, new_tss_base,
2644 &tss_seg.prev_task_link,
2645 sizeof tss_seg.prev_task_link,
2647 if (ret != X86EMUL_CONTINUE)
2648 /* FIXME: need to provide precise fault address */
2652 return load_state_from_tss16(ctxt, &tss_seg);
2655 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2656 struct tss_segment_32 *tss)
2658 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2659 tss->eip = ctxt->_eip;
2660 tss->eflags = ctxt->eflags;
2661 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2662 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2663 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2664 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2665 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2666 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2667 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2668 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2670 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2671 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2672 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2673 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2674 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2675 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2676 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2679 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2680 struct tss_segment_32 *tss)
2684 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2685 return emulate_gp(ctxt, 0);
2686 ctxt->_eip = tss->eip;
2687 ctxt->eflags = tss->eflags | 2;
2689 /* General purpose registers */
2690 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2691 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2692 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2693 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2694 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2695 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2696 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2697 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2700 * SDM says that segment selectors are loaded before segment
2703 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2704 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2705 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2706 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2707 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2708 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2709 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2712 * If we're switching between Protected Mode and VM86, we need to make
2713 * sure to update the mode before loading the segment descriptors so
2714 * that the selectors are interpreted correctly.
2716 * Need to get rflags to the vcpu struct immediately because it
2717 * influences the CPL which is checked at least when loading the segment
2718 * descriptors and when pushing an error code to the new kernel stack.
2720 * TODO Introduce a separate ctxt->ops->set_cpl callback
2722 if (ctxt->eflags & X86_EFLAGS_VM)
2723 ctxt->mode = X86EMUL_MODE_VM86;
2725 ctxt->mode = X86EMUL_MODE_PROT32;
2727 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2730 * Now load segment descriptors. If fault happenes at this stage
2731 * it is handled in a context of new task
2733 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2734 if (ret != X86EMUL_CONTINUE)
2736 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2737 if (ret != X86EMUL_CONTINUE)
2739 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2740 if (ret != X86EMUL_CONTINUE)
2742 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2743 if (ret != X86EMUL_CONTINUE)
2745 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2746 if (ret != X86EMUL_CONTINUE)
2748 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2749 if (ret != X86EMUL_CONTINUE)
2751 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2752 if (ret != X86EMUL_CONTINUE)
2755 return X86EMUL_CONTINUE;
2758 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2759 u16 tss_selector, u16 old_tss_sel,
2760 ulong old_tss_base, struct desc_struct *new_desc)
2762 const struct x86_emulate_ops *ops = ctxt->ops;
2763 struct tss_segment_32 tss_seg;
2765 u32 new_tss_base = get_desc_base(new_desc);
2767 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2769 if (ret != X86EMUL_CONTINUE)
2770 /* FIXME: need to provide precise fault address */
2773 save_state_to_tss32(ctxt, &tss_seg);
2775 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2777 if (ret != X86EMUL_CONTINUE)
2778 /* FIXME: need to provide precise fault address */
2781 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2783 if (ret != X86EMUL_CONTINUE)
2784 /* FIXME: need to provide precise fault address */
2787 if (old_tss_sel != 0xffff) {
2788 tss_seg.prev_task_link = old_tss_sel;
2790 ret = ops->write_std(ctxt, new_tss_base,
2791 &tss_seg.prev_task_link,
2792 sizeof tss_seg.prev_task_link,
2794 if (ret != X86EMUL_CONTINUE)
2795 /* FIXME: need to provide precise fault address */
2799 return load_state_from_tss32(ctxt, &tss_seg);
2802 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2803 u16 tss_selector, int idt_index, int reason,
2804 bool has_error_code, u32 error_code)
2806 const struct x86_emulate_ops *ops = ctxt->ops;
2807 struct desc_struct curr_tss_desc, next_tss_desc;
2809 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2810 ulong old_tss_base =
2811 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2815 /* FIXME: old_tss_base == ~0 ? */
2817 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2818 if (ret != X86EMUL_CONTINUE)
2820 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2821 if (ret != X86EMUL_CONTINUE)
2824 /* FIXME: check that next_tss_desc is tss */
2827 * Check privileges. The three cases are task switch caused by...
2829 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2830 * 2. Exception/IRQ/iret: No check is performed
2831 * 3. jmp/call to TSS: Check against DPL of the TSS
2833 if (reason == TASK_SWITCH_GATE) {
2834 if (idt_index != -1) {
2835 /* Software interrupts */
2836 struct desc_struct task_gate_desc;
2839 ret = read_interrupt_descriptor(ctxt, idt_index,
2841 if (ret != X86EMUL_CONTINUE)
2844 dpl = task_gate_desc.dpl;
2845 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2846 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2848 } else if (reason != TASK_SWITCH_IRET) {
2849 int dpl = next_tss_desc.dpl;
2850 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2851 return emulate_gp(ctxt, tss_selector);
2855 desc_limit = desc_limit_scaled(&next_tss_desc);
2856 if (!next_tss_desc.p ||
2857 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2858 desc_limit < 0x2b)) {
2859 emulate_ts(ctxt, tss_selector & 0xfffc);
2860 return X86EMUL_PROPAGATE_FAULT;
2863 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2864 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2865 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2868 if (reason == TASK_SWITCH_IRET)
2869 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2871 /* set back link to prev task only if NT bit is set in eflags
2872 note that old_tss_sel is not used after this point */
2873 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2874 old_tss_sel = 0xffff;
2876 if (next_tss_desc.type & 8)
2877 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2878 old_tss_base, &next_tss_desc);
2880 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2881 old_tss_base, &next_tss_desc);
2882 if (ret != X86EMUL_CONTINUE)
2885 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2886 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2888 if (reason != TASK_SWITCH_IRET) {
2889 next_tss_desc.type |= (1 << 1); /* set busy flag */
2890 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2893 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2894 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2896 if (has_error_code) {
2897 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2898 ctxt->lock_prefix = 0;
2899 ctxt->src.val = (unsigned long) error_code;
2900 ret = em_push(ctxt);
2906 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2907 u16 tss_selector, int idt_index, int reason,
2908 bool has_error_code, u32 error_code)
2912 invalidate_registers(ctxt);
2913 ctxt->_eip = ctxt->eip;
2914 ctxt->dst.type = OP_NONE;
2916 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2917 has_error_code, error_code);
2919 if (rc == X86EMUL_CONTINUE) {
2920 ctxt->eip = ctxt->_eip;
2921 writeback_registers(ctxt);
2924 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2927 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2930 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2932 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2933 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2936 static int em_das(struct x86_emulate_ctxt *ctxt)
2939 bool af, cf, old_cf;
2941 cf = ctxt->eflags & X86_EFLAGS_CF;
2947 af = ctxt->eflags & X86_EFLAGS_AF;
2948 if ((al & 0x0f) > 9 || af) {
2950 cf = old_cf | (al >= 250);
2955 if (old_al > 0x99 || old_cf) {
2961 /* Set PF, ZF, SF */
2962 ctxt->src.type = OP_IMM;
2964 ctxt->src.bytes = 1;
2965 emulate_2op_SrcV(ctxt, "or");
2966 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2968 ctxt->eflags |= X86_EFLAGS_CF;
2970 ctxt->eflags |= X86_EFLAGS_AF;
2971 return X86EMUL_CONTINUE;
2974 static int em_aad(struct x86_emulate_ctxt *ctxt)
2976 u8 al = ctxt->dst.val & 0xff;
2977 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2979 al = (al + (ah * ctxt->src.val)) & 0xff;
2981 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2983 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
2986 ctxt->eflags |= X86_EFLAGS_ZF;
2988 ctxt->eflags |= X86_EFLAGS_PF;
2990 ctxt->eflags |= X86_EFLAGS_SF;
2992 return X86EMUL_CONTINUE;
2995 static int em_call(struct x86_emulate_ctxt *ctxt)
2997 long rel = ctxt->src.val;
2999 ctxt->src.val = (unsigned long)ctxt->_eip;
3001 return em_push(ctxt);
3004 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3010 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3011 old_eip = ctxt->_eip;
3013 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3014 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3015 return X86EMUL_CONTINUE;
3018 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3020 ctxt->src.val = old_cs;
3022 if (rc != X86EMUL_CONTINUE)
3025 ctxt->src.val = old_eip;
3026 return em_push(ctxt);
3029 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3033 ctxt->dst.type = OP_REG;
3034 ctxt->dst.addr.reg = &ctxt->_eip;
3035 ctxt->dst.bytes = ctxt->op_bytes;
3036 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3037 if (rc != X86EMUL_CONTINUE)
3039 rsp_increment(ctxt, ctxt->src.val);
3040 return X86EMUL_CONTINUE;
3056 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3058 /* Write back the register source. */
3059 ctxt->src.val = ctxt->dst.val;
3060 write_register_operand(&ctxt->src);
3062 /* Write back the memory destination with implicit LOCK prefix. */
3063 ctxt->dst.val = ctxt->src.orig_val;
3064 ctxt->lock_prefix = 1;
3065 return X86EMUL_CONTINUE;
3068 static int em_imul(struct x86_emulate_ctxt *ctxt)
3070 emulate_2op_SrcV_nobyte(ctxt, "imul");
3071 return X86EMUL_CONTINUE;
3074 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3076 ctxt->dst.val = ctxt->src2.val;
3077 return em_imul(ctxt);
3080 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3082 ctxt->dst.type = OP_REG;
3083 ctxt->dst.bytes = ctxt->src.bytes;
3084 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3085 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3087 return X86EMUL_CONTINUE;
3090 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3094 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3095 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3096 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3097 return X86EMUL_CONTINUE;
3100 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3104 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3105 return emulate_gp(ctxt, 0);
3106 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3107 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3108 return X86EMUL_CONTINUE;
3111 static int em_mov(struct x86_emulate_ctxt *ctxt)
3113 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3114 return X86EMUL_CONTINUE;
3117 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3119 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3120 return emulate_gp(ctxt, 0);
3122 /* Disable writeback. */
3123 ctxt->dst.type = OP_NONE;
3124 return X86EMUL_CONTINUE;
3127 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3131 if (ctxt->mode == X86EMUL_MODE_PROT64)
3132 val = ctxt->src.val & ~0ULL;
3134 val = ctxt->src.val & ~0U;
3136 /* #UD condition is already handled. */
3137 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3138 return emulate_gp(ctxt, 0);
3140 /* Disable writeback. */
3141 ctxt->dst.type = OP_NONE;
3142 return X86EMUL_CONTINUE;
3145 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3149 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3150 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3151 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3152 return emulate_gp(ctxt, 0);
3154 return X86EMUL_CONTINUE;
3157 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3161 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3162 return emulate_gp(ctxt, 0);
3164 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3165 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3166 return X86EMUL_CONTINUE;
3169 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3171 if (ctxt->modrm_reg > VCPU_SREG_GS)
3172 return emulate_ud(ctxt);
3174 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3175 return X86EMUL_CONTINUE;
3178 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3180 u16 sel = ctxt->src.val;
3182 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3183 return emulate_ud(ctxt);
3185 if (ctxt->modrm_reg == VCPU_SREG_SS)
3186 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3188 /* Disable writeback. */
3189 ctxt->dst.type = OP_NONE;
3190 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3193 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3195 u16 sel = ctxt->src.val;
3197 /* Disable writeback. */
3198 ctxt->dst.type = OP_NONE;
3199 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3202 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3204 u16 sel = ctxt->src.val;
3206 /* Disable writeback. */
3207 ctxt->dst.type = OP_NONE;
3208 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3211 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3216 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3217 if (rc == X86EMUL_CONTINUE)
3218 ctxt->ops->invlpg(ctxt, linear);
3219 /* Disable writeback. */
3220 ctxt->dst.type = OP_NONE;
3221 return X86EMUL_CONTINUE;
3224 static int em_clts(struct x86_emulate_ctxt *ctxt)
3228 cr0 = ctxt->ops->get_cr(ctxt, 0);
3230 ctxt->ops->set_cr(ctxt, 0, cr0);
3231 return X86EMUL_CONTINUE;
3234 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3238 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3239 return X86EMUL_UNHANDLEABLE;
3241 rc = ctxt->ops->fix_hypercall(ctxt);
3242 if (rc != X86EMUL_CONTINUE)
3245 /* Let the processor re-execute the fixed hypercall */
3246 ctxt->_eip = ctxt->eip;
3247 /* Disable writeback. */
3248 ctxt->dst.type = OP_NONE;
3249 return X86EMUL_CONTINUE;
3252 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3253 void (*get)(struct x86_emulate_ctxt *ctxt,
3254 struct desc_ptr *ptr))
3256 struct desc_ptr desc_ptr;
3258 if (ctxt->mode == X86EMUL_MODE_PROT64)
3260 get(ctxt, &desc_ptr);
3261 if (ctxt->op_bytes == 2) {
3263 desc_ptr.address &= 0x00ffffff;
3265 /* Disable writeback. */
3266 ctxt->dst.type = OP_NONE;
3267 return segmented_write(ctxt, ctxt->dst.addr.mem,
3268 &desc_ptr, 2 + ctxt->op_bytes);
3271 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3273 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3276 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3278 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3281 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3283 struct desc_ptr desc_ptr;
3286 if (ctxt->mode == X86EMUL_MODE_PROT64)
3288 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3289 &desc_ptr.size, &desc_ptr.address,
3291 if (rc != X86EMUL_CONTINUE)
3293 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3294 /* Disable writeback. */
3295 ctxt->dst.type = OP_NONE;
3296 return X86EMUL_CONTINUE;
3299 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3303 rc = ctxt->ops->fix_hypercall(ctxt);
3305 /* Disable writeback. */
3306 ctxt->dst.type = OP_NONE;
3310 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3312 struct desc_ptr desc_ptr;
3315 if (ctxt->mode == X86EMUL_MODE_PROT64)
3317 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3318 &desc_ptr.size, &desc_ptr.address,
3320 if (rc != X86EMUL_CONTINUE)
3322 ctxt->ops->set_idt(ctxt, &desc_ptr);
3323 /* Disable writeback. */
3324 ctxt->dst.type = OP_NONE;
3325 return X86EMUL_CONTINUE;
3328 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3330 ctxt->dst.bytes = 2;
3331 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3332 return X86EMUL_CONTINUE;
3335 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3337 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3338 | (ctxt->src.val & 0x0f));
3339 ctxt->dst.type = OP_NONE;
3340 return X86EMUL_CONTINUE;
3343 static int em_loop(struct x86_emulate_ctxt *ctxt)
3345 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3346 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3347 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3348 jmp_rel(ctxt, ctxt->src.val);
3350 return X86EMUL_CONTINUE;
3353 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3355 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3356 jmp_rel(ctxt, ctxt->src.val);
3358 return X86EMUL_CONTINUE;
3361 static int em_in(struct x86_emulate_ctxt *ctxt)
3363 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3365 return X86EMUL_IO_NEEDED;
3367 return X86EMUL_CONTINUE;
3370 static int em_out(struct x86_emulate_ctxt *ctxt)
3372 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3374 /* Disable writeback. */
3375 ctxt->dst.type = OP_NONE;
3376 return X86EMUL_CONTINUE;
3379 static int em_cli(struct x86_emulate_ctxt *ctxt)
3381 if (emulator_bad_iopl(ctxt))
3382 return emulate_gp(ctxt, 0);
3384 ctxt->eflags &= ~X86_EFLAGS_IF;
3385 return X86EMUL_CONTINUE;
3388 static int em_sti(struct x86_emulate_ctxt *ctxt)
3390 if (emulator_bad_iopl(ctxt))
3391 return emulate_gp(ctxt, 0);
3393 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3394 ctxt->eflags |= X86_EFLAGS_IF;
3395 return X86EMUL_CONTINUE;
3398 static int em_bt(struct x86_emulate_ctxt *ctxt)
3400 /* Disable writeback. */
3401 ctxt->dst.type = OP_NONE;
3402 /* only subword offset */
3403 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3405 emulate_2op_SrcV_nobyte(ctxt, "bt");
3406 return X86EMUL_CONTINUE;
3409 static int em_bts(struct x86_emulate_ctxt *ctxt)
3411 emulate_2op_SrcV_nobyte(ctxt, "bts");
3412 return X86EMUL_CONTINUE;
3415 static int em_btr(struct x86_emulate_ctxt *ctxt)
3417 emulate_2op_SrcV_nobyte(ctxt, "btr");
3418 return X86EMUL_CONTINUE;
3421 static int em_btc(struct x86_emulate_ctxt *ctxt)
3423 emulate_2op_SrcV_nobyte(ctxt, "btc");
3424 return X86EMUL_CONTINUE;
3427 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3429 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3430 return X86EMUL_CONTINUE;
3433 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3435 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3436 return X86EMUL_CONTINUE;
3439 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3441 u32 eax, ebx, ecx, edx;
3443 eax = reg_read(ctxt, VCPU_REGS_RAX);
3444 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3445 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3446 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3447 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3448 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3449 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3450 return X86EMUL_CONTINUE;
3453 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3455 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3456 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3457 return X86EMUL_CONTINUE;
3460 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3462 switch (ctxt->op_bytes) {
3463 #ifdef CONFIG_X86_64
3465 asm("bswap %0" : "+r"(ctxt->dst.val));
3469 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3472 return X86EMUL_CONTINUE;
3475 static bool valid_cr(int nr)
3487 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3489 if (!valid_cr(ctxt->modrm_reg))
3490 return emulate_ud(ctxt);
3492 return X86EMUL_CONTINUE;
3495 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3497 u64 new_val = ctxt->src.val64;
3498 int cr = ctxt->modrm_reg;
3501 static u64 cr_reserved_bits[] = {
3502 0xffffffff00000000ULL,
3503 0, 0, 0, /* CR3 checked later */
3510 return emulate_ud(ctxt);
3512 if (new_val & cr_reserved_bits[cr])
3513 return emulate_gp(ctxt, 0);
3518 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3519 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3520 return emulate_gp(ctxt, 0);
3522 cr4 = ctxt->ops->get_cr(ctxt, 4);
3523 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3525 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3526 !(cr4 & X86_CR4_PAE))
3527 return emulate_gp(ctxt, 0);
3534 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3535 if (efer & EFER_LMA)
3536 rsvd = CR3_L_MODE_RESERVED_BITS;
3537 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3538 rsvd = CR3_PAE_RESERVED_BITS;
3539 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3540 rsvd = CR3_NONPAE_RESERVED_BITS;
3543 return emulate_gp(ctxt, 0);
3548 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3550 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3551 return emulate_gp(ctxt, 0);
3557 return X86EMUL_CONTINUE;
3560 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3564 ctxt->ops->get_dr(ctxt, 7, &dr7);
3566 /* Check if DR7.Global_Enable is set */
3567 return dr7 & (1 << 13);
3570 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3572 int dr = ctxt->modrm_reg;
3576 return emulate_ud(ctxt);
3578 cr4 = ctxt->ops->get_cr(ctxt, 4);
3579 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3580 return emulate_ud(ctxt);
3582 if (check_dr7_gd(ctxt))
3583 return emulate_db(ctxt);
3585 return X86EMUL_CONTINUE;
3588 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3590 u64 new_val = ctxt->src.val64;
3591 int dr = ctxt->modrm_reg;
3593 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3594 return emulate_gp(ctxt, 0);
3596 return check_dr_read(ctxt);
3599 static int check_svme(struct x86_emulate_ctxt *ctxt)
3603 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3605 if (!(efer & EFER_SVME))
3606 return emulate_ud(ctxt);
3608 return X86EMUL_CONTINUE;
3611 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3613 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3615 /* Valid physical address? */
3616 if (rax & 0xffff000000000000ULL)
3617 return emulate_gp(ctxt, 0);
3619 return check_svme(ctxt);
3622 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3624 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3626 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3627 return emulate_ud(ctxt);
3629 return X86EMUL_CONTINUE;
3632 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3634 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3635 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3637 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3639 return emulate_gp(ctxt, 0);
3641 return X86EMUL_CONTINUE;
3644 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3646 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3647 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3648 return emulate_gp(ctxt, 0);
3650 return X86EMUL_CONTINUE;
3653 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3655 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3656 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3657 return emulate_gp(ctxt, 0);
3659 return X86EMUL_CONTINUE;
3662 #define D(_y) { .flags = (_y) }
3663 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3664 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3665 .check_perm = (_p) }
3667 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3668 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3669 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3670 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3671 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3672 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3673 #define II(_f, _e, _i) \
3674 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3675 #define IIP(_f, _e, _i, _p) \
3676 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3677 .check_perm = (_p) }
3678 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3680 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3681 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3682 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3683 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3684 #define I2bvIP(_f, _e, _i, _p) \
3685 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3687 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3688 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3689 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3691 static const struct opcode group7_rm1[] = {
3692 DI(SrcNone | Priv, monitor),
3693 DI(SrcNone | Priv, mwait),
3697 static const struct opcode group7_rm3[] = {
3698 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3699 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3700 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3701 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3702 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3703 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3704 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3705 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3708 static const struct opcode group7_rm7[] = {
3710 DIP(SrcNone, rdtscp, check_rdtsc),
3714 static const struct opcode group1[] = {
3716 F(Lock | PageTable, em_or),
3719 F(Lock | PageTable, em_and),
3725 static const struct opcode group1A[] = {
3726 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3729 static const struct opcode group3[] = {
3730 F(DstMem | SrcImm | NoWrite, em_test),
3731 F(DstMem | SrcImm | NoWrite, em_test),
3732 F(DstMem | SrcNone | Lock, em_not),
3733 F(DstMem | SrcNone | Lock, em_neg),
3734 I(SrcMem, em_mul_ex),
3735 I(SrcMem, em_imul_ex),
3736 I(SrcMem, em_div_ex),
3737 I(SrcMem, em_idiv_ex),
3740 static const struct opcode group4[] = {
3741 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3742 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3746 static const struct opcode group5[] = {
3747 I(DstMem | SrcNone | Lock, em_grp45),
3748 I(DstMem | SrcNone | Lock, em_grp45),
3749 I(SrcMem | Stack, em_grp45),
3750 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3751 I(SrcMem | Stack, em_grp45),
3752 I(SrcMemFAddr | ImplicitOps, em_grp45),
3753 I(SrcMem | Stack, em_grp45), N,
3756 static const struct opcode group6[] = {
3759 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3760 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3764 static const struct group_dual group7 = { {
3765 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3766 II(Mov | DstMem | Priv, em_sidt, sidt),
3767 II(SrcMem | Priv, em_lgdt, lgdt),
3768 II(SrcMem | Priv, em_lidt, lidt),
3769 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3770 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3771 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3773 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3775 N, EXT(0, group7_rm3),
3776 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3777 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3781 static const struct opcode group8[] = {
3783 I(DstMem | SrcImmByte, em_bt),
3784 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3785 I(DstMem | SrcImmByte | Lock, em_btr),
3786 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3789 static const struct group_dual group9 = { {
3790 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3792 N, N, N, N, N, N, N, N,
3795 static const struct opcode group11[] = {
3796 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3800 static const struct gprefix pfx_0f_6f_0f_7f = {
3801 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3804 static const struct gprefix pfx_vmovntpx = {
3805 I(0, em_mov), N, N, N,
3808 static const struct escape escape_d9 = { {
3809 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3812 N, N, N, N, N, N, N, N,
3814 N, N, N, N, N, N, N, N,
3816 N, N, N, N, N, N, N, N,
3818 N, N, N, N, N, N, N, N,
3820 N, N, N, N, N, N, N, N,
3822 N, N, N, N, N, N, N, N,
3824 N, N, N, N, N, N, N, N,
3826 N, N, N, N, N, N, N, N,
3829 static const struct escape escape_db = { {
3830 N, N, N, N, N, N, N, N,
3833 N, N, N, N, N, N, N, N,
3835 N, N, N, N, N, N, N, N,
3837 N, N, N, N, N, N, N, N,
3839 N, N, N, N, N, N, N, N,
3841 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3843 N, N, N, N, N, N, N, N,
3845 N, N, N, N, N, N, N, N,
3847 N, N, N, N, N, N, N, N,
3850 static const struct escape escape_dd = { {
3851 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3854 N, N, N, N, N, N, N, N,
3856 N, N, N, N, N, N, N, N,
3858 N, N, N, N, N, N, N, N,
3860 N, N, N, N, N, N, N, N,
3862 N, N, N, N, N, N, N, N,
3864 N, N, N, N, N, N, N, N,
3866 N, N, N, N, N, N, N, N,
3868 N, N, N, N, N, N, N, N,
3871 static const struct opcode opcode_table[256] = {
3873 F6ALU(Lock, em_add),
3874 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3875 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3877 F6ALU(Lock | PageTable, em_or),
3878 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3881 F6ALU(Lock, em_adc),
3882 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3883 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3885 F6ALU(Lock, em_sbb),
3886 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3887 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3889 F6ALU(Lock | PageTable, em_and), N, N,
3891 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3893 F6ALU(Lock, em_xor), N, N,
3895 F6ALU(NoWrite, em_cmp), N, N,
3899 X8(I(SrcReg | Stack, em_push)),
3901 X8(I(DstReg | Stack, em_pop)),
3903 I(ImplicitOps | Stack | No64, em_pusha),
3904 I(ImplicitOps | Stack | No64, em_popa),
3905 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3908 I(SrcImm | Mov | Stack, em_push),
3909 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3910 I(SrcImmByte | Mov | Stack, em_push),
3911 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3912 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3913 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3917 G(ByteOp | DstMem | SrcImm, group1),
3918 G(DstMem | SrcImm, group1),
3919 G(ByteOp | DstMem | SrcImm | No64, group1),
3920 G(DstMem | SrcImmByte, group1),
3921 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3922 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3924 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3925 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3926 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3927 D(ModRM | SrcMem | NoAccess | DstReg),
3928 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3931 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3933 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3934 I(SrcImmFAddr | No64, em_call_far), N,
3935 II(ImplicitOps | Stack, em_pushf, pushf),
3936 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3938 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3939 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3940 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3941 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3943 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3944 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3945 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3946 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3948 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3950 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3952 D2bv(DstMem | SrcImmByte | ModRM),
3953 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3954 I(ImplicitOps | Stack, em_ret),
3955 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3956 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3957 G(ByteOp, group11), G(0, group11),
3959 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3960 N, I(ImplicitOps | Stack, em_ret_far),
3961 D(ImplicitOps), DI(SrcImmByte, intn),
3962 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3964 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3965 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3967 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3969 X3(I(SrcImmByte, em_loop)),
3970 I(SrcImmByte, em_jcxz),
3971 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3972 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3974 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3975 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3976 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3977 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3979 N, DI(ImplicitOps, icebp), N, N,
3980 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3981 G(ByteOp, group3), G(0, group3),
3983 D(ImplicitOps), D(ImplicitOps),
3984 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3985 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3988 static const struct opcode twobyte_table[256] = {
3990 G(0, group6), GD(0, &group7), N, N,
3991 N, I(ImplicitOps | VendorSpecific, em_syscall),
3992 II(ImplicitOps | Priv, em_clts, clts), N,
3993 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3994 N, D(ImplicitOps | ModRM), N, N,
3996 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3998 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3999 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4000 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4001 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4003 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4006 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4007 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4008 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4009 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4010 I(ImplicitOps | VendorSpecific, em_sysenter),
4011 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4013 N, N, N, N, N, N, N, N,
4015 X16(D(DstReg | SrcMem | ModRM | Mov)),
4017 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4022 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4027 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4031 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4033 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4034 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
4035 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4036 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4038 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4039 DI(ImplicitOps, rsm),
4040 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4041 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4042 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4043 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
4045 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4046 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4047 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4048 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4049 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4050 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4054 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4055 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
4056 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4058 D2bv(DstMem | SrcReg | ModRM | Lock),
4059 N, D(DstMem | SrcReg | ModRM | Mov),
4060 N, N, N, GD(0, &group9),
4062 X8(I(DstReg, em_bswap)),
4064 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4066 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4068 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4085 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4089 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4095 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4096 unsigned size, bool sign_extension)
4098 int rc = X86EMUL_CONTINUE;
4102 op->addr.mem.ea = ctxt->_eip;
4103 /* NB. Immediates are sign-extended as necessary. */
4104 switch (op->bytes) {
4106 op->val = insn_fetch(s8, ctxt);
4109 op->val = insn_fetch(s16, ctxt);
4112 op->val = insn_fetch(s32, ctxt);
4115 op->val = insn_fetch(s64, ctxt);
4118 if (!sign_extension) {
4119 switch (op->bytes) {
4127 op->val &= 0xffffffff;
4135 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4138 int rc = X86EMUL_CONTINUE;
4142 decode_register_operand(ctxt, op);
4145 rc = decode_imm(ctxt, op, 1, false);
4148 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4152 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4153 fetch_bit_operand(ctxt);
4154 op->orig_val = op->val;
4157 ctxt->memop.bytes = 8;
4161 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4162 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4163 fetch_register_operand(op);
4164 op->orig_val = op->val;
4168 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4170 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4171 op->addr.mem.seg = VCPU_SREG_ES;
4178 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4179 fetch_register_operand(op);
4183 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4186 rc = decode_imm(ctxt, op, 1, true);
4193 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4196 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4199 ctxt->memop.bytes = 1;
4202 ctxt->memop.bytes = 2;
4205 ctxt->memop.bytes = 4;
4208 rc = decode_imm(ctxt, op, 2, false);
4211 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4215 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4217 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4218 op->addr.mem.seg = seg_override(ctxt);
4224 op->addr.mem.ea = ctxt->_eip;
4225 op->bytes = ctxt->op_bytes + 2;
4226 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4229 ctxt->memop.bytes = ctxt->op_bytes + 2;
4232 op->val = VCPU_SREG_ES;
4235 op->val = VCPU_SREG_CS;
4238 op->val = VCPU_SREG_SS;
4241 op->val = VCPU_SREG_DS;
4244 op->val = VCPU_SREG_FS;
4247 op->val = VCPU_SREG_GS;
4250 /* Special instructions do their own operand decoding. */
4252 op->type = OP_NONE; /* Disable writeback. */
4260 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4262 int rc = X86EMUL_CONTINUE;
4263 int mode = ctxt->mode;
4264 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4265 bool op_prefix = false;
4266 struct opcode opcode;
4268 ctxt->memop.type = OP_NONE;
4269 ctxt->memopp = NULL;
4270 ctxt->_eip = ctxt->eip;
4271 ctxt->fetch.start = ctxt->_eip;
4272 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4274 memcpy(ctxt->fetch.data, insn, insn_len);
4277 case X86EMUL_MODE_REAL:
4278 case X86EMUL_MODE_VM86:
4279 case X86EMUL_MODE_PROT16:
4280 def_op_bytes = def_ad_bytes = 2;
4282 case X86EMUL_MODE_PROT32:
4283 def_op_bytes = def_ad_bytes = 4;
4285 #ifdef CONFIG_X86_64
4286 case X86EMUL_MODE_PROT64:
4292 return EMULATION_FAILED;
4295 ctxt->op_bytes = def_op_bytes;
4296 ctxt->ad_bytes = def_ad_bytes;
4298 /* Legacy prefixes. */
4300 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4301 case 0x66: /* operand-size override */
4303 /* switch between 2/4 bytes */
4304 ctxt->op_bytes = def_op_bytes ^ 6;
4306 case 0x67: /* address-size override */
4307 if (mode == X86EMUL_MODE_PROT64)
4308 /* switch between 4/8 bytes */
4309 ctxt->ad_bytes = def_ad_bytes ^ 12;
4311 /* switch between 2/4 bytes */
4312 ctxt->ad_bytes = def_ad_bytes ^ 6;
4314 case 0x26: /* ES override */
4315 case 0x2e: /* CS override */
4316 case 0x36: /* SS override */
4317 case 0x3e: /* DS override */
4318 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4320 case 0x64: /* FS override */
4321 case 0x65: /* GS override */
4322 set_seg_override(ctxt, ctxt->b & 7);
4324 case 0x40 ... 0x4f: /* REX */
4325 if (mode != X86EMUL_MODE_PROT64)
4327 ctxt->rex_prefix = ctxt->b;
4329 case 0xf0: /* LOCK */
4330 ctxt->lock_prefix = 1;
4332 case 0xf2: /* REPNE/REPNZ */
4333 case 0xf3: /* REP/REPE/REPZ */
4334 ctxt->rep_prefix = ctxt->b;
4340 /* Any legacy prefix after a REX prefix nullifies its effect. */
4342 ctxt->rex_prefix = 0;
4348 if (ctxt->rex_prefix & 8)
4349 ctxt->op_bytes = 8; /* REX.W */
4351 /* Opcode byte(s). */
4352 opcode = opcode_table[ctxt->b];
4353 /* Two-byte opcode? */
4354 if (ctxt->b == 0x0f) {
4356 ctxt->b = insn_fetch(u8, ctxt);
4357 opcode = twobyte_table[ctxt->b];
4359 ctxt->d = opcode.flags;
4361 if (ctxt->d & ModRM)
4362 ctxt->modrm = insn_fetch(u8, ctxt);
4364 while (ctxt->d & GroupMask) {
4365 switch (ctxt->d & GroupMask) {
4367 goffset = (ctxt->modrm >> 3) & 7;
4368 opcode = opcode.u.group[goffset];
4371 goffset = (ctxt->modrm >> 3) & 7;
4372 if ((ctxt->modrm >> 6) == 3)
4373 opcode = opcode.u.gdual->mod3[goffset];
4375 opcode = opcode.u.gdual->mod012[goffset];
4378 goffset = ctxt->modrm & 7;
4379 opcode = opcode.u.group[goffset];
4382 if (ctxt->rep_prefix && op_prefix)
4383 return EMULATION_FAILED;
4384 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4385 switch (simd_prefix) {
4386 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4387 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4388 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4389 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4393 if (ctxt->modrm > 0xbf)
4394 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4396 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4399 return EMULATION_FAILED;
4402 ctxt->d &= ~(u64)GroupMask;
4403 ctxt->d |= opcode.flags;
4406 ctxt->execute = opcode.u.execute;
4407 ctxt->check_perm = opcode.check_perm;
4408 ctxt->intercept = opcode.intercept;
4411 if (ctxt->d == 0 || (ctxt->d & Undefined))
4412 return EMULATION_FAILED;
4414 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4415 return EMULATION_FAILED;
4417 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4420 if (ctxt->d & Op3264) {
4421 if (mode == X86EMUL_MODE_PROT64)
4428 ctxt->op_bytes = 16;
4429 else if (ctxt->d & Mmx)
4432 /* ModRM and SIB bytes. */
4433 if (ctxt->d & ModRM) {
4434 rc = decode_modrm(ctxt, &ctxt->memop);
4435 if (!ctxt->has_seg_override)
4436 set_seg_override(ctxt, ctxt->modrm_seg);
4437 } else if (ctxt->d & MemAbs)
4438 rc = decode_abs(ctxt, &ctxt->memop);
4439 if (rc != X86EMUL_CONTINUE)
4442 if (!ctxt->has_seg_override)
4443 set_seg_override(ctxt, VCPU_SREG_DS);
4445 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4447 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4448 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4451 * Decode and fetch the source operand: register, memory
4454 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4455 if (rc != X86EMUL_CONTINUE)
4459 * Decode and fetch the second source operand: register, memory
4462 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4463 if (rc != X86EMUL_CONTINUE)
4466 /* Decode and fetch the destination operand: register or memory. */
4467 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4470 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4471 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4473 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4476 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4478 return ctxt->d & PageTable;
4481 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4483 /* The second termination condition only applies for REPE
4484 * and REPNE. Test if the repeat string operation prefix is
4485 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4486 * corresponding termination condition according to:
4487 * - if REPE/REPZ and ZF = 0 then done
4488 * - if REPNE/REPNZ and ZF = 1 then done
4490 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4491 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4492 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4493 ((ctxt->eflags & EFLG_ZF) == 0))
4494 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4495 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4501 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4505 ctxt->ops->get_fpu(ctxt);
4506 asm volatile("1: fwait \n\t"
4508 ".pushsection .fixup,\"ax\" \n\t"
4510 "movb $1, %[fault] \n\t"
4513 _ASM_EXTABLE(1b, 3b)
4514 : [fault]"+qm"(fault));
4515 ctxt->ops->put_fpu(ctxt);
4517 if (unlikely(fault))
4518 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4520 return X86EMUL_CONTINUE;
4523 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4526 if (op->type == OP_MM)
4527 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4530 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4532 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4533 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4534 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4535 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4536 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4537 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4538 return X86EMUL_CONTINUE;
4541 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4543 const struct x86_emulate_ops *ops = ctxt->ops;
4544 int rc = X86EMUL_CONTINUE;
4545 int saved_dst_type = ctxt->dst.type;
4547 ctxt->mem_read.pos = 0;
4549 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4550 rc = emulate_ud(ctxt);
4554 /* LOCK prefix is allowed only with some instructions */
4555 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4556 rc = emulate_ud(ctxt);
4560 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4561 rc = emulate_ud(ctxt);
4565 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4566 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4567 rc = emulate_ud(ctxt);
4571 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4572 rc = emulate_nm(ctxt);
4576 if (ctxt->d & Mmx) {
4577 rc = flush_pending_x87_faults(ctxt);
4578 if (rc != X86EMUL_CONTINUE)
4581 * Now that we know the fpu is exception safe, we can fetch
4584 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4585 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4586 if (!(ctxt->d & Mov))
4587 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4590 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4591 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4592 X86_ICPT_PRE_EXCEPT);
4593 if (rc != X86EMUL_CONTINUE)
4597 /* Privileged instruction can be executed only in CPL=0 */
4598 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4599 rc = emulate_gp(ctxt, 0);
4603 /* Instruction can only be executed in protected mode */
4604 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4605 rc = emulate_ud(ctxt);
4609 /* Do instruction specific permission checks */
4610 if (ctxt->check_perm) {
4611 rc = ctxt->check_perm(ctxt);
4612 if (rc != X86EMUL_CONTINUE)
4616 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4617 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4618 X86_ICPT_POST_EXCEPT);
4619 if (rc != X86EMUL_CONTINUE)
4623 if (ctxt->rep_prefix && (ctxt->d & String)) {
4624 /* All REP prefixes have the same first termination condition */
4625 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4626 ctxt->eip = ctxt->_eip;
4631 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4632 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4633 ctxt->src.valptr, ctxt->src.bytes);
4634 if (rc != X86EMUL_CONTINUE)
4636 ctxt->src.orig_val64 = ctxt->src.val64;
4639 if (ctxt->src2.type == OP_MEM) {
4640 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4641 &ctxt->src2.val, ctxt->src2.bytes);
4642 if (rc != X86EMUL_CONTINUE)
4646 if ((ctxt->d & DstMask) == ImplicitOps)
4650 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4651 /* optimisation - avoid slow emulated read if Mov */
4652 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4653 &ctxt->dst.val, ctxt->dst.bytes);
4654 if (rc != X86EMUL_CONTINUE)
4657 ctxt->dst.orig_val = ctxt->dst.val;
4661 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4662 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4663 X86_ICPT_POST_MEMACCESS);
4664 if (rc != X86EMUL_CONTINUE)
4668 if (ctxt->execute) {
4669 if (ctxt->d & Fastop) {
4670 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4671 rc = fastop(ctxt, fop);
4672 if (rc != X86EMUL_CONTINUE)
4676 rc = ctxt->execute(ctxt);
4677 if (rc != X86EMUL_CONTINUE)
4686 case 0x40 ... 0x47: /* inc r16/r32 */
4687 emulate_1op(ctxt, "inc");
4689 case 0x48 ... 0x4f: /* dec r16/r32 */
4690 emulate_1op(ctxt, "dec");
4692 case 0x63: /* movsxd */
4693 if (ctxt->mode != X86EMUL_MODE_PROT64)
4694 goto cannot_emulate;
4695 ctxt->dst.val = (s32) ctxt->src.val;
4697 case 0x70 ... 0x7f: /* jcc (short) */
4698 if (test_cc(ctxt->b, ctxt->eflags))
4699 jmp_rel(ctxt, ctxt->src.val);
4701 case 0x8d: /* lea r16/r32, m */
4702 ctxt->dst.val = ctxt->src.addr.mem.ea;
4704 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4705 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4709 case 0x98: /* cbw/cwde/cdqe */
4710 switch (ctxt->op_bytes) {
4711 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4712 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4713 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4719 case 0xcc: /* int3 */
4720 rc = emulate_int(ctxt, 3);
4722 case 0xcd: /* int n */
4723 rc = emulate_int(ctxt, ctxt->src.val);
4725 case 0xce: /* into */
4726 if (ctxt->eflags & EFLG_OF)
4727 rc = emulate_int(ctxt, 4);
4729 case 0xd0 ... 0xd1: /* Grp2 */
4732 case 0xd2 ... 0xd3: /* Grp2 */
4733 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4736 case 0xe9: /* jmp rel */
4737 case 0xeb: /* jmp rel short */
4738 jmp_rel(ctxt, ctxt->src.val);
4739 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4741 case 0xf4: /* hlt */
4742 ctxt->ops->halt(ctxt);
4744 case 0xf5: /* cmc */
4745 /* complement carry flag from eflags reg */
4746 ctxt->eflags ^= EFLG_CF;
4748 case 0xf8: /* clc */
4749 ctxt->eflags &= ~EFLG_CF;
4751 case 0xf9: /* stc */
4752 ctxt->eflags |= EFLG_CF;
4754 case 0xfc: /* cld */
4755 ctxt->eflags &= ~EFLG_DF;
4757 case 0xfd: /* std */
4758 ctxt->eflags |= EFLG_DF;
4761 goto cannot_emulate;
4764 if (rc != X86EMUL_CONTINUE)
4768 rc = writeback(ctxt);
4769 if (rc != X86EMUL_CONTINUE)
4773 * restore dst type in case the decoding will be reused
4774 * (happens for string instruction )
4776 ctxt->dst.type = saved_dst_type;
4778 if ((ctxt->d & SrcMask) == SrcSI)
4779 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4781 if ((ctxt->d & DstMask) == DstDI)
4782 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4784 if (ctxt->rep_prefix && (ctxt->d & String)) {
4786 struct read_cache *r = &ctxt->io_read;
4787 if ((ctxt->d & SrcMask) == SrcSI)
4788 count = ctxt->src.count;
4790 count = ctxt->dst.count;
4791 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4794 if (!string_insn_completed(ctxt)) {
4796 * Re-enter guest when pio read ahead buffer is empty
4797 * or, if it is not used, after each 1024 iteration.
4799 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4800 (r->end == 0 || r->end != r->pos)) {
4802 * Reset read cache. Usually happens before
4803 * decode, but since instruction is restarted
4804 * we have to do it here.
4806 ctxt->mem_read.end = 0;
4807 writeback_registers(ctxt);
4808 return EMULATION_RESTART;
4810 goto done; /* skip rip writeback */
4814 ctxt->eip = ctxt->_eip;
4817 if (rc == X86EMUL_PROPAGATE_FAULT)
4818 ctxt->have_exception = true;
4819 if (rc == X86EMUL_INTERCEPTED)
4820 return EMULATION_INTERCEPTED;
4822 if (rc == X86EMUL_CONTINUE)
4823 writeback_registers(ctxt);
4825 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4829 case 0x09: /* wbinvd */
4830 (ctxt->ops->wbinvd)(ctxt);
4832 case 0x08: /* invd */
4833 case 0x0d: /* GrpP (prefetch) */
4834 case 0x18: /* Grp16 (prefetch/nop) */
4836 case 0x20: /* mov cr, reg */
4837 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4839 case 0x21: /* mov from dr to reg */
4840 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4842 case 0x40 ... 0x4f: /* cmov */
4843 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4844 if (!test_cc(ctxt->b, ctxt->eflags))
4845 ctxt->dst.type = OP_NONE; /* no writeback */
4847 case 0x80 ... 0x8f: /* jnz rel, etc*/
4848 if (test_cc(ctxt->b, ctxt->eflags))
4849 jmp_rel(ctxt, ctxt->src.val);
4851 case 0x90 ... 0x9f: /* setcc r/m8 */
4852 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4854 case 0xae: /* clflush */
4856 case 0xb6 ... 0xb7: /* movzx */
4857 ctxt->dst.bytes = ctxt->op_bytes;
4858 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4859 : (u16) ctxt->src.val;
4861 case 0xbe ... 0xbf: /* movsx */
4862 ctxt->dst.bytes = ctxt->op_bytes;
4863 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4864 (s16) ctxt->src.val;
4866 case 0xc0 ... 0xc1: /* xadd */
4867 emulate_2op_SrcV(ctxt, "add");
4868 /* Write back the register source. */
4869 ctxt->src.val = ctxt->dst.orig_val;
4870 write_register_operand(&ctxt->src);
4872 case 0xc3: /* movnti */
4873 ctxt->dst.bytes = ctxt->op_bytes;
4874 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4875 (u64) ctxt->src.val;
4878 goto cannot_emulate;
4881 if (rc != X86EMUL_CONTINUE)
4887 return EMULATION_FAILED;
4890 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4892 invalidate_registers(ctxt);
4895 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4897 writeback_registers(ctxt);