1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpBits 5 /* Width of operand field */
63 #define OpMask ((1ull << OpBits) - 1)
66 * Opcode effective-address decode tables.
67 * Note that we only emulate instructions that have at least one memory
68 * operand (excluding implicit stack references). We assume that stack
69 * references and instruction fetches will never occur in special memory
70 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
74 /* Operand sizes: 8-bit operands or specified/overridden size. */
75 #define ByteOp (1<<0) /* 8-bit operands. */
76 /* Destination operand type. */
78 #define ImplicitOps (OpImplicit << DstShift)
79 #define DstReg (OpReg << DstShift)
80 #define DstMem (OpMem << DstShift)
81 #define DstAcc (OpAcc << DstShift)
82 #define DstDI (OpDI << DstShift)
83 #define DstMem64 (OpMem64 << DstShift)
84 #define DstImmUByte (OpImmUByte << DstShift)
85 #define DstDX (OpDX << DstShift)
86 #define DstMask (OpMask << DstShift)
87 /* Source operand type. */
89 #define SrcNone (OpNone << SrcShift)
90 #define SrcReg (OpReg << SrcShift)
91 #define SrcMem (OpMem << SrcShift)
92 #define SrcMem16 (OpMem16 << SrcShift)
93 #define SrcMem32 (OpMem32 << SrcShift)
94 #define SrcImm (OpImm << SrcShift)
95 #define SrcImmByte (OpImmByte << SrcShift)
96 #define SrcOne (OpOne << SrcShift)
97 #define SrcImmUByte (OpImmUByte << SrcShift)
98 #define SrcImmU (OpImmU << SrcShift)
99 #define SrcSI (OpSI << SrcShift)
100 #define SrcImmFAddr (OpImmFAddr << SrcShift)
101 #define SrcMemFAddr (OpMemFAddr << SrcShift)
102 #define SrcAcc (OpAcc << SrcShift)
103 #define SrcImmU16 (OpImmU16 << SrcShift)
104 #define SrcDX (OpDX << SrcShift)
105 #define SrcMem8 (OpMem8 << SrcShift)
106 #define SrcMask (OpMask << SrcShift)
107 #define BitOp (1<<11)
108 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
109 #define String (1<<13) /* String instruction (rep capable) */
110 #define Stack (1<<14) /* Stack instruction (push/pop) */
111 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
112 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
113 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
114 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
115 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
116 #define Sse (1<<18) /* SSE Vector instruction */
117 /* Generic ModRM decode. */
118 #define ModRM (1<<19)
119 /* Destination is only written; never read. */
122 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
123 #define VendorSpecific (1<<22) /* Vendor specific instruction */
124 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
125 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
126 #define Undefined (1<<25) /* No Such Instruction */
127 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
128 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
130 #define PageTable (1 << 29) /* instruction used to write page table */
131 /* Source 2 operand type */
132 #define Src2Shift (30)
133 #define Src2None (OpNone << Src2Shift)
134 #define Src2CL (OpCL << Src2Shift)
135 #define Src2ImmByte (OpImmByte << Src2Shift)
136 #define Src2One (OpOne << Src2Shift)
137 #define Src2Imm (OpImm << Src2Shift)
138 #define Src2ES (OpES << Src2Shift)
139 #define Src2CS (OpCS << Src2Shift)
140 #define Src2SS (OpSS << Src2Shift)
141 #define Src2DS (OpDS << Src2Shift)
142 #define Src2FS (OpFS << Src2Shift)
143 #define Src2GS (OpGS << Src2Shift)
144 #define Src2Mask (OpMask << Src2Shift)
145 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
146 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
147 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
148 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
150 #define X2(x...) x, x
151 #define X3(x...) X2(x), x
152 #define X4(x...) X2(x), X2(x)
153 #define X5(x...) X4(x), x
154 #define X6(x...) X4(x), X2(x)
155 #define X7(x...) X4(x), X3(x)
156 #define X8(x...) X4(x), X4(x)
157 #define X16(x...) X8(x), X8(x)
163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 const struct opcode *group;
165 const struct group_dual *gdual;
166 const struct gprefix *gprefix;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
172 struct opcode mod012[8];
173 struct opcode mod3[8];
177 struct opcode pfx_no;
178 struct opcode pfx_66;
179 struct opcode pfx_f2;
180 struct opcode pfx_f3;
183 /* EFLAGS bit definitions. */
184 #define EFLG_ID (1<<21)
185 #define EFLG_VIP (1<<20)
186 #define EFLG_VIF (1<<19)
187 #define EFLG_AC (1<<18)
188 #define EFLG_VM (1<<17)
189 #define EFLG_RF (1<<16)
190 #define EFLG_IOPL (3<<12)
191 #define EFLG_NT (1<<14)
192 #define EFLG_OF (1<<11)
193 #define EFLG_DF (1<<10)
194 #define EFLG_IF (1<<9)
195 #define EFLG_TF (1<<8)
196 #define EFLG_SF (1<<7)
197 #define EFLG_ZF (1<<6)
198 #define EFLG_AF (1<<4)
199 #define EFLG_PF (1<<2)
200 #define EFLG_CF (1<<0)
202 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203 #define EFLG_RESERVED_ONE_MASK 2
205 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
207 if (!(ctxt->regs_valid & (1 << nr))) {
208 ctxt->regs_valid |= 1 << nr;
209 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
211 return ctxt->_regs[nr];
214 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
216 ctxt->regs_valid |= 1 << nr;
217 ctxt->regs_dirty |= 1 << nr;
218 return &ctxt->_regs[nr];
221 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
224 return reg_write(ctxt, nr);
227 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
231 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
232 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
235 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
237 ctxt->regs_dirty = 0;
238 ctxt->regs_valid = 0;
242 * Instruction emulation:
243 * Most instructions are emulated directly via a fragment of inline assembly
244 * code. This allows us to save/restore EFLAGS and thus very easily pick up
245 * any modified flags.
248 #if defined(CONFIG_X86_64)
249 #define _LO32 "k" /* force 32-bit operand */
250 #define _STK "%%rsp" /* stack pointer */
251 #elif defined(__i386__)
252 #define _LO32 "" /* force 32-bit operand */
253 #define _STK "%%esp" /* stack pointer */
257 * These EFLAGS bits are restored from saved value during emulation, and
258 * any changes are written back to the saved value after emulation.
260 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
262 /* Before executing instruction: restore necessary bits in EFLAGS. */
263 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
264 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
265 "movl %"_sav",%"_LO32 _tmp"; " \
268 "movl %"_msk",%"_LO32 _tmp"; " \
269 "andl %"_LO32 _tmp",("_STK"); " \
271 "notl %"_LO32 _tmp"; " \
272 "andl %"_LO32 _tmp",("_STK"); " \
273 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
275 "orl %"_LO32 _tmp",("_STK"); " \
279 /* After executing instruction: write-back necessary bits in EFLAGS. */
280 #define _POST_EFLAGS(_sav, _msk, _tmp) \
281 /* _sav |= EFLAGS & _msk; */ \
284 "andl %"_msk",%"_LO32 _tmp"; " \
285 "orl %"_LO32 _tmp",%"_sav"; "
293 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
295 __asm__ __volatile__ ( \
296 _PRE_EFLAGS("0", "4", "2") \
297 _op _suffix " %"_x"3,%1; " \
298 _POST_EFLAGS("0", "4", "2") \
299 : "=m" ((ctxt)->eflags), \
300 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
302 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
306 /* Raw emulation: instruction has two explicit operands. */
307 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
309 unsigned long _tmp; \
311 switch ((ctxt)->dst.bytes) { \
313 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
316 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
319 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
324 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
326 unsigned long _tmp; \
327 switch ((ctxt)->dst.bytes) { \
329 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
332 __emulate_2op_nobyte(ctxt, _op, \
333 _wx, _wy, _lx, _ly, _qx, _qy); \
338 /* Source operand is byte-sized and may be restricted to just %cl. */
339 #define emulate_2op_SrcB(ctxt, _op) \
340 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
342 /* Source operand is byte, word, long or quad sized. */
343 #define emulate_2op_SrcV(ctxt, _op) \
344 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
346 /* Source operand is word, long or quad sized. */
347 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
348 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
350 /* Instruction has three operands and one operand is stored in ECX register */
351 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
353 unsigned long _tmp; \
354 _type _clv = (ctxt)->src2.val; \
355 _type _srcv = (ctxt)->src.val; \
356 _type _dstv = (ctxt)->dst.val; \
358 __asm__ __volatile__ ( \
359 _PRE_EFLAGS("0", "5", "2") \
360 _op _suffix " %4,%1 \n" \
361 _POST_EFLAGS("0", "5", "2") \
362 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
363 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
366 (ctxt)->src2.val = (unsigned long) _clv; \
367 (ctxt)->src2.val = (unsigned long) _srcv; \
368 (ctxt)->dst.val = (unsigned long) _dstv; \
371 #define emulate_2op_cl(ctxt, _op) \
373 switch ((ctxt)->dst.bytes) { \
375 __emulate_2op_cl(ctxt, _op, "w", u16); \
378 __emulate_2op_cl(ctxt, _op, "l", u32); \
381 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
386 #define __emulate_1op(ctxt, _op, _suffix) \
388 unsigned long _tmp; \
390 __asm__ __volatile__ ( \
391 _PRE_EFLAGS("0", "3", "2") \
392 _op _suffix " %1; " \
393 _POST_EFLAGS("0", "3", "2") \
394 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
396 : "i" (EFLAGS_MASK)); \
399 /* Instruction has only one explicit operand (no source operand). */
400 #define emulate_1op(ctxt, _op) \
402 switch ((ctxt)->dst.bytes) { \
403 case 1: __emulate_1op(ctxt, _op, "b"); break; \
404 case 2: __emulate_1op(ctxt, _op, "w"); break; \
405 case 4: __emulate_1op(ctxt, _op, "l"); break; \
406 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
410 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
412 unsigned long _tmp; \
413 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
414 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
416 __asm__ __volatile__ ( \
417 _PRE_EFLAGS("0", "5", "1") \
419 _op _suffix " %6; " \
421 _POST_EFLAGS("0", "5", "1") \
422 ".pushsection .fixup,\"ax\" \n\t" \
423 "3: movb $1, %4 \n\t" \
426 _ASM_EXTABLE(1b, 3b) \
427 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
428 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
429 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
432 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
433 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
435 switch((ctxt)->src.bytes) { \
437 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
440 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
443 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
446 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
451 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
452 enum x86_intercept intercept,
453 enum x86_intercept_stage stage)
455 struct x86_instruction_info info = {
456 .intercept = intercept,
457 .rep_prefix = ctxt->rep_prefix,
458 .modrm_mod = ctxt->modrm_mod,
459 .modrm_reg = ctxt->modrm_reg,
460 .modrm_rm = ctxt->modrm_rm,
461 .src_val = ctxt->src.val64,
462 .src_bytes = ctxt->src.bytes,
463 .dst_bytes = ctxt->dst.bytes,
464 .ad_bytes = ctxt->ad_bytes,
465 .next_rip = ctxt->eip,
468 return ctxt->ops->intercept(ctxt, &info, stage);
471 static void assign_masked(ulong *dest, ulong src, ulong mask)
473 *dest = (*dest & ~mask) | (src & mask);
476 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
478 return (1UL << (ctxt->ad_bytes << 3)) - 1;
481 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
484 struct desc_struct ss;
486 if (ctxt->mode == X86EMUL_MODE_PROT64)
488 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
489 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
492 static int stack_size(struct x86_emulate_ctxt *ctxt)
494 return (__fls(stack_mask(ctxt)) + 1) >> 3;
497 /* Access/update address held in a register, based on addressing mode. */
498 static inline unsigned long
499 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
501 if (ctxt->ad_bytes == sizeof(unsigned long))
504 return reg & ad_mask(ctxt);
507 static inline unsigned long
508 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
510 return address_mask(ctxt, reg);
513 static void masked_increment(ulong *reg, ulong mask, int inc)
515 assign_masked(reg, *reg + inc, mask);
519 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
523 if (ctxt->ad_bytes == sizeof(unsigned long))
526 mask = ad_mask(ctxt);
527 masked_increment(reg, mask, inc);
530 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
532 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
535 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
537 register_address_increment(ctxt, &ctxt->_eip, rel);
540 static u32 desc_limit_scaled(struct desc_struct *desc)
542 u32 limit = get_desc_limit(desc);
544 return desc->g ? (limit << 12) | 0xfff : limit;
547 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
549 ctxt->has_seg_override = true;
550 ctxt->seg_override = seg;
553 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
555 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
558 return ctxt->ops->get_cached_segment_base(ctxt, seg);
561 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
563 if (!ctxt->has_seg_override)
566 return ctxt->seg_override;
569 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
570 u32 error, bool valid)
572 ctxt->exception.vector = vec;
573 ctxt->exception.error_code = error;
574 ctxt->exception.error_code_valid = valid;
575 return X86EMUL_PROPAGATE_FAULT;
578 static int emulate_db(struct x86_emulate_ctxt *ctxt)
580 return emulate_exception(ctxt, DB_VECTOR, 0, false);
583 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
585 return emulate_exception(ctxt, GP_VECTOR, err, true);
588 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
590 return emulate_exception(ctxt, SS_VECTOR, err, true);
593 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
595 return emulate_exception(ctxt, UD_VECTOR, 0, false);
598 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
600 return emulate_exception(ctxt, TS_VECTOR, err, true);
603 static int emulate_de(struct x86_emulate_ctxt *ctxt)
605 return emulate_exception(ctxt, DE_VECTOR, 0, false);
608 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
610 return emulate_exception(ctxt, NM_VECTOR, 0, false);
613 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
616 struct desc_struct desc;
618 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
622 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
627 struct desc_struct desc;
629 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
630 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
634 * x86 defines three classes of vector instructions: explicitly
635 * aligned, explicitly unaligned, and the rest, which change behaviour
636 * depending on whether they're AVX encoded or not.
638 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
639 * subject to the same check.
641 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
643 if (likely(size < 16))
646 if (ctxt->d & Aligned)
648 else if (ctxt->d & Unaligned)
650 else if (ctxt->d & Avx)
656 static int __linearize(struct x86_emulate_ctxt *ctxt,
657 struct segmented_address addr,
658 unsigned size, bool write, bool fetch,
661 struct desc_struct desc;
668 la = seg_base(ctxt, addr.seg) + addr.ea;
669 switch (ctxt->mode) {
670 case X86EMUL_MODE_PROT64:
671 if (((signed long)la << 16) >> 16 != la)
672 return emulate_gp(ctxt, 0);
675 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
679 /* code segment or read-only data segment */
680 if (((desc.type & 8) || !(desc.type & 2)) && write)
682 /* unreadable code segment */
683 if (!fetch && (desc.type & 8) && !(desc.type & 2))
685 lim = desc_limit_scaled(&desc);
686 if ((desc.type & 8) || !(desc.type & 4)) {
687 /* expand-up segment */
688 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
691 /* expand-down segment */
692 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
694 lim = desc.d ? 0xffffffff : 0xffff;
695 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
698 cpl = ctxt->ops->cpl(ctxt);
699 if (ctxt->mode == X86EMUL_MODE_REAL)
704 if (!(desc.type & 8)) {
708 } else if ((desc.type & 8) && !(desc.type & 4)) {
709 /* nonconforming code segment */
712 } else if ((desc.type & 8) && (desc.type & 4)) {
713 /* conforming code segment */
719 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
721 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
722 return emulate_gp(ctxt, 0);
724 return X86EMUL_CONTINUE;
726 if (addr.seg == VCPU_SREG_SS)
727 return emulate_ss(ctxt, sel);
729 return emulate_gp(ctxt, sel);
732 static int linearize(struct x86_emulate_ctxt *ctxt,
733 struct segmented_address addr,
734 unsigned size, bool write,
737 return __linearize(ctxt, addr, size, write, false, linear);
741 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
742 struct segmented_address addr,
749 rc = linearize(ctxt, addr, size, false, &linear);
750 if (rc != X86EMUL_CONTINUE)
752 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
756 * Fetch the next byte of the instruction being emulated which is pointed to
757 * by ctxt->_eip, then increment ctxt->_eip.
759 * Also prefetch the remaining bytes of the instruction without crossing page
760 * boundary if they are not in fetch_cache yet.
762 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
764 struct fetch_cache *fc = &ctxt->fetch;
768 if (ctxt->_eip == fc->end) {
769 unsigned long linear;
770 struct segmented_address addr = { .seg = VCPU_SREG_CS,
772 cur_size = fc->end - fc->start;
773 size = min(15UL - cur_size,
774 PAGE_SIZE - offset_in_page(ctxt->_eip));
775 rc = __linearize(ctxt, addr, size, false, true, &linear);
776 if (unlikely(rc != X86EMUL_CONTINUE))
778 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
779 size, &ctxt->exception);
780 if (unlikely(rc != X86EMUL_CONTINUE))
784 *dest = fc->data[ctxt->_eip - fc->start];
786 return X86EMUL_CONTINUE;
789 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
790 void *dest, unsigned size)
794 /* x86 instructions are limited to 15 bytes. */
795 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
796 return X86EMUL_UNHANDLEABLE;
798 rc = do_insn_fetch_byte(ctxt, dest++);
799 if (rc != X86EMUL_CONTINUE)
802 return X86EMUL_CONTINUE;
805 /* Fetch next part of the instruction being emulated. */
806 #define insn_fetch(_type, _ctxt) \
807 ({ unsigned long _x; \
808 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
809 if (rc != X86EMUL_CONTINUE) \
814 #define insn_fetch_arr(_arr, _size, _ctxt) \
815 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
816 if (rc != X86EMUL_CONTINUE) \
821 * Given the 'reg' portion of a ModRM byte, and a register block, return a
822 * pointer into the block that addresses the relevant register.
823 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
825 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
830 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
831 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
833 p = reg_rmw(ctxt, modrm_reg);
837 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
838 struct segmented_address addr,
839 u16 *size, unsigned long *address, int op_bytes)
846 rc = segmented_read_std(ctxt, addr, size, 2);
847 if (rc != X86EMUL_CONTINUE)
850 rc = segmented_read_std(ctxt, addr, address, op_bytes);
854 static int test_cc(unsigned int condition, unsigned int flags)
858 switch ((condition & 15) >> 1) {
860 rc |= (flags & EFLG_OF);
862 case 1: /* b/c/nae */
863 rc |= (flags & EFLG_CF);
866 rc |= (flags & EFLG_ZF);
869 rc |= (flags & (EFLG_CF|EFLG_ZF));
872 rc |= (flags & EFLG_SF);
875 rc |= (flags & EFLG_PF);
878 rc |= (flags & EFLG_ZF);
881 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
885 /* Odd condition identifiers (lsb == 1) have inverted sense. */
886 return (!!rc ^ (condition & 1));
889 static void fetch_register_operand(struct operand *op)
893 op->val = *(u8 *)op->addr.reg;
896 op->val = *(u16 *)op->addr.reg;
899 op->val = *(u32 *)op->addr.reg;
902 op->val = *(u64 *)op->addr.reg;
907 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
909 ctxt->ops->get_fpu(ctxt);
911 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
912 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
913 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
914 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
915 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
916 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
917 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
918 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
920 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
921 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
922 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
923 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
924 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
925 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
926 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
927 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
931 ctxt->ops->put_fpu(ctxt);
934 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
937 ctxt->ops->get_fpu(ctxt);
939 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
940 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
941 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
942 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
943 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
944 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
945 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
946 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
948 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
949 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
950 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
951 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
952 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
953 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
954 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
955 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
959 ctxt->ops->put_fpu(ctxt);
962 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
964 ctxt->ops->get_fpu(ctxt);
966 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
967 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
968 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
969 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
970 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
971 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
972 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
973 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
976 ctxt->ops->put_fpu(ctxt);
979 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
981 ctxt->ops->get_fpu(ctxt);
983 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
984 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
985 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
986 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
987 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
988 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
989 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
990 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
993 ctxt->ops->put_fpu(ctxt);
996 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
999 unsigned reg = ctxt->modrm_reg;
1000 int highbyte_regs = ctxt->rex_prefix == 0;
1002 if (!(ctxt->d & ModRM))
1003 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1005 if (ctxt->d & Sse) {
1009 read_sse_reg(ctxt, &op->vec_val, reg);
1012 if (ctxt->d & Mmx) {
1021 if (ctxt->d & ByteOp) {
1022 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1025 op->addr.reg = decode_register(ctxt, reg, 0);
1026 op->bytes = ctxt->op_bytes;
1028 fetch_register_operand(op);
1029 op->orig_val = op->val;
1032 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1034 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1035 ctxt->modrm_seg = VCPU_SREG_SS;
1038 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1042 int index_reg = 0, base_reg = 0, scale;
1043 int rc = X86EMUL_CONTINUE;
1046 if (ctxt->rex_prefix) {
1047 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1048 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1049 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1052 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1053 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1054 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1055 ctxt->modrm_seg = VCPU_SREG_DS;
1057 if (ctxt->modrm_mod == 3) {
1059 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1060 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1061 if (ctxt->d & Sse) {
1064 op->addr.xmm = ctxt->modrm_rm;
1065 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1068 if (ctxt->d & Mmx) {
1071 op->addr.xmm = ctxt->modrm_rm & 7;
1074 fetch_register_operand(op);
1080 if (ctxt->ad_bytes == 2) {
1081 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1082 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1083 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1084 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1086 /* 16-bit ModR/M decode. */
1087 switch (ctxt->modrm_mod) {
1089 if (ctxt->modrm_rm == 6)
1090 modrm_ea += insn_fetch(u16, ctxt);
1093 modrm_ea += insn_fetch(s8, ctxt);
1096 modrm_ea += insn_fetch(u16, ctxt);
1099 switch (ctxt->modrm_rm) {
1101 modrm_ea += bx + si;
1104 modrm_ea += bx + di;
1107 modrm_ea += bp + si;
1110 modrm_ea += bp + di;
1119 if (ctxt->modrm_mod != 0)
1126 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1127 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1128 ctxt->modrm_seg = VCPU_SREG_SS;
1129 modrm_ea = (u16)modrm_ea;
1131 /* 32/64-bit ModR/M decode. */
1132 if ((ctxt->modrm_rm & 7) == 4) {
1133 sib = insn_fetch(u8, ctxt);
1134 index_reg |= (sib >> 3) & 7;
1135 base_reg |= sib & 7;
1138 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1139 modrm_ea += insn_fetch(s32, ctxt);
1141 modrm_ea += reg_read(ctxt, base_reg);
1142 adjust_modrm_seg(ctxt, base_reg);
1145 modrm_ea += reg_read(ctxt, index_reg) << scale;
1146 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1147 if (ctxt->mode == X86EMUL_MODE_PROT64)
1148 ctxt->rip_relative = 1;
1150 base_reg = ctxt->modrm_rm;
1151 modrm_ea += reg_read(ctxt, base_reg);
1152 adjust_modrm_seg(ctxt, base_reg);
1154 switch (ctxt->modrm_mod) {
1156 if (ctxt->modrm_rm == 5)
1157 modrm_ea += insn_fetch(s32, ctxt);
1160 modrm_ea += insn_fetch(s8, ctxt);
1163 modrm_ea += insn_fetch(s32, ctxt);
1167 op->addr.mem.ea = modrm_ea;
1172 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1175 int rc = X86EMUL_CONTINUE;
1178 switch (ctxt->ad_bytes) {
1180 op->addr.mem.ea = insn_fetch(u16, ctxt);
1183 op->addr.mem.ea = insn_fetch(u32, ctxt);
1186 op->addr.mem.ea = insn_fetch(u64, ctxt);
1193 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1197 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1198 mask = ~(ctxt->dst.bytes * 8 - 1);
1200 if (ctxt->src.bytes == 2)
1201 sv = (s16)ctxt->src.val & (s16)mask;
1202 else if (ctxt->src.bytes == 4)
1203 sv = (s32)ctxt->src.val & (s32)mask;
1205 ctxt->dst.addr.mem.ea += (sv >> 3);
1208 /* only subword offset */
1209 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1212 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1213 unsigned long addr, void *dest, unsigned size)
1216 struct read_cache *mc = &ctxt->mem_read;
1218 if (mc->pos < mc->end)
1221 WARN_ON((mc->end + size) >= sizeof(mc->data));
1223 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1225 if (rc != X86EMUL_CONTINUE)
1231 memcpy(dest, mc->data + mc->pos, size);
1233 return X86EMUL_CONTINUE;
1236 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1237 struct segmented_address addr,
1244 rc = linearize(ctxt, addr, size, false, &linear);
1245 if (rc != X86EMUL_CONTINUE)
1247 return read_emulated(ctxt, linear, data, size);
1250 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1251 struct segmented_address addr,
1258 rc = linearize(ctxt, addr, size, true, &linear);
1259 if (rc != X86EMUL_CONTINUE)
1261 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1265 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1266 struct segmented_address addr,
1267 const void *orig_data, const void *data,
1273 rc = linearize(ctxt, addr, size, true, &linear);
1274 if (rc != X86EMUL_CONTINUE)
1276 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1277 size, &ctxt->exception);
1280 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1281 unsigned int size, unsigned short port,
1284 struct read_cache *rc = &ctxt->io_read;
1286 if (rc->pos == rc->end) { /* refill pio read ahead */
1287 unsigned int in_page, n;
1288 unsigned int count = ctxt->rep_prefix ?
1289 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1290 in_page = (ctxt->eflags & EFLG_DF) ?
1291 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1292 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1293 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1297 rc->pos = rc->end = 0;
1298 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1303 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1304 ctxt->dst.data = rc->data + rc->pos;
1305 ctxt->dst.type = OP_MEM_STR;
1306 ctxt->dst.count = (rc->end - rc->pos) / size;
1309 memcpy(dest, rc->data + rc->pos, size);
1315 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1316 u16 index, struct desc_struct *desc)
1321 ctxt->ops->get_idt(ctxt, &dt);
1323 if (dt.size < index * 8 + 7)
1324 return emulate_gp(ctxt, index << 3 | 0x2);
1326 addr = dt.address + index * 8;
1327 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1331 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1332 u16 selector, struct desc_ptr *dt)
1334 const struct x86_emulate_ops *ops = ctxt->ops;
1336 if (selector & 1 << 2) {
1337 struct desc_struct desc;
1340 memset (dt, 0, sizeof *dt);
1341 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1344 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1345 dt->address = get_desc_base(&desc);
1347 ops->get_gdt(ctxt, dt);
1350 /* allowed just for 8 bytes segments */
1351 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1352 u16 selector, struct desc_struct *desc,
1356 u16 index = selector >> 3;
1359 get_descriptor_table_ptr(ctxt, selector, &dt);
1361 if (dt.size < index * 8 + 7)
1362 return emulate_gp(ctxt, selector & 0xfffc);
1364 *desc_addr_p = addr = dt.address + index * 8;
1365 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1369 /* allowed just for 8 bytes segments */
1370 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1371 u16 selector, struct desc_struct *desc)
1374 u16 index = selector >> 3;
1377 get_descriptor_table_ptr(ctxt, selector, &dt);
1379 if (dt.size < index * 8 + 7)
1380 return emulate_gp(ctxt, selector & 0xfffc);
1382 addr = dt.address + index * 8;
1383 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1387 /* Does not support long mode */
1388 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1389 u16 selector, int seg)
1391 struct desc_struct seg_desc, old_desc;
1393 unsigned err_vec = GP_VECTOR;
1395 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1400 memset(&seg_desc, 0, sizeof seg_desc);
1402 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1403 || ctxt->mode == X86EMUL_MODE_REAL) {
1404 /* set real mode segment descriptor */
1405 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1406 set_desc_base(&seg_desc, selector << 4);
1411 cpl = ctxt->ops->cpl(ctxt);
1413 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1414 if ((seg == VCPU_SREG_CS
1415 || (seg == VCPU_SREG_SS
1416 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1417 || seg == VCPU_SREG_TR)
1421 /* TR should be in GDT only */
1422 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1425 if (null_selector) /* for NULL selector skip all following checks */
1428 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1429 if (ret != X86EMUL_CONTINUE)
1432 err_code = selector & 0xfffc;
1433 err_vec = GP_VECTOR;
1435 /* can't load system descriptor into segment selector */
1436 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1440 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1449 * segment is not a writable data segment or segment
1450 * selector's RPL != CPL or segment selector's RPL != CPL
1452 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1456 if (!(seg_desc.type & 8))
1459 if (seg_desc.type & 4) {
1465 if (rpl > cpl || dpl != cpl)
1468 /* CS(RPL) <- CPL */
1469 selector = (selector & 0xfffc) | cpl;
1472 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1474 old_desc = seg_desc;
1475 seg_desc.type |= 2; /* busy */
1476 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1477 sizeof(seg_desc), &ctxt->exception);
1478 if (ret != X86EMUL_CONTINUE)
1481 case VCPU_SREG_LDTR:
1482 if (seg_desc.s || seg_desc.type != 2)
1485 default: /* DS, ES, FS, or GS */
1487 * segment is not a data or readable code segment or
1488 * ((segment is a data or nonconforming code segment)
1489 * and (both RPL and CPL > DPL))
1491 if ((seg_desc.type & 0xa) == 0x8 ||
1492 (((seg_desc.type & 0xc) != 0xc) &&
1493 (rpl > dpl && cpl > dpl)))
1499 /* mark segment as accessed */
1501 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1502 if (ret != X86EMUL_CONTINUE)
1506 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1507 return X86EMUL_CONTINUE;
1509 emulate_exception(ctxt, err_vec, err_code, true);
1510 return X86EMUL_PROPAGATE_FAULT;
1513 static void write_register_operand(struct operand *op)
1515 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1516 switch (op->bytes) {
1518 *(u8 *)op->addr.reg = (u8)op->val;
1521 *(u16 *)op->addr.reg = (u16)op->val;
1524 *op->addr.reg = (u32)op->val;
1525 break; /* 64b: zero-extend */
1527 *op->addr.reg = op->val;
1532 static int writeback(struct x86_emulate_ctxt *ctxt)
1536 switch (ctxt->dst.type) {
1538 write_register_operand(&ctxt->dst);
1541 if (ctxt->lock_prefix)
1542 rc = segmented_cmpxchg(ctxt,
1544 &ctxt->dst.orig_val,
1548 rc = segmented_write(ctxt,
1552 if (rc != X86EMUL_CONTINUE)
1556 rc = segmented_write(ctxt,
1559 ctxt->dst.bytes * ctxt->dst.count);
1560 if (rc != X86EMUL_CONTINUE)
1564 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1567 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1575 return X86EMUL_CONTINUE;
1578 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1580 struct segmented_address addr;
1582 rsp_increment(ctxt, -bytes);
1583 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1584 addr.seg = VCPU_SREG_SS;
1586 return segmented_write(ctxt, addr, data, bytes);
1589 static int em_push(struct x86_emulate_ctxt *ctxt)
1591 /* Disable writeback. */
1592 ctxt->dst.type = OP_NONE;
1593 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1596 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1597 void *dest, int len)
1600 struct segmented_address addr;
1602 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1603 addr.seg = VCPU_SREG_SS;
1604 rc = segmented_read(ctxt, addr, dest, len);
1605 if (rc != X86EMUL_CONTINUE)
1608 rsp_increment(ctxt, len);
1612 static int em_pop(struct x86_emulate_ctxt *ctxt)
1614 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1617 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1618 void *dest, int len)
1621 unsigned long val, change_mask;
1622 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1623 int cpl = ctxt->ops->cpl(ctxt);
1625 rc = emulate_pop(ctxt, &val, len);
1626 if (rc != X86EMUL_CONTINUE)
1629 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1630 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1632 switch(ctxt->mode) {
1633 case X86EMUL_MODE_PROT64:
1634 case X86EMUL_MODE_PROT32:
1635 case X86EMUL_MODE_PROT16:
1637 change_mask |= EFLG_IOPL;
1639 change_mask |= EFLG_IF;
1641 case X86EMUL_MODE_VM86:
1643 return emulate_gp(ctxt, 0);
1644 change_mask |= EFLG_IF;
1646 default: /* real mode */
1647 change_mask |= (EFLG_IOPL | EFLG_IF);
1651 *(unsigned long *)dest =
1652 (ctxt->eflags & ~change_mask) | (val & change_mask);
1657 static int em_popf(struct x86_emulate_ctxt *ctxt)
1659 ctxt->dst.type = OP_REG;
1660 ctxt->dst.addr.reg = &ctxt->eflags;
1661 ctxt->dst.bytes = ctxt->op_bytes;
1662 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1665 static int em_enter(struct x86_emulate_ctxt *ctxt)
1668 unsigned frame_size = ctxt->src.val;
1669 unsigned nesting_level = ctxt->src2.val & 31;
1673 return X86EMUL_UNHANDLEABLE;
1675 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1676 rc = push(ctxt, &rbp, stack_size(ctxt));
1677 if (rc != X86EMUL_CONTINUE)
1679 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1681 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1682 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1684 return X86EMUL_CONTINUE;
1687 static int em_leave(struct x86_emulate_ctxt *ctxt)
1689 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1691 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1694 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1696 int seg = ctxt->src2.val;
1698 ctxt->src.val = get_segment_selector(ctxt, seg);
1700 return em_push(ctxt);
1703 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1705 int seg = ctxt->src2.val;
1706 unsigned long selector;
1709 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1710 if (rc != X86EMUL_CONTINUE)
1713 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1717 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1719 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1720 int rc = X86EMUL_CONTINUE;
1721 int reg = VCPU_REGS_RAX;
1723 while (reg <= VCPU_REGS_RDI) {
1724 (reg == VCPU_REGS_RSP) ?
1725 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1728 if (rc != X86EMUL_CONTINUE)
1737 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1739 ctxt->src.val = (unsigned long)ctxt->eflags;
1740 return em_push(ctxt);
1743 static int em_popa(struct x86_emulate_ctxt *ctxt)
1745 int rc = X86EMUL_CONTINUE;
1746 int reg = VCPU_REGS_RDI;
1748 while (reg >= VCPU_REGS_RAX) {
1749 if (reg == VCPU_REGS_RSP) {
1750 rsp_increment(ctxt, ctxt->op_bytes);
1754 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1755 if (rc != X86EMUL_CONTINUE)
1762 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1764 const struct x86_emulate_ops *ops = ctxt->ops;
1771 /* TODO: Add limit checks */
1772 ctxt->src.val = ctxt->eflags;
1774 if (rc != X86EMUL_CONTINUE)
1777 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1779 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1781 if (rc != X86EMUL_CONTINUE)
1784 ctxt->src.val = ctxt->_eip;
1786 if (rc != X86EMUL_CONTINUE)
1789 ops->get_idt(ctxt, &dt);
1791 eip_addr = dt.address + (irq << 2);
1792 cs_addr = dt.address + (irq << 2) + 2;
1794 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1795 if (rc != X86EMUL_CONTINUE)
1798 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1799 if (rc != X86EMUL_CONTINUE)
1802 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1803 if (rc != X86EMUL_CONTINUE)
1811 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1815 invalidate_registers(ctxt);
1816 rc = __emulate_int_real(ctxt, irq);
1817 if (rc == X86EMUL_CONTINUE)
1818 writeback_registers(ctxt);
1822 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1824 switch(ctxt->mode) {
1825 case X86EMUL_MODE_REAL:
1826 return __emulate_int_real(ctxt, irq);
1827 case X86EMUL_MODE_VM86:
1828 case X86EMUL_MODE_PROT16:
1829 case X86EMUL_MODE_PROT32:
1830 case X86EMUL_MODE_PROT64:
1832 /* Protected mode interrupts unimplemented yet */
1833 return X86EMUL_UNHANDLEABLE;
1837 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1839 int rc = X86EMUL_CONTINUE;
1840 unsigned long temp_eip = 0;
1841 unsigned long temp_eflags = 0;
1842 unsigned long cs = 0;
1843 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1844 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1845 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1846 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1848 /* TODO: Add stack limit check */
1850 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1852 if (rc != X86EMUL_CONTINUE)
1855 if (temp_eip & ~0xffff)
1856 return emulate_gp(ctxt, 0);
1858 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1860 if (rc != X86EMUL_CONTINUE)
1863 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1865 if (rc != X86EMUL_CONTINUE)
1868 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1870 if (rc != X86EMUL_CONTINUE)
1873 ctxt->_eip = temp_eip;
1876 if (ctxt->op_bytes == 4)
1877 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1878 else if (ctxt->op_bytes == 2) {
1879 ctxt->eflags &= ~0xffff;
1880 ctxt->eflags |= temp_eflags;
1883 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1884 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1889 static int em_iret(struct x86_emulate_ctxt *ctxt)
1891 switch(ctxt->mode) {
1892 case X86EMUL_MODE_REAL:
1893 return emulate_iret_real(ctxt);
1894 case X86EMUL_MODE_VM86:
1895 case X86EMUL_MODE_PROT16:
1896 case X86EMUL_MODE_PROT32:
1897 case X86EMUL_MODE_PROT64:
1899 /* iret from protected mode unimplemented yet */
1900 return X86EMUL_UNHANDLEABLE;
1904 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1909 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1911 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1912 if (rc != X86EMUL_CONTINUE)
1916 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1917 return X86EMUL_CONTINUE;
1920 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1922 switch (ctxt->modrm_reg) {
1924 emulate_2op_SrcB(ctxt, "rol");
1927 emulate_2op_SrcB(ctxt, "ror");
1930 emulate_2op_SrcB(ctxt, "rcl");
1933 emulate_2op_SrcB(ctxt, "rcr");
1935 case 4: /* sal/shl */
1936 case 6: /* sal/shl */
1937 emulate_2op_SrcB(ctxt, "sal");
1940 emulate_2op_SrcB(ctxt, "shr");
1943 emulate_2op_SrcB(ctxt, "sar");
1946 return X86EMUL_CONTINUE;
1949 static int em_not(struct x86_emulate_ctxt *ctxt)
1951 ctxt->dst.val = ~ctxt->dst.val;
1952 return X86EMUL_CONTINUE;
1955 static int em_neg(struct x86_emulate_ctxt *ctxt)
1957 emulate_1op(ctxt, "neg");
1958 return X86EMUL_CONTINUE;
1961 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1965 emulate_1op_rax_rdx(ctxt, "mul", ex);
1966 return X86EMUL_CONTINUE;
1969 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1973 emulate_1op_rax_rdx(ctxt, "imul", ex);
1974 return X86EMUL_CONTINUE;
1977 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1981 emulate_1op_rax_rdx(ctxt, "div", de);
1983 return emulate_de(ctxt);
1984 return X86EMUL_CONTINUE;
1987 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1991 emulate_1op_rax_rdx(ctxt, "idiv", de);
1993 return emulate_de(ctxt);
1994 return X86EMUL_CONTINUE;
1997 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1999 int rc = X86EMUL_CONTINUE;
2001 switch (ctxt->modrm_reg) {
2003 emulate_1op(ctxt, "inc");
2006 emulate_1op(ctxt, "dec");
2008 case 2: /* call near abs */ {
2010 old_eip = ctxt->_eip;
2011 ctxt->_eip = ctxt->src.val;
2012 ctxt->src.val = old_eip;
2016 case 4: /* jmp abs */
2017 ctxt->_eip = ctxt->src.val;
2019 case 5: /* jmp far */
2020 rc = em_jmp_far(ctxt);
2029 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2031 u64 old = ctxt->dst.orig_val64;
2033 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2034 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2035 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2036 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2037 ctxt->eflags &= ~EFLG_ZF;
2039 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2040 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2042 ctxt->eflags |= EFLG_ZF;
2044 return X86EMUL_CONTINUE;
2047 static int em_ret(struct x86_emulate_ctxt *ctxt)
2049 ctxt->dst.type = OP_REG;
2050 ctxt->dst.addr.reg = &ctxt->_eip;
2051 ctxt->dst.bytes = ctxt->op_bytes;
2052 return em_pop(ctxt);
2055 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2060 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2061 if (rc != X86EMUL_CONTINUE)
2063 if (ctxt->op_bytes == 4)
2064 ctxt->_eip = (u32)ctxt->_eip;
2065 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2066 if (rc != X86EMUL_CONTINUE)
2068 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2072 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2074 /* Save real source value, then compare EAX against destination. */
2075 ctxt->src.orig_val = ctxt->src.val;
2076 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2077 emulate_2op_SrcV(ctxt, "cmp");
2079 if (ctxt->eflags & EFLG_ZF) {
2080 /* Success: write back to memory. */
2081 ctxt->dst.val = ctxt->src.orig_val;
2083 /* Failure: write the value we saw to EAX. */
2084 ctxt->dst.type = OP_REG;
2085 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2087 return X86EMUL_CONTINUE;
2090 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2092 int seg = ctxt->src2.val;
2096 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2098 rc = load_segment_descriptor(ctxt, sel, seg);
2099 if (rc != X86EMUL_CONTINUE)
2102 ctxt->dst.val = ctxt->src.val;
2107 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2108 struct desc_struct *cs, struct desc_struct *ss)
2110 cs->l = 0; /* will be adjusted later */
2111 set_desc_base(cs, 0); /* flat segment */
2112 cs->g = 1; /* 4kb granularity */
2113 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2114 cs->type = 0x0b; /* Read, Execute, Accessed */
2116 cs->dpl = 0; /* will be adjusted later */
2121 set_desc_base(ss, 0); /* flat segment */
2122 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2123 ss->g = 1; /* 4kb granularity */
2125 ss->type = 0x03; /* Read/Write, Accessed */
2126 ss->d = 1; /* 32bit stack segment */
2133 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2135 u32 eax, ebx, ecx, edx;
2138 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2139 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2140 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2141 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2144 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2146 const struct x86_emulate_ops *ops = ctxt->ops;
2147 u32 eax, ebx, ecx, edx;
2150 * syscall should always be enabled in longmode - so only become
2151 * vendor specific (cpuid) if other modes are active...
2153 if (ctxt->mode == X86EMUL_MODE_PROT64)
2158 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2160 * Intel ("GenuineIntel")
2161 * remark: Intel CPUs only support "syscall" in 64bit
2162 * longmode. Also an 64bit guest with a
2163 * 32bit compat-app running will #UD !! While this
2164 * behaviour can be fixed (by emulating) into AMD
2165 * response - CPUs of AMD can't behave like Intel.
2167 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2168 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2169 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2172 /* AMD ("AuthenticAMD") */
2173 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2174 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2175 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2178 /* AMD ("AMDisbetter!") */
2179 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2180 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2181 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2184 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2188 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2190 const struct x86_emulate_ops *ops = ctxt->ops;
2191 struct desc_struct cs, ss;
2196 /* syscall is not available in real mode */
2197 if (ctxt->mode == X86EMUL_MODE_REAL ||
2198 ctxt->mode == X86EMUL_MODE_VM86)
2199 return emulate_ud(ctxt);
2201 if (!(em_syscall_is_enabled(ctxt)))
2202 return emulate_ud(ctxt);
2204 ops->get_msr(ctxt, MSR_EFER, &efer);
2205 setup_syscalls_segments(ctxt, &cs, &ss);
2207 if (!(efer & EFER_SCE))
2208 return emulate_ud(ctxt);
2210 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2212 cs_sel = (u16)(msr_data & 0xfffc);
2213 ss_sel = (u16)(msr_data + 8);
2215 if (efer & EFER_LMA) {
2219 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2220 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2222 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2223 if (efer & EFER_LMA) {
2224 #ifdef CONFIG_X86_64
2225 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2228 ctxt->mode == X86EMUL_MODE_PROT64 ?
2229 MSR_LSTAR : MSR_CSTAR, &msr_data);
2230 ctxt->_eip = msr_data;
2232 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2233 ctxt->eflags &= ~(msr_data | EFLG_RF);
2237 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2238 ctxt->_eip = (u32)msr_data;
2240 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2243 return X86EMUL_CONTINUE;
2246 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2248 const struct x86_emulate_ops *ops = ctxt->ops;
2249 struct desc_struct cs, ss;
2254 ops->get_msr(ctxt, MSR_EFER, &efer);
2255 /* inject #GP if in real mode */
2256 if (ctxt->mode == X86EMUL_MODE_REAL)
2257 return emulate_gp(ctxt, 0);
2260 * Not recognized on AMD in compat mode (but is recognized in legacy
2263 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2264 && !vendor_intel(ctxt))
2265 return emulate_ud(ctxt);
2267 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2268 * Therefore, we inject an #UD.
2270 if (ctxt->mode == X86EMUL_MODE_PROT64)
2271 return emulate_ud(ctxt);
2273 setup_syscalls_segments(ctxt, &cs, &ss);
2275 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2276 switch (ctxt->mode) {
2277 case X86EMUL_MODE_PROT32:
2278 if ((msr_data & 0xfffc) == 0x0)
2279 return emulate_gp(ctxt, 0);
2281 case X86EMUL_MODE_PROT64:
2282 if (msr_data == 0x0)
2283 return emulate_gp(ctxt, 0);
2289 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2290 cs_sel = (u16)msr_data;
2291 cs_sel &= ~SELECTOR_RPL_MASK;
2292 ss_sel = cs_sel + 8;
2293 ss_sel &= ~SELECTOR_RPL_MASK;
2294 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2299 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2300 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2302 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2303 ctxt->_eip = msr_data;
2305 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2306 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2308 return X86EMUL_CONTINUE;
2311 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2313 const struct x86_emulate_ops *ops = ctxt->ops;
2314 struct desc_struct cs, ss;
2317 u16 cs_sel = 0, ss_sel = 0;
2319 /* inject #GP if in real mode or Virtual 8086 mode */
2320 if (ctxt->mode == X86EMUL_MODE_REAL ||
2321 ctxt->mode == X86EMUL_MODE_VM86)
2322 return emulate_gp(ctxt, 0);
2324 setup_syscalls_segments(ctxt, &cs, &ss);
2326 if ((ctxt->rex_prefix & 0x8) != 0x0)
2327 usermode = X86EMUL_MODE_PROT64;
2329 usermode = X86EMUL_MODE_PROT32;
2333 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2335 case X86EMUL_MODE_PROT32:
2336 cs_sel = (u16)(msr_data + 16);
2337 if ((msr_data & 0xfffc) == 0x0)
2338 return emulate_gp(ctxt, 0);
2339 ss_sel = (u16)(msr_data + 24);
2341 case X86EMUL_MODE_PROT64:
2342 cs_sel = (u16)(msr_data + 32);
2343 if (msr_data == 0x0)
2344 return emulate_gp(ctxt, 0);
2345 ss_sel = cs_sel + 8;
2350 cs_sel |= SELECTOR_RPL_MASK;
2351 ss_sel |= SELECTOR_RPL_MASK;
2353 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2354 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2356 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2357 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2359 return X86EMUL_CONTINUE;
2362 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2365 if (ctxt->mode == X86EMUL_MODE_REAL)
2367 if (ctxt->mode == X86EMUL_MODE_VM86)
2369 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2370 return ctxt->ops->cpl(ctxt) > iopl;
2373 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2376 const struct x86_emulate_ops *ops = ctxt->ops;
2377 struct desc_struct tr_seg;
2380 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2381 unsigned mask = (1 << len) - 1;
2384 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2387 if (desc_limit_scaled(&tr_seg) < 103)
2389 base = get_desc_base(&tr_seg);
2390 #ifdef CONFIG_X86_64
2391 base |= ((u64)base3) << 32;
2393 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2394 if (r != X86EMUL_CONTINUE)
2396 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2398 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2399 if (r != X86EMUL_CONTINUE)
2401 if ((perm >> bit_idx) & mask)
2406 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2412 if (emulator_bad_iopl(ctxt))
2413 if (!emulator_io_port_access_allowed(ctxt, port, len))
2416 ctxt->perm_ok = true;
2421 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2422 struct tss_segment_16 *tss)
2424 tss->ip = ctxt->_eip;
2425 tss->flag = ctxt->eflags;
2426 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2427 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2428 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2429 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2430 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2431 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2432 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2433 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2435 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2436 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2437 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2438 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2439 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2442 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2443 struct tss_segment_16 *tss)
2447 ctxt->_eip = tss->ip;
2448 ctxt->eflags = tss->flag | 2;
2449 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2450 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2451 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2452 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2453 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2454 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2455 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2456 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2459 * SDM says that segment selectors are loaded before segment
2462 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2463 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2464 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2465 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2466 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2469 * Now load segment descriptors. If fault happens at this stage
2470 * it is handled in a context of new task
2472 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2473 if (ret != X86EMUL_CONTINUE)
2475 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2476 if (ret != X86EMUL_CONTINUE)
2478 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2479 if (ret != X86EMUL_CONTINUE)
2481 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2482 if (ret != X86EMUL_CONTINUE)
2484 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2485 if (ret != X86EMUL_CONTINUE)
2488 return X86EMUL_CONTINUE;
2491 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2492 u16 tss_selector, u16 old_tss_sel,
2493 ulong old_tss_base, struct desc_struct *new_desc)
2495 const struct x86_emulate_ops *ops = ctxt->ops;
2496 struct tss_segment_16 tss_seg;
2498 u32 new_tss_base = get_desc_base(new_desc);
2500 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2502 if (ret != X86EMUL_CONTINUE)
2503 /* FIXME: need to provide precise fault address */
2506 save_state_to_tss16(ctxt, &tss_seg);
2508 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2510 if (ret != X86EMUL_CONTINUE)
2511 /* FIXME: need to provide precise fault address */
2514 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2516 if (ret != X86EMUL_CONTINUE)
2517 /* FIXME: need to provide precise fault address */
2520 if (old_tss_sel != 0xffff) {
2521 tss_seg.prev_task_link = old_tss_sel;
2523 ret = ops->write_std(ctxt, new_tss_base,
2524 &tss_seg.prev_task_link,
2525 sizeof tss_seg.prev_task_link,
2527 if (ret != X86EMUL_CONTINUE)
2528 /* FIXME: need to provide precise fault address */
2532 return load_state_from_tss16(ctxt, &tss_seg);
2535 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2536 struct tss_segment_32 *tss)
2538 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2539 tss->eip = ctxt->_eip;
2540 tss->eflags = ctxt->eflags;
2541 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2542 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2543 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2544 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2545 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2546 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2547 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2548 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2550 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2551 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2552 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2553 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2554 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2555 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2556 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2559 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2560 struct tss_segment_32 *tss)
2564 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2565 return emulate_gp(ctxt, 0);
2566 ctxt->_eip = tss->eip;
2567 ctxt->eflags = tss->eflags | 2;
2569 /* General purpose registers */
2570 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2571 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2572 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2573 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2574 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2575 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2576 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2577 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2580 * SDM says that segment selectors are loaded before segment
2583 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2584 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2585 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2586 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2587 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2588 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2589 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2592 * If we're switching between Protected Mode and VM86, we need to make
2593 * sure to update the mode before loading the segment descriptors so
2594 * that the selectors are interpreted correctly.
2596 * Need to get rflags to the vcpu struct immediately because it
2597 * influences the CPL which is checked at least when loading the segment
2598 * descriptors and when pushing an error code to the new kernel stack.
2600 * TODO Introduce a separate ctxt->ops->set_cpl callback
2602 if (ctxt->eflags & X86_EFLAGS_VM)
2603 ctxt->mode = X86EMUL_MODE_VM86;
2605 ctxt->mode = X86EMUL_MODE_PROT32;
2607 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2610 * Now load segment descriptors. If fault happenes at this stage
2611 * it is handled in a context of new task
2613 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2614 if (ret != X86EMUL_CONTINUE)
2616 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2617 if (ret != X86EMUL_CONTINUE)
2619 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2620 if (ret != X86EMUL_CONTINUE)
2622 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2626 if (ret != X86EMUL_CONTINUE)
2628 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2632 if (ret != X86EMUL_CONTINUE)
2635 return X86EMUL_CONTINUE;
2638 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2639 u16 tss_selector, u16 old_tss_sel,
2640 ulong old_tss_base, struct desc_struct *new_desc)
2642 const struct x86_emulate_ops *ops = ctxt->ops;
2643 struct tss_segment_32 tss_seg;
2645 u32 new_tss_base = get_desc_base(new_desc);
2647 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2649 if (ret != X86EMUL_CONTINUE)
2650 /* FIXME: need to provide precise fault address */
2653 save_state_to_tss32(ctxt, &tss_seg);
2655 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2657 if (ret != X86EMUL_CONTINUE)
2658 /* FIXME: need to provide precise fault address */
2661 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2663 if (ret != X86EMUL_CONTINUE)
2664 /* FIXME: need to provide precise fault address */
2667 if (old_tss_sel != 0xffff) {
2668 tss_seg.prev_task_link = old_tss_sel;
2670 ret = ops->write_std(ctxt, new_tss_base,
2671 &tss_seg.prev_task_link,
2672 sizeof tss_seg.prev_task_link,
2674 if (ret != X86EMUL_CONTINUE)
2675 /* FIXME: need to provide precise fault address */
2679 return load_state_from_tss32(ctxt, &tss_seg);
2682 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2683 u16 tss_selector, int idt_index, int reason,
2684 bool has_error_code, u32 error_code)
2686 const struct x86_emulate_ops *ops = ctxt->ops;
2687 struct desc_struct curr_tss_desc, next_tss_desc;
2689 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2690 ulong old_tss_base =
2691 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2695 /* FIXME: old_tss_base == ~0 ? */
2697 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2698 if (ret != X86EMUL_CONTINUE)
2700 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2701 if (ret != X86EMUL_CONTINUE)
2704 /* FIXME: check that next_tss_desc is tss */
2707 * Check privileges. The three cases are task switch caused by...
2709 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2710 * 2. Exception/IRQ/iret: No check is performed
2711 * 3. jmp/call to TSS: Check against DPL of the TSS
2713 if (reason == TASK_SWITCH_GATE) {
2714 if (idt_index != -1) {
2715 /* Software interrupts */
2716 struct desc_struct task_gate_desc;
2719 ret = read_interrupt_descriptor(ctxt, idt_index,
2721 if (ret != X86EMUL_CONTINUE)
2724 dpl = task_gate_desc.dpl;
2725 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2726 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2728 } else if (reason != TASK_SWITCH_IRET) {
2729 int dpl = next_tss_desc.dpl;
2730 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2731 return emulate_gp(ctxt, tss_selector);
2735 desc_limit = desc_limit_scaled(&next_tss_desc);
2736 if (!next_tss_desc.p ||
2737 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2738 desc_limit < 0x2b)) {
2739 emulate_ts(ctxt, tss_selector & 0xfffc);
2740 return X86EMUL_PROPAGATE_FAULT;
2743 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2744 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2745 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2748 if (reason == TASK_SWITCH_IRET)
2749 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2751 /* set back link to prev task only if NT bit is set in eflags
2752 note that old_tss_sel is not used after this point */
2753 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2754 old_tss_sel = 0xffff;
2756 if (next_tss_desc.type & 8)
2757 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2758 old_tss_base, &next_tss_desc);
2760 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2761 old_tss_base, &next_tss_desc);
2762 if (ret != X86EMUL_CONTINUE)
2765 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2766 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2768 if (reason != TASK_SWITCH_IRET) {
2769 next_tss_desc.type |= (1 << 1); /* set busy flag */
2770 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2773 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2774 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2776 if (has_error_code) {
2777 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2778 ctxt->lock_prefix = 0;
2779 ctxt->src.val = (unsigned long) error_code;
2780 ret = em_push(ctxt);
2786 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2787 u16 tss_selector, int idt_index, int reason,
2788 bool has_error_code, u32 error_code)
2792 invalidate_registers(ctxt);
2793 ctxt->_eip = ctxt->eip;
2794 ctxt->dst.type = OP_NONE;
2796 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2797 has_error_code, error_code);
2799 if (rc == X86EMUL_CONTINUE) {
2800 ctxt->eip = ctxt->_eip;
2801 writeback_registers(ctxt);
2804 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2807 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2810 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2812 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2813 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2816 static int em_das(struct x86_emulate_ctxt *ctxt)
2819 bool af, cf, old_cf;
2821 cf = ctxt->eflags & X86_EFLAGS_CF;
2827 af = ctxt->eflags & X86_EFLAGS_AF;
2828 if ((al & 0x0f) > 9 || af) {
2830 cf = old_cf | (al >= 250);
2835 if (old_al > 0x99 || old_cf) {
2841 /* Set PF, ZF, SF */
2842 ctxt->src.type = OP_IMM;
2844 ctxt->src.bytes = 1;
2845 emulate_2op_SrcV(ctxt, "or");
2846 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2848 ctxt->eflags |= X86_EFLAGS_CF;
2850 ctxt->eflags |= X86_EFLAGS_AF;
2851 return X86EMUL_CONTINUE;
2854 static int em_call(struct x86_emulate_ctxt *ctxt)
2856 long rel = ctxt->src.val;
2858 ctxt->src.val = (unsigned long)ctxt->_eip;
2860 return em_push(ctxt);
2863 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2869 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2870 old_eip = ctxt->_eip;
2872 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2873 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2874 return X86EMUL_CONTINUE;
2877 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2879 ctxt->src.val = old_cs;
2881 if (rc != X86EMUL_CONTINUE)
2884 ctxt->src.val = old_eip;
2885 return em_push(ctxt);
2888 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2892 ctxt->dst.type = OP_REG;
2893 ctxt->dst.addr.reg = &ctxt->_eip;
2894 ctxt->dst.bytes = ctxt->op_bytes;
2895 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2896 if (rc != X86EMUL_CONTINUE)
2898 rsp_increment(ctxt, ctxt->src.val);
2899 return X86EMUL_CONTINUE;
2902 static int em_add(struct x86_emulate_ctxt *ctxt)
2904 emulate_2op_SrcV(ctxt, "add");
2905 return X86EMUL_CONTINUE;
2908 static int em_or(struct x86_emulate_ctxt *ctxt)
2910 emulate_2op_SrcV(ctxt, "or");
2911 return X86EMUL_CONTINUE;
2914 static int em_adc(struct x86_emulate_ctxt *ctxt)
2916 emulate_2op_SrcV(ctxt, "adc");
2917 return X86EMUL_CONTINUE;
2920 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2922 emulate_2op_SrcV(ctxt, "sbb");
2923 return X86EMUL_CONTINUE;
2926 static int em_and(struct x86_emulate_ctxt *ctxt)
2928 emulate_2op_SrcV(ctxt, "and");
2929 return X86EMUL_CONTINUE;
2932 static int em_sub(struct x86_emulate_ctxt *ctxt)
2934 emulate_2op_SrcV(ctxt, "sub");
2935 return X86EMUL_CONTINUE;
2938 static int em_xor(struct x86_emulate_ctxt *ctxt)
2940 emulate_2op_SrcV(ctxt, "xor");
2941 return X86EMUL_CONTINUE;
2944 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2946 emulate_2op_SrcV(ctxt, "cmp");
2947 /* Disable writeback. */
2948 ctxt->dst.type = OP_NONE;
2949 return X86EMUL_CONTINUE;
2952 static int em_test(struct x86_emulate_ctxt *ctxt)
2954 emulate_2op_SrcV(ctxt, "test");
2955 /* Disable writeback. */
2956 ctxt->dst.type = OP_NONE;
2957 return X86EMUL_CONTINUE;
2960 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2962 /* Write back the register source. */
2963 ctxt->src.val = ctxt->dst.val;
2964 write_register_operand(&ctxt->src);
2966 /* Write back the memory destination with implicit LOCK prefix. */
2967 ctxt->dst.val = ctxt->src.orig_val;
2968 ctxt->lock_prefix = 1;
2969 return X86EMUL_CONTINUE;
2972 static int em_imul(struct x86_emulate_ctxt *ctxt)
2974 emulate_2op_SrcV_nobyte(ctxt, "imul");
2975 return X86EMUL_CONTINUE;
2978 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2980 ctxt->dst.val = ctxt->src2.val;
2981 return em_imul(ctxt);
2984 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2986 ctxt->dst.type = OP_REG;
2987 ctxt->dst.bytes = ctxt->src.bytes;
2988 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2989 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2991 return X86EMUL_CONTINUE;
2994 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2998 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2999 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3000 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3001 return X86EMUL_CONTINUE;
3004 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3008 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3009 return emulate_gp(ctxt, 0);
3010 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3011 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3012 return X86EMUL_CONTINUE;
3015 static int em_mov(struct x86_emulate_ctxt *ctxt)
3017 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3018 return X86EMUL_CONTINUE;
3021 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3023 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3024 return emulate_gp(ctxt, 0);
3026 /* Disable writeback. */
3027 ctxt->dst.type = OP_NONE;
3028 return X86EMUL_CONTINUE;
3031 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3035 if (ctxt->mode == X86EMUL_MODE_PROT64)
3036 val = ctxt->src.val & ~0ULL;
3038 val = ctxt->src.val & ~0U;
3040 /* #UD condition is already handled. */
3041 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3042 return emulate_gp(ctxt, 0);
3044 /* Disable writeback. */
3045 ctxt->dst.type = OP_NONE;
3046 return X86EMUL_CONTINUE;
3049 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3053 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3054 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3055 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3056 return emulate_gp(ctxt, 0);
3058 return X86EMUL_CONTINUE;
3061 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3065 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3066 return emulate_gp(ctxt, 0);
3068 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3069 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3070 return X86EMUL_CONTINUE;
3073 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3075 if (ctxt->modrm_reg > VCPU_SREG_GS)
3076 return emulate_ud(ctxt);
3078 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3079 return X86EMUL_CONTINUE;
3082 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3084 u16 sel = ctxt->src.val;
3086 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3087 return emulate_ud(ctxt);
3089 if (ctxt->modrm_reg == VCPU_SREG_SS)
3090 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3092 /* Disable writeback. */
3093 ctxt->dst.type = OP_NONE;
3094 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3097 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3099 u16 sel = ctxt->src.val;
3101 /* Disable writeback. */
3102 ctxt->dst.type = OP_NONE;
3103 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3106 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3108 u16 sel = ctxt->src.val;
3110 /* Disable writeback. */
3111 ctxt->dst.type = OP_NONE;
3112 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3115 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3120 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3121 if (rc == X86EMUL_CONTINUE)
3122 ctxt->ops->invlpg(ctxt, linear);
3123 /* Disable writeback. */
3124 ctxt->dst.type = OP_NONE;
3125 return X86EMUL_CONTINUE;
3128 static int em_clts(struct x86_emulate_ctxt *ctxt)
3132 cr0 = ctxt->ops->get_cr(ctxt, 0);
3134 ctxt->ops->set_cr(ctxt, 0, cr0);
3135 return X86EMUL_CONTINUE;
3138 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3142 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3143 return X86EMUL_UNHANDLEABLE;
3145 rc = ctxt->ops->fix_hypercall(ctxt);
3146 if (rc != X86EMUL_CONTINUE)
3149 /* Let the processor re-execute the fixed hypercall */
3150 ctxt->_eip = ctxt->eip;
3151 /* Disable writeback. */
3152 ctxt->dst.type = OP_NONE;
3153 return X86EMUL_CONTINUE;
3156 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3157 void (*get)(struct x86_emulate_ctxt *ctxt,
3158 struct desc_ptr *ptr))
3160 struct desc_ptr desc_ptr;
3162 if (ctxt->mode == X86EMUL_MODE_PROT64)
3164 get(ctxt, &desc_ptr);
3165 if (ctxt->op_bytes == 2) {
3167 desc_ptr.address &= 0x00ffffff;
3169 /* Disable writeback. */
3170 ctxt->dst.type = OP_NONE;
3171 return segmented_write(ctxt, ctxt->dst.addr.mem,
3172 &desc_ptr, 2 + ctxt->op_bytes);
3175 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3177 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3180 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3182 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3185 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3187 struct desc_ptr desc_ptr;
3190 if (ctxt->mode == X86EMUL_MODE_PROT64)
3192 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3193 &desc_ptr.size, &desc_ptr.address,
3195 if (rc != X86EMUL_CONTINUE)
3197 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3198 /* Disable writeback. */
3199 ctxt->dst.type = OP_NONE;
3200 return X86EMUL_CONTINUE;
3203 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3207 rc = ctxt->ops->fix_hypercall(ctxt);
3209 /* Disable writeback. */
3210 ctxt->dst.type = OP_NONE;
3214 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3216 struct desc_ptr desc_ptr;
3219 if (ctxt->mode == X86EMUL_MODE_PROT64)
3221 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3222 &desc_ptr.size, &desc_ptr.address,
3224 if (rc != X86EMUL_CONTINUE)
3226 ctxt->ops->set_idt(ctxt, &desc_ptr);
3227 /* Disable writeback. */
3228 ctxt->dst.type = OP_NONE;
3229 return X86EMUL_CONTINUE;
3232 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3234 ctxt->dst.bytes = 2;
3235 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3236 return X86EMUL_CONTINUE;
3239 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3241 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3242 | (ctxt->src.val & 0x0f));
3243 ctxt->dst.type = OP_NONE;
3244 return X86EMUL_CONTINUE;
3247 static int em_loop(struct x86_emulate_ctxt *ctxt)
3249 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3250 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3251 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3252 jmp_rel(ctxt, ctxt->src.val);
3254 return X86EMUL_CONTINUE;
3257 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3259 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3260 jmp_rel(ctxt, ctxt->src.val);
3262 return X86EMUL_CONTINUE;
3265 static int em_in(struct x86_emulate_ctxt *ctxt)
3267 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3269 return X86EMUL_IO_NEEDED;
3271 return X86EMUL_CONTINUE;
3274 static int em_out(struct x86_emulate_ctxt *ctxt)
3276 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3278 /* Disable writeback. */
3279 ctxt->dst.type = OP_NONE;
3280 return X86EMUL_CONTINUE;
3283 static int em_cli(struct x86_emulate_ctxt *ctxt)
3285 if (emulator_bad_iopl(ctxt))
3286 return emulate_gp(ctxt, 0);
3288 ctxt->eflags &= ~X86_EFLAGS_IF;
3289 return X86EMUL_CONTINUE;
3292 static int em_sti(struct x86_emulate_ctxt *ctxt)
3294 if (emulator_bad_iopl(ctxt))
3295 return emulate_gp(ctxt, 0);
3297 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3298 ctxt->eflags |= X86_EFLAGS_IF;
3299 return X86EMUL_CONTINUE;
3302 static int em_bt(struct x86_emulate_ctxt *ctxt)
3304 /* Disable writeback. */
3305 ctxt->dst.type = OP_NONE;
3306 /* only subword offset */
3307 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3309 emulate_2op_SrcV_nobyte(ctxt, "bt");
3310 return X86EMUL_CONTINUE;
3313 static int em_bts(struct x86_emulate_ctxt *ctxt)
3315 emulate_2op_SrcV_nobyte(ctxt, "bts");
3316 return X86EMUL_CONTINUE;
3319 static int em_btr(struct x86_emulate_ctxt *ctxt)
3321 emulate_2op_SrcV_nobyte(ctxt, "btr");
3322 return X86EMUL_CONTINUE;
3325 static int em_btc(struct x86_emulate_ctxt *ctxt)
3327 emulate_2op_SrcV_nobyte(ctxt, "btc");
3328 return X86EMUL_CONTINUE;
3331 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3333 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3334 return X86EMUL_CONTINUE;
3337 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3339 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3340 return X86EMUL_CONTINUE;
3343 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3345 u32 eax, ebx, ecx, edx;
3347 eax = reg_read(ctxt, VCPU_REGS_RAX);
3348 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3349 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3350 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3351 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3352 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3353 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3354 return X86EMUL_CONTINUE;
3357 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3359 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3360 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3361 return X86EMUL_CONTINUE;
3364 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3366 switch (ctxt->op_bytes) {
3367 #ifdef CONFIG_X86_64
3369 asm("bswap %0" : "+r"(ctxt->dst.val));
3373 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3376 return X86EMUL_CONTINUE;
3379 static bool valid_cr(int nr)
3391 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3393 if (!valid_cr(ctxt->modrm_reg))
3394 return emulate_ud(ctxt);
3396 return X86EMUL_CONTINUE;
3399 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3401 u64 new_val = ctxt->src.val64;
3402 int cr = ctxt->modrm_reg;
3405 static u64 cr_reserved_bits[] = {
3406 0xffffffff00000000ULL,
3407 0, 0, 0, /* CR3 checked later */
3414 return emulate_ud(ctxt);
3416 if (new_val & cr_reserved_bits[cr])
3417 return emulate_gp(ctxt, 0);
3422 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3423 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3424 return emulate_gp(ctxt, 0);
3426 cr4 = ctxt->ops->get_cr(ctxt, 4);
3427 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3429 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3430 !(cr4 & X86_CR4_PAE))
3431 return emulate_gp(ctxt, 0);
3438 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3439 if (efer & EFER_LMA)
3440 rsvd = CR3_L_MODE_RESERVED_BITS;
3441 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3442 rsvd = CR3_PAE_RESERVED_BITS;
3443 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3444 rsvd = CR3_NONPAE_RESERVED_BITS;
3447 return emulate_gp(ctxt, 0);
3452 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3454 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3455 return emulate_gp(ctxt, 0);
3461 return X86EMUL_CONTINUE;
3464 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3468 ctxt->ops->get_dr(ctxt, 7, &dr7);
3470 /* Check if DR7.Global_Enable is set */
3471 return dr7 & (1 << 13);
3474 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3476 int dr = ctxt->modrm_reg;
3480 return emulate_ud(ctxt);
3482 cr4 = ctxt->ops->get_cr(ctxt, 4);
3483 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3484 return emulate_ud(ctxt);
3486 if (check_dr7_gd(ctxt))
3487 return emulate_db(ctxt);
3489 return X86EMUL_CONTINUE;
3492 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3494 u64 new_val = ctxt->src.val64;
3495 int dr = ctxt->modrm_reg;
3497 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3498 return emulate_gp(ctxt, 0);
3500 return check_dr_read(ctxt);
3503 static int check_svme(struct x86_emulate_ctxt *ctxt)
3507 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3509 if (!(efer & EFER_SVME))
3510 return emulate_ud(ctxt);
3512 return X86EMUL_CONTINUE;
3515 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3517 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3519 /* Valid physical address? */
3520 if (rax & 0xffff000000000000ULL)
3521 return emulate_gp(ctxt, 0);
3523 return check_svme(ctxt);
3526 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3528 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3530 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3531 return emulate_ud(ctxt);
3533 return X86EMUL_CONTINUE;
3536 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3538 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3539 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3541 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3543 return emulate_gp(ctxt, 0);
3545 return X86EMUL_CONTINUE;
3548 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3550 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3551 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3552 return emulate_gp(ctxt, 0);
3554 return X86EMUL_CONTINUE;
3557 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3559 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3560 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3561 return emulate_gp(ctxt, 0);
3563 return X86EMUL_CONTINUE;
3566 #define D(_y) { .flags = (_y) }
3567 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3568 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3569 .check_perm = (_p) }
3571 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3572 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3573 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3574 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3575 #define II(_f, _e, _i) \
3576 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3577 #define IIP(_f, _e, _i, _p) \
3578 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3579 .check_perm = (_p) }
3580 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3582 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3583 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3584 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3585 #define I2bvIP(_f, _e, _i, _p) \
3586 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3588 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3589 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3590 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3592 static const struct opcode group7_rm1[] = {
3593 DI(SrcNone | Priv, monitor),
3594 DI(SrcNone | Priv, mwait),
3598 static const struct opcode group7_rm3[] = {
3599 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3600 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3601 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3602 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3603 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3604 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3605 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3606 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3609 static const struct opcode group7_rm7[] = {
3611 DIP(SrcNone, rdtscp, check_rdtsc),
3615 static const struct opcode group1[] = {
3617 I(Lock | PageTable, em_or),
3620 I(Lock | PageTable, em_and),
3626 static const struct opcode group1A[] = {
3627 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3630 static const struct opcode group3[] = {
3631 I(DstMem | SrcImm, em_test),
3632 I(DstMem | SrcImm, em_test),
3633 I(DstMem | SrcNone | Lock, em_not),
3634 I(DstMem | SrcNone | Lock, em_neg),
3635 I(SrcMem, em_mul_ex),
3636 I(SrcMem, em_imul_ex),
3637 I(SrcMem, em_div_ex),
3638 I(SrcMem, em_idiv_ex),
3641 static const struct opcode group4[] = {
3642 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3643 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3647 static const struct opcode group5[] = {
3648 I(DstMem | SrcNone | Lock, em_grp45),
3649 I(DstMem | SrcNone | Lock, em_grp45),
3650 I(SrcMem | Stack, em_grp45),
3651 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3652 I(SrcMem | Stack, em_grp45),
3653 I(SrcMemFAddr | ImplicitOps, em_grp45),
3654 I(SrcMem | Stack, em_grp45), N,
3657 static const struct opcode group6[] = {
3660 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3661 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3665 static const struct group_dual group7 = { {
3666 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3667 II(Mov | DstMem | Priv, em_sidt, sidt),
3668 II(SrcMem | Priv, em_lgdt, lgdt),
3669 II(SrcMem | Priv, em_lidt, lidt),
3670 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3671 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3672 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3674 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3676 N, EXT(0, group7_rm3),
3677 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3678 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3682 static const struct opcode group8[] = {
3684 I(DstMem | SrcImmByte, em_bt),
3685 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3686 I(DstMem | SrcImmByte | Lock, em_btr),
3687 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3690 static const struct group_dual group9 = { {
3691 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3693 N, N, N, N, N, N, N, N,
3696 static const struct opcode group11[] = {
3697 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3701 static const struct gprefix pfx_0f_6f_0f_7f = {
3702 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3705 static const struct gprefix pfx_vmovntpx = {
3706 I(0, em_mov), N, N, N,
3709 static const struct opcode opcode_table[256] = {
3711 I6ALU(Lock, em_add),
3712 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3713 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3715 I6ALU(Lock | PageTable, em_or),
3716 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3719 I6ALU(Lock, em_adc),
3720 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3721 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3723 I6ALU(Lock, em_sbb),
3724 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3725 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3727 I6ALU(Lock | PageTable, em_and), N, N,
3729 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3731 I6ALU(Lock, em_xor), N, N,
3733 I6ALU(0, em_cmp), N, N,
3737 X8(I(SrcReg | Stack, em_push)),
3739 X8(I(DstReg | Stack, em_pop)),
3741 I(ImplicitOps | Stack | No64, em_pusha),
3742 I(ImplicitOps | Stack | No64, em_popa),
3743 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3746 I(SrcImm | Mov | Stack, em_push),
3747 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3748 I(SrcImmByte | Mov | Stack, em_push),
3749 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3750 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3751 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3755 G(ByteOp | DstMem | SrcImm, group1),
3756 G(DstMem | SrcImm, group1),
3757 G(ByteOp | DstMem | SrcImm | No64, group1),
3758 G(DstMem | SrcImmByte, group1),
3759 I2bv(DstMem | SrcReg | ModRM, em_test),
3760 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3762 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3763 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3764 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3765 D(ModRM | SrcMem | NoAccess | DstReg),
3766 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3769 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3771 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3772 I(SrcImmFAddr | No64, em_call_far), N,
3773 II(ImplicitOps | Stack, em_pushf, pushf),
3774 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3776 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3777 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3778 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3779 I2bv(SrcSI | DstDI | String, em_cmp),
3781 I2bv(DstAcc | SrcImm, em_test),
3782 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3783 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3784 I2bv(SrcAcc | DstDI | String, em_cmp),
3786 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3788 X8(I(DstReg | SrcImm | Mov, em_mov)),
3790 D2bv(DstMem | SrcImmByte | ModRM),
3791 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3792 I(ImplicitOps | Stack, em_ret),
3793 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3794 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3795 G(ByteOp, group11), G(0, group11),
3797 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3798 N, I(ImplicitOps | Stack, em_ret_far),
3799 D(ImplicitOps), DI(SrcImmByte, intn),
3800 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3802 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3805 N, N, N, N, N, N, N, N,
3807 X3(I(SrcImmByte, em_loop)),
3808 I(SrcImmByte, em_jcxz),
3809 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3810 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3812 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3813 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3814 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3815 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3817 N, DI(ImplicitOps, icebp), N, N,
3818 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3819 G(ByteOp, group3), G(0, group3),
3821 D(ImplicitOps), D(ImplicitOps),
3822 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3823 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3826 static const struct opcode twobyte_table[256] = {
3828 G(0, group6), GD(0, &group7), N, N,
3829 N, I(ImplicitOps | VendorSpecific, em_syscall),
3830 II(ImplicitOps | Priv, em_clts, clts), N,
3831 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3832 N, D(ImplicitOps | ModRM), N, N,
3834 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3836 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3837 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3838 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3839 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3841 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3844 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3845 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3846 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3847 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3848 I(ImplicitOps | VendorSpecific, em_sysenter),
3849 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3851 N, N, N, N, N, N, N, N,
3853 X16(D(DstReg | SrcMem | ModRM | Mov)),
3855 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3860 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3865 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3869 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3871 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3872 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3873 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3874 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3876 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3877 DI(ImplicitOps, rsm),
3878 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3879 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3880 D(DstMem | SrcReg | Src2CL | ModRM),
3881 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3883 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3884 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3885 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3886 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3887 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3888 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3892 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3893 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3894 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3896 D2bv(DstMem | SrcReg | ModRM | Lock),
3897 N, D(DstMem | SrcReg | ModRM | Mov),
3898 N, N, N, GD(0, &group9),
3900 X8(I(DstReg, em_bswap)),
3902 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3904 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3906 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3923 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3927 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3933 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3934 unsigned size, bool sign_extension)
3936 int rc = X86EMUL_CONTINUE;
3940 op->addr.mem.ea = ctxt->_eip;
3941 /* NB. Immediates are sign-extended as necessary. */
3942 switch (op->bytes) {
3944 op->val = insn_fetch(s8, ctxt);
3947 op->val = insn_fetch(s16, ctxt);
3950 op->val = insn_fetch(s32, ctxt);
3953 if (!sign_extension) {
3954 switch (op->bytes) {
3962 op->val &= 0xffffffff;
3970 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3973 int rc = X86EMUL_CONTINUE;
3977 decode_register_operand(ctxt, op);
3980 rc = decode_imm(ctxt, op, 1, false);
3983 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3987 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3988 fetch_bit_operand(ctxt);
3989 op->orig_val = op->val;
3992 ctxt->memop.bytes = 8;
3996 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3997 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3998 fetch_register_operand(op);
3999 op->orig_val = op->val;
4003 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4005 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4006 op->addr.mem.seg = VCPU_SREG_ES;
4013 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4014 fetch_register_operand(op);
4018 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4021 rc = decode_imm(ctxt, op, 1, true);
4028 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4031 ctxt->memop.bytes = 1;
4034 ctxt->memop.bytes = 2;
4037 ctxt->memop.bytes = 4;
4040 rc = decode_imm(ctxt, op, 2, false);
4043 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4047 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4049 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4050 op->addr.mem.seg = seg_override(ctxt);
4056 op->addr.mem.ea = ctxt->_eip;
4057 op->bytes = ctxt->op_bytes + 2;
4058 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4061 ctxt->memop.bytes = ctxt->op_bytes + 2;
4064 op->val = VCPU_SREG_ES;
4067 op->val = VCPU_SREG_CS;
4070 op->val = VCPU_SREG_SS;
4073 op->val = VCPU_SREG_DS;
4076 op->val = VCPU_SREG_FS;
4079 op->val = VCPU_SREG_GS;
4082 /* Special instructions do their own operand decoding. */
4084 op->type = OP_NONE; /* Disable writeback. */
4092 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4094 int rc = X86EMUL_CONTINUE;
4095 int mode = ctxt->mode;
4096 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4097 bool op_prefix = false;
4098 struct opcode opcode;
4100 ctxt->memop.type = OP_NONE;
4101 ctxt->memopp = NULL;
4102 ctxt->_eip = ctxt->eip;
4103 ctxt->fetch.start = ctxt->_eip;
4104 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4106 memcpy(ctxt->fetch.data, insn, insn_len);
4109 case X86EMUL_MODE_REAL:
4110 case X86EMUL_MODE_VM86:
4111 case X86EMUL_MODE_PROT16:
4112 def_op_bytes = def_ad_bytes = 2;
4114 case X86EMUL_MODE_PROT32:
4115 def_op_bytes = def_ad_bytes = 4;
4117 #ifdef CONFIG_X86_64
4118 case X86EMUL_MODE_PROT64:
4124 return EMULATION_FAILED;
4127 ctxt->op_bytes = def_op_bytes;
4128 ctxt->ad_bytes = def_ad_bytes;
4130 /* Legacy prefixes. */
4132 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4133 case 0x66: /* operand-size override */
4135 /* switch between 2/4 bytes */
4136 ctxt->op_bytes = def_op_bytes ^ 6;
4138 case 0x67: /* address-size override */
4139 if (mode == X86EMUL_MODE_PROT64)
4140 /* switch between 4/8 bytes */
4141 ctxt->ad_bytes = def_ad_bytes ^ 12;
4143 /* switch between 2/4 bytes */
4144 ctxt->ad_bytes = def_ad_bytes ^ 6;
4146 case 0x26: /* ES override */
4147 case 0x2e: /* CS override */
4148 case 0x36: /* SS override */
4149 case 0x3e: /* DS override */
4150 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4152 case 0x64: /* FS override */
4153 case 0x65: /* GS override */
4154 set_seg_override(ctxt, ctxt->b & 7);
4156 case 0x40 ... 0x4f: /* REX */
4157 if (mode != X86EMUL_MODE_PROT64)
4159 ctxt->rex_prefix = ctxt->b;
4161 case 0xf0: /* LOCK */
4162 ctxt->lock_prefix = 1;
4164 case 0xf2: /* REPNE/REPNZ */
4165 case 0xf3: /* REP/REPE/REPZ */
4166 ctxt->rep_prefix = ctxt->b;
4172 /* Any legacy prefix after a REX prefix nullifies its effect. */
4174 ctxt->rex_prefix = 0;
4180 if (ctxt->rex_prefix & 8)
4181 ctxt->op_bytes = 8; /* REX.W */
4183 /* Opcode byte(s). */
4184 opcode = opcode_table[ctxt->b];
4185 /* Two-byte opcode? */
4186 if (ctxt->b == 0x0f) {
4188 ctxt->b = insn_fetch(u8, ctxt);
4189 opcode = twobyte_table[ctxt->b];
4191 ctxt->d = opcode.flags;
4193 if (ctxt->d & ModRM)
4194 ctxt->modrm = insn_fetch(u8, ctxt);
4196 while (ctxt->d & GroupMask) {
4197 switch (ctxt->d & GroupMask) {
4199 goffset = (ctxt->modrm >> 3) & 7;
4200 opcode = opcode.u.group[goffset];
4203 goffset = (ctxt->modrm >> 3) & 7;
4204 if ((ctxt->modrm >> 6) == 3)
4205 opcode = opcode.u.gdual->mod3[goffset];
4207 opcode = opcode.u.gdual->mod012[goffset];
4210 goffset = ctxt->modrm & 7;
4211 opcode = opcode.u.group[goffset];
4214 if (ctxt->rep_prefix && op_prefix)
4215 return EMULATION_FAILED;
4216 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4217 switch (simd_prefix) {
4218 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4219 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4220 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4221 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4225 return EMULATION_FAILED;
4228 ctxt->d &= ~(u64)GroupMask;
4229 ctxt->d |= opcode.flags;
4232 ctxt->execute = opcode.u.execute;
4233 ctxt->check_perm = opcode.check_perm;
4234 ctxt->intercept = opcode.intercept;
4237 if (ctxt->d == 0 || (ctxt->d & Undefined))
4238 return EMULATION_FAILED;
4240 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4241 return EMULATION_FAILED;
4243 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4246 if (ctxt->d & Op3264) {
4247 if (mode == X86EMUL_MODE_PROT64)
4254 ctxt->op_bytes = 16;
4255 else if (ctxt->d & Mmx)
4258 /* ModRM and SIB bytes. */
4259 if (ctxt->d & ModRM) {
4260 rc = decode_modrm(ctxt, &ctxt->memop);
4261 if (!ctxt->has_seg_override)
4262 set_seg_override(ctxt, ctxt->modrm_seg);
4263 } else if (ctxt->d & MemAbs)
4264 rc = decode_abs(ctxt, &ctxt->memop);
4265 if (rc != X86EMUL_CONTINUE)
4268 if (!ctxt->has_seg_override)
4269 set_seg_override(ctxt, VCPU_SREG_DS);
4271 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4273 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4274 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4277 * Decode and fetch the source operand: register, memory
4280 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4281 if (rc != X86EMUL_CONTINUE)
4285 * Decode and fetch the second source operand: register, memory
4288 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4289 if (rc != X86EMUL_CONTINUE)
4292 /* Decode and fetch the destination operand: register or memory. */
4293 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4296 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4297 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4299 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4302 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4304 return ctxt->d & PageTable;
4307 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4309 /* The second termination condition only applies for REPE
4310 * and REPNE. Test if the repeat string operation prefix is
4311 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4312 * corresponding termination condition according to:
4313 * - if REPE/REPZ and ZF = 0 then done
4314 * - if REPNE/REPNZ and ZF = 1 then done
4316 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4317 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4318 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4319 ((ctxt->eflags & EFLG_ZF) == 0))
4320 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4321 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4327 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4331 ctxt->ops->get_fpu(ctxt);
4332 asm volatile("1: fwait \n\t"
4334 ".pushsection .fixup,\"ax\" \n\t"
4336 "movb $1, %[fault] \n\t"
4339 _ASM_EXTABLE(1b, 3b)
4340 : [fault]"+qm"(fault));
4341 ctxt->ops->put_fpu(ctxt);
4343 if (unlikely(fault))
4344 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4346 return X86EMUL_CONTINUE;
4349 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4352 if (op->type == OP_MM)
4353 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4357 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4359 const struct x86_emulate_ops *ops = ctxt->ops;
4360 int rc = X86EMUL_CONTINUE;
4361 int saved_dst_type = ctxt->dst.type;
4363 ctxt->mem_read.pos = 0;
4365 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4366 rc = emulate_ud(ctxt);
4370 /* LOCK prefix is allowed only with some instructions */
4371 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4372 rc = emulate_ud(ctxt);
4376 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4377 rc = emulate_ud(ctxt);
4381 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4382 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4383 rc = emulate_ud(ctxt);
4387 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4388 rc = emulate_nm(ctxt);
4392 if (ctxt->d & Mmx) {
4393 rc = flush_pending_x87_faults(ctxt);
4394 if (rc != X86EMUL_CONTINUE)
4397 * Now that we know the fpu is exception safe, we can fetch
4400 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4401 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4402 if (!(ctxt->d & Mov))
4403 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4406 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4407 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4408 X86_ICPT_PRE_EXCEPT);
4409 if (rc != X86EMUL_CONTINUE)
4413 /* Privileged instruction can be executed only in CPL=0 */
4414 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4415 rc = emulate_gp(ctxt, 0);
4419 /* Instruction can only be executed in protected mode */
4420 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4421 rc = emulate_ud(ctxt);
4425 /* Do instruction specific permission checks */
4426 if (ctxt->check_perm) {
4427 rc = ctxt->check_perm(ctxt);
4428 if (rc != X86EMUL_CONTINUE)
4432 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4433 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4434 X86_ICPT_POST_EXCEPT);
4435 if (rc != X86EMUL_CONTINUE)
4439 if (ctxt->rep_prefix && (ctxt->d & String)) {
4440 /* All REP prefixes have the same first termination condition */
4441 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4442 ctxt->eip = ctxt->_eip;
4447 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4448 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4449 ctxt->src.valptr, ctxt->src.bytes);
4450 if (rc != X86EMUL_CONTINUE)
4452 ctxt->src.orig_val64 = ctxt->src.val64;
4455 if (ctxt->src2.type == OP_MEM) {
4456 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4457 &ctxt->src2.val, ctxt->src2.bytes);
4458 if (rc != X86EMUL_CONTINUE)
4462 if ((ctxt->d & DstMask) == ImplicitOps)
4466 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4467 /* optimisation - avoid slow emulated read if Mov */
4468 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4469 &ctxt->dst.val, ctxt->dst.bytes);
4470 if (rc != X86EMUL_CONTINUE)
4473 ctxt->dst.orig_val = ctxt->dst.val;
4477 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4478 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4479 X86_ICPT_POST_MEMACCESS);
4480 if (rc != X86EMUL_CONTINUE)
4484 if (ctxt->execute) {
4485 rc = ctxt->execute(ctxt);
4486 if (rc != X86EMUL_CONTINUE)
4495 case 0x40 ... 0x47: /* inc r16/r32 */
4496 emulate_1op(ctxt, "inc");
4498 case 0x48 ... 0x4f: /* dec r16/r32 */
4499 emulate_1op(ctxt, "dec");
4501 case 0x63: /* movsxd */
4502 if (ctxt->mode != X86EMUL_MODE_PROT64)
4503 goto cannot_emulate;
4504 ctxt->dst.val = (s32) ctxt->src.val;
4506 case 0x70 ... 0x7f: /* jcc (short) */
4507 if (test_cc(ctxt->b, ctxt->eflags))
4508 jmp_rel(ctxt, ctxt->src.val);
4510 case 0x8d: /* lea r16/r32, m */
4511 ctxt->dst.val = ctxt->src.addr.mem.ea;
4513 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4514 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4518 case 0x98: /* cbw/cwde/cdqe */
4519 switch (ctxt->op_bytes) {
4520 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4521 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4522 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4528 case 0xcc: /* int3 */
4529 rc = emulate_int(ctxt, 3);
4531 case 0xcd: /* int n */
4532 rc = emulate_int(ctxt, ctxt->src.val);
4534 case 0xce: /* into */
4535 if (ctxt->eflags & EFLG_OF)
4536 rc = emulate_int(ctxt, 4);
4538 case 0xd0 ... 0xd1: /* Grp2 */
4541 case 0xd2 ... 0xd3: /* Grp2 */
4542 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4545 case 0xe9: /* jmp rel */
4546 case 0xeb: /* jmp rel short */
4547 jmp_rel(ctxt, ctxt->src.val);
4548 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4550 case 0xf4: /* hlt */
4551 ctxt->ops->halt(ctxt);
4553 case 0xf5: /* cmc */
4554 /* complement carry flag from eflags reg */
4555 ctxt->eflags ^= EFLG_CF;
4557 case 0xf8: /* clc */
4558 ctxt->eflags &= ~EFLG_CF;
4560 case 0xf9: /* stc */
4561 ctxt->eflags |= EFLG_CF;
4563 case 0xfc: /* cld */
4564 ctxt->eflags &= ~EFLG_DF;
4566 case 0xfd: /* std */
4567 ctxt->eflags |= EFLG_DF;
4570 goto cannot_emulate;
4573 if (rc != X86EMUL_CONTINUE)
4577 rc = writeback(ctxt);
4578 if (rc != X86EMUL_CONTINUE)
4582 * restore dst type in case the decoding will be reused
4583 * (happens for string instruction )
4585 ctxt->dst.type = saved_dst_type;
4587 if ((ctxt->d & SrcMask) == SrcSI)
4588 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4590 if ((ctxt->d & DstMask) == DstDI)
4591 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4593 if (ctxt->rep_prefix && (ctxt->d & String)) {
4595 struct read_cache *r = &ctxt->io_read;
4596 if ((ctxt->d & SrcMask) == SrcSI)
4597 count = ctxt->src.count;
4599 count = ctxt->dst.count;
4600 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4603 if (!string_insn_completed(ctxt)) {
4605 * Re-enter guest when pio read ahead buffer is empty
4606 * or, if it is not used, after each 1024 iteration.
4608 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4609 (r->end == 0 || r->end != r->pos)) {
4611 * Reset read cache. Usually happens before
4612 * decode, but since instruction is restarted
4613 * we have to do it here.
4615 ctxt->mem_read.end = 0;
4616 writeback_registers(ctxt);
4617 return EMULATION_RESTART;
4619 goto done; /* skip rip writeback */
4623 ctxt->eip = ctxt->_eip;
4626 if (rc == X86EMUL_PROPAGATE_FAULT)
4627 ctxt->have_exception = true;
4628 if (rc == X86EMUL_INTERCEPTED)
4629 return EMULATION_INTERCEPTED;
4631 if (rc == X86EMUL_CONTINUE)
4632 writeback_registers(ctxt);
4634 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4638 case 0x09: /* wbinvd */
4639 (ctxt->ops->wbinvd)(ctxt);
4641 case 0x08: /* invd */
4642 case 0x0d: /* GrpP (prefetch) */
4643 case 0x18: /* Grp16 (prefetch/nop) */
4645 case 0x20: /* mov cr, reg */
4646 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4648 case 0x21: /* mov from dr to reg */
4649 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4651 case 0x40 ... 0x4f: /* cmov */
4652 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4653 if (!test_cc(ctxt->b, ctxt->eflags))
4654 ctxt->dst.type = OP_NONE; /* no writeback */
4656 case 0x80 ... 0x8f: /* jnz rel, etc*/
4657 if (test_cc(ctxt->b, ctxt->eflags))
4658 jmp_rel(ctxt, ctxt->src.val);
4660 case 0x90 ... 0x9f: /* setcc r/m8 */
4661 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4663 case 0xa4: /* shld imm8, r, r/m */
4664 case 0xa5: /* shld cl, r, r/m */
4665 emulate_2op_cl(ctxt, "shld");
4667 case 0xac: /* shrd imm8, r, r/m */
4668 case 0xad: /* shrd cl, r, r/m */
4669 emulate_2op_cl(ctxt, "shrd");
4671 case 0xae: /* clflush */
4673 case 0xb6 ... 0xb7: /* movzx */
4674 ctxt->dst.bytes = ctxt->op_bytes;
4675 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4676 : (u16) ctxt->src.val;
4678 case 0xbe ... 0xbf: /* movsx */
4679 ctxt->dst.bytes = ctxt->op_bytes;
4680 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4681 (s16) ctxt->src.val;
4683 case 0xc0 ... 0xc1: /* xadd */
4684 emulate_2op_SrcV(ctxt, "add");
4685 /* Write back the register source. */
4686 ctxt->src.val = ctxt->dst.orig_val;
4687 write_register_operand(&ctxt->src);
4689 case 0xc3: /* movnti */
4690 ctxt->dst.bytes = ctxt->op_bytes;
4691 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4692 (u64) ctxt->src.val;
4695 goto cannot_emulate;
4698 if (rc != X86EMUL_CONTINUE)
4704 return EMULATION_FAILED;
4707 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4709 invalidate_registers(ctxt);
4712 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4714 writeback_registers(ctxt);