1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
35 #define OpImplicit 1ull /* No generic decode */
36 #define OpReg 2ull /* Register */
37 #define OpMem 3ull /* Memory */
38 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
39 #define OpDI 5ull /* ES:DI/EDI/RDI */
40 #define OpMem64 6ull /* Memory, 64-bit */
41 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
42 #define OpDX 8ull /* DX register */
43 #define OpCL 9ull /* CL register (for shifts) */
44 #define OpImmByte 10ull /* 8-bit sign extended immediate */
45 #define OpOne 11ull /* Implied 1 */
46 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
47 #define OpMem16 13ull /* Memory operand (16-bit). */
48 #define OpMem32 14ull /* Memory operand (32-bit). */
49 #define OpImmU 15ull /* Immediate operand, zero extended */
50 #define OpSI 16ull /* SI/ESI/RSI */
51 #define OpImmFAddr 17ull /* Immediate far address */
52 #define OpMemFAddr 18ull /* Far address in memory */
53 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
54 #define OpES 20ull /* ES */
55 #define OpCS 21ull /* CS */
56 #define OpSS 22ull /* SS */
57 #define OpDS 23ull /* DS */
58 #define OpFS 24ull /* FS */
59 #define OpGS 25ull /* GS */
60 #define OpMem8 26ull /* 8-bit zero extended memory operand */
61 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpBits 5 /* Width of operand field */
64 #define OpMask ((1ull << OpBits) - 1)
67 * Opcode effective-address decode tables.
68 * Note that we only emulate instructions that have at least one memory
69 * operand (excluding implicit stack references). We assume that stack
70 * references and instruction fetches will never occur in special memory
71 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
75 /* Operand sizes: 8-bit operands or specified/overridden size. */
76 #define ByteOp (1<<0) /* 8-bit operands. */
77 /* Destination operand type. */
79 #define ImplicitOps (OpImplicit << DstShift)
80 #define DstReg (OpReg << DstShift)
81 #define DstMem (OpMem << DstShift)
82 #define DstAcc (OpAcc << DstShift)
83 #define DstDI (OpDI << DstShift)
84 #define DstMem64 (OpMem64 << DstShift)
85 #define DstImmUByte (OpImmUByte << DstShift)
86 #define DstDX (OpDX << DstShift)
87 #define DstMask (OpMask << DstShift)
88 /* Source operand type. */
90 #define SrcNone (OpNone << SrcShift)
91 #define SrcReg (OpReg << SrcShift)
92 #define SrcMem (OpMem << SrcShift)
93 #define SrcMem16 (OpMem16 << SrcShift)
94 #define SrcMem32 (OpMem32 << SrcShift)
95 #define SrcImm (OpImm << SrcShift)
96 #define SrcImmByte (OpImmByte << SrcShift)
97 #define SrcOne (OpOne << SrcShift)
98 #define SrcImmUByte (OpImmUByte << SrcShift)
99 #define SrcImmU (OpImmU << SrcShift)
100 #define SrcSI (OpSI << SrcShift)
101 #define SrcImmFAddr (OpImmFAddr << SrcShift)
102 #define SrcMemFAddr (OpMemFAddr << SrcShift)
103 #define SrcAcc (OpAcc << SrcShift)
104 #define SrcImmU16 (OpImmU16 << SrcShift)
105 #define SrcImm64 (OpImm64 << SrcShift)
106 #define SrcDX (OpDX << SrcShift)
107 #define SrcMem8 (OpMem8 << SrcShift)
108 #define SrcMask (OpMask << SrcShift)
109 #define BitOp (1<<11)
110 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
111 #define String (1<<13) /* String instruction (rep capable) */
112 #define Stack (1<<14) /* Stack instruction (push/pop) */
113 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
114 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
115 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
116 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
117 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
118 #define Sse (1<<18) /* SSE Vector instruction */
119 /* Generic ModRM decode. */
120 #define ModRM (1<<19)
121 /* Destination is only written; never read. */
124 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
125 #define VendorSpecific (1<<22) /* Vendor specific instruction */
126 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
127 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
128 #define Undefined (1<<25) /* No Such Instruction */
129 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
130 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
132 #define PageTable (1 << 29) /* instruction used to write page table */
133 /* Source 2 operand type */
134 #define Src2Shift (30)
135 #define Src2None (OpNone << Src2Shift)
136 #define Src2CL (OpCL << Src2Shift)
137 #define Src2ImmByte (OpImmByte << Src2Shift)
138 #define Src2One (OpOne << Src2Shift)
139 #define Src2Imm (OpImm << Src2Shift)
140 #define Src2ES (OpES << Src2Shift)
141 #define Src2CS (OpCS << Src2Shift)
142 #define Src2SS (OpSS << Src2Shift)
143 #define Src2DS (OpDS << Src2Shift)
144 #define Src2FS (OpFS << Src2Shift)
145 #define Src2GS (OpGS << Src2Shift)
146 #define Src2Mask (OpMask << Src2Shift)
147 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
148 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
149 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
150 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
152 #define X2(x...) x, x
153 #define X3(x...) X2(x), x
154 #define X4(x...) X2(x), X2(x)
155 #define X5(x...) X4(x), x
156 #define X6(x...) X4(x), X2(x)
157 #define X7(x...) X4(x), X3(x)
158 #define X8(x...) X4(x), X4(x)
159 #define X16(x...) X8(x), X8(x)
165 int (*execute)(struct x86_emulate_ctxt *ctxt);
166 const struct opcode *group;
167 const struct group_dual *gdual;
168 const struct gprefix *gprefix;
170 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
174 struct opcode mod012[8];
175 struct opcode mod3[8];
179 struct opcode pfx_no;
180 struct opcode pfx_66;
181 struct opcode pfx_f2;
182 struct opcode pfx_f3;
185 /* EFLAGS bit definitions. */
186 #define EFLG_ID (1<<21)
187 #define EFLG_VIP (1<<20)
188 #define EFLG_VIF (1<<19)
189 #define EFLG_AC (1<<18)
190 #define EFLG_VM (1<<17)
191 #define EFLG_RF (1<<16)
192 #define EFLG_IOPL (3<<12)
193 #define EFLG_NT (1<<14)
194 #define EFLG_OF (1<<11)
195 #define EFLG_DF (1<<10)
196 #define EFLG_IF (1<<9)
197 #define EFLG_TF (1<<8)
198 #define EFLG_SF (1<<7)
199 #define EFLG_ZF (1<<6)
200 #define EFLG_AF (1<<4)
201 #define EFLG_PF (1<<2)
202 #define EFLG_CF (1<<0)
204 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
205 #define EFLG_RESERVED_ONE_MASK 2
207 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
209 if (!(ctxt->regs_valid & (1 << nr))) {
210 ctxt->regs_valid |= 1 << nr;
211 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
213 return ctxt->_regs[nr];
216 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
218 ctxt->regs_valid |= 1 << nr;
219 ctxt->regs_dirty |= 1 << nr;
220 return &ctxt->_regs[nr];
223 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
226 return reg_write(ctxt, nr);
229 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
233 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
234 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
237 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
239 ctxt->regs_dirty = 0;
240 ctxt->regs_valid = 0;
244 * Instruction emulation:
245 * Most instructions are emulated directly via a fragment of inline assembly
246 * code. This allows us to save/restore EFLAGS and thus very easily pick up
247 * any modified flags.
250 #if defined(CONFIG_X86_64)
251 #define _LO32 "k" /* force 32-bit operand */
252 #define _STK "%%rsp" /* stack pointer */
253 #elif defined(__i386__)
254 #define _LO32 "" /* force 32-bit operand */
255 #define _STK "%%esp" /* stack pointer */
259 * These EFLAGS bits are restored from saved value during emulation, and
260 * any changes are written back to the saved value after emulation.
262 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
264 /* Before executing instruction: restore necessary bits in EFLAGS. */
265 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
266 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
267 "movl %"_sav",%"_LO32 _tmp"; " \
270 "movl %"_msk",%"_LO32 _tmp"; " \
271 "andl %"_LO32 _tmp",("_STK"); " \
273 "notl %"_LO32 _tmp"; " \
274 "andl %"_LO32 _tmp",("_STK"); " \
275 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
277 "orl %"_LO32 _tmp",("_STK"); " \
281 /* After executing instruction: write-back necessary bits in EFLAGS. */
282 #define _POST_EFLAGS(_sav, _msk, _tmp) \
283 /* _sav |= EFLAGS & _msk; */ \
286 "andl %"_msk",%"_LO32 _tmp"; " \
287 "orl %"_LO32 _tmp",%"_sav"; "
295 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
297 __asm__ __volatile__ ( \
298 _PRE_EFLAGS("0", "4", "2") \
299 _op _suffix " %"_x"3,%1; " \
300 _POST_EFLAGS("0", "4", "2") \
301 : "=m" ((ctxt)->eflags), \
302 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
304 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
308 /* Raw emulation: instruction has two explicit operands. */
309 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
311 unsigned long _tmp; \
313 switch ((ctxt)->dst.bytes) { \
315 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
318 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
321 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
326 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
328 unsigned long _tmp; \
329 switch ((ctxt)->dst.bytes) { \
331 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
334 __emulate_2op_nobyte(ctxt, _op, \
335 _wx, _wy, _lx, _ly, _qx, _qy); \
340 /* Source operand is byte-sized and may be restricted to just %cl. */
341 #define emulate_2op_SrcB(ctxt, _op) \
342 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
344 /* Source operand is byte, word, long or quad sized. */
345 #define emulate_2op_SrcV(ctxt, _op) \
346 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
348 /* Source operand is word, long or quad sized. */
349 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
350 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
352 /* Instruction has three operands and one operand is stored in ECX register */
353 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
355 unsigned long _tmp; \
356 _type _clv = (ctxt)->src2.val; \
357 _type _srcv = (ctxt)->src.val; \
358 _type _dstv = (ctxt)->dst.val; \
360 __asm__ __volatile__ ( \
361 _PRE_EFLAGS("0", "5", "2") \
362 _op _suffix " %4,%1 \n" \
363 _POST_EFLAGS("0", "5", "2") \
364 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
365 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
368 (ctxt)->src2.val = (unsigned long) _clv; \
369 (ctxt)->src2.val = (unsigned long) _srcv; \
370 (ctxt)->dst.val = (unsigned long) _dstv; \
373 #define emulate_2op_cl(ctxt, _op) \
375 switch ((ctxt)->dst.bytes) { \
377 __emulate_2op_cl(ctxt, _op, "w", u16); \
380 __emulate_2op_cl(ctxt, _op, "l", u32); \
383 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
388 #define __emulate_1op(ctxt, _op, _suffix) \
390 unsigned long _tmp; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "3", "2") \
394 _op _suffix " %1; " \
395 _POST_EFLAGS("0", "3", "2") \
396 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
398 : "i" (EFLAGS_MASK)); \
401 /* Instruction has only one explicit operand (no source operand). */
402 #define emulate_1op(ctxt, _op) \
404 switch ((ctxt)->dst.bytes) { \
405 case 1: __emulate_1op(ctxt, _op, "b"); break; \
406 case 2: __emulate_1op(ctxt, _op, "w"); break; \
407 case 4: __emulate_1op(ctxt, _op, "l"); break; \
408 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
412 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
414 unsigned long _tmp; \
415 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
416 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
418 __asm__ __volatile__ ( \
419 _PRE_EFLAGS("0", "5", "1") \
421 _op _suffix " %6; " \
423 _POST_EFLAGS("0", "5", "1") \
424 ".pushsection .fixup,\"ax\" \n\t" \
425 "3: movb $1, %4 \n\t" \
428 _ASM_EXTABLE(1b, 3b) \
429 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
430 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
431 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
434 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
435 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
437 switch((ctxt)->src.bytes) { \
439 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
442 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
445 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
448 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
453 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
454 enum x86_intercept intercept,
455 enum x86_intercept_stage stage)
457 struct x86_instruction_info info = {
458 .intercept = intercept,
459 .rep_prefix = ctxt->rep_prefix,
460 .modrm_mod = ctxt->modrm_mod,
461 .modrm_reg = ctxt->modrm_reg,
462 .modrm_rm = ctxt->modrm_rm,
463 .src_val = ctxt->src.val64,
464 .src_bytes = ctxt->src.bytes,
465 .dst_bytes = ctxt->dst.bytes,
466 .ad_bytes = ctxt->ad_bytes,
467 .next_rip = ctxt->eip,
470 return ctxt->ops->intercept(ctxt, &info, stage);
473 static void assign_masked(ulong *dest, ulong src, ulong mask)
475 *dest = (*dest & ~mask) | (src & mask);
478 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
480 return (1UL << (ctxt->ad_bytes << 3)) - 1;
483 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
486 struct desc_struct ss;
488 if (ctxt->mode == X86EMUL_MODE_PROT64)
490 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
491 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
494 static int stack_size(struct x86_emulate_ctxt *ctxt)
496 return (__fls(stack_mask(ctxt)) + 1) >> 3;
499 /* Access/update address held in a register, based on addressing mode. */
500 static inline unsigned long
501 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
503 if (ctxt->ad_bytes == sizeof(unsigned long))
506 return reg & ad_mask(ctxt);
509 static inline unsigned long
510 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
512 return address_mask(ctxt, reg);
515 static void masked_increment(ulong *reg, ulong mask, int inc)
517 assign_masked(reg, *reg + inc, mask);
521 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
525 if (ctxt->ad_bytes == sizeof(unsigned long))
528 mask = ad_mask(ctxt);
529 masked_increment(reg, mask, inc);
532 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
534 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
537 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
539 register_address_increment(ctxt, &ctxt->_eip, rel);
542 static u32 desc_limit_scaled(struct desc_struct *desc)
544 u32 limit = get_desc_limit(desc);
546 return desc->g ? (limit << 12) | 0xfff : limit;
549 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
551 ctxt->has_seg_override = true;
552 ctxt->seg_override = seg;
555 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
557 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
560 return ctxt->ops->get_cached_segment_base(ctxt, seg);
563 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
565 if (!ctxt->has_seg_override)
568 return ctxt->seg_override;
571 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
572 u32 error, bool valid)
574 ctxt->exception.vector = vec;
575 ctxt->exception.error_code = error;
576 ctxt->exception.error_code_valid = valid;
577 return X86EMUL_PROPAGATE_FAULT;
580 static int emulate_db(struct x86_emulate_ctxt *ctxt)
582 return emulate_exception(ctxt, DB_VECTOR, 0, false);
585 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
587 return emulate_exception(ctxt, GP_VECTOR, err, true);
590 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
592 return emulate_exception(ctxt, SS_VECTOR, err, true);
595 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
597 return emulate_exception(ctxt, UD_VECTOR, 0, false);
600 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
602 return emulate_exception(ctxt, TS_VECTOR, err, true);
605 static int emulate_de(struct x86_emulate_ctxt *ctxt)
607 return emulate_exception(ctxt, DE_VECTOR, 0, false);
610 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
612 return emulate_exception(ctxt, NM_VECTOR, 0, false);
615 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
618 struct desc_struct desc;
620 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
624 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
629 struct desc_struct desc;
631 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
632 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
636 * x86 defines three classes of vector instructions: explicitly
637 * aligned, explicitly unaligned, and the rest, which change behaviour
638 * depending on whether they're AVX encoded or not.
640 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
641 * subject to the same check.
643 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
645 if (likely(size < 16))
648 if (ctxt->d & Aligned)
650 else if (ctxt->d & Unaligned)
652 else if (ctxt->d & Avx)
658 static int __linearize(struct x86_emulate_ctxt *ctxt,
659 struct segmented_address addr,
660 unsigned size, bool write, bool fetch,
663 struct desc_struct desc;
670 la = seg_base(ctxt, addr.seg) + addr.ea;
671 switch (ctxt->mode) {
672 case X86EMUL_MODE_PROT64:
673 if (((signed long)la << 16) >> 16 != la)
674 return emulate_gp(ctxt, 0);
677 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
681 /* code segment in protected mode or read-only data segment */
682 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
683 || !(desc.type & 2)) && write)
685 /* unreadable code segment */
686 if (!fetch && (desc.type & 8) && !(desc.type & 2))
688 lim = desc_limit_scaled(&desc);
689 if ((desc.type & 8) || !(desc.type & 4)) {
690 /* expand-up segment */
691 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
694 /* expand-down segment */
695 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
697 lim = desc.d ? 0xffffffff : 0xffff;
698 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
701 cpl = ctxt->ops->cpl(ctxt);
702 if (ctxt->mode == X86EMUL_MODE_REAL)
707 if (!(desc.type & 8)) {
711 } else if ((desc.type & 8) && !(desc.type & 4)) {
712 /* nonconforming code segment */
715 } else if ((desc.type & 8) && (desc.type & 4)) {
716 /* conforming code segment */
722 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
724 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
725 return emulate_gp(ctxt, 0);
727 return X86EMUL_CONTINUE;
729 if (addr.seg == VCPU_SREG_SS)
730 return emulate_ss(ctxt, sel);
732 return emulate_gp(ctxt, sel);
735 static int linearize(struct x86_emulate_ctxt *ctxt,
736 struct segmented_address addr,
737 unsigned size, bool write,
740 return __linearize(ctxt, addr, size, write, false, linear);
744 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
745 struct segmented_address addr,
752 rc = linearize(ctxt, addr, size, false, &linear);
753 if (rc != X86EMUL_CONTINUE)
755 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
759 * Fetch the next byte of the instruction being emulated which is pointed to
760 * by ctxt->_eip, then increment ctxt->_eip.
762 * Also prefetch the remaining bytes of the instruction without crossing page
763 * boundary if they are not in fetch_cache yet.
765 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
767 struct fetch_cache *fc = &ctxt->fetch;
771 if (ctxt->_eip == fc->end) {
772 unsigned long linear;
773 struct segmented_address addr = { .seg = VCPU_SREG_CS,
775 cur_size = fc->end - fc->start;
776 size = min(15UL - cur_size,
777 PAGE_SIZE - offset_in_page(ctxt->_eip));
778 rc = __linearize(ctxt, addr, size, false, true, &linear);
779 if (unlikely(rc != X86EMUL_CONTINUE))
781 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
782 size, &ctxt->exception);
783 if (unlikely(rc != X86EMUL_CONTINUE))
787 *dest = fc->data[ctxt->_eip - fc->start];
789 return X86EMUL_CONTINUE;
792 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
793 void *dest, unsigned size)
797 /* x86 instructions are limited to 15 bytes. */
798 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
799 return X86EMUL_UNHANDLEABLE;
801 rc = do_insn_fetch_byte(ctxt, dest++);
802 if (rc != X86EMUL_CONTINUE)
805 return X86EMUL_CONTINUE;
808 /* Fetch next part of the instruction being emulated. */
809 #define insn_fetch(_type, _ctxt) \
810 ({ unsigned long _x; \
811 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
812 if (rc != X86EMUL_CONTINUE) \
817 #define insn_fetch_arr(_arr, _size, _ctxt) \
818 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
819 if (rc != X86EMUL_CONTINUE) \
824 * Given the 'reg' portion of a ModRM byte, and a register block, return a
825 * pointer into the block that addresses the relevant register.
826 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
828 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
833 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
834 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
836 p = reg_rmw(ctxt, modrm_reg);
840 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
841 struct segmented_address addr,
842 u16 *size, unsigned long *address, int op_bytes)
849 rc = segmented_read_std(ctxt, addr, size, 2);
850 if (rc != X86EMUL_CONTINUE)
853 rc = segmented_read_std(ctxt, addr, address, op_bytes);
857 static int test_cc(unsigned int condition, unsigned int flags)
861 switch ((condition & 15) >> 1) {
863 rc |= (flags & EFLG_OF);
865 case 1: /* b/c/nae */
866 rc |= (flags & EFLG_CF);
869 rc |= (flags & EFLG_ZF);
872 rc |= (flags & (EFLG_CF|EFLG_ZF));
875 rc |= (flags & EFLG_SF);
878 rc |= (flags & EFLG_PF);
881 rc |= (flags & EFLG_ZF);
884 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
888 /* Odd condition identifiers (lsb == 1) have inverted sense. */
889 return (!!rc ^ (condition & 1));
892 static void fetch_register_operand(struct operand *op)
896 op->val = *(u8 *)op->addr.reg;
899 op->val = *(u16 *)op->addr.reg;
902 op->val = *(u32 *)op->addr.reg;
905 op->val = *(u64 *)op->addr.reg;
910 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
912 ctxt->ops->get_fpu(ctxt);
914 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
915 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
916 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
917 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
918 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
919 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
920 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
921 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
923 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
924 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
925 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
926 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
927 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
928 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
929 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
930 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
934 ctxt->ops->put_fpu(ctxt);
937 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
940 ctxt->ops->get_fpu(ctxt);
942 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
943 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
944 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
945 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
946 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
947 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
948 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
949 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
951 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
952 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
953 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
954 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
955 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
956 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
957 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
958 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
962 ctxt->ops->put_fpu(ctxt);
965 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
967 ctxt->ops->get_fpu(ctxt);
969 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
970 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
971 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
972 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
973 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
974 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
975 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
976 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
979 ctxt->ops->put_fpu(ctxt);
982 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
984 ctxt->ops->get_fpu(ctxt);
986 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
987 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
988 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
989 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
990 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
991 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
992 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
993 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
996 ctxt->ops->put_fpu(ctxt);
999 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1002 unsigned reg = ctxt->modrm_reg;
1003 int highbyte_regs = ctxt->rex_prefix == 0;
1005 if (!(ctxt->d & ModRM))
1006 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1008 if (ctxt->d & Sse) {
1012 read_sse_reg(ctxt, &op->vec_val, reg);
1015 if (ctxt->d & Mmx) {
1024 if (ctxt->d & ByteOp) {
1025 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1028 op->addr.reg = decode_register(ctxt, reg, 0);
1029 op->bytes = ctxt->op_bytes;
1031 fetch_register_operand(op);
1032 op->orig_val = op->val;
1035 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1037 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1038 ctxt->modrm_seg = VCPU_SREG_SS;
1041 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1045 int index_reg = 0, base_reg = 0, scale;
1046 int rc = X86EMUL_CONTINUE;
1049 if (ctxt->rex_prefix) {
1050 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1051 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1052 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1055 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1056 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1057 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1058 ctxt->modrm_seg = VCPU_SREG_DS;
1060 if (ctxt->modrm_mod == 3) {
1062 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1063 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1064 if (ctxt->d & Sse) {
1067 op->addr.xmm = ctxt->modrm_rm;
1068 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1071 if (ctxt->d & Mmx) {
1074 op->addr.xmm = ctxt->modrm_rm & 7;
1077 fetch_register_operand(op);
1083 if (ctxt->ad_bytes == 2) {
1084 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1085 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1086 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1087 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1089 /* 16-bit ModR/M decode. */
1090 switch (ctxt->modrm_mod) {
1092 if (ctxt->modrm_rm == 6)
1093 modrm_ea += insn_fetch(u16, ctxt);
1096 modrm_ea += insn_fetch(s8, ctxt);
1099 modrm_ea += insn_fetch(u16, ctxt);
1102 switch (ctxt->modrm_rm) {
1104 modrm_ea += bx + si;
1107 modrm_ea += bx + di;
1110 modrm_ea += bp + si;
1113 modrm_ea += bp + di;
1122 if (ctxt->modrm_mod != 0)
1129 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1130 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1131 ctxt->modrm_seg = VCPU_SREG_SS;
1132 modrm_ea = (u16)modrm_ea;
1134 /* 32/64-bit ModR/M decode. */
1135 if ((ctxt->modrm_rm & 7) == 4) {
1136 sib = insn_fetch(u8, ctxt);
1137 index_reg |= (sib >> 3) & 7;
1138 base_reg |= sib & 7;
1141 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1142 modrm_ea += insn_fetch(s32, ctxt);
1144 modrm_ea += reg_read(ctxt, base_reg);
1145 adjust_modrm_seg(ctxt, base_reg);
1148 modrm_ea += reg_read(ctxt, index_reg) << scale;
1149 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1150 if (ctxt->mode == X86EMUL_MODE_PROT64)
1151 ctxt->rip_relative = 1;
1153 base_reg = ctxt->modrm_rm;
1154 modrm_ea += reg_read(ctxt, base_reg);
1155 adjust_modrm_seg(ctxt, base_reg);
1157 switch (ctxt->modrm_mod) {
1159 if (ctxt->modrm_rm == 5)
1160 modrm_ea += insn_fetch(s32, ctxt);
1163 modrm_ea += insn_fetch(s8, ctxt);
1166 modrm_ea += insn_fetch(s32, ctxt);
1170 op->addr.mem.ea = modrm_ea;
1175 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1178 int rc = X86EMUL_CONTINUE;
1181 switch (ctxt->ad_bytes) {
1183 op->addr.mem.ea = insn_fetch(u16, ctxt);
1186 op->addr.mem.ea = insn_fetch(u32, ctxt);
1189 op->addr.mem.ea = insn_fetch(u64, ctxt);
1196 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1200 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1201 mask = ~(ctxt->dst.bytes * 8 - 1);
1203 if (ctxt->src.bytes == 2)
1204 sv = (s16)ctxt->src.val & (s16)mask;
1205 else if (ctxt->src.bytes == 4)
1206 sv = (s32)ctxt->src.val & (s32)mask;
1208 ctxt->dst.addr.mem.ea += (sv >> 3);
1211 /* only subword offset */
1212 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1215 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1216 unsigned long addr, void *dest, unsigned size)
1219 struct read_cache *mc = &ctxt->mem_read;
1221 if (mc->pos < mc->end)
1224 WARN_ON((mc->end + size) >= sizeof(mc->data));
1226 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1228 if (rc != X86EMUL_CONTINUE)
1234 memcpy(dest, mc->data + mc->pos, size);
1236 return X86EMUL_CONTINUE;
1239 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1240 struct segmented_address addr,
1247 rc = linearize(ctxt, addr, size, false, &linear);
1248 if (rc != X86EMUL_CONTINUE)
1250 return read_emulated(ctxt, linear, data, size);
1253 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1254 struct segmented_address addr,
1261 rc = linearize(ctxt, addr, size, true, &linear);
1262 if (rc != X86EMUL_CONTINUE)
1264 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1268 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1269 struct segmented_address addr,
1270 const void *orig_data, const void *data,
1276 rc = linearize(ctxt, addr, size, true, &linear);
1277 if (rc != X86EMUL_CONTINUE)
1279 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1280 size, &ctxt->exception);
1283 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1284 unsigned int size, unsigned short port,
1287 struct read_cache *rc = &ctxt->io_read;
1289 if (rc->pos == rc->end) { /* refill pio read ahead */
1290 unsigned int in_page, n;
1291 unsigned int count = ctxt->rep_prefix ?
1292 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1293 in_page = (ctxt->eflags & EFLG_DF) ?
1294 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1295 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1296 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1300 rc->pos = rc->end = 0;
1301 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1306 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1307 ctxt->dst.data = rc->data + rc->pos;
1308 ctxt->dst.type = OP_MEM_STR;
1309 ctxt->dst.count = (rc->end - rc->pos) / size;
1312 memcpy(dest, rc->data + rc->pos, size);
1318 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1319 u16 index, struct desc_struct *desc)
1324 ctxt->ops->get_idt(ctxt, &dt);
1326 if (dt.size < index * 8 + 7)
1327 return emulate_gp(ctxt, index << 3 | 0x2);
1329 addr = dt.address + index * 8;
1330 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1334 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1335 u16 selector, struct desc_ptr *dt)
1337 const struct x86_emulate_ops *ops = ctxt->ops;
1339 if (selector & 1 << 2) {
1340 struct desc_struct desc;
1343 memset (dt, 0, sizeof *dt);
1344 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1347 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1348 dt->address = get_desc_base(&desc);
1350 ops->get_gdt(ctxt, dt);
1353 /* allowed just for 8 bytes segments */
1354 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1355 u16 selector, struct desc_struct *desc,
1359 u16 index = selector >> 3;
1362 get_descriptor_table_ptr(ctxt, selector, &dt);
1364 if (dt.size < index * 8 + 7)
1365 return emulate_gp(ctxt, selector & 0xfffc);
1367 *desc_addr_p = addr = dt.address + index * 8;
1368 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1372 /* allowed just for 8 bytes segments */
1373 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1374 u16 selector, struct desc_struct *desc)
1377 u16 index = selector >> 3;
1380 get_descriptor_table_ptr(ctxt, selector, &dt);
1382 if (dt.size < index * 8 + 7)
1383 return emulate_gp(ctxt, selector & 0xfffc);
1385 addr = dt.address + index * 8;
1386 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1390 /* Does not support long mode */
1391 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1392 u16 selector, int seg)
1394 struct desc_struct seg_desc, old_desc;
1396 unsigned err_vec = GP_VECTOR;
1398 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1403 memset(&seg_desc, 0, sizeof seg_desc);
1405 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1406 || ctxt->mode == X86EMUL_MODE_REAL) {
1407 /* set real mode segment descriptor */
1408 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1409 set_desc_base(&seg_desc, selector << 4);
1414 cpl = ctxt->ops->cpl(ctxt);
1416 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1417 if ((seg == VCPU_SREG_CS
1418 || (seg == VCPU_SREG_SS
1419 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1420 || seg == VCPU_SREG_TR)
1424 /* TR should be in GDT only */
1425 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1428 if (null_selector) /* for NULL selector skip all following checks */
1431 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1432 if (ret != X86EMUL_CONTINUE)
1435 err_code = selector & 0xfffc;
1436 err_vec = GP_VECTOR;
1438 /* can't load system descriptor into segment selector */
1439 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1443 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1452 * segment is not a writable data segment or segment
1453 * selector's RPL != CPL or segment selector's RPL != CPL
1455 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1459 if (!(seg_desc.type & 8))
1462 if (seg_desc.type & 4) {
1468 if (rpl > cpl || dpl != cpl)
1471 /* CS(RPL) <- CPL */
1472 selector = (selector & 0xfffc) | cpl;
1475 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1477 old_desc = seg_desc;
1478 seg_desc.type |= 2; /* busy */
1479 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1480 sizeof(seg_desc), &ctxt->exception);
1481 if (ret != X86EMUL_CONTINUE)
1484 case VCPU_SREG_LDTR:
1485 if (seg_desc.s || seg_desc.type != 2)
1488 default: /* DS, ES, FS, or GS */
1490 * segment is not a data or readable code segment or
1491 * ((segment is a data or nonconforming code segment)
1492 * and (both RPL and CPL > DPL))
1494 if ((seg_desc.type & 0xa) == 0x8 ||
1495 (((seg_desc.type & 0xc) != 0xc) &&
1496 (rpl > dpl && cpl > dpl)))
1502 /* mark segment as accessed */
1504 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1505 if (ret != X86EMUL_CONTINUE)
1509 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1510 return X86EMUL_CONTINUE;
1512 emulate_exception(ctxt, err_vec, err_code, true);
1513 return X86EMUL_PROPAGATE_FAULT;
1516 static void write_register_operand(struct operand *op)
1518 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1519 switch (op->bytes) {
1521 *(u8 *)op->addr.reg = (u8)op->val;
1524 *(u16 *)op->addr.reg = (u16)op->val;
1527 *op->addr.reg = (u32)op->val;
1528 break; /* 64b: zero-extend */
1530 *op->addr.reg = op->val;
1535 static int writeback(struct x86_emulate_ctxt *ctxt)
1539 switch (ctxt->dst.type) {
1541 write_register_operand(&ctxt->dst);
1544 if (ctxt->lock_prefix)
1545 rc = segmented_cmpxchg(ctxt,
1547 &ctxt->dst.orig_val,
1551 rc = segmented_write(ctxt,
1555 if (rc != X86EMUL_CONTINUE)
1559 rc = segmented_write(ctxt,
1562 ctxt->dst.bytes * ctxt->dst.count);
1563 if (rc != X86EMUL_CONTINUE)
1567 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1570 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1578 return X86EMUL_CONTINUE;
1581 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1583 struct segmented_address addr;
1585 rsp_increment(ctxt, -bytes);
1586 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1587 addr.seg = VCPU_SREG_SS;
1589 return segmented_write(ctxt, addr, data, bytes);
1592 static int em_push(struct x86_emulate_ctxt *ctxt)
1594 /* Disable writeback. */
1595 ctxt->dst.type = OP_NONE;
1596 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1599 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1600 void *dest, int len)
1603 struct segmented_address addr;
1605 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1606 addr.seg = VCPU_SREG_SS;
1607 rc = segmented_read(ctxt, addr, dest, len);
1608 if (rc != X86EMUL_CONTINUE)
1611 rsp_increment(ctxt, len);
1615 static int em_pop(struct x86_emulate_ctxt *ctxt)
1617 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1620 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1621 void *dest, int len)
1624 unsigned long val, change_mask;
1625 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1626 int cpl = ctxt->ops->cpl(ctxt);
1628 rc = emulate_pop(ctxt, &val, len);
1629 if (rc != X86EMUL_CONTINUE)
1632 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1633 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1635 switch(ctxt->mode) {
1636 case X86EMUL_MODE_PROT64:
1637 case X86EMUL_MODE_PROT32:
1638 case X86EMUL_MODE_PROT16:
1640 change_mask |= EFLG_IOPL;
1642 change_mask |= EFLG_IF;
1644 case X86EMUL_MODE_VM86:
1646 return emulate_gp(ctxt, 0);
1647 change_mask |= EFLG_IF;
1649 default: /* real mode */
1650 change_mask |= (EFLG_IOPL | EFLG_IF);
1654 *(unsigned long *)dest =
1655 (ctxt->eflags & ~change_mask) | (val & change_mask);
1660 static int em_popf(struct x86_emulate_ctxt *ctxt)
1662 ctxt->dst.type = OP_REG;
1663 ctxt->dst.addr.reg = &ctxt->eflags;
1664 ctxt->dst.bytes = ctxt->op_bytes;
1665 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1668 static int em_enter(struct x86_emulate_ctxt *ctxt)
1671 unsigned frame_size = ctxt->src.val;
1672 unsigned nesting_level = ctxt->src2.val & 31;
1676 return X86EMUL_UNHANDLEABLE;
1678 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1679 rc = push(ctxt, &rbp, stack_size(ctxt));
1680 if (rc != X86EMUL_CONTINUE)
1682 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1684 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1685 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1687 return X86EMUL_CONTINUE;
1690 static int em_leave(struct x86_emulate_ctxt *ctxt)
1692 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1694 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1697 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1699 int seg = ctxt->src2.val;
1701 ctxt->src.val = get_segment_selector(ctxt, seg);
1703 return em_push(ctxt);
1706 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1708 int seg = ctxt->src2.val;
1709 unsigned long selector;
1712 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1713 if (rc != X86EMUL_CONTINUE)
1716 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1720 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1722 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1723 int rc = X86EMUL_CONTINUE;
1724 int reg = VCPU_REGS_RAX;
1726 while (reg <= VCPU_REGS_RDI) {
1727 (reg == VCPU_REGS_RSP) ?
1728 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1731 if (rc != X86EMUL_CONTINUE)
1740 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1742 ctxt->src.val = (unsigned long)ctxt->eflags;
1743 return em_push(ctxt);
1746 static int em_popa(struct x86_emulate_ctxt *ctxt)
1748 int rc = X86EMUL_CONTINUE;
1749 int reg = VCPU_REGS_RDI;
1751 while (reg >= VCPU_REGS_RAX) {
1752 if (reg == VCPU_REGS_RSP) {
1753 rsp_increment(ctxt, ctxt->op_bytes);
1757 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1758 if (rc != X86EMUL_CONTINUE)
1765 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1767 const struct x86_emulate_ops *ops = ctxt->ops;
1774 /* TODO: Add limit checks */
1775 ctxt->src.val = ctxt->eflags;
1777 if (rc != X86EMUL_CONTINUE)
1780 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1782 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1784 if (rc != X86EMUL_CONTINUE)
1787 ctxt->src.val = ctxt->_eip;
1789 if (rc != X86EMUL_CONTINUE)
1792 ops->get_idt(ctxt, &dt);
1794 eip_addr = dt.address + (irq << 2);
1795 cs_addr = dt.address + (irq << 2) + 2;
1797 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1798 if (rc != X86EMUL_CONTINUE)
1801 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1802 if (rc != X86EMUL_CONTINUE)
1805 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1806 if (rc != X86EMUL_CONTINUE)
1814 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1818 invalidate_registers(ctxt);
1819 rc = __emulate_int_real(ctxt, irq);
1820 if (rc == X86EMUL_CONTINUE)
1821 writeback_registers(ctxt);
1825 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1827 switch(ctxt->mode) {
1828 case X86EMUL_MODE_REAL:
1829 return __emulate_int_real(ctxt, irq);
1830 case X86EMUL_MODE_VM86:
1831 case X86EMUL_MODE_PROT16:
1832 case X86EMUL_MODE_PROT32:
1833 case X86EMUL_MODE_PROT64:
1835 /* Protected mode interrupts unimplemented yet */
1836 return X86EMUL_UNHANDLEABLE;
1840 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1842 int rc = X86EMUL_CONTINUE;
1843 unsigned long temp_eip = 0;
1844 unsigned long temp_eflags = 0;
1845 unsigned long cs = 0;
1846 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1847 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1848 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1849 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1851 /* TODO: Add stack limit check */
1853 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1855 if (rc != X86EMUL_CONTINUE)
1858 if (temp_eip & ~0xffff)
1859 return emulate_gp(ctxt, 0);
1861 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1863 if (rc != X86EMUL_CONTINUE)
1866 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1868 if (rc != X86EMUL_CONTINUE)
1871 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1873 if (rc != X86EMUL_CONTINUE)
1876 ctxt->_eip = temp_eip;
1879 if (ctxt->op_bytes == 4)
1880 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1881 else if (ctxt->op_bytes == 2) {
1882 ctxt->eflags &= ~0xffff;
1883 ctxt->eflags |= temp_eflags;
1886 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1887 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1892 static int em_iret(struct x86_emulate_ctxt *ctxt)
1894 switch(ctxt->mode) {
1895 case X86EMUL_MODE_REAL:
1896 return emulate_iret_real(ctxt);
1897 case X86EMUL_MODE_VM86:
1898 case X86EMUL_MODE_PROT16:
1899 case X86EMUL_MODE_PROT32:
1900 case X86EMUL_MODE_PROT64:
1902 /* iret from protected mode unimplemented yet */
1903 return X86EMUL_UNHANDLEABLE;
1907 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1912 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1914 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1915 if (rc != X86EMUL_CONTINUE)
1919 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1920 return X86EMUL_CONTINUE;
1923 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1925 switch (ctxt->modrm_reg) {
1927 emulate_2op_SrcB(ctxt, "rol");
1930 emulate_2op_SrcB(ctxt, "ror");
1933 emulate_2op_SrcB(ctxt, "rcl");
1936 emulate_2op_SrcB(ctxt, "rcr");
1938 case 4: /* sal/shl */
1939 case 6: /* sal/shl */
1940 emulate_2op_SrcB(ctxt, "sal");
1943 emulate_2op_SrcB(ctxt, "shr");
1946 emulate_2op_SrcB(ctxt, "sar");
1949 return X86EMUL_CONTINUE;
1952 static int em_not(struct x86_emulate_ctxt *ctxt)
1954 ctxt->dst.val = ~ctxt->dst.val;
1955 return X86EMUL_CONTINUE;
1958 static int em_neg(struct x86_emulate_ctxt *ctxt)
1960 emulate_1op(ctxt, "neg");
1961 return X86EMUL_CONTINUE;
1964 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
1968 emulate_1op_rax_rdx(ctxt, "mul", ex);
1969 return X86EMUL_CONTINUE;
1972 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
1976 emulate_1op_rax_rdx(ctxt, "imul", ex);
1977 return X86EMUL_CONTINUE;
1980 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
1984 emulate_1op_rax_rdx(ctxt, "div", de);
1986 return emulate_de(ctxt);
1987 return X86EMUL_CONTINUE;
1990 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
1994 emulate_1op_rax_rdx(ctxt, "idiv", de);
1996 return emulate_de(ctxt);
1997 return X86EMUL_CONTINUE;
2000 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2002 int rc = X86EMUL_CONTINUE;
2004 switch (ctxt->modrm_reg) {
2006 emulate_1op(ctxt, "inc");
2009 emulate_1op(ctxt, "dec");
2011 case 2: /* call near abs */ {
2013 old_eip = ctxt->_eip;
2014 ctxt->_eip = ctxt->src.val;
2015 ctxt->src.val = old_eip;
2019 case 4: /* jmp abs */
2020 ctxt->_eip = ctxt->src.val;
2022 case 5: /* jmp far */
2023 rc = em_jmp_far(ctxt);
2032 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2034 u64 old = ctxt->dst.orig_val64;
2036 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2037 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2038 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2039 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2040 ctxt->eflags &= ~EFLG_ZF;
2042 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2043 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2045 ctxt->eflags |= EFLG_ZF;
2047 return X86EMUL_CONTINUE;
2050 static int em_ret(struct x86_emulate_ctxt *ctxt)
2052 ctxt->dst.type = OP_REG;
2053 ctxt->dst.addr.reg = &ctxt->_eip;
2054 ctxt->dst.bytes = ctxt->op_bytes;
2055 return em_pop(ctxt);
2058 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2063 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2064 if (rc != X86EMUL_CONTINUE)
2066 if (ctxt->op_bytes == 4)
2067 ctxt->_eip = (u32)ctxt->_eip;
2068 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2069 if (rc != X86EMUL_CONTINUE)
2071 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2075 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2077 /* Save real source value, then compare EAX against destination. */
2078 ctxt->src.orig_val = ctxt->src.val;
2079 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2080 emulate_2op_SrcV(ctxt, "cmp");
2082 if (ctxt->eflags & EFLG_ZF) {
2083 /* Success: write back to memory. */
2084 ctxt->dst.val = ctxt->src.orig_val;
2086 /* Failure: write the value we saw to EAX. */
2087 ctxt->dst.type = OP_REG;
2088 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2090 return X86EMUL_CONTINUE;
2093 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2095 int seg = ctxt->src2.val;
2099 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2101 rc = load_segment_descriptor(ctxt, sel, seg);
2102 if (rc != X86EMUL_CONTINUE)
2105 ctxt->dst.val = ctxt->src.val;
2110 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2111 struct desc_struct *cs, struct desc_struct *ss)
2113 cs->l = 0; /* will be adjusted later */
2114 set_desc_base(cs, 0); /* flat segment */
2115 cs->g = 1; /* 4kb granularity */
2116 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2117 cs->type = 0x0b; /* Read, Execute, Accessed */
2119 cs->dpl = 0; /* will be adjusted later */
2124 set_desc_base(ss, 0); /* flat segment */
2125 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2126 ss->g = 1; /* 4kb granularity */
2128 ss->type = 0x03; /* Read/Write, Accessed */
2129 ss->d = 1; /* 32bit stack segment */
2136 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2138 u32 eax, ebx, ecx, edx;
2141 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2142 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2143 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2144 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2147 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2149 const struct x86_emulate_ops *ops = ctxt->ops;
2150 u32 eax, ebx, ecx, edx;
2153 * syscall should always be enabled in longmode - so only become
2154 * vendor specific (cpuid) if other modes are active...
2156 if (ctxt->mode == X86EMUL_MODE_PROT64)
2161 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2163 * Intel ("GenuineIntel")
2164 * remark: Intel CPUs only support "syscall" in 64bit
2165 * longmode. Also an 64bit guest with a
2166 * 32bit compat-app running will #UD !! While this
2167 * behaviour can be fixed (by emulating) into AMD
2168 * response - CPUs of AMD can't behave like Intel.
2170 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2171 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2172 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2175 /* AMD ("AuthenticAMD") */
2176 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2177 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2178 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2181 /* AMD ("AMDisbetter!") */
2182 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2183 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2184 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2187 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2191 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2193 const struct x86_emulate_ops *ops = ctxt->ops;
2194 struct desc_struct cs, ss;
2199 /* syscall is not available in real mode */
2200 if (ctxt->mode == X86EMUL_MODE_REAL ||
2201 ctxt->mode == X86EMUL_MODE_VM86)
2202 return emulate_ud(ctxt);
2204 if (!(em_syscall_is_enabled(ctxt)))
2205 return emulate_ud(ctxt);
2207 ops->get_msr(ctxt, MSR_EFER, &efer);
2208 setup_syscalls_segments(ctxt, &cs, &ss);
2210 if (!(efer & EFER_SCE))
2211 return emulate_ud(ctxt);
2213 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2215 cs_sel = (u16)(msr_data & 0xfffc);
2216 ss_sel = (u16)(msr_data + 8);
2218 if (efer & EFER_LMA) {
2222 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2223 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2225 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2226 if (efer & EFER_LMA) {
2227 #ifdef CONFIG_X86_64
2228 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2231 ctxt->mode == X86EMUL_MODE_PROT64 ?
2232 MSR_LSTAR : MSR_CSTAR, &msr_data);
2233 ctxt->_eip = msr_data;
2235 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2236 ctxt->eflags &= ~(msr_data | EFLG_RF);
2240 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2241 ctxt->_eip = (u32)msr_data;
2243 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2246 return X86EMUL_CONTINUE;
2249 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2251 const struct x86_emulate_ops *ops = ctxt->ops;
2252 struct desc_struct cs, ss;
2257 ops->get_msr(ctxt, MSR_EFER, &efer);
2258 /* inject #GP if in real mode */
2259 if (ctxt->mode == X86EMUL_MODE_REAL)
2260 return emulate_gp(ctxt, 0);
2263 * Not recognized on AMD in compat mode (but is recognized in legacy
2266 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2267 && !vendor_intel(ctxt))
2268 return emulate_ud(ctxt);
2270 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2271 * Therefore, we inject an #UD.
2273 if (ctxt->mode == X86EMUL_MODE_PROT64)
2274 return emulate_ud(ctxt);
2276 setup_syscalls_segments(ctxt, &cs, &ss);
2278 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2279 switch (ctxt->mode) {
2280 case X86EMUL_MODE_PROT32:
2281 if ((msr_data & 0xfffc) == 0x0)
2282 return emulate_gp(ctxt, 0);
2284 case X86EMUL_MODE_PROT64:
2285 if (msr_data == 0x0)
2286 return emulate_gp(ctxt, 0);
2292 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2293 cs_sel = (u16)msr_data;
2294 cs_sel &= ~SELECTOR_RPL_MASK;
2295 ss_sel = cs_sel + 8;
2296 ss_sel &= ~SELECTOR_RPL_MASK;
2297 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2302 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2303 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2305 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2306 ctxt->_eip = msr_data;
2308 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2309 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2311 return X86EMUL_CONTINUE;
2314 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2316 const struct x86_emulate_ops *ops = ctxt->ops;
2317 struct desc_struct cs, ss;
2320 u16 cs_sel = 0, ss_sel = 0;
2322 /* inject #GP if in real mode or Virtual 8086 mode */
2323 if (ctxt->mode == X86EMUL_MODE_REAL ||
2324 ctxt->mode == X86EMUL_MODE_VM86)
2325 return emulate_gp(ctxt, 0);
2327 setup_syscalls_segments(ctxt, &cs, &ss);
2329 if ((ctxt->rex_prefix & 0x8) != 0x0)
2330 usermode = X86EMUL_MODE_PROT64;
2332 usermode = X86EMUL_MODE_PROT32;
2336 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2338 case X86EMUL_MODE_PROT32:
2339 cs_sel = (u16)(msr_data + 16);
2340 if ((msr_data & 0xfffc) == 0x0)
2341 return emulate_gp(ctxt, 0);
2342 ss_sel = (u16)(msr_data + 24);
2344 case X86EMUL_MODE_PROT64:
2345 cs_sel = (u16)(msr_data + 32);
2346 if (msr_data == 0x0)
2347 return emulate_gp(ctxt, 0);
2348 ss_sel = cs_sel + 8;
2353 cs_sel |= SELECTOR_RPL_MASK;
2354 ss_sel |= SELECTOR_RPL_MASK;
2356 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2357 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2359 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2360 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2362 return X86EMUL_CONTINUE;
2365 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2368 if (ctxt->mode == X86EMUL_MODE_REAL)
2370 if (ctxt->mode == X86EMUL_MODE_VM86)
2372 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2373 return ctxt->ops->cpl(ctxt) > iopl;
2376 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2379 const struct x86_emulate_ops *ops = ctxt->ops;
2380 struct desc_struct tr_seg;
2383 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2384 unsigned mask = (1 << len) - 1;
2387 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2390 if (desc_limit_scaled(&tr_seg) < 103)
2392 base = get_desc_base(&tr_seg);
2393 #ifdef CONFIG_X86_64
2394 base |= ((u64)base3) << 32;
2396 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2397 if (r != X86EMUL_CONTINUE)
2399 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2401 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2402 if (r != X86EMUL_CONTINUE)
2404 if ((perm >> bit_idx) & mask)
2409 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2415 if (emulator_bad_iopl(ctxt))
2416 if (!emulator_io_port_access_allowed(ctxt, port, len))
2419 ctxt->perm_ok = true;
2424 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2425 struct tss_segment_16 *tss)
2427 tss->ip = ctxt->_eip;
2428 tss->flag = ctxt->eflags;
2429 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2430 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2431 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2432 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2433 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2434 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2435 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2436 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2438 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2439 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2440 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2441 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2442 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2445 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2446 struct tss_segment_16 *tss)
2450 ctxt->_eip = tss->ip;
2451 ctxt->eflags = tss->flag | 2;
2452 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2453 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2454 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2455 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2456 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2457 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2458 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2459 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2462 * SDM says that segment selectors are loaded before segment
2465 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2466 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2467 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2468 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2469 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2472 * Now load segment descriptors. If fault happens at this stage
2473 * it is handled in a context of new task
2475 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2476 if (ret != X86EMUL_CONTINUE)
2478 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2479 if (ret != X86EMUL_CONTINUE)
2481 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2482 if (ret != X86EMUL_CONTINUE)
2484 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2485 if (ret != X86EMUL_CONTINUE)
2487 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2488 if (ret != X86EMUL_CONTINUE)
2491 return X86EMUL_CONTINUE;
2494 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2495 u16 tss_selector, u16 old_tss_sel,
2496 ulong old_tss_base, struct desc_struct *new_desc)
2498 const struct x86_emulate_ops *ops = ctxt->ops;
2499 struct tss_segment_16 tss_seg;
2501 u32 new_tss_base = get_desc_base(new_desc);
2503 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2505 if (ret != X86EMUL_CONTINUE)
2506 /* FIXME: need to provide precise fault address */
2509 save_state_to_tss16(ctxt, &tss_seg);
2511 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2513 if (ret != X86EMUL_CONTINUE)
2514 /* FIXME: need to provide precise fault address */
2517 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2519 if (ret != X86EMUL_CONTINUE)
2520 /* FIXME: need to provide precise fault address */
2523 if (old_tss_sel != 0xffff) {
2524 tss_seg.prev_task_link = old_tss_sel;
2526 ret = ops->write_std(ctxt, new_tss_base,
2527 &tss_seg.prev_task_link,
2528 sizeof tss_seg.prev_task_link,
2530 if (ret != X86EMUL_CONTINUE)
2531 /* FIXME: need to provide precise fault address */
2535 return load_state_from_tss16(ctxt, &tss_seg);
2538 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2539 struct tss_segment_32 *tss)
2541 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2542 tss->eip = ctxt->_eip;
2543 tss->eflags = ctxt->eflags;
2544 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2545 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2546 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2547 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2548 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2549 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2550 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2551 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2553 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2554 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2555 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2556 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2557 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2558 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2559 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2562 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2563 struct tss_segment_32 *tss)
2567 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2568 return emulate_gp(ctxt, 0);
2569 ctxt->_eip = tss->eip;
2570 ctxt->eflags = tss->eflags | 2;
2572 /* General purpose registers */
2573 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2574 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2575 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2576 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2577 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2578 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2579 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2580 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2583 * SDM says that segment selectors are loaded before segment
2586 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2587 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2588 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2589 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2590 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2591 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2592 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2595 * If we're switching between Protected Mode and VM86, we need to make
2596 * sure to update the mode before loading the segment descriptors so
2597 * that the selectors are interpreted correctly.
2599 * Need to get rflags to the vcpu struct immediately because it
2600 * influences the CPL which is checked at least when loading the segment
2601 * descriptors and when pushing an error code to the new kernel stack.
2603 * TODO Introduce a separate ctxt->ops->set_cpl callback
2605 if (ctxt->eflags & X86_EFLAGS_VM)
2606 ctxt->mode = X86EMUL_MODE_VM86;
2608 ctxt->mode = X86EMUL_MODE_PROT32;
2610 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2613 * Now load segment descriptors. If fault happenes at this stage
2614 * it is handled in a context of new task
2616 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2617 if (ret != X86EMUL_CONTINUE)
2619 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2620 if (ret != X86EMUL_CONTINUE)
2622 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2626 if (ret != X86EMUL_CONTINUE)
2628 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2635 if (ret != X86EMUL_CONTINUE)
2638 return X86EMUL_CONTINUE;
2641 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2642 u16 tss_selector, u16 old_tss_sel,
2643 ulong old_tss_base, struct desc_struct *new_desc)
2645 const struct x86_emulate_ops *ops = ctxt->ops;
2646 struct tss_segment_32 tss_seg;
2648 u32 new_tss_base = get_desc_base(new_desc);
2650 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2652 if (ret != X86EMUL_CONTINUE)
2653 /* FIXME: need to provide precise fault address */
2656 save_state_to_tss32(ctxt, &tss_seg);
2658 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2660 if (ret != X86EMUL_CONTINUE)
2661 /* FIXME: need to provide precise fault address */
2664 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2666 if (ret != X86EMUL_CONTINUE)
2667 /* FIXME: need to provide precise fault address */
2670 if (old_tss_sel != 0xffff) {
2671 tss_seg.prev_task_link = old_tss_sel;
2673 ret = ops->write_std(ctxt, new_tss_base,
2674 &tss_seg.prev_task_link,
2675 sizeof tss_seg.prev_task_link,
2677 if (ret != X86EMUL_CONTINUE)
2678 /* FIXME: need to provide precise fault address */
2682 return load_state_from_tss32(ctxt, &tss_seg);
2685 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2686 u16 tss_selector, int idt_index, int reason,
2687 bool has_error_code, u32 error_code)
2689 const struct x86_emulate_ops *ops = ctxt->ops;
2690 struct desc_struct curr_tss_desc, next_tss_desc;
2692 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2693 ulong old_tss_base =
2694 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2698 /* FIXME: old_tss_base == ~0 ? */
2700 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2701 if (ret != X86EMUL_CONTINUE)
2703 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2704 if (ret != X86EMUL_CONTINUE)
2707 /* FIXME: check that next_tss_desc is tss */
2710 * Check privileges. The three cases are task switch caused by...
2712 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2713 * 2. Exception/IRQ/iret: No check is performed
2714 * 3. jmp/call to TSS: Check against DPL of the TSS
2716 if (reason == TASK_SWITCH_GATE) {
2717 if (idt_index != -1) {
2718 /* Software interrupts */
2719 struct desc_struct task_gate_desc;
2722 ret = read_interrupt_descriptor(ctxt, idt_index,
2724 if (ret != X86EMUL_CONTINUE)
2727 dpl = task_gate_desc.dpl;
2728 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2729 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2731 } else if (reason != TASK_SWITCH_IRET) {
2732 int dpl = next_tss_desc.dpl;
2733 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2734 return emulate_gp(ctxt, tss_selector);
2738 desc_limit = desc_limit_scaled(&next_tss_desc);
2739 if (!next_tss_desc.p ||
2740 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2741 desc_limit < 0x2b)) {
2742 emulate_ts(ctxt, tss_selector & 0xfffc);
2743 return X86EMUL_PROPAGATE_FAULT;
2746 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2747 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2748 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2751 if (reason == TASK_SWITCH_IRET)
2752 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2754 /* set back link to prev task only if NT bit is set in eflags
2755 note that old_tss_sel is not used after this point */
2756 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2757 old_tss_sel = 0xffff;
2759 if (next_tss_desc.type & 8)
2760 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2761 old_tss_base, &next_tss_desc);
2763 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2764 old_tss_base, &next_tss_desc);
2765 if (ret != X86EMUL_CONTINUE)
2768 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2769 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2771 if (reason != TASK_SWITCH_IRET) {
2772 next_tss_desc.type |= (1 << 1); /* set busy flag */
2773 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2776 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2777 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2779 if (has_error_code) {
2780 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2781 ctxt->lock_prefix = 0;
2782 ctxt->src.val = (unsigned long) error_code;
2783 ret = em_push(ctxt);
2789 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2790 u16 tss_selector, int idt_index, int reason,
2791 bool has_error_code, u32 error_code)
2795 invalidate_registers(ctxt);
2796 ctxt->_eip = ctxt->eip;
2797 ctxt->dst.type = OP_NONE;
2799 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2800 has_error_code, error_code);
2802 if (rc == X86EMUL_CONTINUE) {
2803 ctxt->eip = ctxt->_eip;
2804 writeback_registers(ctxt);
2807 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2810 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2813 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2815 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2816 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2819 static int em_das(struct x86_emulate_ctxt *ctxt)
2822 bool af, cf, old_cf;
2824 cf = ctxt->eflags & X86_EFLAGS_CF;
2830 af = ctxt->eflags & X86_EFLAGS_AF;
2831 if ((al & 0x0f) > 9 || af) {
2833 cf = old_cf | (al >= 250);
2838 if (old_al > 0x99 || old_cf) {
2844 /* Set PF, ZF, SF */
2845 ctxt->src.type = OP_IMM;
2847 ctxt->src.bytes = 1;
2848 emulate_2op_SrcV(ctxt, "or");
2849 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2851 ctxt->eflags |= X86_EFLAGS_CF;
2853 ctxt->eflags |= X86_EFLAGS_AF;
2854 return X86EMUL_CONTINUE;
2857 static int em_aad(struct x86_emulate_ctxt *ctxt)
2859 u8 al = ctxt->dst.val & 0xff;
2860 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2862 al = (al + (ah * ctxt->src.val)) & 0xff;
2864 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2866 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
2869 ctxt->eflags |= X86_EFLAGS_ZF;
2871 ctxt->eflags |= X86_EFLAGS_PF;
2873 ctxt->eflags |= X86_EFLAGS_SF;
2875 return X86EMUL_CONTINUE;
2878 static int em_call(struct x86_emulate_ctxt *ctxt)
2880 long rel = ctxt->src.val;
2882 ctxt->src.val = (unsigned long)ctxt->_eip;
2884 return em_push(ctxt);
2887 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2893 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2894 old_eip = ctxt->_eip;
2896 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2897 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2898 return X86EMUL_CONTINUE;
2901 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2903 ctxt->src.val = old_cs;
2905 if (rc != X86EMUL_CONTINUE)
2908 ctxt->src.val = old_eip;
2909 return em_push(ctxt);
2912 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2916 ctxt->dst.type = OP_REG;
2917 ctxt->dst.addr.reg = &ctxt->_eip;
2918 ctxt->dst.bytes = ctxt->op_bytes;
2919 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2920 if (rc != X86EMUL_CONTINUE)
2922 rsp_increment(ctxt, ctxt->src.val);
2923 return X86EMUL_CONTINUE;
2926 static int em_add(struct x86_emulate_ctxt *ctxt)
2928 emulate_2op_SrcV(ctxt, "add");
2929 return X86EMUL_CONTINUE;
2932 static int em_or(struct x86_emulate_ctxt *ctxt)
2934 emulate_2op_SrcV(ctxt, "or");
2935 return X86EMUL_CONTINUE;
2938 static int em_adc(struct x86_emulate_ctxt *ctxt)
2940 emulate_2op_SrcV(ctxt, "adc");
2941 return X86EMUL_CONTINUE;
2944 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2946 emulate_2op_SrcV(ctxt, "sbb");
2947 return X86EMUL_CONTINUE;
2950 static int em_and(struct x86_emulate_ctxt *ctxt)
2952 emulate_2op_SrcV(ctxt, "and");
2953 return X86EMUL_CONTINUE;
2956 static int em_sub(struct x86_emulate_ctxt *ctxt)
2958 emulate_2op_SrcV(ctxt, "sub");
2959 return X86EMUL_CONTINUE;
2962 static int em_xor(struct x86_emulate_ctxt *ctxt)
2964 emulate_2op_SrcV(ctxt, "xor");
2965 return X86EMUL_CONTINUE;
2968 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2970 emulate_2op_SrcV(ctxt, "cmp");
2971 /* Disable writeback. */
2972 ctxt->dst.type = OP_NONE;
2973 return X86EMUL_CONTINUE;
2976 static int em_test(struct x86_emulate_ctxt *ctxt)
2978 emulate_2op_SrcV(ctxt, "test");
2979 /* Disable writeback. */
2980 ctxt->dst.type = OP_NONE;
2981 return X86EMUL_CONTINUE;
2984 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2986 /* Write back the register source. */
2987 ctxt->src.val = ctxt->dst.val;
2988 write_register_operand(&ctxt->src);
2990 /* Write back the memory destination with implicit LOCK prefix. */
2991 ctxt->dst.val = ctxt->src.orig_val;
2992 ctxt->lock_prefix = 1;
2993 return X86EMUL_CONTINUE;
2996 static int em_imul(struct x86_emulate_ctxt *ctxt)
2998 emulate_2op_SrcV_nobyte(ctxt, "imul");
2999 return X86EMUL_CONTINUE;
3002 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3004 ctxt->dst.val = ctxt->src2.val;
3005 return em_imul(ctxt);
3008 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3010 ctxt->dst.type = OP_REG;
3011 ctxt->dst.bytes = ctxt->src.bytes;
3012 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3013 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3015 return X86EMUL_CONTINUE;
3018 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3022 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3023 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3024 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3025 return X86EMUL_CONTINUE;
3028 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3032 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3033 return emulate_gp(ctxt, 0);
3034 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3035 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3036 return X86EMUL_CONTINUE;
3039 static int em_mov(struct x86_emulate_ctxt *ctxt)
3041 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3042 return X86EMUL_CONTINUE;
3045 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3047 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3048 return emulate_gp(ctxt, 0);
3050 /* Disable writeback. */
3051 ctxt->dst.type = OP_NONE;
3052 return X86EMUL_CONTINUE;
3055 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3059 if (ctxt->mode == X86EMUL_MODE_PROT64)
3060 val = ctxt->src.val & ~0ULL;
3062 val = ctxt->src.val & ~0U;
3064 /* #UD condition is already handled. */
3065 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3066 return emulate_gp(ctxt, 0);
3068 /* Disable writeback. */
3069 ctxt->dst.type = OP_NONE;
3070 return X86EMUL_CONTINUE;
3073 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3077 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3078 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3079 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3080 return emulate_gp(ctxt, 0);
3082 return X86EMUL_CONTINUE;
3085 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3089 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3090 return emulate_gp(ctxt, 0);
3092 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3093 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3094 return X86EMUL_CONTINUE;
3097 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3099 if (ctxt->modrm_reg > VCPU_SREG_GS)
3100 return emulate_ud(ctxt);
3102 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3103 return X86EMUL_CONTINUE;
3106 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3108 u16 sel = ctxt->src.val;
3110 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3111 return emulate_ud(ctxt);
3113 if (ctxt->modrm_reg == VCPU_SREG_SS)
3114 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3116 /* Disable writeback. */
3117 ctxt->dst.type = OP_NONE;
3118 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3121 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3123 u16 sel = ctxt->src.val;
3125 /* Disable writeback. */
3126 ctxt->dst.type = OP_NONE;
3127 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3130 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3132 u16 sel = ctxt->src.val;
3134 /* Disable writeback. */
3135 ctxt->dst.type = OP_NONE;
3136 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3139 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3144 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3145 if (rc == X86EMUL_CONTINUE)
3146 ctxt->ops->invlpg(ctxt, linear);
3147 /* Disable writeback. */
3148 ctxt->dst.type = OP_NONE;
3149 return X86EMUL_CONTINUE;
3152 static int em_clts(struct x86_emulate_ctxt *ctxt)
3156 cr0 = ctxt->ops->get_cr(ctxt, 0);
3158 ctxt->ops->set_cr(ctxt, 0, cr0);
3159 return X86EMUL_CONTINUE;
3162 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3166 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3167 return X86EMUL_UNHANDLEABLE;
3169 rc = ctxt->ops->fix_hypercall(ctxt);
3170 if (rc != X86EMUL_CONTINUE)
3173 /* Let the processor re-execute the fixed hypercall */
3174 ctxt->_eip = ctxt->eip;
3175 /* Disable writeback. */
3176 ctxt->dst.type = OP_NONE;
3177 return X86EMUL_CONTINUE;
3180 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3181 void (*get)(struct x86_emulate_ctxt *ctxt,
3182 struct desc_ptr *ptr))
3184 struct desc_ptr desc_ptr;
3186 if (ctxt->mode == X86EMUL_MODE_PROT64)
3188 get(ctxt, &desc_ptr);
3189 if (ctxt->op_bytes == 2) {
3191 desc_ptr.address &= 0x00ffffff;
3193 /* Disable writeback. */
3194 ctxt->dst.type = OP_NONE;
3195 return segmented_write(ctxt, ctxt->dst.addr.mem,
3196 &desc_ptr, 2 + ctxt->op_bytes);
3199 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3201 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3204 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3206 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3209 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3211 struct desc_ptr desc_ptr;
3214 if (ctxt->mode == X86EMUL_MODE_PROT64)
3216 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3217 &desc_ptr.size, &desc_ptr.address,
3219 if (rc != X86EMUL_CONTINUE)
3221 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3222 /* Disable writeback. */
3223 ctxt->dst.type = OP_NONE;
3224 return X86EMUL_CONTINUE;
3227 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3231 rc = ctxt->ops->fix_hypercall(ctxt);
3233 /* Disable writeback. */
3234 ctxt->dst.type = OP_NONE;
3238 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3240 struct desc_ptr desc_ptr;
3243 if (ctxt->mode == X86EMUL_MODE_PROT64)
3245 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3246 &desc_ptr.size, &desc_ptr.address,
3248 if (rc != X86EMUL_CONTINUE)
3250 ctxt->ops->set_idt(ctxt, &desc_ptr);
3251 /* Disable writeback. */
3252 ctxt->dst.type = OP_NONE;
3253 return X86EMUL_CONTINUE;
3256 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3258 ctxt->dst.bytes = 2;
3259 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3260 return X86EMUL_CONTINUE;
3263 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3265 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3266 | (ctxt->src.val & 0x0f));
3267 ctxt->dst.type = OP_NONE;
3268 return X86EMUL_CONTINUE;
3271 static int em_loop(struct x86_emulate_ctxt *ctxt)
3273 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3274 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3275 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3276 jmp_rel(ctxt, ctxt->src.val);
3278 return X86EMUL_CONTINUE;
3281 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3283 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3284 jmp_rel(ctxt, ctxt->src.val);
3286 return X86EMUL_CONTINUE;
3289 static int em_in(struct x86_emulate_ctxt *ctxt)
3291 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3293 return X86EMUL_IO_NEEDED;
3295 return X86EMUL_CONTINUE;
3298 static int em_out(struct x86_emulate_ctxt *ctxt)
3300 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3302 /* Disable writeback. */
3303 ctxt->dst.type = OP_NONE;
3304 return X86EMUL_CONTINUE;
3307 static int em_cli(struct x86_emulate_ctxt *ctxt)
3309 if (emulator_bad_iopl(ctxt))
3310 return emulate_gp(ctxt, 0);
3312 ctxt->eflags &= ~X86_EFLAGS_IF;
3313 return X86EMUL_CONTINUE;
3316 static int em_sti(struct x86_emulate_ctxt *ctxt)
3318 if (emulator_bad_iopl(ctxt))
3319 return emulate_gp(ctxt, 0);
3321 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3322 ctxt->eflags |= X86_EFLAGS_IF;
3323 return X86EMUL_CONTINUE;
3326 static int em_bt(struct x86_emulate_ctxt *ctxt)
3328 /* Disable writeback. */
3329 ctxt->dst.type = OP_NONE;
3330 /* only subword offset */
3331 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3333 emulate_2op_SrcV_nobyte(ctxt, "bt");
3334 return X86EMUL_CONTINUE;
3337 static int em_bts(struct x86_emulate_ctxt *ctxt)
3339 emulate_2op_SrcV_nobyte(ctxt, "bts");
3340 return X86EMUL_CONTINUE;
3343 static int em_btr(struct x86_emulate_ctxt *ctxt)
3345 emulate_2op_SrcV_nobyte(ctxt, "btr");
3346 return X86EMUL_CONTINUE;
3349 static int em_btc(struct x86_emulate_ctxt *ctxt)
3351 emulate_2op_SrcV_nobyte(ctxt, "btc");
3352 return X86EMUL_CONTINUE;
3355 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3357 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3358 return X86EMUL_CONTINUE;
3361 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3363 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3364 return X86EMUL_CONTINUE;
3367 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3369 u32 eax, ebx, ecx, edx;
3371 eax = reg_read(ctxt, VCPU_REGS_RAX);
3372 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3373 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3374 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3375 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3376 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3377 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3378 return X86EMUL_CONTINUE;
3381 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3383 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3384 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3385 return X86EMUL_CONTINUE;
3388 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3390 switch (ctxt->op_bytes) {
3391 #ifdef CONFIG_X86_64
3393 asm("bswap %0" : "+r"(ctxt->dst.val));
3397 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3400 return X86EMUL_CONTINUE;
3403 static bool valid_cr(int nr)
3415 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3417 if (!valid_cr(ctxt->modrm_reg))
3418 return emulate_ud(ctxt);
3420 return X86EMUL_CONTINUE;
3423 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3425 u64 new_val = ctxt->src.val64;
3426 int cr = ctxt->modrm_reg;
3429 static u64 cr_reserved_bits[] = {
3430 0xffffffff00000000ULL,
3431 0, 0, 0, /* CR3 checked later */
3438 return emulate_ud(ctxt);
3440 if (new_val & cr_reserved_bits[cr])
3441 return emulate_gp(ctxt, 0);
3446 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3447 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3448 return emulate_gp(ctxt, 0);
3450 cr4 = ctxt->ops->get_cr(ctxt, 4);
3451 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3453 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3454 !(cr4 & X86_CR4_PAE))
3455 return emulate_gp(ctxt, 0);
3462 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3463 if (efer & EFER_LMA)
3464 rsvd = CR3_L_MODE_RESERVED_BITS;
3465 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3466 rsvd = CR3_PAE_RESERVED_BITS;
3467 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3468 rsvd = CR3_NONPAE_RESERVED_BITS;
3471 return emulate_gp(ctxt, 0);
3476 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3478 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3479 return emulate_gp(ctxt, 0);
3485 return X86EMUL_CONTINUE;
3488 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3492 ctxt->ops->get_dr(ctxt, 7, &dr7);
3494 /* Check if DR7.Global_Enable is set */
3495 return dr7 & (1 << 13);
3498 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3500 int dr = ctxt->modrm_reg;
3504 return emulate_ud(ctxt);
3506 cr4 = ctxt->ops->get_cr(ctxt, 4);
3507 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3508 return emulate_ud(ctxt);
3510 if (check_dr7_gd(ctxt))
3511 return emulate_db(ctxt);
3513 return X86EMUL_CONTINUE;
3516 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3518 u64 new_val = ctxt->src.val64;
3519 int dr = ctxt->modrm_reg;
3521 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3522 return emulate_gp(ctxt, 0);
3524 return check_dr_read(ctxt);
3527 static int check_svme(struct x86_emulate_ctxt *ctxt)
3531 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3533 if (!(efer & EFER_SVME))
3534 return emulate_ud(ctxt);
3536 return X86EMUL_CONTINUE;
3539 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3541 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3543 /* Valid physical address? */
3544 if (rax & 0xffff000000000000ULL)
3545 return emulate_gp(ctxt, 0);
3547 return check_svme(ctxt);
3550 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3552 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3554 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3555 return emulate_ud(ctxt);
3557 return X86EMUL_CONTINUE;
3560 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3562 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3563 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3565 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3567 return emulate_gp(ctxt, 0);
3569 return X86EMUL_CONTINUE;
3572 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3574 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3575 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3576 return emulate_gp(ctxt, 0);
3578 return X86EMUL_CONTINUE;
3581 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3583 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3584 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3585 return emulate_gp(ctxt, 0);
3587 return X86EMUL_CONTINUE;
3590 #define D(_y) { .flags = (_y) }
3591 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3592 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3593 .check_perm = (_p) }
3595 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3596 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3597 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3598 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3599 #define II(_f, _e, _i) \
3600 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3601 #define IIP(_f, _e, _i, _p) \
3602 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3603 .check_perm = (_p) }
3604 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3606 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3607 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3608 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3609 #define I2bvIP(_f, _e, _i, _p) \
3610 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3612 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3613 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3614 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3616 static const struct opcode group7_rm1[] = {
3617 DI(SrcNone | Priv, monitor),
3618 DI(SrcNone | Priv, mwait),
3622 static const struct opcode group7_rm3[] = {
3623 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3624 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3625 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3626 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3627 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3628 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3629 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3630 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3633 static const struct opcode group7_rm7[] = {
3635 DIP(SrcNone, rdtscp, check_rdtsc),
3639 static const struct opcode group1[] = {
3641 I(Lock | PageTable, em_or),
3644 I(Lock | PageTable, em_and),
3650 static const struct opcode group1A[] = {
3651 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3654 static const struct opcode group3[] = {
3655 I(DstMem | SrcImm, em_test),
3656 I(DstMem | SrcImm, em_test),
3657 I(DstMem | SrcNone | Lock, em_not),
3658 I(DstMem | SrcNone | Lock, em_neg),
3659 I(SrcMem, em_mul_ex),
3660 I(SrcMem, em_imul_ex),
3661 I(SrcMem, em_div_ex),
3662 I(SrcMem, em_idiv_ex),
3665 static const struct opcode group4[] = {
3666 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3667 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3671 static const struct opcode group5[] = {
3672 I(DstMem | SrcNone | Lock, em_grp45),
3673 I(DstMem | SrcNone | Lock, em_grp45),
3674 I(SrcMem | Stack, em_grp45),
3675 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3676 I(SrcMem | Stack, em_grp45),
3677 I(SrcMemFAddr | ImplicitOps, em_grp45),
3678 I(SrcMem | Stack, em_grp45), N,
3681 static const struct opcode group6[] = {
3684 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3685 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3689 static const struct group_dual group7 = { {
3690 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3691 II(Mov | DstMem | Priv, em_sidt, sidt),
3692 II(SrcMem | Priv, em_lgdt, lgdt),
3693 II(SrcMem | Priv, em_lidt, lidt),
3694 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3695 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3696 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3698 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3700 N, EXT(0, group7_rm3),
3701 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3702 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3706 static const struct opcode group8[] = {
3708 I(DstMem | SrcImmByte, em_bt),
3709 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3710 I(DstMem | SrcImmByte | Lock, em_btr),
3711 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3714 static const struct group_dual group9 = { {
3715 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3717 N, N, N, N, N, N, N, N,
3720 static const struct opcode group11[] = {
3721 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3725 static const struct gprefix pfx_0f_6f_0f_7f = {
3726 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3729 static const struct gprefix pfx_vmovntpx = {
3730 I(0, em_mov), N, N, N,
3733 static const struct opcode opcode_table[256] = {
3735 I6ALU(Lock, em_add),
3736 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3737 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3739 I6ALU(Lock | PageTable, em_or),
3740 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3743 I6ALU(Lock, em_adc),
3744 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3745 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3747 I6ALU(Lock, em_sbb),
3748 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3749 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3751 I6ALU(Lock | PageTable, em_and), N, N,
3753 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3755 I6ALU(Lock, em_xor), N, N,
3757 I6ALU(0, em_cmp), N, N,
3761 X8(I(SrcReg | Stack, em_push)),
3763 X8(I(DstReg | Stack, em_pop)),
3765 I(ImplicitOps | Stack | No64, em_pusha),
3766 I(ImplicitOps | Stack | No64, em_popa),
3767 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3770 I(SrcImm | Mov | Stack, em_push),
3771 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3772 I(SrcImmByte | Mov | Stack, em_push),
3773 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3774 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3775 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3779 G(ByteOp | DstMem | SrcImm, group1),
3780 G(DstMem | SrcImm, group1),
3781 G(ByteOp | DstMem | SrcImm | No64, group1),
3782 G(DstMem | SrcImmByte, group1),
3783 I2bv(DstMem | SrcReg | ModRM, em_test),
3784 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3786 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3787 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3788 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3789 D(ModRM | SrcMem | NoAccess | DstReg),
3790 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3793 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3795 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3796 I(SrcImmFAddr | No64, em_call_far), N,
3797 II(ImplicitOps | Stack, em_pushf, pushf),
3798 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3800 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3801 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3802 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3803 I2bv(SrcSI | DstDI | String, em_cmp),
3805 I2bv(DstAcc | SrcImm, em_test),
3806 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3807 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3808 I2bv(SrcAcc | DstDI | String, em_cmp),
3810 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3812 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3814 D2bv(DstMem | SrcImmByte | ModRM),
3815 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3816 I(ImplicitOps | Stack, em_ret),
3817 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3818 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3819 G(ByteOp, group11), G(0, group11),
3821 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3822 N, I(ImplicitOps | Stack, em_ret_far),
3823 D(ImplicitOps), DI(SrcImmByte, intn),
3824 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3826 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3827 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3829 N, N, N, N, N, N, N, N,
3831 X3(I(SrcImmByte, em_loop)),
3832 I(SrcImmByte, em_jcxz),
3833 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3834 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3836 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3837 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3838 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3839 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3841 N, DI(ImplicitOps, icebp), N, N,
3842 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3843 G(ByteOp, group3), G(0, group3),
3845 D(ImplicitOps), D(ImplicitOps),
3846 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3847 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3850 static const struct opcode twobyte_table[256] = {
3852 G(0, group6), GD(0, &group7), N, N,
3853 N, I(ImplicitOps | VendorSpecific, em_syscall),
3854 II(ImplicitOps | Priv, em_clts, clts), N,
3855 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3856 N, D(ImplicitOps | ModRM), N, N,
3858 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3860 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3861 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3862 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3863 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3865 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3868 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3869 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3870 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3871 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3872 I(ImplicitOps | VendorSpecific, em_sysenter),
3873 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3875 N, N, N, N, N, N, N, N,
3877 X16(D(DstReg | SrcMem | ModRM | Mov)),
3879 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3884 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3889 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3893 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3895 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3896 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
3897 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3898 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3900 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3901 DI(ImplicitOps, rsm),
3902 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3903 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3904 D(DstMem | SrcReg | Src2CL | ModRM),
3905 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3907 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3908 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3909 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3910 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3911 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3912 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3916 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3917 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
3918 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3920 D2bv(DstMem | SrcReg | ModRM | Lock),
3921 N, D(DstMem | SrcReg | ModRM | Mov),
3922 N, N, N, GD(0, &group9),
3924 X8(I(DstReg, em_bswap)),
3926 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3928 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3930 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3947 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3951 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3957 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3958 unsigned size, bool sign_extension)
3960 int rc = X86EMUL_CONTINUE;
3964 op->addr.mem.ea = ctxt->_eip;
3965 /* NB. Immediates are sign-extended as necessary. */
3966 switch (op->bytes) {
3968 op->val = insn_fetch(s8, ctxt);
3971 op->val = insn_fetch(s16, ctxt);
3974 op->val = insn_fetch(s32, ctxt);
3977 op->val = insn_fetch(s64, ctxt);
3980 if (!sign_extension) {
3981 switch (op->bytes) {
3989 op->val &= 0xffffffff;
3997 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4000 int rc = X86EMUL_CONTINUE;
4004 decode_register_operand(ctxt, op);
4007 rc = decode_imm(ctxt, op, 1, false);
4010 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4014 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4015 fetch_bit_operand(ctxt);
4016 op->orig_val = op->val;
4019 ctxt->memop.bytes = 8;
4023 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4024 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4025 fetch_register_operand(op);
4026 op->orig_val = op->val;
4030 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4032 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4033 op->addr.mem.seg = VCPU_SREG_ES;
4040 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4041 fetch_register_operand(op);
4045 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4048 rc = decode_imm(ctxt, op, 1, true);
4055 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4058 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4061 ctxt->memop.bytes = 1;
4064 ctxt->memop.bytes = 2;
4067 ctxt->memop.bytes = 4;
4070 rc = decode_imm(ctxt, op, 2, false);
4073 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4077 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4079 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4080 op->addr.mem.seg = seg_override(ctxt);
4086 op->addr.mem.ea = ctxt->_eip;
4087 op->bytes = ctxt->op_bytes + 2;
4088 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4091 ctxt->memop.bytes = ctxt->op_bytes + 2;
4094 op->val = VCPU_SREG_ES;
4097 op->val = VCPU_SREG_CS;
4100 op->val = VCPU_SREG_SS;
4103 op->val = VCPU_SREG_DS;
4106 op->val = VCPU_SREG_FS;
4109 op->val = VCPU_SREG_GS;
4112 /* Special instructions do their own operand decoding. */
4114 op->type = OP_NONE; /* Disable writeback. */
4122 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4124 int rc = X86EMUL_CONTINUE;
4125 int mode = ctxt->mode;
4126 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4127 bool op_prefix = false;
4128 struct opcode opcode;
4130 ctxt->memop.type = OP_NONE;
4131 ctxt->memopp = NULL;
4132 ctxt->_eip = ctxt->eip;
4133 ctxt->fetch.start = ctxt->_eip;
4134 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4136 memcpy(ctxt->fetch.data, insn, insn_len);
4139 case X86EMUL_MODE_REAL:
4140 case X86EMUL_MODE_VM86:
4141 case X86EMUL_MODE_PROT16:
4142 def_op_bytes = def_ad_bytes = 2;
4144 case X86EMUL_MODE_PROT32:
4145 def_op_bytes = def_ad_bytes = 4;
4147 #ifdef CONFIG_X86_64
4148 case X86EMUL_MODE_PROT64:
4154 return EMULATION_FAILED;
4157 ctxt->op_bytes = def_op_bytes;
4158 ctxt->ad_bytes = def_ad_bytes;
4160 /* Legacy prefixes. */
4162 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4163 case 0x66: /* operand-size override */
4165 /* switch between 2/4 bytes */
4166 ctxt->op_bytes = def_op_bytes ^ 6;
4168 case 0x67: /* address-size override */
4169 if (mode == X86EMUL_MODE_PROT64)
4170 /* switch between 4/8 bytes */
4171 ctxt->ad_bytes = def_ad_bytes ^ 12;
4173 /* switch between 2/4 bytes */
4174 ctxt->ad_bytes = def_ad_bytes ^ 6;
4176 case 0x26: /* ES override */
4177 case 0x2e: /* CS override */
4178 case 0x36: /* SS override */
4179 case 0x3e: /* DS override */
4180 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4182 case 0x64: /* FS override */
4183 case 0x65: /* GS override */
4184 set_seg_override(ctxt, ctxt->b & 7);
4186 case 0x40 ... 0x4f: /* REX */
4187 if (mode != X86EMUL_MODE_PROT64)
4189 ctxt->rex_prefix = ctxt->b;
4191 case 0xf0: /* LOCK */
4192 ctxt->lock_prefix = 1;
4194 case 0xf2: /* REPNE/REPNZ */
4195 case 0xf3: /* REP/REPE/REPZ */
4196 ctxt->rep_prefix = ctxt->b;
4202 /* Any legacy prefix after a REX prefix nullifies its effect. */
4204 ctxt->rex_prefix = 0;
4210 if (ctxt->rex_prefix & 8)
4211 ctxt->op_bytes = 8; /* REX.W */
4213 /* Opcode byte(s). */
4214 opcode = opcode_table[ctxt->b];
4215 /* Two-byte opcode? */
4216 if (ctxt->b == 0x0f) {
4218 ctxt->b = insn_fetch(u8, ctxt);
4219 opcode = twobyte_table[ctxt->b];
4221 ctxt->d = opcode.flags;
4223 if (ctxt->d & ModRM)
4224 ctxt->modrm = insn_fetch(u8, ctxt);
4226 while (ctxt->d & GroupMask) {
4227 switch (ctxt->d & GroupMask) {
4229 goffset = (ctxt->modrm >> 3) & 7;
4230 opcode = opcode.u.group[goffset];
4233 goffset = (ctxt->modrm >> 3) & 7;
4234 if ((ctxt->modrm >> 6) == 3)
4235 opcode = opcode.u.gdual->mod3[goffset];
4237 opcode = opcode.u.gdual->mod012[goffset];
4240 goffset = ctxt->modrm & 7;
4241 opcode = opcode.u.group[goffset];
4244 if (ctxt->rep_prefix && op_prefix)
4245 return EMULATION_FAILED;
4246 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4247 switch (simd_prefix) {
4248 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4249 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4250 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4251 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4255 return EMULATION_FAILED;
4258 ctxt->d &= ~(u64)GroupMask;
4259 ctxt->d |= opcode.flags;
4262 ctxt->execute = opcode.u.execute;
4263 ctxt->check_perm = opcode.check_perm;
4264 ctxt->intercept = opcode.intercept;
4267 if (ctxt->d == 0 || (ctxt->d & Undefined))
4268 return EMULATION_FAILED;
4270 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4271 return EMULATION_FAILED;
4273 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4276 if (ctxt->d & Op3264) {
4277 if (mode == X86EMUL_MODE_PROT64)
4284 ctxt->op_bytes = 16;
4285 else if (ctxt->d & Mmx)
4288 /* ModRM and SIB bytes. */
4289 if (ctxt->d & ModRM) {
4290 rc = decode_modrm(ctxt, &ctxt->memop);
4291 if (!ctxt->has_seg_override)
4292 set_seg_override(ctxt, ctxt->modrm_seg);
4293 } else if (ctxt->d & MemAbs)
4294 rc = decode_abs(ctxt, &ctxt->memop);
4295 if (rc != X86EMUL_CONTINUE)
4298 if (!ctxt->has_seg_override)
4299 set_seg_override(ctxt, VCPU_SREG_DS);
4301 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4303 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4304 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4307 * Decode and fetch the source operand: register, memory
4310 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4311 if (rc != X86EMUL_CONTINUE)
4315 * Decode and fetch the second source operand: register, memory
4318 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4319 if (rc != X86EMUL_CONTINUE)
4322 /* Decode and fetch the destination operand: register or memory. */
4323 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4326 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4327 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4329 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4332 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4334 return ctxt->d & PageTable;
4337 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4339 /* The second termination condition only applies for REPE
4340 * and REPNE. Test if the repeat string operation prefix is
4341 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4342 * corresponding termination condition according to:
4343 * - if REPE/REPZ and ZF = 0 then done
4344 * - if REPNE/REPNZ and ZF = 1 then done
4346 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4347 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4348 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4349 ((ctxt->eflags & EFLG_ZF) == 0))
4350 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4351 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4357 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4361 ctxt->ops->get_fpu(ctxt);
4362 asm volatile("1: fwait \n\t"
4364 ".pushsection .fixup,\"ax\" \n\t"
4366 "movb $1, %[fault] \n\t"
4369 _ASM_EXTABLE(1b, 3b)
4370 : [fault]"+qm"(fault));
4371 ctxt->ops->put_fpu(ctxt);
4373 if (unlikely(fault))
4374 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4376 return X86EMUL_CONTINUE;
4379 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4382 if (op->type == OP_MM)
4383 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4387 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4389 const struct x86_emulate_ops *ops = ctxt->ops;
4390 int rc = X86EMUL_CONTINUE;
4391 int saved_dst_type = ctxt->dst.type;
4393 ctxt->mem_read.pos = 0;
4395 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4396 rc = emulate_ud(ctxt);
4400 /* LOCK prefix is allowed only with some instructions */
4401 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4402 rc = emulate_ud(ctxt);
4406 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4407 rc = emulate_ud(ctxt);
4411 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4412 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4413 rc = emulate_ud(ctxt);
4417 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4418 rc = emulate_nm(ctxt);
4422 if (ctxt->d & Mmx) {
4423 rc = flush_pending_x87_faults(ctxt);
4424 if (rc != X86EMUL_CONTINUE)
4427 * Now that we know the fpu is exception safe, we can fetch
4430 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4431 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4432 if (!(ctxt->d & Mov))
4433 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4436 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4437 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4438 X86_ICPT_PRE_EXCEPT);
4439 if (rc != X86EMUL_CONTINUE)
4443 /* Privileged instruction can be executed only in CPL=0 */
4444 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4445 rc = emulate_gp(ctxt, 0);
4449 /* Instruction can only be executed in protected mode */
4450 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4451 rc = emulate_ud(ctxt);
4455 /* Do instruction specific permission checks */
4456 if (ctxt->check_perm) {
4457 rc = ctxt->check_perm(ctxt);
4458 if (rc != X86EMUL_CONTINUE)
4462 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4463 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4464 X86_ICPT_POST_EXCEPT);
4465 if (rc != X86EMUL_CONTINUE)
4469 if (ctxt->rep_prefix && (ctxt->d & String)) {
4470 /* All REP prefixes have the same first termination condition */
4471 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4472 ctxt->eip = ctxt->_eip;
4477 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4478 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4479 ctxt->src.valptr, ctxt->src.bytes);
4480 if (rc != X86EMUL_CONTINUE)
4482 ctxt->src.orig_val64 = ctxt->src.val64;
4485 if (ctxt->src2.type == OP_MEM) {
4486 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4487 &ctxt->src2.val, ctxt->src2.bytes);
4488 if (rc != X86EMUL_CONTINUE)
4492 if ((ctxt->d & DstMask) == ImplicitOps)
4496 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4497 /* optimisation - avoid slow emulated read if Mov */
4498 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4499 &ctxt->dst.val, ctxt->dst.bytes);
4500 if (rc != X86EMUL_CONTINUE)
4503 ctxt->dst.orig_val = ctxt->dst.val;
4507 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4508 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4509 X86_ICPT_POST_MEMACCESS);
4510 if (rc != X86EMUL_CONTINUE)
4514 if (ctxt->execute) {
4515 rc = ctxt->execute(ctxt);
4516 if (rc != X86EMUL_CONTINUE)
4525 case 0x40 ... 0x47: /* inc r16/r32 */
4526 emulate_1op(ctxt, "inc");
4528 case 0x48 ... 0x4f: /* dec r16/r32 */
4529 emulate_1op(ctxt, "dec");
4531 case 0x63: /* movsxd */
4532 if (ctxt->mode != X86EMUL_MODE_PROT64)
4533 goto cannot_emulate;
4534 ctxt->dst.val = (s32) ctxt->src.val;
4536 case 0x70 ... 0x7f: /* jcc (short) */
4537 if (test_cc(ctxt->b, ctxt->eflags))
4538 jmp_rel(ctxt, ctxt->src.val);
4540 case 0x8d: /* lea r16/r32, m */
4541 ctxt->dst.val = ctxt->src.addr.mem.ea;
4543 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4544 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4548 case 0x98: /* cbw/cwde/cdqe */
4549 switch (ctxt->op_bytes) {
4550 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4551 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4552 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4558 case 0xcc: /* int3 */
4559 rc = emulate_int(ctxt, 3);
4561 case 0xcd: /* int n */
4562 rc = emulate_int(ctxt, ctxt->src.val);
4564 case 0xce: /* into */
4565 if (ctxt->eflags & EFLG_OF)
4566 rc = emulate_int(ctxt, 4);
4568 case 0xd0 ... 0xd1: /* Grp2 */
4571 case 0xd2 ... 0xd3: /* Grp2 */
4572 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4575 case 0xe9: /* jmp rel */
4576 case 0xeb: /* jmp rel short */
4577 jmp_rel(ctxt, ctxt->src.val);
4578 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4580 case 0xf4: /* hlt */
4581 ctxt->ops->halt(ctxt);
4583 case 0xf5: /* cmc */
4584 /* complement carry flag from eflags reg */
4585 ctxt->eflags ^= EFLG_CF;
4587 case 0xf8: /* clc */
4588 ctxt->eflags &= ~EFLG_CF;
4590 case 0xf9: /* stc */
4591 ctxt->eflags |= EFLG_CF;
4593 case 0xfc: /* cld */
4594 ctxt->eflags &= ~EFLG_DF;
4596 case 0xfd: /* std */
4597 ctxt->eflags |= EFLG_DF;
4600 goto cannot_emulate;
4603 if (rc != X86EMUL_CONTINUE)
4607 rc = writeback(ctxt);
4608 if (rc != X86EMUL_CONTINUE)
4612 * restore dst type in case the decoding will be reused
4613 * (happens for string instruction )
4615 ctxt->dst.type = saved_dst_type;
4617 if ((ctxt->d & SrcMask) == SrcSI)
4618 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4620 if ((ctxt->d & DstMask) == DstDI)
4621 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4623 if (ctxt->rep_prefix && (ctxt->d & String)) {
4625 struct read_cache *r = &ctxt->io_read;
4626 if ((ctxt->d & SrcMask) == SrcSI)
4627 count = ctxt->src.count;
4629 count = ctxt->dst.count;
4630 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4633 if (!string_insn_completed(ctxt)) {
4635 * Re-enter guest when pio read ahead buffer is empty
4636 * or, if it is not used, after each 1024 iteration.
4638 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4639 (r->end == 0 || r->end != r->pos)) {
4641 * Reset read cache. Usually happens before
4642 * decode, but since instruction is restarted
4643 * we have to do it here.
4645 ctxt->mem_read.end = 0;
4646 writeback_registers(ctxt);
4647 return EMULATION_RESTART;
4649 goto done; /* skip rip writeback */
4653 ctxt->eip = ctxt->_eip;
4656 if (rc == X86EMUL_PROPAGATE_FAULT)
4657 ctxt->have_exception = true;
4658 if (rc == X86EMUL_INTERCEPTED)
4659 return EMULATION_INTERCEPTED;
4661 if (rc == X86EMUL_CONTINUE)
4662 writeback_registers(ctxt);
4664 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4668 case 0x09: /* wbinvd */
4669 (ctxt->ops->wbinvd)(ctxt);
4671 case 0x08: /* invd */
4672 case 0x0d: /* GrpP (prefetch) */
4673 case 0x18: /* Grp16 (prefetch/nop) */
4675 case 0x20: /* mov cr, reg */
4676 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4678 case 0x21: /* mov from dr to reg */
4679 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4681 case 0x40 ... 0x4f: /* cmov */
4682 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4683 if (!test_cc(ctxt->b, ctxt->eflags))
4684 ctxt->dst.type = OP_NONE; /* no writeback */
4686 case 0x80 ... 0x8f: /* jnz rel, etc*/
4687 if (test_cc(ctxt->b, ctxt->eflags))
4688 jmp_rel(ctxt, ctxt->src.val);
4690 case 0x90 ... 0x9f: /* setcc r/m8 */
4691 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4693 case 0xa4: /* shld imm8, r, r/m */
4694 case 0xa5: /* shld cl, r, r/m */
4695 emulate_2op_cl(ctxt, "shld");
4697 case 0xac: /* shrd imm8, r, r/m */
4698 case 0xad: /* shrd cl, r, r/m */
4699 emulate_2op_cl(ctxt, "shrd");
4701 case 0xae: /* clflush */
4703 case 0xb6 ... 0xb7: /* movzx */
4704 ctxt->dst.bytes = ctxt->op_bytes;
4705 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4706 : (u16) ctxt->src.val;
4708 case 0xbe ... 0xbf: /* movsx */
4709 ctxt->dst.bytes = ctxt->op_bytes;
4710 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4711 (s16) ctxt->src.val;
4713 case 0xc0 ... 0xc1: /* xadd */
4714 emulate_2op_SrcV(ctxt, "add");
4715 /* Write back the register source. */
4716 ctxt->src.val = ctxt->dst.orig_val;
4717 write_register_operand(&ctxt->src);
4719 case 0xc3: /* movnti */
4720 ctxt->dst.bytes = ctxt->op_bytes;
4721 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4722 (u64) ctxt->src.val;
4725 goto cannot_emulate;
4728 if (rc != X86EMUL_CONTINUE)
4734 return EMULATION_FAILED;
4737 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4739 invalidate_registers(ctxt);
4742 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4744 writeback_registers(ctxt);