1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define VendorSpecific (1<<22) /* Vendor specific instruction */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
165 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
167 #define X2(x...) x, x
168 #define X3(x...) X2(x), x
169 #define X4(x...) X2(x), X2(x)
170 #define X5(x...) X4(x), x
171 #define X6(x...) X4(x), X2(x)
172 #define X7(x...) X4(x), X3(x)
173 #define X8(x...) X4(x), X4(x)
174 #define X16(x...) X8(x), X8(x)
176 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177 #define FASTOP_SIZE 8
180 * fastop functions have a special calling convention:
185 * flags: rflags (in/out)
186 * ex: rsi (in:fastop pointer, out:zero if exception)
188 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
189 * different operand sizes can be reached by calculation, rather than a jump
190 * table (which would be bigger than the code).
192 * fastop functions are declared as taking a never-defined fastop parameter,
193 * so they can't be called from C directly.
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
229 /* EFLAGS bit definitions. */
230 #define EFLG_ID (1<<21)
231 #define EFLG_VIP (1<<20)
232 #define EFLG_VIF (1<<19)
233 #define EFLG_AC (1<<18)
234 #define EFLG_VM (1<<17)
235 #define EFLG_RF (1<<16)
236 #define EFLG_IOPL (3<<12)
237 #define EFLG_NT (1<<14)
238 #define EFLG_OF (1<<11)
239 #define EFLG_DF (1<<10)
240 #define EFLG_IF (1<<9)
241 #define EFLG_TF (1<<8)
242 #define EFLG_SF (1<<7)
243 #define EFLG_ZF (1<<6)
244 #define EFLG_AF (1<<4)
245 #define EFLG_PF (1<<2)
246 #define EFLG_CF (1<<0)
248 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249 #define EFLG_RESERVED_ONE_MASK 2
251 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
257 return ctxt->_regs[nr];
260 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
267 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
270 return reg_write(ctxt, nr);
273 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
281 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
288 * These EFLAGS bits are restored from saved value during emulation, and
289 * any changes are written back to the saved value after emulation.
291 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
301 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302 #define FOP_RET "ret \n\t"
304 #define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
314 #define FOPNOP() FOP_ALIGN FOP_RET
316 #define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
319 #define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
322 #define FASTOP1(op) \
327 ON64(FOP1E(op##q, rax)) \
330 /* 1-operand, using src2 (for MUL/DIV r/m) */
331 #define FASTOP1SRC2(op, name) \
336 ON64(FOP1E(op, rcx)) \
339 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
340 #define FASTOP1SRC2EX(op, name) \
345 ON64(FOP1EEX(op, rcx)) \
348 #define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
351 #define FASTOP2(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
359 /* 2 operand, word only */
360 #define FASTOP2W(op) \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
368 /* 2 operand, src is CL */
369 #define FASTOP2CL(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
377 #define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
380 /* 3-operand, word-only, src2=cl */
381 #define FASTOP3WCL(op) \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
389 /* Special case for SETcc - 1 instruction per cc */
390 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
392 asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
414 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
417 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
434 return ctxt->ops->intercept(ctxt, &info, stage);
437 static void assign_masked(ulong *dest, ulong src, ulong mask)
439 *dest = (*dest & ~mask) | (src & mask);
442 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
447 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
450 struct desc_struct ss;
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
458 static int stack_size(struct x86_emulate_ctxt *ctxt)
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
463 /* Access/update address held in a register, based on addressing mode. */
464 static inline unsigned long
465 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
467 if (ctxt->ad_bytes == sizeof(unsigned long))
470 return reg & ad_mask(ctxt);
473 static inline unsigned long
474 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
476 return address_mask(ctxt, reg);
479 static void masked_increment(ulong *reg, ulong mask, int inc)
481 assign_masked(reg, *reg + inc, mask);
485 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
489 if (ctxt->ad_bytes == sizeof(unsigned long))
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
496 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
501 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
503 register_address_increment(ctxt, &ctxt->_eip, rel);
506 static u32 desc_limit_scaled(struct desc_struct *desc)
508 u32 limit = get_desc_limit(desc);
510 return desc->g ? (limit << 12) | 0xfff : limit;
513 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
515 ctxt->has_seg_override = true;
516 ctxt->seg_override = seg;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
529 if (!ctxt->has_seg_override)
532 return ctxt->seg_override;
535 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
536 u32 error, bool valid)
538 ctxt->exception.vector = vec;
539 ctxt->exception.error_code = error;
540 ctxt->exception.error_code_valid = valid;
541 return X86EMUL_PROPAGATE_FAULT;
544 static int emulate_db(struct x86_emulate_ctxt *ctxt)
546 return emulate_exception(ctxt, DB_VECTOR, 0, false);
549 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
551 return emulate_exception(ctxt, GP_VECTOR, err, true);
554 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
556 return emulate_exception(ctxt, SS_VECTOR, err, true);
559 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, UD_VECTOR, 0, false);
564 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
566 return emulate_exception(ctxt, TS_VECTOR, err, true);
569 static int emulate_de(struct x86_emulate_ctxt *ctxt)
571 return emulate_exception(ctxt, DE_VECTOR, 0, false);
574 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
576 return emulate_exception(ctxt, NM_VECTOR, 0, false);
579 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
582 struct desc_struct desc;
584 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
588 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
593 struct desc_struct desc;
595 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
596 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
600 * x86 defines three classes of vector instructions: explicitly
601 * aligned, explicitly unaligned, and the rest, which change behaviour
602 * depending on whether they're AVX encoded or not.
604 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
605 * subject to the same check.
607 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
609 if (likely(size < 16))
612 if (ctxt->d & Aligned)
614 else if (ctxt->d & Unaligned)
616 else if (ctxt->d & Avx)
622 static int __linearize(struct x86_emulate_ctxt *ctxt,
623 struct segmented_address addr,
624 unsigned size, bool write, bool fetch,
627 struct desc_struct desc;
634 la = seg_base(ctxt, addr.seg) + addr.ea;
635 switch (ctxt->mode) {
636 case X86EMUL_MODE_PROT64:
637 if (((signed long)la << 16) >> 16 != la)
638 return emulate_gp(ctxt, 0);
641 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
645 /* code segment in protected mode or read-only data segment */
646 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
647 || !(desc.type & 2)) && write)
649 /* unreadable code segment */
650 if (!fetch && (desc.type & 8) && !(desc.type & 2))
652 lim = desc_limit_scaled(&desc);
653 if ((desc.type & 8) || !(desc.type & 4)) {
654 /* expand-up segment */
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
658 /* expand-down segment */
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
661 lim = desc.d ? 0xffffffff : 0xffff;
662 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
665 cpl = ctxt->ops->cpl(ctxt);
666 if (!(desc.type & 8)) {
670 } else if ((desc.type & 8) && !(desc.type & 4)) {
671 /* nonconforming code segment */
674 } else if ((desc.type & 8) && (desc.type & 4)) {
675 /* conforming code segment */
681 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
683 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
684 return emulate_gp(ctxt, 0);
686 return X86EMUL_CONTINUE;
688 if (addr.seg == VCPU_SREG_SS)
689 return emulate_ss(ctxt, sel);
691 return emulate_gp(ctxt, sel);
694 static int linearize(struct x86_emulate_ctxt *ctxt,
695 struct segmented_address addr,
696 unsigned size, bool write,
699 return __linearize(ctxt, addr, size, write, false, linear);
703 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
704 struct segmented_address addr,
711 rc = linearize(ctxt, addr, size, false, &linear);
712 if (rc != X86EMUL_CONTINUE)
714 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
718 * Fetch the next byte of the instruction being emulated which is pointed to
719 * by ctxt->_eip, then increment ctxt->_eip.
721 * Also prefetch the remaining bytes of the instruction without crossing page
722 * boundary if they are not in fetch_cache yet.
724 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
726 struct fetch_cache *fc = &ctxt->fetch;
730 if (ctxt->_eip == fc->end) {
731 unsigned long linear;
732 struct segmented_address addr = { .seg = VCPU_SREG_CS,
734 cur_size = fc->end - fc->start;
735 size = min(15UL - cur_size,
736 PAGE_SIZE - offset_in_page(ctxt->_eip));
737 rc = __linearize(ctxt, addr, size, false, true, &linear);
738 if (unlikely(rc != X86EMUL_CONTINUE))
740 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
741 size, &ctxt->exception);
742 if (unlikely(rc != X86EMUL_CONTINUE))
746 *dest = fc->data[ctxt->_eip - fc->start];
748 return X86EMUL_CONTINUE;
751 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752 void *dest, unsigned size)
756 /* x86 instructions are limited to 15 bytes. */
757 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758 return X86EMUL_UNHANDLEABLE;
760 rc = do_insn_fetch_byte(ctxt, dest++);
761 if (rc != X86EMUL_CONTINUE)
764 return X86EMUL_CONTINUE;
767 /* Fetch next part of the instruction being emulated. */
768 #define insn_fetch(_type, _ctxt) \
769 ({ unsigned long _x; \
770 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
771 if (rc != X86EMUL_CONTINUE) \
776 #define insn_fetch_arr(_arr, _size, _ctxt) \
777 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
778 if (rc != X86EMUL_CONTINUE) \
783 * Given the 'reg' portion of a ModRM byte, and a register block, return a
784 * pointer into the block that addresses the relevant register.
785 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
787 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
792 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
793 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
795 p = reg_rmw(ctxt, modrm_reg);
799 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
800 struct segmented_address addr,
801 u16 *size, unsigned long *address, int op_bytes)
808 rc = segmented_read_std(ctxt, addr, size, 2);
809 if (rc != X86EMUL_CONTINUE)
812 rc = segmented_read_std(ctxt, addr, address, op_bytes);
826 FASTOP1SRC2(mul, mul_ex);
827 FASTOP1SRC2(imul, imul_ex);
828 FASTOP1SRC2EX(div, div_ex);
829 FASTOP1SRC2EX(idiv, idiv_ex);
858 static u8 test_cc(unsigned int condition, unsigned long flags)
861 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
863 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
864 asm("push %[flags]; popf; call *%[fastop]"
865 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
869 static void fetch_register_operand(struct operand *op)
873 op->val = *(u8 *)op->addr.reg;
876 op->val = *(u16 *)op->addr.reg;
879 op->val = *(u32 *)op->addr.reg;
882 op->val = *(u64 *)op->addr.reg;
887 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
889 ctxt->ops->get_fpu(ctxt);
891 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
892 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
893 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
894 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
895 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
896 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
897 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
898 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
900 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
901 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
902 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
903 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
904 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
905 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
906 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
907 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
911 ctxt->ops->put_fpu(ctxt);
914 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
917 ctxt->ops->get_fpu(ctxt);
919 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
920 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
921 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
922 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
923 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
924 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
925 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
926 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
928 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
929 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
930 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
931 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
932 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
933 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
934 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
935 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
939 ctxt->ops->put_fpu(ctxt);
942 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
944 ctxt->ops->get_fpu(ctxt);
946 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
947 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
948 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
949 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
950 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
951 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
952 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
953 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
956 ctxt->ops->put_fpu(ctxt);
959 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
961 ctxt->ops->get_fpu(ctxt);
963 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
964 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
965 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
966 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
967 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
968 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
969 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
970 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
973 ctxt->ops->put_fpu(ctxt);
976 static int em_fninit(struct x86_emulate_ctxt *ctxt)
978 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
979 return emulate_nm(ctxt);
981 ctxt->ops->get_fpu(ctxt);
982 asm volatile("fninit");
983 ctxt->ops->put_fpu(ctxt);
984 return X86EMUL_CONTINUE;
987 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
991 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
992 return emulate_nm(ctxt);
994 ctxt->ops->get_fpu(ctxt);
995 asm volatile("fnstcw %0": "+m"(fcw));
996 ctxt->ops->put_fpu(ctxt);
998 /* force 2 byte destination */
1000 ctxt->dst.val = fcw;
1002 return X86EMUL_CONTINUE;
1005 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1009 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1010 return emulate_nm(ctxt);
1012 ctxt->ops->get_fpu(ctxt);
1013 asm volatile("fnstsw %0": "+m"(fsw));
1014 ctxt->ops->put_fpu(ctxt);
1016 /* force 2 byte destination */
1017 ctxt->dst.bytes = 2;
1018 ctxt->dst.val = fsw;
1020 return X86EMUL_CONTINUE;
1023 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1026 unsigned reg = ctxt->modrm_reg;
1027 int highbyte_regs = ctxt->rex_prefix == 0;
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1032 if (ctxt->d & Sse) {
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1039 if (ctxt->d & Mmx) {
1048 if (ctxt->d & ByteOp) {
1049 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1052 op->addr.reg = decode_register(ctxt, reg, 0);
1053 op->bytes = ctxt->op_bytes;
1055 fetch_register_operand(op);
1056 op->orig_val = op->val;
1059 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1061 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1062 ctxt->modrm_seg = VCPU_SREG_SS;
1065 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1069 int index_reg = 0, base_reg = 0, scale;
1070 int rc = X86EMUL_CONTINUE;
1073 if (ctxt->rex_prefix) {
1074 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1075 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1076 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1079 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1080 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1081 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1082 ctxt->modrm_seg = VCPU_SREG_DS;
1084 if (ctxt->modrm_mod == 3) {
1085 int highbyte_regs = ctxt->rex_prefix == 0;
1088 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1089 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1090 highbyte_regs && (ctxt->d & ByteOp));
1091 if (ctxt->d & Sse) {
1094 op->addr.xmm = ctxt->modrm_rm;
1095 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1098 if (ctxt->d & Mmx) {
1101 op->addr.xmm = ctxt->modrm_rm & 7;
1104 fetch_register_operand(op);
1110 if (ctxt->ad_bytes == 2) {
1111 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1112 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1113 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1114 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1116 /* 16-bit ModR/M decode. */
1117 switch (ctxt->modrm_mod) {
1119 if (ctxt->modrm_rm == 6)
1120 modrm_ea += insn_fetch(u16, ctxt);
1123 modrm_ea += insn_fetch(s8, ctxt);
1126 modrm_ea += insn_fetch(u16, ctxt);
1129 switch (ctxt->modrm_rm) {
1131 modrm_ea += bx + si;
1134 modrm_ea += bx + di;
1137 modrm_ea += bp + si;
1140 modrm_ea += bp + di;
1149 if (ctxt->modrm_mod != 0)
1156 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1157 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1158 ctxt->modrm_seg = VCPU_SREG_SS;
1159 modrm_ea = (u16)modrm_ea;
1161 /* 32/64-bit ModR/M decode. */
1162 if ((ctxt->modrm_rm & 7) == 4) {
1163 sib = insn_fetch(u8, ctxt);
1164 index_reg |= (sib >> 3) & 7;
1165 base_reg |= sib & 7;
1168 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1169 modrm_ea += insn_fetch(s32, ctxt);
1171 modrm_ea += reg_read(ctxt, base_reg);
1172 adjust_modrm_seg(ctxt, base_reg);
1175 modrm_ea += reg_read(ctxt, index_reg) << scale;
1176 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1177 if (ctxt->mode == X86EMUL_MODE_PROT64)
1178 ctxt->rip_relative = 1;
1180 base_reg = ctxt->modrm_rm;
1181 modrm_ea += reg_read(ctxt, base_reg);
1182 adjust_modrm_seg(ctxt, base_reg);
1184 switch (ctxt->modrm_mod) {
1186 if (ctxt->modrm_rm == 5)
1187 modrm_ea += insn_fetch(s32, ctxt);
1190 modrm_ea += insn_fetch(s8, ctxt);
1193 modrm_ea += insn_fetch(s32, ctxt);
1197 op->addr.mem.ea = modrm_ea;
1202 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1205 int rc = X86EMUL_CONTINUE;
1208 switch (ctxt->ad_bytes) {
1210 op->addr.mem.ea = insn_fetch(u16, ctxt);
1213 op->addr.mem.ea = insn_fetch(u32, ctxt);
1216 op->addr.mem.ea = insn_fetch(u64, ctxt);
1223 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1227 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1228 mask = ~(ctxt->dst.bytes * 8 - 1);
1230 if (ctxt->src.bytes == 2)
1231 sv = (s16)ctxt->src.val & (s16)mask;
1232 else if (ctxt->src.bytes == 4)
1233 sv = (s32)ctxt->src.val & (s32)mask;
1235 ctxt->dst.addr.mem.ea += (sv >> 3);
1238 /* only subword offset */
1239 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1242 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1243 unsigned long addr, void *dest, unsigned size)
1246 struct read_cache *mc = &ctxt->mem_read;
1248 if (mc->pos < mc->end)
1251 WARN_ON((mc->end + size) >= sizeof(mc->data));
1253 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1255 if (rc != X86EMUL_CONTINUE)
1261 memcpy(dest, mc->data + mc->pos, size);
1263 return X86EMUL_CONTINUE;
1266 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1267 struct segmented_address addr,
1274 rc = linearize(ctxt, addr, size, false, &linear);
1275 if (rc != X86EMUL_CONTINUE)
1277 return read_emulated(ctxt, linear, data, size);
1280 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1281 struct segmented_address addr,
1288 rc = linearize(ctxt, addr, size, true, &linear);
1289 if (rc != X86EMUL_CONTINUE)
1291 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1295 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1296 struct segmented_address addr,
1297 const void *orig_data, const void *data,
1303 rc = linearize(ctxt, addr, size, true, &linear);
1304 if (rc != X86EMUL_CONTINUE)
1306 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1307 size, &ctxt->exception);
1310 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1311 unsigned int size, unsigned short port,
1314 struct read_cache *rc = &ctxt->io_read;
1316 if (rc->pos == rc->end) { /* refill pio read ahead */
1317 unsigned int in_page, n;
1318 unsigned int count = ctxt->rep_prefix ?
1319 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1320 in_page = (ctxt->eflags & EFLG_DF) ?
1321 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1322 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1323 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1327 rc->pos = rc->end = 0;
1328 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1333 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1334 ctxt->dst.data = rc->data + rc->pos;
1335 ctxt->dst.type = OP_MEM_STR;
1336 ctxt->dst.count = (rc->end - rc->pos) / size;
1339 memcpy(dest, rc->data + rc->pos, size);
1345 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1346 u16 index, struct desc_struct *desc)
1351 ctxt->ops->get_idt(ctxt, &dt);
1353 if (dt.size < index * 8 + 7)
1354 return emulate_gp(ctxt, index << 3 | 0x2);
1356 addr = dt.address + index * 8;
1357 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1361 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1362 u16 selector, struct desc_ptr *dt)
1364 const struct x86_emulate_ops *ops = ctxt->ops;
1366 if (selector & 1 << 2) {
1367 struct desc_struct desc;
1370 memset (dt, 0, sizeof *dt);
1371 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1374 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1375 dt->address = get_desc_base(&desc);
1377 ops->get_gdt(ctxt, dt);
1380 /* allowed just for 8 bytes segments */
1381 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1382 u16 selector, struct desc_struct *desc,
1386 u16 index = selector >> 3;
1389 get_descriptor_table_ptr(ctxt, selector, &dt);
1391 if (dt.size < index * 8 + 7)
1392 return emulate_gp(ctxt, selector & 0xfffc);
1394 *desc_addr_p = addr = dt.address + index * 8;
1395 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1399 /* allowed just for 8 bytes segments */
1400 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1401 u16 selector, struct desc_struct *desc)
1404 u16 index = selector >> 3;
1407 get_descriptor_table_ptr(ctxt, selector, &dt);
1409 if (dt.size < index * 8 + 7)
1410 return emulate_gp(ctxt, selector & 0xfffc);
1412 addr = dt.address + index * 8;
1413 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1417 /* Does not support long mode */
1418 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1419 u16 selector, int seg)
1421 struct desc_struct seg_desc, old_desc;
1423 unsigned err_vec = GP_VECTOR;
1425 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1430 memset(&seg_desc, 0, sizeof seg_desc);
1432 if (ctxt->mode == X86EMUL_MODE_REAL) {
1433 /* set real mode segment descriptor (keep limit etc. for
1435 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1436 set_desc_base(&seg_desc, selector << 4);
1438 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1439 /* VM86 needs a clean new segment descriptor */
1440 set_desc_base(&seg_desc, selector << 4);
1441 set_desc_limit(&seg_desc, 0xffff);
1450 cpl = ctxt->ops->cpl(ctxt);
1452 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1453 if ((seg == VCPU_SREG_CS
1454 || (seg == VCPU_SREG_SS
1455 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1456 || seg == VCPU_SREG_TR)
1460 /* TR should be in GDT only */
1461 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1464 if (null_selector) /* for NULL selector skip all following checks */
1467 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1468 if (ret != X86EMUL_CONTINUE)
1471 err_code = selector & 0xfffc;
1472 err_vec = GP_VECTOR;
1474 /* can't load system descriptor into segment selector */
1475 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1479 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1488 * segment is not a writable data segment or segment
1489 * selector's RPL != CPL or segment selector's RPL != CPL
1491 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1495 if (!(seg_desc.type & 8))
1498 if (seg_desc.type & 4) {
1504 if (rpl > cpl || dpl != cpl)
1507 /* CS(RPL) <- CPL */
1508 selector = (selector & 0xfffc) | cpl;
1511 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1513 old_desc = seg_desc;
1514 seg_desc.type |= 2; /* busy */
1515 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1516 sizeof(seg_desc), &ctxt->exception);
1517 if (ret != X86EMUL_CONTINUE)
1520 case VCPU_SREG_LDTR:
1521 if (seg_desc.s || seg_desc.type != 2)
1524 default: /* DS, ES, FS, or GS */
1526 * segment is not a data or readable code segment or
1527 * ((segment is a data or nonconforming code segment)
1528 * and (both RPL and CPL > DPL))
1530 if ((seg_desc.type & 0xa) == 0x8 ||
1531 (((seg_desc.type & 0xc) != 0xc) &&
1532 (rpl > dpl && cpl > dpl)))
1538 /* mark segment as accessed */
1540 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1541 if (ret != X86EMUL_CONTINUE)
1545 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1546 return X86EMUL_CONTINUE;
1548 emulate_exception(ctxt, err_vec, err_code, true);
1549 return X86EMUL_PROPAGATE_FAULT;
1552 static void write_register_operand(struct operand *op)
1554 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1555 switch (op->bytes) {
1557 *(u8 *)op->addr.reg = (u8)op->val;
1560 *(u16 *)op->addr.reg = (u16)op->val;
1563 *op->addr.reg = (u32)op->val;
1564 break; /* 64b: zero-extend */
1566 *op->addr.reg = op->val;
1571 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1577 write_register_operand(op);
1580 if (ctxt->lock_prefix)
1581 rc = segmented_cmpxchg(ctxt,
1587 rc = segmented_write(ctxt,
1591 if (rc != X86EMUL_CONTINUE)
1595 rc = segmented_write(ctxt,
1598 op->bytes * op->count);
1599 if (rc != X86EMUL_CONTINUE)
1603 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1606 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1614 return X86EMUL_CONTINUE;
1617 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1619 struct segmented_address addr;
1621 rsp_increment(ctxt, -bytes);
1622 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1623 addr.seg = VCPU_SREG_SS;
1625 return segmented_write(ctxt, addr, data, bytes);
1628 static int em_push(struct x86_emulate_ctxt *ctxt)
1630 /* Disable writeback. */
1631 ctxt->dst.type = OP_NONE;
1632 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1635 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1636 void *dest, int len)
1639 struct segmented_address addr;
1641 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1642 addr.seg = VCPU_SREG_SS;
1643 rc = segmented_read(ctxt, addr, dest, len);
1644 if (rc != X86EMUL_CONTINUE)
1647 rsp_increment(ctxt, len);
1651 static int em_pop(struct x86_emulate_ctxt *ctxt)
1653 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1656 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1657 void *dest, int len)
1660 unsigned long val, change_mask;
1661 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1662 int cpl = ctxt->ops->cpl(ctxt);
1664 rc = emulate_pop(ctxt, &val, len);
1665 if (rc != X86EMUL_CONTINUE)
1668 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1669 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1671 switch(ctxt->mode) {
1672 case X86EMUL_MODE_PROT64:
1673 case X86EMUL_MODE_PROT32:
1674 case X86EMUL_MODE_PROT16:
1676 change_mask |= EFLG_IOPL;
1678 change_mask |= EFLG_IF;
1680 case X86EMUL_MODE_VM86:
1682 return emulate_gp(ctxt, 0);
1683 change_mask |= EFLG_IF;
1685 default: /* real mode */
1686 change_mask |= (EFLG_IOPL | EFLG_IF);
1690 *(unsigned long *)dest =
1691 (ctxt->eflags & ~change_mask) | (val & change_mask);
1696 static int em_popf(struct x86_emulate_ctxt *ctxt)
1698 ctxt->dst.type = OP_REG;
1699 ctxt->dst.addr.reg = &ctxt->eflags;
1700 ctxt->dst.bytes = ctxt->op_bytes;
1701 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1704 static int em_enter(struct x86_emulate_ctxt *ctxt)
1707 unsigned frame_size = ctxt->src.val;
1708 unsigned nesting_level = ctxt->src2.val & 31;
1712 return X86EMUL_UNHANDLEABLE;
1714 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1715 rc = push(ctxt, &rbp, stack_size(ctxt));
1716 if (rc != X86EMUL_CONTINUE)
1718 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1720 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1721 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1723 return X86EMUL_CONTINUE;
1726 static int em_leave(struct x86_emulate_ctxt *ctxt)
1728 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1730 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1733 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1735 int seg = ctxt->src2.val;
1737 ctxt->src.val = get_segment_selector(ctxt, seg);
1739 return em_push(ctxt);
1742 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1744 int seg = ctxt->src2.val;
1745 unsigned long selector;
1748 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1749 if (rc != X86EMUL_CONTINUE)
1752 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1756 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1758 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1759 int rc = X86EMUL_CONTINUE;
1760 int reg = VCPU_REGS_RAX;
1762 while (reg <= VCPU_REGS_RDI) {
1763 (reg == VCPU_REGS_RSP) ?
1764 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1767 if (rc != X86EMUL_CONTINUE)
1776 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1778 ctxt->src.val = (unsigned long)ctxt->eflags;
1779 return em_push(ctxt);
1782 static int em_popa(struct x86_emulate_ctxt *ctxt)
1784 int rc = X86EMUL_CONTINUE;
1785 int reg = VCPU_REGS_RDI;
1787 while (reg >= VCPU_REGS_RAX) {
1788 if (reg == VCPU_REGS_RSP) {
1789 rsp_increment(ctxt, ctxt->op_bytes);
1793 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1794 if (rc != X86EMUL_CONTINUE)
1801 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1803 const struct x86_emulate_ops *ops = ctxt->ops;
1810 /* TODO: Add limit checks */
1811 ctxt->src.val = ctxt->eflags;
1813 if (rc != X86EMUL_CONTINUE)
1816 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1818 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1820 if (rc != X86EMUL_CONTINUE)
1823 ctxt->src.val = ctxt->_eip;
1825 if (rc != X86EMUL_CONTINUE)
1828 ops->get_idt(ctxt, &dt);
1830 eip_addr = dt.address + (irq << 2);
1831 cs_addr = dt.address + (irq << 2) + 2;
1833 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1834 if (rc != X86EMUL_CONTINUE)
1837 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1838 if (rc != X86EMUL_CONTINUE)
1841 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1842 if (rc != X86EMUL_CONTINUE)
1850 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1854 invalidate_registers(ctxt);
1855 rc = __emulate_int_real(ctxt, irq);
1856 if (rc == X86EMUL_CONTINUE)
1857 writeback_registers(ctxt);
1861 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1863 switch(ctxt->mode) {
1864 case X86EMUL_MODE_REAL:
1865 return __emulate_int_real(ctxt, irq);
1866 case X86EMUL_MODE_VM86:
1867 case X86EMUL_MODE_PROT16:
1868 case X86EMUL_MODE_PROT32:
1869 case X86EMUL_MODE_PROT64:
1871 /* Protected mode interrupts unimplemented yet */
1872 return X86EMUL_UNHANDLEABLE;
1876 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1878 int rc = X86EMUL_CONTINUE;
1879 unsigned long temp_eip = 0;
1880 unsigned long temp_eflags = 0;
1881 unsigned long cs = 0;
1882 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1883 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1884 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1885 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1887 /* TODO: Add stack limit check */
1889 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1891 if (rc != X86EMUL_CONTINUE)
1894 if (temp_eip & ~0xffff)
1895 return emulate_gp(ctxt, 0);
1897 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1899 if (rc != X86EMUL_CONTINUE)
1902 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1904 if (rc != X86EMUL_CONTINUE)
1907 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1909 if (rc != X86EMUL_CONTINUE)
1912 ctxt->_eip = temp_eip;
1915 if (ctxt->op_bytes == 4)
1916 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1917 else if (ctxt->op_bytes == 2) {
1918 ctxt->eflags &= ~0xffff;
1919 ctxt->eflags |= temp_eflags;
1922 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1923 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1928 static int em_iret(struct x86_emulate_ctxt *ctxt)
1930 switch(ctxt->mode) {
1931 case X86EMUL_MODE_REAL:
1932 return emulate_iret_real(ctxt);
1933 case X86EMUL_MODE_VM86:
1934 case X86EMUL_MODE_PROT16:
1935 case X86EMUL_MODE_PROT32:
1936 case X86EMUL_MODE_PROT64:
1938 /* iret from protected mode unimplemented yet */
1939 return X86EMUL_UNHANDLEABLE;
1943 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1948 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1950 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1951 if (rc != X86EMUL_CONTINUE)
1955 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1956 return X86EMUL_CONTINUE;
1959 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1961 int rc = X86EMUL_CONTINUE;
1963 switch (ctxt->modrm_reg) {
1964 case 2: /* call near abs */ {
1966 old_eip = ctxt->_eip;
1967 ctxt->_eip = ctxt->src.val;
1968 ctxt->src.val = old_eip;
1972 case 4: /* jmp abs */
1973 ctxt->_eip = ctxt->src.val;
1975 case 5: /* jmp far */
1976 rc = em_jmp_far(ctxt);
1985 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1987 u64 old = ctxt->dst.orig_val64;
1989 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1990 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1991 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1992 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1993 ctxt->eflags &= ~EFLG_ZF;
1995 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1996 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1998 ctxt->eflags |= EFLG_ZF;
2000 return X86EMUL_CONTINUE;
2003 static int em_ret(struct x86_emulate_ctxt *ctxt)
2005 ctxt->dst.type = OP_REG;
2006 ctxt->dst.addr.reg = &ctxt->_eip;
2007 ctxt->dst.bytes = ctxt->op_bytes;
2008 return em_pop(ctxt);
2011 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2016 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2017 if (rc != X86EMUL_CONTINUE)
2019 if (ctxt->op_bytes == 4)
2020 ctxt->_eip = (u32)ctxt->_eip;
2021 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2022 if (rc != X86EMUL_CONTINUE)
2024 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2028 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2032 rc = em_ret_far(ctxt);
2033 if (rc != X86EMUL_CONTINUE)
2035 rsp_increment(ctxt, ctxt->src.val);
2036 return X86EMUL_CONTINUE;
2039 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2041 /* Save real source value, then compare EAX against destination. */
2042 ctxt->src.orig_val = ctxt->src.val;
2043 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2044 fastop(ctxt, em_cmp);
2046 if (ctxt->eflags & EFLG_ZF) {
2047 /* Success: write back to memory. */
2048 ctxt->dst.val = ctxt->src.orig_val;
2050 /* Failure: write the value we saw to EAX. */
2051 ctxt->dst.type = OP_REG;
2052 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2054 return X86EMUL_CONTINUE;
2057 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2059 int seg = ctxt->src2.val;
2063 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2065 rc = load_segment_descriptor(ctxt, sel, seg);
2066 if (rc != X86EMUL_CONTINUE)
2069 ctxt->dst.val = ctxt->src.val;
2074 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2075 struct desc_struct *cs, struct desc_struct *ss)
2077 cs->l = 0; /* will be adjusted later */
2078 set_desc_base(cs, 0); /* flat segment */
2079 cs->g = 1; /* 4kb granularity */
2080 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2081 cs->type = 0x0b; /* Read, Execute, Accessed */
2083 cs->dpl = 0; /* will be adjusted later */
2088 set_desc_base(ss, 0); /* flat segment */
2089 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2090 ss->g = 1; /* 4kb granularity */
2092 ss->type = 0x03; /* Read/Write, Accessed */
2093 ss->d = 1; /* 32bit stack segment */
2100 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2102 u32 eax, ebx, ecx, edx;
2105 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2106 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2107 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2108 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2111 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2113 const struct x86_emulate_ops *ops = ctxt->ops;
2114 u32 eax, ebx, ecx, edx;
2117 * syscall should always be enabled in longmode - so only become
2118 * vendor specific (cpuid) if other modes are active...
2120 if (ctxt->mode == X86EMUL_MODE_PROT64)
2125 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2127 * Intel ("GenuineIntel")
2128 * remark: Intel CPUs only support "syscall" in 64bit
2129 * longmode. Also an 64bit guest with a
2130 * 32bit compat-app running will #UD !! While this
2131 * behaviour can be fixed (by emulating) into AMD
2132 * response - CPUs of AMD can't behave like Intel.
2134 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2135 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2136 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2139 /* AMD ("AuthenticAMD") */
2140 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2141 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2142 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2145 /* AMD ("AMDisbetter!") */
2146 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2147 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2148 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2151 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2155 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2157 const struct x86_emulate_ops *ops = ctxt->ops;
2158 struct desc_struct cs, ss;
2163 /* syscall is not available in real mode */
2164 if (ctxt->mode == X86EMUL_MODE_REAL ||
2165 ctxt->mode == X86EMUL_MODE_VM86)
2166 return emulate_ud(ctxt);
2168 if (!(em_syscall_is_enabled(ctxt)))
2169 return emulate_ud(ctxt);
2171 ops->get_msr(ctxt, MSR_EFER, &efer);
2172 setup_syscalls_segments(ctxt, &cs, &ss);
2174 if (!(efer & EFER_SCE))
2175 return emulate_ud(ctxt);
2177 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2179 cs_sel = (u16)(msr_data & 0xfffc);
2180 ss_sel = (u16)(msr_data + 8);
2182 if (efer & EFER_LMA) {
2186 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2187 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2189 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2190 if (efer & EFER_LMA) {
2191 #ifdef CONFIG_X86_64
2192 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2195 ctxt->mode == X86EMUL_MODE_PROT64 ?
2196 MSR_LSTAR : MSR_CSTAR, &msr_data);
2197 ctxt->_eip = msr_data;
2199 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2200 ctxt->eflags &= ~(msr_data | EFLG_RF);
2204 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2205 ctxt->_eip = (u32)msr_data;
2207 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2210 return X86EMUL_CONTINUE;
2213 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2215 const struct x86_emulate_ops *ops = ctxt->ops;
2216 struct desc_struct cs, ss;
2221 ops->get_msr(ctxt, MSR_EFER, &efer);
2222 /* inject #GP if in real mode */
2223 if (ctxt->mode == X86EMUL_MODE_REAL)
2224 return emulate_gp(ctxt, 0);
2227 * Not recognized on AMD in compat mode (but is recognized in legacy
2230 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2231 && !vendor_intel(ctxt))
2232 return emulate_ud(ctxt);
2234 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2235 * Therefore, we inject an #UD.
2237 if (ctxt->mode == X86EMUL_MODE_PROT64)
2238 return emulate_ud(ctxt);
2240 setup_syscalls_segments(ctxt, &cs, &ss);
2242 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2243 switch (ctxt->mode) {
2244 case X86EMUL_MODE_PROT32:
2245 if ((msr_data & 0xfffc) == 0x0)
2246 return emulate_gp(ctxt, 0);
2248 case X86EMUL_MODE_PROT64:
2249 if (msr_data == 0x0)
2250 return emulate_gp(ctxt, 0);
2256 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2257 cs_sel = (u16)msr_data;
2258 cs_sel &= ~SELECTOR_RPL_MASK;
2259 ss_sel = cs_sel + 8;
2260 ss_sel &= ~SELECTOR_RPL_MASK;
2261 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2266 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2267 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2269 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2270 ctxt->_eip = msr_data;
2272 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2273 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2275 return X86EMUL_CONTINUE;
2278 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2280 const struct x86_emulate_ops *ops = ctxt->ops;
2281 struct desc_struct cs, ss;
2284 u16 cs_sel = 0, ss_sel = 0;
2286 /* inject #GP if in real mode or Virtual 8086 mode */
2287 if (ctxt->mode == X86EMUL_MODE_REAL ||
2288 ctxt->mode == X86EMUL_MODE_VM86)
2289 return emulate_gp(ctxt, 0);
2291 setup_syscalls_segments(ctxt, &cs, &ss);
2293 if ((ctxt->rex_prefix & 0x8) != 0x0)
2294 usermode = X86EMUL_MODE_PROT64;
2296 usermode = X86EMUL_MODE_PROT32;
2300 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2302 case X86EMUL_MODE_PROT32:
2303 cs_sel = (u16)(msr_data + 16);
2304 if ((msr_data & 0xfffc) == 0x0)
2305 return emulate_gp(ctxt, 0);
2306 ss_sel = (u16)(msr_data + 24);
2308 case X86EMUL_MODE_PROT64:
2309 cs_sel = (u16)(msr_data + 32);
2310 if (msr_data == 0x0)
2311 return emulate_gp(ctxt, 0);
2312 ss_sel = cs_sel + 8;
2317 cs_sel |= SELECTOR_RPL_MASK;
2318 ss_sel |= SELECTOR_RPL_MASK;
2320 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2321 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2323 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2324 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2326 return X86EMUL_CONTINUE;
2329 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2332 if (ctxt->mode == X86EMUL_MODE_REAL)
2334 if (ctxt->mode == X86EMUL_MODE_VM86)
2336 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2337 return ctxt->ops->cpl(ctxt) > iopl;
2340 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2343 const struct x86_emulate_ops *ops = ctxt->ops;
2344 struct desc_struct tr_seg;
2347 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2348 unsigned mask = (1 << len) - 1;
2351 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2354 if (desc_limit_scaled(&tr_seg) < 103)
2356 base = get_desc_base(&tr_seg);
2357 #ifdef CONFIG_X86_64
2358 base |= ((u64)base3) << 32;
2360 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2361 if (r != X86EMUL_CONTINUE)
2363 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2365 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2366 if (r != X86EMUL_CONTINUE)
2368 if ((perm >> bit_idx) & mask)
2373 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2379 if (emulator_bad_iopl(ctxt))
2380 if (!emulator_io_port_access_allowed(ctxt, port, len))
2383 ctxt->perm_ok = true;
2388 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2389 struct tss_segment_16 *tss)
2391 tss->ip = ctxt->_eip;
2392 tss->flag = ctxt->eflags;
2393 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2394 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2395 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2396 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2397 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2398 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2399 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2400 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2402 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2403 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2404 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2405 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2406 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2409 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2410 struct tss_segment_16 *tss)
2414 ctxt->_eip = tss->ip;
2415 ctxt->eflags = tss->flag | 2;
2416 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2417 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2418 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2419 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2420 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2421 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2422 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2423 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2426 * SDM says that segment selectors are loaded before segment
2429 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2430 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2431 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2432 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2433 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2436 * Now load segment descriptors. If fault happens at this stage
2437 * it is handled in a context of new task
2439 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2440 if (ret != X86EMUL_CONTINUE)
2442 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2443 if (ret != X86EMUL_CONTINUE)
2445 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2446 if (ret != X86EMUL_CONTINUE)
2448 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2449 if (ret != X86EMUL_CONTINUE)
2451 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2452 if (ret != X86EMUL_CONTINUE)
2455 return X86EMUL_CONTINUE;
2458 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2459 u16 tss_selector, u16 old_tss_sel,
2460 ulong old_tss_base, struct desc_struct *new_desc)
2462 const struct x86_emulate_ops *ops = ctxt->ops;
2463 struct tss_segment_16 tss_seg;
2465 u32 new_tss_base = get_desc_base(new_desc);
2467 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2469 if (ret != X86EMUL_CONTINUE)
2470 /* FIXME: need to provide precise fault address */
2473 save_state_to_tss16(ctxt, &tss_seg);
2475 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2477 if (ret != X86EMUL_CONTINUE)
2478 /* FIXME: need to provide precise fault address */
2481 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2483 if (ret != X86EMUL_CONTINUE)
2484 /* FIXME: need to provide precise fault address */
2487 if (old_tss_sel != 0xffff) {
2488 tss_seg.prev_task_link = old_tss_sel;
2490 ret = ops->write_std(ctxt, new_tss_base,
2491 &tss_seg.prev_task_link,
2492 sizeof tss_seg.prev_task_link,
2494 if (ret != X86EMUL_CONTINUE)
2495 /* FIXME: need to provide precise fault address */
2499 return load_state_from_tss16(ctxt, &tss_seg);
2502 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2503 struct tss_segment_32 *tss)
2505 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2506 tss->eip = ctxt->_eip;
2507 tss->eflags = ctxt->eflags;
2508 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2509 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2510 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2511 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2512 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2513 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2514 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2515 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2517 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2518 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2519 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2520 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2521 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2522 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2523 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2526 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2527 struct tss_segment_32 *tss)
2531 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2532 return emulate_gp(ctxt, 0);
2533 ctxt->_eip = tss->eip;
2534 ctxt->eflags = tss->eflags | 2;
2536 /* General purpose registers */
2537 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2538 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2539 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2540 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2541 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2542 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2543 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2544 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2547 * SDM says that segment selectors are loaded before segment
2550 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2551 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2552 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2553 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2554 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2555 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2556 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2559 * If we're switching between Protected Mode and VM86, we need to make
2560 * sure to update the mode before loading the segment descriptors so
2561 * that the selectors are interpreted correctly.
2563 * Need to get rflags to the vcpu struct immediately because it
2564 * influences the CPL which is checked at least when loading the segment
2565 * descriptors and when pushing an error code to the new kernel stack.
2567 * TODO Introduce a separate ctxt->ops->set_cpl callback
2569 if (ctxt->eflags & X86_EFLAGS_VM)
2570 ctxt->mode = X86EMUL_MODE_VM86;
2572 ctxt->mode = X86EMUL_MODE_PROT32;
2574 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2577 * Now load segment descriptors. If fault happenes at this stage
2578 * it is handled in a context of new task
2580 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2581 if (ret != X86EMUL_CONTINUE)
2583 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2584 if (ret != X86EMUL_CONTINUE)
2586 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2587 if (ret != X86EMUL_CONTINUE)
2589 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2590 if (ret != X86EMUL_CONTINUE)
2592 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2593 if (ret != X86EMUL_CONTINUE)
2595 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2596 if (ret != X86EMUL_CONTINUE)
2598 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2599 if (ret != X86EMUL_CONTINUE)
2602 return X86EMUL_CONTINUE;
2605 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2606 u16 tss_selector, u16 old_tss_sel,
2607 ulong old_tss_base, struct desc_struct *new_desc)
2609 const struct x86_emulate_ops *ops = ctxt->ops;
2610 struct tss_segment_32 tss_seg;
2612 u32 new_tss_base = get_desc_base(new_desc);
2614 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2616 if (ret != X86EMUL_CONTINUE)
2617 /* FIXME: need to provide precise fault address */
2620 save_state_to_tss32(ctxt, &tss_seg);
2622 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2624 if (ret != X86EMUL_CONTINUE)
2625 /* FIXME: need to provide precise fault address */
2628 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2630 if (ret != X86EMUL_CONTINUE)
2631 /* FIXME: need to provide precise fault address */
2634 if (old_tss_sel != 0xffff) {
2635 tss_seg.prev_task_link = old_tss_sel;
2637 ret = ops->write_std(ctxt, new_tss_base,
2638 &tss_seg.prev_task_link,
2639 sizeof tss_seg.prev_task_link,
2641 if (ret != X86EMUL_CONTINUE)
2642 /* FIXME: need to provide precise fault address */
2646 return load_state_from_tss32(ctxt, &tss_seg);
2649 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2650 u16 tss_selector, int idt_index, int reason,
2651 bool has_error_code, u32 error_code)
2653 const struct x86_emulate_ops *ops = ctxt->ops;
2654 struct desc_struct curr_tss_desc, next_tss_desc;
2656 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2657 ulong old_tss_base =
2658 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2662 /* FIXME: old_tss_base == ~0 ? */
2664 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2665 if (ret != X86EMUL_CONTINUE)
2667 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2668 if (ret != X86EMUL_CONTINUE)
2671 /* FIXME: check that next_tss_desc is tss */
2674 * Check privileges. The three cases are task switch caused by...
2676 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2677 * 2. Exception/IRQ/iret: No check is performed
2678 * 3. jmp/call to TSS: Check against DPL of the TSS
2680 if (reason == TASK_SWITCH_GATE) {
2681 if (idt_index != -1) {
2682 /* Software interrupts */
2683 struct desc_struct task_gate_desc;
2686 ret = read_interrupt_descriptor(ctxt, idt_index,
2688 if (ret != X86EMUL_CONTINUE)
2691 dpl = task_gate_desc.dpl;
2692 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2693 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2695 } else if (reason != TASK_SWITCH_IRET) {
2696 int dpl = next_tss_desc.dpl;
2697 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2698 return emulate_gp(ctxt, tss_selector);
2702 desc_limit = desc_limit_scaled(&next_tss_desc);
2703 if (!next_tss_desc.p ||
2704 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2705 desc_limit < 0x2b)) {
2706 emulate_ts(ctxt, tss_selector & 0xfffc);
2707 return X86EMUL_PROPAGATE_FAULT;
2710 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2711 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2712 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2715 if (reason == TASK_SWITCH_IRET)
2716 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2718 /* set back link to prev task only if NT bit is set in eflags
2719 note that old_tss_sel is not used after this point */
2720 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2721 old_tss_sel = 0xffff;
2723 if (next_tss_desc.type & 8)
2724 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2725 old_tss_base, &next_tss_desc);
2727 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2728 old_tss_base, &next_tss_desc);
2729 if (ret != X86EMUL_CONTINUE)
2732 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2733 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2735 if (reason != TASK_SWITCH_IRET) {
2736 next_tss_desc.type |= (1 << 1); /* set busy flag */
2737 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2740 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2741 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2743 if (has_error_code) {
2744 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2745 ctxt->lock_prefix = 0;
2746 ctxt->src.val = (unsigned long) error_code;
2747 ret = em_push(ctxt);
2753 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2754 u16 tss_selector, int idt_index, int reason,
2755 bool has_error_code, u32 error_code)
2759 invalidate_registers(ctxt);
2760 ctxt->_eip = ctxt->eip;
2761 ctxt->dst.type = OP_NONE;
2763 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2764 has_error_code, error_code);
2766 if (rc == X86EMUL_CONTINUE) {
2767 ctxt->eip = ctxt->_eip;
2768 writeback_registers(ctxt);
2771 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2774 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2777 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2779 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2780 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2783 static int em_das(struct x86_emulate_ctxt *ctxt)
2786 bool af, cf, old_cf;
2788 cf = ctxt->eflags & X86_EFLAGS_CF;
2794 af = ctxt->eflags & X86_EFLAGS_AF;
2795 if ((al & 0x0f) > 9 || af) {
2797 cf = old_cf | (al >= 250);
2802 if (old_al > 0x99 || old_cf) {
2808 /* Set PF, ZF, SF */
2809 ctxt->src.type = OP_IMM;
2811 ctxt->src.bytes = 1;
2812 fastop(ctxt, em_or);
2813 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2815 ctxt->eflags |= X86_EFLAGS_CF;
2817 ctxt->eflags |= X86_EFLAGS_AF;
2818 return X86EMUL_CONTINUE;
2821 static int em_aam(struct x86_emulate_ctxt *ctxt)
2825 if (ctxt->src.val == 0)
2826 return emulate_de(ctxt);
2828 al = ctxt->dst.val & 0xff;
2829 ah = al / ctxt->src.val;
2830 al %= ctxt->src.val;
2832 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2834 /* Set PF, ZF, SF */
2835 ctxt->src.type = OP_IMM;
2837 ctxt->src.bytes = 1;
2838 fastop(ctxt, em_or);
2840 return X86EMUL_CONTINUE;
2843 static int em_aad(struct x86_emulate_ctxt *ctxt)
2845 u8 al = ctxt->dst.val & 0xff;
2846 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2848 al = (al + (ah * ctxt->src.val)) & 0xff;
2850 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2852 /* Set PF, ZF, SF */
2853 ctxt->src.type = OP_IMM;
2855 ctxt->src.bytes = 1;
2856 fastop(ctxt, em_or);
2858 return X86EMUL_CONTINUE;
2861 static int em_call(struct x86_emulate_ctxt *ctxt)
2863 long rel = ctxt->src.val;
2865 ctxt->src.val = (unsigned long)ctxt->_eip;
2867 return em_push(ctxt);
2870 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2876 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2877 old_eip = ctxt->_eip;
2879 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2880 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2881 return X86EMUL_CONTINUE;
2884 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2886 ctxt->src.val = old_cs;
2888 if (rc != X86EMUL_CONTINUE)
2891 ctxt->src.val = old_eip;
2892 return em_push(ctxt);
2895 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2899 ctxt->dst.type = OP_REG;
2900 ctxt->dst.addr.reg = &ctxt->_eip;
2901 ctxt->dst.bytes = ctxt->op_bytes;
2902 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2903 if (rc != X86EMUL_CONTINUE)
2905 rsp_increment(ctxt, ctxt->src.val);
2906 return X86EMUL_CONTINUE;
2909 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2911 /* Write back the register source. */
2912 ctxt->src.val = ctxt->dst.val;
2913 write_register_operand(&ctxt->src);
2915 /* Write back the memory destination with implicit LOCK prefix. */
2916 ctxt->dst.val = ctxt->src.orig_val;
2917 ctxt->lock_prefix = 1;
2918 return X86EMUL_CONTINUE;
2921 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2923 ctxt->dst.val = ctxt->src2.val;
2924 return fastop(ctxt, em_imul);
2927 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2929 ctxt->dst.type = OP_REG;
2930 ctxt->dst.bytes = ctxt->src.bytes;
2931 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2932 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2934 return X86EMUL_CONTINUE;
2937 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2941 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2942 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2943 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2944 return X86EMUL_CONTINUE;
2947 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2951 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2952 return emulate_gp(ctxt, 0);
2953 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2954 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2955 return X86EMUL_CONTINUE;
2958 static int em_mov(struct x86_emulate_ctxt *ctxt)
2960 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2961 return X86EMUL_CONTINUE;
2964 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2966 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2967 return emulate_gp(ctxt, 0);
2969 /* Disable writeback. */
2970 ctxt->dst.type = OP_NONE;
2971 return X86EMUL_CONTINUE;
2974 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2978 if (ctxt->mode == X86EMUL_MODE_PROT64)
2979 val = ctxt->src.val & ~0ULL;
2981 val = ctxt->src.val & ~0U;
2983 /* #UD condition is already handled. */
2984 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2985 return emulate_gp(ctxt, 0);
2987 /* Disable writeback. */
2988 ctxt->dst.type = OP_NONE;
2989 return X86EMUL_CONTINUE;
2992 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2996 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
2997 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
2998 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
2999 return emulate_gp(ctxt, 0);
3001 return X86EMUL_CONTINUE;
3004 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3008 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3009 return emulate_gp(ctxt, 0);
3011 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3012 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3013 return X86EMUL_CONTINUE;
3016 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3018 if (ctxt->modrm_reg > VCPU_SREG_GS)
3019 return emulate_ud(ctxt);
3021 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3022 return X86EMUL_CONTINUE;
3025 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3027 u16 sel = ctxt->src.val;
3029 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3030 return emulate_ud(ctxt);
3032 if (ctxt->modrm_reg == VCPU_SREG_SS)
3033 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3035 /* Disable writeback. */
3036 ctxt->dst.type = OP_NONE;
3037 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3040 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3042 u16 sel = ctxt->src.val;
3044 /* Disable writeback. */
3045 ctxt->dst.type = OP_NONE;
3046 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3049 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3051 u16 sel = ctxt->src.val;
3053 /* Disable writeback. */
3054 ctxt->dst.type = OP_NONE;
3055 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3058 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3063 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3064 if (rc == X86EMUL_CONTINUE)
3065 ctxt->ops->invlpg(ctxt, linear);
3066 /* Disable writeback. */
3067 ctxt->dst.type = OP_NONE;
3068 return X86EMUL_CONTINUE;
3071 static int em_clts(struct x86_emulate_ctxt *ctxt)
3075 cr0 = ctxt->ops->get_cr(ctxt, 0);
3077 ctxt->ops->set_cr(ctxt, 0, cr0);
3078 return X86EMUL_CONTINUE;
3081 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3085 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3086 return X86EMUL_UNHANDLEABLE;
3088 rc = ctxt->ops->fix_hypercall(ctxt);
3089 if (rc != X86EMUL_CONTINUE)
3092 /* Let the processor re-execute the fixed hypercall */
3093 ctxt->_eip = ctxt->eip;
3094 /* Disable writeback. */
3095 ctxt->dst.type = OP_NONE;
3096 return X86EMUL_CONTINUE;
3099 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3100 void (*get)(struct x86_emulate_ctxt *ctxt,
3101 struct desc_ptr *ptr))
3103 struct desc_ptr desc_ptr;
3105 if (ctxt->mode == X86EMUL_MODE_PROT64)
3107 get(ctxt, &desc_ptr);
3108 if (ctxt->op_bytes == 2) {
3110 desc_ptr.address &= 0x00ffffff;
3112 /* Disable writeback. */
3113 ctxt->dst.type = OP_NONE;
3114 return segmented_write(ctxt, ctxt->dst.addr.mem,
3115 &desc_ptr, 2 + ctxt->op_bytes);
3118 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3120 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3123 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3125 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3128 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3130 struct desc_ptr desc_ptr;
3133 if (ctxt->mode == X86EMUL_MODE_PROT64)
3135 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3136 &desc_ptr.size, &desc_ptr.address,
3138 if (rc != X86EMUL_CONTINUE)
3140 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3141 /* Disable writeback. */
3142 ctxt->dst.type = OP_NONE;
3143 return X86EMUL_CONTINUE;
3146 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3150 rc = ctxt->ops->fix_hypercall(ctxt);
3152 /* Disable writeback. */
3153 ctxt->dst.type = OP_NONE;
3157 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3159 struct desc_ptr desc_ptr;
3162 if (ctxt->mode == X86EMUL_MODE_PROT64)
3164 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3165 &desc_ptr.size, &desc_ptr.address,
3167 if (rc != X86EMUL_CONTINUE)
3169 ctxt->ops->set_idt(ctxt, &desc_ptr);
3170 /* Disable writeback. */
3171 ctxt->dst.type = OP_NONE;
3172 return X86EMUL_CONTINUE;
3175 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3177 ctxt->dst.bytes = 2;
3178 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3179 return X86EMUL_CONTINUE;
3182 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3184 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3185 | (ctxt->src.val & 0x0f));
3186 ctxt->dst.type = OP_NONE;
3187 return X86EMUL_CONTINUE;
3190 static int em_loop(struct x86_emulate_ctxt *ctxt)
3192 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3193 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3194 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3195 jmp_rel(ctxt, ctxt->src.val);
3197 return X86EMUL_CONTINUE;
3200 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3202 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3203 jmp_rel(ctxt, ctxt->src.val);
3205 return X86EMUL_CONTINUE;
3208 static int em_in(struct x86_emulate_ctxt *ctxt)
3210 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3212 return X86EMUL_IO_NEEDED;
3214 return X86EMUL_CONTINUE;
3217 static int em_out(struct x86_emulate_ctxt *ctxt)
3219 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3221 /* Disable writeback. */
3222 ctxt->dst.type = OP_NONE;
3223 return X86EMUL_CONTINUE;
3226 static int em_cli(struct x86_emulate_ctxt *ctxt)
3228 if (emulator_bad_iopl(ctxt))
3229 return emulate_gp(ctxt, 0);
3231 ctxt->eflags &= ~X86_EFLAGS_IF;
3232 return X86EMUL_CONTINUE;
3235 static int em_sti(struct x86_emulate_ctxt *ctxt)
3237 if (emulator_bad_iopl(ctxt))
3238 return emulate_gp(ctxt, 0);
3240 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3241 ctxt->eflags |= X86_EFLAGS_IF;
3242 return X86EMUL_CONTINUE;
3245 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3247 u32 eax, ebx, ecx, edx;
3249 eax = reg_read(ctxt, VCPU_REGS_RAX);
3250 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3251 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3252 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3253 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3254 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3255 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3256 return X86EMUL_CONTINUE;
3259 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3261 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3262 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3263 return X86EMUL_CONTINUE;
3266 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3268 switch (ctxt->op_bytes) {
3269 #ifdef CONFIG_X86_64
3271 asm("bswap %0" : "+r"(ctxt->dst.val));
3275 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3278 return X86EMUL_CONTINUE;
3281 static bool valid_cr(int nr)
3293 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3295 if (!valid_cr(ctxt->modrm_reg))
3296 return emulate_ud(ctxt);
3298 return X86EMUL_CONTINUE;
3301 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3303 u64 new_val = ctxt->src.val64;
3304 int cr = ctxt->modrm_reg;
3307 static u64 cr_reserved_bits[] = {
3308 0xffffffff00000000ULL,
3309 0, 0, 0, /* CR3 checked later */
3316 return emulate_ud(ctxt);
3318 if (new_val & cr_reserved_bits[cr])
3319 return emulate_gp(ctxt, 0);
3324 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3325 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3326 return emulate_gp(ctxt, 0);
3328 cr4 = ctxt->ops->get_cr(ctxt, 4);
3329 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3331 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3332 !(cr4 & X86_CR4_PAE))
3333 return emulate_gp(ctxt, 0);
3340 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3341 if (efer & EFER_LMA)
3342 rsvd = CR3_L_MODE_RESERVED_BITS;
3343 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3344 rsvd = CR3_PAE_RESERVED_BITS;
3345 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3346 rsvd = CR3_NONPAE_RESERVED_BITS;
3349 return emulate_gp(ctxt, 0);
3354 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3356 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3357 return emulate_gp(ctxt, 0);
3363 return X86EMUL_CONTINUE;
3366 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3370 ctxt->ops->get_dr(ctxt, 7, &dr7);
3372 /* Check if DR7.Global_Enable is set */
3373 return dr7 & (1 << 13);
3376 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3378 int dr = ctxt->modrm_reg;
3382 return emulate_ud(ctxt);
3384 cr4 = ctxt->ops->get_cr(ctxt, 4);
3385 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3386 return emulate_ud(ctxt);
3388 if (check_dr7_gd(ctxt))
3389 return emulate_db(ctxt);
3391 return X86EMUL_CONTINUE;
3394 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3396 u64 new_val = ctxt->src.val64;
3397 int dr = ctxt->modrm_reg;
3399 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3400 return emulate_gp(ctxt, 0);
3402 return check_dr_read(ctxt);
3405 static int check_svme(struct x86_emulate_ctxt *ctxt)
3409 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3411 if (!(efer & EFER_SVME))
3412 return emulate_ud(ctxt);
3414 return X86EMUL_CONTINUE;
3417 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3419 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3421 /* Valid physical address? */
3422 if (rax & 0xffff000000000000ULL)
3423 return emulate_gp(ctxt, 0);
3425 return check_svme(ctxt);
3428 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3430 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3432 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3433 return emulate_ud(ctxt);
3435 return X86EMUL_CONTINUE;
3438 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3440 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3441 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3443 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3445 return emulate_gp(ctxt, 0);
3447 return X86EMUL_CONTINUE;
3450 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3452 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3453 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3454 return emulate_gp(ctxt, 0);
3456 return X86EMUL_CONTINUE;
3459 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3461 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3462 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3463 return emulate_gp(ctxt, 0);
3465 return X86EMUL_CONTINUE;
3468 #define D(_y) { .flags = (_y) }
3469 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3470 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3471 .check_perm = (_p) }
3472 #define N D(NotImpl)
3473 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3474 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3475 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3476 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3477 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3478 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3479 #define II(_f, _e, _i) \
3480 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3481 #define IIP(_f, _e, _i, _p) \
3482 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3483 .check_perm = (_p) }
3484 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3486 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3487 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3488 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3489 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3490 #define I2bvIP(_f, _e, _i, _p) \
3491 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3493 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3494 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3495 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3497 static const struct opcode group7_rm1[] = {
3498 DI(SrcNone | Priv, monitor),
3499 DI(SrcNone | Priv, mwait),
3503 static const struct opcode group7_rm3[] = {
3504 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3505 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3506 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3507 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3508 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3509 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3510 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3511 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3514 static const struct opcode group7_rm7[] = {
3516 DIP(SrcNone, rdtscp, check_rdtsc),
3520 static const struct opcode group1[] = {
3522 F(Lock | PageTable, em_or),
3525 F(Lock | PageTable, em_and),
3531 static const struct opcode group1A[] = {
3532 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3535 static const struct opcode group2[] = {
3536 F(DstMem | ModRM, em_rol),
3537 F(DstMem | ModRM, em_ror),
3538 F(DstMem | ModRM, em_rcl),
3539 F(DstMem | ModRM, em_rcr),
3540 F(DstMem | ModRM, em_shl),
3541 F(DstMem | ModRM, em_shr),
3542 F(DstMem | ModRM, em_shl),
3543 F(DstMem | ModRM, em_sar),
3546 static const struct opcode group3[] = {
3547 F(DstMem | SrcImm | NoWrite, em_test),
3548 F(DstMem | SrcImm | NoWrite, em_test),
3549 F(DstMem | SrcNone | Lock, em_not),
3550 F(DstMem | SrcNone | Lock, em_neg),
3551 F(DstXacc | Src2Mem, em_mul_ex),
3552 F(DstXacc | Src2Mem, em_imul_ex),
3553 F(DstXacc | Src2Mem, em_div_ex),
3554 F(DstXacc | Src2Mem, em_idiv_ex),
3557 static const struct opcode group4[] = {
3558 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3559 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3563 static const struct opcode group5[] = {
3564 F(DstMem | SrcNone | Lock, em_inc),
3565 F(DstMem | SrcNone | Lock, em_dec),
3566 I(SrcMem | Stack, em_grp45),
3567 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3568 I(SrcMem | Stack, em_grp45),
3569 I(SrcMemFAddr | ImplicitOps, em_grp45),
3570 I(SrcMem | Stack, em_grp45), D(Undefined),
3573 static const struct opcode group6[] = {
3576 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3577 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3581 static const struct group_dual group7 = { {
3582 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3583 II(Mov | DstMem | Priv, em_sidt, sidt),
3584 II(SrcMem | Priv, em_lgdt, lgdt),
3585 II(SrcMem | Priv, em_lidt, lidt),
3586 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3587 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3588 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3590 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3592 N, EXT(0, group7_rm3),
3593 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3594 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3598 static const struct opcode group8[] = {
3600 F(DstMem | SrcImmByte | NoWrite, em_bt),
3601 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3602 F(DstMem | SrcImmByte | Lock, em_btr),
3603 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3606 static const struct group_dual group9 = { {
3607 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3609 N, N, N, N, N, N, N, N,
3612 static const struct opcode group11[] = {
3613 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3617 static const struct gprefix pfx_0f_6f_0f_7f = {
3618 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3621 static const struct gprefix pfx_vmovntpx = {
3622 I(0, em_mov), N, N, N,
3625 static const struct escape escape_d9 = { {
3626 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3629 N, N, N, N, N, N, N, N,
3631 N, N, N, N, N, N, N, N,
3633 N, N, N, N, N, N, N, N,
3635 N, N, N, N, N, N, N, N,
3637 N, N, N, N, N, N, N, N,
3639 N, N, N, N, N, N, N, N,
3641 N, N, N, N, N, N, N, N,
3643 N, N, N, N, N, N, N, N,
3646 static const struct escape escape_db = { {
3647 N, N, N, N, N, N, N, N,
3650 N, N, N, N, N, N, N, N,
3652 N, N, N, N, N, N, N, N,
3654 N, N, N, N, N, N, N, N,
3656 N, N, N, N, N, N, N, N,
3658 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3660 N, N, N, N, N, N, N, N,
3662 N, N, N, N, N, N, N, N,
3664 N, N, N, N, N, N, N, N,
3667 static const struct escape escape_dd = { {
3668 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3671 N, N, N, N, N, N, N, N,
3673 N, N, N, N, N, N, N, N,
3675 N, N, N, N, N, N, N, N,
3677 N, N, N, N, N, N, N, N,
3679 N, N, N, N, N, N, N, N,
3681 N, N, N, N, N, N, N, N,
3683 N, N, N, N, N, N, N, N,
3685 N, N, N, N, N, N, N, N,
3688 static const struct opcode opcode_table[256] = {
3690 F6ALU(Lock, em_add),
3691 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3692 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3694 F6ALU(Lock | PageTable, em_or),
3695 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3698 F6ALU(Lock, em_adc),
3699 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3700 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3702 F6ALU(Lock, em_sbb),
3703 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3704 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3706 F6ALU(Lock | PageTable, em_and), N, N,
3708 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3710 F6ALU(Lock, em_xor), N, N,
3712 F6ALU(NoWrite, em_cmp), N, N,
3714 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3716 X8(I(SrcReg | Stack, em_push)),
3718 X8(I(DstReg | Stack, em_pop)),
3720 I(ImplicitOps | Stack | No64, em_pusha),
3721 I(ImplicitOps | Stack | No64, em_popa),
3722 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3725 I(SrcImm | Mov | Stack, em_push),
3726 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3727 I(SrcImmByte | Mov | Stack, em_push),
3728 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3729 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3730 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3734 G(ByteOp | DstMem | SrcImm, group1),
3735 G(DstMem | SrcImm, group1),
3736 G(ByteOp | DstMem | SrcImm | No64, group1),
3737 G(DstMem | SrcImmByte, group1),
3738 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3739 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3741 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3742 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3743 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3744 D(ModRM | SrcMem | NoAccess | DstReg),
3745 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3748 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3750 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3751 I(SrcImmFAddr | No64, em_call_far), N,
3752 II(ImplicitOps | Stack, em_pushf, pushf),
3753 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3755 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3756 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3757 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3758 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3760 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3761 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3762 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3763 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3765 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3767 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3769 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3770 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3771 I(ImplicitOps | Stack, em_ret),
3772 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3773 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3774 G(ByteOp, group11), G(0, group11),
3776 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3777 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3778 I(ImplicitOps | Stack, em_ret_far),
3779 D(ImplicitOps), DI(SrcImmByte, intn),
3780 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3782 G(Src2One | ByteOp, group2), G(Src2One, group2),
3783 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3784 I(DstAcc | SrcImmUByte | No64, em_aam),
3785 I(DstAcc | SrcImmUByte | No64, em_aad),
3786 F(DstAcc | ByteOp | No64, em_salc),
3787 I(DstAcc | SrcXLat | ByteOp, em_mov),
3789 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3791 X3(I(SrcImmByte, em_loop)),
3792 I(SrcImmByte, em_jcxz),
3793 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3794 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3796 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3797 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3798 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3799 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3801 N, DI(ImplicitOps, icebp), N, N,
3802 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3803 G(ByteOp, group3), G(0, group3),
3805 D(ImplicitOps), D(ImplicitOps),
3806 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3807 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3810 static const struct opcode twobyte_table[256] = {
3812 G(0, group6), GD(0, &group7), N, N,
3813 N, I(ImplicitOps | VendorSpecific, em_syscall),
3814 II(ImplicitOps | Priv, em_clts, clts), N,
3815 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3816 N, D(ImplicitOps | ModRM), N, N,
3818 N, N, N, N, N, N, N, N,
3819 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3821 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3822 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3823 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3824 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3826 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3829 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3830 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3831 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3832 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3833 I(ImplicitOps | VendorSpecific, em_sysenter),
3834 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3836 N, N, N, N, N, N, N, N,
3838 X16(D(DstReg | SrcMem | ModRM | Mov)),
3840 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3845 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3850 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3854 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3856 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3857 II(ImplicitOps, em_cpuid, cpuid),
3858 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3859 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3860 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3862 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3863 DI(ImplicitOps, rsm),
3864 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3865 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3866 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3867 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3869 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3870 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3871 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3872 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3873 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3874 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3878 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3879 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3880 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3882 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3883 N, D(DstMem | SrcReg | ModRM | Mov),
3884 N, N, N, GD(0, &group9),
3886 X8(I(DstReg, em_bswap)),
3888 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3890 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3892 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3909 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3913 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3919 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3920 unsigned size, bool sign_extension)
3922 int rc = X86EMUL_CONTINUE;
3926 op->addr.mem.ea = ctxt->_eip;
3927 /* NB. Immediates are sign-extended as necessary. */
3928 switch (op->bytes) {
3930 op->val = insn_fetch(s8, ctxt);
3933 op->val = insn_fetch(s16, ctxt);
3936 op->val = insn_fetch(s32, ctxt);
3939 op->val = insn_fetch(s64, ctxt);
3942 if (!sign_extension) {
3943 switch (op->bytes) {
3951 op->val &= 0xffffffff;
3959 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3962 int rc = X86EMUL_CONTINUE;
3966 decode_register_operand(ctxt, op);
3969 rc = decode_imm(ctxt, op, 1, false);
3972 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3976 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3977 fetch_bit_operand(ctxt);
3978 op->orig_val = op->val;
3981 ctxt->memop.bytes = 8;
3985 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3986 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3987 fetch_register_operand(op);
3988 op->orig_val = op->val;
3992 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
3993 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3994 fetch_register_operand(op);
3995 op->orig_val = op->val;
3998 if (ctxt->d & ByteOp) {
4003 op->bytes = ctxt->op_bytes;
4004 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4005 fetch_register_operand(op);
4006 op->orig_val = op->val;
4010 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4012 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4013 op->addr.mem.seg = VCPU_SREG_ES;
4020 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4021 fetch_register_operand(op);
4025 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4028 rc = decode_imm(ctxt, op, 1, true);
4035 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4038 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4041 ctxt->memop.bytes = 1;
4042 if (ctxt->memop.type == OP_REG) {
4043 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4044 fetch_register_operand(&ctxt->memop);
4048 ctxt->memop.bytes = 2;
4051 ctxt->memop.bytes = 4;
4054 rc = decode_imm(ctxt, op, 2, false);
4057 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4061 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4063 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4064 op->addr.mem.seg = seg_override(ctxt);
4070 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4072 register_address(ctxt,
4073 reg_read(ctxt, VCPU_REGS_RBX) +
4074 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4075 op->addr.mem.seg = seg_override(ctxt);
4080 op->addr.mem.ea = ctxt->_eip;
4081 op->bytes = ctxt->op_bytes + 2;
4082 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4085 ctxt->memop.bytes = ctxt->op_bytes + 2;
4088 op->val = VCPU_SREG_ES;
4091 op->val = VCPU_SREG_CS;
4094 op->val = VCPU_SREG_SS;
4097 op->val = VCPU_SREG_DS;
4100 op->val = VCPU_SREG_FS;
4103 op->val = VCPU_SREG_GS;
4106 /* Special instructions do their own operand decoding. */
4108 op->type = OP_NONE; /* Disable writeback. */
4116 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4118 int rc = X86EMUL_CONTINUE;
4119 int mode = ctxt->mode;
4120 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4121 bool op_prefix = false;
4122 struct opcode opcode;
4124 ctxt->memop.type = OP_NONE;
4125 ctxt->memopp = NULL;
4126 ctxt->_eip = ctxt->eip;
4127 ctxt->fetch.start = ctxt->_eip;
4128 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4130 memcpy(ctxt->fetch.data, insn, insn_len);
4133 case X86EMUL_MODE_REAL:
4134 case X86EMUL_MODE_VM86:
4135 case X86EMUL_MODE_PROT16:
4136 def_op_bytes = def_ad_bytes = 2;
4138 case X86EMUL_MODE_PROT32:
4139 def_op_bytes = def_ad_bytes = 4;
4141 #ifdef CONFIG_X86_64
4142 case X86EMUL_MODE_PROT64:
4148 return EMULATION_FAILED;
4151 ctxt->op_bytes = def_op_bytes;
4152 ctxt->ad_bytes = def_ad_bytes;
4154 /* Legacy prefixes. */
4156 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4157 case 0x66: /* operand-size override */
4159 /* switch between 2/4 bytes */
4160 ctxt->op_bytes = def_op_bytes ^ 6;
4162 case 0x67: /* address-size override */
4163 if (mode == X86EMUL_MODE_PROT64)
4164 /* switch between 4/8 bytes */
4165 ctxt->ad_bytes = def_ad_bytes ^ 12;
4167 /* switch between 2/4 bytes */
4168 ctxt->ad_bytes = def_ad_bytes ^ 6;
4170 case 0x26: /* ES override */
4171 case 0x2e: /* CS override */
4172 case 0x36: /* SS override */
4173 case 0x3e: /* DS override */
4174 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4176 case 0x64: /* FS override */
4177 case 0x65: /* GS override */
4178 set_seg_override(ctxt, ctxt->b & 7);
4180 case 0x40 ... 0x4f: /* REX */
4181 if (mode != X86EMUL_MODE_PROT64)
4183 ctxt->rex_prefix = ctxt->b;
4185 case 0xf0: /* LOCK */
4186 ctxt->lock_prefix = 1;
4188 case 0xf2: /* REPNE/REPNZ */
4189 case 0xf3: /* REP/REPE/REPZ */
4190 ctxt->rep_prefix = ctxt->b;
4196 /* Any legacy prefix after a REX prefix nullifies its effect. */
4198 ctxt->rex_prefix = 0;
4204 if (ctxt->rex_prefix & 8)
4205 ctxt->op_bytes = 8; /* REX.W */
4207 /* Opcode byte(s). */
4208 opcode = opcode_table[ctxt->b];
4209 /* Two-byte opcode? */
4210 if (ctxt->b == 0x0f) {
4212 ctxt->b = insn_fetch(u8, ctxt);
4213 opcode = twobyte_table[ctxt->b];
4215 ctxt->d = opcode.flags;
4217 if (ctxt->d & ModRM)
4218 ctxt->modrm = insn_fetch(u8, ctxt);
4220 while (ctxt->d & GroupMask) {
4221 switch (ctxt->d & GroupMask) {
4223 goffset = (ctxt->modrm >> 3) & 7;
4224 opcode = opcode.u.group[goffset];
4227 goffset = (ctxt->modrm >> 3) & 7;
4228 if ((ctxt->modrm >> 6) == 3)
4229 opcode = opcode.u.gdual->mod3[goffset];
4231 opcode = opcode.u.gdual->mod012[goffset];
4234 goffset = ctxt->modrm & 7;
4235 opcode = opcode.u.group[goffset];
4238 if (ctxt->rep_prefix && op_prefix)
4239 return EMULATION_FAILED;
4240 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4241 switch (simd_prefix) {
4242 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4243 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4244 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4245 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4249 if (ctxt->modrm > 0xbf)
4250 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4252 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4255 return EMULATION_FAILED;
4258 ctxt->d &= ~(u64)GroupMask;
4259 ctxt->d |= opcode.flags;
4262 ctxt->execute = opcode.u.execute;
4263 ctxt->check_perm = opcode.check_perm;
4264 ctxt->intercept = opcode.intercept;
4267 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4268 return EMULATION_FAILED;
4270 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4271 return EMULATION_FAILED;
4273 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4276 if (ctxt->d & Op3264) {
4277 if (mode == X86EMUL_MODE_PROT64)
4284 ctxt->op_bytes = 16;
4285 else if (ctxt->d & Mmx)
4288 /* ModRM and SIB bytes. */
4289 if (ctxt->d & ModRM) {
4290 rc = decode_modrm(ctxt, &ctxt->memop);
4291 if (!ctxt->has_seg_override)
4292 set_seg_override(ctxt, ctxt->modrm_seg);
4293 } else if (ctxt->d & MemAbs)
4294 rc = decode_abs(ctxt, &ctxt->memop);
4295 if (rc != X86EMUL_CONTINUE)
4298 if (!ctxt->has_seg_override)
4299 set_seg_override(ctxt, VCPU_SREG_DS);
4301 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4303 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4304 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4307 * Decode and fetch the source operand: register, memory
4310 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4311 if (rc != X86EMUL_CONTINUE)
4315 * Decode and fetch the second source operand: register, memory
4318 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4319 if (rc != X86EMUL_CONTINUE)
4322 /* Decode and fetch the destination operand: register or memory. */
4323 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4326 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4327 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4329 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4332 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4334 return ctxt->d & PageTable;
4337 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4339 /* The second termination condition only applies for REPE
4340 * and REPNE. Test if the repeat string operation prefix is
4341 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4342 * corresponding termination condition according to:
4343 * - if REPE/REPZ and ZF = 0 then done
4344 * - if REPNE/REPNZ and ZF = 1 then done
4346 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4347 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4348 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4349 ((ctxt->eflags & EFLG_ZF) == 0))
4350 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4351 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4357 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4361 ctxt->ops->get_fpu(ctxt);
4362 asm volatile("1: fwait \n\t"
4364 ".pushsection .fixup,\"ax\" \n\t"
4366 "movb $1, %[fault] \n\t"
4369 _ASM_EXTABLE(1b, 3b)
4370 : [fault]"+qm"(fault));
4371 ctxt->ops->put_fpu(ctxt);
4373 if (unlikely(fault))
4374 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4376 return X86EMUL_CONTINUE;
4379 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4382 if (op->type == OP_MM)
4383 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4386 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4388 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4389 if (!(ctxt->d & ByteOp))
4390 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4391 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4392 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4394 : "c"(ctxt->src2.val));
4395 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4396 if (!fop) /* exception is returned in fop variable */
4397 return emulate_de(ctxt);
4398 return X86EMUL_CONTINUE;
4401 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4403 const struct x86_emulate_ops *ops = ctxt->ops;
4404 int rc = X86EMUL_CONTINUE;
4405 int saved_dst_type = ctxt->dst.type;
4407 ctxt->mem_read.pos = 0;
4409 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4410 (ctxt->d & Undefined)) {
4411 rc = emulate_ud(ctxt);
4415 /* LOCK prefix is allowed only with some instructions */
4416 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4417 rc = emulate_ud(ctxt);
4421 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4422 rc = emulate_ud(ctxt);
4426 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4427 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4428 rc = emulate_ud(ctxt);
4432 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4433 rc = emulate_nm(ctxt);
4437 if (ctxt->d & Mmx) {
4438 rc = flush_pending_x87_faults(ctxt);
4439 if (rc != X86EMUL_CONTINUE)
4442 * Now that we know the fpu is exception safe, we can fetch
4445 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4446 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4447 if (!(ctxt->d & Mov))
4448 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4451 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4452 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4453 X86_ICPT_PRE_EXCEPT);
4454 if (rc != X86EMUL_CONTINUE)
4458 /* Privileged instruction can be executed only in CPL=0 */
4459 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4460 rc = emulate_gp(ctxt, 0);
4464 /* Instruction can only be executed in protected mode */
4465 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4466 rc = emulate_ud(ctxt);
4470 /* Do instruction specific permission checks */
4471 if (ctxt->check_perm) {
4472 rc = ctxt->check_perm(ctxt);
4473 if (rc != X86EMUL_CONTINUE)
4477 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4478 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4479 X86_ICPT_POST_EXCEPT);
4480 if (rc != X86EMUL_CONTINUE)
4484 if (ctxt->rep_prefix && (ctxt->d & String)) {
4485 /* All REP prefixes have the same first termination condition */
4486 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4487 ctxt->eip = ctxt->_eip;
4492 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4493 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4494 ctxt->src.valptr, ctxt->src.bytes);
4495 if (rc != X86EMUL_CONTINUE)
4497 ctxt->src.orig_val64 = ctxt->src.val64;
4500 if (ctxt->src2.type == OP_MEM) {
4501 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4502 &ctxt->src2.val, ctxt->src2.bytes);
4503 if (rc != X86EMUL_CONTINUE)
4507 if ((ctxt->d & DstMask) == ImplicitOps)
4511 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4512 /* optimisation - avoid slow emulated read if Mov */
4513 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4514 &ctxt->dst.val, ctxt->dst.bytes);
4515 if (rc != X86EMUL_CONTINUE)
4518 ctxt->dst.orig_val = ctxt->dst.val;
4522 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4523 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4524 X86_ICPT_POST_MEMACCESS);
4525 if (rc != X86EMUL_CONTINUE)
4529 if (ctxt->execute) {
4530 if (ctxt->d & Fastop) {
4531 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4532 rc = fastop(ctxt, fop);
4533 if (rc != X86EMUL_CONTINUE)
4537 rc = ctxt->execute(ctxt);
4538 if (rc != X86EMUL_CONTINUE)
4547 case 0x63: /* movsxd */
4548 if (ctxt->mode != X86EMUL_MODE_PROT64)
4549 goto cannot_emulate;
4550 ctxt->dst.val = (s32) ctxt->src.val;
4552 case 0x70 ... 0x7f: /* jcc (short) */
4553 if (test_cc(ctxt->b, ctxt->eflags))
4554 jmp_rel(ctxt, ctxt->src.val);
4556 case 0x8d: /* lea r16/r32, m */
4557 ctxt->dst.val = ctxt->src.addr.mem.ea;
4559 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4560 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4564 case 0x98: /* cbw/cwde/cdqe */
4565 switch (ctxt->op_bytes) {
4566 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4567 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4568 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4571 case 0xcc: /* int3 */
4572 rc = emulate_int(ctxt, 3);
4574 case 0xcd: /* int n */
4575 rc = emulate_int(ctxt, ctxt->src.val);
4577 case 0xce: /* into */
4578 if (ctxt->eflags & EFLG_OF)
4579 rc = emulate_int(ctxt, 4);
4581 case 0xe9: /* jmp rel */
4582 case 0xeb: /* jmp rel short */
4583 jmp_rel(ctxt, ctxt->src.val);
4584 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4586 case 0xf4: /* hlt */
4587 ctxt->ops->halt(ctxt);
4589 case 0xf5: /* cmc */
4590 /* complement carry flag from eflags reg */
4591 ctxt->eflags ^= EFLG_CF;
4593 case 0xf8: /* clc */
4594 ctxt->eflags &= ~EFLG_CF;
4596 case 0xf9: /* stc */
4597 ctxt->eflags |= EFLG_CF;
4599 case 0xfc: /* cld */
4600 ctxt->eflags &= ~EFLG_DF;
4602 case 0xfd: /* std */
4603 ctxt->eflags |= EFLG_DF;
4606 goto cannot_emulate;
4609 if (rc != X86EMUL_CONTINUE)
4613 if (!(ctxt->d & NoWrite)) {
4614 rc = writeback(ctxt, &ctxt->dst);
4615 if (rc != X86EMUL_CONTINUE)
4618 if (ctxt->d & SrcWrite) {
4619 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4620 rc = writeback(ctxt, &ctxt->src);
4621 if (rc != X86EMUL_CONTINUE)
4626 * restore dst type in case the decoding will be reused
4627 * (happens for string instruction )
4629 ctxt->dst.type = saved_dst_type;
4631 if ((ctxt->d & SrcMask) == SrcSI)
4632 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4634 if ((ctxt->d & DstMask) == DstDI)
4635 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4637 if (ctxt->rep_prefix && (ctxt->d & String)) {
4639 struct read_cache *r = &ctxt->io_read;
4640 if ((ctxt->d & SrcMask) == SrcSI)
4641 count = ctxt->src.count;
4643 count = ctxt->dst.count;
4644 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4647 if (!string_insn_completed(ctxt)) {
4649 * Re-enter guest when pio read ahead buffer is empty
4650 * or, if it is not used, after each 1024 iteration.
4652 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4653 (r->end == 0 || r->end != r->pos)) {
4655 * Reset read cache. Usually happens before
4656 * decode, but since instruction is restarted
4657 * we have to do it here.
4659 ctxt->mem_read.end = 0;
4660 writeback_registers(ctxt);
4661 return EMULATION_RESTART;
4663 goto done; /* skip rip writeback */
4667 ctxt->eip = ctxt->_eip;
4670 if (rc == X86EMUL_PROPAGATE_FAULT)
4671 ctxt->have_exception = true;
4672 if (rc == X86EMUL_INTERCEPTED)
4673 return EMULATION_INTERCEPTED;
4675 if (rc == X86EMUL_CONTINUE)
4676 writeback_registers(ctxt);
4678 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4682 case 0x09: /* wbinvd */
4683 (ctxt->ops->wbinvd)(ctxt);
4685 case 0x08: /* invd */
4686 case 0x0d: /* GrpP (prefetch) */
4687 case 0x18: /* Grp16 (prefetch/nop) */
4688 case 0x1f: /* nop */
4690 case 0x20: /* mov cr, reg */
4691 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4693 case 0x21: /* mov from dr to reg */
4694 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4696 case 0x40 ... 0x4f: /* cmov */
4697 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4698 if (!test_cc(ctxt->b, ctxt->eflags))
4699 ctxt->dst.type = OP_NONE; /* no writeback */
4701 case 0x80 ... 0x8f: /* jnz rel, etc*/
4702 if (test_cc(ctxt->b, ctxt->eflags))
4703 jmp_rel(ctxt, ctxt->src.val);
4705 case 0x90 ... 0x9f: /* setcc r/m8 */
4706 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4708 case 0xae: /* clflush */
4710 case 0xb6 ... 0xb7: /* movzx */
4711 ctxt->dst.bytes = ctxt->op_bytes;
4712 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4713 : (u16) ctxt->src.val;
4715 case 0xbe ... 0xbf: /* movsx */
4716 ctxt->dst.bytes = ctxt->op_bytes;
4717 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4718 (s16) ctxt->src.val;
4720 case 0xc3: /* movnti */
4721 ctxt->dst.bytes = ctxt->op_bytes;
4722 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4723 (u64) ctxt->src.val;
4726 goto cannot_emulate;
4729 if (rc != X86EMUL_CONTINUE)
4735 return EMULATION_FAILED;
4738 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4740 invalidate_registers(ctxt);
4743 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4745 writeback_registers(ctxt);