1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define GroupMask (7<<14) /* Opcode uses one of the group mechanisms */
77 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
78 #define GroupDual (2<<14) /* Alternate decoding of mod == 3 */
79 #define Prefix (3<<14) /* Instruction varies with 66/f2/f3 prefix */
80 #define RMExt (4<<14) /* Opcode extension in ModRM r/m if mod == 3 */
81 #define Sse (1<<17) /* SSE Vector instruction */
83 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
84 #define VendorSpecific (1<<22) /* Vendor specific instruction */
85 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
86 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Imm (4<<29)
97 #define Src2Mask (7<<29)
100 #define X3(x...) X2(x), x
101 #define X4(x...) X2(x), X2(x)
102 #define X5(x...) X4(x), x
103 #define X6(x...) X4(x), X2(x)
104 #define X7(x...) X4(x), X3(x)
105 #define X8(x...) X4(x), X4(x)
106 #define X16(x...) X8(x), X8(x)
112 int (*execute)(struct x86_emulate_ctxt *ctxt);
113 struct opcode *group;
114 struct group_dual *gdual;
115 struct gprefix *gprefix;
117 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
121 struct opcode mod012[8];
122 struct opcode mod3[8];
126 struct opcode pfx_no;
127 struct opcode pfx_66;
128 struct opcode pfx_f2;
129 struct opcode pfx_f3;
132 /* EFLAGS bit definitions. */
133 #define EFLG_ID (1<<21)
134 #define EFLG_VIP (1<<20)
135 #define EFLG_VIF (1<<19)
136 #define EFLG_AC (1<<18)
137 #define EFLG_VM (1<<17)
138 #define EFLG_RF (1<<16)
139 #define EFLG_IOPL (3<<12)
140 #define EFLG_NT (1<<14)
141 #define EFLG_OF (1<<11)
142 #define EFLG_DF (1<<10)
143 #define EFLG_IF (1<<9)
144 #define EFLG_TF (1<<8)
145 #define EFLG_SF (1<<7)
146 #define EFLG_ZF (1<<6)
147 #define EFLG_AF (1<<4)
148 #define EFLG_PF (1<<2)
149 #define EFLG_CF (1<<0)
151 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
152 #define EFLG_RESERVED_ONE_MASK 2
155 * Instruction emulation:
156 * Most instructions are emulated directly via a fragment of inline assembly
157 * code. This allows us to save/restore EFLAGS and thus very easily pick up
158 * any modified flags.
161 #if defined(CONFIG_X86_64)
162 #define _LO32 "k" /* force 32-bit operand */
163 #define _STK "%%rsp" /* stack pointer */
164 #elif defined(__i386__)
165 #define _LO32 "" /* force 32-bit operand */
166 #define _STK "%%esp" /* stack pointer */
170 * These EFLAGS bits are restored from saved value during emulation, and
171 * any changes are written back to the saved value after emulation.
173 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
175 /* Before executing instruction: restore necessary bits in EFLAGS. */
176 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
177 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
178 "movl %"_sav",%"_LO32 _tmp"; " \
181 "movl %"_msk",%"_LO32 _tmp"; " \
182 "andl %"_LO32 _tmp",("_STK"); " \
184 "notl %"_LO32 _tmp"; " \
185 "andl %"_LO32 _tmp",("_STK"); " \
186 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
188 "orl %"_LO32 _tmp",("_STK"); " \
192 /* After executing instruction: write-back necessary bits in EFLAGS. */
193 #define _POST_EFLAGS(_sav, _msk, _tmp) \
194 /* _sav |= EFLAGS & _msk; */ \
197 "andl %"_msk",%"_LO32 _tmp"; " \
198 "orl %"_LO32 _tmp",%"_sav"; "
206 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
208 __asm__ __volatile__ ( \
209 _PRE_EFLAGS("0", "4", "2") \
210 _op _suffix " %"_x"3,%1; " \
211 _POST_EFLAGS("0", "4", "2") \
212 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
214 : _y ((_src).val), "i" (EFLAGS_MASK)); \
218 /* Raw emulation: instruction has two explicit operands. */
219 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
221 unsigned long _tmp; \
223 switch ((_dst).bytes) { \
225 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
228 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
231 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
236 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
238 unsigned long _tmp; \
239 switch ((_dst).bytes) { \
241 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
244 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
245 _wx, _wy, _lx, _ly, _qx, _qy); \
250 /* Source operand is byte-sized and may be restricted to just %cl. */
251 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
252 __emulate_2op(_op, _src, _dst, _eflags, \
253 "b", "c", "b", "c", "b", "c", "b", "c")
255 /* Source operand is byte, word, long or quad sized. */
256 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
257 __emulate_2op(_op, _src, _dst, _eflags, \
258 "b", "q", "w", "r", _LO32, "r", "", "r")
260 /* Source operand is word, long or quad sized. */
261 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
262 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
263 "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (_cl).val; \
270 _type _srcv = (_src).val; \
271 _type _dstv = (_dst).val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (_cl).val = (unsigned long) _clv; \
282 (_src).val = (unsigned long) _srcv; \
283 (_dst).val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
288 switch ((_dst).bytes) { \
290 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
291 "w", unsigned short); \
294 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
295 "l", unsigned int); \
298 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
299 "q", unsigned long)); \
304 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
306 unsigned long _tmp; \
308 __asm__ __volatile__ ( \
309 _PRE_EFLAGS("0", "3", "2") \
310 _op _suffix " %1; " \
311 _POST_EFLAGS("0", "3", "2") \
312 : "=m" (_eflags), "+m" ((_dst).val), \
314 : "i" (EFLAGS_MASK)); \
317 /* Instruction has only one explicit operand (no source operand). */
318 #define emulate_1op(_op, _dst, _eflags) \
320 switch ((_dst).bytes) { \
321 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
322 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
323 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
324 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
328 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
330 unsigned long _tmp; \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "1") \
334 _op _suffix " %5; " \
335 _POST_EFLAGS("0", "4", "1") \
336 : "=m" (_eflags), "=&r" (_tmp), \
337 "+a" (_rax), "+d" (_rdx) \
338 : "i" (EFLAGS_MASK), "m" ((_src).val), \
339 "a" (_rax), "d" (_rdx)); \
342 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
344 unsigned long _tmp; \
346 __asm__ __volatile__ ( \
347 _PRE_EFLAGS("0", "5", "1") \
349 _op _suffix " %6; " \
351 _POST_EFLAGS("0", "5", "1") \
352 ".pushsection .fixup,\"ax\" \n\t" \
353 "3: movb $1, %4 \n\t" \
356 _ASM_EXTABLE(1b, 3b) \
357 : "=m" (_eflags), "=&r" (_tmp), \
358 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
359 : "i" (EFLAGS_MASK), "m" ((_src).val), \
360 "a" (_rax), "d" (_rdx)); \
363 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
364 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
366 switch((_src).bytes) { \
368 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
372 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
376 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
380 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
386 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
388 switch((_src).bytes) { \
390 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
391 _eflags, "b", _ex); \
394 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
395 _eflags, "w", _ex); \
398 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
399 _eflags, "l", _ex); \
402 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
403 _eflags, "q", _ex)); \
408 /* Fetch next part of the instruction being emulated. */
409 #define insn_fetch(_type, _size, _eip) \
410 ({ unsigned long _x; \
411 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
412 if (rc != X86EMUL_CONTINUE) \
418 #define insn_fetch_arr(_arr, _size, _eip) \
419 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
420 if (rc != X86EMUL_CONTINUE) \
425 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
426 enum x86_intercept intercept,
427 enum x86_intercept_stage stage)
429 struct x86_instruction_info info = {
430 .intercept = intercept,
431 .rep_prefix = ctxt->decode.rep_prefix,
432 .modrm_mod = ctxt->decode.modrm_mod,
433 .modrm_reg = ctxt->decode.modrm_reg,
434 .modrm_rm = ctxt->decode.modrm_rm,
435 .src_val = ctxt->decode.src.val64,
436 .src_bytes = ctxt->decode.src.bytes,
437 .dst_bytes = ctxt->decode.dst.bytes,
438 .ad_bytes = ctxt->decode.ad_bytes,
439 .next_rip = ctxt->eip,
442 return ctxt->ops->intercept(ctxt, &info, stage);
445 static inline unsigned long ad_mask(struct decode_cache *c)
447 return (1UL << (c->ad_bytes << 3)) - 1;
450 /* Access/update address held in a register, based on addressing mode. */
451 static inline unsigned long
452 address_mask(struct decode_cache *c, unsigned long reg)
454 if (c->ad_bytes == sizeof(unsigned long))
457 return reg & ad_mask(c);
460 static inline unsigned long
461 register_address(struct decode_cache *c, unsigned long reg)
463 return address_mask(c, reg);
467 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
469 if (c->ad_bytes == sizeof(unsigned long))
472 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
475 static inline void jmp_rel(struct decode_cache *c, int rel)
477 register_address_increment(c, &c->eip, rel);
480 static u32 desc_limit_scaled(struct desc_struct *desc)
482 u32 limit = get_desc_limit(desc);
484 return desc->g ? (limit << 12) | 0xfff : limit;
487 static void set_seg_override(struct decode_cache *c, int seg)
489 c->has_seg_override = true;
490 c->seg_override = seg;
493 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
494 struct x86_emulate_ops *ops, int seg)
496 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
499 return ops->get_cached_segment_base(ctxt, seg);
502 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
503 struct decode_cache *c)
505 if (!c->has_seg_override)
508 return c->seg_override;
511 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
512 u32 error, bool valid)
514 ctxt->exception.vector = vec;
515 ctxt->exception.error_code = error;
516 ctxt->exception.error_code_valid = valid;
517 return X86EMUL_PROPAGATE_FAULT;
520 static int emulate_db(struct x86_emulate_ctxt *ctxt)
522 return emulate_exception(ctxt, DB_VECTOR, 0, false);
525 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
527 return emulate_exception(ctxt, GP_VECTOR, err, true);
530 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
532 return emulate_exception(ctxt, SS_VECTOR, err, true);
535 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
537 return emulate_exception(ctxt, UD_VECTOR, 0, false);
540 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
542 return emulate_exception(ctxt, TS_VECTOR, err, true);
545 static int emulate_de(struct x86_emulate_ctxt *ctxt)
547 return emulate_exception(ctxt, DE_VECTOR, 0, false);
550 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
552 return emulate_exception(ctxt, NM_VECTOR, 0, false);
555 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
558 struct desc_struct desc;
560 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
564 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
569 struct desc_struct desc;
571 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
572 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
575 static int __linearize(struct x86_emulate_ctxt *ctxt,
576 struct segmented_address addr,
577 unsigned size, bool write, bool fetch,
580 struct decode_cache *c = &ctxt->decode;
581 struct desc_struct desc;
588 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
589 switch (ctxt->mode) {
590 case X86EMUL_MODE_REAL:
592 case X86EMUL_MODE_PROT64:
593 if (((signed long)la << 16) >> 16 != la)
594 return emulate_gp(ctxt, 0);
597 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
601 /* code segment or read-only data segment */
602 if (((desc.type & 8) || !(desc.type & 2)) && write)
604 /* unreadable code segment */
605 if (!fetch && (desc.type & 8) && !(desc.type & 2))
607 lim = desc_limit_scaled(&desc);
608 if ((desc.type & 8) || !(desc.type & 4)) {
609 /* expand-up segment */
610 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
613 /* exapand-down segment */
614 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
616 lim = desc.d ? 0xffffffff : 0xffff;
617 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
620 cpl = ctxt->ops->cpl(ctxt);
623 if (!(desc.type & 8)) {
627 } else if ((desc.type & 8) && !(desc.type & 4)) {
628 /* nonconforming code segment */
631 } else if ((desc.type & 8) && (desc.type & 4)) {
632 /* conforming code segment */
638 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
641 return X86EMUL_CONTINUE;
643 if (addr.seg == VCPU_SREG_SS)
644 return emulate_ss(ctxt, addr.seg);
646 return emulate_gp(ctxt, addr.seg);
649 static int linearize(struct x86_emulate_ctxt *ctxt,
650 struct segmented_address addr,
651 unsigned size, bool write,
654 return __linearize(ctxt, addr, size, write, false, linear);
658 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
659 struct segmented_address addr,
666 rc = linearize(ctxt, addr, size, false, &linear);
667 if (rc != X86EMUL_CONTINUE)
669 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
672 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
673 struct x86_emulate_ops *ops,
674 unsigned long eip, u8 *dest)
676 struct fetch_cache *fc = &ctxt->decode.fetch;
680 if (eip == fc->end) {
681 unsigned long linear;
682 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
683 cur_size = fc->end - fc->start;
684 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
685 rc = __linearize(ctxt, addr, size, false, true, &linear);
686 if (rc != X86EMUL_CONTINUE)
688 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
689 size, &ctxt->exception);
690 if (rc != X86EMUL_CONTINUE)
694 *dest = fc->data[eip - fc->start];
695 return X86EMUL_CONTINUE;
698 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
699 struct x86_emulate_ops *ops,
700 unsigned long eip, void *dest, unsigned size)
704 /* x86 instructions are limited to 15 bytes. */
705 if (eip + size - ctxt->eip > 15)
706 return X86EMUL_UNHANDLEABLE;
708 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
709 if (rc != X86EMUL_CONTINUE)
712 return X86EMUL_CONTINUE;
716 * Given the 'reg' portion of a ModRM byte, and a register block, return a
717 * pointer into the block that addresses the relevant register.
718 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
720 static void *decode_register(u8 modrm_reg, unsigned long *regs,
725 p = ®s[modrm_reg];
726 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
727 p = (unsigned char *)®s[modrm_reg & 3] + 1;
731 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
732 struct segmented_address addr,
733 u16 *size, unsigned long *address, int op_bytes)
740 rc = segmented_read_std(ctxt, addr, size, 2);
741 if (rc != X86EMUL_CONTINUE)
744 rc = segmented_read_std(ctxt, addr, address, op_bytes);
748 static int test_cc(unsigned int condition, unsigned int flags)
752 switch ((condition & 15) >> 1) {
754 rc |= (flags & EFLG_OF);
756 case 1: /* b/c/nae */
757 rc |= (flags & EFLG_CF);
760 rc |= (flags & EFLG_ZF);
763 rc |= (flags & (EFLG_CF|EFLG_ZF));
766 rc |= (flags & EFLG_SF);
769 rc |= (flags & EFLG_PF);
772 rc |= (flags & EFLG_ZF);
775 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
779 /* Odd condition identifiers (lsb == 1) have inverted sense. */
780 return (!!rc ^ (condition & 1));
783 static void fetch_register_operand(struct operand *op)
787 op->val = *(u8 *)op->addr.reg;
790 op->val = *(u16 *)op->addr.reg;
793 op->val = *(u32 *)op->addr.reg;
796 op->val = *(u64 *)op->addr.reg;
801 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
803 ctxt->ops->get_fpu(ctxt);
805 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
806 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
807 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
808 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
809 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
810 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
811 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
812 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
814 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
815 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
816 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
817 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
818 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
819 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
820 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
821 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
825 ctxt->ops->put_fpu(ctxt);
828 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
831 ctxt->ops->get_fpu(ctxt);
833 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
834 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
835 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
836 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
837 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
838 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
839 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
840 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
842 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
843 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
844 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
845 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
846 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
847 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
848 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
849 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
853 ctxt->ops->put_fpu(ctxt);
856 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
858 struct decode_cache *c,
861 unsigned reg = c->modrm_reg;
862 int highbyte_regs = c->rex_prefix == 0;
865 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
871 read_sse_reg(ctxt, &op->vec_val, reg);
876 if ((c->d & ByteOp) && !inhibit_bytereg) {
877 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
880 op->addr.reg = decode_register(reg, c->regs, 0);
881 op->bytes = c->op_bytes;
883 fetch_register_operand(op);
884 op->orig_val = op->val;
887 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
888 struct x86_emulate_ops *ops,
891 struct decode_cache *c = &ctxt->decode;
893 int index_reg = 0, base_reg = 0, scale;
894 int rc = X86EMUL_CONTINUE;
898 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
899 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
900 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
903 c->modrm = insn_fetch(u8, 1, c->eip);
904 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
905 c->modrm_reg |= (c->modrm & 0x38) >> 3;
906 c->modrm_rm |= (c->modrm & 0x07);
907 c->modrm_seg = VCPU_SREG_DS;
909 if (c->modrm_mod == 3) {
911 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
912 op->addr.reg = decode_register(c->modrm_rm,
913 c->regs, c->d & ByteOp);
917 op->addr.xmm = c->modrm_rm;
918 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
921 fetch_register_operand(op);
927 if (c->ad_bytes == 2) {
928 unsigned bx = c->regs[VCPU_REGS_RBX];
929 unsigned bp = c->regs[VCPU_REGS_RBP];
930 unsigned si = c->regs[VCPU_REGS_RSI];
931 unsigned di = c->regs[VCPU_REGS_RDI];
933 /* 16-bit ModR/M decode. */
934 switch (c->modrm_mod) {
936 if (c->modrm_rm == 6)
937 modrm_ea += insn_fetch(u16, 2, c->eip);
940 modrm_ea += insn_fetch(s8, 1, c->eip);
943 modrm_ea += insn_fetch(u16, 2, c->eip);
946 switch (c->modrm_rm) {
966 if (c->modrm_mod != 0)
973 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
974 (c->modrm_rm == 6 && c->modrm_mod != 0))
975 c->modrm_seg = VCPU_SREG_SS;
976 modrm_ea = (u16)modrm_ea;
978 /* 32/64-bit ModR/M decode. */
979 if ((c->modrm_rm & 7) == 4) {
980 sib = insn_fetch(u8, 1, c->eip);
981 index_reg |= (sib >> 3) & 7;
985 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
986 modrm_ea += insn_fetch(s32, 4, c->eip);
988 modrm_ea += c->regs[base_reg];
990 modrm_ea += c->regs[index_reg] << scale;
991 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
992 if (ctxt->mode == X86EMUL_MODE_PROT64)
995 modrm_ea += c->regs[c->modrm_rm];
996 switch (c->modrm_mod) {
998 if (c->modrm_rm == 5)
999 modrm_ea += insn_fetch(s32, 4, c->eip);
1002 modrm_ea += insn_fetch(s8, 1, c->eip);
1005 modrm_ea += insn_fetch(s32, 4, c->eip);
1009 op->addr.mem.ea = modrm_ea;
1014 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1015 struct x86_emulate_ops *ops,
1018 struct decode_cache *c = &ctxt->decode;
1019 int rc = X86EMUL_CONTINUE;
1022 switch (c->ad_bytes) {
1024 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1027 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1030 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1037 static void fetch_bit_operand(struct decode_cache *c)
1041 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1042 mask = ~(c->dst.bytes * 8 - 1);
1044 if (c->src.bytes == 2)
1045 sv = (s16)c->src.val & (s16)mask;
1046 else if (c->src.bytes == 4)
1047 sv = (s32)c->src.val & (s32)mask;
1049 c->dst.addr.mem.ea += (sv >> 3);
1052 /* only subword offset */
1053 c->src.val &= (c->dst.bytes << 3) - 1;
1056 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1057 struct x86_emulate_ops *ops,
1058 unsigned long addr, void *dest, unsigned size)
1061 struct read_cache *mc = &ctxt->decode.mem_read;
1064 int n = min(size, 8u);
1066 if (mc->pos < mc->end)
1069 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1071 if (rc != X86EMUL_CONTINUE)
1076 memcpy(dest, mc->data + mc->pos, n);
1081 return X86EMUL_CONTINUE;
1084 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1085 struct segmented_address addr,
1092 rc = linearize(ctxt, addr, size, false, &linear);
1093 if (rc != X86EMUL_CONTINUE)
1095 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1098 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1099 struct segmented_address addr,
1106 rc = linearize(ctxt, addr, size, true, &linear);
1107 if (rc != X86EMUL_CONTINUE)
1109 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1113 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1114 struct segmented_address addr,
1115 const void *orig_data, const void *data,
1121 rc = linearize(ctxt, addr, size, true, &linear);
1122 if (rc != X86EMUL_CONTINUE)
1124 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1125 size, &ctxt->exception);
1128 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1129 struct x86_emulate_ops *ops,
1130 unsigned int size, unsigned short port,
1133 struct read_cache *rc = &ctxt->decode.io_read;
1135 if (rc->pos == rc->end) { /* refill pio read ahead */
1136 struct decode_cache *c = &ctxt->decode;
1137 unsigned int in_page, n;
1138 unsigned int count = c->rep_prefix ?
1139 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1140 in_page = (ctxt->eflags & EFLG_DF) ?
1141 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1142 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1143 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1147 rc->pos = rc->end = 0;
1148 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1153 memcpy(dest, rc->data + rc->pos, size);
1158 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1159 struct x86_emulate_ops *ops,
1160 u16 selector, struct desc_ptr *dt)
1162 if (selector & 1 << 2) {
1163 struct desc_struct desc;
1166 memset (dt, 0, sizeof *dt);
1167 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1170 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1171 dt->address = get_desc_base(&desc);
1173 ops->get_gdt(ctxt, dt);
1176 /* allowed just for 8 bytes segments */
1177 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1178 struct x86_emulate_ops *ops,
1179 u16 selector, struct desc_struct *desc)
1182 u16 index = selector >> 3;
1186 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1188 if (dt.size < index * 8 + 7)
1189 return emulate_gp(ctxt, selector & 0xfffc);
1190 addr = dt.address + index * 8;
1191 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1196 /* allowed just for 8 bytes segments */
1197 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1198 struct x86_emulate_ops *ops,
1199 u16 selector, struct desc_struct *desc)
1202 u16 index = selector >> 3;
1206 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1208 if (dt.size < index * 8 + 7)
1209 return emulate_gp(ctxt, selector & 0xfffc);
1211 addr = dt.address + index * 8;
1212 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1217 /* Does not support long mode */
1218 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1219 struct x86_emulate_ops *ops,
1220 u16 selector, int seg)
1222 struct desc_struct seg_desc;
1224 unsigned err_vec = GP_VECTOR;
1226 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1229 memset(&seg_desc, 0, sizeof seg_desc);
1231 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1232 || ctxt->mode == X86EMUL_MODE_REAL) {
1233 /* set real mode segment descriptor */
1234 set_desc_base(&seg_desc, selector << 4);
1235 set_desc_limit(&seg_desc, 0xffff);
1242 /* NULL selector is not valid for TR, CS and SS */
1243 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1247 /* TR should be in GDT only */
1248 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1251 if (null_selector) /* for NULL selector skip all following checks */
1254 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1255 if (ret != X86EMUL_CONTINUE)
1258 err_code = selector & 0xfffc;
1259 err_vec = GP_VECTOR;
1261 /* can't load system descriptor into segment selecor */
1262 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1266 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1272 cpl = ops->cpl(ctxt);
1277 * segment is not a writable data segment or segment
1278 * selector's RPL != CPL or segment selector's RPL != CPL
1280 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1284 if (!(seg_desc.type & 8))
1287 if (seg_desc.type & 4) {
1293 if (rpl > cpl || dpl != cpl)
1296 /* CS(RPL) <- CPL */
1297 selector = (selector & 0xfffc) | cpl;
1300 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1303 case VCPU_SREG_LDTR:
1304 if (seg_desc.s || seg_desc.type != 2)
1307 default: /* DS, ES, FS, or GS */
1309 * segment is not a data or readable code segment or
1310 * ((segment is a data or nonconforming code segment)
1311 * and (both RPL and CPL > DPL))
1313 if ((seg_desc.type & 0xa) == 0x8 ||
1314 (((seg_desc.type & 0xc) != 0xc) &&
1315 (rpl > dpl && cpl > dpl)))
1321 /* mark segment as accessed */
1323 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1324 if (ret != X86EMUL_CONTINUE)
1328 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1329 return X86EMUL_CONTINUE;
1331 emulate_exception(ctxt, err_vec, err_code, true);
1332 return X86EMUL_PROPAGATE_FAULT;
1335 static void write_register_operand(struct operand *op)
1337 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1338 switch (op->bytes) {
1340 *(u8 *)op->addr.reg = (u8)op->val;
1343 *(u16 *)op->addr.reg = (u16)op->val;
1346 *op->addr.reg = (u32)op->val;
1347 break; /* 64b: zero-extend */
1349 *op->addr.reg = op->val;
1354 static int writeback(struct x86_emulate_ctxt *ctxt)
1357 struct decode_cache *c = &ctxt->decode;
1359 switch (c->dst.type) {
1361 write_register_operand(&c->dst);
1365 rc = segmented_cmpxchg(ctxt,
1371 rc = segmented_write(ctxt,
1375 if (rc != X86EMUL_CONTINUE)
1379 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1387 return X86EMUL_CONTINUE;
1390 static int em_push(struct x86_emulate_ctxt *ctxt)
1392 struct decode_cache *c = &ctxt->decode;
1393 struct segmented_address addr;
1395 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1396 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1397 addr.seg = VCPU_SREG_SS;
1399 /* Disable writeback. */
1400 c->dst.type = OP_NONE;
1401 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1404 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1405 void *dest, int len)
1407 struct decode_cache *c = &ctxt->decode;
1409 struct segmented_address addr;
1411 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1412 addr.seg = VCPU_SREG_SS;
1413 rc = segmented_read(ctxt, addr, dest, len);
1414 if (rc != X86EMUL_CONTINUE)
1417 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1421 static int em_pop(struct x86_emulate_ctxt *ctxt)
1423 struct decode_cache *c = &ctxt->decode;
1425 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1428 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1429 struct x86_emulate_ops *ops,
1430 void *dest, int len)
1433 unsigned long val, change_mask;
1434 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1435 int cpl = ops->cpl(ctxt);
1437 rc = emulate_pop(ctxt, &val, len);
1438 if (rc != X86EMUL_CONTINUE)
1441 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1442 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1444 switch(ctxt->mode) {
1445 case X86EMUL_MODE_PROT64:
1446 case X86EMUL_MODE_PROT32:
1447 case X86EMUL_MODE_PROT16:
1449 change_mask |= EFLG_IOPL;
1451 change_mask |= EFLG_IF;
1453 case X86EMUL_MODE_VM86:
1455 return emulate_gp(ctxt, 0);
1456 change_mask |= EFLG_IF;
1458 default: /* real mode */
1459 change_mask |= (EFLG_IOPL | EFLG_IF);
1463 *(unsigned long *)dest =
1464 (ctxt->eflags & ~change_mask) | (val & change_mask);
1469 static int em_popf(struct x86_emulate_ctxt *ctxt)
1471 struct decode_cache *c = &ctxt->decode;
1473 c->dst.type = OP_REG;
1474 c->dst.addr.reg = &ctxt->eflags;
1475 c->dst.bytes = c->op_bytes;
1476 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1479 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1480 struct x86_emulate_ops *ops, int seg)
1482 struct decode_cache *c = &ctxt->decode;
1484 c->src.val = get_segment_selector(ctxt, seg);
1486 return em_push(ctxt);
1489 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1490 struct x86_emulate_ops *ops, int seg)
1492 struct decode_cache *c = &ctxt->decode;
1493 unsigned long selector;
1496 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1497 if (rc != X86EMUL_CONTINUE)
1500 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1504 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1506 struct decode_cache *c = &ctxt->decode;
1507 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1508 int rc = X86EMUL_CONTINUE;
1509 int reg = VCPU_REGS_RAX;
1511 while (reg <= VCPU_REGS_RDI) {
1512 (reg == VCPU_REGS_RSP) ?
1513 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1516 if (rc != X86EMUL_CONTINUE)
1525 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1527 struct decode_cache *c = &ctxt->decode;
1529 c->src.val = (unsigned long)ctxt->eflags;
1530 return em_push(ctxt);
1533 static int em_popa(struct x86_emulate_ctxt *ctxt)
1535 struct decode_cache *c = &ctxt->decode;
1536 int rc = X86EMUL_CONTINUE;
1537 int reg = VCPU_REGS_RDI;
1539 while (reg >= VCPU_REGS_RAX) {
1540 if (reg == VCPU_REGS_RSP) {
1541 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1546 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1547 if (rc != X86EMUL_CONTINUE)
1554 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1555 struct x86_emulate_ops *ops, int irq)
1557 struct decode_cache *c = &ctxt->decode;
1564 /* TODO: Add limit checks */
1565 c->src.val = ctxt->eflags;
1567 if (rc != X86EMUL_CONTINUE)
1570 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1572 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1574 if (rc != X86EMUL_CONTINUE)
1577 c->src.val = c->eip;
1579 if (rc != X86EMUL_CONTINUE)
1582 ops->get_idt(ctxt, &dt);
1584 eip_addr = dt.address + (irq << 2);
1585 cs_addr = dt.address + (irq << 2) + 2;
1587 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1588 if (rc != X86EMUL_CONTINUE)
1591 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1592 if (rc != X86EMUL_CONTINUE)
1595 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1596 if (rc != X86EMUL_CONTINUE)
1604 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1605 struct x86_emulate_ops *ops, int irq)
1607 switch(ctxt->mode) {
1608 case X86EMUL_MODE_REAL:
1609 return emulate_int_real(ctxt, ops, irq);
1610 case X86EMUL_MODE_VM86:
1611 case X86EMUL_MODE_PROT16:
1612 case X86EMUL_MODE_PROT32:
1613 case X86EMUL_MODE_PROT64:
1615 /* Protected mode interrupts unimplemented yet */
1616 return X86EMUL_UNHANDLEABLE;
1620 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1621 struct x86_emulate_ops *ops)
1623 struct decode_cache *c = &ctxt->decode;
1624 int rc = X86EMUL_CONTINUE;
1625 unsigned long temp_eip = 0;
1626 unsigned long temp_eflags = 0;
1627 unsigned long cs = 0;
1628 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1629 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1630 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1631 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1633 /* TODO: Add stack limit check */
1635 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1637 if (rc != X86EMUL_CONTINUE)
1640 if (temp_eip & ~0xffff)
1641 return emulate_gp(ctxt, 0);
1643 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1645 if (rc != X86EMUL_CONTINUE)
1648 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1650 if (rc != X86EMUL_CONTINUE)
1653 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1655 if (rc != X86EMUL_CONTINUE)
1661 if (c->op_bytes == 4)
1662 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1663 else if (c->op_bytes == 2) {
1664 ctxt->eflags &= ~0xffff;
1665 ctxt->eflags |= temp_eflags;
1668 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1669 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1674 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1675 struct x86_emulate_ops* ops)
1677 switch(ctxt->mode) {
1678 case X86EMUL_MODE_REAL:
1679 return emulate_iret_real(ctxt, ops);
1680 case X86EMUL_MODE_VM86:
1681 case X86EMUL_MODE_PROT16:
1682 case X86EMUL_MODE_PROT32:
1683 case X86EMUL_MODE_PROT64:
1685 /* iret from protected mode unimplemented yet */
1686 return X86EMUL_UNHANDLEABLE;
1690 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1692 struct decode_cache *c = &ctxt->decode;
1696 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1698 rc = load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS);
1699 if (rc != X86EMUL_CONTINUE)
1703 memcpy(&c->eip, c->src.valptr, c->op_bytes);
1704 return X86EMUL_CONTINUE;
1707 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1709 struct decode_cache *c = &ctxt->decode;
1711 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1714 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1716 struct decode_cache *c = &ctxt->decode;
1717 switch (c->modrm_reg) {
1719 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1722 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1725 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1728 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1730 case 4: /* sal/shl */
1731 case 6: /* sal/shl */
1732 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1735 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1738 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1741 return X86EMUL_CONTINUE;
1744 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1746 struct decode_cache *c = &ctxt->decode;
1747 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1748 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1751 switch (c->modrm_reg) {
1752 case 0 ... 1: /* test */
1753 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1756 c->dst.val = ~c->dst.val;
1759 emulate_1op("neg", c->dst, ctxt->eflags);
1762 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1765 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1768 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1772 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1776 return X86EMUL_UNHANDLEABLE;
1779 return emulate_de(ctxt);
1780 return X86EMUL_CONTINUE;
1783 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1785 struct decode_cache *c = &ctxt->decode;
1786 int rc = X86EMUL_CONTINUE;
1788 switch (c->modrm_reg) {
1790 emulate_1op("inc", c->dst, ctxt->eflags);
1793 emulate_1op("dec", c->dst, ctxt->eflags);
1795 case 2: /* call near abs */ {
1798 c->eip = c->src.val;
1799 c->src.val = old_eip;
1803 case 4: /* jmp abs */
1804 c->eip = c->src.val;
1806 case 5: /* jmp far */
1807 rc = em_jmp_far(ctxt);
1816 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1818 struct decode_cache *c = &ctxt->decode;
1819 u64 old = c->dst.orig_val64;
1821 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1822 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1823 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1824 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1825 ctxt->eflags &= ~EFLG_ZF;
1827 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1828 (u32) c->regs[VCPU_REGS_RBX];
1830 ctxt->eflags |= EFLG_ZF;
1832 return X86EMUL_CONTINUE;
1835 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1836 struct x86_emulate_ops *ops)
1838 struct decode_cache *c = &ctxt->decode;
1842 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1843 if (rc != X86EMUL_CONTINUE)
1845 if (c->op_bytes == 4)
1846 c->eip = (u32)c->eip;
1847 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1848 if (rc != X86EMUL_CONTINUE)
1850 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1854 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1855 struct x86_emulate_ops *ops, int seg)
1857 struct decode_cache *c = &ctxt->decode;
1861 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1863 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1864 if (rc != X86EMUL_CONTINUE)
1867 c->dst.val = c->src.val;
1872 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1873 struct x86_emulate_ops *ops, struct desc_struct *cs,
1874 struct desc_struct *ss)
1878 memset(cs, 0, sizeof(struct desc_struct));
1879 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1880 memset(ss, 0, sizeof(struct desc_struct));
1882 cs->l = 0; /* will be adjusted later */
1883 set_desc_base(cs, 0); /* flat segment */
1884 cs->g = 1; /* 4kb granularity */
1885 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1886 cs->type = 0x0b; /* Read, Execute, Accessed */
1888 cs->dpl = 0; /* will be adjusted later */
1892 set_desc_base(ss, 0); /* flat segment */
1893 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1894 ss->g = 1; /* 4kb granularity */
1896 ss->type = 0x03; /* Read/Write, Accessed */
1897 ss->d = 1; /* 32bit stack segment */
1903 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1905 struct decode_cache *c = &ctxt->decode;
1906 struct desc_struct cs, ss;
1911 /* syscall is not available in real mode */
1912 if (ctxt->mode == X86EMUL_MODE_REAL ||
1913 ctxt->mode == X86EMUL_MODE_VM86)
1914 return emulate_ud(ctxt);
1916 ops->get_msr(ctxt, MSR_EFER, &efer);
1917 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1919 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1921 cs_sel = (u16)(msr_data & 0xfffc);
1922 ss_sel = (u16)(msr_data + 8);
1924 if (efer & EFER_LMA) {
1928 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1929 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1931 c->regs[VCPU_REGS_RCX] = c->eip;
1932 if (efer & EFER_LMA) {
1933 #ifdef CONFIG_X86_64
1934 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1937 ctxt->mode == X86EMUL_MODE_PROT64 ?
1938 MSR_LSTAR : MSR_CSTAR, &msr_data);
1941 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1942 ctxt->eflags &= ~(msr_data | EFLG_RF);
1946 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1947 c->eip = (u32)msr_data;
1949 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1952 return X86EMUL_CONTINUE;
1956 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1958 struct decode_cache *c = &ctxt->decode;
1959 struct desc_struct cs, ss;
1964 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1965 /* inject #GP if in real mode */
1966 if (ctxt->mode == X86EMUL_MODE_REAL)
1967 return emulate_gp(ctxt, 0);
1969 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1970 * Therefore, we inject an #UD.
1972 if (ctxt->mode == X86EMUL_MODE_PROT64)
1973 return emulate_ud(ctxt);
1975 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1977 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1978 switch (ctxt->mode) {
1979 case X86EMUL_MODE_PROT32:
1980 if ((msr_data & 0xfffc) == 0x0)
1981 return emulate_gp(ctxt, 0);
1983 case X86EMUL_MODE_PROT64:
1984 if (msr_data == 0x0)
1985 return emulate_gp(ctxt, 0);
1989 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1990 cs_sel = (u16)msr_data;
1991 cs_sel &= ~SELECTOR_RPL_MASK;
1992 ss_sel = cs_sel + 8;
1993 ss_sel &= ~SELECTOR_RPL_MASK;
1994 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1999 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2000 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2002 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2005 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2006 c->regs[VCPU_REGS_RSP] = msr_data;
2008 return X86EMUL_CONTINUE;
2012 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2014 struct decode_cache *c = &ctxt->decode;
2015 struct desc_struct cs, ss;
2020 /* inject #GP if in real mode or Virtual 8086 mode */
2021 if (ctxt->mode == X86EMUL_MODE_REAL ||
2022 ctxt->mode == X86EMUL_MODE_VM86)
2023 return emulate_gp(ctxt, 0);
2025 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2027 if ((c->rex_prefix & 0x8) != 0x0)
2028 usermode = X86EMUL_MODE_PROT64;
2030 usermode = X86EMUL_MODE_PROT32;
2034 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2036 case X86EMUL_MODE_PROT32:
2037 cs_sel = (u16)(msr_data + 16);
2038 if ((msr_data & 0xfffc) == 0x0)
2039 return emulate_gp(ctxt, 0);
2040 ss_sel = (u16)(msr_data + 24);
2042 case X86EMUL_MODE_PROT64:
2043 cs_sel = (u16)(msr_data + 32);
2044 if (msr_data == 0x0)
2045 return emulate_gp(ctxt, 0);
2046 ss_sel = cs_sel + 8;
2051 cs_sel |= SELECTOR_RPL_MASK;
2052 ss_sel |= SELECTOR_RPL_MASK;
2054 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2055 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2057 c->eip = c->regs[VCPU_REGS_RDX];
2058 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2060 return X86EMUL_CONTINUE;
2063 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2064 struct x86_emulate_ops *ops)
2067 if (ctxt->mode == X86EMUL_MODE_REAL)
2069 if (ctxt->mode == X86EMUL_MODE_VM86)
2071 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2072 return ops->cpl(ctxt) > iopl;
2075 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2076 struct x86_emulate_ops *ops,
2079 struct desc_struct tr_seg;
2082 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2083 unsigned mask = (1 << len) - 1;
2086 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2089 if (desc_limit_scaled(&tr_seg) < 103)
2091 base = get_desc_base(&tr_seg);
2092 #ifdef CONFIG_X86_64
2093 base |= ((u64)base3) << 32;
2095 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2096 if (r != X86EMUL_CONTINUE)
2098 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2100 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2101 if (r != X86EMUL_CONTINUE)
2103 if ((perm >> bit_idx) & mask)
2108 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2109 struct x86_emulate_ops *ops,
2115 if (emulator_bad_iopl(ctxt, ops))
2116 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2119 ctxt->perm_ok = true;
2124 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2125 struct x86_emulate_ops *ops,
2126 struct tss_segment_16 *tss)
2128 struct decode_cache *c = &ctxt->decode;
2131 tss->flag = ctxt->eflags;
2132 tss->ax = c->regs[VCPU_REGS_RAX];
2133 tss->cx = c->regs[VCPU_REGS_RCX];
2134 tss->dx = c->regs[VCPU_REGS_RDX];
2135 tss->bx = c->regs[VCPU_REGS_RBX];
2136 tss->sp = c->regs[VCPU_REGS_RSP];
2137 tss->bp = c->regs[VCPU_REGS_RBP];
2138 tss->si = c->regs[VCPU_REGS_RSI];
2139 tss->di = c->regs[VCPU_REGS_RDI];
2141 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2142 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2143 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2144 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2145 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2148 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2149 struct x86_emulate_ops *ops,
2150 struct tss_segment_16 *tss)
2152 struct decode_cache *c = &ctxt->decode;
2156 ctxt->eflags = tss->flag | 2;
2157 c->regs[VCPU_REGS_RAX] = tss->ax;
2158 c->regs[VCPU_REGS_RCX] = tss->cx;
2159 c->regs[VCPU_REGS_RDX] = tss->dx;
2160 c->regs[VCPU_REGS_RBX] = tss->bx;
2161 c->regs[VCPU_REGS_RSP] = tss->sp;
2162 c->regs[VCPU_REGS_RBP] = tss->bp;
2163 c->regs[VCPU_REGS_RSI] = tss->si;
2164 c->regs[VCPU_REGS_RDI] = tss->di;
2167 * SDM says that segment selectors are loaded before segment
2170 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2171 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2172 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2173 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2174 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2177 * Now load segment descriptors. If fault happenes at this stage
2178 * it is handled in a context of new task
2180 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2181 if (ret != X86EMUL_CONTINUE)
2183 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2184 if (ret != X86EMUL_CONTINUE)
2186 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2187 if (ret != X86EMUL_CONTINUE)
2189 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2190 if (ret != X86EMUL_CONTINUE)
2192 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2193 if (ret != X86EMUL_CONTINUE)
2196 return X86EMUL_CONTINUE;
2199 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2200 struct x86_emulate_ops *ops,
2201 u16 tss_selector, u16 old_tss_sel,
2202 ulong old_tss_base, struct desc_struct *new_desc)
2204 struct tss_segment_16 tss_seg;
2206 u32 new_tss_base = get_desc_base(new_desc);
2208 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2210 if (ret != X86EMUL_CONTINUE)
2211 /* FIXME: need to provide precise fault address */
2214 save_state_to_tss16(ctxt, ops, &tss_seg);
2216 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2218 if (ret != X86EMUL_CONTINUE)
2219 /* FIXME: need to provide precise fault address */
2222 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2224 if (ret != X86EMUL_CONTINUE)
2225 /* FIXME: need to provide precise fault address */
2228 if (old_tss_sel != 0xffff) {
2229 tss_seg.prev_task_link = old_tss_sel;
2231 ret = ops->write_std(ctxt, new_tss_base,
2232 &tss_seg.prev_task_link,
2233 sizeof tss_seg.prev_task_link,
2235 if (ret != X86EMUL_CONTINUE)
2236 /* FIXME: need to provide precise fault address */
2240 return load_state_from_tss16(ctxt, ops, &tss_seg);
2243 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2244 struct x86_emulate_ops *ops,
2245 struct tss_segment_32 *tss)
2247 struct decode_cache *c = &ctxt->decode;
2249 tss->cr3 = ops->get_cr(ctxt, 3);
2251 tss->eflags = ctxt->eflags;
2252 tss->eax = c->regs[VCPU_REGS_RAX];
2253 tss->ecx = c->regs[VCPU_REGS_RCX];
2254 tss->edx = c->regs[VCPU_REGS_RDX];
2255 tss->ebx = c->regs[VCPU_REGS_RBX];
2256 tss->esp = c->regs[VCPU_REGS_RSP];
2257 tss->ebp = c->regs[VCPU_REGS_RBP];
2258 tss->esi = c->regs[VCPU_REGS_RSI];
2259 tss->edi = c->regs[VCPU_REGS_RDI];
2261 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2262 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2263 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2264 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2265 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2266 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2267 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2270 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2271 struct x86_emulate_ops *ops,
2272 struct tss_segment_32 *tss)
2274 struct decode_cache *c = &ctxt->decode;
2277 if (ops->set_cr(ctxt, 3, tss->cr3))
2278 return emulate_gp(ctxt, 0);
2280 ctxt->eflags = tss->eflags | 2;
2281 c->regs[VCPU_REGS_RAX] = tss->eax;
2282 c->regs[VCPU_REGS_RCX] = tss->ecx;
2283 c->regs[VCPU_REGS_RDX] = tss->edx;
2284 c->regs[VCPU_REGS_RBX] = tss->ebx;
2285 c->regs[VCPU_REGS_RSP] = tss->esp;
2286 c->regs[VCPU_REGS_RBP] = tss->ebp;
2287 c->regs[VCPU_REGS_RSI] = tss->esi;
2288 c->regs[VCPU_REGS_RDI] = tss->edi;
2291 * SDM says that segment selectors are loaded before segment
2294 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2295 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2296 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2297 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2298 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2299 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2300 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2303 * Now load segment descriptors. If fault happenes at this stage
2304 * it is handled in a context of new task
2306 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2307 if (ret != X86EMUL_CONTINUE)
2309 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2310 if (ret != X86EMUL_CONTINUE)
2312 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2313 if (ret != X86EMUL_CONTINUE)
2315 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2316 if (ret != X86EMUL_CONTINUE)
2318 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2319 if (ret != X86EMUL_CONTINUE)
2321 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2322 if (ret != X86EMUL_CONTINUE)
2324 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2325 if (ret != X86EMUL_CONTINUE)
2328 return X86EMUL_CONTINUE;
2331 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2332 struct x86_emulate_ops *ops,
2333 u16 tss_selector, u16 old_tss_sel,
2334 ulong old_tss_base, struct desc_struct *new_desc)
2336 struct tss_segment_32 tss_seg;
2338 u32 new_tss_base = get_desc_base(new_desc);
2340 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2342 if (ret != X86EMUL_CONTINUE)
2343 /* FIXME: need to provide precise fault address */
2346 save_state_to_tss32(ctxt, ops, &tss_seg);
2348 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2350 if (ret != X86EMUL_CONTINUE)
2351 /* FIXME: need to provide precise fault address */
2354 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2356 if (ret != X86EMUL_CONTINUE)
2357 /* FIXME: need to provide precise fault address */
2360 if (old_tss_sel != 0xffff) {
2361 tss_seg.prev_task_link = old_tss_sel;
2363 ret = ops->write_std(ctxt, new_tss_base,
2364 &tss_seg.prev_task_link,
2365 sizeof tss_seg.prev_task_link,
2367 if (ret != X86EMUL_CONTINUE)
2368 /* FIXME: need to provide precise fault address */
2372 return load_state_from_tss32(ctxt, ops, &tss_seg);
2375 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2376 struct x86_emulate_ops *ops,
2377 u16 tss_selector, int reason,
2378 bool has_error_code, u32 error_code)
2380 struct desc_struct curr_tss_desc, next_tss_desc;
2382 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2383 ulong old_tss_base =
2384 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2387 /* FIXME: old_tss_base == ~0 ? */
2389 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2390 if (ret != X86EMUL_CONTINUE)
2392 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2393 if (ret != X86EMUL_CONTINUE)
2396 /* FIXME: check that next_tss_desc is tss */
2398 if (reason != TASK_SWITCH_IRET) {
2399 if ((tss_selector & 3) > next_tss_desc.dpl ||
2400 ops->cpl(ctxt) > next_tss_desc.dpl)
2401 return emulate_gp(ctxt, 0);
2404 desc_limit = desc_limit_scaled(&next_tss_desc);
2405 if (!next_tss_desc.p ||
2406 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2407 desc_limit < 0x2b)) {
2408 emulate_ts(ctxt, tss_selector & 0xfffc);
2409 return X86EMUL_PROPAGATE_FAULT;
2412 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2413 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2414 write_segment_descriptor(ctxt, ops, old_tss_sel,
2418 if (reason == TASK_SWITCH_IRET)
2419 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2421 /* set back link to prev task only if NT bit is set in eflags
2422 note that old_tss_sel is not used afetr this point */
2423 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2424 old_tss_sel = 0xffff;
2426 if (next_tss_desc.type & 8)
2427 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2428 old_tss_base, &next_tss_desc);
2430 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2431 old_tss_base, &next_tss_desc);
2432 if (ret != X86EMUL_CONTINUE)
2435 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2436 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2438 if (reason != TASK_SWITCH_IRET) {
2439 next_tss_desc.type |= (1 << 1); /* set busy flag */
2440 write_segment_descriptor(ctxt, ops, tss_selector,
2444 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2445 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2447 if (has_error_code) {
2448 struct decode_cache *c = &ctxt->decode;
2450 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2452 c->src.val = (unsigned long) error_code;
2453 ret = em_push(ctxt);
2459 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2460 u16 tss_selector, int reason,
2461 bool has_error_code, u32 error_code)
2463 struct x86_emulate_ops *ops = ctxt->ops;
2464 struct decode_cache *c = &ctxt->decode;
2468 c->dst.type = OP_NONE;
2470 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2471 has_error_code, error_code);
2473 if (rc == X86EMUL_CONTINUE)
2476 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2479 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2480 int reg, struct operand *op)
2482 struct decode_cache *c = &ctxt->decode;
2483 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2485 register_address_increment(c, &c->regs[reg], df * op->bytes);
2486 op->addr.mem.ea = register_address(c, c->regs[reg]);
2487 op->addr.mem.seg = seg;
2490 static int em_das(struct x86_emulate_ctxt *ctxt)
2492 struct decode_cache *c = &ctxt->decode;
2494 bool af, cf, old_cf;
2496 cf = ctxt->eflags & X86_EFLAGS_CF;
2502 af = ctxt->eflags & X86_EFLAGS_AF;
2503 if ((al & 0x0f) > 9 || af) {
2505 cf = old_cf | (al >= 250);
2510 if (old_al > 0x99 || old_cf) {
2516 /* Set PF, ZF, SF */
2517 c->src.type = OP_IMM;
2520 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2521 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2523 ctxt->eflags |= X86_EFLAGS_CF;
2525 ctxt->eflags |= X86_EFLAGS_AF;
2526 return X86EMUL_CONTINUE;
2529 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2531 struct decode_cache *c = &ctxt->decode;
2536 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2539 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2540 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2541 return X86EMUL_CONTINUE;
2544 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2546 c->src.val = old_cs;
2548 if (rc != X86EMUL_CONTINUE)
2551 c->src.val = old_eip;
2552 return em_push(ctxt);
2555 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2557 struct decode_cache *c = &ctxt->decode;
2560 c->dst.type = OP_REG;
2561 c->dst.addr.reg = &c->eip;
2562 c->dst.bytes = c->op_bytes;
2563 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2564 if (rc != X86EMUL_CONTINUE)
2566 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2567 return X86EMUL_CONTINUE;
2570 static int em_add(struct x86_emulate_ctxt *ctxt)
2572 struct decode_cache *c = &ctxt->decode;
2574 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2575 return X86EMUL_CONTINUE;
2578 static int em_or(struct x86_emulate_ctxt *ctxt)
2580 struct decode_cache *c = &ctxt->decode;
2582 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2583 return X86EMUL_CONTINUE;
2586 static int em_adc(struct x86_emulate_ctxt *ctxt)
2588 struct decode_cache *c = &ctxt->decode;
2590 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2591 return X86EMUL_CONTINUE;
2594 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2596 struct decode_cache *c = &ctxt->decode;
2598 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2599 return X86EMUL_CONTINUE;
2602 static int em_and(struct x86_emulate_ctxt *ctxt)
2604 struct decode_cache *c = &ctxt->decode;
2606 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2607 return X86EMUL_CONTINUE;
2610 static int em_sub(struct x86_emulate_ctxt *ctxt)
2612 struct decode_cache *c = &ctxt->decode;
2614 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2615 return X86EMUL_CONTINUE;
2618 static int em_xor(struct x86_emulate_ctxt *ctxt)
2620 struct decode_cache *c = &ctxt->decode;
2622 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2623 return X86EMUL_CONTINUE;
2626 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2628 struct decode_cache *c = &ctxt->decode;
2630 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2631 /* Disable writeback. */
2632 c->dst.type = OP_NONE;
2633 return X86EMUL_CONTINUE;
2636 static int em_imul(struct x86_emulate_ctxt *ctxt)
2638 struct decode_cache *c = &ctxt->decode;
2640 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2641 return X86EMUL_CONTINUE;
2644 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2646 struct decode_cache *c = &ctxt->decode;
2648 c->dst.val = c->src2.val;
2649 return em_imul(ctxt);
2652 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2654 struct decode_cache *c = &ctxt->decode;
2656 c->dst.type = OP_REG;
2657 c->dst.bytes = c->src.bytes;
2658 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2659 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2661 return X86EMUL_CONTINUE;
2664 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2666 struct decode_cache *c = &ctxt->decode;
2669 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2670 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2671 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2672 return X86EMUL_CONTINUE;
2675 static int em_mov(struct x86_emulate_ctxt *ctxt)
2677 struct decode_cache *c = &ctxt->decode;
2678 c->dst.val = c->src.val;
2679 return X86EMUL_CONTINUE;
2682 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2684 struct decode_cache *c = &ctxt->decode;
2685 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2686 return X86EMUL_CONTINUE;
2689 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2691 struct decode_cache *c = &ctxt->decode;
2695 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2696 if (rc == X86EMUL_CONTINUE)
2697 ctxt->ops->invlpg(ctxt, linear);
2698 /* Disable writeback. */
2699 c->dst.type = OP_NONE;
2700 return X86EMUL_CONTINUE;
2703 static int em_clts(struct x86_emulate_ctxt *ctxt)
2707 cr0 = ctxt->ops->get_cr(ctxt, 0);
2709 ctxt->ops->set_cr(ctxt, 0, cr0);
2710 return X86EMUL_CONTINUE;
2713 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2715 struct decode_cache *c = &ctxt->decode;
2718 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2719 return X86EMUL_UNHANDLEABLE;
2721 rc = ctxt->ops->fix_hypercall(ctxt);
2722 if (rc != X86EMUL_CONTINUE)
2725 /* Let the processor re-execute the fixed hypercall */
2727 /* Disable writeback. */
2728 c->dst.type = OP_NONE;
2729 return X86EMUL_CONTINUE;
2732 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2734 struct decode_cache *c = &ctxt->decode;
2735 struct desc_ptr desc_ptr;
2738 rc = read_descriptor(ctxt, c->src.addr.mem,
2739 &desc_ptr.size, &desc_ptr.address,
2741 if (rc != X86EMUL_CONTINUE)
2743 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2744 /* Disable writeback. */
2745 c->dst.type = OP_NONE;
2746 return X86EMUL_CONTINUE;
2749 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2751 struct decode_cache *c = &ctxt->decode;
2754 rc = ctxt->ops->fix_hypercall(ctxt);
2756 /* Disable writeback. */
2757 c->dst.type = OP_NONE;
2761 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2763 struct decode_cache *c = &ctxt->decode;
2764 struct desc_ptr desc_ptr;
2767 rc = read_descriptor(ctxt, c->src.addr.mem,
2768 &desc_ptr.size, &desc_ptr.address,
2770 if (rc != X86EMUL_CONTINUE)
2772 ctxt->ops->set_idt(ctxt, &desc_ptr);
2773 /* Disable writeback. */
2774 c->dst.type = OP_NONE;
2775 return X86EMUL_CONTINUE;
2778 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2780 struct decode_cache *c = &ctxt->decode;
2783 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2784 return X86EMUL_CONTINUE;
2787 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2789 struct decode_cache *c = &ctxt->decode;
2790 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2791 | (c->src.val & 0x0f));
2792 c->dst.type = OP_NONE;
2793 return X86EMUL_CONTINUE;
2796 static bool valid_cr(int nr)
2808 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2810 struct decode_cache *c = &ctxt->decode;
2812 if (!valid_cr(c->modrm_reg))
2813 return emulate_ud(ctxt);
2815 return X86EMUL_CONTINUE;
2818 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2820 struct decode_cache *c = &ctxt->decode;
2821 u64 new_val = c->src.val64;
2822 int cr = c->modrm_reg;
2825 static u64 cr_reserved_bits[] = {
2826 0xffffffff00000000ULL,
2827 0, 0, 0, /* CR3 checked later */
2834 return emulate_ud(ctxt);
2836 if (new_val & cr_reserved_bits[cr])
2837 return emulate_gp(ctxt, 0);
2842 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2843 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2844 return emulate_gp(ctxt, 0);
2846 cr4 = ctxt->ops->get_cr(ctxt, 4);
2847 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2849 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2850 !(cr4 & X86_CR4_PAE))
2851 return emulate_gp(ctxt, 0);
2858 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2859 if (efer & EFER_LMA)
2860 rsvd = CR3_L_MODE_RESERVED_BITS;
2861 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2862 rsvd = CR3_PAE_RESERVED_BITS;
2863 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2864 rsvd = CR3_NONPAE_RESERVED_BITS;
2867 return emulate_gp(ctxt, 0);
2874 cr4 = ctxt->ops->get_cr(ctxt, 4);
2875 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2877 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2878 return emulate_gp(ctxt, 0);
2884 return X86EMUL_CONTINUE;
2887 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2891 ctxt->ops->get_dr(ctxt, 7, &dr7);
2893 /* Check if DR7.Global_Enable is set */
2894 return dr7 & (1 << 13);
2897 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2899 struct decode_cache *c = &ctxt->decode;
2900 int dr = c->modrm_reg;
2904 return emulate_ud(ctxt);
2906 cr4 = ctxt->ops->get_cr(ctxt, 4);
2907 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2908 return emulate_ud(ctxt);
2910 if (check_dr7_gd(ctxt))
2911 return emulate_db(ctxt);
2913 return X86EMUL_CONTINUE;
2916 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2918 struct decode_cache *c = &ctxt->decode;
2919 u64 new_val = c->src.val64;
2920 int dr = c->modrm_reg;
2922 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2923 return emulate_gp(ctxt, 0);
2925 return check_dr_read(ctxt);
2928 static int check_svme(struct x86_emulate_ctxt *ctxt)
2932 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2934 if (!(efer & EFER_SVME))
2935 return emulate_ud(ctxt);
2937 return X86EMUL_CONTINUE;
2940 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2942 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2944 /* Valid physical address? */
2945 if (rax & 0xffff000000000000ULL)
2946 return emulate_gp(ctxt, 0);
2948 return check_svme(ctxt);
2951 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2953 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2955 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2956 return emulate_ud(ctxt);
2958 return X86EMUL_CONTINUE;
2961 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2963 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2964 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2966 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2968 return emulate_gp(ctxt, 0);
2970 return X86EMUL_CONTINUE;
2973 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2975 struct decode_cache *c = &ctxt->decode;
2977 c->dst.bytes = min(c->dst.bytes, 4u);
2978 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2979 return emulate_gp(ctxt, 0);
2981 return X86EMUL_CONTINUE;
2984 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2986 struct decode_cache *c = &ctxt->decode;
2988 c->src.bytes = min(c->src.bytes, 4u);
2989 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2990 return emulate_gp(ctxt, 0);
2992 return X86EMUL_CONTINUE;
2995 #define D(_y) { .flags = (_y) }
2996 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2997 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2998 .check_perm = (_p) }
3000 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3001 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3002 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3003 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3004 #define II(_f, _e, _i) \
3005 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3006 #define IIP(_f, _e, _i, _p) \
3007 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3008 .check_perm = (_p) }
3009 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3011 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3012 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3013 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3015 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3016 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3017 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3019 static struct opcode group7_rm1[] = {
3020 DI(SrcNone | ModRM | Priv, monitor),
3021 DI(SrcNone | ModRM | Priv, mwait),
3025 static struct opcode group7_rm3[] = {
3026 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3027 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3028 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3029 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3030 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3031 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3032 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3033 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3036 static struct opcode group7_rm7[] = {
3038 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3042 static struct opcode group1[] = {
3053 static struct opcode group1A[] = {
3054 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3057 static struct opcode group3[] = {
3058 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3059 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3060 X4(D(SrcMem | ModRM)),
3063 static struct opcode group4[] = {
3064 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3068 static struct opcode group5[] = {
3069 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3070 D(SrcMem | ModRM | Stack),
3071 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3072 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3073 D(SrcMem | ModRM | Stack), N,
3076 static struct opcode group6[] = {
3077 DI(ModRM | Prot, sldt),
3078 DI(ModRM | Prot, str),
3079 DI(ModRM | Prot | Priv, lldt),
3080 DI(ModRM | Prot | Priv, ltr),
3084 static struct group_dual group7 = { {
3085 DI(ModRM | Mov | DstMem | Priv, sgdt),
3086 DI(ModRM | Mov | DstMem | Priv, sidt),
3087 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3088 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3089 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3090 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3091 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3093 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3095 N, EXT(0, group7_rm3),
3096 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3097 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3100 static struct opcode group8[] = {
3102 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3103 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3106 static struct group_dual group9 = { {
3107 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3109 N, N, N, N, N, N, N, N,
3112 static struct opcode group11[] = {
3113 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3116 static struct gprefix pfx_0f_6f_0f_7f = {
3117 N, N, N, I(Sse, em_movdqu),
3120 static struct opcode opcode_table[256] = {
3122 I6ALU(Lock, em_add),
3123 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3126 D(ImplicitOps | Stack | No64), N,
3128 I6ALU(Lock, em_adc),
3129 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3131 I6ALU(Lock, em_sbb),
3132 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3134 I6ALU(Lock, em_and), N, N,
3136 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3138 I6ALU(Lock, em_xor), N, N,
3140 I6ALU(0, em_cmp), N, N,
3144 X8(I(SrcReg | Stack, em_push)),
3146 X8(I(DstReg | Stack, em_pop)),
3148 I(ImplicitOps | Stack | No64, em_pusha),
3149 I(ImplicitOps | Stack | No64, em_popa),
3150 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3153 I(SrcImm | Mov | Stack, em_push),
3154 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3155 I(SrcImmByte | Mov | Stack, em_push),
3156 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3157 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3158 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
3162 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3163 G(DstMem | SrcImm | ModRM | Group, group1),
3164 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3165 G(DstMem | SrcImmByte | ModRM | Group, group1),
3166 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3168 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3169 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3170 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3171 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3173 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3175 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3176 I(SrcImmFAddr | No64, em_call_far), N,
3177 II(ImplicitOps | Stack, em_pushf, pushf),
3178 II(ImplicitOps | Stack, em_popf, popf), N, N,
3180 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3181 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3182 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3183 I2bv(SrcSI | DstDI | String, em_cmp),
3185 D2bv(DstAcc | SrcImm),
3186 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3187 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3188 I2bv(SrcAcc | DstDI | String, em_cmp),
3190 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3192 X8(I(DstReg | SrcImm | Mov, em_mov)),
3194 D2bv(DstMem | SrcImmByte | ModRM),
3195 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3196 D(ImplicitOps | Stack),
3197 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3198 G(ByteOp, group11), G(0, group11),
3200 N, N, N, D(ImplicitOps | Stack),
3201 D(ImplicitOps), DI(SrcImmByte, intn),
3202 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3204 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3207 N, N, N, N, N, N, N, N,
3210 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3211 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3213 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3214 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3215 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
3216 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
3218 N, DI(ImplicitOps, icebp), N, N,
3219 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3220 G(ByteOp, group3), G(0, group3),
3222 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3223 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3226 static struct opcode twobyte_table[256] = {
3228 G(0, group6), GD(0, &group7), N, N,
3229 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3230 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3231 N, D(ImplicitOps | ModRM), N, N,
3233 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3235 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3236 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3237 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3238 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3240 N, N, N, N, N, N, N, N,
3242 DI(ImplicitOps | Priv, wrmsr),
3243 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3244 DI(ImplicitOps | Priv, rdmsr),
3245 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3246 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3248 N, N, N, N, N, N, N, N,
3250 X16(D(DstReg | SrcMem | ModRM | Mov)),
3252 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3257 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3262 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3266 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3268 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3269 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3270 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3271 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3273 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3274 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3275 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3276 D(DstMem | SrcReg | Src2CL | ModRM),
3277 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3279 D2bv(DstMem | SrcReg | ModRM | Lock),
3280 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3281 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3282 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3285 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3286 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3287 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3289 D2bv(DstMem | SrcReg | ModRM | Lock),
3290 N, D(DstMem | SrcReg | ModRM | Mov),
3291 N, N, N, GD(0, &group9),
3292 N, N, N, N, N, N, N, N,
3294 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3296 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3298 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3314 static unsigned imm_size(struct decode_cache *c)
3318 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3324 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3325 unsigned size, bool sign_extension)
3327 struct decode_cache *c = &ctxt->decode;
3328 struct x86_emulate_ops *ops = ctxt->ops;
3329 int rc = X86EMUL_CONTINUE;
3333 op->addr.mem.ea = c->eip;
3334 /* NB. Immediates are sign-extended as necessary. */
3335 switch (op->bytes) {
3337 op->val = insn_fetch(s8, 1, c->eip);
3340 op->val = insn_fetch(s16, 2, c->eip);
3343 op->val = insn_fetch(s32, 4, c->eip);
3346 if (!sign_extension) {
3347 switch (op->bytes) {
3355 op->val &= 0xffffffff;
3364 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3366 struct x86_emulate_ops *ops = ctxt->ops;
3367 struct decode_cache *c = &ctxt->decode;
3368 int rc = X86EMUL_CONTINUE;
3369 int mode = ctxt->mode;
3370 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3371 bool op_prefix = false;
3372 struct opcode opcode;
3373 struct operand memop = { .type = OP_NONE };
3376 c->fetch.start = c->eip;
3377 c->fetch.end = c->fetch.start + insn_len;
3379 memcpy(c->fetch.data, insn, insn_len);
3382 case X86EMUL_MODE_REAL:
3383 case X86EMUL_MODE_VM86:
3384 case X86EMUL_MODE_PROT16:
3385 def_op_bytes = def_ad_bytes = 2;
3387 case X86EMUL_MODE_PROT32:
3388 def_op_bytes = def_ad_bytes = 4;
3390 #ifdef CONFIG_X86_64
3391 case X86EMUL_MODE_PROT64:
3400 c->op_bytes = def_op_bytes;
3401 c->ad_bytes = def_ad_bytes;
3403 /* Legacy prefixes. */
3405 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3406 case 0x66: /* operand-size override */
3408 /* switch between 2/4 bytes */
3409 c->op_bytes = def_op_bytes ^ 6;
3411 case 0x67: /* address-size override */
3412 if (mode == X86EMUL_MODE_PROT64)
3413 /* switch between 4/8 bytes */
3414 c->ad_bytes = def_ad_bytes ^ 12;
3416 /* switch between 2/4 bytes */
3417 c->ad_bytes = def_ad_bytes ^ 6;
3419 case 0x26: /* ES override */
3420 case 0x2e: /* CS override */
3421 case 0x36: /* SS override */
3422 case 0x3e: /* DS override */
3423 set_seg_override(c, (c->b >> 3) & 3);
3425 case 0x64: /* FS override */
3426 case 0x65: /* GS override */
3427 set_seg_override(c, c->b & 7);
3429 case 0x40 ... 0x4f: /* REX */
3430 if (mode != X86EMUL_MODE_PROT64)
3432 c->rex_prefix = c->b;
3434 case 0xf0: /* LOCK */
3437 case 0xf2: /* REPNE/REPNZ */
3438 case 0xf3: /* REP/REPE/REPZ */
3439 c->rep_prefix = c->b;
3445 /* Any legacy prefix after a REX prefix nullifies its effect. */
3453 if (c->rex_prefix & 8)
3454 c->op_bytes = 8; /* REX.W */
3456 /* Opcode byte(s). */
3457 opcode = opcode_table[c->b];
3458 /* Two-byte opcode? */
3461 c->b = insn_fetch(u8, 1, c->eip);
3462 opcode = twobyte_table[c->b];
3464 c->d = opcode.flags;
3466 while (c->d & GroupMask) {
3467 switch (c->d & GroupMask) {
3469 c->modrm = insn_fetch(u8, 1, c->eip);
3471 goffset = (c->modrm >> 3) & 7;
3472 opcode = opcode.u.group[goffset];
3475 c->modrm = insn_fetch(u8, 1, c->eip);
3477 goffset = (c->modrm >> 3) & 7;
3478 if ((c->modrm >> 6) == 3)
3479 opcode = opcode.u.gdual->mod3[goffset];
3481 opcode = opcode.u.gdual->mod012[goffset];
3484 goffset = c->modrm & 7;
3485 opcode = opcode.u.group[goffset];
3488 if (c->rep_prefix && op_prefix)
3489 return X86EMUL_UNHANDLEABLE;
3490 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3491 switch (simd_prefix) {
3492 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3493 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3494 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3495 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3499 return X86EMUL_UNHANDLEABLE;
3503 c->d |= opcode.flags;
3506 c->execute = opcode.u.execute;
3507 c->check_perm = opcode.check_perm;
3508 c->intercept = opcode.intercept;
3511 if (c->d == 0 || (c->d & Undefined))
3514 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3517 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3520 if (c->d & Op3264) {
3521 if (mode == X86EMUL_MODE_PROT64)
3530 /* ModRM and SIB bytes. */
3532 rc = decode_modrm(ctxt, ops, &memop);
3533 if (!c->has_seg_override)
3534 set_seg_override(c, c->modrm_seg);
3535 } else if (c->d & MemAbs)
3536 rc = decode_abs(ctxt, ops, &memop);
3537 if (rc != X86EMUL_CONTINUE)
3540 if (!c->has_seg_override)
3541 set_seg_override(c, VCPU_SREG_DS);
3543 memop.addr.mem.seg = seg_override(ctxt, c);
3545 if (memop.type == OP_MEM && c->ad_bytes != 8)
3546 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3548 if (memop.type == OP_MEM && c->rip_relative)
3549 memop.addr.mem.ea += c->eip;
3552 * Decode and fetch the source operand: register, memory
3555 switch (c->d & SrcMask) {
3559 decode_register_operand(ctxt, &c->src, c, 0);
3568 memop.bytes = (c->d & ByteOp) ? 1 :
3574 rc = decode_imm(ctxt, &c->src, 2, false);
3577 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3580 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3583 rc = decode_imm(ctxt, &c->src, 1, true);
3586 rc = decode_imm(ctxt, &c->src, 1, false);
3589 c->src.type = OP_REG;
3590 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3591 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3592 fetch_register_operand(&c->src);
3599 c->src.type = OP_MEM;
3600 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3601 c->src.addr.mem.ea =
3602 register_address(c, c->regs[VCPU_REGS_RSI]);
3603 c->src.addr.mem.seg = seg_override(ctxt, c);
3607 c->src.type = OP_IMM;
3608 c->src.addr.mem.ea = c->eip;
3609 c->src.bytes = c->op_bytes + 2;
3610 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3613 memop.bytes = c->op_bytes + 2;
3618 if (rc != X86EMUL_CONTINUE)
3622 * Decode and fetch the second source operand: register, memory
3625 switch (c->d & Src2Mask) {
3630 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3633 rc = decode_imm(ctxt, &c->src2, 1, true);
3640 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3644 if (rc != X86EMUL_CONTINUE)
3647 /* Decode and fetch the destination operand: register or memory. */
3648 switch (c->d & DstMask) {
3650 decode_register_operand(ctxt, &c->dst, c,
3651 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3654 c->dst.type = OP_IMM;
3655 c->dst.addr.mem.ea = c->eip;
3657 c->dst.val = insn_fetch(u8, 1, c->eip);
3662 if ((c->d & DstMask) == DstMem64)
3665 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3667 fetch_bit_operand(c);
3668 c->dst.orig_val = c->dst.val;
3671 c->dst.type = OP_REG;
3672 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3673 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3674 fetch_register_operand(&c->dst);
3675 c->dst.orig_val = c->dst.val;
3678 c->dst.type = OP_MEM;
3679 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3680 c->dst.addr.mem.ea =
3681 register_address(c, c->regs[VCPU_REGS_RDI]);
3682 c->dst.addr.mem.seg = VCPU_SREG_ES;
3686 /* Special instructions do their own operand decoding. */
3688 c->dst.type = OP_NONE; /* Disable writeback. */
3693 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3696 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3698 struct decode_cache *c = &ctxt->decode;
3700 /* The second termination condition only applies for REPE
3701 * and REPNE. Test if the repeat string operation prefix is
3702 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3703 * corresponding termination condition according to:
3704 * - if REPE/REPZ and ZF = 0 then done
3705 * - if REPNE/REPNZ and ZF = 1 then done
3707 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3708 (c->b == 0xae) || (c->b == 0xaf))
3709 && (((c->rep_prefix == REPE_PREFIX) &&
3710 ((ctxt->eflags & EFLG_ZF) == 0))
3711 || ((c->rep_prefix == REPNE_PREFIX) &&
3712 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3719 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3721 struct x86_emulate_ops *ops = ctxt->ops;
3723 struct decode_cache *c = &ctxt->decode;
3724 int rc = X86EMUL_CONTINUE;
3725 int saved_dst_type = c->dst.type;
3726 int irq; /* Used for int 3, int, and into */
3728 ctxt->decode.mem_read.pos = 0;
3730 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3731 rc = emulate_ud(ctxt);
3735 /* LOCK prefix is allowed only with some instructions */
3736 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3737 rc = emulate_ud(ctxt);
3741 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3742 rc = emulate_ud(ctxt);
3747 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3748 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3749 rc = emulate_ud(ctxt);
3753 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3754 rc = emulate_nm(ctxt);
3758 if (unlikely(ctxt->guest_mode) && c->intercept) {
3759 rc = emulator_check_intercept(ctxt, c->intercept,
3760 X86_ICPT_PRE_EXCEPT);
3761 if (rc != X86EMUL_CONTINUE)
3765 /* Privileged instruction can be executed only in CPL=0 */
3766 if ((c->d & Priv) && ops->cpl(ctxt)) {
3767 rc = emulate_gp(ctxt, 0);
3771 /* Instruction can only be executed in protected mode */
3772 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3773 rc = emulate_ud(ctxt);
3777 /* Do instruction specific permission checks */
3778 if (c->check_perm) {
3779 rc = c->check_perm(ctxt);
3780 if (rc != X86EMUL_CONTINUE)
3784 if (unlikely(ctxt->guest_mode) && c->intercept) {
3785 rc = emulator_check_intercept(ctxt, c->intercept,
3786 X86_ICPT_POST_EXCEPT);
3787 if (rc != X86EMUL_CONTINUE)
3791 if (c->rep_prefix && (c->d & String)) {
3792 /* All REP prefixes have the same first termination condition */
3793 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3799 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3800 rc = segmented_read(ctxt, c->src.addr.mem,
3801 c->src.valptr, c->src.bytes);
3802 if (rc != X86EMUL_CONTINUE)
3804 c->src.orig_val64 = c->src.val64;
3807 if (c->src2.type == OP_MEM) {
3808 rc = segmented_read(ctxt, c->src2.addr.mem,
3809 &c->src2.val, c->src2.bytes);
3810 if (rc != X86EMUL_CONTINUE)
3814 if ((c->d & DstMask) == ImplicitOps)
3818 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3819 /* optimisation - avoid slow emulated read if Mov */
3820 rc = segmented_read(ctxt, c->dst.addr.mem,
3821 &c->dst.val, c->dst.bytes);
3822 if (rc != X86EMUL_CONTINUE)
3825 c->dst.orig_val = c->dst.val;
3829 if (unlikely(ctxt->guest_mode) && c->intercept) {
3830 rc = emulator_check_intercept(ctxt, c->intercept,
3831 X86_ICPT_POST_MEMACCESS);
3832 if (rc != X86EMUL_CONTINUE)
3837 rc = c->execute(ctxt);
3838 if (rc != X86EMUL_CONTINUE)
3847 case 0x06: /* push es */
3848 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3850 case 0x07: /* pop es */
3851 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3853 case 0x0e: /* push cs */
3854 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3856 case 0x16: /* push ss */
3857 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3859 case 0x17: /* pop ss */
3860 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3862 case 0x1e: /* push ds */
3863 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3865 case 0x1f: /* pop ds */
3866 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3868 case 0x40 ... 0x47: /* inc r16/r32 */
3869 emulate_1op("inc", c->dst, ctxt->eflags);
3871 case 0x48 ... 0x4f: /* dec r16/r32 */
3872 emulate_1op("dec", c->dst, ctxt->eflags);
3874 case 0x63: /* movsxd */
3875 if (ctxt->mode != X86EMUL_MODE_PROT64)
3876 goto cannot_emulate;
3877 c->dst.val = (s32) c->src.val;
3879 case 0x6c: /* insb */
3880 case 0x6d: /* insw/insd */
3881 c->src.val = c->regs[VCPU_REGS_RDX];
3883 case 0x6e: /* outsb */
3884 case 0x6f: /* outsw/outsd */
3885 c->dst.val = c->regs[VCPU_REGS_RDX];
3888 case 0x70 ... 0x7f: /* jcc (short) */
3889 if (test_cc(c->b, ctxt->eflags))
3890 jmp_rel(c, c->src.val);
3894 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3896 case 0x86 ... 0x87: /* xchg */
3898 /* Write back the register source. */
3899 c->src.val = c->dst.val;
3900 write_register_operand(&c->src);
3902 * Write back the memory destination with implicit LOCK
3905 c->dst.val = c->src.orig_val;
3908 case 0x8c: /* mov r/m, sreg */
3909 if (c->modrm_reg > VCPU_SREG_GS) {
3910 rc = emulate_ud(ctxt);
3913 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3915 case 0x8d: /* lea r16/r32, m */
3916 c->dst.val = c->src.addr.mem.ea;
3918 case 0x8e: { /* mov seg, r/m16 */
3923 if (c->modrm_reg == VCPU_SREG_CS ||
3924 c->modrm_reg > VCPU_SREG_GS) {
3925 rc = emulate_ud(ctxt);
3929 if (c->modrm_reg == VCPU_SREG_SS)
3930 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3932 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3934 c->dst.type = OP_NONE; /* Disable writeback. */
3937 case 0x8f: /* pop (sole member of Grp1a) */
3938 rc = em_grp1a(ctxt);
3940 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3941 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3944 case 0x98: /* cbw/cwde/cdqe */
3945 switch (c->op_bytes) {
3946 case 2: c->dst.val = (s8)c->dst.val; break;
3947 case 4: c->dst.val = (s16)c->dst.val; break;
3948 case 8: c->dst.val = (s32)c->dst.val; break;
3951 case 0xa8 ... 0xa9: /* test ax, imm */
3956 case 0xc3: /* ret */
3957 c->dst.type = OP_REG;
3958 c->dst.addr.reg = &c->eip;
3959 c->dst.bytes = c->op_bytes;
3962 case 0xc4: /* les */
3963 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3965 case 0xc5: /* lds */
3966 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3968 case 0xcb: /* ret far */
3969 rc = emulate_ret_far(ctxt, ops);
3971 case 0xcc: /* int3 */
3974 case 0xcd: /* int n */
3977 rc = emulate_int(ctxt, ops, irq);
3979 case 0xce: /* into */
3980 if (ctxt->eflags & EFLG_OF) {
3985 case 0xcf: /* iret */
3986 rc = emulate_iret(ctxt, ops);
3988 case 0xd0 ... 0xd1: /* Grp2 */
3991 case 0xd2 ... 0xd3: /* Grp2 */
3992 c->src.val = c->regs[VCPU_REGS_RCX];
3995 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3996 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3997 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3998 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3999 jmp_rel(c, c->src.val);
4001 case 0xe3: /* jcxz/jecxz/jrcxz */
4002 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
4003 jmp_rel(c, c->src.val);
4005 case 0xe4: /* inb */
4008 case 0xe6: /* outb */
4009 case 0xe7: /* out */
4011 case 0xe8: /* call (near) */ {
4012 long int rel = c->src.val;
4013 c->src.val = (unsigned long) c->eip;
4018 case 0xe9: /* jmp rel */
4020 case 0xea: /* jmp far */
4021 rc = em_jmp_far(ctxt);
4024 jmp: /* jmp rel short */
4025 jmp_rel(c, c->src.val);
4026 c->dst.type = OP_NONE; /* Disable writeback. */
4028 case 0xec: /* in al,dx */
4029 case 0xed: /* in (e/r)ax,dx */
4030 c->src.val = c->regs[VCPU_REGS_RDX];
4032 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4034 goto done; /* IO is needed */
4036 case 0xee: /* out dx,al */
4037 case 0xef: /* out dx,(e/r)ax */
4038 c->dst.val = c->regs[VCPU_REGS_RDX];
4040 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4042 c->dst.type = OP_NONE; /* Disable writeback. */
4044 case 0xf4: /* hlt */
4045 ctxt->ops->halt(ctxt);
4047 case 0xf5: /* cmc */
4048 /* complement carry flag from eflags reg */
4049 ctxt->eflags ^= EFLG_CF;
4051 case 0xf6 ... 0xf7: /* Grp3 */
4054 case 0xf8: /* clc */
4055 ctxt->eflags &= ~EFLG_CF;
4057 case 0xf9: /* stc */
4058 ctxt->eflags |= EFLG_CF;
4060 case 0xfa: /* cli */
4061 if (emulator_bad_iopl(ctxt, ops)) {
4062 rc = emulate_gp(ctxt, 0);
4065 ctxt->eflags &= ~X86_EFLAGS_IF;
4067 case 0xfb: /* sti */
4068 if (emulator_bad_iopl(ctxt, ops)) {
4069 rc = emulate_gp(ctxt, 0);
4072 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4073 ctxt->eflags |= X86_EFLAGS_IF;
4076 case 0xfc: /* cld */
4077 ctxt->eflags &= ~EFLG_DF;
4079 case 0xfd: /* std */
4080 ctxt->eflags |= EFLG_DF;
4082 case 0xfe: /* Grp4 */
4083 rc = em_grp45(ctxt);
4085 case 0xff: /* Grp5 */
4086 rc = em_grp45(ctxt);
4089 goto cannot_emulate;
4092 if (rc != X86EMUL_CONTINUE)
4096 rc = writeback(ctxt);
4097 if (rc != X86EMUL_CONTINUE)
4101 * restore dst type in case the decoding will be reused
4102 * (happens for string instruction )
4104 c->dst.type = saved_dst_type;
4106 if ((c->d & SrcMask) == SrcSI)
4107 string_addr_inc(ctxt, seg_override(ctxt, c),
4108 VCPU_REGS_RSI, &c->src);
4110 if ((c->d & DstMask) == DstDI)
4111 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4114 if (c->rep_prefix && (c->d & String)) {
4115 struct read_cache *r = &ctxt->decode.io_read;
4116 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4118 if (!string_insn_completed(ctxt)) {
4120 * Re-enter guest when pio read ahead buffer is empty
4121 * or, if it is not used, after each 1024 iteration.
4123 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4124 (r->end == 0 || r->end != r->pos)) {
4126 * Reset read cache. Usually happens before
4127 * decode, but since instruction is restarted
4128 * we have to do it here.
4130 ctxt->decode.mem_read.end = 0;
4131 return EMULATION_RESTART;
4133 goto done; /* skip rip writeback */
4140 if (rc == X86EMUL_PROPAGATE_FAULT)
4141 ctxt->have_exception = true;
4142 if (rc == X86EMUL_INTERCEPTED)
4143 return EMULATION_INTERCEPTED;
4145 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4149 case 0x05: /* syscall */
4150 rc = emulate_syscall(ctxt, ops);
4155 case 0x09: /* wbinvd */
4156 (ctxt->ops->wbinvd)(ctxt);
4158 case 0x08: /* invd */
4159 case 0x0d: /* GrpP (prefetch) */
4160 case 0x18: /* Grp16 (prefetch/nop) */
4162 case 0x20: /* mov cr, reg */
4163 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4165 case 0x21: /* mov from dr to reg */
4166 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4168 case 0x22: /* mov reg, cr */
4169 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4170 emulate_gp(ctxt, 0);
4171 rc = X86EMUL_PROPAGATE_FAULT;
4174 c->dst.type = OP_NONE;
4176 case 0x23: /* mov from reg to dr */
4177 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4178 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4179 ~0ULL : ~0U)) < 0) {
4180 /* #UD condition is already handled by the code above */
4181 emulate_gp(ctxt, 0);
4182 rc = X86EMUL_PROPAGATE_FAULT;
4186 c->dst.type = OP_NONE; /* no writeback */
4190 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4191 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4192 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4193 emulate_gp(ctxt, 0);
4194 rc = X86EMUL_PROPAGATE_FAULT;
4197 rc = X86EMUL_CONTINUE;
4201 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4202 emulate_gp(ctxt, 0);
4203 rc = X86EMUL_PROPAGATE_FAULT;
4206 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4207 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4209 rc = X86EMUL_CONTINUE;
4211 case 0x34: /* sysenter */
4212 rc = emulate_sysenter(ctxt, ops);
4214 case 0x35: /* sysexit */
4215 rc = emulate_sysexit(ctxt, ops);
4217 case 0x40 ... 0x4f: /* cmov */
4218 c->dst.val = c->dst.orig_val = c->src.val;
4219 if (!test_cc(c->b, ctxt->eflags))
4220 c->dst.type = OP_NONE; /* no writeback */
4222 case 0x80 ... 0x8f: /* jnz rel, etc*/
4223 if (test_cc(c->b, ctxt->eflags))
4224 jmp_rel(c, c->src.val);
4226 case 0x90 ... 0x9f: /* setcc r/m8 */
4227 c->dst.val = test_cc(c->b, ctxt->eflags);
4229 case 0xa0: /* push fs */
4230 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4232 case 0xa1: /* pop fs */
4233 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4237 c->dst.type = OP_NONE;
4238 /* only subword offset */
4239 c->src.val &= (c->dst.bytes << 3) - 1;
4240 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4242 case 0xa4: /* shld imm8, r, r/m */
4243 case 0xa5: /* shld cl, r, r/m */
4244 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4246 case 0xa8: /* push gs */
4247 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4249 case 0xa9: /* pop gs */
4250 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4254 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4256 case 0xac: /* shrd imm8, r, r/m */
4257 case 0xad: /* shrd cl, r, r/m */
4258 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4260 case 0xae: /* clflush */
4262 case 0xb0 ... 0xb1: /* cmpxchg */
4264 * Save real source value, then compare EAX against
4267 c->src.orig_val = c->src.val;
4268 c->src.val = c->regs[VCPU_REGS_RAX];
4269 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4270 if (ctxt->eflags & EFLG_ZF) {
4271 /* Success: write back to memory. */
4272 c->dst.val = c->src.orig_val;
4274 /* Failure: write the value we saw to EAX. */
4275 c->dst.type = OP_REG;
4276 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4279 case 0xb2: /* lss */
4280 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4284 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4286 case 0xb4: /* lfs */
4287 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4289 case 0xb5: /* lgs */
4290 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4292 case 0xb6 ... 0xb7: /* movzx */
4293 c->dst.bytes = c->op_bytes;
4294 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4297 case 0xba: /* Grp8 */
4298 switch (c->modrm_reg & 3) {
4311 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4313 case 0xbc: { /* bsf */
4315 __asm__ ("bsf %2, %0; setz %1"
4316 : "=r"(c->dst.val), "=q"(zf)
4318 ctxt->eflags &= ~X86_EFLAGS_ZF;
4320 ctxt->eflags |= X86_EFLAGS_ZF;
4321 c->dst.type = OP_NONE; /* Disable writeback. */
4325 case 0xbd: { /* bsr */
4327 __asm__ ("bsr %2, %0; setz %1"
4328 : "=r"(c->dst.val), "=q"(zf)
4330 ctxt->eflags &= ~X86_EFLAGS_ZF;
4332 ctxt->eflags |= X86_EFLAGS_ZF;
4333 c->dst.type = OP_NONE; /* Disable writeback. */
4337 case 0xbe ... 0xbf: /* movsx */
4338 c->dst.bytes = c->op_bytes;
4339 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4342 case 0xc0 ... 0xc1: /* xadd */
4343 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4344 /* Write back the register source. */
4345 c->src.val = c->dst.orig_val;
4346 write_register_operand(&c->src);
4348 case 0xc3: /* movnti */
4349 c->dst.bytes = c->op_bytes;
4350 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4353 case 0xc7: /* Grp9 (cmpxchg8b) */
4357 goto cannot_emulate;
4360 if (rc != X86EMUL_CONTINUE)
4366 return EMULATION_FAILED;