1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstMask (7<<1)
51 /* Source operand type. */
52 #define SrcNone (0<<4) /* No source operand. */
53 #define SrcReg (1<<4) /* Register operand. */
54 #define SrcMem (2<<4) /* Memory operand. */
55 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
56 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
57 #define SrcImm (5<<4) /* Immediate operand. */
58 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
59 #define SrcOne (7<<4) /* Implied '1' */
60 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
61 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
62 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
63 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
64 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
65 #define SrcAcc (0xd<<4) /* Source Accumulator */
66 #define SrcImmU16 (0xe<<4) /* Immediate operand, unsigned, 16 bits */
67 #define SrcMask (0xf<<4)
68 /* Generic ModRM decode. */
70 /* Destination is only written; never read. */
73 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
74 #define String (1<<12) /* String instruction (rep capable) */
75 #define Stack (1<<13) /* Stack instruction (push/pop) */
76 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
77 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
78 #define Prefix (1<<16) /* Instruction varies with 66/f2/f3 prefix */
79 #define Sse (1<<17) /* SSE Vector instruction */
80 #define RMExt (1<<18) /* Opcode extension in ModRM r/m if mod == 3 */
82 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
83 #define VendorSpecific (1<<22) /* Vendor specific instruction */
84 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
85 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
86 #define Undefined (1<<25) /* No Such Instruction */
87 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
88 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
90 /* Source 2 operand type */
91 #define Src2None (0<<29)
92 #define Src2CL (1<<29)
93 #define Src2ImmByte (2<<29)
94 #define Src2One (3<<29)
95 #define Src2Imm (4<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x...) X2(x), x
100 #define X4(x...) X2(x), X2(x)
101 #define X5(x...) X4(x), x
102 #define X6(x...) X4(x), X2(x)
103 #define X7(x...) X4(x), X3(x)
104 #define X8(x...) X4(x), X4(x)
105 #define X16(x...) X8(x), X8(x)
111 int (*execute)(struct x86_emulate_ctxt *ctxt);
112 struct opcode *group;
113 struct group_dual *gdual;
114 struct gprefix *gprefix;
116 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
120 struct opcode mod012[8];
121 struct opcode mod3[8];
125 struct opcode pfx_no;
126 struct opcode pfx_66;
127 struct opcode pfx_f2;
128 struct opcode pfx_f3;
131 /* EFLAGS bit definitions. */
132 #define EFLG_ID (1<<21)
133 #define EFLG_VIP (1<<20)
134 #define EFLG_VIF (1<<19)
135 #define EFLG_AC (1<<18)
136 #define EFLG_VM (1<<17)
137 #define EFLG_RF (1<<16)
138 #define EFLG_IOPL (3<<12)
139 #define EFLG_NT (1<<14)
140 #define EFLG_OF (1<<11)
141 #define EFLG_DF (1<<10)
142 #define EFLG_IF (1<<9)
143 #define EFLG_TF (1<<8)
144 #define EFLG_SF (1<<7)
145 #define EFLG_ZF (1<<6)
146 #define EFLG_AF (1<<4)
147 #define EFLG_PF (1<<2)
148 #define EFLG_CF (1<<0)
150 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
151 #define EFLG_RESERVED_ONE_MASK 2
154 * Instruction emulation:
155 * Most instructions are emulated directly via a fragment of inline assembly
156 * code. This allows us to save/restore EFLAGS and thus very easily pick up
157 * any modified flags.
160 #if defined(CONFIG_X86_64)
161 #define _LO32 "k" /* force 32-bit operand */
162 #define _STK "%%rsp" /* stack pointer */
163 #elif defined(__i386__)
164 #define _LO32 "" /* force 32-bit operand */
165 #define _STK "%%esp" /* stack pointer */
169 * These EFLAGS bits are restored from saved value during emulation, and
170 * any changes are written back to the saved value after emulation.
172 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
174 /* Before executing instruction: restore necessary bits in EFLAGS. */
175 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
176 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
177 "movl %"_sav",%"_LO32 _tmp"; " \
180 "movl %"_msk",%"_LO32 _tmp"; " \
181 "andl %"_LO32 _tmp",("_STK"); " \
183 "notl %"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
185 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
187 "orl %"_LO32 _tmp",("_STK"); " \
191 /* After executing instruction: write-back necessary bits in EFLAGS. */
192 #define _POST_EFLAGS(_sav, _msk, _tmp) \
193 /* _sav |= EFLAGS & _msk; */ \
196 "andl %"_msk",%"_LO32 _tmp"; " \
197 "orl %"_LO32 _tmp",%"_sav"; "
205 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
207 __asm__ __volatile__ ( \
208 _PRE_EFLAGS("0", "4", "2") \
209 _op _suffix " %"_x"3,%1; " \
210 _POST_EFLAGS("0", "4", "2") \
211 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
213 : _y ((_src).val), "i" (EFLAGS_MASK)); \
217 /* Raw emulation: instruction has two explicit operands. */
218 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
220 unsigned long _tmp; \
222 switch ((_dst).bytes) { \
224 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
227 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
230 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
235 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
237 unsigned long _tmp; \
238 switch ((_dst).bytes) { \
240 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
243 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
244 _wx, _wy, _lx, _ly, _qx, _qy); \
249 /* Source operand is byte-sized and may be restricted to just %cl. */
250 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
251 __emulate_2op(_op, _src, _dst, _eflags, \
252 "b", "c", "b", "c", "b", "c", "b", "c")
254 /* Source operand is byte, word, long or quad sized. */
255 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
256 __emulate_2op(_op, _src, _dst, _eflags, \
257 "b", "q", "w", "r", _LO32, "r", "", "r")
259 /* Source operand is word, long or quad sized. */
260 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
261 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
262 "w", "r", _LO32, "r", "", "r")
264 /* Instruction has three operands and one operand is stored in ECX register */
265 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
267 unsigned long _tmp; \
268 _type _clv = (_cl).val; \
269 _type _srcv = (_src).val; \
270 _type _dstv = (_dst).val; \
272 __asm__ __volatile__ ( \
273 _PRE_EFLAGS("0", "5", "2") \
274 _op _suffix " %4,%1 \n" \
275 _POST_EFLAGS("0", "5", "2") \
276 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
277 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
280 (_cl).val = (unsigned long) _clv; \
281 (_src).val = (unsigned long) _srcv; \
282 (_dst).val = (unsigned long) _dstv; \
285 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
287 switch ((_dst).bytes) { \
289 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
290 "w", unsigned short); \
293 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
294 "l", unsigned int); \
297 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
298 "q", unsigned long)); \
303 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
305 unsigned long _tmp; \
307 __asm__ __volatile__ ( \
308 _PRE_EFLAGS("0", "3", "2") \
309 _op _suffix " %1; " \
310 _POST_EFLAGS("0", "3", "2") \
311 : "=m" (_eflags), "+m" ((_dst).val), \
313 : "i" (EFLAGS_MASK)); \
316 /* Instruction has only one explicit operand (no source operand). */
317 #define emulate_1op(_op, _dst, _eflags) \
319 switch ((_dst).bytes) { \
320 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
321 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
322 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
323 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
327 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
329 unsigned long _tmp; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "4", "1") \
333 _op _suffix " %5; " \
334 _POST_EFLAGS("0", "4", "1") \
335 : "=m" (_eflags), "=&r" (_tmp), \
336 "+a" (_rax), "+d" (_rdx) \
337 : "i" (EFLAGS_MASK), "m" ((_src).val), \
338 "a" (_rax), "d" (_rdx)); \
341 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
343 unsigned long _tmp; \
345 __asm__ __volatile__ ( \
346 _PRE_EFLAGS("0", "5", "1") \
348 _op _suffix " %6; " \
350 _POST_EFLAGS("0", "5", "1") \
351 ".pushsection .fixup,\"ax\" \n\t" \
352 "3: movb $1, %4 \n\t" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=m" (_eflags), "=&r" (_tmp), \
357 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
358 : "i" (EFLAGS_MASK), "m" ((_src).val), \
359 "a" (_rax), "d" (_rdx)); \
362 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
363 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
365 switch((_src).bytes) { \
366 case 1: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "b"); break; \
367 case 2: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "w"); break; \
368 case 4: __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "l"); break; \
369 case 8: ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, "q")); break; \
373 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
375 switch((_src).bytes) { \
377 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
378 _eflags, "b", _ex); \
381 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
382 _eflags, "w", _ex); \
385 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
386 _eflags, "l", _ex); \
389 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
390 _eflags, "q", _ex)); \
395 /* Fetch next part of the instruction being emulated. */
396 #define insn_fetch(_type, _size, _eip) \
397 ({ unsigned long _x; \
398 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
399 if (rc != X86EMUL_CONTINUE) \
405 #define insn_fetch_arr(_arr, _size, _eip) \
406 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
407 if (rc != X86EMUL_CONTINUE) \
412 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
413 enum x86_intercept intercept,
414 enum x86_intercept_stage stage)
416 struct x86_instruction_info info = {
417 .intercept = intercept,
418 .rep_prefix = ctxt->decode.rep_prefix,
419 .modrm_mod = ctxt->decode.modrm_mod,
420 .modrm_reg = ctxt->decode.modrm_reg,
421 .modrm_rm = ctxt->decode.modrm_rm,
422 .src_val = ctxt->decode.src.val64,
423 .src_bytes = ctxt->decode.src.bytes,
424 .dst_bytes = ctxt->decode.dst.bytes,
425 .ad_bytes = ctxt->decode.ad_bytes,
426 .next_rip = ctxt->eip,
429 return ctxt->ops->intercept(ctxt->vcpu, &info, stage);
432 static inline unsigned long ad_mask(struct decode_cache *c)
434 return (1UL << (c->ad_bytes << 3)) - 1;
437 /* Access/update address held in a register, based on addressing mode. */
438 static inline unsigned long
439 address_mask(struct decode_cache *c, unsigned long reg)
441 if (c->ad_bytes == sizeof(unsigned long))
444 return reg & ad_mask(c);
447 static inline unsigned long
448 register_address(struct decode_cache *c, unsigned long reg)
450 return address_mask(c, reg);
454 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
456 if (c->ad_bytes == sizeof(unsigned long))
459 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
462 static inline void jmp_rel(struct decode_cache *c, int rel)
464 register_address_increment(c, &c->eip, rel);
467 static u32 desc_limit_scaled(struct desc_struct *desc)
469 u32 limit = get_desc_limit(desc);
471 return desc->g ? (limit << 12) | 0xfff : limit;
474 static void set_seg_override(struct decode_cache *c, int seg)
476 c->has_seg_override = true;
477 c->seg_override = seg;
480 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
481 struct x86_emulate_ops *ops, int seg)
483 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
486 return ops->get_cached_segment_base(seg, ctxt->vcpu);
489 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
490 struct x86_emulate_ops *ops,
491 struct decode_cache *c)
493 if (!c->has_seg_override)
496 return c->seg_override;
499 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
500 u32 error, bool valid)
502 ctxt->exception.vector = vec;
503 ctxt->exception.error_code = error;
504 ctxt->exception.error_code_valid = valid;
505 return X86EMUL_PROPAGATE_FAULT;
508 static int emulate_db(struct x86_emulate_ctxt *ctxt)
510 return emulate_exception(ctxt, DB_VECTOR, 0, false);
513 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
515 return emulate_exception(ctxt, GP_VECTOR, err, true);
518 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
520 return emulate_exception(ctxt, SS_VECTOR, err, true);
523 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
525 return emulate_exception(ctxt, UD_VECTOR, 0, false);
528 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
530 return emulate_exception(ctxt, TS_VECTOR, err, true);
533 static int emulate_de(struct x86_emulate_ctxt *ctxt)
535 return emulate_exception(ctxt, DE_VECTOR, 0, false);
538 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
540 return emulate_exception(ctxt, NM_VECTOR, 0, false);
543 static int linearize(struct x86_emulate_ctxt *ctxt,
544 struct segmented_address addr,
545 unsigned size, bool write,
548 struct decode_cache *c = &ctxt->decode;
549 struct desc_struct desc;
555 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
556 switch (ctxt->mode) {
557 case X86EMUL_MODE_REAL:
559 case X86EMUL_MODE_PROT64:
560 if (((signed long)la << 16) >> 16 != la)
561 return emulate_gp(ctxt, 0);
564 usable = ctxt->ops->get_cached_descriptor(&desc, NULL, addr.seg,
568 /* code segment or read-only data segment */
569 if (((desc.type & 8) || !(desc.type & 2)) && write)
571 /* unreadable code segment */
572 if ((desc.type & 8) && !(desc.type & 2))
574 lim = desc_limit_scaled(&desc);
575 if ((desc.type & 8) || !(desc.type & 4)) {
576 /* expand-up segment */
577 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
580 /* exapand-down segment */
581 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
583 lim = desc.d ? 0xffffffff : 0xffff;
584 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
587 cpl = ctxt->ops->cpl(ctxt->vcpu);
588 rpl = ctxt->ops->get_segment_selector(addr.seg, ctxt->vcpu) & 3;
590 if (!(desc.type & 8)) {
594 } else if ((desc.type & 8) && !(desc.type & 4)) {
595 /* nonconforming code segment */
598 } else if ((desc.type & 8) && (desc.type & 4)) {
599 /* conforming code segment */
605 if (c->ad_bytes != 8)
608 return X86EMUL_CONTINUE;
610 if (addr.seg == VCPU_SREG_SS)
611 return emulate_ss(ctxt, addr.seg);
613 return emulate_gp(ctxt, addr.seg);
616 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
617 struct segmented_address addr,
624 rc = linearize(ctxt, addr, size, false, &linear);
625 if (rc != X86EMUL_CONTINUE)
627 return ctxt->ops->read_std(linear, data, size, ctxt->vcpu,
631 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
632 struct x86_emulate_ops *ops,
633 unsigned long eip, u8 *dest)
635 struct fetch_cache *fc = &ctxt->decode.fetch;
639 if (eip == fc->end) {
640 cur_size = fc->end - fc->start;
641 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
642 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
643 size, ctxt->vcpu, &ctxt->exception);
644 if (rc != X86EMUL_CONTINUE)
648 *dest = fc->data[eip - fc->start];
649 return X86EMUL_CONTINUE;
652 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
653 struct x86_emulate_ops *ops,
654 unsigned long eip, void *dest, unsigned size)
658 /* x86 instructions are limited to 15 bytes. */
659 if (eip + size - ctxt->eip > 15)
660 return X86EMUL_UNHANDLEABLE;
662 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
663 if (rc != X86EMUL_CONTINUE)
666 return X86EMUL_CONTINUE;
670 * Given the 'reg' portion of a ModRM byte, and a register block, return a
671 * pointer into the block that addresses the relevant register.
672 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
674 static void *decode_register(u8 modrm_reg, unsigned long *regs,
679 p = ®s[modrm_reg];
680 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
681 p = (unsigned char *)®s[modrm_reg & 3] + 1;
685 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
686 struct x86_emulate_ops *ops,
687 struct segmented_address addr,
688 u16 *size, unsigned long *address, int op_bytes)
695 rc = segmented_read_std(ctxt, addr, size, 2);
696 if (rc != X86EMUL_CONTINUE)
699 rc = segmented_read_std(ctxt, addr, address, op_bytes);
703 static int test_cc(unsigned int condition, unsigned int flags)
707 switch ((condition & 15) >> 1) {
709 rc |= (flags & EFLG_OF);
711 case 1: /* b/c/nae */
712 rc |= (flags & EFLG_CF);
715 rc |= (flags & EFLG_ZF);
718 rc |= (flags & (EFLG_CF|EFLG_ZF));
721 rc |= (flags & EFLG_SF);
724 rc |= (flags & EFLG_PF);
727 rc |= (flags & EFLG_ZF);
730 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
734 /* Odd condition identifiers (lsb == 1) have inverted sense. */
735 return (!!rc ^ (condition & 1));
738 static void fetch_register_operand(struct operand *op)
742 op->val = *(u8 *)op->addr.reg;
745 op->val = *(u16 *)op->addr.reg;
748 op->val = *(u32 *)op->addr.reg;
751 op->val = *(u64 *)op->addr.reg;
756 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
758 ctxt->ops->get_fpu(ctxt);
760 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
761 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
762 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
763 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
764 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
765 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
766 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
767 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
769 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
770 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
771 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
772 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
773 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
774 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
775 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
776 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
780 ctxt->ops->put_fpu(ctxt);
783 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
786 ctxt->ops->get_fpu(ctxt);
788 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
789 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
790 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
791 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
792 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
793 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
794 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
795 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
797 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
798 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
799 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
800 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
801 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
802 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
803 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
804 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
808 ctxt->ops->put_fpu(ctxt);
811 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
813 struct decode_cache *c,
816 unsigned reg = c->modrm_reg;
817 int highbyte_regs = c->rex_prefix == 0;
820 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
826 read_sse_reg(ctxt, &op->vec_val, reg);
831 if ((c->d & ByteOp) && !inhibit_bytereg) {
832 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
835 op->addr.reg = decode_register(reg, c->regs, 0);
836 op->bytes = c->op_bytes;
838 fetch_register_operand(op);
839 op->orig_val = op->val;
842 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
843 struct x86_emulate_ops *ops,
846 struct decode_cache *c = &ctxt->decode;
848 int index_reg = 0, base_reg = 0, scale;
849 int rc = X86EMUL_CONTINUE;
853 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
854 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
855 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
858 c->modrm = insn_fetch(u8, 1, c->eip);
859 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
860 c->modrm_reg |= (c->modrm & 0x38) >> 3;
861 c->modrm_rm |= (c->modrm & 0x07);
862 c->modrm_seg = VCPU_SREG_DS;
864 if (c->modrm_mod == 3) {
866 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
867 op->addr.reg = decode_register(c->modrm_rm,
868 c->regs, c->d & ByteOp);
872 op->addr.xmm = c->modrm_rm;
873 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
876 fetch_register_operand(op);
882 if (c->ad_bytes == 2) {
883 unsigned bx = c->regs[VCPU_REGS_RBX];
884 unsigned bp = c->regs[VCPU_REGS_RBP];
885 unsigned si = c->regs[VCPU_REGS_RSI];
886 unsigned di = c->regs[VCPU_REGS_RDI];
888 /* 16-bit ModR/M decode. */
889 switch (c->modrm_mod) {
891 if (c->modrm_rm == 6)
892 modrm_ea += insn_fetch(u16, 2, c->eip);
895 modrm_ea += insn_fetch(s8, 1, c->eip);
898 modrm_ea += insn_fetch(u16, 2, c->eip);
901 switch (c->modrm_rm) {
921 if (c->modrm_mod != 0)
928 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
929 (c->modrm_rm == 6 && c->modrm_mod != 0))
930 c->modrm_seg = VCPU_SREG_SS;
931 modrm_ea = (u16)modrm_ea;
933 /* 32/64-bit ModR/M decode. */
934 if ((c->modrm_rm & 7) == 4) {
935 sib = insn_fetch(u8, 1, c->eip);
936 index_reg |= (sib >> 3) & 7;
940 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
941 modrm_ea += insn_fetch(s32, 4, c->eip);
943 modrm_ea += c->regs[base_reg];
945 modrm_ea += c->regs[index_reg] << scale;
946 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
947 if (ctxt->mode == X86EMUL_MODE_PROT64)
950 modrm_ea += c->regs[c->modrm_rm];
951 switch (c->modrm_mod) {
953 if (c->modrm_rm == 5)
954 modrm_ea += insn_fetch(s32, 4, c->eip);
957 modrm_ea += insn_fetch(s8, 1, c->eip);
960 modrm_ea += insn_fetch(s32, 4, c->eip);
964 op->addr.mem.ea = modrm_ea;
969 static int decode_abs(struct x86_emulate_ctxt *ctxt,
970 struct x86_emulate_ops *ops,
973 struct decode_cache *c = &ctxt->decode;
974 int rc = X86EMUL_CONTINUE;
977 switch (c->ad_bytes) {
979 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
982 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
985 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
992 static void fetch_bit_operand(struct decode_cache *c)
996 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
997 mask = ~(c->dst.bytes * 8 - 1);
999 if (c->src.bytes == 2)
1000 sv = (s16)c->src.val & (s16)mask;
1001 else if (c->src.bytes == 4)
1002 sv = (s32)c->src.val & (s32)mask;
1004 c->dst.addr.mem.ea += (sv >> 3);
1007 /* only subword offset */
1008 c->src.val &= (c->dst.bytes << 3) - 1;
1011 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1012 struct x86_emulate_ops *ops,
1013 unsigned long addr, void *dest, unsigned size)
1016 struct read_cache *mc = &ctxt->decode.mem_read;
1019 int n = min(size, 8u);
1021 if (mc->pos < mc->end)
1024 rc = ops->read_emulated(addr, mc->data + mc->end, n,
1025 &ctxt->exception, ctxt->vcpu);
1026 if (rc != X86EMUL_CONTINUE)
1031 memcpy(dest, mc->data + mc->pos, n);
1036 return X86EMUL_CONTINUE;
1039 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1040 struct segmented_address addr,
1047 rc = linearize(ctxt, addr, size, false, &linear);
1048 if (rc != X86EMUL_CONTINUE)
1050 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1053 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1054 struct segmented_address addr,
1061 rc = linearize(ctxt, addr, size, true, &linear);
1062 if (rc != X86EMUL_CONTINUE)
1064 return ctxt->ops->write_emulated(linear, data, size,
1065 &ctxt->exception, ctxt->vcpu);
1068 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1069 struct segmented_address addr,
1070 const void *orig_data, const void *data,
1076 rc = linearize(ctxt, addr, size, true, &linear);
1077 if (rc != X86EMUL_CONTINUE)
1079 return ctxt->ops->cmpxchg_emulated(linear, orig_data, data,
1080 size, &ctxt->exception, ctxt->vcpu);
1083 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1084 struct x86_emulate_ops *ops,
1085 unsigned int size, unsigned short port,
1088 struct read_cache *rc = &ctxt->decode.io_read;
1090 if (rc->pos == rc->end) { /* refill pio read ahead */
1091 struct decode_cache *c = &ctxt->decode;
1092 unsigned int in_page, n;
1093 unsigned int count = c->rep_prefix ?
1094 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1095 in_page = (ctxt->eflags & EFLG_DF) ?
1096 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1097 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1098 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1102 rc->pos = rc->end = 0;
1103 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1108 memcpy(dest, rc->data + rc->pos, size);
1113 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1114 struct x86_emulate_ops *ops,
1115 u16 selector, struct desc_ptr *dt)
1117 if (selector & 1 << 2) {
1118 struct desc_struct desc;
1119 memset (dt, 0, sizeof *dt);
1120 if (!ops->get_cached_descriptor(&desc, NULL, VCPU_SREG_LDTR,
1124 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1125 dt->address = get_desc_base(&desc);
1127 ops->get_gdt(dt, ctxt->vcpu);
1130 /* allowed just for 8 bytes segments */
1131 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1132 struct x86_emulate_ops *ops,
1133 u16 selector, struct desc_struct *desc)
1136 u16 index = selector >> 3;
1140 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1142 if (dt.size < index * 8 + 7)
1143 return emulate_gp(ctxt, selector & 0xfffc);
1144 addr = dt.address + index * 8;
1145 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu,
1151 /* allowed just for 8 bytes segments */
1152 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1153 struct x86_emulate_ops *ops,
1154 u16 selector, struct desc_struct *desc)
1157 u16 index = selector >> 3;
1161 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1163 if (dt.size < index * 8 + 7)
1164 return emulate_gp(ctxt, selector & 0xfffc);
1166 addr = dt.address + index * 8;
1167 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu,
1173 /* Does not support long mode */
1174 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1175 struct x86_emulate_ops *ops,
1176 u16 selector, int seg)
1178 struct desc_struct seg_desc;
1180 unsigned err_vec = GP_VECTOR;
1182 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1185 memset(&seg_desc, 0, sizeof seg_desc);
1187 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1188 || ctxt->mode == X86EMUL_MODE_REAL) {
1189 /* set real mode segment descriptor */
1190 set_desc_base(&seg_desc, selector << 4);
1191 set_desc_limit(&seg_desc, 0xffff);
1198 /* NULL selector is not valid for TR, CS and SS */
1199 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1203 /* TR should be in GDT only */
1204 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1207 if (null_selector) /* for NULL selector skip all following checks */
1210 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1211 if (ret != X86EMUL_CONTINUE)
1214 err_code = selector & 0xfffc;
1215 err_vec = GP_VECTOR;
1217 /* can't load system descriptor into segment selecor */
1218 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1222 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1228 cpl = ops->cpl(ctxt->vcpu);
1233 * segment is not a writable data segment or segment
1234 * selector's RPL != CPL or segment selector's RPL != CPL
1236 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1240 if (!(seg_desc.type & 8))
1243 if (seg_desc.type & 4) {
1249 if (rpl > cpl || dpl != cpl)
1252 /* CS(RPL) <- CPL */
1253 selector = (selector & 0xfffc) | cpl;
1256 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1259 case VCPU_SREG_LDTR:
1260 if (seg_desc.s || seg_desc.type != 2)
1263 default: /* DS, ES, FS, or GS */
1265 * segment is not a data or readable code segment or
1266 * ((segment is a data or nonconforming code segment)
1267 * and (both RPL and CPL > DPL))
1269 if ((seg_desc.type & 0xa) == 0x8 ||
1270 (((seg_desc.type & 0xc) != 0xc) &&
1271 (rpl > dpl && cpl > dpl)))
1277 /* mark segment as accessed */
1279 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1280 if (ret != X86EMUL_CONTINUE)
1284 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1285 ops->set_cached_descriptor(&seg_desc, 0, seg, ctxt->vcpu);
1286 return X86EMUL_CONTINUE;
1288 emulate_exception(ctxt, err_vec, err_code, true);
1289 return X86EMUL_PROPAGATE_FAULT;
1292 static void write_register_operand(struct operand *op)
1294 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1295 switch (op->bytes) {
1297 *(u8 *)op->addr.reg = (u8)op->val;
1300 *(u16 *)op->addr.reg = (u16)op->val;
1303 *op->addr.reg = (u32)op->val;
1304 break; /* 64b: zero-extend */
1306 *op->addr.reg = op->val;
1311 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1312 struct x86_emulate_ops *ops)
1315 struct decode_cache *c = &ctxt->decode;
1317 switch (c->dst.type) {
1319 write_register_operand(&c->dst);
1323 rc = segmented_cmpxchg(ctxt,
1329 rc = segmented_write(ctxt,
1333 if (rc != X86EMUL_CONTINUE)
1337 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1345 return X86EMUL_CONTINUE;
1348 static int em_push(struct x86_emulate_ctxt *ctxt)
1350 struct decode_cache *c = &ctxt->decode;
1351 struct segmented_address addr;
1353 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1354 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1355 addr.seg = VCPU_SREG_SS;
1357 /* Disable writeback. */
1358 c->dst.type = OP_NONE;
1359 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1362 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1363 struct x86_emulate_ops *ops,
1364 void *dest, int len)
1366 struct decode_cache *c = &ctxt->decode;
1368 struct segmented_address addr;
1370 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1371 addr.seg = VCPU_SREG_SS;
1372 rc = segmented_read(ctxt, addr, dest, len);
1373 if (rc != X86EMUL_CONTINUE)
1376 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1380 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1381 struct x86_emulate_ops *ops,
1382 void *dest, int len)
1385 unsigned long val, change_mask;
1386 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1387 int cpl = ops->cpl(ctxt->vcpu);
1389 rc = emulate_pop(ctxt, ops, &val, len);
1390 if (rc != X86EMUL_CONTINUE)
1393 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1394 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1396 switch(ctxt->mode) {
1397 case X86EMUL_MODE_PROT64:
1398 case X86EMUL_MODE_PROT32:
1399 case X86EMUL_MODE_PROT16:
1401 change_mask |= EFLG_IOPL;
1403 change_mask |= EFLG_IF;
1405 case X86EMUL_MODE_VM86:
1407 return emulate_gp(ctxt, 0);
1408 change_mask |= EFLG_IF;
1410 default: /* real mode */
1411 change_mask |= (EFLG_IOPL | EFLG_IF);
1415 *(unsigned long *)dest =
1416 (ctxt->eflags & ~change_mask) | (val & change_mask);
1421 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1422 struct x86_emulate_ops *ops, int seg)
1424 struct decode_cache *c = &ctxt->decode;
1426 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1428 return em_push(ctxt);
1431 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1432 struct x86_emulate_ops *ops, int seg)
1434 struct decode_cache *c = &ctxt->decode;
1435 unsigned long selector;
1438 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1439 if (rc != X86EMUL_CONTINUE)
1442 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1446 static int emulate_pusha(struct x86_emulate_ctxt *ctxt)
1448 struct decode_cache *c = &ctxt->decode;
1449 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1450 int rc = X86EMUL_CONTINUE;
1451 int reg = VCPU_REGS_RAX;
1453 while (reg <= VCPU_REGS_RDI) {
1454 (reg == VCPU_REGS_RSP) ?
1455 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1458 if (rc != X86EMUL_CONTINUE)
1467 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1468 struct x86_emulate_ops *ops)
1470 struct decode_cache *c = &ctxt->decode;
1471 int rc = X86EMUL_CONTINUE;
1472 int reg = VCPU_REGS_RDI;
1474 while (reg >= VCPU_REGS_RAX) {
1475 if (reg == VCPU_REGS_RSP) {
1476 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1481 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1482 if (rc != X86EMUL_CONTINUE)
1489 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1490 struct x86_emulate_ops *ops, int irq)
1492 struct decode_cache *c = &ctxt->decode;
1499 /* TODO: Add limit checks */
1500 c->src.val = ctxt->eflags;
1502 if (rc != X86EMUL_CONTINUE)
1505 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1507 c->src.val = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
1509 if (rc != X86EMUL_CONTINUE)
1512 c->src.val = c->eip;
1514 if (rc != X86EMUL_CONTINUE)
1517 ops->get_idt(&dt, ctxt->vcpu);
1519 eip_addr = dt.address + (irq << 2);
1520 cs_addr = dt.address + (irq << 2) + 2;
1522 rc = ops->read_std(cs_addr, &cs, 2, ctxt->vcpu, &ctxt->exception);
1523 if (rc != X86EMUL_CONTINUE)
1526 rc = ops->read_std(eip_addr, &eip, 2, ctxt->vcpu, &ctxt->exception);
1527 if (rc != X86EMUL_CONTINUE)
1530 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1531 if (rc != X86EMUL_CONTINUE)
1539 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1540 struct x86_emulate_ops *ops, int irq)
1542 switch(ctxt->mode) {
1543 case X86EMUL_MODE_REAL:
1544 return emulate_int_real(ctxt, ops, irq);
1545 case X86EMUL_MODE_VM86:
1546 case X86EMUL_MODE_PROT16:
1547 case X86EMUL_MODE_PROT32:
1548 case X86EMUL_MODE_PROT64:
1550 /* Protected mode interrupts unimplemented yet */
1551 return X86EMUL_UNHANDLEABLE;
1555 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1556 struct x86_emulate_ops *ops)
1558 struct decode_cache *c = &ctxt->decode;
1559 int rc = X86EMUL_CONTINUE;
1560 unsigned long temp_eip = 0;
1561 unsigned long temp_eflags = 0;
1562 unsigned long cs = 0;
1563 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1564 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1565 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1566 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1568 /* TODO: Add stack limit check */
1570 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1572 if (rc != X86EMUL_CONTINUE)
1575 if (temp_eip & ~0xffff)
1576 return emulate_gp(ctxt, 0);
1578 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1580 if (rc != X86EMUL_CONTINUE)
1583 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1585 if (rc != X86EMUL_CONTINUE)
1588 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1590 if (rc != X86EMUL_CONTINUE)
1596 if (c->op_bytes == 4)
1597 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1598 else if (c->op_bytes == 2) {
1599 ctxt->eflags &= ~0xffff;
1600 ctxt->eflags |= temp_eflags;
1603 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1604 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1609 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1610 struct x86_emulate_ops* ops)
1612 switch(ctxt->mode) {
1613 case X86EMUL_MODE_REAL:
1614 return emulate_iret_real(ctxt, ops);
1615 case X86EMUL_MODE_VM86:
1616 case X86EMUL_MODE_PROT16:
1617 case X86EMUL_MODE_PROT32:
1618 case X86EMUL_MODE_PROT64:
1620 /* iret from protected mode unimplemented yet */
1621 return X86EMUL_UNHANDLEABLE;
1625 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1626 struct x86_emulate_ops *ops)
1628 struct decode_cache *c = &ctxt->decode;
1630 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1633 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1635 struct decode_cache *c = &ctxt->decode;
1636 switch (c->modrm_reg) {
1638 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1641 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1644 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1647 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1649 case 4: /* sal/shl */
1650 case 6: /* sal/shl */
1651 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1654 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1657 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1662 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1663 struct x86_emulate_ops *ops)
1665 struct decode_cache *c = &ctxt->decode;
1666 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1667 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1670 switch (c->modrm_reg) {
1671 case 0 ... 1: /* test */
1672 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1675 c->dst.val = ~c->dst.val;
1678 emulate_1op("neg", c->dst, ctxt->eflags);
1681 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1684 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1687 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1691 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1695 return X86EMUL_UNHANDLEABLE;
1698 return emulate_de(ctxt);
1699 return X86EMUL_CONTINUE;
1702 static int emulate_grp45(struct x86_emulate_ctxt *ctxt)
1704 struct decode_cache *c = &ctxt->decode;
1705 int rc = X86EMUL_CONTINUE;
1707 switch (c->modrm_reg) {
1709 emulate_1op("inc", c->dst, ctxt->eflags);
1712 emulate_1op("dec", c->dst, ctxt->eflags);
1714 case 2: /* call near abs */ {
1717 c->eip = c->src.val;
1718 c->src.val = old_eip;
1722 case 4: /* jmp abs */
1723 c->eip = c->src.val;
1732 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1733 struct x86_emulate_ops *ops)
1735 struct decode_cache *c = &ctxt->decode;
1736 u64 old = c->dst.orig_val64;
1738 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1739 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1740 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1741 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1742 ctxt->eflags &= ~EFLG_ZF;
1744 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1745 (u32) c->regs[VCPU_REGS_RBX];
1747 ctxt->eflags |= EFLG_ZF;
1749 return X86EMUL_CONTINUE;
1752 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1753 struct x86_emulate_ops *ops)
1755 struct decode_cache *c = &ctxt->decode;
1759 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1760 if (rc != X86EMUL_CONTINUE)
1762 if (c->op_bytes == 4)
1763 c->eip = (u32)c->eip;
1764 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1765 if (rc != X86EMUL_CONTINUE)
1767 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1771 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1772 struct x86_emulate_ops *ops, int seg)
1774 struct decode_cache *c = &ctxt->decode;
1778 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1780 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1781 if (rc != X86EMUL_CONTINUE)
1784 c->dst.val = c->src.val;
1789 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1790 struct x86_emulate_ops *ops, struct desc_struct *cs,
1791 struct desc_struct *ss)
1793 memset(cs, 0, sizeof(struct desc_struct));
1794 ops->get_cached_descriptor(cs, NULL, VCPU_SREG_CS, ctxt->vcpu);
1795 memset(ss, 0, sizeof(struct desc_struct));
1797 cs->l = 0; /* will be adjusted later */
1798 set_desc_base(cs, 0); /* flat segment */
1799 cs->g = 1; /* 4kb granularity */
1800 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1801 cs->type = 0x0b; /* Read, Execute, Accessed */
1803 cs->dpl = 0; /* will be adjusted later */
1807 set_desc_base(ss, 0); /* flat segment */
1808 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1809 ss->g = 1; /* 4kb granularity */
1811 ss->type = 0x03; /* Read/Write, Accessed */
1812 ss->d = 1; /* 32bit stack segment */
1818 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1820 struct decode_cache *c = &ctxt->decode;
1821 struct desc_struct cs, ss;
1825 /* syscall is not available in real mode */
1826 if (ctxt->mode == X86EMUL_MODE_REAL ||
1827 ctxt->mode == X86EMUL_MODE_VM86)
1828 return emulate_ud(ctxt);
1830 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1832 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1834 cs_sel = (u16)(msr_data & 0xfffc);
1835 ss_sel = (u16)(msr_data + 8);
1837 if (is_long_mode(ctxt->vcpu)) {
1841 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1842 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1843 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1844 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1846 c->regs[VCPU_REGS_RCX] = c->eip;
1847 if (is_long_mode(ctxt->vcpu)) {
1848 #ifdef CONFIG_X86_64
1849 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1851 ops->get_msr(ctxt->vcpu,
1852 ctxt->mode == X86EMUL_MODE_PROT64 ?
1853 MSR_LSTAR : MSR_CSTAR, &msr_data);
1856 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
1857 ctxt->eflags &= ~(msr_data | EFLG_RF);
1861 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
1862 c->eip = (u32)msr_data;
1864 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1867 return X86EMUL_CONTINUE;
1871 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1873 struct decode_cache *c = &ctxt->decode;
1874 struct desc_struct cs, ss;
1878 /* inject #GP if in real mode */
1879 if (ctxt->mode == X86EMUL_MODE_REAL)
1880 return emulate_gp(ctxt, 0);
1882 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1883 * Therefore, we inject an #UD.
1885 if (ctxt->mode == X86EMUL_MODE_PROT64)
1886 return emulate_ud(ctxt);
1888 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1890 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1891 switch (ctxt->mode) {
1892 case X86EMUL_MODE_PROT32:
1893 if ((msr_data & 0xfffc) == 0x0)
1894 return emulate_gp(ctxt, 0);
1896 case X86EMUL_MODE_PROT64:
1897 if (msr_data == 0x0)
1898 return emulate_gp(ctxt, 0);
1902 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1903 cs_sel = (u16)msr_data;
1904 cs_sel &= ~SELECTOR_RPL_MASK;
1905 ss_sel = cs_sel + 8;
1906 ss_sel &= ~SELECTOR_RPL_MASK;
1907 if (ctxt->mode == X86EMUL_MODE_PROT64
1908 || is_long_mode(ctxt->vcpu)) {
1913 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1914 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1915 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1916 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1918 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
1921 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
1922 c->regs[VCPU_REGS_RSP] = msr_data;
1924 return X86EMUL_CONTINUE;
1928 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1930 struct decode_cache *c = &ctxt->decode;
1931 struct desc_struct cs, ss;
1936 /* inject #GP if in real mode or Virtual 8086 mode */
1937 if (ctxt->mode == X86EMUL_MODE_REAL ||
1938 ctxt->mode == X86EMUL_MODE_VM86)
1939 return emulate_gp(ctxt, 0);
1941 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1943 if ((c->rex_prefix & 0x8) != 0x0)
1944 usermode = X86EMUL_MODE_PROT64;
1946 usermode = X86EMUL_MODE_PROT32;
1950 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
1952 case X86EMUL_MODE_PROT32:
1953 cs_sel = (u16)(msr_data + 16);
1954 if ((msr_data & 0xfffc) == 0x0)
1955 return emulate_gp(ctxt, 0);
1956 ss_sel = (u16)(msr_data + 24);
1958 case X86EMUL_MODE_PROT64:
1959 cs_sel = (u16)(msr_data + 32);
1960 if (msr_data == 0x0)
1961 return emulate_gp(ctxt, 0);
1962 ss_sel = cs_sel + 8;
1967 cs_sel |= SELECTOR_RPL_MASK;
1968 ss_sel |= SELECTOR_RPL_MASK;
1970 ops->set_cached_descriptor(&cs, 0, VCPU_SREG_CS, ctxt->vcpu);
1971 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
1972 ops->set_cached_descriptor(&ss, 0, VCPU_SREG_SS, ctxt->vcpu);
1973 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
1975 c->eip = c->regs[VCPU_REGS_RDX];
1976 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
1978 return X86EMUL_CONTINUE;
1981 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
1982 struct x86_emulate_ops *ops)
1985 if (ctxt->mode == X86EMUL_MODE_REAL)
1987 if (ctxt->mode == X86EMUL_MODE_VM86)
1989 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1990 return ops->cpl(ctxt->vcpu) > iopl;
1993 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1994 struct x86_emulate_ops *ops,
1997 struct desc_struct tr_seg;
2000 u16 io_bitmap_ptr, perm, bit_idx = port & 0x7;
2001 unsigned mask = (1 << len) - 1;
2004 ops->get_cached_descriptor(&tr_seg, &base3, VCPU_SREG_TR, ctxt->vcpu);
2007 if (desc_limit_scaled(&tr_seg) < 103)
2009 base = get_desc_base(&tr_seg);
2010 #ifdef CONFIG_X86_64
2011 base |= ((u64)base3) << 32;
2013 r = ops->read_std(base + 102, &io_bitmap_ptr, 2, ctxt->vcpu, NULL);
2014 if (r != X86EMUL_CONTINUE)
2016 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2018 r = ops->read_std(base + io_bitmap_ptr + port/8, &perm, 2, ctxt->vcpu,
2020 if (r != X86EMUL_CONTINUE)
2022 if ((perm >> bit_idx) & mask)
2027 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2028 struct x86_emulate_ops *ops,
2034 if (emulator_bad_iopl(ctxt, ops))
2035 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2038 ctxt->perm_ok = true;
2043 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2044 struct x86_emulate_ops *ops,
2045 struct tss_segment_16 *tss)
2047 struct decode_cache *c = &ctxt->decode;
2050 tss->flag = ctxt->eflags;
2051 tss->ax = c->regs[VCPU_REGS_RAX];
2052 tss->cx = c->regs[VCPU_REGS_RCX];
2053 tss->dx = c->regs[VCPU_REGS_RDX];
2054 tss->bx = c->regs[VCPU_REGS_RBX];
2055 tss->sp = c->regs[VCPU_REGS_RSP];
2056 tss->bp = c->regs[VCPU_REGS_RBP];
2057 tss->si = c->regs[VCPU_REGS_RSI];
2058 tss->di = c->regs[VCPU_REGS_RDI];
2060 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2061 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2062 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2063 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2064 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2067 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2068 struct x86_emulate_ops *ops,
2069 struct tss_segment_16 *tss)
2071 struct decode_cache *c = &ctxt->decode;
2075 ctxt->eflags = tss->flag | 2;
2076 c->regs[VCPU_REGS_RAX] = tss->ax;
2077 c->regs[VCPU_REGS_RCX] = tss->cx;
2078 c->regs[VCPU_REGS_RDX] = tss->dx;
2079 c->regs[VCPU_REGS_RBX] = tss->bx;
2080 c->regs[VCPU_REGS_RSP] = tss->sp;
2081 c->regs[VCPU_REGS_RBP] = tss->bp;
2082 c->regs[VCPU_REGS_RSI] = tss->si;
2083 c->regs[VCPU_REGS_RDI] = tss->di;
2086 * SDM says that segment selectors are loaded before segment
2089 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2090 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2091 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2092 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2093 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2096 * Now load segment descriptors. If fault happenes at this stage
2097 * it is handled in a context of new task
2099 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2100 if (ret != X86EMUL_CONTINUE)
2102 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2103 if (ret != X86EMUL_CONTINUE)
2105 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2106 if (ret != X86EMUL_CONTINUE)
2108 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2109 if (ret != X86EMUL_CONTINUE)
2111 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2112 if (ret != X86EMUL_CONTINUE)
2115 return X86EMUL_CONTINUE;
2118 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2119 struct x86_emulate_ops *ops,
2120 u16 tss_selector, u16 old_tss_sel,
2121 ulong old_tss_base, struct desc_struct *new_desc)
2123 struct tss_segment_16 tss_seg;
2125 u32 new_tss_base = get_desc_base(new_desc);
2127 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2129 if (ret != X86EMUL_CONTINUE)
2130 /* FIXME: need to provide precise fault address */
2133 save_state_to_tss16(ctxt, ops, &tss_seg);
2135 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2137 if (ret != X86EMUL_CONTINUE)
2138 /* FIXME: need to provide precise fault address */
2141 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2143 if (ret != X86EMUL_CONTINUE)
2144 /* FIXME: need to provide precise fault address */
2147 if (old_tss_sel != 0xffff) {
2148 tss_seg.prev_task_link = old_tss_sel;
2150 ret = ops->write_std(new_tss_base,
2151 &tss_seg.prev_task_link,
2152 sizeof tss_seg.prev_task_link,
2153 ctxt->vcpu, &ctxt->exception);
2154 if (ret != X86EMUL_CONTINUE)
2155 /* FIXME: need to provide precise fault address */
2159 return load_state_from_tss16(ctxt, ops, &tss_seg);
2162 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2163 struct x86_emulate_ops *ops,
2164 struct tss_segment_32 *tss)
2166 struct decode_cache *c = &ctxt->decode;
2168 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2170 tss->eflags = ctxt->eflags;
2171 tss->eax = c->regs[VCPU_REGS_RAX];
2172 tss->ecx = c->regs[VCPU_REGS_RCX];
2173 tss->edx = c->regs[VCPU_REGS_RDX];
2174 tss->ebx = c->regs[VCPU_REGS_RBX];
2175 tss->esp = c->regs[VCPU_REGS_RSP];
2176 tss->ebp = c->regs[VCPU_REGS_RBP];
2177 tss->esi = c->regs[VCPU_REGS_RSI];
2178 tss->edi = c->regs[VCPU_REGS_RDI];
2180 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2181 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2182 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2183 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2184 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2185 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2186 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2189 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2190 struct x86_emulate_ops *ops,
2191 struct tss_segment_32 *tss)
2193 struct decode_cache *c = &ctxt->decode;
2196 if (ops->set_cr(3, tss->cr3, ctxt->vcpu))
2197 return emulate_gp(ctxt, 0);
2199 ctxt->eflags = tss->eflags | 2;
2200 c->regs[VCPU_REGS_RAX] = tss->eax;
2201 c->regs[VCPU_REGS_RCX] = tss->ecx;
2202 c->regs[VCPU_REGS_RDX] = tss->edx;
2203 c->regs[VCPU_REGS_RBX] = tss->ebx;
2204 c->regs[VCPU_REGS_RSP] = tss->esp;
2205 c->regs[VCPU_REGS_RBP] = tss->ebp;
2206 c->regs[VCPU_REGS_RSI] = tss->esi;
2207 c->regs[VCPU_REGS_RDI] = tss->edi;
2210 * SDM says that segment selectors are loaded before segment
2213 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2214 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2215 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2216 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2217 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2218 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2219 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2222 * Now load segment descriptors. If fault happenes at this stage
2223 * it is handled in a context of new task
2225 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2226 if (ret != X86EMUL_CONTINUE)
2228 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2229 if (ret != X86EMUL_CONTINUE)
2231 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2232 if (ret != X86EMUL_CONTINUE)
2234 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2235 if (ret != X86EMUL_CONTINUE)
2237 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2238 if (ret != X86EMUL_CONTINUE)
2240 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2241 if (ret != X86EMUL_CONTINUE)
2243 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2244 if (ret != X86EMUL_CONTINUE)
2247 return X86EMUL_CONTINUE;
2250 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2251 struct x86_emulate_ops *ops,
2252 u16 tss_selector, u16 old_tss_sel,
2253 ulong old_tss_base, struct desc_struct *new_desc)
2255 struct tss_segment_32 tss_seg;
2257 u32 new_tss_base = get_desc_base(new_desc);
2259 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2261 if (ret != X86EMUL_CONTINUE)
2262 /* FIXME: need to provide precise fault address */
2265 save_state_to_tss32(ctxt, ops, &tss_seg);
2267 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2269 if (ret != X86EMUL_CONTINUE)
2270 /* FIXME: need to provide precise fault address */
2273 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2275 if (ret != X86EMUL_CONTINUE)
2276 /* FIXME: need to provide precise fault address */
2279 if (old_tss_sel != 0xffff) {
2280 tss_seg.prev_task_link = old_tss_sel;
2282 ret = ops->write_std(new_tss_base,
2283 &tss_seg.prev_task_link,
2284 sizeof tss_seg.prev_task_link,
2285 ctxt->vcpu, &ctxt->exception);
2286 if (ret != X86EMUL_CONTINUE)
2287 /* FIXME: need to provide precise fault address */
2291 return load_state_from_tss32(ctxt, ops, &tss_seg);
2294 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2295 struct x86_emulate_ops *ops,
2296 u16 tss_selector, int reason,
2297 bool has_error_code, u32 error_code)
2299 struct desc_struct curr_tss_desc, next_tss_desc;
2301 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2302 ulong old_tss_base =
2303 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2306 /* FIXME: old_tss_base == ~0 ? */
2308 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2309 if (ret != X86EMUL_CONTINUE)
2311 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2312 if (ret != X86EMUL_CONTINUE)
2315 /* FIXME: check that next_tss_desc is tss */
2317 if (reason != TASK_SWITCH_IRET) {
2318 if ((tss_selector & 3) > next_tss_desc.dpl ||
2319 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl)
2320 return emulate_gp(ctxt, 0);
2323 desc_limit = desc_limit_scaled(&next_tss_desc);
2324 if (!next_tss_desc.p ||
2325 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2326 desc_limit < 0x2b)) {
2327 emulate_ts(ctxt, tss_selector & 0xfffc);
2328 return X86EMUL_PROPAGATE_FAULT;
2331 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2332 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2333 write_segment_descriptor(ctxt, ops, old_tss_sel,
2337 if (reason == TASK_SWITCH_IRET)
2338 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2340 /* set back link to prev task only if NT bit is set in eflags
2341 note that old_tss_sel is not used afetr this point */
2342 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2343 old_tss_sel = 0xffff;
2345 if (next_tss_desc.type & 8)
2346 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2347 old_tss_base, &next_tss_desc);
2349 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2350 old_tss_base, &next_tss_desc);
2351 if (ret != X86EMUL_CONTINUE)
2354 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2355 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2357 if (reason != TASK_SWITCH_IRET) {
2358 next_tss_desc.type |= (1 << 1); /* set busy flag */
2359 write_segment_descriptor(ctxt, ops, tss_selector,
2363 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2364 ops->set_cached_descriptor(&next_tss_desc, 0, VCPU_SREG_TR, ctxt->vcpu);
2365 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2367 if (has_error_code) {
2368 struct decode_cache *c = &ctxt->decode;
2370 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2372 c->src.val = (unsigned long) error_code;
2373 ret = em_push(ctxt);
2379 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2380 u16 tss_selector, int reason,
2381 bool has_error_code, u32 error_code)
2383 struct x86_emulate_ops *ops = ctxt->ops;
2384 struct decode_cache *c = &ctxt->decode;
2388 c->dst.type = OP_NONE;
2390 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2391 has_error_code, error_code);
2393 if (rc == X86EMUL_CONTINUE)
2396 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2399 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2400 int reg, struct operand *op)
2402 struct decode_cache *c = &ctxt->decode;
2403 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2405 register_address_increment(c, &c->regs[reg], df * op->bytes);
2406 op->addr.mem.ea = register_address(c, c->regs[reg]);
2407 op->addr.mem.seg = seg;
2410 static int em_das(struct x86_emulate_ctxt *ctxt)
2412 struct decode_cache *c = &ctxt->decode;
2414 bool af, cf, old_cf;
2416 cf = ctxt->eflags & X86_EFLAGS_CF;
2422 af = ctxt->eflags & X86_EFLAGS_AF;
2423 if ((al & 0x0f) > 9 || af) {
2425 cf = old_cf | (al >= 250);
2430 if (old_al > 0x99 || old_cf) {
2436 /* Set PF, ZF, SF */
2437 c->src.type = OP_IMM;
2440 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2441 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2443 ctxt->eflags |= X86_EFLAGS_CF;
2445 ctxt->eflags |= X86_EFLAGS_AF;
2446 return X86EMUL_CONTINUE;
2449 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2451 struct decode_cache *c = &ctxt->decode;
2456 old_cs = ctxt->ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2459 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2460 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2461 return X86EMUL_CONTINUE;
2464 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2466 c->src.val = old_cs;
2468 if (rc != X86EMUL_CONTINUE)
2471 c->src.val = old_eip;
2472 return em_push(ctxt);
2475 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2477 struct decode_cache *c = &ctxt->decode;
2480 c->dst.type = OP_REG;
2481 c->dst.addr.reg = &c->eip;
2482 c->dst.bytes = c->op_bytes;
2483 rc = emulate_pop(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
2484 if (rc != X86EMUL_CONTINUE)
2486 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2487 return X86EMUL_CONTINUE;
2490 static int em_imul(struct x86_emulate_ctxt *ctxt)
2492 struct decode_cache *c = &ctxt->decode;
2494 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2495 return X86EMUL_CONTINUE;
2498 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2500 struct decode_cache *c = &ctxt->decode;
2502 c->dst.val = c->src2.val;
2503 return em_imul(ctxt);
2506 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2508 struct decode_cache *c = &ctxt->decode;
2510 c->dst.type = OP_REG;
2511 c->dst.bytes = c->src.bytes;
2512 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2513 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2515 return X86EMUL_CONTINUE;
2518 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2520 struct decode_cache *c = &ctxt->decode;
2523 ctxt->ops->get_msr(ctxt->vcpu, MSR_IA32_TSC, &tsc);
2524 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2525 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2526 return X86EMUL_CONTINUE;
2529 static int em_mov(struct x86_emulate_ctxt *ctxt)
2531 struct decode_cache *c = &ctxt->decode;
2532 c->dst.val = c->src.val;
2533 return X86EMUL_CONTINUE;
2536 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2538 struct decode_cache *c = &ctxt->decode;
2539 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2540 return X86EMUL_CONTINUE;
2543 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2545 struct decode_cache *c = &ctxt->decode;
2549 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2550 if (rc == X86EMUL_CONTINUE)
2551 emulate_invlpg(ctxt->vcpu, linear);
2552 /* Disable writeback. */
2553 c->dst.type = OP_NONE;
2554 return X86EMUL_CONTINUE;
2557 static bool valid_cr(int nr)
2569 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2571 struct decode_cache *c = &ctxt->decode;
2573 if (!valid_cr(c->modrm_reg))
2574 return emulate_ud(ctxt);
2576 return X86EMUL_CONTINUE;
2579 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2581 struct decode_cache *c = &ctxt->decode;
2582 u64 new_val = c->src.val64;
2583 int cr = c->modrm_reg;
2585 static u64 cr_reserved_bits[] = {
2586 0xffffffff00000000ULL,
2587 0, 0, 0, /* CR3 checked later */
2594 return emulate_ud(ctxt);
2596 if (new_val & cr_reserved_bits[cr])
2597 return emulate_gp(ctxt, 0);
2602 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2603 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2604 return emulate_gp(ctxt, 0);
2606 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2607 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2609 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2610 !(cr4 & X86_CR4_PAE))
2611 return emulate_gp(ctxt, 0);
2618 if (is_long_mode(ctxt->vcpu))
2619 rsvd = CR3_L_MODE_RESERVED_BITS;
2620 else if (is_pae(ctxt->vcpu))
2621 rsvd = CR3_PAE_RESERVED_BITS;
2622 else if (is_paging(ctxt->vcpu))
2623 rsvd = CR3_NONPAE_RESERVED_BITS;
2626 return emulate_gp(ctxt, 0);
2633 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2634 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2636 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2637 return emulate_gp(ctxt, 0);
2643 return X86EMUL_CONTINUE;
2646 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2650 ctxt->ops->get_dr(7, &dr7, ctxt->vcpu);
2652 /* Check if DR7.Global_Enable is set */
2653 return dr7 & (1 << 13);
2656 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2658 struct decode_cache *c = &ctxt->decode;
2659 int dr = c->modrm_reg;
2663 return emulate_ud(ctxt);
2665 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2666 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2667 return emulate_ud(ctxt);
2669 if (check_dr7_gd(ctxt))
2670 return emulate_db(ctxt);
2672 return X86EMUL_CONTINUE;
2675 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2677 struct decode_cache *c = &ctxt->decode;
2678 u64 new_val = c->src.val64;
2679 int dr = c->modrm_reg;
2681 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2682 return emulate_gp(ctxt, 0);
2684 return check_dr_read(ctxt);
2687 static int check_svme(struct x86_emulate_ctxt *ctxt)
2691 ctxt->ops->get_msr(ctxt->vcpu, MSR_EFER, &efer);
2693 if (!(efer & EFER_SVME))
2694 return emulate_ud(ctxt);
2696 return X86EMUL_CONTINUE;
2699 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2701 u64 rax = kvm_register_read(ctxt->vcpu, VCPU_REGS_RAX);
2703 /* Valid physical address? */
2704 if (rax & 0xffff000000000000)
2705 return emulate_gp(ctxt, 0);
2707 return check_svme(ctxt);
2710 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2712 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2714 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt->vcpu))
2715 return emulate_ud(ctxt);
2717 return X86EMUL_CONTINUE;
2720 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2722 u64 cr4 = ctxt->ops->get_cr(4, ctxt->vcpu);
2723 u64 rcx = kvm_register_read(ctxt->vcpu, VCPU_REGS_RCX);
2725 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt->vcpu)) ||
2727 return emulate_gp(ctxt, 0);
2729 return X86EMUL_CONTINUE;
2732 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2734 struct decode_cache *c = &ctxt->decode;
2736 c->dst.bytes = min(c->dst.bytes, 4u);
2737 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
2738 return emulate_gp(ctxt, 0);
2740 return X86EMUL_CONTINUE;
2743 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2745 struct decode_cache *c = &ctxt->decode;
2747 c->src.bytes = min(c->src.bytes, 4u);
2748 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
2749 return emulate_gp(ctxt, 0);
2751 return X86EMUL_CONTINUE;
2754 #define D(_y) { .flags = (_y) }
2755 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2756 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2757 .check_perm = (_p) }
2759 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2760 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2761 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
2762 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2763 #define II(_f, _e, _i) \
2764 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2765 #define IIP(_f, _e, _i, _p) \
2766 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2767 .check_perm = (_p) }
2768 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2770 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2771 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2772 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2774 #define D6ALU(_f) D2bv((_f) | DstMem | SrcReg | ModRM), \
2775 D2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock), \
2776 D2bv(((_f) & ~Lock) | DstAcc | SrcImm)
2778 static struct opcode group7_rm1[] = {
2779 DI(SrcNone | ModRM | Priv, monitor),
2780 DI(SrcNone | ModRM | Priv, mwait),
2784 static struct opcode group7_rm3[] = {
2785 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2786 DI(SrcNone | ModRM | Prot | VendorSpecific, vmmcall),
2787 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2788 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2789 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2790 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2791 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2792 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2795 static struct opcode group7_rm7[] = {
2797 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2800 static struct opcode group1[] = {
2804 static struct opcode group1A[] = {
2805 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2808 static struct opcode group3[] = {
2809 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2810 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2811 X4(D(SrcMem | ModRM)),
2814 static struct opcode group4[] = {
2815 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
2819 static struct opcode group5[] = {
2820 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2821 D(SrcMem | ModRM | Stack),
2822 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
2823 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
2824 D(SrcMem | ModRM | Stack), N,
2827 static struct opcode group6[] = {
2828 DI(ModRM | Prot, sldt),
2829 DI(ModRM | Prot, str),
2830 DI(ModRM | Prot | Priv, lldt),
2831 DI(ModRM | Prot | Priv, ltr),
2835 static struct group_dual group7 = { {
2836 DI(ModRM | Mov | DstMem | Priv, sgdt),
2837 DI(ModRM | Mov | DstMem | Priv, sidt),
2838 DI(ModRM | SrcMem | Priv, lgdt), DI(ModRM | SrcMem | Priv, lidt),
2839 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2840 DI(SrcMem16 | ModRM | Mov | Priv, lmsw),
2841 DI(SrcMem | ModRM | ByteOp | Priv | NoAccess, invlpg),
2843 D(SrcNone | ModRM | Priv | VendorSpecific), EXT(0, group7_rm1),
2844 N, EXT(0, group7_rm3),
2845 DI(SrcNone | ModRM | DstMem | Mov, smsw), N,
2846 DI(SrcMem16 | ModRM | Mov | Priv, lmsw), EXT(0, group7_rm7),
2849 static struct opcode group8[] = {
2851 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
2852 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
2855 static struct group_dual group9 = { {
2856 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
2858 N, N, N, N, N, N, N, N,
2861 static struct opcode group11[] = {
2862 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
2865 static struct gprefix pfx_0f_6f_0f_7f = {
2866 N, N, N, I(Sse, em_movdqu),
2869 static struct opcode opcode_table[256] = {
2872 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2875 D(ImplicitOps | Stack | No64), N,
2878 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2881 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2885 D6ALU(Lock), N, I(ByteOp | DstAcc | No64, em_das),
2893 X8(I(SrcReg | Stack, em_push)),
2895 X8(D(DstReg | Stack)),
2897 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
2898 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
2901 I(SrcImm | Mov | Stack, em_push),
2902 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
2903 I(SrcImmByte | Mov | Stack, em_push),
2904 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
2905 D2bvIP(DstDI | Mov | String, ins, check_perm_in), /* insb, insw/insd */
2906 D2bvIP(SrcSI | ImplicitOps | String, outs, check_perm_out), /* outsb, outsw/outsd */
2910 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
2911 G(DstMem | SrcImm | ModRM | Group, group1),
2912 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
2913 G(DstMem | SrcImmByte | ModRM | Group, group1),
2914 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
2916 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
2917 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
2918 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
2919 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
2921 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
2923 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
2924 I(SrcImmFAddr | No64, em_call_far), N,
2925 DI(ImplicitOps | Stack, pushf), DI(ImplicitOps | Stack, popf), N, N,
2927 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
2928 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
2929 I2bv(SrcSI | DstDI | Mov | String, em_mov),
2930 D2bv(SrcSI | DstDI | String),
2932 D2bv(DstAcc | SrcImm),
2933 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
2934 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
2935 D2bv(SrcAcc | DstDI | String),
2937 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
2939 X8(I(DstReg | SrcImm | Mov, em_mov)),
2941 D2bv(DstMem | SrcImmByte | ModRM),
2942 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
2943 D(ImplicitOps | Stack),
2944 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
2945 G(ByteOp, group11), G(0, group11),
2947 N, N, N, D(ImplicitOps | Stack),
2948 D(ImplicitOps), DI(SrcImmByte, intn),
2949 D(ImplicitOps | No64), DI(ImplicitOps, iret),
2951 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
2954 N, N, N, N, N, N, N, N,
2957 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
2958 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
2960 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
2961 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
2962 D2bvIP(SrcNone | DstAcc, in, check_perm_in),
2963 D2bvIP(SrcAcc | ImplicitOps, out, check_perm_out),
2965 N, DI(ImplicitOps, icebp), N, N,
2966 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
2967 G(ByteOp, group3), G(0, group3),
2969 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
2970 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
2973 static struct opcode twobyte_table[256] = {
2975 G(0, group6), GD(0, &group7), N, N,
2976 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
2977 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
2978 N, D(ImplicitOps | ModRM), N, N,
2980 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
2982 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
2983 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
2984 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
2985 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
2987 N, N, N, N, N, N, N, N,
2989 DI(ImplicitOps | Priv, wrmsr),
2990 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
2991 DI(ImplicitOps | Priv, rdmsr),
2992 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
2993 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
2995 N, N, N, N, N, N, N, N,
2997 X16(D(DstReg | SrcMem | ModRM | Mov)),
2999 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3004 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3009 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3013 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3015 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3016 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3017 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3018 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3020 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3021 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3022 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3023 D(DstMem | SrcReg | Src2CL | ModRM),
3024 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3026 D2bv(DstMem | SrcReg | ModRM | Lock),
3027 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3028 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3029 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3032 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3033 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3034 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3036 D2bv(DstMem | SrcReg | ModRM | Lock),
3037 N, D(DstMem | SrcReg | ModRM | Mov),
3038 N, N, N, GD(0, &group9),
3039 N, N, N, N, N, N, N, N,
3041 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3043 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3045 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3061 static unsigned imm_size(struct decode_cache *c)
3065 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3071 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3072 unsigned size, bool sign_extension)
3074 struct decode_cache *c = &ctxt->decode;
3075 struct x86_emulate_ops *ops = ctxt->ops;
3076 int rc = X86EMUL_CONTINUE;
3080 op->addr.mem.ea = c->eip;
3081 /* NB. Immediates are sign-extended as necessary. */
3082 switch (op->bytes) {
3084 op->val = insn_fetch(s8, 1, c->eip);
3087 op->val = insn_fetch(s16, 2, c->eip);
3090 op->val = insn_fetch(s32, 4, c->eip);
3093 if (!sign_extension) {
3094 switch (op->bytes) {
3102 op->val &= 0xffffffff;
3111 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3113 struct x86_emulate_ops *ops = ctxt->ops;
3114 struct decode_cache *c = &ctxt->decode;
3115 int rc = X86EMUL_CONTINUE;
3116 int mode = ctxt->mode;
3117 int def_op_bytes, def_ad_bytes, dual, goffset, simd_prefix;
3118 bool op_prefix = false;
3119 struct opcode opcode, *g_mod012, *g_mod3;
3120 struct operand memop = { .type = OP_NONE };
3123 c->fetch.start = c->eip;
3124 c->fetch.end = c->fetch.start + insn_len;
3126 memcpy(c->fetch.data, insn, insn_len);
3127 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
3130 case X86EMUL_MODE_REAL:
3131 case X86EMUL_MODE_VM86:
3132 case X86EMUL_MODE_PROT16:
3133 def_op_bytes = def_ad_bytes = 2;
3135 case X86EMUL_MODE_PROT32:
3136 def_op_bytes = def_ad_bytes = 4;
3138 #ifdef CONFIG_X86_64
3139 case X86EMUL_MODE_PROT64:
3148 c->op_bytes = def_op_bytes;
3149 c->ad_bytes = def_ad_bytes;
3151 /* Legacy prefixes. */
3153 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3154 case 0x66: /* operand-size override */
3156 /* switch between 2/4 bytes */
3157 c->op_bytes = def_op_bytes ^ 6;
3159 case 0x67: /* address-size override */
3160 if (mode == X86EMUL_MODE_PROT64)
3161 /* switch between 4/8 bytes */
3162 c->ad_bytes = def_ad_bytes ^ 12;
3164 /* switch between 2/4 bytes */
3165 c->ad_bytes = def_ad_bytes ^ 6;
3167 case 0x26: /* ES override */
3168 case 0x2e: /* CS override */
3169 case 0x36: /* SS override */
3170 case 0x3e: /* DS override */
3171 set_seg_override(c, (c->b >> 3) & 3);
3173 case 0x64: /* FS override */
3174 case 0x65: /* GS override */
3175 set_seg_override(c, c->b & 7);
3177 case 0x40 ... 0x4f: /* REX */
3178 if (mode != X86EMUL_MODE_PROT64)
3180 c->rex_prefix = c->b;
3182 case 0xf0: /* LOCK */
3185 case 0xf2: /* REPNE/REPNZ */
3186 case 0xf3: /* REP/REPE/REPZ */
3187 c->rep_prefix = c->b;
3193 /* Any legacy prefix after a REX prefix nullifies its effect. */
3201 if (c->rex_prefix & 8)
3202 c->op_bytes = 8; /* REX.W */
3204 /* Opcode byte(s). */
3205 opcode = opcode_table[c->b];
3206 /* Two-byte opcode? */
3209 c->b = insn_fetch(u8, 1, c->eip);
3210 opcode = twobyte_table[c->b];
3212 c->d = opcode.flags;
3215 dual = c->d & GroupDual;
3216 c->modrm = insn_fetch(u8, 1, c->eip);
3219 if (c->d & GroupDual) {
3220 g_mod012 = opcode.u.gdual->mod012;
3221 g_mod3 = opcode.u.gdual->mod3;
3223 g_mod012 = g_mod3 = opcode.u.group;
3225 c->d &= ~(Group | GroupDual);
3227 goffset = (c->modrm >> 3) & 7;
3229 if ((c->modrm >> 6) == 3)
3230 opcode = g_mod3[goffset];
3232 opcode = g_mod012[goffset];
3234 if (opcode.flags & RMExt) {
3235 goffset = c->modrm & 7;
3236 opcode = opcode.u.group[goffset];
3239 c->d |= opcode.flags;
3242 if (c->d & Prefix) {
3243 if (c->rep_prefix && op_prefix)
3244 return X86EMUL_UNHANDLEABLE;
3245 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3246 switch (simd_prefix) {
3247 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3248 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3249 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3250 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3252 c->d |= opcode.flags;
3255 c->execute = opcode.u.execute;
3256 c->check_perm = opcode.check_perm;
3257 c->intercept = opcode.intercept;
3260 if (c->d == 0 || (c->d & Undefined))
3263 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3266 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3269 if (c->d & Op3264) {
3270 if (mode == X86EMUL_MODE_PROT64)
3279 /* ModRM and SIB bytes. */
3281 rc = decode_modrm(ctxt, ops, &memop);
3282 if (!c->has_seg_override)
3283 set_seg_override(c, c->modrm_seg);
3284 } else if (c->d & MemAbs)
3285 rc = decode_abs(ctxt, ops, &memop);
3286 if (rc != X86EMUL_CONTINUE)
3289 if (!c->has_seg_override)
3290 set_seg_override(c, VCPU_SREG_DS);
3292 memop.addr.mem.seg = seg_override(ctxt, ops, c);
3294 if (memop.type == OP_MEM && c->ad_bytes != 8)
3295 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3297 if (memop.type == OP_MEM && c->rip_relative)
3298 memop.addr.mem.ea += c->eip;
3301 * Decode and fetch the source operand: register, memory
3304 switch (c->d & SrcMask) {
3308 decode_register_operand(ctxt, &c->src, c, 0);
3317 memop.bytes = (c->d & ByteOp) ? 1 :
3323 rc = decode_imm(ctxt, &c->src, 2, false);
3326 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3329 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3332 rc = decode_imm(ctxt, &c->src, 1, true);
3335 rc = decode_imm(ctxt, &c->src, 1, false);
3338 c->src.type = OP_REG;
3339 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3340 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3341 fetch_register_operand(&c->src);
3348 c->src.type = OP_MEM;
3349 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3350 c->src.addr.mem.ea =
3351 register_address(c, c->regs[VCPU_REGS_RSI]);
3352 c->src.addr.mem.seg = seg_override(ctxt, ops, c),
3356 c->src.type = OP_IMM;
3357 c->src.addr.mem.ea = c->eip;
3358 c->src.bytes = c->op_bytes + 2;
3359 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3362 memop.bytes = c->op_bytes + 2;
3367 if (rc != X86EMUL_CONTINUE)
3371 * Decode and fetch the second source operand: register, memory
3374 switch (c->d & Src2Mask) {
3379 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3382 rc = decode_imm(ctxt, &c->src2, 1, true);
3389 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3393 if (rc != X86EMUL_CONTINUE)
3396 /* Decode and fetch the destination operand: register or memory. */
3397 switch (c->d & DstMask) {
3399 decode_register_operand(ctxt, &c->dst, c,
3400 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3403 c->dst.type = OP_IMM;
3404 c->dst.addr.mem.ea = c->eip;
3406 c->dst.val = insn_fetch(u8, 1, c->eip);
3411 if ((c->d & DstMask) == DstMem64)
3414 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3416 fetch_bit_operand(c);
3417 c->dst.orig_val = c->dst.val;
3420 c->dst.type = OP_REG;
3421 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3422 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3423 fetch_register_operand(&c->dst);
3424 c->dst.orig_val = c->dst.val;
3427 c->dst.type = OP_MEM;
3428 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3429 c->dst.addr.mem.ea =
3430 register_address(c, c->regs[VCPU_REGS_RDI]);
3431 c->dst.addr.mem.seg = VCPU_SREG_ES;
3435 /* Special instructions do their own operand decoding. */
3437 c->dst.type = OP_NONE; /* Disable writeback. */
3442 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3445 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3447 struct decode_cache *c = &ctxt->decode;
3449 /* The second termination condition only applies for REPE
3450 * and REPNE. Test if the repeat string operation prefix is
3451 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3452 * corresponding termination condition according to:
3453 * - if REPE/REPZ and ZF = 0 then done
3454 * - if REPNE/REPNZ and ZF = 1 then done
3456 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3457 (c->b == 0xae) || (c->b == 0xaf))
3458 && (((c->rep_prefix == REPE_PREFIX) &&
3459 ((ctxt->eflags & EFLG_ZF) == 0))
3460 || ((c->rep_prefix == REPNE_PREFIX) &&
3461 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3468 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3470 struct x86_emulate_ops *ops = ctxt->ops;
3472 struct decode_cache *c = &ctxt->decode;
3473 int rc = X86EMUL_CONTINUE;
3474 int saved_dst_type = c->dst.type;
3475 int irq; /* Used for int 3, int, and into */
3477 ctxt->decode.mem_read.pos = 0;
3479 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3480 rc = emulate_ud(ctxt);
3484 /* LOCK prefix is allowed only with some instructions */
3485 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3486 rc = emulate_ud(ctxt);
3490 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3491 rc = emulate_ud(ctxt);
3496 && ((ops->get_cr(0, ctxt->vcpu) & X86_CR0_EM)
3497 || !(ops->get_cr(4, ctxt->vcpu) & X86_CR4_OSFXSR))) {
3498 rc = emulate_ud(ctxt);
3502 if ((c->d & Sse) && (ops->get_cr(0, ctxt->vcpu) & X86_CR0_TS)) {
3503 rc = emulate_nm(ctxt);
3507 if (unlikely(ctxt->guest_mode) && c->intercept) {
3508 rc = emulator_check_intercept(ctxt, c->intercept,
3509 X86_ICPT_PRE_EXCEPT);
3510 if (rc != X86EMUL_CONTINUE)
3514 /* Privileged instruction can be executed only in CPL=0 */
3515 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
3516 rc = emulate_gp(ctxt, 0);
3520 /* Instruction can only be executed in protected mode */
3521 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3522 rc = emulate_ud(ctxt);
3526 /* Do instruction specific permission checks */
3527 if (c->check_perm) {
3528 rc = c->check_perm(ctxt);
3529 if (rc != X86EMUL_CONTINUE)
3533 if (unlikely(ctxt->guest_mode) && c->intercept) {
3534 rc = emulator_check_intercept(ctxt, c->intercept,
3535 X86_ICPT_POST_EXCEPT);
3536 if (rc != X86EMUL_CONTINUE)
3540 if (c->rep_prefix && (c->d & String)) {
3541 /* All REP prefixes have the same first termination condition */
3542 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3548 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3549 rc = segmented_read(ctxt, c->src.addr.mem,
3550 c->src.valptr, c->src.bytes);
3551 if (rc != X86EMUL_CONTINUE)
3553 c->src.orig_val64 = c->src.val64;
3556 if (c->src2.type == OP_MEM) {
3557 rc = segmented_read(ctxt, c->src2.addr.mem,
3558 &c->src2.val, c->src2.bytes);
3559 if (rc != X86EMUL_CONTINUE)
3563 if ((c->d & DstMask) == ImplicitOps)
3567 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3568 /* optimisation - avoid slow emulated read if Mov */
3569 rc = segmented_read(ctxt, c->dst.addr.mem,
3570 &c->dst.val, c->dst.bytes);
3571 if (rc != X86EMUL_CONTINUE)
3574 c->dst.orig_val = c->dst.val;
3578 if (unlikely(ctxt->guest_mode) && c->intercept) {
3579 rc = emulator_check_intercept(ctxt, c->intercept,
3580 X86_ICPT_POST_MEMACCESS);
3581 if (rc != X86EMUL_CONTINUE)
3586 rc = c->execute(ctxt);
3587 if (rc != X86EMUL_CONTINUE)
3598 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
3600 case 0x06: /* push es */
3601 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3603 case 0x07: /* pop es */
3604 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3608 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
3610 case 0x0e: /* push cs */
3611 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3615 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
3617 case 0x16: /* push ss */
3618 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3620 case 0x17: /* pop ss */
3621 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3625 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
3627 case 0x1e: /* push ds */
3628 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3630 case 0x1f: /* pop ds */
3631 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3635 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
3639 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
3643 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
3647 c->dst.type = OP_NONE; /* Disable writeback. */
3648 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3650 case 0x40 ... 0x47: /* inc r16/r32 */
3651 emulate_1op("inc", c->dst, ctxt->eflags);
3653 case 0x48 ... 0x4f: /* dec r16/r32 */
3654 emulate_1op("dec", c->dst, ctxt->eflags);
3656 case 0x58 ... 0x5f: /* pop reg */
3658 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
3660 case 0x60: /* pusha */
3661 rc = emulate_pusha(ctxt);
3663 case 0x61: /* popa */
3664 rc = emulate_popa(ctxt, ops);
3666 case 0x63: /* movsxd */
3667 if (ctxt->mode != X86EMUL_MODE_PROT64)
3668 goto cannot_emulate;
3669 c->dst.val = (s32) c->src.val;
3671 case 0x6c: /* insb */
3672 case 0x6d: /* insw/insd */
3673 c->src.val = c->regs[VCPU_REGS_RDX];
3675 case 0x6e: /* outsb */
3676 case 0x6f: /* outsw/outsd */
3677 c->dst.val = c->regs[VCPU_REGS_RDX];
3680 case 0x70 ... 0x7f: /* jcc (short) */
3681 if (test_cc(c->b, ctxt->eflags))
3682 jmp_rel(c, c->src.val);
3684 case 0x80 ... 0x83: /* Grp1 */
3685 switch (c->modrm_reg) {
3706 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3708 case 0x86 ... 0x87: /* xchg */
3710 /* Write back the register source. */
3711 c->src.val = c->dst.val;
3712 write_register_operand(&c->src);
3714 * Write back the memory destination with implicit LOCK
3717 c->dst.val = c->src.orig_val;
3720 case 0x8c: /* mov r/m, sreg */
3721 if (c->modrm_reg > VCPU_SREG_GS) {
3722 rc = emulate_ud(ctxt);
3725 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
3727 case 0x8d: /* lea r16/r32, m */
3728 c->dst.val = c->src.addr.mem.ea;
3730 case 0x8e: { /* mov seg, r/m16 */
3735 if (c->modrm_reg == VCPU_SREG_CS ||
3736 c->modrm_reg > VCPU_SREG_GS) {
3737 rc = emulate_ud(ctxt);
3741 if (c->modrm_reg == VCPU_SREG_SS)
3742 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3744 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
3746 c->dst.type = OP_NONE; /* Disable writeback. */
3749 case 0x8f: /* pop (sole member of Grp1a) */
3750 rc = emulate_grp1a(ctxt, ops);
3752 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3753 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3756 case 0x98: /* cbw/cwde/cdqe */
3757 switch (c->op_bytes) {
3758 case 2: c->dst.val = (s8)c->dst.val; break;
3759 case 4: c->dst.val = (s16)c->dst.val; break;
3760 case 8: c->dst.val = (s32)c->dst.val; break;
3763 case 0x9c: /* pushf */
3764 c->src.val = (unsigned long) ctxt->eflags;
3767 case 0x9d: /* popf */
3768 c->dst.type = OP_REG;
3769 c->dst.addr.reg = &ctxt->eflags;
3770 c->dst.bytes = c->op_bytes;
3771 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
3773 case 0xa6 ... 0xa7: /* cmps */
3775 case 0xa8 ... 0xa9: /* test ax, imm */
3777 case 0xae ... 0xaf: /* scas */
3782 case 0xc3: /* ret */
3783 c->dst.type = OP_REG;
3784 c->dst.addr.reg = &c->eip;
3785 c->dst.bytes = c->op_bytes;
3786 goto pop_instruction;
3787 case 0xc4: /* les */
3788 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
3790 case 0xc5: /* lds */
3791 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
3793 case 0xcb: /* ret far */
3794 rc = emulate_ret_far(ctxt, ops);
3796 case 0xcc: /* int3 */
3799 case 0xcd: /* int n */
3802 rc = emulate_int(ctxt, ops, irq);
3804 case 0xce: /* into */
3805 if (ctxt->eflags & EFLG_OF) {
3810 case 0xcf: /* iret */
3811 rc = emulate_iret(ctxt, ops);
3813 case 0xd0 ... 0xd1: /* Grp2 */
3816 case 0xd2 ... 0xd3: /* Grp2 */
3817 c->src.val = c->regs[VCPU_REGS_RCX];
3820 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3821 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3822 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3823 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3824 jmp_rel(c, c->src.val);
3826 case 0xe3: /* jcxz/jecxz/jrcxz */
3827 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3828 jmp_rel(c, c->src.val);
3830 case 0xe4: /* inb */
3833 case 0xe6: /* outb */
3834 case 0xe7: /* out */
3836 case 0xe8: /* call (near) */ {
3837 long int rel = c->src.val;
3838 c->src.val = (unsigned long) c->eip;
3843 case 0xe9: /* jmp rel */
3845 case 0xea: { /* jmp far */
3848 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3850 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3854 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3858 jmp: /* jmp rel short */
3859 jmp_rel(c, c->src.val);
3860 c->dst.type = OP_NONE; /* Disable writeback. */
3862 case 0xec: /* in al,dx */
3863 case 0xed: /* in (e/r)ax,dx */
3864 c->src.val = c->regs[VCPU_REGS_RDX];
3866 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3868 goto done; /* IO is needed */
3870 case 0xee: /* out dx,al */
3871 case 0xef: /* out dx,(e/r)ax */
3872 c->dst.val = c->regs[VCPU_REGS_RDX];
3874 ops->pio_out_emulated(c->src.bytes, c->dst.val,
3875 &c->src.val, 1, ctxt->vcpu);
3876 c->dst.type = OP_NONE; /* Disable writeback. */
3878 case 0xf4: /* hlt */
3879 ctxt->vcpu->arch.halt_request = 1;
3881 case 0xf5: /* cmc */
3882 /* complement carry flag from eflags reg */
3883 ctxt->eflags ^= EFLG_CF;
3885 case 0xf6 ... 0xf7: /* Grp3 */
3886 rc = emulate_grp3(ctxt, ops);
3888 case 0xf8: /* clc */
3889 ctxt->eflags &= ~EFLG_CF;
3891 case 0xf9: /* stc */
3892 ctxt->eflags |= EFLG_CF;
3894 case 0xfa: /* cli */
3895 if (emulator_bad_iopl(ctxt, ops)) {
3896 rc = emulate_gp(ctxt, 0);
3899 ctxt->eflags &= ~X86_EFLAGS_IF;
3901 case 0xfb: /* sti */
3902 if (emulator_bad_iopl(ctxt, ops)) {
3903 rc = emulate_gp(ctxt, 0);
3906 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3907 ctxt->eflags |= X86_EFLAGS_IF;
3910 case 0xfc: /* cld */
3911 ctxt->eflags &= ~EFLG_DF;
3913 case 0xfd: /* std */
3914 ctxt->eflags |= EFLG_DF;
3916 case 0xfe: /* Grp4 */
3918 rc = emulate_grp45(ctxt);
3920 case 0xff: /* Grp5 */
3921 if (c->modrm_reg == 5)
3925 goto cannot_emulate;
3928 if (rc != X86EMUL_CONTINUE)
3932 rc = writeback(ctxt, ops);
3933 if (rc != X86EMUL_CONTINUE)
3937 * restore dst type in case the decoding will be reused
3938 * (happens for string instruction )
3940 c->dst.type = saved_dst_type;
3942 if ((c->d & SrcMask) == SrcSI)
3943 string_addr_inc(ctxt, seg_override(ctxt, ops, c),
3944 VCPU_REGS_RSI, &c->src);
3946 if ((c->d & DstMask) == DstDI)
3947 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3950 if (c->rep_prefix && (c->d & String)) {
3951 struct read_cache *r = &ctxt->decode.io_read;
3952 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3954 if (!string_insn_completed(ctxt)) {
3956 * Re-enter guest when pio read ahead buffer is empty
3957 * or, if it is not used, after each 1024 iteration.
3959 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
3960 (r->end == 0 || r->end != r->pos)) {
3962 * Reset read cache. Usually happens before
3963 * decode, but since instruction is restarted
3964 * we have to do it here.
3966 ctxt->decode.mem_read.end = 0;
3967 return EMULATION_RESTART;
3969 goto done; /* skip rip writeback */
3976 if (rc == X86EMUL_PROPAGATE_FAULT)
3977 ctxt->have_exception = true;
3978 if (rc == X86EMUL_INTERCEPTED)
3979 return EMULATION_INTERCEPTED;
3981 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3985 case 0x01: /* lgdt, lidt, lmsw */
3986 switch (c->modrm_reg) {
3988 unsigned long address;
3990 case 0: /* vmcall */
3991 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3992 goto cannot_emulate;
3994 rc = kvm_fix_hypercall(ctxt->vcpu);
3995 if (rc != X86EMUL_CONTINUE)
3998 /* Let the processor re-execute the fixed hypercall */
4000 /* Disable writeback. */
4001 c->dst.type = OP_NONE;
4004 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4005 &size, &address, c->op_bytes);
4006 if (rc != X86EMUL_CONTINUE)
4008 realmode_lgdt(ctxt->vcpu, size, address);
4009 /* Disable writeback. */
4010 c->dst.type = OP_NONE;
4012 case 3: /* lidt/vmmcall */
4013 if (c->modrm_mod == 3) {
4014 switch (c->modrm_rm) {
4016 rc = kvm_fix_hypercall(ctxt->vcpu);
4019 goto cannot_emulate;
4022 rc = read_descriptor(ctxt, ops, c->src.addr.mem,
4025 if (rc != X86EMUL_CONTINUE)
4027 realmode_lidt(ctxt->vcpu, size, address);
4029 /* Disable writeback. */
4030 c->dst.type = OP_NONE;
4034 c->dst.val = ops->get_cr(0, ctxt->vcpu);
4037 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0eul) |
4038 (c->src.val & 0x0f), ctxt->vcpu);
4039 c->dst.type = OP_NONE;
4041 case 5: /* not defined */
4043 rc = X86EMUL_PROPAGATE_FAULT;
4046 rc = em_invlpg(ctxt);
4049 goto cannot_emulate;
4052 case 0x05: /* syscall */
4053 rc = emulate_syscall(ctxt, ops);
4056 emulate_clts(ctxt->vcpu);
4058 case 0x09: /* wbinvd */
4059 kvm_emulate_wbinvd(ctxt->vcpu);
4061 case 0x08: /* invd */
4062 case 0x0d: /* GrpP (prefetch) */
4063 case 0x18: /* Grp16 (prefetch/nop) */
4065 case 0x20: /* mov cr, reg */
4066 c->dst.val = ops->get_cr(c->modrm_reg, ctxt->vcpu);
4068 case 0x21: /* mov from dr to reg */
4069 ops->get_dr(c->modrm_reg, &c->dst.val, ctxt->vcpu);
4071 case 0x22: /* mov reg, cr */
4072 if (ops->set_cr(c->modrm_reg, c->src.val, ctxt->vcpu)) {
4073 emulate_gp(ctxt, 0);
4074 rc = X86EMUL_PROPAGATE_FAULT;
4077 c->dst.type = OP_NONE;
4079 case 0x23: /* mov from reg to dr */
4080 if (ops->set_dr(c->modrm_reg, c->src.val &
4081 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4082 ~0ULL : ~0U), ctxt->vcpu) < 0) {
4083 /* #UD condition is already handled by the code above */
4084 emulate_gp(ctxt, 0);
4085 rc = X86EMUL_PROPAGATE_FAULT;
4089 c->dst.type = OP_NONE; /* no writeback */
4093 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4094 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4095 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
4096 emulate_gp(ctxt, 0);
4097 rc = X86EMUL_PROPAGATE_FAULT;
4100 rc = X86EMUL_CONTINUE;
4104 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
4105 emulate_gp(ctxt, 0);
4106 rc = X86EMUL_PROPAGATE_FAULT;
4109 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4110 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4112 rc = X86EMUL_CONTINUE;
4114 case 0x34: /* sysenter */
4115 rc = emulate_sysenter(ctxt, ops);
4117 case 0x35: /* sysexit */
4118 rc = emulate_sysexit(ctxt, ops);
4120 case 0x40 ... 0x4f: /* cmov */
4121 c->dst.val = c->dst.orig_val = c->src.val;
4122 if (!test_cc(c->b, ctxt->eflags))
4123 c->dst.type = OP_NONE; /* no writeback */
4125 case 0x80 ... 0x8f: /* jnz rel, etc*/
4126 if (test_cc(c->b, ctxt->eflags))
4127 jmp_rel(c, c->src.val);
4129 case 0x90 ... 0x9f: /* setcc r/m8 */
4130 c->dst.val = test_cc(c->b, ctxt->eflags);
4132 case 0xa0: /* push fs */
4133 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4135 case 0xa1: /* pop fs */
4136 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4140 c->dst.type = OP_NONE;
4141 /* only subword offset */
4142 c->src.val &= (c->dst.bytes << 3) - 1;
4143 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4145 case 0xa4: /* shld imm8, r, r/m */
4146 case 0xa5: /* shld cl, r, r/m */
4147 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4149 case 0xa8: /* push gs */
4150 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4152 case 0xa9: /* pop gs */
4153 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4157 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4159 case 0xac: /* shrd imm8, r, r/m */
4160 case 0xad: /* shrd cl, r, r/m */
4161 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4163 case 0xae: /* clflush */
4165 case 0xb0 ... 0xb1: /* cmpxchg */
4167 * Save real source value, then compare EAX against
4170 c->src.orig_val = c->src.val;
4171 c->src.val = c->regs[VCPU_REGS_RAX];
4172 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4173 if (ctxt->eflags & EFLG_ZF) {
4174 /* Success: write back to memory. */
4175 c->dst.val = c->src.orig_val;
4177 /* Failure: write the value we saw to EAX. */
4178 c->dst.type = OP_REG;
4179 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4182 case 0xb2: /* lss */
4183 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4187 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4189 case 0xb4: /* lfs */
4190 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4192 case 0xb5: /* lgs */
4193 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4195 case 0xb6 ... 0xb7: /* movzx */
4196 c->dst.bytes = c->op_bytes;
4197 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4200 case 0xba: /* Grp8 */
4201 switch (c->modrm_reg & 3) {
4214 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4216 case 0xbc: { /* bsf */
4218 __asm__ ("bsf %2, %0; setz %1"
4219 : "=r"(c->dst.val), "=q"(zf)
4221 ctxt->eflags &= ~X86_EFLAGS_ZF;
4223 ctxt->eflags |= X86_EFLAGS_ZF;
4224 c->dst.type = OP_NONE; /* Disable writeback. */
4228 case 0xbd: { /* bsr */
4230 __asm__ ("bsr %2, %0; setz %1"
4231 : "=r"(c->dst.val), "=q"(zf)
4233 ctxt->eflags &= ~X86_EFLAGS_ZF;
4235 ctxt->eflags |= X86_EFLAGS_ZF;
4236 c->dst.type = OP_NONE; /* Disable writeback. */
4240 case 0xbe ... 0xbf: /* movsx */
4241 c->dst.bytes = c->op_bytes;
4242 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4245 case 0xc0 ... 0xc1: /* xadd */
4246 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4247 /* Write back the register source. */
4248 c->src.val = c->dst.orig_val;
4249 write_register_operand(&c->src);
4251 case 0xc3: /* movnti */
4252 c->dst.bytes = c->op_bytes;
4253 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4256 case 0xc7: /* Grp9 (cmpxchg8b) */
4257 rc = emulate_grp9(ctxt, ops);
4260 goto cannot_emulate;
4263 if (rc != X86EMUL_CONTINUE)
4269 return EMULATION_FAILED;