Pileus Git - ~andy/linux/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37 #include <linux/sysdev.h>
  38 #include <linux/cpu.h>
  39 #include <linux/file.h>
  40 #include <linux/fs.h>
  41 #include <linux/mount.h>
  42
  43 #include "x86_emulate.h"
  44 #include "segment_descriptor.h"
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static DEFINE_SPINLOCK(kvm_lock);
  50 static LIST_HEAD(vm_list);
  51
  52 struct kvm_arch_ops *kvm_arch_ops;
  53 struct kvm_stat kvm_stat;
  54 EXPORT_SYMBOL_GPL(kvm_stat);
  55
  56 static struct kvm_stats_debugfs_item {
  57         const char *name;
  58         u32 *data;
  59         struct dentry *dentry;
  60 } debugfs_entries[] = {
  61         { "pf_fixed", &kvm_stat.pf_fixed },
  62         { "pf_guest", &kvm_stat.pf_guest },
  63         { "tlb_flush", &kvm_stat.tlb_flush },
  64         { "invlpg", &kvm_stat.invlpg },
  65         { "exits", &kvm_stat.exits },
  66         { "io_exits", &kvm_stat.io_exits },
  67         { "mmio_exits", &kvm_stat.mmio_exits },
  68         { "signal_exits", &kvm_stat.signal_exits },
  69         { "irq_window", &kvm_stat.irq_window_exits },
  70         { "halt_exits", &kvm_stat.halt_exits },
  71         { "request_irq", &kvm_stat.request_irq_exits },
  72         { "irq_exits", &kvm_stat.irq_exits },
  73         { NULL, NULL }
  74 };
  75
  76 static struct dentry *debugfs_dir;
  77
  78 #define KVMFS_MAGIC 0x19700426
  79 struct vfsmount *kvmfs_mnt;
  80
  81 #define MAX_IO_MSRS 256
  82
  83 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  84 #define LMSW_GUEST_MASK 0x0eULL
  85 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  86 #define CR8_RESEVED_BITS (~0x0fULL)
  87 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  88
  89 #ifdef CONFIG_X86_64
  90 // LDT or TSS descriptor in the GDT. 16 bytes.
  91 struct segment_descriptor_64 {
  92         struct segment_descriptor s;
  93         u32 base_higher;
  94         u32 pad_zero;
  95 };
  96
  97 #endif
  98
  99 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 100                            unsigned long arg);
 101
 102 static struct inode *kvmfs_inode(struct file_operations *fops)
 103 {
 104         int error = -ENOMEM;
 105         struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
 106
 107         if (!inode)
 108                 goto eexit_1;
 109
 110         inode->i_fop = fops;
 111
 112         /*
 113          * Mark the inode dirty from the very beginning,
 114          * that way it will never be moved to the dirty
 115          * list because mark_inode_dirty() will think
 116          * that it already _is_ on the dirty list.
 117          */
 118         inode->i_state = I_DIRTY;
 119         inode->i_mode = S_IRUSR | S_IWUSR;
 120         inode->i_uid = current->fsuid;
 121         inode->i_gid = current->fsgid;
 122         inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 123         return inode;
 124
 125 eexit_1:
 126         return ERR_PTR(error);
 127 }
 128
 129 static struct file *kvmfs_file(struct inode *inode, void *private_data)
 130 {
 131         struct file *file = get_empty_filp();
 132
 133         if (!file)
 134                 return ERR_PTR(-ENFILE);
 135
 136         file->f_path.mnt = mntget(kvmfs_mnt);
 137         file->f_path.dentry = d_alloc_anon(inode);
 138         if (!file->f_path.dentry)
 139                 return ERR_PTR(-ENOMEM);
 140         file->f_mapping = inode->i_mapping;
 141
 142         file->f_pos = 0;
 143         file->f_flags = O_RDWR;
 144         file->f_op = inode->i_fop;
 145         file->f_mode = FMODE_READ | FMODE_WRITE;
 146         file->f_version = 0;
 147         file->private_data = private_data;
 148         return file;
 149 }
 150
 151 unsigned long segment_base(u16 selector)
 152 {
 153         struct descriptor_table gdt;
 154         struct segment_descriptor *d;
 155         unsigned long table_base;
 156         typedef unsigned long ul;
 157         unsigned long v;
 158
 159         if (selector == 0)
 160                 return 0;
 161
 162         asm ("sgdt %0" : "=m"(gdt));
 163         table_base = gdt.base;
 164
 165         if (selector & 4) {           /* from ldt */
 166                 u16 ldt_selector;
 167
 168                 asm ("sldt %0" : "=g"(ldt_selector));
 169                 table_base = segment_base(ldt_selector);
 170         }
 171         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 172         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 173 #ifdef CONFIG_X86_64
 174         if (d->system == 0
 175             && (d->type == 2 || d->type == 9 || d->type == 11))
 176                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 177 #endif
 178         return v;
 179 }
 180 EXPORT_SYMBOL_GPL(segment_base);
 181
 182 static inline int valid_vcpu(int n)
 183 {
 184         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 185 }
 186
 187 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 188                    void *dest)
 189 {
 190         unsigned char *host_buf = dest;
 191         unsigned long req_size = size;
 192
 193         while (size) {
 194                 hpa_t paddr;
 195                 unsigned now;
 196                 unsigned offset;
 197                 hva_t guest_buf;
 198
 199                 paddr = gva_to_hpa(vcpu, addr);
 200
 201                 if (is_error_hpa(paddr))
 202                         break;
 203
 204                 guest_buf = (hva_t)kmap_atomic(
 205                                         pfn_to_page(paddr >> PAGE_SHIFT),
 206                                         KM_USER0);
 207                 offset = addr & ~PAGE_MASK;
 208                 guest_buf |= offset;
 209                 now = min(size, PAGE_SIZE - offset);
 210                 memcpy(host_buf, (void*)guest_buf, now);
 211                 host_buf += now;
 212                 addr += now;
 213                 size -= now;
 214                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 215         }
 216         return req_size - size;
 217 }
 218 EXPORT_SYMBOL_GPL(kvm_read_guest);
 219
 220 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 221                     void *data)
 222 {
 223         unsigned char *host_buf = data;
 224         unsigned long req_size = size;
 225
 226         while (size) {
 227                 hpa_t paddr;
 228                 unsigned now;
 229                 unsigned offset;
 230                 hva_t guest_buf;
 231
 232                 paddr = gva_to_hpa(vcpu, addr);
 233
 234                 if (is_error_hpa(paddr))
 235                         break;
 236
 237                 guest_buf = (hva_t)kmap_atomic(
 238                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 239                 offset = addr & ~PAGE_MASK;
 240                 guest_buf |= offset;
 241                 now = min(size, PAGE_SIZE - offset);
 242                 memcpy((void*)guest_buf, host_buf, now);
 243                 host_buf += now;
 244                 addr += now;
 245                 size -= now;
 246                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 247         }
 248         return req_size - size;
 249 }
 250 EXPORT_SYMBOL_GPL(kvm_write_guest);
 251
 252 /*
 253  * Switches to specified vcpu, until a matching vcpu_put()
 254  */
 255 static void vcpu_load(struct kvm_vcpu *vcpu)
 256 {
 257         mutex_lock(&vcpu->mutex);
 258         kvm_arch_ops->vcpu_load(vcpu);
 259 }
 260
 261 /*
 262  * Switches to specified vcpu, until a matching vcpu_put(). Will return NULL
 263  * if the slot is not populated.
 264  */
 265 static struct kvm_vcpu *vcpu_load_slot(struct kvm *kvm, int slot)
 266 {
 267         struct kvm_vcpu *vcpu = &kvm->vcpus[slot];
 268
 269         mutex_lock(&vcpu->mutex);
 270         if (!vcpu->vmcs) {
 271                 mutex_unlock(&vcpu->mutex);
 272                 return NULL;
 273         }
 274         kvm_arch_ops->vcpu_load(vcpu);
 275         return vcpu;
 276 }
 277
 278 static void vcpu_put(struct kvm_vcpu *vcpu)
 279 {
 280         kvm_arch_ops->vcpu_put(vcpu);
 281         mutex_unlock(&vcpu->mutex);
 282 }
 283
 284 static struct kvm *kvm_create_vm(void)
 285 {
 286         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 287         int i;
 288
 289         if (!kvm)
 290                 return ERR_PTR(-ENOMEM);
 291
 292         spin_lock_init(&kvm->lock);
 293         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 294         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 295                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 296
 297                 mutex_init(&vcpu->mutex);
 298                 vcpu->cpu = -1;
 299                 vcpu->kvm = kvm;
 300                 vcpu->mmu.root_hpa = INVALID_PAGE;
 301                 INIT_LIST_HEAD(&vcpu->free_pages);
 302                 spin_lock(&kvm_lock);
 303                 list_add(&kvm->vm_list, &vm_list);
 304                 spin_unlock(&kvm_lock);
 305         }
 306         return kvm;
 307 }
 308
 309 static int kvm_dev_open(struct inode *inode, struct file *filp)
 310 {
 311         return 0;
 312 }
 313
 314 /*
 315  * Free any memory in @free but not in @dont.
 316  */
 317 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 318                                   struct kvm_memory_slot *dont)
 319 {
 320         int i;
 321
 322         if (!dont || free->phys_mem != dont->phys_mem)
 323                 if (free->phys_mem) {
 324                         for (i = 0; i < free->npages; ++i)
 325                                 if (free->phys_mem[i])
 326                                         __free_page(free->phys_mem[i]);
 327                         vfree(free->phys_mem);
 328                 }
 329
 330         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 331                 vfree(free->dirty_bitmap);
 332
 333         free->phys_mem = NULL;
 334         free->npages = 0;
 335         free->dirty_bitmap = NULL;
 336 }
 337
 338 static void kvm_free_physmem(struct kvm *kvm)
 339 {
 340         int i;
 341
 342         for (i = 0; i < kvm->nmemslots; ++i)
 343                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 344 }
 345
 346 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 347 {
 348         if (!vcpu->vmcs)
 349                 return;
 350
 351         vcpu_load(vcpu);
 352         kvm_mmu_destroy(vcpu);
 353         vcpu_put(vcpu);
 354         kvm_arch_ops->vcpu_free(vcpu);
 355 }
 356
 357 static void kvm_free_vcpus(struct kvm *kvm)
 358 {
 359         unsigned int i;
 360
 361         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 362                 kvm_free_vcpu(&kvm->vcpus[i]);
 363 }
 364
 365 static int kvm_dev_release(struct inode *inode, struct file *filp)
 366 {
 367         return 0;
 368 }
 369
 370 static void kvm_destroy_vm(struct kvm *kvm)
 371 {
 372         spin_lock(&kvm_lock);
 373         list_del(&kvm->vm_list);
 374         spin_unlock(&kvm_lock);
 375         kvm_free_vcpus(kvm);
 376         kvm_free_physmem(kvm);
 377         kfree(kvm);
 378 }
 379
 380 static int kvm_vm_release(struct inode *inode, struct file *filp)
 381 {
 382         struct kvm *kvm = filp->private_data;
 383
 384         kvm_destroy_vm(kvm);
 385         return 0;
 386 }
 387
 388 static void inject_gp(struct kvm_vcpu *vcpu)
 389 {
 390         kvm_arch_ops->inject_gp(vcpu, 0);
 391 }
 392
 393 /*
 394  * Load the pae pdptrs.  Return true is they are all valid.
 395  */
 396 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 397 {
 398         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 399         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 400         int i;
 401         u64 pdpte;
 402         u64 *pdpt;
 403         int ret;
 404         struct kvm_memory_slot *memslot;
 405
 406         spin_lock(&vcpu->kvm->lock);
 407         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 408         /* FIXME: !memslot - emulate? 0xff? */
 409         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 410
 411         ret = 1;
 412         for (i = 0; i < 4; ++i) {
 413                 pdpte = pdpt[offset + i];
 414                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
 415                         ret = 0;
 416                         goto out;
 417                 }
 418         }
 419
 420         for (i = 0; i < 4; ++i)
 421                 vcpu->pdptrs[i] = pdpt[offset + i];
 422
 423 out:
 424         kunmap_atomic(pdpt, KM_USER0);
 425         spin_unlock(&vcpu->kvm->lock);
 426
 427         return ret;
 428 }
 429
 430 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 431 {
 432         if (cr0 & CR0_RESEVED_BITS) {
 433                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 434                        cr0, vcpu->cr0);
 435                 inject_gp(vcpu);
 436                 return;
 437         }
 438
 439         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 440                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 441                 inject_gp(vcpu);
 442                 return;
 443         }
 444
 445         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 446                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 447                        "and a clear PE flag\n");
 448                 inject_gp(vcpu);
 449                 return;
 450         }
 451
 452         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 453 #ifdef CONFIG_X86_64
 454                 if ((vcpu->shadow_efer & EFER_LME)) {
 455                         int cs_db, cs_l;
 456
 457                         if (!is_pae(vcpu)) {
 458                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 459                                        "in long mode while PAE is disabled\n");
 460                                 inject_gp(vcpu);
 461                                 return;
 462                         }
 463                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 464                         if (cs_l) {
 465                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 466                                        "in long mode while CS.L == 1\n");
 467                                 inject_gp(vcpu);
 468                                 return;
 469
 470                         }
 471                 } else
 472 #endif
 473                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 474                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 475                                "reserved bits\n");
 476                         inject_gp(vcpu);
 477                         return;
 478                 }
 479
 480         }
 481
 482         kvm_arch_ops->set_cr0(vcpu, cr0);
 483         vcpu->cr0 = cr0;
 484
 485         spin_lock(&vcpu->kvm->lock);
 486         kvm_mmu_reset_context(vcpu);
 487         spin_unlock(&vcpu->kvm->lock);
 488         return;
 489 }
 490 EXPORT_SYMBOL_GPL(set_cr0);
 491
 492 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 493 {
 494         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
 495         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 496 }
 497 EXPORT_SYMBOL_GPL(lmsw);
 498
 499 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 500 {
 501         if (cr4 & CR4_RESEVED_BITS) {
 502                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 503                 inject_gp(vcpu);
 504                 return;
 505         }
 506
 507         if (is_long_mode(vcpu)) {
 508                 if (!(cr4 & CR4_PAE_MASK)) {
 509                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 510                                "in long mode\n");
 511                         inject_gp(vcpu);
 512                         return;
 513                 }
 514         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 515                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 516                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 517                 inject_gp(vcpu);
 518         }
 519
 520         if (cr4 & CR4_VMXE_MASK) {
 521                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 522                 inject_gp(vcpu);
 523                 return;
 524         }
 525         kvm_arch_ops->set_cr4(vcpu, cr4);
 526         spin_lock(&vcpu->kvm->lock);
 527         kvm_mmu_reset_context(vcpu);
 528         spin_unlock(&vcpu->kvm->lock);
 529 }
 530 EXPORT_SYMBOL_GPL(set_cr4);
 531
 532 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 533 {
 534         if (is_long_mode(vcpu)) {
 535                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
 536                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 537                         inject_gp(vcpu);
 538                         return;
 539                 }
 540         } else {
 541                 if (cr3 & CR3_RESEVED_BITS) {
 542                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 543                         inject_gp(vcpu);
 544                         return;
 545                 }
 546                 if (is_paging(vcpu) && is_pae(vcpu) &&
 547                     !load_pdptrs(vcpu, cr3)) {
 548                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 549                                "reserved bits\n");
 550                         inject_gp(vcpu);
 551                         return;
 552                 }
 553         }
 554
 555         vcpu->cr3 = cr3;
 556         spin_lock(&vcpu->kvm->lock);
 557         /*
 558          * Does the new cr3 value map to physical memory? (Note, we
 559          * catch an invalid cr3 even in real-mode, because it would
 560          * cause trouble later on when we turn on paging anyway.)
 561          *
 562          * A real CPU would silently accept an invalid cr3 and would
 563          * attempt to use it - with largely undefined (and often hard
 564          * to debug) behavior on the guest side.
 565          */
 566         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 567                 inject_gp(vcpu);
 568         else
 569                 vcpu->mmu.new_cr3(vcpu);
 570         spin_unlock(&vcpu->kvm->lock);
 571 }
 572 EXPORT_SYMBOL_GPL(set_cr3);
 573
 574 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 575 {
 576         if ( cr8 & CR8_RESEVED_BITS) {
 577                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 578                 inject_gp(vcpu);
 579                 return;
 580         }
 581         vcpu->cr8 = cr8;
 582 }
 583 EXPORT_SYMBOL_GPL(set_cr8);
 584
 585 void fx_init(struct kvm_vcpu *vcpu)
 586 {
 587         struct __attribute__ ((__packed__)) fx_image_s {
 588                 u16 control; //fcw
 589                 u16 status; //fsw
 590                 u16 tag; // ftw
 591                 u16 opcode; //fop
 592                 u64 ip; // fpu ip
 593                 u64 operand;// fpu dp
 594                 u32 mxcsr;
 595                 u32 mxcsr_mask;
 596
 597         } *fx_image;
 598
 599         fx_save(vcpu->host_fx_image);
 600         fpu_init();
 601         fx_save(vcpu->guest_fx_image);
 602         fx_restore(vcpu->host_fx_image);
 603
 604         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 605         fx_image->mxcsr = 0x1f80;
 606         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 607                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 608 }
 609 EXPORT_SYMBOL_GPL(fx_init);
 610
 611 /*
 612  * Allocate some memory and give it an address in the guest physical address
 613  * space.
 614  *
 615  * Discontiguous memory is allowed, mostly for framebuffers.
 616  */
 617 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 618                                           struct kvm_memory_region *mem)
 619 {
 620         int r;
 621         gfn_t base_gfn;
 622         unsigned long npages;
 623         unsigned long i;
 624         struct kvm_memory_slot *memslot;
 625         struct kvm_memory_slot old, new;
 626         int memory_config_version;
 627
 628         r = -EINVAL;
 629         /* General sanity checks */
 630         if (mem->memory_size & (PAGE_SIZE - 1))
 631                 goto out;
 632         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 633                 goto out;
 634         if (mem->slot >= KVM_MEMORY_SLOTS)
 635                 goto out;
 636         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 637                 goto out;
 638
 639         memslot = &kvm->memslots[mem->slot];
 640         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 641         npages = mem->memory_size >> PAGE_SHIFT;
 642
 643         if (!npages)
 644                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 645
 646 raced:
 647         spin_lock(&kvm->lock);
 648
 649         memory_config_version = kvm->memory_config_version;
 650         new = old = *memslot;
 651
 652         new.base_gfn = base_gfn;
 653         new.npages = npages;
 654         new.flags = mem->flags;
 655
 656         /* Disallow changing a memory slot's size. */
 657         r = -EINVAL;
 658         if (npages && old.npages && npages != old.npages)
 659                 goto out_unlock;
 660
 661         /* Check for overlaps */
 662         r = -EEXIST;
 663         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 664                 struct kvm_memory_slot *s = &kvm->memslots[i];
 665
 666                 if (s == memslot)
 667                         continue;
 668                 if (!((base_gfn + npages <= s->base_gfn) ||
 669                       (base_gfn >= s->base_gfn + s->npages)))
 670                         goto out_unlock;
 671         }
 672         /*
 673          * Do memory allocations outside lock.  memory_config_version will
 674          * detect any races.
 675          */
 676         spin_unlock(&kvm->lock);
 677
 678         /* Deallocate if slot is being removed */
 679         if (!npages)
 680                 new.phys_mem = NULL;
 681
 682         /* Free page dirty bitmap if unneeded */
 683         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 684                 new.dirty_bitmap = NULL;
 685
 686         r = -ENOMEM;
 687
 688         /* Allocate if a slot is being created */
 689         if (npages && !new.phys_mem) {
 690                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 691
 692                 if (!new.phys_mem)
 693                         goto out_free;
 694
 695                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 696                 for (i = 0; i < npages; ++i) {
 697                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 698                                                      | __GFP_ZERO);
 699                         if (!new.phys_mem[i])
 700                                 goto out_free;
 701                         set_page_private(new.phys_mem[i],0);
 702                 }
 703         }
 704
 705         /* Allocate page dirty bitmap if needed */
 706         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 707                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 708
 709                 new.dirty_bitmap = vmalloc(dirty_bytes);
 710                 if (!new.dirty_bitmap)
 711                         goto out_free;
 712                 memset(new.dirty_bitmap, 0, dirty_bytes);
 713         }
 714
 715         spin_lock(&kvm->lock);
 716
 717         if (memory_config_version != kvm->memory_config_version) {
 718                 spin_unlock(&kvm->lock);
 719                 kvm_free_physmem_slot(&new, &old);
 720                 goto raced;
 721         }
 722
 723         r = -EAGAIN;
 724         if (kvm->busy)
 725                 goto out_unlock;
 726
 727         if (mem->slot >= kvm->nmemslots)
 728                 kvm->nmemslots = mem->slot + 1;
 729
 730         *memslot = new;
 731         ++kvm->memory_config_version;
 732
 733         spin_unlock(&kvm->lock);
 734
 735         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 736                 struct kvm_vcpu *vcpu;
 737
 738                 vcpu = vcpu_load_slot(kvm, i);
 739                 if (!vcpu)
 740                         continue;
 741                 kvm_mmu_reset_context(vcpu);
 742                 vcpu_put(vcpu);
 743         }
 744
 745         kvm_free_physmem_slot(&old, &new);
 746         return 0;
 747
 748 out_unlock:
 749         spin_unlock(&kvm->lock);
 750 out_free:
 751         kvm_free_physmem_slot(&new, &old);
 752 out:
 753         return r;
 754 }
 755
 756 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 757 {
 758         spin_lock(&vcpu->kvm->lock);
 759         kvm_mmu_slot_remove_write_access(vcpu, slot);
 760         spin_unlock(&vcpu->kvm->lock);
 761 }
 762
 763 /*
 764  * Get (and clear) the dirty memory log for a memory slot.
 765  */
 766 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 767                                       struct kvm_dirty_log *log)
 768 {
 769         struct kvm_memory_slot *memslot;
 770         int r, i;
 771         int n;
 772         int cleared;
 773         unsigned long any = 0;
 774
 775         spin_lock(&kvm->lock);
 776
 777         /*
 778          * Prevent changes to guest memory configuration even while the lock
 779          * is not taken.
 780          */
 781         ++kvm->busy;
 782         spin_unlock(&kvm->lock);
 783         r = -EINVAL;
 784         if (log->slot >= KVM_MEMORY_SLOTS)
 785                 goto out;
 786
 787         memslot = &kvm->memslots[log->slot];
 788         r = -ENOENT;
 789         if (!memslot->dirty_bitmap)
 790                 goto out;
 791
 792         n = ALIGN(memslot->npages, 8) / 8;
 793
 794         for (i = 0; !any && i < n; ++i)
 795                 any = memslot->dirty_bitmap[i];
 796
 797         r = -EFAULT;
 798         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 799                 goto out;
 800
 801         if (any) {
 802                 cleared = 0;
 803                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 804                         struct kvm_vcpu *vcpu;
 805
 806                         vcpu = vcpu_load_slot(kvm, i);
 807                         if (!vcpu)
 808                                 continue;
 809                         if (!cleared) {
 810                                 do_remove_write_access(vcpu, log->slot);
 811                                 memset(memslot->dirty_bitmap, 0, n);
 812                                 cleared = 1;
 813                         }
 814                         kvm_arch_ops->tlb_flush(vcpu);
 815                         vcpu_put(vcpu);
 816                 }
 817         }
 818
 819         r = 0;
 820
 821 out:
 822         spin_lock(&kvm->lock);
 823         --kvm->busy;
 824         spin_unlock(&kvm->lock);
 825         return r;
 826 }
 827
 828 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 829 {
 830         int i;
 831
 832         for (i = 0; i < kvm->nmemslots; ++i) {
 833                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 834
 835                 if (gfn >= memslot->base_gfn
 836                     && gfn < memslot->base_gfn + memslot->npages)
 837                         return memslot;
 838         }
 839         return NULL;
 840 }
 841 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 842
 843 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 844 {
 845         int i;
 846         struct kvm_memory_slot *memslot = NULL;
 847         unsigned long rel_gfn;
 848
 849         for (i = 0; i < kvm->nmemslots; ++i) {
 850                 memslot = &kvm->memslots[i];
 851
 852                 if (gfn >= memslot->base_gfn
 853                     && gfn < memslot->base_gfn + memslot->npages) {
 854
 855                         if (!memslot || !memslot->dirty_bitmap)
 856                                 return;
 857
 858                         rel_gfn = gfn - memslot->base_gfn;
 859
 860                         /* avoid RMW */
 861                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 862                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 863                         return;
 864                 }
 865         }
 866 }
 867
 868 static int emulator_read_std(unsigned long addr,
 869                              unsigned long *val,
 870                              unsigned int bytes,
 871                              struct x86_emulate_ctxt *ctxt)
 872 {
 873         struct kvm_vcpu *vcpu = ctxt->vcpu;
 874         void *data = val;
 875
 876         while (bytes) {
 877                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 878                 unsigned offset = addr & (PAGE_SIZE-1);
 879                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 880                 unsigned long pfn;
 881                 struct kvm_memory_slot *memslot;
 882                 void *page;
 883
 884                 if (gpa == UNMAPPED_GVA)
 885                         return X86EMUL_PROPAGATE_FAULT;
 886                 pfn = gpa >> PAGE_SHIFT;
 887                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 888                 if (!memslot)
 889                         return X86EMUL_UNHANDLEABLE;
 890                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 891
 892                 memcpy(data, page + offset, tocopy);
 893
 894                 kunmap_atomic(page, KM_USER0);
 895
 896                 bytes -= tocopy;
 897                 data += tocopy;
 898                 addr += tocopy;
 899         }
 900
 901         return X86EMUL_CONTINUE;
 902 }
 903
 904 static int emulator_write_std(unsigned long addr,
 905                               unsigned long val,
 906                               unsigned int bytes,
 907                               struct x86_emulate_ctxt *ctxt)
 908 {
 909         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 910                addr, bytes);
 911         return X86EMUL_UNHANDLEABLE;
 912 }
 913
 914 static int emulator_read_emulated(unsigned long addr,
 915                                   unsigned long *val,
 916                                   unsigned int bytes,
 917                                   struct x86_emulate_ctxt *ctxt)
 918 {
 919         struct kvm_vcpu *vcpu = ctxt->vcpu;
 920
 921         if (vcpu->mmio_read_completed) {
 922                 memcpy(val, vcpu->mmio_data, bytes);
 923                 vcpu->mmio_read_completed = 0;
 924                 return X86EMUL_CONTINUE;
 925         } else if (emulator_read_std(addr, val, bytes, ctxt)
 926                    == X86EMUL_CONTINUE)
 927                 return X86EMUL_CONTINUE;
 928         else {
 929                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 930
 931                 if (gpa == UNMAPPED_GVA)
 932                         return X86EMUL_PROPAGATE_FAULT;
 933                 vcpu->mmio_needed = 1;
 934                 vcpu->mmio_phys_addr = gpa;
 935                 vcpu->mmio_size = bytes;
 936                 vcpu->mmio_is_write = 0;
 937
 938                 return X86EMUL_UNHANDLEABLE;
 939         }
 940 }
 941
 942 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 943                                unsigned long val, int bytes)
 944 {
 945         struct kvm_memory_slot *m;
 946         struct page *page;
 947         void *virt;
 948
 949         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
 950                 return 0;
 951         m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
 952         if (!m)
 953                 return 0;
 954         page = gfn_to_page(m, gpa >> PAGE_SHIFT);
 955         kvm_mmu_pre_write(vcpu, gpa, bytes);
 956         virt = kmap_atomic(page, KM_USER0);
 957         memcpy(virt + offset_in_page(gpa), &val, bytes);
 958         kunmap_atomic(virt, KM_USER0);
 959         kvm_mmu_post_write(vcpu, gpa, bytes);
 960         return 1;
 961 }
 962
 963 static int emulator_write_emulated(unsigned long addr,
 964                                    unsigned long val,
 965                                    unsigned int bytes,
 966                                    struct x86_emulate_ctxt *ctxt)
 967 {
 968         struct kvm_vcpu *vcpu = ctxt->vcpu;
 969         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 970
 971         if (gpa == UNMAPPED_GVA)
 972                 return X86EMUL_PROPAGATE_FAULT;
 973
 974         if (emulator_write_phys(vcpu, gpa, val, bytes))
 975                 return X86EMUL_CONTINUE;
 976
 977         vcpu->mmio_needed = 1;
 978         vcpu->mmio_phys_addr = gpa;
 979         vcpu->mmio_size = bytes;
 980         vcpu->mmio_is_write = 1;
 981         memcpy(vcpu->mmio_data, &val, bytes);
 982
 983         return X86EMUL_CONTINUE;
 984 }
 985
 986 static int emulator_cmpxchg_emulated(unsigned long addr,
 987                                      unsigned long old,
 988                                      unsigned long new,
 989                                      unsigned int bytes,
 990                                      struct x86_emulate_ctxt *ctxt)
 991 {
 992         static int reported;
 993
 994         if (!reported) {
 995                 reported = 1;
 996                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
 997         }
 998         return emulator_write_emulated(addr, new, bytes, ctxt);
 999 }
1000
1001 #ifdef CONFIG_X86_32
1002
1003 static int emulator_cmpxchg8b_emulated(unsigned long addr,
1004                                        unsigned long old_lo,
1005                                        unsigned long old_hi,
1006                                        unsigned long new_lo,
1007                                        unsigned long new_hi,
1008                                        struct x86_emulate_ctxt *ctxt)
1009 {
1010         static int reported;
1011         int r;
1012
1013         if (!reported) {
1014                 reported = 1;
1015                 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
1016         }
1017         r = emulator_write_emulated(addr, new_lo, 4, ctxt);
1018         if (r != X86EMUL_CONTINUE)
1019                 return r;
1020         return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
1021 }
1022
1023 #endif
1024
1025 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1026 {
1027         return kvm_arch_ops->get_segment_base(vcpu, seg);
1028 }
1029
1030 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1031 {
1032         return X86EMUL_CONTINUE;
1033 }
1034
1035 int emulate_clts(struct kvm_vcpu *vcpu)
1036 {
1037         unsigned long cr0;
1038
1039         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1040         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1041         kvm_arch_ops->set_cr0(vcpu, cr0);
1042         return X86EMUL_CONTINUE;
1043 }
1044
1045 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1046 {
1047         struct kvm_vcpu *vcpu = ctxt->vcpu;
1048
1049         switch (dr) {
1050         case 0 ... 3:
1051                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1052                 return X86EMUL_CONTINUE;
1053         default:
1054                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1055                        __FUNCTION__, dr);
1056                 return X86EMUL_UNHANDLEABLE;
1057         }
1058 }
1059
1060 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1061 {
1062         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1063         int exception;
1064
1065         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1066         if (exception) {
1067                 /* FIXME: better handling */
1068                 return X86EMUL_UNHANDLEABLE;
1069         }
1070         return X86EMUL_CONTINUE;
1071 }
1072
1073 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1074 {
1075         static int reported;
1076         u8 opcodes[4];
1077         unsigned long rip = ctxt->vcpu->rip;
1078         unsigned long rip_linear;
1079
1080         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1081
1082         if (reported)
1083                 return;
1084
1085         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1086
1087         printk(KERN_ERR "emulation failed but !mmio_needed?"
1088                " rip %lx %02x %02x %02x %02x\n",
1089                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1090         reported = 1;
1091 }
1092
1093 struct x86_emulate_ops emulate_ops = {
1094         .read_std            = emulator_read_std,
1095         .write_std           = emulator_write_std,
1096         .read_emulated       = emulator_read_emulated,
1097         .write_emulated      = emulator_write_emulated,
1098         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1099 #ifdef CONFIG_X86_32
1100         .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
1101 #endif
1102 };
1103
1104 int emulate_instruction(struct kvm_vcpu *vcpu,
1105                         struct kvm_run *run,
1106                         unsigned long cr2,
1107                         u16 error_code)
1108 {
1109         struct x86_emulate_ctxt emulate_ctxt;
1110         int r;
1111         int cs_db, cs_l;
1112
1113         kvm_arch_ops->cache_regs(vcpu);
1114
1115         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1116
1117         emulate_ctxt.vcpu = vcpu;
1118         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1119         emulate_ctxt.cr2 = cr2;
1120         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1121                 ? X86EMUL_MODE_REAL : cs_l
1122                 ? X86EMUL_MODE_PROT64 : cs_db
1123                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1124
1125         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1126                 emulate_ctxt.cs_base = 0;
1127                 emulate_ctxt.ds_base = 0;
1128                 emulate_ctxt.es_base = 0;
1129                 emulate_ctxt.ss_base = 0;
1130         } else {
1131                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1132                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1133                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1134                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1135         }
1136
1137         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1138         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1139
1140         vcpu->mmio_is_write = 0;
1141         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1142
1143         if ((r || vcpu->mmio_is_write) && run) {
1144                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1145                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1146                 run->mmio.len = vcpu->mmio_size;
1147                 run->mmio.is_write = vcpu->mmio_is_write;
1148         }
1149
1150         if (r) {
1151                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1152                         return EMULATE_DONE;
1153                 if (!vcpu->mmio_needed) {
1154                         report_emulation_failure(&emulate_ctxt);
1155                         return EMULATE_FAIL;
1156                 }
1157                 return EMULATE_DO_MMIO;
1158         }
1159
1160         kvm_arch_ops->decache_regs(vcpu);
1161         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1162
1163         if (vcpu->mmio_is_write)
1164                 return EMULATE_DO_MMIO;
1165
1166         return EMULATE_DONE;
1167 }
1168 EXPORT_SYMBOL_GPL(emulate_instruction);
1169
1170 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1171 {
1172         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1173
1174         kvm_arch_ops->decache_regs(vcpu);
1175         ret = -KVM_EINVAL;
1176 #ifdef CONFIG_X86_64
1177         if (is_long_mode(vcpu)) {
1178                 nr = vcpu->regs[VCPU_REGS_RAX];
1179                 a0 = vcpu->regs[VCPU_REGS_RDI];
1180                 a1 = vcpu->regs[VCPU_REGS_RSI];
1181                 a2 = vcpu->regs[VCPU_REGS_RDX];
1182                 a3 = vcpu->regs[VCPU_REGS_RCX];
1183                 a4 = vcpu->regs[VCPU_REGS_R8];
1184                 a5 = vcpu->regs[VCPU_REGS_R9];
1185         } else
1186 #endif
1187         {
1188                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1189                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1190                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1191                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1192                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1193                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1194                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1195         }
1196         switch (nr) {
1197         default:
1198                 ;
1199         }
1200         vcpu->regs[VCPU_REGS_RAX] = ret;
1201         kvm_arch_ops->cache_regs(vcpu);
1202         return 1;
1203 }
1204 EXPORT_SYMBOL_GPL(kvm_hypercall);
1205
1206 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1207 {
1208         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1209 }
1210
1211 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1212 {
1213         struct descriptor_table dt = { limit, base };
1214
1215         kvm_arch_ops->set_gdt(vcpu, &dt);
1216 }
1217
1218 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1219 {
1220         struct descriptor_table dt = { limit, base };
1221
1222         kvm_arch_ops->set_idt(vcpu, &dt);
1223 }
1224
1225 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1226                    unsigned long *rflags)
1227 {
1228         lmsw(vcpu, msw);
1229         *rflags = kvm_arch_ops->get_rflags(vcpu);
1230 }
1231
1232 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1233 {
1234         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1235         switch (cr) {
1236         case 0:
1237                 return vcpu->cr0;
1238         case 2:
1239                 return vcpu->cr2;
1240         case 3:
1241                 return vcpu->cr3;
1242         case 4:
1243                 return vcpu->cr4;
1244         default:
1245                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1246                 return 0;
1247         }
1248 }
1249
1250 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1251                      unsigned long *rflags)
1252 {
1253         switch (cr) {
1254         case 0:
1255                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1256                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1257                 break;
1258         case 2:
1259                 vcpu->cr2 = val;
1260                 break;
1261         case 3:
1262                 set_cr3(vcpu, val);
1263                 break;
1264         case 4:
1265                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1266                 break;
1267         default:
1268                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1269         }
1270 }
1271
1272 /*
1273  * Register the para guest with the host:
1274  */
1275 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1276 {
1277         struct kvm_vcpu_para_state *para_state;
1278         hpa_t para_state_hpa, hypercall_hpa;
1279         struct page *para_state_page;
1280         unsigned char *hypercall;
1281         gpa_t hypercall_gpa;
1282
1283         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1284         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1285
1286         /*
1287          * Needs to be page aligned:
1288          */
1289         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1290                 goto err_gp;
1291
1292         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1293         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1294         if (is_error_hpa(para_state_hpa))
1295                 goto err_gp;
1296
1297         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1298         para_state = kmap_atomic(para_state_page, KM_USER0);
1299
1300         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1301         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1302
1303         para_state->host_version = KVM_PARA_API_VERSION;
1304         /*
1305          * We cannot support guests that try to register themselves
1306          * with a newer API version than the host supports:
1307          */
1308         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1309                 para_state->ret = -KVM_EINVAL;
1310                 goto err_kunmap_skip;
1311         }
1312
1313         hypercall_gpa = para_state->hypercall_gpa;
1314         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1315         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1316         if (is_error_hpa(hypercall_hpa)) {
1317                 para_state->ret = -KVM_EINVAL;
1318                 goto err_kunmap_skip;
1319         }
1320
1321         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1322         vcpu->para_state_page = para_state_page;
1323         vcpu->para_state_gpa = para_state_gpa;
1324         vcpu->hypercall_gpa = hypercall_gpa;
1325
1326         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1327                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1328         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1329         kunmap_atomic(hypercall, KM_USER1);
1330
1331         para_state->ret = 0;
1332 err_kunmap_skip:
1333         kunmap_atomic(para_state, KM_USER0);
1334         return 0;
1335 err_gp:
1336         return 1;
1337 }
1338
1339 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1340 {
1341         u64 data;
1342
1343         switch (msr) {
1344         case 0xc0010010: /* SYSCFG */
1345         case 0xc0010015: /* HWCR */
1346         case MSR_IA32_PLATFORM_ID:
1347         case MSR_IA32_P5_MC_ADDR:
1348         case MSR_IA32_P5_MC_TYPE:
1349         case MSR_IA32_MC0_CTL:
1350         case MSR_IA32_MCG_STATUS:
1351         case MSR_IA32_MCG_CAP:
1352         case MSR_IA32_MC0_MISC:
1353         case MSR_IA32_MC0_MISC+4:
1354         case MSR_IA32_MC0_MISC+8:
1355         case MSR_IA32_MC0_MISC+12:
1356         case MSR_IA32_MC0_MISC+16:
1357         case MSR_IA32_UCODE_REV:
1358         case MSR_IA32_PERF_STATUS:
1359                 /* MTRR registers */
1360         case 0xfe:
1361         case 0x200 ... 0x2ff:
1362                 data = 0;
1363                 break;
1364         case 0xcd: /* fsb frequency */
1365                 data = 3;
1366                 break;
1367         case MSR_IA32_APICBASE:
1368                 data = vcpu->apic_base;
1369                 break;
1370         case MSR_IA32_MISC_ENABLE:
1371                 data = vcpu->ia32_misc_enable_msr;
1372                 break;
1373 #ifdef CONFIG_X86_64
1374         case MSR_EFER:
1375                 data = vcpu->shadow_efer;
1376                 break;
1377 #endif
1378         default:
1379                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1380                 return 1;
1381         }
1382         *pdata = data;
1383         return 0;
1384 }
1385 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1386
1387 /*
1388  * Reads an msr value (of 'msr_index') into 'pdata'.
1389  * Returns 0 on success, non-0 otherwise.
1390  * Assumes vcpu_load() was already called.
1391  */
1392 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1393 {
1394         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1395 }
1396
1397 #ifdef CONFIG_X86_64
1398
1399 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1400 {
1401         if (efer & EFER_RESERVED_BITS) {
1402                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1403                        efer);
1404                 inject_gp(vcpu);
1405                 return;
1406         }
1407
1408         if (is_paging(vcpu)
1409             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1410                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1411                 inject_gp(vcpu);
1412                 return;
1413         }
1414
1415         kvm_arch_ops->set_efer(vcpu, efer);
1416
1417         efer &= ~EFER_LMA;
1418         efer |= vcpu->shadow_efer & EFER_LMA;
1419
1420         vcpu->shadow_efer = efer;
1421 }
1422
1423 #endif
1424
1425 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1426 {
1427         switch (msr) {
1428 #ifdef CONFIG_X86_64
1429         case MSR_EFER:
1430                 set_efer(vcpu, data);
1431                 break;
1432 #endif
1433         case MSR_IA32_MC0_STATUS:
1434                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1435                        __FUNCTION__, data);
1436                 break;
1437         case MSR_IA32_UCODE_REV:
1438         case MSR_IA32_UCODE_WRITE:
1439         case 0x200 ... 0x2ff: /* MTRRs */
1440                 break;
1441         case MSR_IA32_APICBASE:
1442                 vcpu->apic_base = data;
1443                 break;
1444         case MSR_IA32_MISC_ENABLE:
1445                 vcpu->ia32_misc_enable_msr = data;
1446                 break;
1447         /*
1448          * This is the 'probe whether the host is KVM' logic:
1449          */
1450         case MSR_KVM_API_MAGIC:
1451                 return vcpu_register_para(vcpu, data);
1452
1453         default:
1454                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1455                 return 1;
1456         }
1457         return 0;
1458 }
1459 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1460
1461 /*
1462  * Writes msr value into into the appropriate "register".
1463  * Returns 0 on success, non-0 otherwise.
1464  * Assumes vcpu_load() was already called.
1465  */
1466 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1467 {
1468         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1469 }
1470
1471 void kvm_resched(struct kvm_vcpu *vcpu)
1472 {
1473         vcpu_put(vcpu);
1474         cond_resched();
1475         vcpu_load(vcpu);
1476 }
1477 EXPORT_SYMBOL_GPL(kvm_resched);
1478
1479 void load_msrs(struct vmx_msr_entry *e, int n)
1480 {
1481         int i;
1482
1483         for (i = 0; i < n; ++i)
1484                 wrmsrl(e[i].index, e[i].data);
1485 }
1486 EXPORT_SYMBOL_GPL(load_msrs);
1487
1488 void save_msrs(struct vmx_msr_entry *e, int n)
1489 {
1490         int i;
1491
1492         for (i = 0; i < n; ++i)
1493                 rdmsrl(e[i].index, e[i].data);
1494 }
1495 EXPORT_SYMBOL_GPL(save_msrs);
1496
1497 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1498 {
1499         int r;
1500
1501         vcpu_load(vcpu);
1502
1503         /* re-sync apic's tpr */
1504         vcpu->cr8 = kvm_run->cr8;
1505
1506         if (kvm_run->emulated) {
1507                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1508                 kvm_run->emulated = 0;
1509         }
1510
1511         if (kvm_run->mmio_completed) {
1512                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1513                 vcpu->mmio_read_completed = 1;
1514         }
1515
1516         vcpu->mmio_needed = 0;
1517
1518         r = kvm_arch_ops->run(vcpu, kvm_run);
1519
1520         vcpu_put(vcpu);
1521         return r;
1522 }
1523
1524 static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
1525                                    struct kvm_regs *regs)
1526 {
1527         vcpu_load(vcpu);
1528
1529         kvm_arch_ops->cache_regs(vcpu);
1530
1531         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1532         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1533         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1534         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1535         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1536         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1537         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1538         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1539 #ifdef CONFIG_X86_64
1540         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1541         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1542         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1543         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1544         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1545         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1546         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1547         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1548 #endif
1549
1550         regs->rip = vcpu->rip;
1551         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1552
1553         /*
1554          * Don't leak debug flags in case they were set for guest debugging
1555          */
1556         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1557                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1558
1559         vcpu_put(vcpu);
1560
1561         return 0;
1562 }
1563
1564 static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
1565                                    struct kvm_regs *regs)
1566 {
1567         vcpu_load(vcpu);
1568
1569         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1570         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1571         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1572         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1573         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1574         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1575         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1576         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1577 #ifdef CONFIG_X86_64
1578         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1579         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1580         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1581         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1582         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1583         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1584         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1585         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1586 #endif
1587
1588         vcpu->rip = regs->rip;
1589         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1590
1591         kvm_arch_ops->decache_regs(vcpu);
1592
1593         vcpu_put(vcpu);
1594
1595         return 0;
1596 }
1597
1598 static void get_segment(struct kvm_vcpu *vcpu,
1599                         struct kvm_segment *var, int seg)
1600 {
1601         return kvm_arch_ops->get_segment(vcpu, var, seg);
1602 }
1603
1604 static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1605                                     struct kvm_sregs *sregs)
1606 {
1607         struct descriptor_table dt;
1608
1609         vcpu_load(vcpu);
1610
1611         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1612         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1613         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1614         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1615         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1616         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1617
1618         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1619         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1620
1621         kvm_arch_ops->get_idt(vcpu, &dt);
1622         sregs->idt.limit = dt.limit;
1623         sregs->idt.base = dt.base;
1624         kvm_arch_ops->get_gdt(vcpu, &dt);
1625         sregs->gdt.limit = dt.limit;
1626         sregs->gdt.base = dt.base;
1627
1628         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1629         sregs->cr0 = vcpu->cr0;
1630         sregs->cr2 = vcpu->cr2;
1631         sregs->cr3 = vcpu->cr3;
1632         sregs->cr4 = vcpu->cr4;
1633         sregs->cr8 = vcpu->cr8;
1634         sregs->efer = vcpu->shadow_efer;
1635         sregs->apic_base = vcpu->apic_base;
1636
1637         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1638                sizeof sregs->interrupt_bitmap);
1639
1640         vcpu_put(vcpu);
1641
1642         return 0;
1643 }
1644
1645 static void set_segment(struct kvm_vcpu *vcpu,
1646                         struct kvm_segment *var, int seg)
1647 {
1648         return kvm_arch_ops->set_segment(vcpu, var, seg);
1649 }
1650
1651 static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1652                                     struct kvm_sregs *sregs)
1653 {
1654         int mmu_reset_needed = 0;
1655         int i;
1656         struct descriptor_table dt;
1657
1658         vcpu_load(vcpu);
1659
1660         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1661         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1662         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1663         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1664         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1665         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1666
1667         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1668         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1669
1670         dt.limit = sregs->idt.limit;
1671         dt.base = sregs->idt.base;
1672         kvm_arch_ops->set_idt(vcpu, &dt);
1673         dt.limit = sregs->gdt.limit;
1674         dt.base = sregs->gdt.base;
1675         kvm_arch_ops->set_gdt(vcpu, &dt);
1676
1677         vcpu->cr2 = sregs->cr2;
1678         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1679         vcpu->cr3 = sregs->cr3;
1680
1681         vcpu->cr8 = sregs->cr8;
1682
1683         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1684 #ifdef CONFIG_X86_64
1685         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1686 #endif
1687         vcpu->apic_base = sregs->apic_base;
1688
1689         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1690
1691         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1692         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1693
1694         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1695         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1696         if (!is_long_mode(vcpu) && is_pae(vcpu))
1697                 load_pdptrs(vcpu, vcpu->cr3);
1698
1699         if (mmu_reset_needed)
1700                 kvm_mmu_reset_context(vcpu);
1701
1702         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1703                sizeof vcpu->irq_pending);
1704         vcpu->irq_summary = 0;
1705         for (i = 0; i < NR_IRQ_WORDS; ++i)
1706                 if (vcpu->irq_pending[i])
1707                         __set_bit(i, &vcpu->irq_summary);
1708
1709         vcpu_put(vcpu);
1710
1711         return 0;
1712 }
1713
1714 /*
1715  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1716  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1717  *
1718  * This list is modified at module load time to reflect the
1719  * capabilities of the host cpu.
1720  */
1721 static u32 msrs_to_save[] = {
1722         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1723         MSR_K6_STAR,
1724 #ifdef CONFIG_X86_64
1725         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1726 #endif
1727         MSR_IA32_TIME_STAMP_COUNTER,
1728 };
1729
1730 static unsigned num_msrs_to_save;
1731
1732 static u32 emulated_msrs[] = {
1733         MSR_IA32_MISC_ENABLE,
1734 };
1735
1736 static __init void kvm_init_msr_list(void)
1737 {
1738         u32 dummy[2];
1739         unsigned i, j;
1740
1741         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1742                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1743                         continue;
1744                 if (j < i)
1745                         msrs_to_save[j] = msrs_to_save[i];
1746                 j++;
1747         }
1748         num_msrs_to_save = j;
1749 }
1750
1751 /*
1752  * Adapt set_msr() to msr_io()'s calling convention
1753  */
1754 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1755 {
1756         return set_msr(vcpu, index, *data);
1757 }
1758
1759 /*
1760  * Read or write a bunch of msrs. All parameters are kernel addresses.
1761  *
1762  * @return number of msrs set successfully.
1763  */
1764 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
1765                     struct kvm_msr_entry *entries,
1766                     int (*do_msr)(struct kvm_vcpu *vcpu,
1767                                   unsigned index, u64 *data))
1768 {
1769         int i;
1770
1771         vcpu_load(vcpu);
1772
1773         for (i = 0; i < msrs->nmsrs; ++i)
1774                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1775                         break;
1776
1777         vcpu_put(vcpu);
1778
1779         return i;
1780 }
1781
1782 /*
1783  * Read or write a bunch of msrs. Parameters are user addresses.
1784  *
1785  * @return number of msrs set successfully.
1786  */
1787 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
1788                   int (*do_msr)(struct kvm_vcpu *vcpu,
1789                                 unsigned index, u64 *data),
1790                   int writeback)
1791 {
1792         struct kvm_msrs msrs;
1793         struct kvm_msr_entry *entries;
1794         int r, n;
1795         unsigned size;
1796
1797         r = -EFAULT;
1798         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1799                 goto out;
1800
1801         r = -E2BIG;
1802         if (msrs.nmsrs >= MAX_IO_MSRS)
1803                 goto out;
1804
1805         r = -ENOMEM;
1806         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1807         entries = vmalloc(size);
1808         if (!entries)
1809                 goto out;
1810
1811         r = -EFAULT;
1812         if (copy_from_user(entries, user_msrs->entries, size))
1813                 goto out_free;
1814
1815         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
1816         if (r < 0)
1817                 goto out_free;
1818
1819         r = -EFAULT;
1820         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1821                 goto out_free;
1822
1823         r = n;
1824
1825 out_free:
1826         vfree(entries);
1827 out:
1828         return r;
1829 }
1830
1831 /*
1832  * Translate a guest virtual address to a guest physical address.
1833  */
1834 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1835                                     struct kvm_translation *tr)
1836 {
1837         unsigned long vaddr = tr->linear_address;
1838         gpa_t gpa;
1839
1840         vcpu_load(vcpu);
1841         spin_lock(&vcpu->kvm->lock);
1842         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1843         tr->physical_address = gpa;
1844         tr->valid = gpa != UNMAPPED_GVA;
1845         tr->writeable = 1;
1846         tr->usermode = 0;
1847         spin_unlock(&vcpu->kvm->lock);
1848         vcpu_put(vcpu);
1849
1850         return 0;
1851 }
1852
1853 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
1854                                     struct kvm_interrupt *irq)
1855 {
1856         if (irq->irq < 0 || irq->irq >= 256)
1857                 return -EINVAL;
1858         vcpu_load(vcpu);
1859
1860         set_bit(irq->irq, vcpu->irq_pending);
1861         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1862
1863         vcpu_put(vcpu);
1864
1865         return 0;
1866 }
1867
1868 static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
1869                                       struct kvm_debug_guest *dbg)
1870 {
1871         int r;
1872
1873         vcpu_load(vcpu);
1874
1875         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1876
1877         vcpu_put(vcpu);
1878
1879         return r;
1880 }
1881
1882 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1883 {
1884         struct kvm_vcpu *vcpu = filp->private_data;
1885
1886         fput(vcpu->kvm->filp);
1887         return 0;
1888 }
1889
1890 static struct file_operations kvm_vcpu_fops = {
1891         .release        = kvm_vcpu_release,
1892         .unlocked_ioctl = kvm_vcpu_ioctl,
1893         .compat_ioctl   = kvm_vcpu_ioctl,
1894 };
1895
1896 /*
1897  * Allocates an inode for the vcpu.
1898  */
1899 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1900 {
1901         int fd, r;
1902         struct inode *inode;
1903         struct file *file;
1904
1905         atomic_inc(&vcpu->kvm->filp->f_count);
1906         inode = kvmfs_inode(&kvm_vcpu_fops);
1907         if (IS_ERR(inode)) {
1908                 r = PTR_ERR(inode);
1909                 goto out1;
1910         }
1911
1912         file = kvmfs_file(inode, vcpu);
1913         if (IS_ERR(file)) {
1914                 r = PTR_ERR(file);
1915                 goto out2;
1916         }
1917
1918         r = get_unused_fd();
1919         if (r < 0)
1920                 goto out3;
1921         fd = r;
1922         fd_install(fd, file);
1923
1924         return fd;
1925
1926 out3:
1927         fput(file);
1928 out2:
1929         iput(inode);
1930 out1:
1931         fput(vcpu->kvm->filp);
1932         return r;
1933 }
1934
1935 /*
1936  * Creates some virtual cpus.  Good luck creating more than one.
1937  */
1938 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
1939 {
1940         int r;
1941         struct kvm_vcpu *vcpu;
1942
1943         r = -EINVAL;
1944         if (!valid_vcpu(n))
1945                 goto out;
1946
1947         vcpu = &kvm->vcpus[n];
1948
1949         mutex_lock(&vcpu->mutex);
1950
1951         if (vcpu->vmcs) {
1952                 mutex_unlock(&vcpu->mutex);
1953                 return -EEXIST;
1954         }
1955
1956         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
1957                                            FX_IMAGE_ALIGN);
1958         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
1959
1960         r = kvm_arch_ops->vcpu_create(vcpu);
1961         if (r < 0)
1962                 goto out_free_vcpus;
1963
1964         r = kvm_mmu_create(vcpu);
1965         if (r < 0)
1966                 goto out_free_vcpus;
1967
1968         kvm_arch_ops->vcpu_load(vcpu);
1969         r = kvm_mmu_setup(vcpu);
1970         if (r >= 0)
1971                 r = kvm_arch_ops->vcpu_setup(vcpu);
1972         vcpu_put(vcpu);
1973
1974         if (r < 0)
1975                 goto out_free_vcpus;
1976
1977         r = create_vcpu_fd(vcpu);
1978         if (r < 0)
1979                 goto out_free_vcpus;
1980
1981         return r;
1982
1983 out_free_vcpus:
1984         kvm_free_vcpu(vcpu);
1985         mutex_unlock(&vcpu->mutex);
1986 out:
1987         return r;
1988 }
1989
1990 static long kvm_vcpu_ioctl(struct file *filp,
1991                            unsigned int ioctl, unsigned long arg)
1992 {
1993         struct kvm_vcpu *vcpu = filp->private_data;
1994         void __user *argp = (void __user *)arg;
1995         int r = -EINVAL;
1996
1997         switch (ioctl) {
1998         case KVM_RUN: {
1999                 struct kvm_run kvm_run;
2000
2001                 r = -EFAULT;
2002                 if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
2003                         goto out;
2004                 r = kvm_vcpu_ioctl_run(vcpu, &kvm_run);
2005                 if (r < 0 &&  r != -EINTR)
2006                         goto out;
2007                 if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
2008                         r = -EFAULT;
2009                         goto out;
2010                 }
2011                 break;
2012         }
2013         case KVM_GET_REGS: {
2014                 struct kvm_regs kvm_regs;
2015
2016                 memset(&kvm_regs, 0, sizeof kvm_regs);
2017                 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
2018                 if (r)
2019                         goto out;
2020                 r = -EFAULT;
2021                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2022                         goto out;
2023                 r = 0;
2024                 break;
2025         }
2026         case KVM_SET_REGS: {
2027                 struct kvm_regs kvm_regs;
2028
2029                 r = -EFAULT;
2030                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2031                         goto out;
2032                 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
2033                 if (r)
2034                         goto out;
2035                 r = 0;
2036                 break;
2037         }
2038         case KVM_GET_SREGS: {
2039                 struct kvm_sregs kvm_sregs;
2040
2041                 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2042                 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
2043                 if (r)
2044                         goto out;
2045                 r = -EFAULT;
2046                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2047                         goto out;
2048                 r = 0;
2049                 break;
2050         }
2051         case KVM_SET_SREGS: {
2052                 struct kvm_sregs kvm_sregs;
2053
2054                 r = -EFAULT;
2055                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2056                         goto out;
2057                 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
2058                 if (r)
2059                         goto out;
2060                 r = 0;
2061                 break;
2062         }
2063         case KVM_TRANSLATE: {
2064                 struct kvm_translation tr;
2065
2066                 r = -EFAULT;
2067                 if (copy_from_user(&tr, argp, sizeof tr))
2068                         goto out;
2069                 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
2070                 if (r)
2071                         goto out;
2072                 r = -EFAULT;
2073                 if (copy_to_user(argp, &tr, sizeof tr))
2074                         goto out;
2075                 r = 0;
2076                 break;
2077         }
2078         case KVM_INTERRUPT: {
2079                 struct kvm_interrupt irq;
2080
2081                 r = -EFAULT;
2082                 if (copy_from_user(&irq, argp, sizeof irq))
2083                         goto out;
2084                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2085                 if (r)
2086                         goto out;
2087                 r = 0;
2088                 break;
2089         }
2090         case KVM_DEBUG_GUEST: {
2091                 struct kvm_debug_guest dbg;
2092
2093                 r = -EFAULT;
2094                 if (copy_from_user(&dbg, argp, sizeof dbg))
2095                         goto out;
2096                 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
2097                 if (r)
2098                         goto out;
2099                 r = 0;
2100                 break;
2101         }
2102         case KVM_GET_MSRS:
2103                 r = msr_io(vcpu, argp, get_msr, 1);
2104                 break;
2105         case KVM_SET_MSRS:
2106                 r = msr_io(vcpu, argp, do_set_msr, 0);
2107                 break;
2108         default:
2109                 ;
2110         }
2111 out:
2112         return r;
2113 }
2114
2115 static long kvm_vm_ioctl(struct file *filp,
2116                            unsigned int ioctl, unsigned long arg)
2117 {
2118         struct kvm *kvm = filp->private_data;
2119         void __user *argp = (void __user *)arg;
2120         int r = -EINVAL;
2121
2122         switch (ioctl) {
2123         case KVM_CREATE_VCPU:
2124                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2125                 if (r < 0)
2126                         goto out;
2127                 break;
2128         case KVM_SET_MEMORY_REGION: {
2129                 struct kvm_memory_region kvm_mem;
2130
2131                 r = -EFAULT;
2132                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2133                         goto out;
2134                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2135                 if (r)
2136                         goto out;
2137                 break;
2138         }
2139         case KVM_GET_DIRTY_LOG: {
2140                 struct kvm_dirty_log log;
2141
2142                 r = -EFAULT;
2143                 if (copy_from_user(&log, argp, sizeof log))
2144                         goto out;
2145                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2146                 if (r)
2147                         goto out;
2148                 break;
2149         }
2150         default:
2151                 ;
2152         }
2153 out:
2154         return r;
2155 }
2156
2157 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2158                                   unsigned long address,
2159                                   int *type)
2160 {
2161         struct kvm *kvm = vma->vm_file->private_data;
2162         unsigned long pgoff;
2163         struct kvm_memory_slot *slot;
2164         struct page *page;
2165
2166         *type = VM_FAULT_MINOR;
2167         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2168         slot = gfn_to_memslot(kvm, pgoff);
2169         if (!slot)
2170                 return NOPAGE_SIGBUS;
2171         page = gfn_to_page(slot, pgoff);
2172         if (!page)
2173                 return NOPAGE_SIGBUS;
2174         get_page(page);
2175         return page;
2176 }
2177
2178 static struct vm_operations_struct kvm_vm_vm_ops = {
2179         .nopage = kvm_vm_nopage,
2180 };
2181
2182 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2183 {
2184         vma->vm_ops = &kvm_vm_vm_ops;
2185         return 0;
2186 }
2187
2188 static struct file_operations kvm_vm_fops = {
2189         .release        = kvm_vm_release,
2190         .unlocked_ioctl = kvm_vm_ioctl,
2191         .compat_ioctl   = kvm_vm_ioctl,
2192         .mmap           = kvm_vm_mmap,
2193 };
2194
2195 static int kvm_dev_ioctl_create_vm(void)
2196 {
2197         int fd, r;
2198         struct inode *inode;
2199         struct file *file;
2200         struct kvm *kvm;
2201
2202         inode = kvmfs_inode(&kvm_vm_fops);
2203         if (IS_ERR(inode)) {
2204                 r = PTR_ERR(inode);
2205                 goto out1;
2206         }
2207
2208         kvm = kvm_create_vm();
2209         if (IS_ERR(kvm)) {
2210                 r = PTR_ERR(kvm);
2211                 goto out2;
2212         }
2213
2214         file = kvmfs_file(inode, kvm);
2215         if (IS_ERR(file)) {
2216                 r = PTR_ERR(file);
2217                 goto out3;
2218         }
2219         kvm->filp = file;
2220
2221         r = get_unused_fd();
2222         if (r < 0)
2223                 goto out4;
2224         fd = r;
2225         fd_install(fd, file);
2226
2227         return fd;
2228
2229 out4:
2230         fput(file);
2231 out3:
2232         kvm_destroy_vm(kvm);
2233 out2:
2234         iput(inode);
2235 out1:
2236         return r;
2237 }
2238
2239 static long kvm_dev_ioctl(struct file *filp,
2240                           unsigned int ioctl, unsigned long arg)
2241 {
2242         void __user *argp = (void __user *)arg;
2243         int r = -EINVAL;
2244
2245         switch (ioctl) {
2246         case KVM_GET_API_VERSION:
2247                 r = KVM_API_VERSION;
2248                 break;
2249         case KVM_CREATE_VM:
2250                 r = kvm_dev_ioctl_create_vm();
2251                 break;
2252         case KVM_GET_MSR_INDEX_LIST: {
2253                 struct kvm_msr_list __user *user_msr_list = argp;
2254                 struct kvm_msr_list msr_list;
2255                 unsigned n;
2256
2257                 r = -EFAULT;
2258                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2259                         goto out;
2260                 n = msr_list.nmsrs;
2261                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2262                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2263                         goto out;
2264                 r = -E2BIG;
2265                 if (n < num_msrs_to_save)
2266                         goto out;
2267                 r = -EFAULT;
2268                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2269                                  num_msrs_to_save * sizeof(u32)))
2270                         goto out;
2271                 if (copy_to_user(user_msr_list->indices
2272                                  + num_msrs_to_save * sizeof(u32),
2273                                  &emulated_msrs,
2274                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2275                         goto out;
2276                 r = 0;
2277                 break;
2278         }
2279         default:
2280                 ;
2281         }
2282 out:
2283         return r;
2284 }
2285
2286 static struct file_operations kvm_chardev_ops = {
2287         .open           = kvm_dev_open,
2288         .release        = kvm_dev_release,
2289         .unlocked_ioctl = kvm_dev_ioctl,
2290         .compat_ioctl   = kvm_dev_ioctl,
2291 };
2292
2293 static struct miscdevice kvm_dev = {
2294         MISC_DYNAMIC_MINOR,
2295         "kvm",
2296         &kvm_chardev_ops,
2297 };
2298
2299 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2300                        void *v)
2301 {
2302         if (val == SYS_RESTART) {
2303                 /*
2304                  * Some (well, at least mine) BIOSes hang on reboot if
2305                  * in vmx root mode.
2306                  */
2307                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2308                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2309         }
2310         return NOTIFY_OK;
2311 }
2312
2313 static struct notifier_block kvm_reboot_notifier = {
2314         .notifier_call = kvm_reboot,
2315         .priority = 0,
2316 };
2317
2318 /*
2319  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2320  * cached on it.
2321  */
2322 static void decache_vcpus_on_cpu(int cpu)
2323 {
2324         struct kvm *vm;
2325         struct kvm_vcpu *vcpu;
2326         int i;
2327
2328         spin_lock(&kvm_lock);
2329         list_for_each_entry(vm, &vm_list, vm_list)
2330                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2331                         vcpu = &vm->vcpus[i];
2332                         /*
2333                          * If the vcpu is locked, then it is running on some
2334                          * other cpu and therefore it is not cached on the
2335                          * cpu in question.
2336                          *
2337                          * If it's not locked, check the last cpu it executed
2338                          * on.
2339                          */
2340                         if (mutex_trylock(&vcpu->mutex)) {
2341                                 if (vcpu->cpu == cpu) {
2342                                         kvm_arch_ops->vcpu_decache(vcpu);
2343                                         vcpu->cpu = -1;
2344                                 }
2345                                 mutex_unlock(&vcpu->mutex);
2346                         }
2347                 }
2348         spin_unlock(&kvm_lock);
2349 }
2350
2351 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2352                            void *v)
2353 {
2354         int cpu = (long)v;
2355
2356         switch (val) {
2357         case CPU_DOWN_PREPARE:
2358         case CPU_UP_CANCELED:
2359                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2360                        cpu);
2361                 decache_vcpus_on_cpu(cpu);
2362                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
2363                                          NULL, 0, 1);
2364                 break;
2365         case CPU_ONLINE:
2366                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2367                        cpu);
2368                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
2369                                          NULL, 0, 1);
2370                 break;
2371         }
2372         return NOTIFY_OK;
2373 }
2374
2375 static struct notifier_block kvm_cpu_notifier = {
2376         .notifier_call = kvm_cpu_hotplug,
2377         .priority = 20, /* must be > scheduler priority */
2378 };
2379
2380 static __init void kvm_init_debug(void)
2381 {
2382         struct kvm_stats_debugfs_item *p;
2383
2384         debugfs_dir = debugfs_create_dir("kvm", NULL);
2385         for (p = debugfs_entries; p->name; ++p)
2386                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2387                                                p->data);
2388 }
2389
2390 static void kvm_exit_debug(void)
2391 {
2392         struct kvm_stats_debugfs_item *p;
2393
2394         for (p = debugfs_entries; p->name; ++p)
2395                 debugfs_remove(p->dentry);
2396         debugfs_remove(debugfs_dir);
2397 }
2398
2399 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2400 {
2401         decache_vcpus_on_cpu(raw_smp_processor_id());
2402         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2403         return 0;
2404 }
2405
2406 static int kvm_resume(struct sys_device *dev)
2407 {
2408         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2409         return 0;
2410 }
2411
2412 static struct sysdev_class kvm_sysdev_class = {
2413         set_kset_name("kvm"),
2414         .suspend = kvm_suspend,
2415         .resume = kvm_resume,
2416 };
2417
2418 static struct sys_device kvm_sysdev = {
2419         .id = 0,
2420         .cls = &kvm_sysdev_class,
2421 };
2422
2423 hpa_t bad_page_address;
2424
2425 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
2426                         const char *dev_name, void *data, struct vfsmount *mnt)
2427 {
2428         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_MAGIC, mnt);
2429 }
2430
2431 static struct file_system_type kvm_fs_type = {
2432         .name           = "kvmfs",
2433         .get_sb         = kvmfs_get_sb,
2434         .kill_sb        = kill_anon_super,
2435 };
2436
2437 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2438 {
2439         int r;
2440
2441         if (kvm_arch_ops) {
2442                 printk(KERN_ERR "kvm: already loaded the other module\n");
2443                 return -EEXIST;
2444         }
2445
2446         if (!ops->cpu_has_kvm_support()) {
2447                 printk(KERN_ERR "kvm: no hardware support\n");
2448                 return -EOPNOTSUPP;
2449         }
2450         if (ops->disabled_by_bios()) {
2451                 printk(KERN_ERR "kvm: disabled by bios\n");
2452                 return -EOPNOTSUPP;
2453         }
2454
2455         kvm_arch_ops = ops;
2456
2457         r = kvm_arch_ops->hardware_setup();
2458         if (r < 0)
2459             return r;
2460
2461         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2462         r = register_cpu_notifier(&kvm_cpu_notifier);
2463         if (r)
2464                 goto out_free_1;
2465         register_reboot_notifier(&kvm_reboot_notifier);
2466
2467         r = sysdev_class_register(&kvm_sysdev_class);
2468         if (r)
2469                 goto out_free_2;
2470
2471         r = sysdev_register(&kvm_sysdev);
2472         if (r)
2473                 goto out_free_3;
2474
2475         kvm_chardev_ops.owner = module;
2476
2477         r = misc_register(&kvm_dev);
2478         if (r) {
2479                 printk (KERN_ERR "kvm: misc device register failed\n");
2480                 goto out_free;
2481         }
2482
2483         return r;
2484
2485 out_free:
2486         sysdev_unregister(&kvm_sysdev);
2487 out_free_3:
2488         sysdev_class_unregister(&kvm_sysdev_class);
2489 out_free_2:
2490         unregister_reboot_notifier(&kvm_reboot_notifier);
2491         unregister_cpu_notifier(&kvm_cpu_notifier);
2492 out_free_1:
2493         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2494         kvm_arch_ops->hardware_unsetup();
2495         return r;
2496 }
2497
2498 void kvm_exit_arch(void)
2499 {
2500         misc_deregister(&kvm_dev);
2501         sysdev_unregister(&kvm_sysdev);
2502         sysdev_class_unregister(&kvm_sysdev_class);
2503         unregister_reboot_notifier(&kvm_reboot_notifier);
2504         unregister_cpu_notifier(&kvm_cpu_notifier);
2505         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2506         kvm_arch_ops->hardware_unsetup();
2507         kvm_arch_ops = NULL;
2508 }
2509
2510 static __init int kvm_init(void)
2511 {
2512         static struct page *bad_page;
2513         int r;
2514
2515         r = register_filesystem(&kvm_fs_type);
2516         if (r)
2517                 goto out3;
2518
2519         kvmfs_mnt = kern_mount(&kvm_fs_type);
2520         r = PTR_ERR(kvmfs_mnt);
2521         if (IS_ERR(kvmfs_mnt))
2522                 goto out2;
2523         kvm_init_debug();
2524
2525         kvm_init_msr_list();
2526
2527         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2528                 r = -ENOMEM;
2529                 goto out;
2530         }
2531
2532         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2533         memset(__va(bad_page_address), 0, PAGE_SIZE);
2534
2535         return r;
2536
2537 out:
2538         kvm_exit_debug();
2539         mntput(kvmfs_mnt);
2540 out2:
2541         unregister_filesystem(&kvm_fs_type);
2542 out3:
2543         return r;
2544 }
2545
2546 static __exit void kvm_exit(void)
2547 {
2548         kvm_exit_debug();
2549         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2550         mntput(kvmfs_mnt);
2551         unregister_filesystem(&kvm_fs_type);
2552 }
2553
2554 module_init(kvm_init)
2555 module_exit(kvm_exit)
2556
2557 EXPORT_SYMBOL_GPL(kvm_init_arch);
2558 EXPORT_SYMBOL_GPL(kvm_exit_arch);