Pileus Git - ~andy/linux/blob - arch/powerpc/kvm/booke.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright IBM Corp. 2007
  16  * Copyright 2010-2011 Freescale Semiconductor, Inc.
  17  *
  18  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  19  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  20  *          Scott Wood <scottwood@freescale.com>
  21  *          Varun Sethi <varun.sethi@freescale.com>
  22  */
  23
  24 #include <linux/errno.h>
  25 #include <linux/err.h>
  26 #include <linux/kvm_host.h>
  27 #include <linux/gfp.h>
  28 #include <linux/module.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/fs.h>
  31
  32 #include <asm/cputable.h>
  33 #include <asm/uaccess.h>
  34 #include <asm/kvm_ppc.h>
  35 #include <asm/cacheflush.h>
  36 #include <asm/dbell.h>
  37 #include <asm/hw_irq.h>
  38 #include <asm/irq.h>
  39
  40 #include "timing.h"
  41 #include "booke.h"
  42 #include "trace.h"
  43
  44 unsigned long kvmppc_booke_handlers;
  45
  46 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  47 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  48
  49 struct kvm_stats_debugfs_item debugfs_entries[] = {
  50         { "mmio",       VCPU_STAT(mmio_exits) },
  51         { "dcr",        VCPU_STAT(dcr_exits) },
  52         { "sig",        VCPU_STAT(signal_exits) },
  53         { "itlb_r",     VCPU_STAT(itlb_real_miss_exits) },
  54         { "itlb_v",     VCPU_STAT(itlb_virt_miss_exits) },
  55         { "dtlb_r",     VCPU_STAT(dtlb_real_miss_exits) },
  56         { "dtlb_v",     VCPU_STAT(dtlb_virt_miss_exits) },
  57         { "sysc",       VCPU_STAT(syscall_exits) },
  58         { "isi",        VCPU_STAT(isi_exits) },
  59         { "dsi",        VCPU_STAT(dsi_exits) },
  60         { "inst_emu",   VCPU_STAT(emulated_inst_exits) },
  61         { "dec",        VCPU_STAT(dec_exits) },
  62         { "ext_intr",   VCPU_STAT(ext_intr_exits) },
  63         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  64         { "doorbell", VCPU_STAT(dbell_exits) },
  65         { "guest doorbell", VCPU_STAT(gdbell_exits) },
  66         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  67         { NULL }
  68 };
  69
  70 /* TODO: use vcpu_printf() */
  71 void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
  72 {
  73         int i;
  74
  75         printk("pc:   %08lx msr:  %08llx\n", vcpu->arch.pc, vcpu->arch.shared->msr);
  76         printk("lr:   %08lx ctr:  %08lx\n", vcpu->arch.lr, vcpu->arch.ctr);
  77         printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
  78                                             vcpu->arch.shared->srr1);
  79
  80         printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
  81
  82         for (i = 0; i < 32; i += 4) {
  83                 printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
  84                        kvmppc_get_gpr(vcpu, i),
  85                        kvmppc_get_gpr(vcpu, i+1),
  86                        kvmppc_get_gpr(vcpu, i+2),
  87                        kvmppc_get_gpr(vcpu, i+3));
  88         }
  89 }
  90
  91 #ifdef CONFIG_SPE
  92 void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
  93 {
  94         preempt_disable();
  95         enable_kernel_spe();
  96         kvmppc_save_guest_spe(vcpu);
  97         vcpu->arch.shadow_msr &= ~MSR_SPE;
  98         preempt_enable();
  99 }
 100
 101 static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
 102 {
 103         preempt_disable();
 104         enable_kernel_spe();
 105         kvmppc_load_guest_spe(vcpu);
 106         vcpu->arch.shadow_msr |= MSR_SPE;
 107         preempt_enable();
 108 }
 109
 110 static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
 111 {
 112         if (vcpu->arch.shared->msr & MSR_SPE) {
 113                 if (!(vcpu->arch.shadow_msr & MSR_SPE))
 114                         kvmppc_vcpu_enable_spe(vcpu);
 115         } else if (vcpu->arch.shadow_msr & MSR_SPE) {
 116                 kvmppc_vcpu_disable_spe(vcpu);
 117         }
 118 }
 119 #else
 120 static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
 121 {
 122 }
 123 #endif
 124
 125 /*
 126  * Helper function for "full" MSR writes.  No need to call this if only
 127  * EE/CE/ME/DE/RI are changing.
 128  */
 129 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
 130 {
 131         u32 old_msr = vcpu->arch.shared->msr;
 132
 133 #ifdef CONFIG_KVM_BOOKE_HV
 134         new_msr |= MSR_GS;
 135 #endif
 136
 137         vcpu->arch.shared->msr = new_msr;
 138
 139         kvmppc_mmu_msr_notify(vcpu, old_msr);
 140         kvmppc_vcpu_sync_spe(vcpu);
 141 }
 142
 143 static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
 144                                        unsigned int priority)
 145 {
 146         set_bit(priority, &vcpu->arch.pending_exceptions);
 147 }
 148
 149 static void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
 150                                         ulong dear_flags, ulong esr_flags)
 151 {
 152         vcpu->arch.queued_dear = dear_flags;
 153         vcpu->arch.queued_esr = esr_flags;
 154         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
 155 }
 156
 157 static void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu,
 158                                            ulong dear_flags, ulong esr_flags)
 159 {
 160         vcpu->arch.queued_dear = dear_flags;
 161         vcpu->arch.queued_esr = esr_flags;
 162         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
 163 }
 164
 165 static void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu,
 166                                            ulong esr_flags)
 167 {
 168         vcpu->arch.queued_esr = esr_flags;
 169         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
 170 }
 171
 172 void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
 173 {
 174         vcpu->arch.queued_esr = esr_flags;
 175         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
 176 }
 177
 178 void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
 179 {
 180         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
 181 }
 182
 183 int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
 184 {
 185         return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 186 }
 187
 188 void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
 189 {
 190         clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 191 }
 192
 193 void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
 194                                 struct kvm_interrupt *irq)
 195 {
 196         unsigned int prio = BOOKE_IRQPRIO_EXTERNAL;
 197
 198         if (irq->irq == KVM_INTERRUPT_SET_LEVEL)
 199                 prio = BOOKE_IRQPRIO_EXTERNAL_LEVEL;
 200
 201         kvmppc_booke_queue_irqprio(vcpu, prio);
 202 }
 203
 204 void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu,
 205                                   struct kvm_interrupt *irq)
 206 {
 207         clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
 208         clear_bit(BOOKE_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
 209 }
 210
 211 static void set_guest_srr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 212 {
 213 #ifdef CONFIG_KVM_BOOKE_HV
 214         mtspr(SPRN_GSRR0, srr0);
 215         mtspr(SPRN_GSRR1, srr1);
 216 #else
 217         vcpu->arch.shared->srr0 = srr0;
 218         vcpu->arch.shared->srr1 = srr1;
 219 #endif
 220 }
 221
 222 static void set_guest_csrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 223 {
 224         vcpu->arch.csrr0 = srr0;
 225         vcpu->arch.csrr1 = srr1;
 226 }
 227
 228 static void set_guest_dsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 229 {
 230         if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) {
 231                 vcpu->arch.dsrr0 = srr0;
 232                 vcpu->arch.dsrr1 = srr1;
 233         } else {
 234                 set_guest_csrr(vcpu, srr0, srr1);
 235         }
 236 }
 237
 238 static void set_guest_mcsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
 239 {
 240         vcpu->arch.mcsrr0 = srr0;
 241         vcpu->arch.mcsrr1 = srr1;
 242 }
 243
 244 static unsigned long get_guest_dear(struct kvm_vcpu *vcpu)
 245 {
 246 #ifdef CONFIG_KVM_BOOKE_HV
 247         return mfspr(SPRN_GDEAR);
 248 #else
 249         return vcpu->arch.shared->dar;
 250 #endif
 251 }
 252
 253 static void set_guest_dear(struct kvm_vcpu *vcpu, unsigned long dear)
 254 {
 255 #ifdef CONFIG_KVM_BOOKE_HV
 256         mtspr(SPRN_GDEAR, dear);
 257 #else
 258         vcpu->arch.shared->dar = dear;
 259 #endif
 260 }
 261
 262 static unsigned long get_guest_esr(struct kvm_vcpu *vcpu)
 263 {
 264 #ifdef CONFIG_KVM_BOOKE_HV
 265         return mfspr(SPRN_GESR);
 266 #else
 267         return vcpu->arch.shared->esr;
 268 #endif
 269 }
 270
 271 static void set_guest_esr(struct kvm_vcpu *vcpu, u32 esr)
 272 {
 273 #ifdef CONFIG_KVM_BOOKE_HV
 274         mtspr(SPRN_GESR, esr);
 275 #else
 276         vcpu->arch.shared->esr = esr;
 277 #endif
 278 }
 279
 280 /* Deliver the interrupt of the corresponding priority, if possible. */
 281 static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
 282                                         unsigned int priority)
 283 {
 284         int allowed = 0;
 285         ulong msr_mask = 0;
 286         bool update_esr = false, update_dear = false;
 287         ulong crit_raw = vcpu->arch.shared->critical;
 288         ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
 289         bool crit;
 290         bool keep_irq = false;
 291         enum int_class int_class;
 292
 293         /* Truncate crit indicators in 32 bit mode */
 294         if (!(vcpu->arch.shared->msr & MSR_SF)) {
 295                 crit_raw &= 0xffffffff;
 296                 crit_r1 &= 0xffffffff;
 297         }
 298
 299         /* Critical section when crit == r1 */
 300         crit = (crit_raw == crit_r1);
 301         /* ... and we're in supervisor mode */
 302         crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
 303
 304         if (priority == BOOKE_IRQPRIO_EXTERNAL_LEVEL) {
 305                 priority = BOOKE_IRQPRIO_EXTERNAL;
 306                 keep_irq = true;
 307         }
 308
 309         switch (priority) {
 310         case BOOKE_IRQPRIO_DTLB_MISS:
 311         case BOOKE_IRQPRIO_DATA_STORAGE:
 312                 update_dear = true;
 313                 /* fall through */
 314         case BOOKE_IRQPRIO_INST_STORAGE:
 315         case BOOKE_IRQPRIO_PROGRAM:
 316                 update_esr = true;
 317                 /* fall through */
 318         case BOOKE_IRQPRIO_ITLB_MISS:
 319         case BOOKE_IRQPRIO_SYSCALL:
 320         case BOOKE_IRQPRIO_FP_UNAVAIL:
 321         case BOOKE_IRQPRIO_SPE_UNAVAIL:
 322         case BOOKE_IRQPRIO_SPE_FP_DATA:
 323         case BOOKE_IRQPRIO_SPE_FP_ROUND:
 324         case BOOKE_IRQPRIO_AP_UNAVAIL:
 325         case BOOKE_IRQPRIO_ALIGNMENT:
 326                 allowed = 1;
 327                 msr_mask = MSR_CE | MSR_ME | MSR_DE;
 328                 int_class = INT_CLASS_NONCRIT;
 329                 break;
 330         case BOOKE_IRQPRIO_CRITICAL:
 331         case BOOKE_IRQPRIO_DBELL_CRIT:
 332                 allowed = vcpu->arch.shared->msr & MSR_CE;
 333                 allowed = allowed && !crit;
 334                 msr_mask = MSR_ME;
 335                 int_class = INT_CLASS_CRIT;
 336                 break;
 337         case BOOKE_IRQPRIO_MACHINE_CHECK:
 338                 allowed = vcpu->arch.shared->msr & MSR_ME;
 339                 allowed = allowed && !crit;
 340                 int_class = INT_CLASS_MC;
 341                 break;
 342         case BOOKE_IRQPRIO_DECREMENTER:
 343         case BOOKE_IRQPRIO_FIT:
 344                 keep_irq = true;
 345                 /* fall through */
 346         case BOOKE_IRQPRIO_EXTERNAL:
 347         case BOOKE_IRQPRIO_DBELL:
 348                 allowed = vcpu->arch.shared->msr & MSR_EE;
 349                 allowed = allowed && !crit;
 350                 msr_mask = MSR_CE | MSR_ME | MSR_DE;
 351                 int_class = INT_CLASS_NONCRIT;
 352                 break;
 353         case BOOKE_IRQPRIO_DEBUG:
 354                 allowed = vcpu->arch.shared->msr & MSR_DE;
 355                 allowed = allowed && !crit;
 356                 msr_mask = MSR_ME;
 357                 int_class = INT_CLASS_CRIT;
 358                 break;
 359         }
 360
 361         if (allowed) {
 362                 switch (int_class) {
 363                 case INT_CLASS_NONCRIT:
 364                         set_guest_srr(vcpu, vcpu->arch.pc,
 365                                       vcpu->arch.shared->msr);
 366                         break;
 367                 case INT_CLASS_CRIT:
 368                         set_guest_csrr(vcpu, vcpu->arch.pc,
 369                                        vcpu->arch.shared->msr);
 370                         break;
 371                 case INT_CLASS_DBG:
 372                         set_guest_dsrr(vcpu, vcpu->arch.pc,
 373                                        vcpu->arch.shared->msr);
 374                         break;
 375                 case INT_CLASS_MC:
 376                         set_guest_mcsrr(vcpu, vcpu->arch.pc,
 377                                         vcpu->arch.shared->msr);
 378                         break;
 379                 }
 380
 381                 vcpu->arch.pc = vcpu->arch.ivpr | vcpu->arch.ivor[priority];
 382                 if (update_esr == true)
 383                         set_guest_esr(vcpu, vcpu->arch.queued_esr);
 384                 if (update_dear == true)
 385                         set_guest_dear(vcpu, vcpu->arch.queued_dear);
 386                 kvmppc_set_msr(vcpu, vcpu->arch.shared->msr & msr_mask);
 387
 388                 if (!keep_irq)
 389                         clear_bit(priority, &vcpu->arch.pending_exceptions);
 390         }
 391
 392 #ifdef CONFIG_KVM_BOOKE_HV
 393         /*
 394          * If an interrupt is pending but masked, raise a guest doorbell
 395          * so that we are notified when the guest enables the relevant
 396          * MSR bit.
 397          */
 398         if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_EE)
 399                 kvmppc_set_pending_interrupt(vcpu, INT_CLASS_NONCRIT);
 400         if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_CE)
 401                 kvmppc_set_pending_interrupt(vcpu, INT_CLASS_CRIT);
 402         if (vcpu->arch.pending_exceptions & BOOKE_IRQPRIO_MACHINE_CHECK)
 403                 kvmppc_set_pending_interrupt(vcpu, INT_CLASS_MC);
 404 #endif
 405
 406         return allowed;
 407 }
 408
 409 static void update_timer_ints(struct kvm_vcpu *vcpu)
 410 {
 411         if ((vcpu->arch.tcr & TCR_DIE) && (vcpu->arch.tsr & TSR_DIS))
 412                 kvmppc_core_queue_dec(vcpu);
 413         else
 414                 kvmppc_core_dequeue_dec(vcpu);
 415 }
 416
 417 static void kvmppc_core_check_exceptions(struct kvm_vcpu *vcpu)
 418 {
 419         unsigned long *pending = &vcpu->arch.pending_exceptions;
 420         unsigned int priority;
 421
 422         priority = __ffs(*pending);
 423         while (priority < BOOKE_IRQPRIO_MAX) {
 424                 if (kvmppc_booke_irqprio_deliver(vcpu, priority))
 425                         break;
 426
 427                 priority = find_next_bit(pending,
 428                                          BITS_PER_BYTE * sizeof(*pending),
 429                                          priority + 1);
 430         }
 431
 432         /* Tell the guest about our interrupt status */
 433         vcpu->arch.shared->int_pending = !!*pending;
 434 }
 435
 436 /* Check pending exceptions and deliver one, if possible. */
 437 int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu)
 438 {
 439         int r = 0;
 440         WARN_ON_ONCE(!irqs_disabled());
 441
 442         kvmppc_core_check_exceptions(vcpu);
 443
 444         if (vcpu->arch.shared->msr & MSR_WE) {
 445                 local_irq_enable();
 446                 kvm_vcpu_block(vcpu);
 447                 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 448                 local_irq_disable();
 449
 450                 kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
 451                 r = 1;
 452         };
 453
 454         return r;
 455 }
 456
 457 static void kvmppc_check_requests(struct kvm_vcpu *vcpu)
 458 {
 459         if (vcpu->requests) {
 460                 if (kvm_check_request(KVM_REQ_PENDING_TIMER, vcpu))
 461                         update_timer_ints(vcpu);
 462         }
 463 }
 464
 465 /*
 466  * Common checks before entering the guest world.  Call with interrupts
 467  * disabled.
 468  *
 469  * returns !0 if a signal is pending and check_signal is true
 470  */
 471 static int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
 472 {
 473         int r = 0;
 474
 475         WARN_ON_ONCE(!irqs_disabled());
 476         while (true) {
 477                 if (need_resched()) {
 478                         local_irq_enable();
 479                         cond_resched();
 480                         local_irq_disable();
 481                         continue;
 482                 }
 483
 484                 if (signal_pending(current)) {
 485                         r = 1;
 486                         break;
 487                 }
 488
 489                 smp_mb();
 490                 if (vcpu->requests) {
 491                         /* Make sure we process requests preemptable */
 492                         local_irq_enable();
 493                         kvmppc_check_requests(vcpu);
 494                         local_irq_disable();
 495                         continue;
 496                 }
 497
 498                 if (kvmppc_core_prepare_to_enter(vcpu)) {
 499                         /* interrupts got enabled in between, so we
 500                            are back at square 1 */
 501                         continue;
 502                 }
 503
 504                 if (vcpu->mode == EXITING_GUEST_MODE) {
 505                         r = 1;
 506                         break;
 507                 }
 508
 509                 /* Going into guest context! Yay! */
 510                 vcpu->mode = IN_GUEST_MODE;
 511                 smp_wmb();
 512
 513                 break;
 514         }
 515
 516         return r;
 517 }
 518
 519 int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 520 {
 521         int ret;
 522 #ifdef CONFIG_PPC_FPU
 523         unsigned int fpscr;
 524         int fpexc_mode;
 525         u64 fpr[32];
 526 #endif
 527
 528         if (!vcpu->arch.sane) {
 529                 kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 530                 return -EINVAL;
 531         }
 532
 533         local_irq_disable();
 534         if (kvmppc_prepare_to_enter(vcpu)) {
 535                 kvm_run->exit_reason = KVM_EXIT_INTR;
 536                 ret = -EINTR;
 537                 goto out;
 538         }
 539
 540         kvm_guest_enter();
 541
 542 #ifdef CONFIG_PPC_FPU
 543         /* Save userspace FPU state in stack */
 544         enable_kernel_fp();
 545         memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr));
 546         fpscr = current->thread.fpscr.val;
 547         fpexc_mode = current->thread.fpexc_mode;
 548
 549         /* Restore guest FPU state to thread */
 550         memcpy(current->thread.fpr, vcpu->arch.fpr, sizeof(vcpu->arch.fpr));
 551         current->thread.fpscr.val = vcpu->arch.fpscr;
 552
 553         /*
 554          * Since we can't trap on MSR_FP in GS-mode, we consider the guest
 555          * as always using the FPU.  Kernel usage of FP (via
 556          * enable_kernel_fp()) in this thread must not occur while
 557          * vcpu->fpu_active is set.
 558          */
 559         vcpu->fpu_active = 1;
 560
 561         kvmppc_load_guest_fp(vcpu);
 562 #endif
 563
 564         ret = __kvmppc_vcpu_run(kvm_run, vcpu);
 565
 566 #ifdef CONFIG_PPC_FPU
 567         kvmppc_save_guest_fp(vcpu);
 568
 569         vcpu->fpu_active = 0;
 570
 571         /* Save guest FPU state from thread */
 572         memcpy(vcpu->arch.fpr, current->thread.fpr, sizeof(vcpu->arch.fpr));
 573         vcpu->arch.fpscr = current->thread.fpscr.val;
 574
 575         /* Restore userspace FPU state from stack */
 576         memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr));
 577         current->thread.fpscr.val = fpscr;
 578         current->thread.fpexc_mode = fpexc_mode;
 579 #endif
 580
 581         kvm_guest_exit();
 582
 583 out:
 584         vcpu->mode = OUTSIDE_GUEST_MODE;
 585         smp_wmb();
 586         local_irq_enable();
 587         return ret;
 588 }
 589
 590 static int emulation_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
 591 {
 592         enum emulation_result er;
 593
 594         er = kvmppc_emulate_instruction(run, vcpu);
 595         switch (er) {
 596         case EMULATE_DONE:
 597                 /* don't overwrite subtypes, just account kvm_stats */
 598                 kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
 599                 /* Future optimization: only reload non-volatiles if
 600                  * they were actually modified by emulation. */
 601                 return RESUME_GUEST_NV;
 602
 603         case EMULATE_DO_DCR:
 604                 run->exit_reason = KVM_EXIT_DCR;
 605                 return RESUME_HOST;
 606
 607         case EMULATE_FAIL:
 608                 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
 609                        __func__, vcpu->arch.pc, vcpu->arch.last_inst);
 610                 /* For debugging, encode the failing instruction and
 611                  * report it to userspace. */
 612                 run->hw.hardware_exit_reason = ~0ULL << 32;
 613                 run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
 614                 kvmppc_core_queue_program(vcpu, ESR_PIL);
 615                 return RESUME_HOST;
 616
 617         default:
 618                 BUG();
 619         }
 620 }
 621
 622 static void kvmppc_fill_pt_regs(struct pt_regs *regs)
 623 {
 624         ulong r1, ip, msr, lr;
 625
 626         asm("mr %0, 1" : "=r"(r1));
 627         asm("mflr %0" : "=r"(lr));
 628         asm("mfmsr %0" : "=r"(msr));
 629         asm("bl 1f; 1: mflr %0" : "=r"(ip));
 630
 631         memset(regs, 0, sizeof(*regs));
 632         regs->gpr[1] = r1;
 633         regs->nip = ip;
 634         regs->msr = msr;
 635         regs->link = lr;
 636 }
 637
 638 /*
 639  * For interrupts needed to be handled by host interrupt handlers,
 640  * corresponding host handler are called from here in similar way
 641  * (but not exact) as they are called from low level handler
 642  * (such as from arch/powerpc/kernel/head_fsl_booke.S).
 643  */
 644 static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu,
 645                                      unsigned int exit_nr)
 646 {
 647         struct pt_regs regs;
 648
 649         switch (exit_nr) {
 650         case BOOKE_INTERRUPT_EXTERNAL:
 651                 kvmppc_fill_pt_regs(&regs);
 652                 do_IRQ(&regs);
 653                 break;
 654         case BOOKE_INTERRUPT_DECREMENTER:
 655                 kvmppc_fill_pt_regs(&regs);
 656                 timer_interrupt(&regs);
 657                 break;
 658 #if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_BOOK3E_64)
 659         case BOOKE_INTERRUPT_DOORBELL:
 660                 kvmppc_fill_pt_regs(&regs);
 661                 doorbell_exception(&regs);
 662                 break;
 663 #endif
 664         case BOOKE_INTERRUPT_MACHINE_CHECK:
 665                 /* FIXME */
 666                 break;
 667         case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
 668                 kvmppc_fill_pt_regs(&regs);
 669                 performance_monitor_exception(&regs);
 670                 break;
 671         case BOOKE_INTERRUPT_WATCHDOG:
 672                 kvmppc_fill_pt_regs(&regs);
 673 #ifdef CONFIG_BOOKE_WDT
 674                 WatchdogException(&regs);
 675 #else
 676                 unknown_exception(&regs);
 677 #endif
 678                 break;
 679         case BOOKE_INTERRUPT_CRITICAL:
 680                 unknown_exception(&regs);
 681                 break;
 682         }
 683 }
 684
 685 /**
 686  * kvmppc_handle_exit
 687  *
 688  * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
 689  */
 690 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
 691                        unsigned int exit_nr)
 692 {
 693         int r = RESUME_HOST;
 694
 695         /* update before a new last_exit_type is rewritten */
 696         kvmppc_update_timing_stats(vcpu);
 697
 698         /* restart interrupts if they were meant for the host */
 699         kvmppc_restart_interrupt(vcpu, exit_nr);
 700
 701         local_irq_enable();
 702
 703         trace_kvm_exit(exit_nr, vcpu);
 704
 705         run->exit_reason = KVM_EXIT_UNKNOWN;
 706         run->ready_for_interrupt_injection = 1;
 707
 708         switch (exit_nr) {
 709         case BOOKE_INTERRUPT_MACHINE_CHECK:
 710                 printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
 711                 kvmppc_dump_vcpu(vcpu);
 712                 /* For debugging, send invalid exit reason to user space */
 713                 run->hw.hardware_exit_reason = ~1ULL << 32;
 714                 run->hw.hardware_exit_reason |= mfspr(SPRN_MCSR);
 715                 r = RESUME_HOST;
 716                 break;
 717
 718         case BOOKE_INTERRUPT_EXTERNAL:
 719                 kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
 720                 r = RESUME_GUEST;
 721                 break;
 722
 723         case BOOKE_INTERRUPT_DECREMENTER:
 724                 kvmppc_account_exit(vcpu, DEC_EXITS);
 725                 r = RESUME_GUEST;
 726                 break;
 727
 728         case BOOKE_INTERRUPT_WATCHDOG:
 729                 r = RESUME_GUEST;
 730                 break;
 731
 732         case BOOKE_INTERRUPT_DOORBELL:
 733                 kvmppc_account_exit(vcpu, DBELL_EXITS);
 734                 r = RESUME_GUEST;
 735                 break;
 736
 737         case BOOKE_INTERRUPT_GUEST_DBELL_CRIT:
 738                 kvmppc_account_exit(vcpu, GDBELL_EXITS);
 739
 740                 /*
 741                  * We are here because there is a pending guest interrupt
 742                  * which could not be delivered as MSR_CE or MSR_ME was not
 743                  * set.  Once we break from here we will retry delivery.
 744                  */
 745                 r = RESUME_GUEST;
 746                 break;
 747
 748         case BOOKE_INTERRUPT_GUEST_DBELL:
 749                 kvmppc_account_exit(vcpu, GDBELL_EXITS);
 750
 751                 /*
 752                  * We are here because there is a pending guest interrupt
 753                  * which could not be delivered as MSR_EE was not set.  Once
 754                  * we break from here we will retry delivery.
 755                  */
 756                 r = RESUME_GUEST;
 757                 break;
 758
 759         case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
 760                 r = RESUME_GUEST;
 761                 break;
 762
 763         case BOOKE_INTERRUPT_HV_PRIV:
 764                 r = emulation_exit(run, vcpu);
 765                 break;
 766
 767         case BOOKE_INTERRUPT_PROGRAM:
 768                 if (vcpu->arch.shared->msr & (MSR_PR | MSR_GS)) {
 769                         /*
 770                          * Program traps generated by user-level software must
 771                          * be handled by the guest kernel.
 772                          *
 773                          * In GS mode, hypervisor privileged instructions trap
 774                          * on BOOKE_INTERRUPT_HV_PRIV, not here, so these are
 775                          * actual program interrupts, handled by the guest.
 776                          */
 777                         kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
 778                         r = RESUME_GUEST;
 779                         kvmppc_account_exit(vcpu, USR_PR_INST);
 780                         break;
 781                 }
 782
 783                 r = emulation_exit(run, vcpu);
 784                 break;
 785
 786         case BOOKE_INTERRUPT_FP_UNAVAIL:
 787                 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
 788                 kvmppc_account_exit(vcpu, FP_UNAVAIL);
 789                 r = RESUME_GUEST;
 790                 break;
 791
 792 #ifdef CONFIG_SPE
 793         case BOOKE_INTERRUPT_SPE_UNAVAIL: {
 794                 if (vcpu->arch.shared->msr & MSR_SPE)
 795                         kvmppc_vcpu_enable_spe(vcpu);
 796                 else
 797                         kvmppc_booke_queue_irqprio(vcpu,
 798                                                    BOOKE_IRQPRIO_SPE_UNAVAIL);
 799                 r = RESUME_GUEST;
 800                 break;
 801         }
 802
 803         case BOOKE_INTERRUPT_SPE_FP_DATA:
 804                 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
 805                 r = RESUME_GUEST;
 806                 break;
 807
 808         case BOOKE_INTERRUPT_SPE_FP_ROUND:
 809                 kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
 810                 r = RESUME_GUEST;
 811                 break;
 812 #else
 813         case BOOKE_INTERRUPT_SPE_UNAVAIL:
 814                 /*
 815                  * Guest wants SPE, but host kernel doesn't support it.  Send
 816                  * an "unimplemented operation" program check to the guest.
 817                  */
 818                 kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
 819                 r = RESUME_GUEST;
 820                 break;
 821
 822         /*
 823          * These really should never happen without CONFIG_SPE,
 824          * as we should never enable the real MSR[SPE] in the guest.
 825          */
 826         case BOOKE_INTERRUPT_SPE_FP_DATA:
 827         case BOOKE_INTERRUPT_SPE_FP_ROUND:
 828                 printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
 829                        __func__, exit_nr, vcpu->arch.pc);
 830                 run->hw.hardware_exit_reason = exit_nr;
 831                 r = RESUME_HOST;
 832                 break;
 833 #endif
 834
 835         case BOOKE_INTERRUPT_DATA_STORAGE:
 836                 kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
 837                                                vcpu->arch.fault_esr);
 838                 kvmppc_account_exit(vcpu, DSI_EXITS);
 839                 r = RESUME_GUEST;
 840                 break;
 841
 842         case BOOKE_INTERRUPT_INST_STORAGE:
 843                 kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
 844                 kvmppc_account_exit(vcpu, ISI_EXITS);
 845                 r = RESUME_GUEST;
 846                 break;
 847
 848 #ifdef CONFIG_KVM_BOOKE_HV
 849         case BOOKE_INTERRUPT_HV_SYSCALL:
 850                 if (!(vcpu->arch.shared->msr & MSR_PR)) {
 851                         kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
 852                 } else {
 853                         /*
 854                          * hcall from guest userspace -- send privileged
 855                          * instruction program check.
 856                          */
 857                         kvmppc_core_queue_program(vcpu, ESR_PPR);
 858                 }
 859
 860                 r = RESUME_GUEST;
 861                 break;
 862 #else
 863         case BOOKE_INTERRUPT_SYSCALL:
 864                 if (!(vcpu->arch.shared->msr & MSR_PR) &&
 865                     (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
 866                         /* KVM PV hypercalls */
 867                         kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
 868                         r = RESUME_GUEST;
 869                 } else {
 870                         /* Guest syscalls */
 871                         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
 872                 }
 873                 kvmppc_account_exit(vcpu, SYSCALL_EXITS);
 874                 r = RESUME_GUEST;
 875                 break;
 876 #endif
 877
 878         case BOOKE_INTERRUPT_DTLB_MISS: {
 879                 unsigned long eaddr = vcpu->arch.fault_dear;
 880                 int gtlb_index;
 881                 gpa_t gpaddr;
 882                 gfn_t gfn;
 883
 884 #ifdef CONFIG_KVM_E500V2
 885                 if (!(vcpu->arch.shared->msr & MSR_PR) &&
 886                     (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
 887                         kvmppc_map_magic(vcpu);
 888                         kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
 889                         r = RESUME_GUEST;
 890
 891                         break;
 892                 }
 893 #endif
 894
 895                 /* Check the guest TLB. */
 896                 gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
 897                 if (gtlb_index < 0) {
 898                         /* The guest didn't have a mapping for it. */
 899                         kvmppc_core_queue_dtlb_miss(vcpu,
 900                                                     vcpu->arch.fault_dear,
 901                                                     vcpu->arch.fault_esr);
 902                         kvmppc_mmu_dtlb_miss(vcpu);
 903                         kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
 904                         r = RESUME_GUEST;
 905                         break;
 906                 }
 907
 908                 gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
 909                 gfn = gpaddr >> PAGE_SHIFT;
 910
 911                 if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
 912                         /* The guest TLB had a mapping, but the shadow TLB
 913                          * didn't, and it is RAM. This could be because:
 914                          * a) the entry is mapping the host kernel, or
 915                          * b) the guest used a large mapping which we're faking
 916                          * Either way, we need to satisfy the fault without
 917                          * invoking the guest. */
 918                         kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
 919                         kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
 920                         r = RESUME_GUEST;
 921                 } else {
 922                         /* Guest has mapped and accessed a page which is not
 923                          * actually RAM. */
 924                         vcpu->arch.paddr_accessed = gpaddr;
 925                         vcpu->arch.vaddr_accessed = eaddr;
 926                         r = kvmppc_emulate_mmio(run, vcpu);
 927                         kvmppc_account_exit(vcpu, MMIO_EXITS);
 928                 }
 929
 930                 break;
 931         }
 932
 933         case BOOKE_INTERRUPT_ITLB_MISS: {
 934                 unsigned long eaddr = vcpu->arch.pc;
 935                 gpa_t gpaddr;
 936                 gfn_t gfn;
 937                 int gtlb_index;
 938
 939                 r = RESUME_GUEST;
 940
 941                 /* Check the guest TLB. */
 942                 gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
 943                 if (gtlb_index < 0) {
 944                         /* The guest didn't have a mapping for it. */
 945                         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
 946                         kvmppc_mmu_itlb_miss(vcpu);
 947                         kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
 948                         break;
 949                 }
 950
 951                 kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
 952
 953                 gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
 954                 gfn = gpaddr >> PAGE_SHIFT;
 955
 956                 if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
 957                         /* The guest TLB had a mapping, but the shadow TLB
 958                          * didn't. This could be because:
 959                          * a) the entry is mapping the host kernel, or
 960                          * b) the guest used a large mapping which we're faking
 961                          * Either way, we need to satisfy the fault without
 962                          * invoking the guest. */
 963                         kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
 964                 } else {
 965                         /* Guest mapped and leaped at non-RAM! */
 966                         kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
 967                 }
 968
 969                 break;
 970         }
 971
 972         case BOOKE_INTERRUPT_DEBUG: {
 973                 u32 dbsr;
 974
 975                 vcpu->arch.pc = mfspr(SPRN_CSRR0);
 976
 977                 /* clear IAC events in DBSR register */
 978                 dbsr = mfspr(SPRN_DBSR);
 979                 dbsr &= DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4;
 980                 mtspr(SPRN_DBSR, dbsr);
 981
 982                 run->exit_reason = KVM_EXIT_DEBUG;
 983                 kvmppc_account_exit(vcpu, DEBUG_EXITS);
 984                 r = RESUME_HOST;
 985                 break;
 986         }
 987
 988         default:
 989                 printk(KERN_EMERG "exit_nr %d\n", exit_nr);
 990                 BUG();
 991         }
 992
 993         /*
 994          * To avoid clobbering exit_reason, only check for signals if we
 995          * aren't already exiting to userspace for some other reason.
 996          */
 997         if (!(r & RESUME_HOST)) {
 998                 local_irq_disable();
 999                 if (kvmppc_prepare_to_enter(vcpu)) {
1000                         run->exit_reason = KVM_EXIT_INTR;
1001                         r = (-EINTR << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
1002                         kvmppc_account_exit(vcpu, SIGNAL_EXITS);
1003                 }
1004         }
1005
1006         return r;
1007 }
1008
1009 /* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
1010 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1011 {
1012         int i;
1013         int r;
1014
1015         vcpu->arch.pc = 0;
1016         vcpu->arch.shared->pir = vcpu->vcpu_id;
1017         kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
1018         kvmppc_set_msr(vcpu, 0);
1019
1020 #ifndef CONFIG_KVM_BOOKE_HV
1021         vcpu->arch.shadow_msr = MSR_USER | MSR_DE | MSR_IS | MSR_DS;
1022         vcpu->arch.shadow_pid = 1;
1023         vcpu->arch.shared->msr = 0;
1024 #endif
1025
1026         /* Eye-catching numbers so we know if the guest takes an interrupt
1027          * before it's programmed its own IVPR/IVORs. */
1028         vcpu->arch.ivpr = 0x55550000;
1029         for (i = 0; i < BOOKE_IRQPRIO_MAX; i++)
1030                 vcpu->arch.ivor[i] = 0x7700 | i * 4;
1031
1032         kvmppc_init_timing_stats(vcpu);
1033
1034         r = kvmppc_core_vcpu_setup(vcpu);
1035         kvmppc_sanity_check(vcpu);
1036         return r;
1037 }
1038
1039 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1040 {
1041         int i;
1042
1043         regs->pc = vcpu->arch.pc;
1044         regs->cr = kvmppc_get_cr(vcpu);
1045         regs->ctr = vcpu->arch.ctr;
1046         regs->lr = vcpu->arch.lr;
1047         regs->xer = kvmppc_get_xer(vcpu);
1048         regs->msr = vcpu->arch.shared->msr;
1049         regs->srr0 = vcpu->arch.shared->srr0;
1050         regs->srr1 = vcpu->arch.shared->srr1;
1051         regs->pid = vcpu->arch.pid;
1052         regs->sprg0 = vcpu->arch.shared->sprg0;
1053         regs->sprg1 = vcpu->arch.shared->sprg1;
1054         regs->sprg2 = vcpu->arch.shared->sprg2;
1055         regs->sprg3 = vcpu->arch.shared->sprg3;
1056         regs->sprg4 = vcpu->arch.shared->sprg4;
1057         regs->sprg5 = vcpu->arch.shared->sprg5;
1058         regs->sprg6 = vcpu->arch.shared->sprg6;
1059         regs->sprg7 = vcpu->arch.shared->sprg7;
1060
1061         for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
1062                 regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
1063
1064         return 0;
1065 }
1066
1067 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1068 {
1069         int i;
1070
1071         vcpu->arch.pc = regs->pc;
1072         kvmppc_set_cr(vcpu, regs->cr);
1073         vcpu->arch.ctr = regs->ctr;
1074         vcpu->arch.lr = regs->lr;
1075         kvmppc_set_xer(vcpu, regs->xer);
1076         kvmppc_set_msr(vcpu, regs->msr);
1077         vcpu->arch.shared->srr0 = regs->srr0;
1078         vcpu->arch.shared->srr1 = regs->srr1;
1079         kvmppc_set_pid(vcpu, regs->pid);
1080         vcpu->arch.shared->sprg0 = regs->sprg0;
1081         vcpu->arch.shared->sprg1 = regs->sprg1;
1082         vcpu->arch.shared->sprg2 = regs->sprg2;
1083         vcpu->arch.shared->sprg3 = regs->sprg3;
1084         vcpu->arch.shared->sprg4 = regs->sprg4;
1085         vcpu->arch.shared->sprg5 = regs->sprg5;
1086         vcpu->arch.shared->sprg6 = regs->sprg6;
1087         vcpu->arch.shared->sprg7 = regs->sprg7;
1088
1089         for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
1090                 kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
1091
1092         return 0;
1093 }
1094
1095 static void get_sregs_base(struct kvm_vcpu *vcpu,
1096                            struct kvm_sregs *sregs)
1097 {
1098         u64 tb = get_tb();
1099
1100         sregs->u.e.features |= KVM_SREGS_E_BASE;
1101
1102         sregs->u.e.csrr0 = vcpu->arch.csrr0;
1103         sregs->u.e.csrr1 = vcpu->arch.csrr1;
1104         sregs->u.e.mcsr = vcpu->arch.mcsr;
1105         sregs->u.e.esr = get_guest_esr(vcpu);
1106         sregs->u.e.dear = get_guest_dear(vcpu);
1107         sregs->u.e.tsr = vcpu->arch.tsr;
1108         sregs->u.e.tcr = vcpu->arch.tcr;
1109         sregs->u.e.dec = kvmppc_get_dec(vcpu, tb);
1110         sregs->u.e.tb = tb;
1111         sregs->u.e.vrsave = vcpu->arch.vrsave;
1112 }
1113
1114 static int set_sregs_base(struct kvm_vcpu *vcpu,
1115                           struct kvm_sregs *sregs)
1116 {
1117         if (!(sregs->u.e.features & KVM_SREGS_E_BASE))
1118                 return 0;
1119
1120         vcpu->arch.csrr0 = sregs->u.e.csrr0;
1121         vcpu->arch.csrr1 = sregs->u.e.csrr1;
1122         vcpu->arch.mcsr = sregs->u.e.mcsr;
1123         set_guest_esr(vcpu, sregs->u.e.esr);
1124         set_guest_dear(vcpu, sregs->u.e.dear);
1125         vcpu->arch.vrsave = sregs->u.e.vrsave;
1126         kvmppc_set_tcr(vcpu, sregs->u.e.tcr);
1127
1128         if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_DEC) {
1129                 vcpu->arch.dec = sregs->u.e.dec;
1130                 kvmppc_emulate_dec(vcpu);
1131         }
1132
1133         if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_TSR) {
1134                 vcpu->arch.tsr = sregs->u.e.tsr;
1135                 update_timer_ints(vcpu);
1136         }
1137
1138         return 0;
1139 }
1140
1141 static void get_sregs_arch206(struct kvm_vcpu *vcpu,
1142                               struct kvm_sregs *sregs)
1143 {
1144         sregs->u.e.features |= KVM_SREGS_E_ARCH206;
1145
1146         sregs->u.e.pir = vcpu->vcpu_id;
1147         sregs->u.e.mcsrr0 = vcpu->arch.mcsrr0;
1148         sregs->u.e.mcsrr1 = vcpu->arch.mcsrr1;
1149         sregs->u.e.decar = vcpu->arch.decar;
1150         sregs->u.e.ivpr = vcpu->arch.ivpr;
1151 }
1152
1153 static int set_sregs_arch206(struct kvm_vcpu *vcpu,
1154                              struct kvm_sregs *sregs)
1155 {
1156         if (!(sregs->u.e.features & KVM_SREGS_E_ARCH206))
1157                 return 0;
1158
1159         if (sregs->u.e.pir != vcpu->vcpu_id)
1160                 return -EINVAL;
1161
1162         vcpu->arch.mcsrr0 = sregs->u.e.mcsrr0;
1163         vcpu->arch.mcsrr1 = sregs->u.e.mcsrr1;
1164         vcpu->arch.decar = sregs->u.e.decar;
1165         vcpu->arch.ivpr = sregs->u.e.ivpr;
1166
1167         return 0;
1168 }
1169
1170 void kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
1171 {
1172         sregs->u.e.features |= KVM_SREGS_E_IVOR;
1173
1174         sregs->u.e.ivor_low[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL];
1175         sregs->u.e.ivor_low[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK];
1176         sregs->u.e.ivor_low[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE];
1177         sregs->u.e.ivor_low[3] = vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE];
1178         sregs->u.e.ivor_low[4] = vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL];
1179         sregs->u.e.ivor_low[5] = vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT];
1180         sregs->u.e.ivor_low[6] = vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM];
1181         sregs->u.e.ivor_low[7] = vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL];
1182         sregs->u.e.ivor_low[8] = vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL];
1183         sregs->u.e.ivor_low[9] = vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL];
1184         sregs->u.e.ivor_low[10] = vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER];
1185         sregs->u.e.ivor_low[11] = vcpu->arch.ivor[BOOKE_IRQPRIO_FIT];
1186         sregs->u.e.ivor_low[12] = vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG];
1187         sregs->u.e.ivor_low[13] = vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS];
1188         sregs->u.e.ivor_low[14] = vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS];
1189         sregs->u.e.ivor_low[15] = vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG];
1190 }
1191
1192 int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
1193 {
1194         if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
1195                 return 0;
1196
1197         vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL] = sregs->u.e.ivor_low[0];
1198         vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK] = sregs->u.e.ivor_low[1];
1199         vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE] = sregs->u.e.ivor_low[2];
1200         vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE] = sregs->u.e.ivor_low[3];
1201         vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL] = sregs->u.e.ivor_low[4];
1202         vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT] = sregs->u.e.ivor_low[5];
1203         vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM] = sregs->u.e.ivor_low[6];
1204         vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL] = sregs->u.e.ivor_low[7];
1205         vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL] = sregs->u.e.ivor_low[8];
1206         vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL] = sregs->u.e.ivor_low[9];
1207         vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER] = sregs->u.e.ivor_low[10];
1208         vcpu->arch.ivor[BOOKE_IRQPRIO_FIT] = sregs->u.e.ivor_low[11];
1209         vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG] = sregs->u.e.ivor_low[12];
1210         vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS] = sregs->u.e.ivor_low[13];
1211         vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS] = sregs->u.e.ivor_low[14];
1212         vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG] = sregs->u.e.ivor_low[15];
1213
1214         return 0;
1215 }
1216
1217 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1218                                   struct kvm_sregs *sregs)
1219 {
1220         sregs->pvr = vcpu->arch.pvr;
1221
1222         get_sregs_base(vcpu, sregs);
1223         get_sregs_arch206(vcpu, sregs);
1224         kvmppc_core_get_sregs(vcpu, sregs);
1225         return 0;
1226 }
1227
1228 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1229                                   struct kvm_sregs *sregs)
1230 {
1231         int ret;
1232
1233         if (vcpu->arch.pvr != sregs->pvr)
1234                 return -EINVAL;
1235
1236         ret = set_sregs_base(vcpu, sregs);
1237         if (ret < 0)
1238                 return ret;
1239
1240         ret = set_sregs_arch206(vcpu, sregs);
1241         if (ret < 0)
1242                 return ret;
1243
1244         return kvmppc_core_set_sregs(vcpu, sregs);
1245 }
1246
1247 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1248 {
1249         return -EINVAL;
1250 }
1251
1252 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1253 {
1254         return -EINVAL;
1255 }
1256
1257 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1258 {
1259         return -ENOTSUPP;
1260 }
1261
1262 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1263 {
1264         return -ENOTSUPP;
1265 }
1266
1267 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1268                                   struct kvm_translation *tr)
1269 {
1270         int r;
1271
1272         r = kvmppc_core_vcpu_translate(vcpu, tr);
1273         return r;
1274 }
1275
1276 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1277 {
1278         return -ENOTSUPP;
1279 }
1280
1281 int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1282                                       struct kvm_userspace_memory_region *mem)
1283 {
1284         return 0;
1285 }
1286
1287 void kvmppc_core_commit_memory_region(struct kvm *kvm,
1288                                 struct kvm_userspace_memory_region *mem)
1289 {
1290 }
1291
1292 void kvmppc_set_tcr(struct kvm_vcpu *vcpu, u32 new_tcr)
1293 {
1294         vcpu->arch.tcr = new_tcr;
1295         update_timer_ints(vcpu);
1296 }
1297
1298 void kvmppc_set_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
1299 {
1300         set_bits(tsr_bits, &vcpu->arch.tsr);
1301         smp_wmb();
1302         kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1303         kvm_vcpu_kick(vcpu);
1304 }
1305
1306 void kvmppc_clr_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
1307 {
1308         clear_bits(tsr_bits, &vcpu->arch.tsr);
1309         update_timer_ints(vcpu);
1310 }
1311
1312 void kvmppc_decrementer_func(unsigned long data)
1313 {
1314         struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data;
1315
1316         if (vcpu->arch.tcr & TCR_ARE) {
1317                 vcpu->arch.dec = vcpu->arch.decar;
1318                 kvmppc_emulate_dec(vcpu);
1319         }
1320
1321         kvmppc_set_tsr_bits(vcpu, TSR_DIS);
1322 }
1323
1324 void kvmppc_booke_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1325 {
1326         current->thread.kvm_vcpu = vcpu;
1327 }
1328
1329 void kvmppc_booke_vcpu_put(struct kvm_vcpu *vcpu)
1330 {
1331         current->thread.kvm_vcpu = NULL;
1332 }
1333
1334 int __init kvmppc_booke_init(void)
1335 {
1336 #ifndef CONFIG_KVM_BOOKE_HV
1337         unsigned long ivor[16];
1338         unsigned long max_ivor = 0;
1339         int i;
1340
1341         /* We install our own exception handlers by hijacking IVPR. IVPR must
1342          * be 16-bit aligned, so we need a 64KB allocation. */
1343         kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
1344                                                  VCPU_SIZE_ORDER);
1345         if (!kvmppc_booke_handlers)
1346                 return -ENOMEM;
1347
1348         /* XXX make sure our handlers are smaller than Linux's */
1349
1350         /* Copy our interrupt handlers to match host IVORs. That way we don't
1351          * have to swap the IVORs on every guest/host transition. */
1352         ivor[0] = mfspr(SPRN_IVOR0);
1353         ivor[1] = mfspr(SPRN_IVOR1);
1354         ivor[2] = mfspr(SPRN_IVOR2);
1355         ivor[3] = mfspr(SPRN_IVOR3);
1356         ivor[4] = mfspr(SPRN_IVOR4);
1357         ivor[5] = mfspr(SPRN_IVOR5);
1358         ivor[6] = mfspr(SPRN_IVOR6);
1359         ivor[7] = mfspr(SPRN_IVOR7);
1360         ivor[8] = mfspr(SPRN_IVOR8);
1361         ivor[9] = mfspr(SPRN_IVOR9);
1362         ivor[10] = mfspr(SPRN_IVOR10);
1363         ivor[11] = mfspr(SPRN_IVOR11);
1364         ivor[12] = mfspr(SPRN_IVOR12);
1365         ivor[13] = mfspr(SPRN_IVOR13);
1366         ivor[14] = mfspr(SPRN_IVOR14);
1367         ivor[15] = mfspr(SPRN_IVOR15);
1368
1369         for (i = 0; i < 16; i++) {
1370                 if (ivor[i] > max_ivor)
1371                         max_ivor = ivor[i];
1372
1373                 memcpy((void *)kvmppc_booke_handlers + ivor[i],
1374                        kvmppc_handlers_start + i * kvmppc_handler_len,
1375                        kvmppc_handler_len);
1376         }
1377         flush_icache_range(kvmppc_booke_handlers,
1378                            kvmppc_booke_handlers + max_ivor + kvmppc_handler_len);
1379 #endif /* !BOOKE_HV */
1380         return 0;
1381 }
1382
1383 void __exit kvmppc_booke_exit(void)
1384 {
1385         free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
1386         kvm_exit();
1387 }