Pileus Git - ~andy/linux/blob - drivers/misc/vmw_vmci/vmci_guest.c

   1 /*
   2  * VMware VMCI Driver
   3  *
   4  * Copyright (C) 2012 VMware, Inc. All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the
   8  * Free Software Foundation version 2 and no later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * for more details.
  14  */
  15
  16 #include <linux/vmw_vmci_defs.h>
  17 #include <linux/vmw_vmci_api.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/highmem.h>
  21 #include <linux/kernel.h>
  22 #include <linux/mm.h>
  23 #include <linux/module.h>
  24 #include <linux/sched.h>
  25 #include <linux/slab.h>
  26 #include <linux/init.h>
  27 #include <linux/pci.h>
  28 #include <linux/smp.h>
  29 #include <linux/io.h>
  30 #include <linux/vmalloc.h>
  31
  32 #include "vmci_datagram.h"
  33 #include "vmci_doorbell.h"
  34 #include "vmci_context.h"
  35 #include "vmci_driver.h"
  36 #include "vmci_event.h"
  37
  38 #define PCI_VENDOR_ID_VMWARE            0x15AD
  39 #define PCI_DEVICE_ID_VMWARE_VMCI       0x0740
  40
  41 #define VMCI_UTIL_NUM_RESOURCES 1
  42
  43 static bool vmci_disable_msi;
  44 module_param_named(disable_msi, vmci_disable_msi, bool, 0);
  45 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)");
  46
  47 static bool vmci_disable_msix;
  48 module_param_named(disable_msix, vmci_disable_msix, bool, 0);
  49 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)");
  50
  51 static u32 ctx_update_sub_id = VMCI_INVALID_ID;
  52 static u32 vm_context_id = VMCI_INVALID_ID;
  53
  54 struct vmci_guest_device {
  55         struct device *dev;     /* PCI device we are attached to */
  56         void __iomem *iobase;
  57
  58         unsigned int irq;
  59         unsigned int intr_type;
  60         bool exclusive_vectors;
  61         struct msix_entry msix_entries[VMCI_MAX_INTRS];
  62
  63         struct tasklet_struct datagram_tasklet;
  64         struct tasklet_struct bm_tasklet;
  65
  66         void *data_buffer;
  67         void *notification_bitmap;
  68 };
  69
  70 /* vmci_dev singleton device and supporting data*/
  71 static struct vmci_guest_device *vmci_dev_g;
  72 static DEFINE_SPINLOCK(vmci_dev_spinlock);
  73
  74 static atomic_t vmci_num_guest_devices = ATOMIC_INIT(0);
  75
  76 bool vmci_guest_code_active(void)
  77 {
  78         return atomic_read(&vmci_num_guest_devices) != 0;
  79 }
  80
  81 u32 vmci_get_vm_context_id(void)
  82 {
  83         if (vm_context_id == VMCI_INVALID_ID) {
  84                 int result;
  85                 struct vmci_datagram get_cid_msg;
  86                 get_cid_msg.dst =
  87                     vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
  88                                      VMCI_GET_CONTEXT_ID);
  89                 get_cid_msg.src = VMCI_ANON_SRC_HANDLE;
  90                 get_cid_msg.payload_size = 0;
  91                 result = vmci_send_datagram(&get_cid_msg);
  92                 if (result >= 0)
  93                         vm_context_id = result;
  94         }
  95         return vm_context_id;
  96 }
  97
  98 /*
  99  * VM to hypervisor call mechanism. We use the standard VMware naming
 100  * convention since shared code is calling this function as well.
 101  */
 102 int vmci_send_datagram(struct vmci_datagram *dg)
 103 {
 104         unsigned long flags;
 105         int result;
 106
 107         /* Check args. */
 108         if (dg == NULL)
 109                 return VMCI_ERROR_INVALID_ARGS;
 110
 111         /*
 112          * Need to acquire spinlock on the device because the datagram
 113          * data may be spread over multiple pages and the monitor may
 114          * interleave device user rpc calls from multiple
 115          * VCPUs. Acquiring the spinlock precludes that
 116          * possibility. Disabling interrupts to avoid incoming
 117          * datagrams during a "rep out" and possibly landing up in
 118          * this function.
 119          */
 120         spin_lock_irqsave(&vmci_dev_spinlock, flags);
 121
 122         if (vmci_dev_g) {
 123                 iowrite8_rep(vmci_dev_g->iobase + VMCI_DATA_OUT_ADDR,
 124                              dg, VMCI_DG_SIZE(dg));
 125                 result = ioread32(vmci_dev_g->iobase + VMCI_RESULT_LOW_ADDR);
 126         } else {
 127                 result = VMCI_ERROR_UNAVAILABLE;
 128         }
 129
 130         spin_unlock_irqrestore(&vmci_dev_spinlock, flags);
 131
 132         return result;
 133 }
 134 EXPORT_SYMBOL_GPL(vmci_send_datagram);
 135
 136 /*
 137  * Gets called with the new context id if updated or resumed.
 138  * Context id.
 139  */
 140 static void vmci_guest_cid_update(u32 sub_id,
 141                                   const struct vmci_event_data *event_data,
 142                                   void *client_data)
 143 {
 144         const struct vmci_event_payld_ctx *ev_payload =
 145                                 vmci_event_data_const_payload(event_data);
 146
 147         if (sub_id != ctx_update_sub_id) {
 148                 pr_devel("Invalid subscriber (ID=0x%x)\n", sub_id);
 149                 return;
 150         }
 151
 152         if (!event_data || ev_payload->context_id == VMCI_INVALID_ID) {
 153                 pr_devel("Invalid event data\n");
 154                 return;
 155         }
 156
 157         pr_devel("Updating context from (ID=0x%x) to (ID=0x%x) on event (type=%d)\n",
 158                  vm_context_id, ev_payload->context_id, event_data->event);
 159
 160         vm_context_id = ev_payload->context_id;
 161 }
 162
 163 /*
 164  * Verify that the host supports the hypercalls we need. If it does not,
 165  * try to find fallback hypercalls and use those instead.  Returns
 166  * true if required hypercalls (or fallback hypercalls) are
 167  * supported by the host, false otherwise.
 168  */
 169 static bool vmci_check_host_caps(struct pci_dev *pdev)
 170 {
 171         bool result;
 172         struct vmci_resource_query_msg *msg;
 173         u32 msg_size = sizeof(struct vmci_resource_query_hdr) +
 174                                 VMCI_UTIL_NUM_RESOURCES * sizeof(u32);
 175         struct vmci_datagram *check_msg;
 176
 177         check_msg = kmalloc(msg_size, GFP_KERNEL);
 178         if (!check_msg) {
 179                 dev_err(&pdev->dev, "%s: Insufficient memory\n", __func__);
 180                 return false;
 181         }
 182
 183         check_msg->dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 184                                           VMCI_RESOURCES_QUERY);
 185         check_msg->src = VMCI_ANON_SRC_HANDLE;
 186         check_msg->payload_size = msg_size - VMCI_DG_HEADERSIZE;
 187         msg = (struct vmci_resource_query_msg *)VMCI_DG_PAYLOAD(check_msg);
 188
 189         msg->num_resources = VMCI_UTIL_NUM_RESOURCES;
 190         msg->resources[0] = VMCI_GET_CONTEXT_ID;
 191
 192         /* Checks that hyper calls are supported */
 193         result = vmci_send_datagram(check_msg) == 0x01;
 194         kfree(check_msg);
 195
 196         dev_dbg(&pdev->dev, "%s: Host capability check: %s\n",
 197                 __func__, result ? "PASSED" : "FAILED");
 198
 199         /* We need the vector. There are no fallbacks. */
 200         return result;
 201 }
 202
 203 /*
 204  * Reads datagrams from the data in port and dispatches them. We
 205  * always start reading datagrams into only the first page of the
 206  * datagram buffer. If the datagrams don't fit into one page, we
 207  * use the maximum datagram buffer size for the remainder of the
 208  * invocation. This is a simple heuristic for not penalizing
 209  * small datagrams.
 210  *
 211  * This function assumes that it has exclusive access to the data
 212  * in port for the duration of the call.
 213  */
 214 static void vmci_dispatch_dgs(unsigned long data)
 215 {
 216         struct vmci_guest_device *vmci_dev = (struct vmci_guest_device *)data;
 217         u8 *dg_in_buffer = vmci_dev->data_buffer;
 218         struct vmci_datagram *dg;
 219         size_t dg_in_buffer_size = VMCI_MAX_DG_SIZE;
 220         size_t current_dg_in_buffer_size = PAGE_SIZE;
 221         size_t remaining_bytes;
 222
 223         BUILD_BUG_ON(VMCI_MAX_DG_SIZE < PAGE_SIZE);
 224
 225         ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR,
 226                     vmci_dev->data_buffer, current_dg_in_buffer_size);
 227         dg = (struct vmci_datagram *)dg_in_buffer;
 228         remaining_bytes = current_dg_in_buffer_size;
 229
 230         while (dg->dst.resource != VMCI_INVALID_ID ||
 231                remaining_bytes > PAGE_SIZE) {
 232                 unsigned dg_in_size;
 233
 234                 /*
 235                  * When the input buffer spans multiple pages, a datagram can
 236                  * start on any page boundary in the buffer.
 237                  */
 238                 if (dg->dst.resource == VMCI_INVALID_ID) {
 239                         dg = (struct vmci_datagram *)roundup(
 240                                 (uintptr_t)dg + 1, PAGE_SIZE);
 241                         remaining_bytes =
 242                                 (size_t)(dg_in_buffer +
 243                                          current_dg_in_buffer_size -
 244                                          (u8 *)dg);
 245                         continue;
 246                 }
 247
 248                 dg_in_size = VMCI_DG_SIZE_ALIGNED(dg);
 249
 250                 if (dg_in_size <= dg_in_buffer_size) {
 251                         int result;
 252
 253                         /*
 254                          * If the remaining bytes in the datagram
 255                          * buffer doesn't contain the complete
 256                          * datagram, we first make sure we have enough
 257                          * room for it and then we read the reminder
 258                          * of the datagram and possibly any following
 259                          * datagrams.
 260                          */
 261                         if (dg_in_size > remaining_bytes) {
 262                                 if (remaining_bytes !=
 263                                     current_dg_in_buffer_size) {
 264
 265                                         /*
 266                                          * We move the partial
 267                                          * datagram to the front and
 268                                          * read the reminder of the
 269                                          * datagram and possibly
 270                                          * following calls into the
 271                                          * following bytes.
 272                                          */
 273                                         memmove(dg_in_buffer, dg_in_buffer +
 274                                                 current_dg_in_buffer_size -
 275                                                 remaining_bytes,
 276                                                 remaining_bytes);
 277                                         dg = (struct vmci_datagram *)
 278                                             dg_in_buffer;
 279                                 }
 280
 281                                 if (current_dg_in_buffer_size !=
 282                                     dg_in_buffer_size)
 283                                         current_dg_in_buffer_size =
 284                                             dg_in_buffer_size;
 285
 286                                 ioread8_rep(vmci_dev->iobase +
 287                                                 VMCI_DATA_IN_ADDR,
 288                                         vmci_dev->data_buffer +
 289                                                 remaining_bytes,
 290                                         current_dg_in_buffer_size -
 291                                                 remaining_bytes);
 292                         }
 293
 294                         /*
 295                          * We special case event datagrams from the
 296                          * hypervisor.
 297                          */
 298                         if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID &&
 299                             dg->dst.resource == VMCI_EVENT_HANDLER) {
 300                                 result = vmci_event_dispatch(dg);
 301                         } else {
 302                                 result = vmci_datagram_invoke_guest_handler(dg);
 303                         }
 304                         if (result < VMCI_SUCCESS)
 305                                 dev_dbg(vmci_dev->dev,
 306                                         "Datagram with resource (ID=0x%x) failed (err=%d)\n",
 307                                          dg->dst.resource, result);
 308
 309                         /* On to the next datagram. */
 310                         dg = (struct vmci_datagram *)((u8 *)dg +
 311                                                       dg_in_size);
 312                 } else {
 313                         size_t bytes_to_skip;
 314
 315                         /*
 316                          * Datagram doesn't fit in datagram buffer of maximal
 317                          * size. We drop it.
 318                          */
 319                         dev_dbg(vmci_dev->dev,
 320                                 "Failed to receive datagram (size=%u bytes)\n",
 321                                  dg_in_size);
 322
 323                         bytes_to_skip = dg_in_size - remaining_bytes;
 324                         if (current_dg_in_buffer_size != dg_in_buffer_size)
 325                                 current_dg_in_buffer_size = dg_in_buffer_size;
 326
 327                         for (;;) {
 328                                 ioread8_rep(vmci_dev->iobase +
 329                                                 VMCI_DATA_IN_ADDR,
 330                                         vmci_dev->data_buffer,
 331                                         current_dg_in_buffer_size);
 332                                 if (bytes_to_skip <= current_dg_in_buffer_size)
 333                                         break;
 334
 335                                 bytes_to_skip -= current_dg_in_buffer_size;
 336                         }
 337                         dg = (struct vmci_datagram *)(dg_in_buffer +
 338                                                       bytes_to_skip);
 339                 }
 340
 341                 remaining_bytes =
 342                     (size_t) (dg_in_buffer + current_dg_in_buffer_size -
 343                               (u8 *)dg);
 344
 345                 if (remaining_bytes < VMCI_DG_HEADERSIZE) {
 346                         /* Get the next batch of datagrams. */
 347
 348                         ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR,
 349                                     vmci_dev->data_buffer,
 350                                     current_dg_in_buffer_size);
 351                         dg = (struct vmci_datagram *)dg_in_buffer;
 352                         remaining_bytes = current_dg_in_buffer_size;
 353                 }
 354         }
 355 }
 356
 357 /*
 358  * Scans the notification bitmap for raised flags, clears them
 359  * and handles the notifications.
 360  */
 361 static void vmci_process_bitmap(unsigned long data)
 362 {
 363         struct vmci_guest_device *dev = (struct vmci_guest_device *)data;
 364
 365         if (!dev->notification_bitmap) {
 366                 dev_dbg(dev->dev, "No bitmap present in %s\n", __func__);
 367                 return;
 368         }
 369
 370         vmci_dbell_scan_notification_entries(dev->notification_bitmap);
 371 }
 372
 373 /*
 374  * Enable MSI-X.  Try exclusive vectors first, then shared vectors.
 375  */
 376 static int vmci_enable_msix(struct pci_dev *pdev,
 377                             struct vmci_guest_device *vmci_dev)
 378 {
 379         int i;
 380         int result;
 381
 382         for (i = 0; i < VMCI_MAX_INTRS; ++i) {
 383                 vmci_dev->msix_entries[i].entry = i;
 384                 vmci_dev->msix_entries[i].vector = i;
 385         }
 386
 387         result = pci_enable_msix(pdev, vmci_dev->msix_entries, VMCI_MAX_INTRS);
 388         if (result == 0)
 389                 vmci_dev->exclusive_vectors = true;
 390         else if (result > 0)
 391                 result = pci_enable_msix(pdev, vmci_dev->msix_entries, 1);
 392
 393         return result;
 394 }
 395
 396 /*
 397  * Interrupt handler for legacy or MSI interrupt, or for first MSI-X
 398  * interrupt (vector VMCI_INTR_DATAGRAM).
 399  */
 400 static irqreturn_t vmci_interrupt(int irq, void *_dev)
 401 {
 402         struct vmci_guest_device *dev = _dev;
 403
 404         /*
 405          * If we are using MSI-X with exclusive vectors then we simply schedule
 406          * the datagram tasklet, since we know the interrupt was meant for us.
 407          * Otherwise we must read the ICR to determine what to do.
 408          */
 409
 410         if (dev->intr_type == VMCI_INTR_TYPE_MSIX && dev->exclusive_vectors) {
 411                 tasklet_schedule(&dev->datagram_tasklet);
 412         } else {
 413                 unsigned int icr;
 414
 415                 /* Acknowledge interrupt and determine what needs doing. */
 416                 icr = ioread32(dev->iobase + VMCI_ICR_ADDR);
 417                 if (icr == 0 || icr == ~0)
 418                         return IRQ_NONE;
 419
 420                 if (icr & VMCI_ICR_DATAGRAM) {
 421                         tasklet_schedule(&dev->datagram_tasklet);
 422                         icr &= ~VMCI_ICR_DATAGRAM;
 423                 }
 424
 425                 if (icr & VMCI_ICR_NOTIFICATION) {
 426                         tasklet_schedule(&dev->bm_tasklet);
 427                         icr &= ~VMCI_ICR_NOTIFICATION;
 428                 }
 429
 430                 if (icr != 0)
 431                         dev_warn(dev->dev,
 432                                  "Ignoring unknown interrupt cause (%d)\n",
 433                                  icr);
 434         }
 435
 436         return IRQ_HANDLED;
 437 }
 438
 439 /*
 440  * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
 441  * which is for the notification bitmap.  Will only get called if we are
 442  * using MSI-X with exclusive vectors.
 443  */
 444 static irqreturn_t vmci_interrupt_bm(int irq, void *_dev)
 445 {
 446         struct vmci_guest_device *dev = _dev;
 447
 448         /* For MSI-X we can just assume it was meant for us. */
 449         tasklet_schedule(&dev->bm_tasklet);
 450
 451         return IRQ_HANDLED;
 452 }
 453
 454 /*
 455  * Most of the initialization at module load time is done here.
 456  */
 457 static int vmci_guest_probe_device(struct pci_dev *pdev,
 458                                    const struct pci_device_id *id)
 459 {
 460         struct vmci_guest_device *vmci_dev;
 461         void __iomem *iobase;
 462         unsigned int capabilities;
 463         unsigned long cmd;
 464         int vmci_err;
 465         int error;
 466
 467         dev_dbg(&pdev->dev, "Probing for vmci/PCI guest device\n");
 468
 469         error = pcim_enable_device(pdev);
 470         if (error) {
 471                 dev_err(&pdev->dev,
 472                         "Failed to enable VMCI device: %d\n", error);
 473                 return error;
 474         }
 475
 476         error = pcim_iomap_regions(pdev, 1 << 0, KBUILD_MODNAME);
 477         if (error) {
 478                 dev_err(&pdev->dev, "Failed to reserve/map IO regions\n");
 479                 return error;
 480         }
 481
 482         iobase = pcim_iomap_table(pdev)[0];
 483
 484         dev_info(&pdev->dev, "Found VMCI PCI device at %#lx, irq %u\n",
 485                  (unsigned long)iobase, pdev->irq);
 486
 487         vmci_dev = devm_kzalloc(&pdev->dev, sizeof(*vmci_dev), GFP_KERNEL);
 488         if (!vmci_dev) {
 489                 dev_err(&pdev->dev,
 490                         "Can't allocate memory for VMCI device\n");
 491                 return -ENOMEM;
 492         }
 493
 494         vmci_dev->dev = &pdev->dev;
 495         vmci_dev->intr_type = VMCI_INTR_TYPE_INTX;
 496         vmci_dev->exclusive_vectors = false;
 497         vmci_dev->iobase = iobase;
 498
 499         tasklet_init(&vmci_dev->datagram_tasklet,
 500                      vmci_dispatch_dgs, (unsigned long)vmci_dev);
 501         tasklet_init(&vmci_dev->bm_tasklet,
 502                      vmci_process_bitmap, (unsigned long)vmci_dev);
 503
 504         vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE);
 505         if (!vmci_dev->data_buffer) {
 506                 dev_err(&pdev->dev,
 507                         "Can't allocate memory for datagram buffer\n");
 508                 return -ENOMEM;
 509         }
 510
 511         pci_set_master(pdev);   /* To enable queue_pair functionality. */
 512
 513         /*
 514          * Verify that the VMCI Device supports the capabilities that
 515          * we need. If the device is missing capabilities that we would
 516          * like to use, check for fallback capabilities and use those
 517          * instead (so we can run a new VM on old hosts). Fail the load if
 518          * a required capability is missing and there is no fallback.
 519          *
 520          * Right now, we need datagrams. There are no fallbacks.
 521          */
 522         capabilities = ioread32(vmci_dev->iobase + VMCI_CAPS_ADDR);
 523         if (!(capabilities & VMCI_CAPS_DATAGRAM)) {
 524                 dev_err(&pdev->dev, "Device does not support datagrams\n");
 525                 error = -ENXIO;
 526                 goto err_free_data_buffer;
 527         }
 528
 529         /*
 530          * If the hardware supports notifications, we will use that as
 531          * well.
 532          */
 533         if (capabilities & VMCI_CAPS_NOTIFICATIONS) {
 534                 vmci_dev->notification_bitmap = vmalloc(PAGE_SIZE);
 535                 if (!vmci_dev->notification_bitmap) {
 536                         dev_warn(&pdev->dev,
 537                                  "Unable to allocate notification bitmap\n");
 538                 } else {
 539                         memset(vmci_dev->notification_bitmap, 0, PAGE_SIZE);
 540                         capabilities |= VMCI_CAPS_NOTIFICATIONS;
 541                 }
 542         }
 543
 544         dev_info(&pdev->dev, "Using capabilities 0x%x\n", capabilities);
 545
 546         /* Let the host know which capabilities we intend to use. */
 547         iowrite32(capabilities, vmci_dev->iobase + VMCI_CAPS_ADDR);
 548
 549         /* Set up global device so that we can start sending datagrams */
 550         spin_lock_irq(&vmci_dev_spinlock);
 551         vmci_dev_g = vmci_dev;
 552         spin_unlock_irq(&vmci_dev_spinlock);
 553
 554         /*
 555          * Register notification bitmap with device if that capability is
 556          * used.
 557          */
 558         if (capabilities & VMCI_CAPS_NOTIFICATIONS) {
 559                 struct page *page =
 560                         vmalloc_to_page(vmci_dev->notification_bitmap);
 561                 unsigned long bitmap_ppn = page_to_pfn(page);
 562                 if (!vmci_dbell_register_notification_bitmap(bitmap_ppn)) {
 563                         dev_warn(&pdev->dev,
 564                                  "VMCI device unable to register notification bitmap with PPN 0x%x\n",
 565                                  (u32) bitmap_ppn);
 566                         goto err_remove_vmci_dev_g;
 567                 }
 568         }
 569
 570         /* Check host capabilities. */
 571         if (!vmci_check_host_caps(pdev))
 572                 goto err_remove_bitmap;
 573
 574         /* Enable device. */
 575
 576         /*
 577          * We subscribe to the VMCI_EVENT_CTX_ID_UPDATE here so we can
 578          * update the internal context id when needed.
 579          */
 580         vmci_err = vmci_event_subscribe(VMCI_EVENT_CTX_ID_UPDATE,
 581                                         vmci_guest_cid_update, NULL,
 582                                         &ctx_update_sub_id);
 583         if (vmci_err < VMCI_SUCCESS)
 584                 dev_warn(&pdev->dev,
 585                          "Failed to subscribe to event (type=%d): %d\n",
 586                          VMCI_EVENT_CTX_ID_UPDATE, vmci_err);
 587
 588         /*
 589          * Enable interrupts.  Try MSI-X first, then MSI, and then fallback on
 590          * legacy interrupts.
 591          */
 592         if (!vmci_disable_msix && !vmci_enable_msix(pdev, vmci_dev)) {
 593                 vmci_dev->intr_type = VMCI_INTR_TYPE_MSIX;
 594                 vmci_dev->irq = vmci_dev->msix_entries[0].vector;
 595         } else if (!vmci_disable_msi && !pci_enable_msi(pdev)) {
 596                 vmci_dev->intr_type = VMCI_INTR_TYPE_MSI;
 597                 vmci_dev->irq = pdev->irq;
 598         } else {
 599                 vmci_dev->intr_type = VMCI_INTR_TYPE_INTX;
 600                 vmci_dev->irq = pdev->irq;
 601         }
 602
 603         /*
 604          * Request IRQ for legacy or MSI interrupts, or for first
 605          * MSI-X vector.
 606          */
 607         error = request_irq(vmci_dev->irq, vmci_interrupt, IRQF_SHARED,
 608                             KBUILD_MODNAME, vmci_dev);
 609         if (error) {
 610                 dev_err(&pdev->dev, "Irq %u in use: %d\n",
 611                         vmci_dev->irq, error);
 612                 goto err_disable_msi;
 613         }
 614
 615         /*
 616          * For MSI-X with exclusive vectors we need to request an
 617          * interrupt for each vector so that we get a separate
 618          * interrupt handler routine.  This allows us to distinguish
 619          * between the vectors.
 620          */
 621         if (vmci_dev->exclusive_vectors) {
 622                 error = request_irq(vmci_dev->msix_entries[1].vector,
 623                                     vmci_interrupt_bm, 0, KBUILD_MODNAME,
 624                                     vmci_dev);
 625                 if (error) {
 626                         dev_err(&pdev->dev,
 627                                 "Failed to allocate irq %u: %d\n",
 628                                 vmci_dev->msix_entries[1].vector, error);
 629                         goto err_free_irq;
 630                 }
 631         }
 632
 633         dev_dbg(&pdev->dev, "Registered device\n");
 634
 635         atomic_inc(&vmci_num_guest_devices);
 636
 637         /* Enable specific interrupt bits. */
 638         cmd = VMCI_IMR_DATAGRAM;
 639         if (capabilities & VMCI_CAPS_NOTIFICATIONS)
 640                 cmd |= VMCI_IMR_NOTIFICATION;
 641         iowrite32(cmd, vmci_dev->iobase + VMCI_IMR_ADDR);
 642
 643         /* Enable interrupts. */
 644         iowrite32(VMCI_CONTROL_INT_ENABLE,
 645                   vmci_dev->iobase + VMCI_CONTROL_ADDR);
 646
 647         pci_set_drvdata(pdev, vmci_dev);
 648         return 0;
 649
 650 err_free_irq:
 651         free_irq(vmci_dev->irq, &vmci_dev);
 652         tasklet_kill(&vmci_dev->datagram_tasklet);
 653         tasklet_kill(&vmci_dev->bm_tasklet);
 654
 655 err_disable_msi:
 656         if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX)
 657                 pci_disable_msix(pdev);
 658         else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI)
 659                 pci_disable_msi(pdev);
 660
 661         vmci_err = vmci_event_unsubscribe(ctx_update_sub_id);
 662         if (vmci_err < VMCI_SUCCESS)
 663                 dev_warn(&pdev->dev,
 664                          "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n",
 665                          VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err);
 666
 667 err_remove_bitmap:
 668         if (vmci_dev->notification_bitmap) {
 669                 iowrite32(VMCI_CONTROL_RESET,
 670                           vmci_dev->iobase + VMCI_CONTROL_ADDR);
 671                 vfree(vmci_dev->notification_bitmap);
 672         }
 673
 674 err_remove_vmci_dev_g:
 675         spin_lock_irq(&vmci_dev_spinlock);
 676         vmci_dev_g = NULL;
 677         spin_unlock_irq(&vmci_dev_spinlock);
 678
 679 err_free_data_buffer:
 680         vfree(vmci_dev->data_buffer);
 681
 682         /* The rest are managed resources and will be freed by PCI core */
 683         return error;
 684 }
 685
 686 static void vmci_guest_remove_device(struct pci_dev *pdev)
 687 {
 688         struct vmci_guest_device *vmci_dev = pci_get_drvdata(pdev);
 689         int vmci_err;
 690
 691         dev_dbg(&pdev->dev, "Removing device\n");
 692
 693         atomic_dec(&vmci_num_guest_devices);
 694
 695         vmci_qp_guest_endpoints_exit();
 696
 697         vmci_err = vmci_event_unsubscribe(ctx_update_sub_id);
 698         if (vmci_err < VMCI_SUCCESS)
 699                 dev_warn(&pdev->dev,
 700                          "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n",
 701                          VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err);
 702
 703         spin_lock_irq(&vmci_dev_spinlock);
 704         vmci_dev_g = NULL;
 705         spin_unlock_irq(&vmci_dev_spinlock);
 706
 707         dev_dbg(&pdev->dev, "Resetting vmci device\n");
 708         iowrite32(VMCI_CONTROL_RESET, vmci_dev->iobase + VMCI_CONTROL_ADDR);
 709
 710         /*
 711          * Free IRQ and then disable MSI/MSI-X as appropriate.  For
 712          * MSI-X, we might have multiple vectors, each with their own
 713          * IRQ, which we must free too.
 714          */
 715         free_irq(vmci_dev->irq, vmci_dev);
 716         if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX) {
 717                 if (vmci_dev->exclusive_vectors)
 718                         free_irq(vmci_dev->msix_entries[1].vector, vmci_dev);
 719                 pci_disable_msix(pdev);
 720         } else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI) {
 721                 pci_disable_msi(pdev);
 722         }
 723
 724         tasklet_kill(&vmci_dev->datagram_tasklet);
 725         tasklet_kill(&vmci_dev->bm_tasklet);
 726
 727         if (vmci_dev->notification_bitmap) {
 728                 /*
 729                  * The device reset above cleared the bitmap state of the
 730                  * device, so we can safely free it here.
 731                  */
 732
 733                 vfree(vmci_dev->notification_bitmap);
 734         }
 735
 736         vfree(vmci_dev->data_buffer);
 737
 738         /* The rest are managed resources and will be freed by PCI core */
 739 }
 740
 741 static DEFINE_PCI_DEVICE_TABLE(vmci_ids) = {
 742         { PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_VMCI), },
 743         { 0 },
 744 };
 745 MODULE_DEVICE_TABLE(pci, vmci_ids);
 746
 747 static struct pci_driver vmci_guest_driver = {
 748         .name           = KBUILD_MODNAME,
 749         .id_table       = vmci_ids,
 750         .probe          = vmci_guest_probe_device,
 751         .remove         = vmci_guest_remove_device,
 752 };
 753
 754 int __init vmci_guest_init(void)
 755 {
 756         return pci_register_driver(&vmci_guest_driver);
 757 }
 758
 759 void __exit vmci_guest_exit(void)
 760 {
 761         pci_unregister_driver(&vmci_guest_driver);
 762 }