Pileus Git - ~andy/linux/blob - drivers/block/nvme-core.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include <linux/nvme.h>
  20 #include <linux/bio.h>
  21 #include <linux/bitops.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/delay.h>
  24 #include <linux/errno.h>
  25 #include <linux/fs.h>
  26 #include <linux/genhd.h>
  27 #include <linux/idr.h>
  28 #include <linux/init.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/io.h>
  31 #include <linux/kdev_t.h>
  32 #include <linux/kthread.h>
  33 #include <linux/kernel.h>
  34 #include <linux/mm.h>
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/pci.h>
  38 #include <linux/poison.h>
  39 #include <linux/ptrace.h>
  40 #include <linux/sched.h>
  41 #include <linux/slab.h>
  42 #include <linux/types.h>
  43 #include <scsi/sg.h>
  44 #include <asm-generic/io-64-nonatomic-lo-hi.h>
  45
  46 #define NVME_Q_DEPTH 1024
  47 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
  48 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
  49 #define NVME_MINORS 64
  50 #define ADMIN_TIMEOUT   (60 * HZ)
  51
  52 static int nvme_major;
  53 module_param(nvme_major, int, 0);
  54
  55 static int use_threaded_interrupts;
  56 module_param(use_threaded_interrupts, int, 0);
  57
  58 static DEFINE_SPINLOCK(dev_list_lock);
  59 static LIST_HEAD(dev_list);
  60 static struct task_struct *nvme_thread;
  61
  62 /*
  63  * An NVM Express queue.  Each device has at least two (one for admin
  64  * commands and one for I/O commands).
  65  */
  66 struct nvme_queue {
  67         struct device *q_dmadev;
  68         struct nvme_dev *dev;
  69         spinlock_t q_lock;
  70         struct nvme_command *sq_cmds;
  71         volatile struct nvme_completion *cqes;
  72         dma_addr_t sq_dma_addr;
  73         dma_addr_t cq_dma_addr;
  74         wait_queue_head_t sq_full;
  75         wait_queue_t sq_cong_wait;
  76         struct bio_list sq_cong;
  77         u32 __iomem *q_db;
  78         u16 q_depth;
  79         u16 cq_vector;
  80         u16 sq_head;
  81         u16 sq_tail;
  82         u16 cq_head;
  83         u8 cq_phase;
  84         u8 cqe_seen;
  85         u8 q_suspended;
  86         unsigned long cmdid_data[];
  87 };
  88
  89 /*
  90  * Check we didin't inadvertently grow the command struct
  91  */
  92 static inline void _nvme_check_size(void)
  93 {
  94         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
  95         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
  96         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
  97         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
  98         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
  99         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
 100         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 101         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
 102         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
 103         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
 104         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 105 }
 106
 107 typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
 108                                                 struct nvme_completion *);
 109
 110 struct nvme_cmd_info {
 111         nvme_completion_fn fn;
 112         void *ctx;
 113         unsigned long timeout;
 114 };
 115
 116 static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
 117 {
 118         return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
 119 }
 120
 121 static unsigned nvme_queue_extra(int depth)
 122 {
 123         return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info));
 124 }
 125
 126 /**
 127  * alloc_cmdid() - Allocate a Command ID
 128  * @nvmeq: The queue that will be used for this command
 129  * @ctx: A pointer that will be passed to the handler
 130  * @handler: The function to call on completion
 131  *
 132  * Allocate a Command ID for a queue.  The data passed in will
 133  * be passed to the completion handler.  This is implemented by using
 134  * the bottom two bits of the ctx pointer to store the handler ID.
 135  * Passing in a pointer that's not 4-byte aligned will cause a BUG.
 136  * We can change this if it becomes a problem.
 137  *
 138  * May be called with local interrupts disabled and the q_lock held,
 139  * or with interrupts enabled and no locks held.
 140  */
 141 static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
 142                                 nvme_completion_fn handler, unsigned timeout)
 143 {
 144         int depth = nvmeq->q_depth - 1;
 145         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 146         int cmdid;
 147
 148         do {
 149                 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
 150                 if (cmdid >= depth)
 151                         return -EBUSY;
 152         } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
 153
 154         info[cmdid].fn = handler;
 155         info[cmdid].ctx = ctx;
 156         info[cmdid].timeout = jiffies + timeout;
 157         return cmdid;
 158 }
 159
 160 static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
 161                                 nvme_completion_fn handler, unsigned timeout)
 162 {
 163         int cmdid;
 164         wait_event_killable(nvmeq->sq_full,
 165                 (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
 166         return (cmdid < 0) ? -EINTR : cmdid;
 167 }
 168
 169 /* Special values must be less than 0x1000 */
 170 #define CMD_CTX_BASE            ((void *)POISON_POINTER_DELTA)
 171 #define CMD_CTX_CANCELLED       (0x30C + CMD_CTX_BASE)
 172 #define CMD_CTX_COMPLETED       (0x310 + CMD_CTX_BASE)
 173 #define CMD_CTX_INVALID         (0x314 + CMD_CTX_BASE)
 174 #define CMD_CTX_FLUSH           (0x318 + CMD_CTX_BASE)
 175
 176 static void special_completion(struct nvme_dev *dev, void *ctx,
 177                                                 struct nvme_completion *cqe)
 178 {
 179         if (ctx == CMD_CTX_CANCELLED)
 180                 return;
 181         if (ctx == CMD_CTX_FLUSH)
 182                 return;
 183         if (ctx == CMD_CTX_COMPLETED) {
 184                 dev_warn(&dev->pci_dev->dev,
 185                                 "completed id %d twice on queue %d\n",
 186                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 187                 return;
 188         }
 189         if (ctx == CMD_CTX_INVALID) {
 190                 dev_warn(&dev->pci_dev->dev,
 191                                 "invalid id %d completed on queue %d\n",
 192                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 193                 return;
 194         }
 195
 196         dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
 197 }
 198
 199 /*
 200  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 201  */
 202 static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
 203                                                 nvme_completion_fn *fn)
 204 {
 205         void *ctx;
 206         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 207
 208         if (cmdid >= nvmeq->q_depth) {
 209                 *fn = special_completion;
 210                 return CMD_CTX_INVALID;
 211         }
 212         if (fn)
 213                 *fn = info[cmdid].fn;
 214         ctx = info[cmdid].ctx;
 215         info[cmdid].fn = special_completion;
 216         info[cmdid].ctx = CMD_CTX_COMPLETED;
 217         clear_bit(cmdid, nvmeq->cmdid_data);
 218         wake_up(&nvmeq->sq_full);
 219         return ctx;
 220 }
 221
 222 static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
 223                                                 nvme_completion_fn *fn)
 224 {
 225         void *ctx;
 226         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 227         if (fn)
 228                 *fn = info[cmdid].fn;
 229         ctx = info[cmdid].ctx;
 230         info[cmdid].fn = special_completion;
 231         info[cmdid].ctx = CMD_CTX_CANCELLED;
 232         return ctx;
 233 }
 234
 235 struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
 236 {
 237         return dev->queues[get_cpu() + 1];
 238 }
 239
 240 void put_nvmeq(struct nvme_queue *nvmeq)
 241 {
 242         put_cpu();
 243 }
 244
 245 /**
 246  * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 247  * @nvmeq: The queue to use
 248  * @cmd: The command to send
 249  *
 250  * Safe to use from interrupt context
 251  */
 252 static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
 253 {
 254         unsigned long flags;
 255         u16 tail;
 256         spin_lock_irqsave(&nvmeq->q_lock, flags);
 257         tail = nvmeq->sq_tail;
 258         memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 259         if (++tail == nvmeq->q_depth)
 260                 tail = 0;
 261         writel(tail, nvmeq->q_db);
 262         nvmeq->sq_tail = tail;
 263         spin_unlock_irqrestore(&nvmeq->q_lock, flags);
 264
 265         return 0;
 266 }
 267
 268 static __le64 **iod_list(struct nvme_iod *iod)
 269 {
 270         return ((void *)iod) + iod->offset;
 271 }
 272
 273 /*
 274  * Will slightly overestimate the number of pages needed.  This is OK
 275  * as it only leads to a small amount of wasted memory for the lifetime of
 276  * the I/O.
 277  */
 278 static int nvme_npages(unsigned size)
 279 {
 280         unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
 281         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 282 }
 283
 284 static struct nvme_iod *
 285 nvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
 286 {
 287         struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
 288                                 sizeof(__le64 *) * nvme_npages(nbytes) +
 289                                 sizeof(struct scatterlist) * nseg, gfp);
 290
 291         if (iod) {
 292                 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
 293                 iod->npages = -1;
 294                 iod->length = nbytes;
 295                 iod->nents = 0;
 296                 iod->start_time = jiffies;
 297         }
 298
 299         return iod;
 300 }
 301
 302 void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
 303 {
 304         const int last_prp = PAGE_SIZE / 8 - 1;
 305         int i;
 306         __le64 **list = iod_list(iod);
 307         dma_addr_t prp_dma = iod->first_dma;
 308
 309         if (iod->npages == 0)
 310                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 311         for (i = 0; i < iod->npages; i++) {
 312                 __le64 *prp_list = list[i];
 313                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
 314                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
 315                 prp_dma = next_prp_dma;
 316         }
 317         kfree(iod);
 318 }
 319
 320 static void nvme_start_io_acct(struct bio *bio)
 321 {
 322         struct gendisk *disk = bio->bi_bdev->bd_disk;
 323         const int rw = bio_data_dir(bio);
 324         int cpu = part_stat_lock();
 325         part_round_stats(cpu, &disk->part0);
 326         part_stat_inc(cpu, &disk->part0, ios[rw]);
 327         part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
 328         part_inc_in_flight(&disk->part0, rw);
 329         part_stat_unlock();
 330 }
 331
 332 static void nvme_end_io_acct(struct bio *bio, unsigned long start_time)
 333 {
 334         struct gendisk *disk = bio->bi_bdev->bd_disk;
 335         const int rw = bio_data_dir(bio);
 336         unsigned long duration = jiffies - start_time;
 337         int cpu = part_stat_lock();
 338         part_stat_add(cpu, &disk->part0, ticks[rw], duration);
 339         part_round_stats(cpu, &disk->part0);
 340         part_dec_in_flight(&disk->part0, rw);
 341         part_stat_unlock();
 342 }
 343
 344 static void bio_completion(struct nvme_dev *dev, void *ctx,
 345                                                 struct nvme_completion *cqe)
 346 {
 347         struct nvme_iod *iod = ctx;
 348         struct bio *bio = iod->private;
 349         u16 status = le16_to_cpup(&cqe->status) >> 1;
 350
 351         if (iod->nents) {
 352                 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
 353                         bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 354                 nvme_end_io_acct(bio, iod->start_time);
 355         }
 356         nvme_free_iod(dev, iod);
 357         if (status)
 358                 bio_endio(bio, -EIO);
 359         else
 360                 bio_endio(bio, 0);
 361 }
 362
 363 /* length is in bytes.  gfp flags indicates whether we may sleep. */
 364 int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
 365                         struct nvme_iod *iod, int total_len, gfp_t gfp)
 366 {
 367         struct dma_pool *pool;
 368         int length = total_len;
 369         struct scatterlist *sg = iod->sg;
 370         int dma_len = sg_dma_len(sg);
 371         u64 dma_addr = sg_dma_address(sg);
 372         int offset = offset_in_page(dma_addr);
 373         __le64 *prp_list;
 374         __le64 **list = iod_list(iod);
 375         dma_addr_t prp_dma;
 376         int nprps, i;
 377
 378         cmd->prp1 = cpu_to_le64(dma_addr);
 379         length -= (PAGE_SIZE - offset);
 380         if (length <= 0)
 381                 return total_len;
 382
 383         dma_len -= (PAGE_SIZE - offset);
 384         if (dma_len) {
 385                 dma_addr += (PAGE_SIZE - offset);
 386         } else {
 387                 sg = sg_next(sg);
 388                 dma_addr = sg_dma_address(sg);
 389                 dma_len = sg_dma_len(sg);
 390         }
 391
 392         if (length <= PAGE_SIZE) {
 393                 cmd->prp2 = cpu_to_le64(dma_addr);
 394                 return total_len;
 395         }
 396
 397         nprps = DIV_ROUND_UP(length, PAGE_SIZE);
 398         if (nprps <= (256 / 8)) {
 399                 pool = dev->prp_small_pool;
 400                 iod->npages = 0;
 401         } else {
 402                 pool = dev->prp_page_pool;
 403                 iod->npages = 1;
 404         }
 405
 406         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 407         if (!prp_list) {
 408                 cmd->prp2 = cpu_to_le64(dma_addr);
 409                 iod->npages = -1;
 410                 return (total_len - length) + PAGE_SIZE;
 411         }
 412         list[0] = prp_list;
 413         iod->first_dma = prp_dma;
 414         cmd->prp2 = cpu_to_le64(prp_dma);
 415         i = 0;
 416         for (;;) {
 417                 if (i == PAGE_SIZE / 8) {
 418                         __le64 *old_prp_list = prp_list;
 419                         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 420                         if (!prp_list)
 421                                 return total_len - length;
 422                         list[iod->npages++] = prp_list;
 423                         prp_list[0] = old_prp_list[i - 1];
 424                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 425                         i = 1;
 426                 }
 427                 prp_list[i++] = cpu_to_le64(dma_addr);
 428                 dma_len -= PAGE_SIZE;
 429                 dma_addr += PAGE_SIZE;
 430                 length -= PAGE_SIZE;
 431                 if (length <= 0)
 432                         break;
 433                 if (dma_len > 0)
 434                         continue;
 435                 BUG_ON(dma_len < 0);
 436                 sg = sg_next(sg);
 437                 dma_addr = sg_dma_address(sg);
 438                 dma_len = sg_dma_len(sg);
 439         }
 440
 441         return total_len;
 442 }
 443
 444 struct nvme_bio_pair {
 445         struct bio b1, b2, *parent;
 446         struct bio_vec *bv1, *bv2;
 447         int err;
 448         atomic_t cnt;
 449 };
 450
 451 static void nvme_bio_pair_endio(struct bio *bio, int err)
 452 {
 453         struct nvme_bio_pair *bp = bio->bi_private;
 454
 455         if (err)
 456                 bp->err = err;
 457
 458         if (atomic_dec_and_test(&bp->cnt)) {
 459                 bio_endio(bp->parent, bp->err);
 460                 kfree(bp->bv1);
 461                 kfree(bp->bv2);
 462                 kfree(bp);
 463         }
 464 }
 465
 466 static struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
 467                                                         int len, int offset)
 468 {
 469         struct nvme_bio_pair *bp;
 470
 471         BUG_ON(len > bio->bi_size);
 472         BUG_ON(idx > bio->bi_vcnt);
 473
 474         bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
 475         if (!bp)
 476                 return NULL;
 477         bp->err = 0;
 478
 479         bp->b1 = *bio;
 480         bp->b2 = *bio;
 481
 482         bp->b1.bi_size = len;
 483         bp->b2.bi_size -= len;
 484         bp->b1.bi_vcnt = idx;
 485         bp->b2.bi_idx = idx;
 486         bp->b2.bi_sector += len >> 9;
 487
 488         if (offset) {
 489                 bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 490                                                                 GFP_ATOMIC);
 491                 if (!bp->bv1)
 492                         goto split_fail_1;
 493
 494                 bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 495                                                                 GFP_ATOMIC);
 496                 if (!bp->bv2)
 497                         goto split_fail_2;
 498
 499                 memcpy(bp->bv1, bio->bi_io_vec,
 500                         bio->bi_max_vecs * sizeof(struct bio_vec));
 501                 memcpy(bp->bv2, bio->bi_io_vec,
 502                         bio->bi_max_vecs * sizeof(struct bio_vec));
 503
 504                 bp->b1.bi_io_vec = bp->bv1;
 505                 bp->b2.bi_io_vec = bp->bv2;
 506                 bp->b2.bi_io_vec[idx].bv_offset += offset;
 507                 bp->b2.bi_io_vec[idx].bv_len -= offset;
 508                 bp->b1.bi_io_vec[idx].bv_len = offset;
 509                 bp->b1.bi_vcnt++;
 510         } else
 511                 bp->bv1 = bp->bv2 = NULL;
 512
 513         bp->b1.bi_private = bp;
 514         bp->b2.bi_private = bp;
 515
 516         bp->b1.bi_end_io = nvme_bio_pair_endio;
 517         bp->b2.bi_end_io = nvme_bio_pair_endio;
 518
 519         bp->parent = bio;
 520         atomic_set(&bp->cnt, 2);
 521
 522         return bp;
 523
 524  split_fail_2:
 525         kfree(bp->bv1);
 526  split_fail_1:
 527         kfree(bp);
 528         return NULL;
 529 }
 530
 531 static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
 532                                                 int idx, int len, int offset)
 533 {
 534         struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
 535         if (!bp)
 536                 return -ENOMEM;
 537
 538         if (bio_list_empty(&nvmeq->sq_cong))
 539                 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 540         bio_list_add(&nvmeq->sq_cong, &bp->b1);
 541         bio_list_add(&nvmeq->sq_cong, &bp->b2);
 542
 543         return 0;
 544 }
 545
 546 /* NVMe scatterlists require no holes in the virtual address */
 547 #define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2)   ((vec2)->bv_offset || \
 548                         (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
 549
 550 static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
 551                 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
 552 {
 553         struct bio_vec *bvec, *bvprv = NULL;
 554         struct scatterlist *sg = NULL;
 555         int i, length = 0, nsegs = 0, split_len = bio->bi_size;
 556
 557         if (nvmeq->dev->stripe_size)
 558                 split_len = nvmeq->dev->stripe_size -
 559                         ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1));
 560
 561         sg_init_table(iod->sg, psegs);
 562         bio_for_each_segment(bvec, bio, i) {
 563                 if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
 564                         sg->length += bvec->bv_len;
 565                 } else {
 566                         if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
 567                                 return nvme_split_and_submit(bio, nvmeq, i,
 568                                                                 length, 0);
 569
 570                         sg = sg ? sg + 1 : iod->sg;
 571                         sg_set_page(sg, bvec->bv_page, bvec->bv_len,
 572                                                         bvec->bv_offset);
 573                         nsegs++;
 574                 }
 575
 576                 if (split_len - length < bvec->bv_len)
 577                         return nvme_split_and_submit(bio, nvmeq, i, split_len,
 578                                                         split_len - length);
 579                 length += bvec->bv_len;
 580                 bvprv = bvec;
 581         }
 582         iod->nents = nsegs;
 583         sg_mark_end(sg);
 584         if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
 585                 return -ENOMEM;
 586
 587         BUG_ON(length != bio->bi_size);
 588         return length;
 589 }
 590
 591 /*
 592  * We reuse the small pool to allocate the 16-byte range here as it is not
 593  * worth having a special pool for these or additional cases to handle freeing
 594  * the iod.
 595  */
 596 static int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 597                 struct bio *bio, struct nvme_iod *iod, int cmdid)
 598 {
 599         struct nvme_dsm_range *range;
 600         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 601
 602         range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
 603                                                         &iod->first_dma);
 604         if (!range)
 605                 return -ENOMEM;
 606
 607         iod_list(iod)[0] = (__le64 *)range;
 608         iod->npages = 0;
 609
 610         range->cattr = cpu_to_le32(0);
 611         range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
 612         range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 613
 614         memset(cmnd, 0, sizeof(*cmnd));
 615         cmnd->dsm.opcode = nvme_cmd_dsm;
 616         cmnd->dsm.command_id = cmdid;
 617         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
 618         cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
 619         cmnd->dsm.nr = 0;
 620         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 621
 622         if (++nvmeq->sq_tail == nvmeq->q_depth)
 623                 nvmeq->sq_tail = 0;
 624         writel(nvmeq->sq_tail, nvmeq->q_db);
 625
 626         return 0;
 627 }
 628
 629 static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 630                                                                 int cmdid)
 631 {
 632         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 633
 634         memset(cmnd, 0, sizeof(*cmnd));
 635         cmnd->common.opcode = nvme_cmd_flush;
 636         cmnd->common.command_id = cmdid;
 637         cmnd->common.nsid = cpu_to_le32(ns->ns_id);
 638
 639         if (++nvmeq->sq_tail == nvmeq->q_depth)
 640                 nvmeq->sq_tail = 0;
 641         writel(nvmeq->sq_tail, nvmeq->q_db);
 642
 643         return 0;
 644 }
 645
 646 int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
 647 {
 648         int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
 649                                         special_completion, NVME_IO_TIMEOUT);
 650         if (unlikely(cmdid < 0))
 651                 return cmdid;
 652
 653         return nvme_submit_flush(nvmeq, ns, cmdid);
 654 }
 655
 656 /*
 657  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 658  */
 659 static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 660                                                                 struct bio *bio)
 661 {
 662         struct nvme_command *cmnd;
 663         struct nvme_iod *iod;
 664         enum dma_data_direction dma_dir;
 665         int cmdid, length, result;
 666         u16 control;
 667         u32 dsmgmt;
 668         int psegs = bio_phys_segments(ns->queue, bio);
 669
 670         if ((bio->bi_rw & REQ_FLUSH) && psegs) {
 671                 result = nvme_submit_flush_data(nvmeq, ns);
 672                 if (result)
 673                         return result;
 674         }
 675
 676         result = -ENOMEM;
 677         iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
 678         if (!iod)
 679                 goto nomem;
 680         iod->private = bio;
 681
 682         result = -EBUSY;
 683         cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
 684         if (unlikely(cmdid < 0))
 685                 goto free_iod;
 686
 687         if (bio->bi_rw & REQ_DISCARD) {
 688                 result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
 689                 if (result)
 690                         goto free_cmdid;
 691                 return result;
 692         }
 693         if ((bio->bi_rw & REQ_FLUSH) && !psegs)
 694                 return nvme_submit_flush(nvmeq, ns, cmdid);
 695
 696         control = 0;
 697         if (bio->bi_rw & REQ_FUA)
 698                 control |= NVME_RW_FUA;
 699         if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 700                 control |= NVME_RW_LR;
 701
 702         dsmgmt = 0;
 703         if (bio->bi_rw & REQ_RAHEAD)
 704                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 705
 706         cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 707
 708         memset(cmnd, 0, sizeof(*cmnd));
 709         if (bio_data_dir(bio)) {
 710                 cmnd->rw.opcode = nvme_cmd_write;
 711                 dma_dir = DMA_TO_DEVICE;
 712         } else {
 713                 cmnd->rw.opcode = nvme_cmd_read;
 714                 dma_dir = DMA_FROM_DEVICE;
 715         }
 716
 717         result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs);
 718         if (result <= 0)
 719                 goto free_cmdid;
 720         length = result;
 721
 722         cmnd->rw.command_id = cmdid;
 723         cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
 724         length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
 725                                                                 GFP_ATOMIC);
 726         cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 727         cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
 728         cmnd->rw.control = cpu_to_le16(control);
 729         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 730
 731         nvme_start_io_acct(bio);
 732         if (++nvmeq->sq_tail == nvmeq->q_depth)
 733                 nvmeq->sq_tail = 0;
 734         writel(nvmeq->sq_tail, nvmeq->q_db);
 735
 736         return 0;
 737
 738  free_cmdid:
 739         free_cmdid(nvmeq, cmdid, NULL);
 740  free_iod:
 741         nvme_free_iod(nvmeq->dev, iod);
 742  nomem:
 743         return result;
 744 }
 745
 746 static int nvme_process_cq(struct nvme_queue *nvmeq)
 747 {
 748         u16 head, phase;
 749
 750         head = nvmeq->cq_head;
 751         phase = nvmeq->cq_phase;
 752
 753         for (;;) {
 754                 void *ctx;
 755                 nvme_completion_fn fn;
 756                 struct nvme_completion cqe = nvmeq->cqes[head];
 757                 if ((le16_to_cpu(cqe.status) & 1) != phase)
 758                         break;
 759                 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
 760                 if (++head == nvmeq->q_depth) {
 761                         head = 0;
 762                         phase = !phase;
 763                 }
 764
 765                 ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
 766                 fn(nvmeq->dev, ctx, &cqe);
 767         }
 768
 769         /* If the controller ignores the cq head doorbell and continuously
 770          * writes to the queue, it is theoretically possible to wrap around
 771          * the queue twice and mistakenly return IRQ_NONE.  Linux only
 772          * requires that 0.1% of your interrupts are handled, so this isn't
 773          * a big problem.
 774          */
 775         if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
 776                 return 0;
 777
 778         writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
 779         nvmeq->cq_head = head;
 780         nvmeq->cq_phase = phase;
 781
 782         nvmeq->cqe_seen = 1;
 783         return 1;
 784 }
 785
 786 static void nvme_make_request(struct request_queue *q, struct bio *bio)
 787 {
 788         struct nvme_ns *ns = q->queuedata;
 789         struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
 790         int result = -EBUSY;
 791
 792         if (!nvmeq) {
 793                 put_nvmeq(NULL);
 794                 bio_endio(bio, -EIO);
 795                 return;
 796         }
 797
 798         spin_lock_irq(&nvmeq->q_lock);
 799         if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong))
 800                 result = nvme_submit_bio_queue(nvmeq, ns, bio);
 801         if (unlikely(result)) {
 802                 if (bio_list_empty(&nvmeq->sq_cong))
 803                         add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 804                 bio_list_add(&nvmeq->sq_cong, bio);
 805         }
 806
 807         nvme_process_cq(nvmeq);
 808         spin_unlock_irq(&nvmeq->q_lock);
 809         put_nvmeq(nvmeq);
 810 }
 811
 812 static irqreturn_t nvme_irq(int irq, void *data)
 813 {
 814         irqreturn_t result;
 815         struct nvme_queue *nvmeq = data;
 816         spin_lock(&nvmeq->q_lock);
 817         nvme_process_cq(nvmeq);
 818         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
 819         nvmeq->cqe_seen = 0;
 820         spin_unlock(&nvmeq->q_lock);
 821         return result;
 822 }
 823
 824 static irqreturn_t nvme_irq_check(int irq, void *data)
 825 {
 826         struct nvme_queue *nvmeq = data;
 827         struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
 828         if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
 829                 return IRQ_NONE;
 830         return IRQ_WAKE_THREAD;
 831 }
 832
 833 static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
 834 {
 835         spin_lock_irq(&nvmeq->q_lock);
 836         cancel_cmdid(nvmeq, cmdid, NULL);
 837         spin_unlock_irq(&nvmeq->q_lock);
 838 }
 839
 840 struct sync_cmd_info {
 841         struct task_struct *task;
 842         u32 result;
 843         int status;
 844 };
 845
 846 static void sync_completion(struct nvme_dev *dev, void *ctx,
 847                                                 struct nvme_completion *cqe)
 848 {
 849         struct sync_cmd_info *cmdinfo = ctx;
 850         cmdinfo->result = le32_to_cpup(&cqe->result);
 851         cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
 852         wake_up_process(cmdinfo->task);
 853 }
 854
 855 /*
 856  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 857  * if the result is positive, it's an NVM Express status code
 858  */
 859 int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
 860                                                 u32 *result, unsigned timeout)
 861 {
 862         int cmdid;
 863         struct sync_cmd_info cmdinfo;
 864
 865         cmdinfo.task = current;
 866         cmdinfo.status = -EINTR;
 867
 868         cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
 869                                                                 timeout);
 870         if (cmdid < 0)
 871                 return cmdid;
 872         cmd->common.command_id = cmdid;
 873
 874         set_current_state(TASK_KILLABLE);
 875         nvme_submit_cmd(nvmeq, cmd);
 876         schedule_timeout(timeout);
 877
 878         if (cmdinfo.status == -EINTR) {
 879                 nvme_abort_command(nvmeq, cmdid);
 880                 return -EINTR;
 881         }
 882
 883         if (result)
 884                 *result = cmdinfo.result;
 885
 886         return cmdinfo.status;
 887 }
 888
 889 int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
 890                                                                 u32 *result)
 891 {
 892         return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
 893 }
 894
 895 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 896 {
 897         int status;
 898         struct nvme_command c;
 899
 900         memset(&c, 0, sizeof(c));
 901         c.delete_queue.opcode = opcode;
 902         c.delete_queue.qid = cpu_to_le16(id);
 903
 904         status = nvme_submit_admin_cmd(dev, &c, NULL);
 905         if (status)
 906                 return -EIO;
 907         return 0;
 908 }
 909
 910 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 911                                                 struct nvme_queue *nvmeq)
 912 {
 913         int status;
 914         struct nvme_command c;
 915         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 916
 917         memset(&c, 0, sizeof(c));
 918         c.create_cq.opcode = nvme_admin_create_cq;
 919         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
 920         c.create_cq.cqid = cpu_to_le16(qid);
 921         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 922         c.create_cq.cq_flags = cpu_to_le16(flags);
 923         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 924
 925         status = nvme_submit_admin_cmd(dev, &c, NULL);
 926         if (status)
 927                 return -EIO;
 928         return 0;
 929 }
 930
 931 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 932                                                 struct nvme_queue *nvmeq)
 933 {
 934         int status;
 935         struct nvme_command c;
 936         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
 937
 938         memset(&c, 0, sizeof(c));
 939         c.create_sq.opcode = nvme_admin_create_sq;
 940         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
 941         c.create_sq.sqid = cpu_to_le16(qid);
 942         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 943         c.create_sq.sq_flags = cpu_to_le16(flags);
 944         c.create_sq.cqid = cpu_to_le16(qid);
 945
 946         status = nvme_submit_admin_cmd(dev, &c, NULL);
 947         if (status)
 948                 return -EIO;
 949         return 0;
 950 }
 951
 952 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
 953 {
 954         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
 955 }
 956
 957 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 958 {
 959         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 960 }
 961
 962 int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
 963                                                         dma_addr_t dma_addr)
 964 {
 965         struct nvme_command c;
 966
 967         memset(&c, 0, sizeof(c));
 968         c.identify.opcode = nvme_admin_identify;
 969         c.identify.nsid = cpu_to_le32(nsid);
 970         c.identify.prp1 = cpu_to_le64(dma_addr);
 971         c.identify.cns = cpu_to_le32(cns);
 972
 973         return nvme_submit_admin_cmd(dev, &c, NULL);
 974 }
 975
 976 int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
 977                                         dma_addr_t dma_addr, u32 *result)
 978 {
 979         struct nvme_command c;
 980
 981         memset(&c, 0, sizeof(c));
 982         c.features.opcode = nvme_admin_get_features;
 983         c.features.nsid = cpu_to_le32(nsid);
 984         c.features.prp1 = cpu_to_le64(dma_addr);
 985         c.features.fid = cpu_to_le32(fid);
 986
 987         return nvme_submit_admin_cmd(dev, &c, result);
 988 }
 989
 990 int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
 991                                         dma_addr_t dma_addr, u32 *result)
 992 {
 993         struct nvme_command c;
 994
 995         memset(&c, 0, sizeof(c));
 996         c.features.opcode = nvme_admin_set_features;
 997         c.features.prp1 = cpu_to_le64(dma_addr);
 998         c.features.fid = cpu_to_le32(fid);
 999         c.features.dword11 = cpu_to_le32(dword11);
1000
1001         return nvme_submit_admin_cmd(dev, &c, result);
1002 }
1003
1004 /**
1005  * nvme_cancel_ios - Cancel outstanding I/Os
1006  * @queue: The queue to cancel I/Os on
1007  * @timeout: True to only cancel I/Os which have timed out
1008  */
1009 static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
1010 {
1011         int depth = nvmeq->q_depth - 1;
1012         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
1013         unsigned long now = jiffies;
1014         int cmdid;
1015
1016         for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
1017                 void *ctx;
1018                 nvme_completion_fn fn;
1019                 static struct nvme_completion cqe = {
1020                         .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
1021                 };
1022
1023                 if (timeout && !time_after(now, info[cmdid].timeout))
1024                         continue;
1025                 if (info[cmdid].ctx == CMD_CTX_CANCELLED)
1026                         continue;
1027                 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
1028                 ctx = cancel_cmdid(nvmeq, cmdid, &fn);
1029                 fn(nvmeq->dev, ctx, &cqe);
1030         }
1031 }
1032
1033 static void nvme_free_queue(struct nvme_queue *nvmeq)
1034 {
1035         spin_lock_irq(&nvmeq->q_lock);
1036         while (bio_list_peek(&nvmeq->sq_cong)) {
1037                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1038                 bio_endio(bio, -EIO);
1039         }
1040         spin_unlock_irq(&nvmeq->q_lock);
1041
1042         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1043                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1044         dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1045                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1046         kfree(nvmeq);
1047 }
1048
1049 static void nvme_free_queues(struct nvme_dev *dev)
1050 {
1051         int i;
1052
1053         for (i = dev->queue_count - 1; i >= 0; i--) {
1054                 nvme_free_queue(dev->queues[i]);
1055                 dev->queue_count--;
1056                 dev->queues[i] = NULL;
1057         }
1058 }
1059
1060 static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1061 {
1062         struct nvme_queue *nvmeq = dev->queues[qid];
1063         int vector = dev->entry[nvmeq->cq_vector].vector;
1064
1065         spin_lock_irq(&nvmeq->q_lock);
1066         if (nvmeq->q_suspended) {
1067                 spin_unlock_irq(&nvmeq->q_lock);
1068                 return;
1069         }
1070         nvmeq->q_suspended = 1;
1071         spin_unlock_irq(&nvmeq->q_lock);
1072
1073         irq_set_affinity_hint(vector, NULL);
1074         free_irq(vector, nvmeq);
1075
1076         /* Don't tell the adapter to delete the admin queue */
1077         if (qid) {
1078                 adapter_delete_sq(dev, qid);
1079                 adapter_delete_cq(dev, qid);
1080         }
1081
1082         spin_lock_irq(&nvmeq->q_lock);
1083         nvme_process_cq(nvmeq);
1084         nvme_cancel_ios(nvmeq, false);
1085         spin_unlock_irq(&nvmeq->q_lock);
1086 }
1087
1088 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1089                                                         int depth, int vector)
1090 {
1091         struct device *dmadev = &dev->pci_dev->dev;
1092         unsigned extra = nvme_queue_extra(depth);
1093         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
1094         if (!nvmeq)
1095                 return NULL;
1096
1097         nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
1098                                         &nvmeq->cq_dma_addr, GFP_KERNEL);
1099         if (!nvmeq->cqes)
1100                 goto free_nvmeq;
1101         memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
1102
1103         nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1104                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1105         if (!nvmeq->sq_cmds)
1106                 goto free_cqdma;
1107
1108         nvmeq->q_dmadev = dmadev;
1109         nvmeq->dev = dev;
1110         spin_lock_init(&nvmeq->q_lock);
1111         nvmeq->cq_head = 0;
1112         nvmeq->cq_phase = 1;
1113         init_waitqueue_head(&nvmeq->sq_full);
1114         init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
1115         bio_list_init(&nvmeq->sq_cong);
1116         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1117         nvmeq->q_depth = depth;
1118         nvmeq->cq_vector = vector;
1119         nvmeq->q_suspended = 1;
1120         dev->queue_count++;
1121
1122         return nvmeq;
1123
1124  free_cqdma:
1125         dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1126                                                         nvmeq->cq_dma_addr);
1127  free_nvmeq:
1128         kfree(nvmeq);
1129         return NULL;
1130 }
1131
1132 static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1133                                                         const char *name)
1134 {
1135         if (use_threaded_interrupts)
1136                 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1137                                         nvme_irq_check, nvme_irq, IRQF_SHARED,
1138                                         name, nvmeq);
1139         return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1140                                 IRQF_SHARED, name, nvmeq);
1141 }
1142
1143 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1144 {
1145         struct nvme_dev *dev = nvmeq->dev;
1146         unsigned extra = nvme_queue_extra(nvmeq->q_depth);
1147
1148         nvmeq->sq_tail = 0;
1149         nvmeq->cq_head = 0;
1150         nvmeq->cq_phase = 1;
1151         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1152         memset(nvmeq->cmdid_data, 0, extra);
1153         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1154         nvme_cancel_ios(nvmeq, false);
1155         nvmeq->q_suspended = 0;
1156 }
1157
1158 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1159 {
1160         struct nvme_dev *dev = nvmeq->dev;
1161         int result;
1162
1163         result = adapter_alloc_cq(dev, qid, nvmeq);
1164         if (result < 0)
1165                 return result;
1166
1167         result = adapter_alloc_sq(dev, qid, nvmeq);
1168         if (result < 0)
1169                 goto release_cq;
1170
1171         result = queue_request_irq(dev, nvmeq, "nvme");
1172         if (result < 0)
1173                 goto release_sq;
1174
1175         spin_lock_irq(&nvmeq->q_lock);
1176         nvme_init_queue(nvmeq, qid);
1177         spin_unlock_irq(&nvmeq->q_lock);
1178
1179         return result;
1180
1181  release_sq:
1182         adapter_delete_sq(dev, qid);
1183  release_cq:
1184         adapter_delete_cq(dev, qid);
1185         return result;
1186 }
1187
1188 static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1189 {
1190         unsigned long timeout;
1191         u32 bit = enabled ? NVME_CSTS_RDY : 0;
1192
1193         timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1194
1195         while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1196                 msleep(100);
1197                 if (fatal_signal_pending(current))
1198                         return -EINTR;
1199                 if (time_after(jiffies, timeout)) {
1200                         dev_err(&dev->pci_dev->dev,
1201                                 "Device not ready; aborting initialisation\n");
1202                         return -ENODEV;
1203                 }
1204         }
1205
1206         return 0;
1207 }
1208
1209 /*
1210  * If the device has been passed off to us in an enabled state, just clear
1211  * the enabled bit.  The spec says we should set the 'shutdown notification
1212  * bits', but doing so may cause the device to complete commands to the
1213  * admin queue ... and we don't know what memory that might be pointing at!
1214  */
1215 static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1216 {
1217         u32 cc = readl(&dev->bar->cc);
1218
1219         if (cc & NVME_CC_ENABLE)
1220                 writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc);
1221         return nvme_wait_ready(dev, cap, false);
1222 }
1223
1224 static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1225 {
1226         return nvme_wait_ready(dev, cap, true);
1227 }
1228
1229 static int nvme_shutdown_ctrl(struct nvme_dev *dev)
1230 {
1231         unsigned long timeout;
1232         u32 cc;
1233
1234         cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL;
1235         writel(cc, &dev->bar->cc);
1236
1237         timeout = 2 * HZ + jiffies;
1238         while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
1239                                                         NVME_CSTS_SHST_CMPLT) {
1240                 msleep(100);
1241                 if (fatal_signal_pending(current))
1242                         return -EINTR;
1243                 if (time_after(jiffies, timeout)) {
1244                         dev_err(&dev->pci_dev->dev,
1245                                 "Device shutdown incomplete; abort shutdown\n");
1246                         return -ENODEV;
1247                 }
1248         }
1249
1250         return 0;
1251 }
1252
1253 static int nvme_configure_admin_queue(struct nvme_dev *dev)
1254 {
1255         int result;
1256         u32 aqa;
1257         u64 cap = readq(&dev->bar->cap);
1258         struct nvme_queue *nvmeq;
1259
1260         result = nvme_disable_ctrl(dev, cap);
1261         if (result < 0)
1262                 return result;
1263
1264         nvmeq = dev->queues[0];
1265         if (!nvmeq) {
1266                 nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
1267                 if (!nvmeq)
1268                         return -ENOMEM;
1269                 dev->queues[0] = nvmeq;
1270         }
1271
1272         aqa = nvmeq->q_depth - 1;
1273         aqa |= aqa << 16;
1274
1275         dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1276         dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1277         dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1278         dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1279
1280         writel(aqa, &dev->bar->aqa);
1281         writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1282         writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1283         writel(dev->ctrl_config, &dev->bar->cc);
1284
1285         result = nvme_enable_ctrl(dev, cap);
1286         if (result)
1287                 return result;
1288
1289         result = queue_request_irq(dev, nvmeq, "nvme admin");
1290         if (result)
1291                 return result;
1292
1293         spin_lock_irq(&nvmeq->q_lock);
1294         nvme_init_queue(nvmeq, 0);
1295         spin_unlock_irq(&nvmeq->q_lock);
1296         return result;
1297 }
1298
1299 struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
1300                                 unsigned long addr, unsigned length)
1301 {
1302         int i, err, count, nents, offset;
1303         struct scatterlist *sg;
1304         struct page **pages;
1305         struct nvme_iod *iod;
1306
1307         if (addr & 3)
1308                 return ERR_PTR(-EINVAL);
1309         if (!length || length > INT_MAX - PAGE_SIZE)
1310                 return ERR_PTR(-EINVAL);
1311
1312         offset = offset_in_page(addr);
1313         count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1314         pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
1315         if (!pages)
1316                 return ERR_PTR(-ENOMEM);
1317
1318         err = get_user_pages_fast(addr, count, 1, pages);
1319         if (err < count) {
1320                 count = err;
1321                 err = -EFAULT;
1322                 goto put_pages;
1323         }
1324
1325         iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1326         sg = iod->sg;
1327         sg_init_table(sg, count);
1328         for (i = 0; i < count; i++) {
1329                 sg_set_page(&sg[i], pages[i],
1330                             min_t(unsigned, length, PAGE_SIZE - offset),
1331                             offset);
1332                 length -= (PAGE_SIZE - offset);
1333                 offset = 0;
1334         }
1335         sg_mark_end(&sg[i - 1]);
1336         iod->nents = count;
1337
1338         err = -ENOMEM;
1339         nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1340                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1341         if (!nents)
1342                 goto free_iod;
1343
1344         kfree(pages);
1345         return iod;
1346
1347  free_iod:
1348         kfree(iod);
1349  put_pages:
1350         for (i = 0; i < count; i++)
1351                 put_page(pages[i]);
1352         kfree(pages);
1353         return ERR_PTR(err);
1354 }
1355
1356 void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1357                         struct nvme_iod *iod)
1358 {
1359         int i;
1360
1361         dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1362                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1363
1364         for (i = 0; i < iod->nents; i++)
1365                 put_page(sg_page(&iod->sg[i]));
1366 }
1367
1368 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1369 {
1370         struct nvme_dev *dev = ns->dev;
1371         struct nvme_queue *nvmeq;
1372         struct nvme_user_io io;
1373         struct nvme_command c;
1374         unsigned length, meta_len;
1375         int status, i;
1376         struct nvme_iod *iod, *meta_iod = NULL;
1377         dma_addr_t meta_dma_addr;
1378         void *meta, *uninitialized_var(meta_mem);
1379
1380         if (copy_from_user(&io, uio, sizeof(io)))
1381                 return -EFAULT;
1382         length = (io.nblocks + 1) << ns->lba_shift;
1383         meta_len = (io.nblocks + 1) * ns->ms;
1384
1385         if (meta_len && ((io.metadata & 3) || !io.metadata))
1386                 return -EINVAL;
1387
1388         switch (io.opcode) {
1389         case nvme_cmd_write:
1390         case nvme_cmd_read:
1391         case nvme_cmd_compare:
1392                 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
1393                 break;
1394         default:
1395                 return -EINVAL;
1396         }
1397
1398         if (IS_ERR(iod))
1399                 return PTR_ERR(iod);
1400
1401         memset(&c, 0, sizeof(c));
1402         c.rw.opcode = io.opcode;
1403         c.rw.flags = io.flags;
1404         c.rw.nsid = cpu_to_le32(ns->ns_id);
1405         c.rw.slba = cpu_to_le64(io.slba);
1406         c.rw.length = cpu_to_le16(io.nblocks);
1407         c.rw.control = cpu_to_le16(io.control);
1408         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1409         c.rw.reftag = cpu_to_le32(io.reftag);
1410         c.rw.apptag = cpu_to_le16(io.apptag);
1411         c.rw.appmask = cpu_to_le16(io.appmask);
1412
1413         if (meta_len) {
1414                 meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
1415                                                                 meta_len);
1416                 if (IS_ERR(meta_iod)) {
1417                         status = PTR_ERR(meta_iod);
1418                         meta_iod = NULL;
1419                         goto unmap;
1420                 }
1421
1422                 meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1423                                                 &meta_dma_addr, GFP_KERNEL);
1424                 if (!meta_mem) {
1425                         status = -ENOMEM;
1426                         goto unmap;
1427                 }
1428
1429                 if (io.opcode & 1) {
1430                         int meta_offset = 0;
1431
1432                         for (i = 0; i < meta_iod->nents; i++) {
1433                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1434                                                 meta_iod->sg[i].offset;
1435                                 memcpy(meta_mem + meta_offset, meta,
1436                                                 meta_iod->sg[i].length);
1437                                 kunmap_atomic(meta);
1438                                 meta_offset += meta_iod->sg[i].length;
1439                         }
1440                 }
1441
1442                 c.rw.metadata = cpu_to_le64(meta_dma_addr);
1443         }
1444
1445         length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
1446
1447         nvmeq = get_nvmeq(dev);
1448         /*
1449          * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
1450          * disabled.  We may be preempted at any point, and be rescheduled
1451          * to a different CPU.  That will cause cacheline bouncing, but no
1452          * additional races since q_lock already protects against other CPUs.
1453          */
1454         put_nvmeq(nvmeq);
1455         if (length != (io.nblocks + 1) << ns->lba_shift)
1456                 status = -ENOMEM;
1457         else if (!nvmeq || nvmeq->q_suspended)
1458                 status = -EBUSY;
1459         else
1460                 status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
1461
1462         if (meta_len) {
1463                 if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1464                         int meta_offset = 0;
1465
1466                         for (i = 0; i < meta_iod->nents; i++) {
1467                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1468                                                 meta_iod->sg[i].offset;
1469                                 memcpy(meta, meta_mem + meta_offset,
1470                                                 meta_iod->sg[i].length);
1471                                 kunmap_atomic(meta);
1472                                 meta_offset += meta_iod->sg[i].length;
1473                         }
1474                 }
1475
1476                 dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1477                                                                 meta_dma_addr);
1478         }
1479
1480  unmap:
1481         nvme_unmap_user_pages(dev, io.opcode & 1, iod);
1482         nvme_free_iod(dev, iod);
1483
1484         if (meta_iod) {
1485                 nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1486                 nvme_free_iod(dev, meta_iod);
1487         }
1488
1489         return status;
1490 }
1491
1492 static int nvme_user_admin_cmd(struct nvme_dev *dev,
1493                                         struct nvme_admin_cmd __user *ucmd)
1494 {
1495         struct nvme_admin_cmd cmd;
1496         struct nvme_command c;
1497         int status, length;
1498         struct nvme_iod *uninitialized_var(iod);
1499         unsigned timeout;
1500
1501         if (!capable(CAP_SYS_ADMIN))
1502                 return -EACCES;
1503         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1504                 return -EFAULT;
1505
1506         memset(&c, 0, sizeof(c));
1507         c.common.opcode = cmd.opcode;
1508         c.common.flags = cmd.flags;
1509         c.common.nsid = cpu_to_le32(cmd.nsid);
1510         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1511         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1512         c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1513         c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1514         c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1515         c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1516         c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1517         c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1518
1519         length = cmd.data_len;
1520         if (cmd.data_len) {
1521                 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1522                                                                 length);
1523                 if (IS_ERR(iod))
1524                         return PTR_ERR(iod);
1525                 length = nvme_setup_prps(dev, &c.common, iod, length,
1526                                                                 GFP_KERNEL);
1527         }
1528
1529         timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
1530                                                                 ADMIN_TIMEOUT;
1531         if (length != cmd.data_len)
1532                 status = -ENOMEM;
1533         else
1534                 status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result,
1535                                                                 timeout);
1536
1537         if (cmd.data_len) {
1538                 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
1539                 nvme_free_iod(dev, iod);
1540         }
1541
1542         if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
1543                                                         sizeof(cmd.result)))
1544                 status = -EFAULT;
1545
1546         return status;
1547 }
1548
1549 static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1550                                                         unsigned long arg)
1551 {
1552         struct nvme_ns *ns = bdev->bd_disk->private_data;
1553
1554         switch (cmd) {
1555         case NVME_IOCTL_ID:
1556                 force_successful_syscall_return();
1557                 return ns->ns_id;
1558         case NVME_IOCTL_ADMIN_CMD:
1559                 return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
1560         case NVME_IOCTL_SUBMIT_IO:
1561                 return nvme_submit_io(ns, (void __user *)arg);
1562         case SG_GET_VERSION_NUM:
1563                 return nvme_sg_get_version_num((void __user *)arg);
1564         case SG_IO:
1565                 return nvme_sg_io(ns, (void __user *)arg);
1566         default:
1567                 return -ENOTTY;
1568         }
1569 }
1570
1571 #ifdef CONFIG_COMPAT
1572 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1573                                         unsigned int cmd, unsigned long arg)
1574 {
1575         struct nvme_ns *ns = bdev->bd_disk->private_data;
1576
1577         switch (cmd) {
1578         case SG_IO:
1579                 return nvme_sg_io32(ns, arg);
1580         }
1581         return nvme_ioctl(bdev, mode, cmd, arg);
1582 }
1583 #else
1584 #define nvme_compat_ioctl       NULL
1585 #endif
1586
1587 static const struct block_device_operations nvme_fops = {
1588         .owner          = THIS_MODULE,
1589         .ioctl          = nvme_ioctl,
1590         .compat_ioctl   = nvme_compat_ioctl,
1591 };
1592
1593 static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
1594 {
1595         while (bio_list_peek(&nvmeq->sq_cong)) {
1596                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1597                 struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
1598
1599                 if (bio_list_empty(&nvmeq->sq_cong))
1600                         remove_wait_queue(&nvmeq->sq_full,
1601                                                         &nvmeq->sq_cong_wait);
1602                 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
1603                         if (bio_list_empty(&nvmeq->sq_cong))
1604                                 add_wait_queue(&nvmeq->sq_full,
1605                                                         &nvmeq->sq_cong_wait);
1606                         bio_list_add_head(&nvmeq->sq_cong, bio);
1607                         break;
1608                 }
1609         }
1610 }
1611
1612 static int nvme_kthread(void *data)
1613 {
1614         struct nvme_dev *dev;
1615
1616         while (!kthread_should_stop()) {
1617                 set_current_state(TASK_INTERRUPTIBLE);
1618                 spin_lock(&dev_list_lock);
1619                 list_for_each_entry(dev, &dev_list, node) {
1620                         int i;
1621                         for (i = 0; i < dev->queue_count; i++) {
1622                                 struct nvme_queue *nvmeq = dev->queues[i];
1623                                 if (!nvmeq)
1624                                         continue;
1625                                 spin_lock_irq(&nvmeq->q_lock);
1626                                 if (nvmeq->q_suspended)
1627                                         goto unlock;
1628                                 nvme_process_cq(nvmeq);
1629                                 nvme_cancel_ios(nvmeq, true);
1630                                 nvme_resubmit_bios(nvmeq);
1631  unlock:
1632                                 spin_unlock_irq(&nvmeq->q_lock);
1633                         }
1634                 }
1635                 spin_unlock(&dev_list_lock);
1636                 schedule_timeout(round_jiffies_relative(HZ));
1637         }
1638         return 0;
1639 }
1640
1641 static DEFINE_IDA(nvme_index_ida);
1642
1643 static int nvme_get_ns_idx(void)
1644 {
1645         int index, error;
1646
1647         do {
1648                 if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
1649                         return -1;
1650
1651                 spin_lock(&dev_list_lock);
1652                 error = ida_get_new(&nvme_index_ida, &index);
1653                 spin_unlock(&dev_list_lock);
1654         } while (error == -EAGAIN);
1655
1656         if (error)
1657                 index = -1;
1658         return index;
1659 }
1660
1661 static void nvme_put_ns_idx(int index)
1662 {
1663         spin_lock(&dev_list_lock);
1664         ida_remove(&nvme_index_ida, index);
1665         spin_unlock(&dev_list_lock);
1666 }
1667
1668 static void nvme_config_discard(struct nvme_ns *ns)
1669 {
1670         u32 logical_block_size = queue_logical_block_size(ns->queue);
1671         ns->queue->limits.discard_zeroes_data = 0;
1672         ns->queue->limits.discard_alignment = logical_block_size;
1673         ns->queue->limits.discard_granularity = logical_block_size;
1674         ns->queue->limits.max_discard_sectors = 0xffffffff;
1675         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1676 }
1677
1678 static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid,
1679                         struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1680 {
1681         struct nvme_ns *ns;
1682         struct gendisk *disk;
1683         int lbaf;
1684
1685         if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1686                 return NULL;
1687
1688         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1689         if (!ns)
1690                 return NULL;
1691         ns->queue = blk_alloc_queue(GFP_KERNEL);
1692         if (!ns->queue)
1693                 goto out_free_ns;
1694         ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
1695         queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1696         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1697         blk_queue_make_request(ns->queue, nvme_make_request);
1698         ns->dev = dev;
1699         ns->queue->queuedata = ns;
1700
1701         disk = alloc_disk(NVME_MINORS);
1702         if (!disk)
1703                 goto out_free_queue;
1704         ns->ns_id = nsid;
1705         ns->disk = disk;
1706         lbaf = id->flbas & 0xf;
1707         ns->lba_shift = id->lbaf[lbaf].ds;
1708         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1709         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1710         if (dev->max_hw_sectors)
1711                 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
1712
1713         disk->major = nvme_major;
1714         disk->minors = NVME_MINORS;
1715         disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
1716         disk->fops = &nvme_fops;
1717         disk->private_data = ns;
1718         disk->queue = ns->queue;
1719         disk->driverfs_dev = &dev->pci_dev->dev;
1720         sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
1721         set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1722
1723         if (dev->oncs & NVME_CTRL_ONCS_DSM)
1724                 nvme_config_discard(ns);
1725
1726         return ns;
1727
1728  out_free_queue:
1729         blk_cleanup_queue(ns->queue);
1730  out_free_ns:
1731         kfree(ns);
1732         return NULL;
1733 }
1734
1735 static void nvme_ns_free(struct nvme_ns *ns)
1736 {
1737         int index = ns->disk->first_minor / NVME_MINORS;
1738         put_disk(ns->disk);
1739         nvme_put_ns_idx(index);
1740         blk_cleanup_queue(ns->queue);
1741         kfree(ns);
1742 }
1743
1744 static int set_queue_count(struct nvme_dev *dev, int count)
1745 {
1746         int status;
1747         u32 result;
1748         u32 q_count = (count - 1) | ((count - 1) << 16);
1749
1750         status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
1751                                                                 &result);
1752         if (status)
1753                 return status < 0 ? -EIO : -EBUSY;
1754         return min(result & 0xffff, result >> 16) + 1;
1755 }
1756
1757 static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1758 {
1759         return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
1760 }
1761
1762 static int nvme_setup_io_queues(struct nvme_dev *dev)
1763 {
1764         struct pci_dev *pdev = dev->pci_dev;
1765         int result, cpu, i, vecs, nr_io_queues, size, q_depth;
1766
1767         nr_io_queues = num_online_cpus();
1768         result = set_queue_count(dev, nr_io_queues);
1769         if (result < 0)
1770                 return result;
1771         if (result < nr_io_queues)
1772                 nr_io_queues = result;
1773
1774         size = db_bar_size(dev, nr_io_queues);
1775         if (size > 8192) {
1776                 iounmap(dev->bar);
1777                 do {
1778                         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1779                         if (dev->bar)
1780                                 break;
1781                         if (!--nr_io_queues)
1782                                 return -ENOMEM;
1783                         size = db_bar_size(dev, nr_io_queues);
1784                 } while (1);
1785                 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1786                 dev->queues[0]->q_db = dev->dbs;
1787         }
1788
1789         /* Deregister the admin queue's interrupt */
1790         free_irq(dev->entry[0].vector, dev->queues[0]);
1791
1792         vecs = nr_io_queues;
1793         for (i = 0; i < vecs; i++)
1794                 dev->entry[i].entry = i;
1795         for (;;) {
1796                 result = pci_enable_msix(pdev, dev->entry, vecs);
1797                 if (result <= 0)
1798                         break;
1799                 vecs = result;
1800         }
1801
1802         if (result < 0) {
1803                 vecs = nr_io_queues;
1804                 if (vecs > 32)
1805                         vecs = 32;
1806                 for (;;) {
1807                         result = pci_enable_msi_block(pdev, vecs);
1808                         if (result == 0) {
1809                                 for (i = 0; i < vecs; i++)
1810                                         dev->entry[i].vector = i + pdev->irq;
1811                                 break;
1812                         } else if (result < 0) {
1813                                 vecs = 1;
1814                                 break;
1815                         }
1816                         vecs = result;
1817                 }
1818         }
1819
1820         /*
1821          * Should investigate if there's a performance win from allocating
1822          * more queues than interrupt vectors; it might allow the submission
1823          * path to scale better, even if the receive path is limited by the
1824          * number of interrupts.
1825          */
1826         nr_io_queues = vecs;
1827
1828         result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1829         if (result) {
1830                 dev->queues[0]->q_suspended = 1;
1831                 goto free_queues;
1832         }
1833
1834         /* Free previously allocated queues that are no longer usable */
1835         spin_lock(&dev_list_lock);
1836         for (i = dev->queue_count - 1; i > nr_io_queues; i--) {
1837                 struct nvme_queue *nvmeq = dev->queues[i];
1838
1839                 spin_lock_irq(&nvmeq->q_lock);
1840                 nvme_cancel_ios(nvmeq, false);
1841                 spin_unlock_irq(&nvmeq->q_lock);
1842
1843                 nvme_free_queue(nvmeq);
1844                 dev->queue_count--;
1845                 dev->queues[i] = NULL;
1846         }
1847         spin_unlock(&dev_list_lock);
1848
1849         cpu = cpumask_first(cpu_online_mask);
1850         for (i = 0; i < nr_io_queues; i++) {
1851                 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1852                 cpu = cpumask_next(cpu, cpu_online_mask);
1853         }
1854
1855         q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1856                                                                 NVME_Q_DEPTH);
1857         for (i = dev->queue_count - 1; i < nr_io_queues; i++) {
1858                 dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i);
1859                 if (!dev->queues[i + 1]) {
1860                         result = -ENOMEM;
1861                         goto free_queues;
1862                 }
1863         }
1864
1865         for (; i < num_possible_cpus(); i++) {
1866                 int target = i % rounddown_pow_of_two(dev->queue_count - 1);
1867                 dev->queues[i + 1] = dev->queues[target + 1];
1868         }
1869
1870         for (i = 1; i < dev->queue_count; i++) {
1871                 result = nvme_create_queue(dev->queues[i], i);
1872                 if (result) {
1873                         for (--i; i > 0; i--)
1874                                 nvme_disable_queue(dev, i);
1875                         goto free_queues;
1876                 }
1877         }
1878
1879         return 0;
1880
1881  free_queues:
1882         nvme_free_queues(dev);
1883         return result;
1884 }
1885
1886 /*
1887  * Return: error value if an error occurred setting up the queues or calling
1888  * Identify Device.  0 if these succeeded, even if adding some of the
1889  * namespaces failed.  At the moment, these failures are silent.  TBD which
1890  * failures should be reported.
1891  */
1892 static int nvme_dev_add(struct nvme_dev *dev)
1893 {
1894         int res;
1895         unsigned nn, i;
1896         struct nvme_ns *ns;
1897         struct nvme_id_ctrl *ctrl;
1898         struct nvme_id_ns *id_ns;
1899         void *mem;
1900         dma_addr_t dma_addr;
1901         int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
1902
1903         mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
1904                                                                 GFP_KERNEL);
1905         if (!mem)
1906                 return -ENOMEM;
1907
1908         res = nvme_identify(dev, 0, 1, dma_addr);
1909         if (res) {
1910                 res = -EIO;
1911                 goto out;
1912         }
1913
1914         ctrl = mem;
1915         nn = le32_to_cpup(&ctrl->nn);
1916         dev->oncs = le16_to_cpup(&ctrl->oncs);
1917         memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1918         memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1919         memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
1920         if (ctrl->mdts)
1921                 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1922         if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) &&
1923                         (dev->pci_dev->device == 0x0953) && ctrl->vs[3])
1924                 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
1925
1926         id_ns = mem;
1927         for (i = 1; i <= nn; i++) {
1928                 res = nvme_identify(dev, i, 0, dma_addr);
1929                 if (res)
1930                         continue;
1931
1932                 if (id_ns->ncap == 0)
1933                         continue;
1934
1935                 res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
1936                                                         dma_addr + 4096, NULL);
1937                 if (res)
1938                         memset(mem + 4096, 0, 4096);
1939
1940                 ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
1941                 if (ns)
1942                         list_add_tail(&ns->list, &dev->namespaces);
1943         }
1944         list_for_each_entry(ns, &dev->namespaces, list)
1945                 add_disk(ns->disk);
1946         res = 0;
1947
1948  out:
1949         dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
1950         return res;
1951 }
1952
1953 static int nvme_dev_map(struct nvme_dev *dev)
1954 {
1955         int bars, result = -ENOMEM;
1956         struct pci_dev *pdev = dev->pci_dev;
1957
1958         if (pci_enable_device_mem(pdev))
1959                 return result;
1960
1961         dev->entry[0].vector = pdev->irq;
1962         pci_set_master(pdev);
1963         bars = pci_select_bars(pdev, IORESOURCE_MEM);
1964         if (pci_request_selected_regions(pdev, bars, "nvme"))
1965                 goto disable_pci;
1966
1967         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) &&
1968             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
1969                 goto disable;
1970
1971         pci_set_drvdata(pdev, dev);
1972         dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1973         if (!dev->bar)
1974                 goto disable;
1975
1976         dev->db_stride = 1 << NVME_CAP_STRIDE(readq(&dev->bar->cap));
1977         dev->dbs = ((void __iomem *)dev->bar) + 4096;
1978
1979         return 0;
1980
1981  disable:
1982         pci_release_regions(pdev);
1983  disable_pci:
1984         pci_disable_device(pdev);
1985         return result;
1986 }
1987
1988 static void nvme_dev_unmap(struct nvme_dev *dev)
1989 {
1990         if (dev->pci_dev->msi_enabled)
1991                 pci_disable_msi(dev->pci_dev);
1992         else if (dev->pci_dev->msix_enabled)
1993                 pci_disable_msix(dev->pci_dev);
1994
1995         if (dev->bar) {
1996                 iounmap(dev->bar);
1997                 dev->bar = NULL;
1998         }
1999
2000         pci_release_regions(dev->pci_dev);
2001         if (pci_is_enabled(dev->pci_dev))
2002                 pci_disable_device(dev->pci_dev);
2003 }
2004
2005 static void nvme_dev_shutdown(struct nvme_dev *dev)
2006 {
2007         int i;
2008
2009         for (i = dev->queue_count - 1; i >= 0; i--)
2010                 nvme_disable_queue(dev, i);
2011
2012         spin_lock(&dev_list_lock);
2013         list_del_init(&dev->node);
2014         spin_unlock(&dev_list_lock);
2015
2016         if (dev->bar)
2017                 nvme_shutdown_ctrl(dev);
2018         nvme_dev_unmap(dev);
2019 }
2020
2021 static void nvme_dev_remove(struct nvme_dev *dev)
2022 {
2023         struct nvme_ns *ns, *next;
2024
2025         list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2026                 list_del(&ns->list);
2027                 del_gendisk(ns->disk);
2028                 nvme_ns_free(ns);
2029         }
2030 }
2031
2032 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2033 {
2034         struct device *dmadev = &dev->pci_dev->dev;
2035         dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
2036                                                 PAGE_SIZE, PAGE_SIZE, 0);
2037         if (!dev->prp_page_pool)
2038                 return -ENOMEM;
2039
2040         /* Optimisation for I/Os between 4k and 128k */
2041         dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
2042                                                 256, 256, 0);
2043         if (!dev->prp_small_pool) {
2044                 dma_pool_destroy(dev->prp_page_pool);
2045                 return -ENOMEM;
2046         }
2047         return 0;
2048 }
2049
2050 static void nvme_release_prp_pools(struct nvme_dev *dev)
2051 {
2052         dma_pool_destroy(dev->prp_page_pool);
2053         dma_pool_destroy(dev->prp_small_pool);
2054 }
2055
2056 static DEFINE_IDA(nvme_instance_ida);
2057
2058 static int nvme_set_instance(struct nvme_dev *dev)
2059 {
2060         int instance, error;
2061
2062         do {
2063                 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2064                         return -ENODEV;
2065
2066                 spin_lock(&dev_list_lock);
2067                 error = ida_get_new(&nvme_instance_ida, &instance);
2068                 spin_unlock(&dev_list_lock);
2069         } while (error == -EAGAIN);
2070
2071         if (error)
2072                 return -ENODEV;
2073
2074         dev->instance = instance;
2075         return 0;
2076 }
2077
2078 static void nvme_release_instance(struct nvme_dev *dev)
2079 {
2080         spin_lock(&dev_list_lock);
2081         ida_remove(&nvme_instance_ida, dev->instance);
2082         spin_unlock(&dev_list_lock);
2083 }
2084
2085 static void nvme_free_dev(struct kref *kref)
2086 {
2087         struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2088         nvme_dev_remove(dev);
2089         nvme_dev_shutdown(dev);
2090         nvme_free_queues(dev);
2091         nvme_release_instance(dev);
2092         nvme_release_prp_pools(dev);
2093         kfree(dev->queues);
2094         kfree(dev->entry);
2095         kfree(dev);
2096 }
2097
2098 static int nvme_dev_open(struct inode *inode, struct file *f)
2099 {
2100         struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
2101                                                                 miscdev);
2102         kref_get(&dev->kref);
2103         f->private_data = dev;
2104         return 0;
2105 }
2106
2107 static int nvme_dev_release(struct inode *inode, struct file *f)
2108 {
2109         struct nvme_dev *dev = f->private_data;
2110         kref_put(&dev->kref, nvme_free_dev);
2111         return 0;
2112 }
2113
2114 static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2115 {
2116         struct nvme_dev *dev = f->private_data;
2117         switch (cmd) {
2118         case NVME_IOCTL_ADMIN_CMD:
2119                 return nvme_user_admin_cmd(dev, (void __user *)arg);
2120         default:
2121                 return -ENOTTY;
2122         }
2123 }
2124
2125 static const struct file_operations nvme_dev_fops = {
2126         .owner          = THIS_MODULE,
2127         .open           = nvme_dev_open,
2128         .release        = nvme_dev_release,
2129         .unlocked_ioctl = nvme_dev_ioctl,
2130         .compat_ioctl   = nvme_dev_ioctl,
2131 };
2132
2133 static int nvme_dev_start(struct nvme_dev *dev)
2134 {
2135         int result;
2136
2137         result = nvme_dev_map(dev);
2138         if (result)
2139                 return result;
2140
2141         result = nvme_configure_admin_queue(dev);
2142         if (result)
2143                 goto unmap;
2144
2145         spin_lock(&dev_list_lock);
2146         list_add(&dev->node, &dev_list);
2147         spin_unlock(&dev_list_lock);
2148
2149         result = nvme_setup_io_queues(dev);
2150         if (result && result != -EBUSY)
2151                 goto disable;
2152
2153         return result;
2154
2155  disable:
2156         spin_lock(&dev_list_lock);
2157         list_del_init(&dev->node);
2158         spin_unlock(&dev_list_lock);
2159  unmap:
2160         nvme_dev_unmap(dev);
2161         return result;
2162 }
2163
2164 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2165 {
2166         int result = -ENOMEM;
2167         struct nvme_dev *dev;
2168
2169         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2170         if (!dev)
2171                 return -ENOMEM;
2172         dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
2173                                                                 GFP_KERNEL);
2174         if (!dev->entry)
2175                 goto free;
2176         dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
2177                                                                 GFP_KERNEL);
2178         if (!dev->queues)
2179                 goto free;
2180
2181         INIT_LIST_HEAD(&dev->namespaces);
2182         dev->pci_dev = pdev;
2183
2184         result = nvme_set_instance(dev);
2185         if (result)
2186                 goto free;
2187
2188         result = nvme_setup_prp_pools(dev);
2189         if (result)
2190                 goto release;
2191
2192         result = nvme_dev_start(dev);
2193         if (result) {
2194                 if (result == -EBUSY)
2195                         goto create_cdev;
2196                 goto release_pools;
2197         }
2198
2199         result = nvme_dev_add(dev);
2200         if (result)
2201                 goto shutdown;
2202
2203  create_cdev:
2204         scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
2205         dev->miscdev.minor = MISC_DYNAMIC_MINOR;
2206         dev->miscdev.parent = &pdev->dev;
2207         dev->miscdev.name = dev->name;
2208         dev->miscdev.fops = &nvme_dev_fops;
2209         result = misc_register(&dev->miscdev);
2210         if (result)
2211                 goto remove;
2212
2213         kref_init(&dev->kref);
2214         return 0;
2215
2216  remove:
2217         nvme_dev_remove(dev);
2218  shutdown:
2219         nvme_dev_shutdown(dev);
2220  release_pools:
2221         nvme_free_queues(dev);
2222         nvme_release_prp_pools(dev);
2223  release:
2224         nvme_release_instance(dev);
2225  free:
2226         kfree(dev->queues);
2227         kfree(dev->entry);
2228         kfree(dev);
2229         return result;
2230 }
2231
2232 static void nvme_remove(struct pci_dev *pdev)
2233 {
2234         struct nvme_dev *dev = pci_get_drvdata(pdev);
2235         misc_deregister(&dev->miscdev);
2236         kref_put(&dev->kref, nvme_free_dev);
2237 }
2238
2239 /* These functions are yet to be implemented */
2240 #define nvme_error_detected NULL
2241 #define nvme_dump_registers NULL
2242 #define nvme_link_reset NULL
2243 #define nvme_slot_reset NULL
2244 #define nvme_error_resume NULL
2245
2246 static int nvme_suspend(struct device *dev)
2247 {
2248         struct pci_dev *pdev = to_pci_dev(dev);
2249         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2250
2251         nvme_dev_shutdown(ndev);
2252         return 0;
2253 }
2254
2255 static int nvme_resume(struct device *dev)
2256 {
2257         struct pci_dev *pdev = to_pci_dev(dev);
2258         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2259         int ret;
2260
2261         ret = nvme_dev_start(ndev);
2262         /* XXX: should remove gendisks if resume fails */
2263         if (ret)
2264                 nvme_free_queues(ndev);
2265         return ret;
2266 }
2267
2268 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2269
2270 static const struct pci_error_handlers nvme_err_handler = {
2271         .error_detected = nvme_error_detected,
2272         .mmio_enabled   = nvme_dump_registers,
2273         .link_reset     = nvme_link_reset,
2274         .slot_reset     = nvme_slot_reset,
2275         .resume         = nvme_error_resume,
2276 };
2277
2278 /* Move to pci_ids.h later */
2279 #define PCI_CLASS_STORAGE_EXPRESS       0x010802
2280
2281 static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
2282         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2283         { 0, }
2284 };
2285 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2286
2287 static struct pci_driver nvme_driver = {
2288         .name           = "nvme",
2289         .id_table       = nvme_id_table,
2290         .probe          = nvme_probe,
2291         .remove         = nvme_remove,
2292         .driver         = {
2293                 .pm     = &nvme_dev_pm_ops,
2294         },
2295         .err_handler    = &nvme_err_handler,
2296 };
2297
2298 static int __init nvme_init(void)
2299 {
2300         int result;
2301
2302         nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2303         if (IS_ERR(nvme_thread))
2304                 return PTR_ERR(nvme_thread);
2305
2306         result = register_blkdev(nvme_major, "nvme");
2307         if (result < 0)
2308                 goto kill_kthread;
2309         else if (result > 0)
2310                 nvme_major = result;
2311
2312         result = pci_register_driver(&nvme_driver);
2313         if (result)
2314                 goto unregister_blkdev;
2315         return 0;
2316
2317  unregister_blkdev:
2318         unregister_blkdev(nvme_major, "nvme");
2319  kill_kthread:
2320         kthread_stop(nvme_thread);
2321         return result;
2322 }
2323
2324 static void __exit nvme_exit(void)
2325 {
2326         pci_unregister_driver(&nvme_driver);
2327         unregister_blkdev(nvme_major, "nvme");
2328         kthread_stop(nvme_thread);
2329 }
2330
2331 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2332 MODULE_LICENSE("GPL");
2333 MODULE_VERSION("0.8");
2334 module_init(nvme_init);
2335 module_exit(nvme_exit);