Pileus Git - ~andy/linux/blob - drivers/block/nvme-core.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include <linux/nvme.h>
  20 #include <linux/bio.h>
  21 #include <linux/bitops.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/delay.h>
  24 #include <linux/errno.h>
  25 #include <linux/fs.h>
  26 #include <linux/genhd.h>
  27 #include <linux/idr.h>
  28 #include <linux/init.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/io.h>
  31 #include <linux/kdev_t.h>
  32 #include <linux/kthread.h>
  33 #include <linux/kernel.h>
  34 #include <linux/mm.h>
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/pci.h>
  38 #include <linux/poison.h>
  39 #include <linux/ptrace.h>
  40 #include <linux/sched.h>
  41 #include <linux/slab.h>
  42 #include <linux/types.h>
  43 #include <scsi/sg.h>
  44 #include <asm-generic/io-64-nonatomic-lo-hi.h>
  45
  46 #define NVME_Q_DEPTH 1024
  47 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
  48 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
  49 #define NVME_MINORS 64
  50 #define ADMIN_TIMEOUT   (60 * HZ)
  51
  52 static int nvme_major;
  53 module_param(nvme_major, int, 0);
  54
  55 static int use_threaded_interrupts;
  56 module_param(use_threaded_interrupts, int, 0);
  57
  58 static DEFINE_SPINLOCK(dev_list_lock);
  59 static LIST_HEAD(dev_list);
  60 static struct task_struct *nvme_thread;
  61
  62 /*
  63  * An NVM Express queue.  Each device has at least two (one for admin
  64  * commands and one for I/O commands).
  65  */
  66 struct nvme_queue {
  67         struct device *q_dmadev;
  68         struct nvme_dev *dev;
  69         spinlock_t q_lock;
  70         struct nvme_command *sq_cmds;
  71         volatile struct nvme_completion *cqes;
  72         dma_addr_t sq_dma_addr;
  73         dma_addr_t cq_dma_addr;
  74         wait_queue_head_t sq_full;
  75         wait_queue_t sq_cong_wait;
  76         struct bio_list sq_cong;
  77         u32 __iomem *q_db;
  78         u16 q_depth;
  79         u16 cq_vector;
  80         u16 sq_head;
  81         u16 sq_tail;
  82         u16 cq_head;
  83         u8 cq_phase;
  84         u8 cqe_seen;
  85         u8 q_suspended;
  86         unsigned long cmdid_data[];
  87 };
  88
  89 /*
  90  * Check we didin't inadvertently grow the command struct
  91  */
  92 static inline void _nvme_check_size(void)
  93 {
  94         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
  95         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
  96         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
  97         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
  98         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
  99         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
 100         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 101         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
 102         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
 103         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
 104         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 105 }
 106
 107 typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
 108                                                 struct nvme_completion *);
 109
 110 struct nvme_cmd_info {
 111         nvme_completion_fn fn;
 112         void *ctx;
 113         unsigned long timeout;
 114 };
 115
 116 static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
 117 {
 118         return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
 119 }
 120
 121 static unsigned nvme_queue_extra(int depth)
 122 {
 123         return DIV_ROUND_UP(depth, 8) + (depth * sizeof(struct nvme_cmd_info));
 124 }
 125
 126 /**
 127  * alloc_cmdid() - Allocate a Command ID
 128  * @nvmeq: The queue that will be used for this command
 129  * @ctx: A pointer that will be passed to the handler
 130  * @handler: The function to call on completion
 131  *
 132  * Allocate a Command ID for a queue.  The data passed in will
 133  * be passed to the completion handler.  This is implemented by using
 134  * the bottom two bits of the ctx pointer to store the handler ID.
 135  * Passing in a pointer that's not 4-byte aligned will cause a BUG.
 136  * We can change this if it becomes a problem.
 137  *
 138  * May be called with local interrupts disabled and the q_lock held,
 139  * or with interrupts enabled and no locks held.
 140  */
 141 static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
 142                                 nvme_completion_fn handler, unsigned timeout)
 143 {
 144         int depth = nvmeq->q_depth - 1;
 145         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 146         int cmdid;
 147
 148         do {
 149                 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
 150                 if (cmdid >= depth)
 151                         return -EBUSY;
 152         } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
 153
 154         info[cmdid].fn = handler;
 155         info[cmdid].ctx = ctx;
 156         info[cmdid].timeout = jiffies + timeout;
 157         return cmdid;
 158 }
 159
 160 static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
 161                                 nvme_completion_fn handler, unsigned timeout)
 162 {
 163         int cmdid;
 164         wait_event_killable(nvmeq->sq_full,
 165                 (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
 166         return (cmdid < 0) ? -EINTR : cmdid;
 167 }
 168
 169 /* Special values must be less than 0x1000 */
 170 #define CMD_CTX_BASE            ((void *)POISON_POINTER_DELTA)
 171 #define CMD_CTX_CANCELLED       (0x30C + CMD_CTX_BASE)
 172 #define CMD_CTX_COMPLETED       (0x310 + CMD_CTX_BASE)
 173 #define CMD_CTX_INVALID         (0x314 + CMD_CTX_BASE)
 174 #define CMD_CTX_FLUSH           (0x318 + CMD_CTX_BASE)
 175
 176 static void special_completion(struct nvme_dev *dev, void *ctx,
 177                                                 struct nvme_completion *cqe)
 178 {
 179         if (ctx == CMD_CTX_CANCELLED)
 180                 return;
 181         if (ctx == CMD_CTX_FLUSH)
 182                 return;
 183         if (ctx == CMD_CTX_COMPLETED) {
 184                 dev_warn(&dev->pci_dev->dev,
 185                                 "completed id %d twice on queue %d\n",
 186                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 187                 return;
 188         }
 189         if (ctx == CMD_CTX_INVALID) {
 190                 dev_warn(&dev->pci_dev->dev,
 191                                 "invalid id %d completed on queue %d\n",
 192                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
 193                 return;
 194         }
 195
 196         dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
 197 }
 198
 199 /*
 200  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 201  */
 202 static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
 203                                                 nvme_completion_fn *fn)
 204 {
 205         void *ctx;
 206         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 207
 208         if (cmdid >= nvmeq->q_depth) {
 209                 *fn = special_completion;
 210                 return CMD_CTX_INVALID;
 211         }
 212         if (fn)
 213                 *fn = info[cmdid].fn;
 214         ctx = info[cmdid].ctx;
 215         info[cmdid].fn = special_completion;
 216         info[cmdid].ctx = CMD_CTX_COMPLETED;
 217         clear_bit(cmdid, nvmeq->cmdid_data);
 218         wake_up(&nvmeq->sq_full);
 219         return ctx;
 220 }
 221
 222 static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
 223                                                 nvme_completion_fn *fn)
 224 {
 225         void *ctx;
 226         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
 227         if (fn)
 228                 *fn = info[cmdid].fn;
 229         ctx = info[cmdid].ctx;
 230         info[cmdid].fn = special_completion;
 231         info[cmdid].ctx = CMD_CTX_CANCELLED;
 232         return ctx;
 233 }
 234
 235 struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
 236 {
 237         return dev->queues[get_cpu() + 1];
 238 }
 239
 240 void put_nvmeq(struct nvme_queue *nvmeq)
 241 {
 242         put_cpu();
 243 }
 244
 245 /**
 246  * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 247  * @nvmeq: The queue to use
 248  * @cmd: The command to send
 249  *
 250  * Safe to use from interrupt context
 251  */
 252 static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
 253 {
 254         unsigned long flags;
 255         u16 tail;
 256         spin_lock_irqsave(&nvmeq->q_lock, flags);
 257         tail = nvmeq->sq_tail;
 258         memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 259         if (++tail == nvmeq->q_depth)
 260                 tail = 0;
 261         writel(tail, nvmeq->q_db);
 262         nvmeq->sq_tail = tail;
 263         spin_unlock_irqrestore(&nvmeq->q_lock, flags);
 264
 265         return 0;
 266 }
 267
 268 static __le64 **iod_list(struct nvme_iod *iod)
 269 {
 270         return ((void *)iod) + iod->offset;
 271 }
 272
 273 /*
 274  * Will slightly overestimate the number of pages needed.  This is OK
 275  * as it only leads to a small amount of wasted memory for the lifetime of
 276  * the I/O.
 277  */
 278 static int nvme_npages(unsigned size)
 279 {
 280         unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
 281         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 282 }
 283
 284 static struct nvme_iod *
 285 nvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
 286 {
 287         struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
 288                                 sizeof(__le64 *) * nvme_npages(nbytes) +
 289                                 sizeof(struct scatterlist) * nseg, gfp);
 290
 291         if (iod) {
 292                 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
 293                 iod->npages = -1;
 294                 iod->length = nbytes;
 295                 iod->nents = 0;
 296                 iod->start_time = jiffies;
 297         }
 298
 299         return iod;
 300 }
 301
 302 void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
 303 {
 304         const int last_prp = PAGE_SIZE / 8 - 1;
 305         int i;
 306         __le64 **list = iod_list(iod);
 307         dma_addr_t prp_dma = iod->first_dma;
 308
 309         if (iod->npages == 0)
 310                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 311         for (i = 0; i < iod->npages; i++) {
 312                 __le64 *prp_list = list[i];
 313                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
 314                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
 315                 prp_dma = next_prp_dma;
 316         }
 317         kfree(iod);
 318 }
 319
 320 static void nvme_start_io_acct(struct bio *bio)
 321 {
 322         struct gendisk *disk = bio->bi_bdev->bd_disk;
 323         const int rw = bio_data_dir(bio);
 324         int cpu = part_stat_lock();
 325         part_round_stats(cpu, &disk->part0);
 326         part_stat_inc(cpu, &disk->part0, ios[rw]);
 327         part_stat_add(cpu, &disk->part0, sectors[rw], bio_sectors(bio));
 328         part_inc_in_flight(&disk->part0, rw);
 329         part_stat_unlock();
 330 }
 331
 332 static void nvme_end_io_acct(struct bio *bio, unsigned long start_time)
 333 {
 334         struct gendisk *disk = bio->bi_bdev->bd_disk;
 335         const int rw = bio_data_dir(bio);
 336         unsigned long duration = jiffies - start_time;
 337         int cpu = part_stat_lock();
 338         part_stat_add(cpu, &disk->part0, ticks[rw], duration);
 339         part_round_stats(cpu, &disk->part0);
 340         part_dec_in_flight(&disk->part0, rw);
 341         part_stat_unlock();
 342 }
 343
 344 static void bio_completion(struct nvme_dev *dev, void *ctx,
 345                                                 struct nvme_completion *cqe)
 346 {
 347         struct nvme_iod *iod = ctx;
 348         struct bio *bio = iod->private;
 349         u16 status = le16_to_cpup(&cqe->status) >> 1;
 350
 351         if (iod->nents) {
 352                 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
 353                         bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 354                 nvme_end_io_acct(bio, iod->start_time);
 355         }
 356         nvme_free_iod(dev, iod);
 357         if (status)
 358                 bio_endio(bio, -EIO);
 359         else
 360                 bio_endio(bio, 0);
 361 }
 362
 363 /* length is in bytes.  gfp flags indicates whether we may sleep. */
 364 int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
 365                         struct nvme_iod *iod, int total_len, gfp_t gfp)
 366 {
 367         struct dma_pool *pool;
 368         int length = total_len;
 369         struct scatterlist *sg = iod->sg;
 370         int dma_len = sg_dma_len(sg);
 371         u64 dma_addr = sg_dma_address(sg);
 372         int offset = offset_in_page(dma_addr);
 373         __le64 *prp_list;
 374         __le64 **list = iod_list(iod);
 375         dma_addr_t prp_dma;
 376         int nprps, i;
 377
 378         cmd->prp1 = cpu_to_le64(dma_addr);
 379         length -= (PAGE_SIZE - offset);
 380         if (length <= 0)
 381                 return total_len;
 382
 383         dma_len -= (PAGE_SIZE - offset);
 384         if (dma_len) {
 385                 dma_addr += (PAGE_SIZE - offset);
 386         } else {
 387                 sg = sg_next(sg);
 388                 dma_addr = sg_dma_address(sg);
 389                 dma_len = sg_dma_len(sg);
 390         }
 391
 392         if (length <= PAGE_SIZE) {
 393                 cmd->prp2 = cpu_to_le64(dma_addr);
 394                 return total_len;
 395         }
 396
 397         nprps = DIV_ROUND_UP(length, PAGE_SIZE);
 398         if (nprps <= (256 / 8)) {
 399                 pool = dev->prp_small_pool;
 400                 iod->npages = 0;
 401         } else {
 402                 pool = dev->prp_page_pool;
 403                 iod->npages = 1;
 404         }
 405
 406         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 407         if (!prp_list) {
 408                 cmd->prp2 = cpu_to_le64(dma_addr);
 409                 iod->npages = -1;
 410                 return (total_len - length) + PAGE_SIZE;
 411         }
 412         list[0] = prp_list;
 413         iod->first_dma = prp_dma;
 414         cmd->prp2 = cpu_to_le64(prp_dma);
 415         i = 0;
 416         for (;;) {
 417                 if (i == PAGE_SIZE / 8) {
 418                         __le64 *old_prp_list = prp_list;
 419                         prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
 420                         if (!prp_list)
 421                                 return total_len - length;
 422                         list[iod->npages++] = prp_list;
 423                         prp_list[0] = old_prp_list[i - 1];
 424                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 425                         i = 1;
 426                 }
 427                 prp_list[i++] = cpu_to_le64(dma_addr);
 428                 dma_len -= PAGE_SIZE;
 429                 dma_addr += PAGE_SIZE;
 430                 length -= PAGE_SIZE;
 431                 if (length <= 0)
 432                         break;
 433                 if (dma_len > 0)
 434                         continue;
 435                 BUG_ON(dma_len < 0);
 436                 sg = sg_next(sg);
 437                 dma_addr = sg_dma_address(sg);
 438                 dma_len = sg_dma_len(sg);
 439         }
 440
 441         return total_len;
 442 }
 443
 444 struct nvme_bio_pair {
 445         struct bio b1, b2, *parent;
 446         struct bio_vec *bv1, *bv2;
 447         int err;
 448         atomic_t cnt;
 449 };
 450
 451 static void nvme_bio_pair_endio(struct bio *bio, int err)
 452 {
 453         struct nvme_bio_pair *bp = bio->bi_private;
 454
 455         if (err)
 456                 bp->err = err;
 457
 458         if (atomic_dec_and_test(&bp->cnt)) {
 459                 bio_endio(bp->parent, bp->err);
 460                 kfree(bp->bv1);
 461                 kfree(bp->bv2);
 462                 kfree(bp);
 463         }
 464 }
 465
 466 static struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
 467                                                         int len, int offset)
 468 {
 469         struct nvme_bio_pair *bp;
 470
 471         BUG_ON(len > bio->bi_size);
 472         BUG_ON(idx > bio->bi_vcnt);
 473
 474         bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
 475         if (!bp)
 476                 return NULL;
 477         bp->err = 0;
 478
 479         bp->b1 = *bio;
 480         bp->b2 = *bio;
 481
 482         bp->b1.bi_size = len;
 483         bp->b2.bi_size -= len;
 484         bp->b1.bi_vcnt = idx;
 485         bp->b2.bi_idx = idx;
 486         bp->b2.bi_sector += len >> 9;
 487
 488         if (offset) {
 489                 bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 490                                                                 GFP_ATOMIC);
 491                 if (!bp->bv1)
 492                         goto split_fail_1;
 493
 494                 bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
 495                                                                 GFP_ATOMIC);
 496                 if (!bp->bv2)
 497                         goto split_fail_2;
 498
 499                 memcpy(bp->bv1, bio->bi_io_vec,
 500                         bio->bi_max_vecs * sizeof(struct bio_vec));
 501                 memcpy(bp->bv2, bio->bi_io_vec,
 502                         bio->bi_max_vecs * sizeof(struct bio_vec));
 503
 504                 bp->b1.bi_io_vec = bp->bv1;
 505                 bp->b2.bi_io_vec = bp->bv2;
 506                 bp->b2.bi_io_vec[idx].bv_offset += offset;
 507                 bp->b2.bi_io_vec[idx].bv_len -= offset;
 508                 bp->b1.bi_io_vec[idx].bv_len = offset;
 509                 bp->b1.bi_vcnt++;
 510         } else
 511                 bp->bv1 = bp->bv2 = NULL;
 512
 513         bp->b1.bi_private = bp;
 514         bp->b2.bi_private = bp;
 515
 516         bp->b1.bi_end_io = nvme_bio_pair_endio;
 517         bp->b2.bi_end_io = nvme_bio_pair_endio;
 518
 519         bp->parent = bio;
 520         atomic_set(&bp->cnt, 2);
 521
 522         return bp;
 523
 524  split_fail_2:
 525         kfree(bp->bv1);
 526  split_fail_1:
 527         kfree(bp);
 528         return NULL;
 529 }
 530
 531 static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
 532                                                 int idx, int len, int offset)
 533 {
 534         struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
 535         if (!bp)
 536                 return -ENOMEM;
 537
 538         if (bio_list_empty(&nvmeq->sq_cong))
 539                 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 540         bio_list_add(&nvmeq->sq_cong, &bp->b1);
 541         bio_list_add(&nvmeq->sq_cong, &bp->b2);
 542
 543         return 0;
 544 }
 545
 546 /* NVMe scatterlists require no holes in the virtual address */
 547 #define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2)   ((vec2)->bv_offset || \
 548                         (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
 549
 550 static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
 551                 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
 552 {
 553         struct bio_vec *bvec, *bvprv = NULL;
 554         struct scatterlist *sg = NULL;
 555         int i, length = 0, nsegs = 0, split_len = bio->bi_size;
 556
 557         if (nvmeq->dev->stripe_size)
 558                 split_len = nvmeq->dev->stripe_size -
 559                         ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1));
 560
 561         sg_init_table(iod->sg, psegs);
 562         bio_for_each_segment(bvec, bio, i) {
 563                 if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
 564                         sg->length += bvec->bv_len;
 565                 } else {
 566                         if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
 567                                 return nvme_split_and_submit(bio, nvmeq, i,
 568                                                                 length, 0);
 569
 570                         sg = sg ? sg + 1 : iod->sg;
 571                         sg_set_page(sg, bvec->bv_page, bvec->bv_len,
 572                                                         bvec->bv_offset);
 573                         nsegs++;
 574                 }
 575
 576                 if (split_len - length < bvec->bv_len)
 577                         return nvme_split_and_submit(bio, nvmeq, i, split_len,
 578                                                         split_len - length);
 579                 length += bvec->bv_len;
 580                 bvprv = bvec;
 581         }
 582         iod->nents = nsegs;
 583         sg_mark_end(sg);
 584         if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
 585                 return -ENOMEM;
 586
 587         BUG_ON(length != bio->bi_size);
 588         return length;
 589 }
 590
 591 /*
 592  * We reuse the small pool to allocate the 16-byte range here as it is not
 593  * worth having a special pool for these or additional cases to handle freeing
 594  * the iod.
 595  */
 596 static int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 597                 struct bio *bio, struct nvme_iod *iod, int cmdid)
 598 {
 599         struct nvme_dsm_range *range;
 600         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 601
 602         range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
 603                                                         &iod->first_dma);
 604         if (!range)
 605                 return -ENOMEM;
 606
 607         iod_list(iod)[0] = (__le64 *)range;
 608         iod->npages = 0;
 609
 610         range->cattr = cpu_to_le32(0);
 611         range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
 612         range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 613
 614         memset(cmnd, 0, sizeof(*cmnd));
 615         cmnd->dsm.opcode = nvme_cmd_dsm;
 616         cmnd->dsm.command_id = cmdid;
 617         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
 618         cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
 619         cmnd->dsm.nr = 0;
 620         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 621
 622         if (++nvmeq->sq_tail == nvmeq->q_depth)
 623                 nvmeq->sq_tail = 0;
 624         writel(nvmeq->sq_tail, nvmeq->q_db);
 625
 626         return 0;
 627 }
 628
 629 static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 630                                                                 int cmdid)
 631 {
 632         struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 633
 634         memset(cmnd, 0, sizeof(*cmnd));
 635         cmnd->common.opcode = nvme_cmd_flush;
 636         cmnd->common.command_id = cmdid;
 637         cmnd->common.nsid = cpu_to_le32(ns->ns_id);
 638
 639         if (++nvmeq->sq_tail == nvmeq->q_depth)
 640                 nvmeq->sq_tail = 0;
 641         writel(nvmeq->sq_tail, nvmeq->q_db);
 642
 643         return 0;
 644 }
 645
 646 int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
 647 {
 648         int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
 649                                         special_completion, NVME_IO_TIMEOUT);
 650         if (unlikely(cmdid < 0))
 651                 return cmdid;
 652
 653         return nvme_submit_flush(nvmeq, ns, cmdid);
 654 }
 655
 656 /*
 657  * Called with local interrupts disabled and the q_lock held.  May not sleep.
 658  */
 659 static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 660                                                                 struct bio *bio)
 661 {
 662         struct nvme_command *cmnd;
 663         struct nvme_iod *iod;
 664         enum dma_data_direction dma_dir;
 665         int cmdid, length, result;
 666         u16 control;
 667         u32 dsmgmt;
 668         int psegs = bio_phys_segments(ns->queue, bio);
 669
 670         if ((bio->bi_rw & REQ_FLUSH) && psegs) {
 671                 result = nvme_submit_flush_data(nvmeq, ns);
 672                 if (result)
 673                         return result;
 674         }
 675
 676         result = -ENOMEM;
 677         iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
 678         if (!iod)
 679                 goto nomem;
 680         iod->private = bio;
 681
 682         result = -EBUSY;
 683         cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
 684         if (unlikely(cmdid < 0))
 685                 goto free_iod;
 686
 687         if (bio->bi_rw & REQ_DISCARD) {
 688                 result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
 689                 if (result)
 690                         goto free_cmdid;
 691                 return result;
 692         }
 693         if ((bio->bi_rw & REQ_FLUSH) && !psegs)
 694                 return nvme_submit_flush(nvmeq, ns, cmdid);
 695
 696         control = 0;
 697         if (bio->bi_rw & REQ_FUA)
 698                 control |= NVME_RW_FUA;
 699         if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 700                 control |= NVME_RW_LR;
 701
 702         dsmgmt = 0;
 703         if (bio->bi_rw & REQ_RAHEAD)
 704                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 705
 706         cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
 707
 708         memset(cmnd, 0, sizeof(*cmnd));
 709         if (bio_data_dir(bio)) {
 710                 cmnd->rw.opcode = nvme_cmd_write;
 711                 dma_dir = DMA_TO_DEVICE;
 712         } else {
 713                 cmnd->rw.opcode = nvme_cmd_read;
 714                 dma_dir = DMA_FROM_DEVICE;
 715         }
 716
 717         result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs);
 718         if (result <= 0)
 719                 goto free_cmdid;
 720         length = result;
 721
 722         cmnd->rw.command_id = cmdid;
 723         cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
 724         length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
 725                                                                 GFP_ATOMIC);
 726         cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
 727         cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
 728         cmnd->rw.control = cpu_to_le16(control);
 729         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 730
 731         nvme_start_io_acct(bio);
 732         if (++nvmeq->sq_tail == nvmeq->q_depth)
 733                 nvmeq->sq_tail = 0;
 734         writel(nvmeq->sq_tail, nvmeq->q_db);
 735
 736         return 0;
 737
 738  free_cmdid:
 739         free_cmdid(nvmeq, cmdid, NULL);
 740  free_iod:
 741         nvme_free_iod(nvmeq->dev, iod);
 742  nomem:
 743         return result;
 744 }
 745
 746 static int nvme_process_cq(struct nvme_queue *nvmeq)
 747 {
 748         u16 head, phase;
 749
 750         head = nvmeq->cq_head;
 751         phase = nvmeq->cq_phase;
 752
 753         for (;;) {
 754                 void *ctx;
 755                 nvme_completion_fn fn;
 756                 struct nvme_completion cqe = nvmeq->cqes[head];
 757                 if ((le16_to_cpu(cqe.status) & 1) != phase)
 758                         break;
 759                 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
 760                 if (++head == nvmeq->q_depth) {
 761                         head = 0;
 762                         phase = !phase;
 763                 }
 764
 765                 ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
 766                 fn(nvmeq->dev, ctx, &cqe);
 767         }
 768
 769         /* If the controller ignores the cq head doorbell and continuously
 770          * writes to the queue, it is theoretically possible to wrap around
 771          * the queue twice and mistakenly return IRQ_NONE.  Linux only
 772          * requires that 0.1% of your interrupts are handled, so this isn't
 773          * a big problem.
 774          */
 775         if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
 776                 return 0;
 777
 778         writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride));
 779         nvmeq->cq_head = head;
 780         nvmeq->cq_phase = phase;
 781
 782         nvmeq->cqe_seen = 1;
 783         return 1;
 784 }
 785
 786 static void nvme_make_request(struct request_queue *q, struct bio *bio)
 787 {
 788         struct nvme_ns *ns = q->queuedata;
 789         struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
 790         int result = -EBUSY;
 791
 792         if (!nvmeq) {
 793                 put_nvmeq(NULL);
 794                 bio_endio(bio, -EIO);
 795                 return;
 796         }
 797
 798         spin_lock_irq(&nvmeq->q_lock);
 799         if (!nvmeq->q_suspended && bio_list_empty(&nvmeq->sq_cong))
 800                 result = nvme_submit_bio_queue(nvmeq, ns, bio);
 801         if (unlikely(result)) {
 802                 if (bio_list_empty(&nvmeq->sq_cong))
 803                         add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
 804                 bio_list_add(&nvmeq->sq_cong, bio);
 805         }
 806
 807         nvme_process_cq(nvmeq);
 808         spin_unlock_irq(&nvmeq->q_lock);
 809         put_nvmeq(nvmeq);
 810 }
 811
 812 static irqreturn_t nvme_irq(int irq, void *data)
 813 {
 814         irqreturn_t result;
 815         struct nvme_queue *nvmeq = data;
 816         spin_lock(&nvmeq->q_lock);
 817         nvme_process_cq(nvmeq);
 818         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
 819         nvmeq->cqe_seen = 0;
 820         spin_unlock(&nvmeq->q_lock);
 821         return result;
 822 }
 823
 824 static irqreturn_t nvme_irq_check(int irq, void *data)
 825 {
 826         struct nvme_queue *nvmeq = data;
 827         struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
 828         if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
 829                 return IRQ_NONE;
 830         return IRQ_WAKE_THREAD;
 831 }
 832
 833 static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
 834 {
 835         spin_lock_irq(&nvmeq->q_lock);
 836         cancel_cmdid(nvmeq, cmdid, NULL);
 837         spin_unlock_irq(&nvmeq->q_lock);
 838 }
 839
 840 struct sync_cmd_info {
 841         struct task_struct *task;
 842         u32 result;
 843         int status;
 844 };
 845
 846 static void sync_completion(struct nvme_dev *dev, void *ctx,
 847                                                 struct nvme_completion *cqe)
 848 {
 849         struct sync_cmd_info *cmdinfo = ctx;
 850         cmdinfo->result = le32_to_cpup(&cqe->result);
 851         cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
 852         wake_up_process(cmdinfo->task);
 853 }
 854
 855 /*
 856  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 857  * if the result is positive, it's an NVM Express status code
 858  */
 859 int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
 860                                                 u32 *result, unsigned timeout)
 861 {
 862         int cmdid;
 863         struct sync_cmd_info cmdinfo;
 864
 865         cmdinfo.task = current;
 866         cmdinfo.status = -EINTR;
 867
 868         cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
 869                                                                 timeout);
 870         if (cmdid < 0)
 871                 return cmdid;
 872         cmd->common.command_id = cmdid;
 873
 874         set_current_state(TASK_KILLABLE);
 875         nvme_submit_cmd(nvmeq, cmd);
 876         schedule_timeout(timeout);
 877
 878         if (cmdinfo.status == -EINTR) {
 879                 nvme_abort_command(nvmeq, cmdid);
 880                 return -EINTR;
 881         }
 882
 883         if (result)
 884                 *result = cmdinfo.result;
 885
 886         return cmdinfo.status;
 887 }
 888
 889 int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
 890                                                                 u32 *result)
 891 {
 892         return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
 893 }
 894
 895 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 896 {
 897         int status;
 898         struct nvme_command c;
 899
 900         memset(&c, 0, sizeof(c));
 901         c.delete_queue.opcode = opcode;
 902         c.delete_queue.qid = cpu_to_le16(id);
 903
 904         status = nvme_submit_admin_cmd(dev, &c, NULL);
 905         if (status)
 906                 return -EIO;
 907         return 0;
 908 }
 909
 910 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 911                                                 struct nvme_queue *nvmeq)
 912 {
 913         int status;
 914         struct nvme_command c;
 915         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 916
 917         memset(&c, 0, sizeof(c));
 918         c.create_cq.opcode = nvme_admin_create_cq;
 919         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
 920         c.create_cq.cqid = cpu_to_le16(qid);
 921         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 922         c.create_cq.cq_flags = cpu_to_le16(flags);
 923         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 924
 925         status = nvme_submit_admin_cmd(dev, &c, NULL);
 926         if (status)
 927                 return -EIO;
 928         return 0;
 929 }
 930
 931 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 932                                                 struct nvme_queue *nvmeq)
 933 {
 934         int status;
 935         struct nvme_command c;
 936         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
 937
 938         memset(&c, 0, sizeof(c));
 939         c.create_sq.opcode = nvme_admin_create_sq;
 940         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
 941         c.create_sq.sqid = cpu_to_le16(qid);
 942         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 943         c.create_sq.sq_flags = cpu_to_le16(flags);
 944         c.create_sq.cqid = cpu_to_le16(qid);
 945
 946         status = nvme_submit_admin_cmd(dev, &c, NULL);
 947         if (status)
 948                 return -EIO;
 949         return 0;
 950 }
 951
 952 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
 953 {
 954         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
 955 }
 956
 957 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 958 {
 959         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 960 }
 961
 962 int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
 963                                                         dma_addr_t dma_addr)
 964 {
 965         struct nvme_command c;
 966
 967         memset(&c, 0, sizeof(c));
 968         c.identify.opcode = nvme_admin_identify;
 969         c.identify.nsid = cpu_to_le32(nsid);
 970         c.identify.prp1 = cpu_to_le64(dma_addr);
 971         c.identify.cns = cpu_to_le32(cns);
 972
 973         return nvme_submit_admin_cmd(dev, &c, NULL);
 974 }
 975
 976 int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
 977                                         dma_addr_t dma_addr, u32 *result)
 978 {
 979         struct nvme_command c;
 980
 981         memset(&c, 0, sizeof(c));
 982         c.features.opcode = nvme_admin_get_features;
 983         c.features.nsid = cpu_to_le32(nsid);
 984         c.features.prp1 = cpu_to_le64(dma_addr);
 985         c.features.fid = cpu_to_le32(fid);
 986
 987         return nvme_submit_admin_cmd(dev, &c, result);
 988 }
 989
 990 int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
 991                                         dma_addr_t dma_addr, u32 *result)
 992 {
 993         struct nvme_command c;
 994
 995         memset(&c, 0, sizeof(c));
 996         c.features.opcode = nvme_admin_set_features;
 997         c.features.prp1 = cpu_to_le64(dma_addr);
 998         c.features.fid = cpu_to_le32(fid);
 999         c.features.dword11 = cpu_to_le32(dword11);
1000
1001         return nvme_submit_admin_cmd(dev, &c, result);
1002 }
1003
1004 /**
1005  * nvme_cancel_ios - Cancel outstanding I/Os
1006  * @queue: The queue to cancel I/Os on
1007  * @timeout: True to only cancel I/Os which have timed out
1008  */
1009 static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
1010 {
1011         int depth = nvmeq->q_depth - 1;
1012         struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
1013         unsigned long now = jiffies;
1014         int cmdid;
1015
1016         for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
1017                 void *ctx;
1018                 nvme_completion_fn fn;
1019                 static struct nvme_completion cqe = {
1020                         .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
1021                 };
1022
1023                 if (timeout && !time_after(now, info[cmdid].timeout))
1024                         continue;
1025                 if (info[cmdid].ctx == CMD_CTX_CANCELLED)
1026                         continue;
1027                 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
1028                 ctx = cancel_cmdid(nvmeq, cmdid, &fn);
1029                 fn(nvmeq->dev, ctx, &cqe);
1030         }
1031 }
1032
1033 static void nvme_free_queue(struct nvme_queue *nvmeq)
1034 {
1035         spin_lock_irq(&nvmeq->q_lock);
1036         while (bio_list_peek(&nvmeq->sq_cong)) {
1037                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1038                 bio_endio(bio, -EIO);
1039         }
1040         spin_unlock_irq(&nvmeq->q_lock);
1041
1042         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1043                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1044         dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1045                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1046         kfree(nvmeq);
1047 }
1048
1049 static void nvme_free_queues(struct nvme_dev *dev)
1050 {
1051         int i;
1052
1053         for (i = dev->queue_count - 1; i >= 0; i--) {
1054                 nvme_free_queue(dev->queues[i]);
1055                 dev->queue_count--;
1056                 dev->queues[i] = NULL;
1057         }
1058 }
1059
1060 static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1061 {
1062         struct nvme_queue *nvmeq = dev->queues[qid];
1063         int vector = dev->entry[nvmeq->cq_vector].vector;
1064
1065         spin_lock_irq(&nvmeq->q_lock);
1066         if (nvmeq->q_suspended) {
1067                 spin_unlock_irq(&nvmeq->q_lock);
1068                 return;
1069         }
1070         nvmeq->q_suspended = 1;
1071         spin_unlock_irq(&nvmeq->q_lock);
1072
1073         irq_set_affinity_hint(vector, NULL);
1074         free_irq(vector, nvmeq);
1075
1076         /* Don't tell the adapter to delete the admin queue */
1077         if (qid) {
1078                 adapter_delete_sq(dev, qid);
1079                 adapter_delete_cq(dev, qid);
1080         }
1081
1082         spin_lock_irq(&nvmeq->q_lock);
1083         nvme_process_cq(nvmeq);
1084         nvme_cancel_ios(nvmeq, false);
1085         spin_unlock_irq(&nvmeq->q_lock);
1086 }
1087
1088 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1089                                                         int depth, int vector)
1090 {
1091         struct device *dmadev = &dev->pci_dev->dev;
1092         unsigned extra = nvme_queue_extra(depth);
1093         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
1094         if (!nvmeq)
1095                 return NULL;
1096
1097         nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
1098                                         &nvmeq->cq_dma_addr, GFP_KERNEL);
1099         if (!nvmeq->cqes)
1100                 goto free_nvmeq;
1101         memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
1102
1103         nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1104                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1105         if (!nvmeq->sq_cmds)
1106                 goto free_cqdma;
1107
1108         nvmeq->q_dmadev = dmadev;
1109         nvmeq->dev = dev;
1110         spin_lock_init(&nvmeq->q_lock);
1111         nvmeq->cq_head = 0;
1112         nvmeq->cq_phase = 1;
1113         init_waitqueue_head(&nvmeq->sq_full);
1114         init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
1115         bio_list_init(&nvmeq->sq_cong);
1116         nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
1117         nvmeq->q_depth = depth;
1118         nvmeq->cq_vector = vector;
1119         nvmeq->q_suspended = 1;
1120         dev->queue_count++;
1121
1122         return nvmeq;
1123
1124  free_cqdma:
1125         dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1126                                                         nvmeq->cq_dma_addr);
1127  free_nvmeq:
1128         kfree(nvmeq);
1129         return NULL;
1130 }
1131
1132 static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1133                                                         const char *name)
1134 {
1135         if (use_threaded_interrupts)
1136                 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1137                                         nvme_irq_check, nvme_irq,
1138                                         IRQF_DISABLED | IRQF_SHARED,
1139                                         name, nvmeq);
1140         return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1141                                 IRQF_DISABLED | IRQF_SHARED, name, nvmeq);
1142 }
1143
1144 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1145 {
1146         struct nvme_dev *dev = nvmeq->dev;
1147         unsigned extra = nvme_queue_extra(nvmeq->q_depth);
1148
1149         nvmeq->sq_tail = 0;
1150         nvmeq->cq_head = 0;
1151         nvmeq->cq_phase = 1;
1152         nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
1153         memset(nvmeq->cmdid_data, 0, extra);
1154         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1155         nvme_cancel_ios(nvmeq, false);
1156         nvmeq->q_suspended = 0;
1157 }
1158
1159 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1160 {
1161         struct nvme_dev *dev = nvmeq->dev;
1162         int result;
1163
1164         result = adapter_alloc_cq(dev, qid, nvmeq);
1165         if (result < 0)
1166                 return result;
1167
1168         result = adapter_alloc_sq(dev, qid, nvmeq);
1169         if (result < 0)
1170                 goto release_cq;
1171
1172         result = queue_request_irq(dev, nvmeq, "nvme");
1173         if (result < 0)
1174                 goto release_sq;
1175
1176         spin_lock(&nvmeq->q_lock);
1177         nvme_init_queue(nvmeq, qid);
1178         spin_unlock(&nvmeq->q_lock);
1179
1180         return result;
1181
1182  release_sq:
1183         adapter_delete_sq(dev, qid);
1184  release_cq:
1185         adapter_delete_cq(dev, qid);
1186         return result;
1187 }
1188
1189 static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1190 {
1191         unsigned long timeout;
1192         u32 bit = enabled ? NVME_CSTS_RDY : 0;
1193
1194         timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1195
1196         while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1197                 msleep(100);
1198                 if (fatal_signal_pending(current))
1199                         return -EINTR;
1200                 if (time_after(jiffies, timeout)) {
1201                         dev_err(&dev->pci_dev->dev,
1202                                 "Device not ready; aborting initialisation\n");
1203                         return -ENODEV;
1204                 }
1205         }
1206
1207         return 0;
1208 }
1209
1210 /*
1211  * If the device has been passed off to us in an enabled state, just clear
1212  * the enabled bit.  The spec says we should set the 'shutdown notification
1213  * bits', but doing so may cause the device to complete commands to the
1214  * admin queue ... and we don't know what memory that might be pointing at!
1215  */
1216 static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1217 {
1218         u32 cc = readl(&dev->bar->cc);
1219
1220         if (cc & NVME_CC_ENABLE)
1221                 writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc);
1222         return nvme_wait_ready(dev, cap, false);
1223 }
1224
1225 static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1226 {
1227         return nvme_wait_ready(dev, cap, true);
1228 }
1229
1230 static int nvme_shutdown_ctrl(struct nvme_dev *dev)
1231 {
1232         unsigned long timeout;
1233         u32 cc;
1234
1235         cc = (readl(&dev->bar->cc) & ~NVME_CC_SHN_MASK) | NVME_CC_SHN_NORMAL;
1236         writel(cc, &dev->bar->cc);
1237
1238         timeout = 2 * HZ + jiffies;
1239         while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
1240                                                         NVME_CSTS_SHST_CMPLT) {
1241                 msleep(100);
1242                 if (fatal_signal_pending(current))
1243                         return -EINTR;
1244                 if (time_after(jiffies, timeout)) {
1245                         dev_err(&dev->pci_dev->dev,
1246                                 "Device shutdown incomplete; abort shutdown\n");
1247                         return -ENODEV;
1248                 }
1249         }
1250
1251         return 0;
1252 }
1253
1254 static int nvme_configure_admin_queue(struct nvme_dev *dev)
1255 {
1256         int result;
1257         u32 aqa;
1258         u64 cap = readq(&dev->bar->cap);
1259         struct nvme_queue *nvmeq;
1260
1261         result = nvme_disable_ctrl(dev, cap);
1262         if (result < 0)
1263                 return result;
1264
1265         nvmeq = dev->queues[0];
1266         if (!nvmeq) {
1267                 nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
1268                 if (!nvmeq)
1269                         return -ENOMEM;
1270                 dev->queues[0] = nvmeq;
1271         }
1272
1273         aqa = nvmeq->q_depth - 1;
1274         aqa |= aqa << 16;
1275
1276         dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1277         dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1278         dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1279         dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
1280
1281         writel(aqa, &dev->bar->aqa);
1282         writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1283         writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1284         writel(dev->ctrl_config, &dev->bar->cc);
1285
1286         result = nvme_enable_ctrl(dev, cap);
1287         if (result)
1288                 return result;
1289
1290         result = queue_request_irq(dev, nvmeq, "nvme admin");
1291         if (result)
1292                 return result;
1293
1294         spin_lock(&nvmeq->q_lock);
1295         nvme_init_queue(nvmeq, 0);
1296         spin_unlock(&nvmeq->q_lock);
1297         return result;
1298 }
1299
1300 struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
1301                                 unsigned long addr, unsigned length)
1302 {
1303         int i, err, count, nents, offset;
1304         struct scatterlist *sg;
1305         struct page **pages;
1306         struct nvme_iod *iod;
1307
1308         if (addr & 3)
1309                 return ERR_PTR(-EINVAL);
1310         if (!length || length > INT_MAX - PAGE_SIZE)
1311                 return ERR_PTR(-EINVAL);
1312
1313         offset = offset_in_page(addr);
1314         count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1315         pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
1316         if (!pages)
1317                 return ERR_PTR(-ENOMEM);
1318
1319         err = get_user_pages_fast(addr, count, 1, pages);
1320         if (err < count) {
1321                 count = err;
1322                 err = -EFAULT;
1323                 goto put_pages;
1324         }
1325
1326         iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1327         sg = iod->sg;
1328         sg_init_table(sg, count);
1329         for (i = 0; i < count; i++) {
1330                 sg_set_page(&sg[i], pages[i],
1331                             min_t(unsigned, length, PAGE_SIZE - offset),
1332                             offset);
1333                 length -= (PAGE_SIZE - offset);
1334                 offset = 0;
1335         }
1336         sg_mark_end(&sg[i - 1]);
1337         iod->nents = count;
1338
1339         err = -ENOMEM;
1340         nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1341                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1342         if (!nents)
1343                 goto free_iod;
1344
1345         kfree(pages);
1346         return iod;
1347
1348  free_iod:
1349         kfree(iod);
1350  put_pages:
1351         for (i = 0; i < count; i++)
1352                 put_page(pages[i]);
1353         kfree(pages);
1354         return ERR_PTR(err);
1355 }
1356
1357 void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1358                         struct nvme_iod *iod)
1359 {
1360         int i;
1361
1362         dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1363                                 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1364
1365         for (i = 0; i < iod->nents; i++)
1366                 put_page(sg_page(&iod->sg[i]));
1367 }
1368
1369 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1370 {
1371         struct nvme_dev *dev = ns->dev;
1372         struct nvme_queue *nvmeq;
1373         struct nvme_user_io io;
1374         struct nvme_command c;
1375         unsigned length, meta_len;
1376         int status, i;
1377         struct nvme_iod *iod, *meta_iod = NULL;
1378         dma_addr_t meta_dma_addr;
1379         void *meta, *uninitialized_var(meta_mem);
1380
1381         if (copy_from_user(&io, uio, sizeof(io)))
1382                 return -EFAULT;
1383         length = (io.nblocks + 1) << ns->lba_shift;
1384         meta_len = (io.nblocks + 1) * ns->ms;
1385
1386         if (meta_len && ((io.metadata & 3) || !io.metadata))
1387                 return -EINVAL;
1388
1389         switch (io.opcode) {
1390         case nvme_cmd_write:
1391         case nvme_cmd_read:
1392         case nvme_cmd_compare:
1393                 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
1394                 break;
1395         default:
1396                 return -EINVAL;
1397         }
1398
1399         if (IS_ERR(iod))
1400                 return PTR_ERR(iod);
1401
1402         memset(&c, 0, sizeof(c));
1403         c.rw.opcode = io.opcode;
1404         c.rw.flags = io.flags;
1405         c.rw.nsid = cpu_to_le32(ns->ns_id);
1406         c.rw.slba = cpu_to_le64(io.slba);
1407         c.rw.length = cpu_to_le16(io.nblocks);
1408         c.rw.control = cpu_to_le16(io.control);
1409         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1410         c.rw.reftag = cpu_to_le32(io.reftag);
1411         c.rw.apptag = cpu_to_le16(io.apptag);
1412         c.rw.appmask = cpu_to_le16(io.appmask);
1413
1414         if (meta_len) {
1415                 meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
1416                                                                 meta_len);
1417                 if (IS_ERR(meta_iod)) {
1418                         status = PTR_ERR(meta_iod);
1419                         meta_iod = NULL;
1420                         goto unmap;
1421                 }
1422
1423                 meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1424                                                 &meta_dma_addr, GFP_KERNEL);
1425                 if (!meta_mem) {
1426                         status = -ENOMEM;
1427                         goto unmap;
1428                 }
1429
1430                 if (io.opcode & 1) {
1431                         int meta_offset = 0;
1432
1433                         for (i = 0; i < meta_iod->nents; i++) {
1434                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1435                                                 meta_iod->sg[i].offset;
1436                                 memcpy(meta_mem + meta_offset, meta,
1437                                                 meta_iod->sg[i].length);
1438                                 kunmap_atomic(meta);
1439                                 meta_offset += meta_iod->sg[i].length;
1440                         }
1441                 }
1442
1443                 c.rw.metadata = cpu_to_le64(meta_dma_addr);
1444         }
1445
1446         length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
1447
1448         nvmeq = get_nvmeq(dev);
1449         /*
1450          * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
1451          * disabled.  We may be preempted at any point, and be rescheduled
1452          * to a different CPU.  That will cause cacheline bouncing, but no
1453          * additional races since q_lock already protects against other CPUs.
1454          */
1455         put_nvmeq(nvmeq);
1456         if (length != (io.nblocks + 1) << ns->lba_shift)
1457                 status = -ENOMEM;
1458         else if (!nvmeq || nvmeq->q_suspended)
1459                 status = -EBUSY;
1460         else
1461                 status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
1462
1463         if (meta_len) {
1464                 if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1465                         int meta_offset = 0;
1466
1467                         for (i = 0; i < meta_iod->nents; i++) {
1468                                 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1469                                                 meta_iod->sg[i].offset;
1470                                 memcpy(meta, meta_mem + meta_offset,
1471                                                 meta_iod->sg[i].length);
1472                                 kunmap_atomic(meta);
1473                                 meta_offset += meta_iod->sg[i].length;
1474                         }
1475                 }
1476
1477                 dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1478                                                                 meta_dma_addr);
1479         }
1480
1481  unmap:
1482         nvme_unmap_user_pages(dev, io.opcode & 1, iod);
1483         nvme_free_iod(dev, iod);
1484
1485         if (meta_iod) {
1486                 nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1487                 nvme_free_iod(dev, meta_iod);
1488         }
1489
1490         return status;
1491 }
1492
1493 static int nvme_user_admin_cmd(struct nvme_dev *dev,
1494                                         struct nvme_admin_cmd __user *ucmd)
1495 {
1496         struct nvme_admin_cmd cmd;
1497         struct nvme_command c;
1498         int status, length;
1499         struct nvme_iod *uninitialized_var(iod);
1500         unsigned timeout;
1501
1502         if (!capable(CAP_SYS_ADMIN))
1503                 return -EACCES;
1504         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1505                 return -EFAULT;
1506
1507         memset(&c, 0, sizeof(c));
1508         c.common.opcode = cmd.opcode;
1509         c.common.flags = cmd.flags;
1510         c.common.nsid = cpu_to_le32(cmd.nsid);
1511         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1512         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1513         c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1514         c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1515         c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1516         c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1517         c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1518         c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1519
1520         length = cmd.data_len;
1521         if (cmd.data_len) {
1522                 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1523                                                                 length);
1524                 if (IS_ERR(iod))
1525                         return PTR_ERR(iod);
1526                 length = nvme_setup_prps(dev, &c.common, iod, length,
1527                                                                 GFP_KERNEL);
1528         }
1529
1530         timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
1531                                                                 ADMIN_TIMEOUT;
1532         if (length != cmd.data_len)
1533                 status = -ENOMEM;
1534         else
1535                 status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result,
1536                                                                 timeout);
1537
1538         if (cmd.data_len) {
1539                 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
1540                 nvme_free_iod(dev, iod);
1541         }
1542
1543         if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
1544                                                         sizeof(cmd.result)))
1545                 status = -EFAULT;
1546
1547         return status;
1548 }
1549
1550 static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1551                                                         unsigned long arg)
1552 {
1553         struct nvme_ns *ns = bdev->bd_disk->private_data;
1554
1555         switch (cmd) {
1556         case NVME_IOCTL_ID:
1557                 force_successful_syscall_return();
1558                 return ns->ns_id;
1559         case NVME_IOCTL_ADMIN_CMD:
1560                 return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
1561         case NVME_IOCTL_SUBMIT_IO:
1562                 return nvme_submit_io(ns, (void __user *)arg);
1563         case SG_GET_VERSION_NUM:
1564                 return nvme_sg_get_version_num((void __user *)arg);
1565         case SG_IO:
1566                 return nvme_sg_io(ns, (void __user *)arg);
1567         default:
1568                 return -ENOTTY;
1569         }
1570 }
1571
1572 static const struct block_device_operations nvme_fops = {
1573         .owner          = THIS_MODULE,
1574         .ioctl          = nvme_ioctl,
1575         .compat_ioctl   = nvme_ioctl,
1576 };
1577
1578 static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
1579 {
1580         while (bio_list_peek(&nvmeq->sq_cong)) {
1581                 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1582                 struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
1583
1584                 if (bio_list_empty(&nvmeq->sq_cong))
1585                         remove_wait_queue(&nvmeq->sq_full,
1586                                                         &nvmeq->sq_cong_wait);
1587                 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
1588                         if (bio_list_empty(&nvmeq->sq_cong))
1589                                 add_wait_queue(&nvmeq->sq_full,
1590                                                         &nvmeq->sq_cong_wait);
1591                         bio_list_add_head(&nvmeq->sq_cong, bio);
1592                         break;
1593                 }
1594         }
1595 }
1596
1597 static int nvme_kthread(void *data)
1598 {
1599         struct nvme_dev *dev;
1600
1601         while (!kthread_should_stop()) {
1602                 set_current_state(TASK_INTERRUPTIBLE);
1603                 spin_lock(&dev_list_lock);
1604                 list_for_each_entry(dev, &dev_list, node) {
1605                         int i;
1606                         for (i = 0; i < dev->queue_count; i++) {
1607                                 struct nvme_queue *nvmeq = dev->queues[i];
1608                                 if (!nvmeq)
1609                                         continue;
1610                                 spin_lock_irq(&nvmeq->q_lock);
1611                                 if (nvmeq->q_suspended)
1612                                         goto unlock;
1613                                 nvme_process_cq(nvmeq);
1614                                 nvme_cancel_ios(nvmeq, true);
1615                                 nvme_resubmit_bios(nvmeq);
1616  unlock:
1617                                 spin_unlock_irq(&nvmeq->q_lock);
1618                         }
1619                 }
1620                 spin_unlock(&dev_list_lock);
1621                 schedule_timeout(round_jiffies_relative(HZ));
1622         }
1623         return 0;
1624 }
1625
1626 static DEFINE_IDA(nvme_index_ida);
1627
1628 static int nvme_get_ns_idx(void)
1629 {
1630         int index, error;
1631
1632         do {
1633                 if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
1634                         return -1;
1635
1636                 spin_lock(&dev_list_lock);
1637                 error = ida_get_new(&nvme_index_ida, &index);
1638                 spin_unlock(&dev_list_lock);
1639         } while (error == -EAGAIN);
1640
1641         if (error)
1642                 index = -1;
1643         return index;
1644 }
1645
1646 static void nvme_put_ns_idx(int index)
1647 {
1648         spin_lock(&dev_list_lock);
1649         ida_remove(&nvme_index_ida, index);
1650         spin_unlock(&dev_list_lock);
1651 }
1652
1653 static void nvme_config_discard(struct nvme_ns *ns)
1654 {
1655         u32 logical_block_size = queue_logical_block_size(ns->queue);
1656         ns->queue->limits.discard_zeroes_data = 0;
1657         ns->queue->limits.discard_alignment = logical_block_size;
1658         ns->queue->limits.discard_granularity = logical_block_size;
1659         ns->queue->limits.max_discard_sectors = 0xffffffff;
1660         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1661 }
1662
1663 static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid,
1664                         struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1665 {
1666         struct nvme_ns *ns;
1667         struct gendisk *disk;
1668         int lbaf;
1669
1670         if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1671                 return NULL;
1672
1673         ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1674         if (!ns)
1675                 return NULL;
1676         ns->queue = blk_alloc_queue(GFP_KERNEL);
1677         if (!ns->queue)
1678                 goto out_free_ns;
1679         ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
1680         queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1681         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1682         blk_queue_make_request(ns->queue, nvme_make_request);
1683         ns->dev = dev;
1684         ns->queue->queuedata = ns;
1685
1686         disk = alloc_disk(NVME_MINORS);
1687         if (!disk)
1688                 goto out_free_queue;
1689         ns->ns_id = nsid;
1690         ns->disk = disk;
1691         lbaf = id->flbas & 0xf;
1692         ns->lba_shift = id->lbaf[lbaf].ds;
1693         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1694         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1695         if (dev->max_hw_sectors)
1696                 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
1697
1698         disk->major = nvme_major;
1699         disk->minors = NVME_MINORS;
1700         disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
1701         disk->fops = &nvme_fops;
1702         disk->private_data = ns;
1703         disk->queue = ns->queue;
1704         disk->driverfs_dev = &dev->pci_dev->dev;
1705         sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
1706         set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1707
1708         if (dev->oncs & NVME_CTRL_ONCS_DSM)
1709                 nvme_config_discard(ns);
1710
1711         return ns;
1712
1713  out_free_queue:
1714         blk_cleanup_queue(ns->queue);
1715  out_free_ns:
1716         kfree(ns);
1717         return NULL;
1718 }
1719
1720 static void nvme_ns_free(struct nvme_ns *ns)
1721 {
1722         int index = ns->disk->first_minor / NVME_MINORS;
1723         put_disk(ns->disk);
1724         nvme_put_ns_idx(index);
1725         blk_cleanup_queue(ns->queue);
1726         kfree(ns);
1727 }
1728
1729 static int set_queue_count(struct nvme_dev *dev, int count)
1730 {
1731         int status;
1732         u32 result;
1733         u32 q_count = (count - 1) | ((count - 1) << 16);
1734
1735         status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
1736                                                                 &result);
1737         if (status)
1738                 return status < 0 ? -EIO : -EBUSY;
1739         return min(result & 0xffff, result >> 16) + 1;
1740 }
1741
1742 static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1743 {
1744         return 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3));
1745 }
1746
1747 static int nvme_setup_io_queues(struct nvme_dev *dev)
1748 {
1749         struct pci_dev *pdev = dev->pci_dev;
1750         int result, cpu, i, vecs, nr_io_queues, size, q_depth;
1751
1752         nr_io_queues = num_online_cpus();
1753         result = set_queue_count(dev, nr_io_queues);
1754         if (result < 0)
1755                 return result;
1756         if (result < nr_io_queues)
1757                 nr_io_queues = result;
1758
1759         size = db_bar_size(dev, nr_io_queues);
1760         if (size > 8192) {
1761                 iounmap(dev->bar);
1762                 do {
1763                         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1764                         if (dev->bar)
1765                                 break;
1766                         if (!--nr_io_queues)
1767                                 return -ENOMEM;
1768                         size = db_bar_size(dev, nr_io_queues);
1769                 } while (1);
1770                 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1771                 dev->queues[0]->q_db = dev->dbs;
1772         }
1773
1774         /* Deregister the admin queue's interrupt */
1775         free_irq(dev->entry[0].vector, dev->queues[0]);
1776
1777         vecs = nr_io_queues;
1778         for (i = 0; i < vecs; i++)
1779                 dev->entry[i].entry = i;
1780         for (;;) {
1781                 result = pci_enable_msix(pdev, dev->entry, vecs);
1782                 if (result <= 0)
1783                         break;
1784                 vecs = result;
1785         }
1786
1787         if (result < 0) {
1788                 vecs = nr_io_queues;
1789                 if (vecs > 32)
1790                         vecs = 32;
1791                 for (;;) {
1792                         result = pci_enable_msi_block(pdev, vecs);
1793                         if (result == 0) {
1794                                 for (i = 0; i < vecs; i++)
1795                                         dev->entry[i].vector = i + pdev->irq;
1796                                 break;
1797                         } else if (result < 0) {
1798                                 vecs = 1;
1799                                 break;
1800                         }
1801                         vecs = result;
1802                 }
1803         }
1804
1805         /*
1806          * Should investigate if there's a performance win from allocating
1807          * more queues than interrupt vectors; it might allow the submission
1808          * path to scale better, even if the receive path is limited by the
1809          * number of interrupts.
1810          */
1811         nr_io_queues = vecs;
1812
1813         result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1814         if (result) {
1815                 dev->queues[0]->q_suspended = 1;
1816                 goto free_queues;
1817         }
1818
1819         /* Free previously allocated queues that are no longer usable */
1820         spin_lock(&dev_list_lock);
1821         for (i = dev->queue_count - 1; i > nr_io_queues; i--) {
1822                 struct nvme_queue *nvmeq = dev->queues[i];
1823
1824                 spin_lock(&nvmeq->q_lock);
1825                 nvme_cancel_ios(nvmeq, false);
1826                 spin_unlock(&nvmeq->q_lock);
1827
1828                 nvme_free_queue(nvmeq);
1829                 dev->queue_count--;
1830                 dev->queues[i] = NULL;
1831         }
1832         spin_unlock(&dev_list_lock);
1833
1834         cpu = cpumask_first(cpu_online_mask);
1835         for (i = 0; i < nr_io_queues; i++) {
1836                 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1837                 cpu = cpumask_next(cpu, cpu_online_mask);
1838         }
1839
1840         q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1841                                                                 NVME_Q_DEPTH);
1842         for (i = dev->queue_count - 1; i < nr_io_queues; i++) {
1843                 dev->queues[i + 1] = nvme_alloc_queue(dev, i + 1, q_depth, i);
1844                 if (!dev->queues[i + 1]) {
1845                         result = -ENOMEM;
1846                         goto free_queues;
1847                 }
1848         }
1849
1850         for (; i < num_possible_cpus(); i++) {
1851                 int target = i % rounddown_pow_of_two(dev->queue_count - 1);
1852                 dev->queues[i + 1] = dev->queues[target + 1];
1853         }
1854
1855         for (i = 1; i < dev->queue_count; i++) {
1856                 result = nvme_create_queue(dev->queues[i], i);
1857                 if (result) {
1858                         for (--i; i > 0; i--)
1859                                 nvme_disable_queue(dev, i);
1860                         goto free_queues;
1861                 }
1862         }
1863
1864         return 0;
1865
1866  free_queues:
1867         nvme_free_queues(dev);
1868         return result;
1869 }
1870
1871 /*
1872  * Return: error value if an error occurred setting up the queues or calling
1873  * Identify Device.  0 if these succeeded, even if adding some of the
1874  * namespaces failed.  At the moment, these failures are silent.  TBD which
1875  * failures should be reported.
1876  */
1877 static int nvme_dev_add(struct nvme_dev *dev)
1878 {
1879         int res;
1880         unsigned nn, i;
1881         struct nvme_ns *ns;
1882         struct nvme_id_ctrl *ctrl;
1883         struct nvme_id_ns *id_ns;
1884         void *mem;
1885         dma_addr_t dma_addr;
1886         int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
1887
1888         mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
1889                                                                 GFP_KERNEL);
1890         if (!mem)
1891                 return -ENOMEM;
1892
1893         res = nvme_identify(dev, 0, 1, dma_addr);
1894         if (res) {
1895                 res = -EIO;
1896                 goto out;
1897         }
1898
1899         ctrl = mem;
1900         nn = le32_to_cpup(&ctrl->nn);
1901         dev->oncs = le16_to_cpup(&ctrl->oncs);
1902         memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1903         memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1904         memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
1905         if (ctrl->mdts)
1906                 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1907         if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) &&
1908                         (dev->pci_dev->device == 0x0953) && ctrl->vs[3])
1909                 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
1910
1911         id_ns = mem;
1912         for (i = 1; i <= nn; i++) {
1913                 res = nvme_identify(dev, i, 0, dma_addr);
1914                 if (res)
1915                         continue;
1916
1917                 if (id_ns->ncap == 0)
1918                         continue;
1919
1920                 res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
1921                                                         dma_addr + 4096, NULL);
1922                 if (res)
1923                         memset(mem + 4096, 0, 4096);
1924
1925                 ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
1926                 if (ns)
1927                         list_add_tail(&ns->list, &dev->namespaces);
1928         }
1929         list_for_each_entry(ns, &dev->namespaces, list)
1930                 add_disk(ns->disk);
1931         res = 0;
1932
1933  out:
1934         dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
1935         return res;
1936 }
1937
1938 static int nvme_dev_map(struct nvme_dev *dev)
1939 {
1940         int bars, result = -ENOMEM;
1941         struct pci_dev *pdev = dev->pci_dev;
1942
1943         if (pci_enable_device_mem(pdev))
1944                 return result;
1945
1946         dev->entry[0].vector = pdev->irq;
1947         pci_set_master(pdev);
1948         bars = pci_select_bars(pdev, IORESOURCE_MEM);
1949         if (pci_request_selected_regions(pdev, bars, "nvme"))
1950                 goto disable_pci;
1951
1952         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) &&
1953             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
1954                 goto disable;
1955
1956         pci_set_drvdata(pdev, dev);
1957         dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1958         if (!dev->bar)
1959                 goto disable;
1960
1961         dev->db_stride = NVME_CAP_STRIDE(readq(&dev->bar->cap));
1962         dev->dbs = ((void __iomem *)dev->bar) + 4096;
1963
1964         return 0;
1965
1966  disable:
1967         pci_release_regions(pdev);
1968  disable_pci:
1969         pci_disable_device(pdev);
1970         return result;
1971 }
1972
1973 static void nvme_dev_unmap(struct nvme_dev *dev)
1974 {
1975         if (dev->pci_dev->msi_enabled)
1976                 pci_disable_msi(dev->pci_dev);
1977         else if (dev->pci_dev->msix_enabled)
1978                 pci_disable_msix(dev->pci_dev);
1979
1980         if (dev->bar) {
1981                 iounmap(dev->bar);
1982                 dev->bar = NULL;
1983         }
1984
1985         pci_release_regions(dev->pci_dev);
1986         if (pci_is_enabled(dev->pci_dev))
1987                 pci_disable_device(dev->pci_dev);
1988 }
1989
1990 static void nvme_dev_shutdown(struct nvme_dev *dev)
1991 {
1992         int i;
1993
1994         for (i = dev->queue_count - 1; i >= 0; i--)
1995                 nvme_disable_queue(dev, i);
1996
1997         spin_lock(&dev_list_lock);
1998         list_del_init(&dev->node);
1999         spin_unlock(&dev_list_lock);
2000
2001         if (dev->bar)
2002                 nvme_shutdown_ctrl(dev);
2003         nvme_dev_unmap(dev);
2004 }
2005
2006 static void nvme_dev_remove(struct nvme_dev *dev)
2007 {
2008         struct nvme_ns *ns, *next;
2009
2010         list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2011                 list_del(&ns->list);
2012                 del_gendisk(ns->disk);
2013                 nvme_ns_free(ns);
2014         }
2015 }
2016
2017 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2018 {
2019         struct device *dmadev = &dev->pci_dev->dev;
2020         dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
2021                                                 PAGE_SIZE, PAGE_SIZE, 0);
2022         if (!dev->prp_page_pool)
2023                 return -ENOMEM;
2024
2025         /* Optimisation for I/Os between 4k and 128k */
2026         dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
2027                                                 256, 256, 0);
2028         if (!dev->prp_small_pool) {
2029                 dma_pool_destroy(dev->prp_page_pool);
2030                 return -ENOMEM;
2031         }
2032         return 0;
2033 }
2034
2035 static void nvme_release_prp_pools(struct nvme_dev *dev)
2036 {
2037         dma_pool_destroy(dev->prp_page_pool);
2038         dma_pool_destroy(dev->prp_small_pool);
2039 }
2040
2041 static DEFINE_IDA(nvme_instance_ida);
2042
2043 static int nvme_set_instance(struct nvme_dev *dev)
2044 {
2045         int instance, error;
2046
2047         do {
2048                 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2049                         return -ENODEV;
2050
2051                 spin_lock(&dev_list_lock);
2052                 error = ida_get_new(&nvme_instance_ida, &instance);
2053                 spin_unlock(&dev_list_lock);
2054         } while (error == -EAGAIN);
2055
2056         if (error)
2057                 return -ENODEV;
2058
2059         dev->instance = instance;
2060         return 0;
2061 }
2062
2063 static void nvme_release_instance(struct nvme_dev *dev)
2064 {
2065         spin_lock(&dev_list_lock);
2066         ida_remove(&nvme_instance_ida, dev->instance);
2067         spin_unlock(&dev_list_lock);
2068 }
2069
2070 static void nvme_free_dev(struct kref *kref)
2071 {
2072         struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2073         nvme_dev_remove(dev);
2074         nvme_dev_shutdown(dev);
2075         nvme_free_queues(dev);
2076         nvme_release_instance(dev);
2077         nvme_release_prp_pools(dev);
2078         kfree(dev->queues);
2079         kfree(dev->entry);
2080         kfree(dev);
2081 }
2082
2083 static int nvme_dev_open(struct inode *inode, struct file *f)
2084 {
2085         struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
2086                                                                 miscdev);
2087         kref_get(&dev->kref);
2088         f->private_data = dev;
2089         return 0;
2090 }
2091
2092 static int nvme_dev_release(struct inode *inode, struct file *f)
2093 {
2094         struct nvme_dev *dev = f->private_data;
2095         kref_put(&dev->kref, nvme_free_dev);
2096         return 0;
2097 }
2098
2099 static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2100 {
2101         struct nvme_dev *dev = f->private_data;
2102         switch (cmd) {
2103         case NVME_IOCTL_ADMIN_CMD:
2104                 return nvme_user_admin_cmd(dev, (void __user *)arg);
2105         default:
2106                 return -ENOTTY;
2107         }
2108 }
2109
2110 static const struct file_operations nvme_dev_fops = {
2111         .owner          = THIS_MODULE,
2112         .open           = nvme_dev_open,
2113         .release        = nvme_dev_release,
2114         .unlocked_ioctl = nvme_dev_ioctl,
2115         .compat_ioctl   = nvme_dev_ioctl,
2116 };
2117
2118 static int nvme_dev_start(struct nvme_dev *dev)
2119 {
2120         int result;
2121
2122         result = nvme_dev_map(dev);
2123         if (result)
2124                 return result;
2125
2126         result = nvme_configure_admin_queue(dev);
2127         if (result)
2128                 goto unmap;
2129
2130         spin_lock(&dev_list_lock);
2131         list_add(&dev->node, &dev_list);
2132         spin_unlock(&dev_list_lock);
2133
2134         result = nvme_setup_io_queues(dev);
2135         if (result && result != -EBUSY)
2136                 goto disable;
2137
2138         return result;
2139
2140  disable:
2141         spin_lock(&dev_list_lock);
2142         list_del_init(&dev->node);
2143         spin_unlock(&dev_list_lock);
2144  unmap:
2145         nvme_dev_unmap(dev);
2146         return result;
2147 }
2148
2149 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2150 {
2151         int result = -ENOMEM;
2152         struct nvme_dev *dev;
2153
2154         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2155         if (!dev)
2156                 return -ENOMEM;
2157         dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
2158                                                                 GFP_KERNEL);
2159         if (!dev->entry)
2160                 goto free;
2161         dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
2162                                                                 GFP_KERNEL);
2163         if (!dev->queues)
2164                 goto free;
2165
2166         INIT_LIST_HEAD(&dev->namespaces);
2167         dev->pci_dev = pdev;
2168
2169         result = nvme_set_instance(dev);
2170         if (result)
2171                 goto free;
2172
2173         result = nvme_setup_prp_pools(dev);
2174         if (result)
2175                 goto release;
2176
2177         result = nvme_dev_start(dev);
2178         if (result) {
2179                 if (result == -EBUSY)
2180                         goto create_cdev;
2181                 goto release_pools;
2182         }
2183
2184         result = nvme_dev_add(dev);
2185         if (result)
2186                 goto shutdown;
2187
2188  create_cdev:
2189         scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
2190         dev->miscdev.minor = MISC_DYNAMIC_MINOR;
2191         dev->miscdev.parent = &pdev->dev;
2192         dev->miscdev.name = dev->name;
2193         dev->miscdev.fops = &nvme_dev_fops;
2194         result = misc_register(&dev->miscdev);
2195         if (result)
2196                 goto remove;
2197
2198         kref_init(&dev->kref);
2199         return 0;
2200
2201  remove:
2202         nvme_dev_remove(dev);
2203  shutdown:
2204         nvme_dev_shutdown(dev);
2205  release_pools:
2206         nvme_free_queues(dev);
2207         nvme_release_prp_pools(dev);
2208  release:
2209         nvme_release_instance(dev);
2210  free:
2211         kfree(dev->queues);
2212         kfree(dev->entry);
2213         kfree(dev);
2214         return result;
2215 }
2216
2217 static void nvme_remove(struct pci_dev *pdev)
2218 {
2219         struct nvme_dev *dev = pci_get_drvdata(pdev);
2220         misc_deregister(&dev->miscdev);
2221         kref_put(&dev->kref, nvme_free_dev);
2222 }
2223
2224 /* These functions are yet to be implemented */
2225 #define nvme_error_detected NULL
2226 #define nvme_dump_registers NULL
2227 #define nvme_link_reset NULL
2228 #define nvme_slot_reset NULL
2229 #define nvme_error_resume NULL
2230
2231 static int nvme_suspend(struct device *dev)
2232 {
2233         struct pci_dev *pdev = to_pci_dev(dev);
2234         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2235
2236         nvme_dev_shutdown(ndev);
2237         return 0;
2238 }
2239
2240 static int nvme_resume(struct device *dev)
2241 {
2242         struct pci_dev *pdev = to_pci_dev(dev);
2243         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2244         int ret;
2245
2246         ret = nvme_dev_start(ndev);
2247         /* XXX: should remove gendisks if resume fails */
2248         if (ret)
2249                 nvme_free_queues(ndev);
2250         return ret;
2251 }
2252
2253 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2254
2255 static const struct pci_error_handlers nvme_err_handler = {
2256         .error_detected = nvme_error_detected,
2257         .mmio_enabled   = nvme_dump_registers,
2258         .link_reset     = nvme_link_reset,
2259         .slot_reset     = nvme_slot_reset,
2260         .resume         = nvme_error_resume,
2261 };
2262
2263 /* Move to pci_ids.h later */
2264 #define PCI_CLASS_STORAGE_EXPRESS       0x010802
2265
2266 static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
2267         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2268         { 0, }
2269 };
2270 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2271
2272 static struct pci_driver nvme_driver = {
2273         .name           = "nvme",
2274         .id_table       = nvme_id_table,
2275         .probe          = nvme_probe,
2276         .remove         = nvme_remove,
2277         .driver         = {
2278                 .pm     = &nvme_dev_pm_ops,
2279         },
2280         .err_handler    = &nvme_err_handler,
2281 };
2282
2283 static int __init nvme_init(void)
2284 {
2285         int result;
2286
2287         nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2288         if (IS_ERR(nvme_thread))
2289                 return PTR_ERR(nvme_thread);
2290
2291         result = register_blkdev(nvme_major, "nvme");
2292         if (result < 0)
2293                 goto kill_kthread;
2294         else if (result > 0)
2295                 nvme_major = result;
2296
2297         result = pci_register_driver(&nvme_driver);
2298         if (result)
2299                 goto unregister_blkdev;
2300         return 0;
2301
2302  unregister_blkdev:
2303         unregister_blkdev(nvme_major, "nvme");
2304  kill_kthread:
2305         kthread_stop(nvme_thread);
2306         return result;
2307 }
2308
2309 static void __exit nvme_exit(void)
2310 {
2311         pci_unregister_driver(&nvme_driver);
2312         unregister_blkdev(nvme_major, "nvme");
2313         kthread_stop(nvme_thread);
2314 }
2315
2316 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2317 MODULE_LICENSE("GPL");
2318 MODULE_VERSION("0.8");
2319 module_init(nvme_init);
2320 module_exit(nvme_exit);