X-Git-Url: http://pileus.org/git/?a=blobdiff_plain;f=drivers%2Fblock%2Fnvme.c;h=a8549dff469147f132ef9ccfeadea841261e8c44;hb=b57ab0fada358357571f0eb448cdf2f144785321;hp=f44d6cd87ea266648b52e6604d2c16cb31c5bca4;hpb=7fc3cdabba75c2516b8b645eb0ca7907aea70415;p=~andy%2Flinux diff --git a/drivers/block/nvme.c b/drivers/block/nvme.c index f44d6cd87ea..a8549dff469 100644 --- a/drivers/block/nvme.c +++ b/drivers/block/nvme.c @@ -26,11 +26,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -40,10 +42,19 @@ #define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) #define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) #define NVME_MINORS 64 +#define IO_TIMEOUT (5 * HZ) +#define ADMIN_TIMEOUT (60 * HZ) static int nvme_major; module_param(nvme_major, int, 0); +static int use_threaded_interrupts; +module_param(use_threaded_interrupts, int, 0); + +static DEFINE_SPINLOCK(dev_list_lock); +static LIST_HEAD(dev_list); +static struct task_struct *nvme_thread; + /* * Represents an NVM Express device. Each nvme_dev is a PCI function. */ @@ -52,12 +63,17 @@ struct nvme_dev { struct nvme_queue **queues; u32 __iomem *dbs; struct pci_dev *pci_dev; + struct dma_pool *prp_page_pool; + struct dma_pool *prp_small_pool; int instance; int queue_count; u32 ctrl_config; struct msix_entry *entry; struct nvme_bar __iomem *bar; struct list_head namespaces; + char serial[20]; + char model[40]; + char firmware_rev[8]; }; /* @@ -80,12 +96,14 @@ struct nvme_ns { */ struct nvme_queue { struct device *q_dmadev; + struct nvme_dev *dev; spinlock_t q_lock; struct nvme_command *sq_cmds; volatile struct nvme_completion *cqes; dma_addr_t sq_dma_addr; dma_addr_t cq_dma_addr; wait_queue_head_t sq_full; + wait_queue_t sq_cong_wait; struct bio_list sq_cong; u32 __iomem *q_db; u16 q_depth; @@ -113,6 +131,16 @@ static inline void _nvme_check_size(void) BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); } +struct nvme_cmd_info { + unsigned long ctx; + unsigned long timeout; +}; + +static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) +{ + return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)]; +} + /** * alloc_cmdid - Allocate a Command ID * @param nvmeq The queue that will be used for this command @@ -125,10 +153,11 @@ static inline void _nvme_check_size(void) * Passing in a pointer that's not 4-byte aligned will cause a BUG. * We can change this if it becomes a problem. */ -static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler) +static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler, + unsigned timeout) { - int depth = nvmeq->q_depth; - unsigned long data = (unsigned long)ctx | handler; + int depth = nvmeq->q_depth - 1; + struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); int cmdid; BUG_ON((unsigned long)ctx & 3); @@ -139,37 +168,56 @@ static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler) return -EBUSY; } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); - nvmeq->cmdid_data[cmdid + BITS_TO_LONGS(depth)] = data; + info[cmdid].ctx = (unsigned long)ctx | handler; + info[cmdid].timeout = jiffies + timeout; return cmdid; } static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, - int handler) + int handler, unsigned timeout) { int cmdid; wait_event_killable(nvmeq->sq_full, - (cmdid = alloc_cmdid(nvmeq, ctx, handler)) >= 0); + (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0); return (cmdid < 0) ? -EINTR : cmdid; } /* If you need more than four handlers, you'll need to change how - * alloc_cmdid and nvme_process_cq work + * alloc_cmdid and nvme_process_cq work. Consider using a special + * CMD_CTX value instead, if that works for your situation. */ enum { sync_completion_id = 0, bio_completion_id, }; +/* Special values must be a multiple of 4, and less than 0x1000 */ +#define CMD_CTX_BASE (POISON_POINTER_DELTA + sync_completion_id) +#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) +#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) +#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) +#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) + static unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid) { unsigned long data; + struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); - data = nvmeq->cmdid_data[cmdid + BITS_TO_LONGS(nvmeq->q_depth)]; + if (cmdid >= nvmeq->q_depth) + return CMD_CTX_INVALID; + data = info[cmdid].ctx; + info[cmdid].ctx = CMD_CTX_COMPLETED; clear_bit(cmdid, nvmeq->cmdid_data); wake_up(&nvmeq->sq_full); return data; } +static void cancel_cmdid_data(struct nvme_queue *nvmeq, int cmdid) +{ + struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); + info[cmdid].ctx = CMD_CTX_CANCELLED; +} + static struct nvme_queue *get_nvmeq(struct nvme_ns *ns) { int qid, cpu = get_cpu(); @@ -200,58 +248,101 @@ static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) spin_lock_irqsave(&nvmeq->q_lock, flags); tail = nvmeq->sq_tail; memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); - writel(tail, nvmeq->q_db); if (++tail == nvmeq->q_depth) tail = 0; + writel(tail, nvmeq->q_db); nvmeq->sq_tail = tail; spin_unlock_irqrestore(&nvmeq->q_lock, flags); return 0; } -struct nvme_req_info { +struct nvme_prps { + int npages; + dma_addr_t first_dma; + __le64 *list[0]; +}; + +static void nvme_free_prps(struct nvme_dev *dev, struct nvme_prps *prps) +{ + const int last_prp = PAGE_SIZE / 8 - 1; + int i; + dma_addr_t prp_dma; + + if (!prps) + return; + + prp_dma = prps->first_dma; + + if (prps->npages == 0) + dma_pool_free(dev->prp_small_pool, prps->list[0], prp_dma); + for (i = 0; i < prps->npages; i++) { + __le64 *prp_list = prps->list[i]; + dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); + dma_pool_free(dev->prp_page_pool, prp_list, prp_dma); + prp_dma = next_prp_dma; + } + kfree(prps); +} + +struct nvme_bio { struct bio *bio; int nents; + struct nvme_prps *prps; struct scatterlist sg[0]; }; /* XXX: use a mempool */ -static struct nvme_req_info *alloc_info(unsigned nseg, gfp_t gfp) +static struct nvme_bio *alloc_nbio(unsigned nseg, gfp_t gfp) { - return kmalloc(sizeof(struct nvme_req_info) + + return kzalloc(sizeof(struct nvme_bio) + sizeof(struct scatterlist) * nseg, gfp); } -static void free_info(struct nvme_req_info *info) +static void free_nbio(struct nvme_queue *nvmeq, struct nvme_bio *nbio) { - kfree(info); + nvme_free_prps(nvmeq->dev, nbio->prps); + kfree(nbio); } static void bio_completion(struct nvme_queue *nvmeq, void *ctx, struct nvme_completion *cqe) { - struct nvme_req_info *info = ctx; - struct bio *bio = info->bio; + struct nvme_bio *nbio = ctx; + struct bio *bio = nbio->bio; u16 status = le16_to_cpup(&cqe->status) >> 1; - dma_unmap_sg(nvmeq->q_dmadev, info->sg, info->nents, + dma_unmap_sg(nvmeq->q_dmadev, nbio->sg, nbio->nents, bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); - free_info(info); - bio_endio(bio, status ? -EIO : 0); + free_nbio(nvmeq, nbio); + if (status) + bio_endio(bio, -EIO); + if (bio->bi_vcnt > bio->bi_idx) { + bio_list_add(&nvmeq->sq_cong, bio); + wake_up_process(nvme_thread); + } else { + bio_endio(bio, 0); + } } /* length is in bytes */ -static void nvme_setup_prps(struct nvme_common_command *cmd, +static struct nvme_prps *nvme_setup_prps(struct nvme_dev *dev, + struct nvme_common_command *cmd, struct scatterlist *sg, int length) { + struct dma_pool *pool; int dma_len = sg_dma_len(sg); u64 dma_addr = sg_dma_address(sg); int offset = offset_in_page(dma_addr); + __le64 *prp_list; + dma_addr_t prp_dma; + int nprps, npages, i, prp_page; + struct nvme_prps *prps = NULL; cmd->prp1 = cpu_to_le64(dma_addr); length -= (PAGE_SIZE - offset); if (length <= 0) - return; + return prps; dma_len -= (PAGE_SIZE - offset); if (dma_len) { @@ -264,50 +355,144 @@ static void nvme_setup_prps(struct nvme_common_command *cmd, if (length <= PAGE_SIZE) { cmd->prp2 = cpu_to_le64(dma_addr); - return; + return prps; + } + + nprps = DIV_ROUND_UP(length, PAGE_SIZE); + npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE); + prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, GFP_ATOMIC); + prp_page = 0; + if (nprps <= (256 / 8)) { + pool = dev->prp_small_pool; + prps->npages = 0; + } else { + pool = dev->prp_page_pool; + prps->npages = npages; } - /* XXX: support PRP lists */ + prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); + prps->list[prp_page++] = prp_list; + prps->first_dma = prp_dma; + cmd->prp2 = cpu_to_le64(prp_dma); + i = 0; + for (;;) { + if (i == PAGE_SIZE / 8 - 1) { + __le64 *old_prp_list = prp_list; + prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); + prps->list[prp_page++] = prp_list; + old_prp_list[i] = cpu_to_le64(prp_dma); + i = 0; + } + prp_list[i++] = cpu_to_le64(dma_addr); + dma_len -= PAGE_SIZE; + dma_addr += PAGE_SIZE; + length -= PAGE_SIZE; + if (length <= 0) + break; + if (dma_len > 0) + continue; + BUG_ON(dma_len < 0); + sg = sg_next(sg); + dma_addr = sg_dma_address(sg); + dma_len = sg_dma_len(sg); + } + + return prps; } -static int nvme_map_bio(struct device *dev, struct nvme_req_info *info, +/* NVMe scatterlists require no holes in the virtual address */ +#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \ + (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE)) + +static int nvme_map_bio(struct device *dev, struct nvme_bio *nbio, struct bio *bio, enum dma_data_direction dma_dir, int psegs) { - struct bio_vec *bvec; - struct scatterlist *sg = info->sg; - int i, nsegs; + struct bio_vec *bvec, *bvprv = NULL; + struct scatterlist *sg = NULL; + int i, old_idx, length = 0, nsegs = 0; - sg_init_table(sg, psegs); + sg_init_table(nbio->sg, psegs); + old_idx = bio->bi_idx; bio_for_each_segment(bvec, bio, i) { - sg_set_page(sg, bvec->bv_page, bvec->bv_len, bvec->bv_offset); - /* XXX: handle non-mergable here */ - nsegs++; + if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) { + sg->length += bvec->bv_len; + } else { + if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec)) + break; + sg = sg ? sg + 1 : nbio->sg; + sg_set_page(sg, bvec->bv_page, bvec->bv_len, + bvec->bv_offset); + nsegs++; + } + length += bvec->bv_len; + bvprv = bvec; + } + bio->bi_idx = i; + nbio->nents = nsegs; + sg_mark_end(sg); + if (dma_map_sg(dev, nbio->sg, nbio->nents, dma_dir) == 0) { + bio->bi_idx = old_idx; + return -ENOMEM; } - info->nents = nsegs; + return length; +} + +static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, + int cmdid) +{ + struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; + + memset(cmnd, 0, sizeof(*cmnd)); + cmnd->common.opcode = nvme_cmd_flush; + cmnd->common.command_id = cmdid; + cmnd->common.nsid = cpu_to_le32(ns->ns_id); - return dma_map_sg(dev, info->sg, info->nents, dma_dir); + if (++nvmeq->sq_tail == nvmeq->q_depth) + nvmeq->sq_tail = 0; + writel(nvmeq->sq_tail, nvmeq->q_db); + + return 0; +} + +static int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) +{ + int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, + sync_completion_id, IO_TIMEOUT); + if (unlikely(cmdid < 0)) + return cmdid; + + return nvme_submit_flush(nvmeq, ns, cmdid); } static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, struct bio *bio) { struct nvme_command *cmnd; - struct nvme_req_info *info; + struct nvme_bio *nbio; enum dma_data_direction dma_dir; - int cmdid; + int cmdid, length, result = -ENOMEM; u16 control; u32 dsmgmt; - unsigned long flags; int psegs = bio_phys_segments(ns->queue, bio); - info = alloc_info(psegs, GFP_NOIO); - if (!info) - goto congestion; - info->bio = bio; + if ((bio->bi_rw & REQ_FLUSH) && psegs) { + result = nvme_submit_flush_data(nvmeq, ns); + if (result) + return result; + } - cmdid = alloc_cmdid(nvmeq, info, bio_completion_id); + nbio = alloc_nbio(psegs, GFP_ATOMIC); + if (!nbio) + goto nomem; + nbio->bio = bio; + + result = -EBUSY; + cmdid = alloc_cmdid(nvmeq, nbio, bio_completion_id, IO_TIMEOUT); if (unlikely(cmdid < 0)) - goto free_info; + goto free_nbio; + + if ((bio->bi_rw & REQ_FLUSH) && !psegs) + return nvme_submit_flush(nvmeq, ns, cmdid); control = 0; if (bio->bi_rw & REQ_FUA) @@ -319,7 +504,6 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, if (bio->bi_rw & REQ_RAHEAD) dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; - spin_lock_irqsave(&nvmeq->q_lock, flags); cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; memset(cmnd, 0, sizeof(*cmnd)); @@ -331,29 +515,32 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, dma_dir = DMA_FROM_DEVICE; } - nvme_map_bio(nvmeq->q_dmadev, info, bio, dma_dir, psegs); + result = nvme_map_bio(nvmeq->q_dmadev, nbio, bio, dma_dir, psegs); + if (result < 0) + goto free_nbio; + length = result; - cmnd->rw.flags = 1; cmnd->rw.command_id = cmdid; cmnd->rw.nsid = cpu_to_le32(ns->ns_id); - nvme_setup_prps(&cmnd->common, info->sg, bio->bi_size); + nbio->prps = nvme_setup_prps(nvmeq->dev, &cmnd->common, nbio->sg, + length); cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); - cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1); + cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1); cmnd->rw.control = cpu_to_le16(control); cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); - writel(nvmeq->sq_tail, nvmeq->q_db); + bio->bi_sector += length >> 9; + if (++nvmeq->sq_tail == nvmeq->q_depth) nvmeq->sq_tail = 0; - - spin_unlock_irqrestore(&nvmeq->q_lock, flags); + writel(nvmeq->sq_tail, nvmeq->q_db); return 0; - free_info: - free_info(info); - congestion: - return -EBUSY; + free_nbio: + free_nbio(nvmeq, nbio); + nomem: + return result; } /* @@ -364,11 +551,18 @@ static int nvme_make_request(struct request_queue *q, struct bio *bio) { struct nvme_ns *ns = q->queuedata; struct nvme_queue *nvmeq = get_nvmeq(ns); - - if (nvme_submit_bio_queue(nvmeq, ns, bio)) { - blk_set_queue_congested(q, rw_is_sync(bio->bi_rw)); + int result = -EBUSY; + + spin_lock_irq(&nvmeq->q_lock); + if (bio_list_empty(&nvmeq->sq_cong)) + result = nvme_submit_bio_queue(nvmeq, ns, bio); + if (unlikely(result)) { + if (bio_list_empty(&nvmeq->sq_cong)) + add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait); bio_list_add(&nvmeq->sq_cong, bio); } + + spin_unlock_irq(&nvmeq->q_lock); put_nvmeq(nvmeq); return 0; @@ -384,6 +578,22 @@ static void sync_completion(struct nvme_queue *nvmeq, void *ctx, struct nvme_completion *cqe) { struct sync_cmd_info *cmdinfo = ctx; + if (unlikely((unsigned long)cmdinfo == CMD_CTX_CANCELLED)) + return; + if ((unsigned long)cmdinfo == CMD_CTX_FLUSH) + return; + if (unlikely((unsigned long)cmdinfo == CMD_CTX_COMPLETED)) { + dev_warn(nvmeq->q_dmadev, + "completed id %d twice on queue %d\n", + cqe->command_id, le16_to_cpup(&cqe->sq_id)); + return; + } + if (unlikely((unsigned long)cmdinfo == CMD_CTX_INVALID)) { + dev_warn(nvmeq->q_dmadev, + "invalid id %d completed on queue %d\n", + cqe->command_id, le16_to_cpup(&cqe->sq_id)); + return; + } cmdinfo->result = le32_to_cpup(&cqe->result); cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; wake_up_process(cmdinfo->task); @@ -441,15 +651,36 @@ static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) static irqreturn_t nvme_irq(int irq, void *data) { - return nvme_process_cq(data); + irqreturn_t result; + struct nvme_queue *nvmeq = data; + spin_lock(&nvmeq->q_lock); + result = nvme_process_cq(nvmeq); + spin_unlock(&nvmeq->q_lock); + return result; +} + +static irqreturn_t nvme_irq_check(int irq, void *data) +{ + struct nvme_queue *nvmeq = data; + struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head]; + if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase) + return IRQ_NONE; + return IRQ_WAKE_THREAD; +} + +static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) +{ + spin_lock_irq(&nvmeq->q_lock); + cancel_cmdid_data(nvmeq, cmdid); + spin_unlock_irq(&nvmeq->q_lock); } /* * Returns 0 on success. If the result is negative, it's a Linux error code; * if the result is positive, it's an NVM Express status code */ -static int nvme_submit_sync_cmd(struct nvme_queue *q, struct nvme_command *cmd, - u32 *result) +static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, + struct nvme_command *cmd, u32 *result, unsigned timeout) { int cmdid; struct sync_cmd_info cmdinfo; @@ -457,15 +688,21 @@ static int nvme_submit_sync_cmd(struct nvme_queue *q, struct nvme_command *cmd, cmdinfo.task = current; cmdinfo.status = -EINTR; - cmdid = alloc_cmdid_killable(q, &cmdinfo, sync_completion_id); + cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion_id, + timeout); if (cmdid < 0) return cmdid; cmd->common.command_id = cmdid; - set_current_state(TASK_UNINTERRUPTIBLE); - nvme_submit_cmd(q, cmd); + set_current_state(TASK_KILLABLE); + nvme_submit_cmd(nvmeq, cmd); schedule(); + if (cmdinfo.status == -EINTR) { + nvme_abort_command(nvmeq, cmdid); + return -EINTR; + } + if (result) *result = cmdinfo.result; @@ -475,7 +712,7 @@ static int nvme_submit_sync_cmd(struct nvme_queue *q, struct nvme_command *cmd, static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd, u32 *result) { - return nvme_submit_sync_cmd(dev->queues[0], cmd, result); + return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT); } static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) @@ -568,7 +805,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth, int vector) { struct device *dmadev = &dev->pci_dev->dev; - unsigned extra = (depth + BITS_TO_LONGS(depth)) * sizeof(long); + unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info)); struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL); if (!nvmeq) return NULL; @@ -585,10 +822,12 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, goto free_cqdma; nvmeq->q_dmadev = dmadev; + nvmeq->dev = dev; spin_lock_init(&nvmeq->q_lock); nvmeq->cq_head = 0; nvmeq->cq_phase = 1; init_waitqueue_head(&nvmeq->sq_full); + init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread); bio_list_init(&nvmeq->sq_cong); nvmeq->q_db = &dev->dbs[qid * 2]; nvmeq->q_depth = depth; @@ -607,6 +846,11 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq, const char *name) { + if (use_threaded_interrupts) + return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector, + nvme_irq_check, nvme_irq, + IRQF_DISABLED | IRQF_SHARED, + name, nvmeq); return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq, IRQF_DISABLED | IRQF_SHARED, name, nvmeq); } @@ -617,6 +861,9 @@ static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, int result; struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector); + if (!nvmeq) + return NULL; + result = adapter_alloc_cq(dev, qid, nvmeq); if (result < 0) goto free_nvmeq; @@ -653,6 +900,8 @@ static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) dev->dbs = ((void __iomem *)dev->bar) + 4096; nvmeq = nvme_alloc_queue(dev, 0, 64, 0); + if (!nvmeq) + return -ENOMEM; aqa = nvmeq->q_depth - 1; aqa |= aqa << 16; @@ -661,6 +910,7 @@ static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev) dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT; dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; + writel(0, &dev->bar->cc); writel(aqa, &dev->bar->aqa); writeq(nvmeq->sq_dma_addr, &dev->bar->asq); writeq(nvmeq->cq_dma_addr, &dev->bar->acq); @@ -746,13 +996,15 @@ static int nvme_submit_user_admin_command(struct nvme_dev *dev, { int err, nents; struct scatterlist *sg; + struct nvme_prps *prps; nents = nvme_map_user_pages(dev, 0, addr, length, &sg); if (nents < 0) return nents; - nvme_setup_prps(&cmd->common, sg, length); + prps = nvme_setup_prps(dev, &cmd->common, sg, length); err = nvme_submit_admin_cmd(dev, cmd, NULL); nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); + nvme_free_prps(dev, prps); return err ? -EIO : 0; } @@ -780,6 +1032,97 @@ static int nvme_get_range_type(struct nvme_ns *ns, unsigned long addr) return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c); } +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +{ + struct nvme_dev *dev = ns->dev; + struct nvme_queue *nvmeq; + struct nvme_user_io io; + struct nvme_command c; + unsigned length; + u32 result; + int nents, status; + struct scatterlist *sg; + struct nvme_prps *prps; + + if (copy_from_user(&io, uio, sizeof(io))) + return -EFAULT; + length = io.nblocks << io.block_shift; + nents = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length, &sg); + if (nents < 0) + return nents; + + memset(&c, 0, sizeof(c)); + c.rw.opcode = io.opcode; + c.rw.flags = io.flags; + c.rw.nsid = cpu_to_le32(io.nsid); + c.rw.slba = cpu_to_le64(io.slba); + c.rw.length = cpu_to_le16(io.nblocks - 1); + c.rw.control = cpu_to_le16(io.control); + c.rw.dsmgmt = cpu_to_le16(io.dsmgmt); + c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */ + c.rw.apptag = cpu_to_le16(io.apptag); + c.rw.appmask = cpu_to_le16(io.appmask); + /* XXX: metadata */ + prps = nvme_setup_prps(dev, &c.common, sg, length); + + nvmeq = get_nvmeq(ns); + /* Since nvme_submit_sync_cmd sleeps, we can't keep preemption + * disabled. We may be preempted at any point, and be rescheduled + * to a different CPU. That will cause cacheline bouncing, but no + * additional races since q_lock already protects against other CPUs. + */ + put_nvmeq(nvmeq); + status = nvme_submit_sync_cmd(nvmeq, &c, &result, IO_TIMEOUT); + + nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); + nvme_free_prps(dev, prps); + put_user(result, &uio->result); + return status; +} + +static int nvme_download_firmware(struct nvme_ns *ns, + struct nvme_dlfw __user *udlfw) +{ + struct nvme_dev *dev = ns->dev; + struct nvme_dlfw dlfw; + struct nvme_command c; + int nents, status; + struct scatterlist *sg; + struct nvme_prps *prps; + + if (copy_from_user(&dlfw, udlfw, sizeof(dlfw))) + return -EFAULT; + if (dlfw.length >= (1 << 30)) + return -EINVAL; + + nents = nvme_map_user_pages(dev, 1, dlfw.addr, dlfw.length * 4, &sg); + if (nents < 0) + return nents; + + memset(&c, 0, sizeof(c)); + c.dlfw.opcode = nvme_admin_download_fw; + c.dlfw.numd = cpu_to_le32(dlfw.length); + c.dlfw.offset = cpu_to_le32(dlfw.offset); + prps = nvme_setup_prps(dev, &c.common, sg, dlfw.length * 4); + + status = nvme_submit_admin_cmd(dev, &c, NULL); + nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents); + nvme_free_prps(dev, prps); + return status; +} + +static int nvme_activate_firmware(struct nvme_ns *ns, unsigned long arg) +{ + struct nvme_dev *dev = ns->dev; + struct nvme_command c; + + memset(&c, 0, sizeof(c)); + c.common.opcode = nvme_admin_activate_fw; + c.common.rsvd10[0] = cpu_to_le32(arg); + + return nvme_submit_admin_cmd(dev, &c, NULL); +} + static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { @@ -792,6 +1135,12 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, return nvme_identify(ns, arg, 1); case NVME_IOCTL_GET_RANGE_TYPE: return nvme_get_range_type(ns, arg); + case NVME_IOCTL_SUBMIT_IO: + return nvme_submit_io(ns, (void __user *)arg); + case NVME_IOCTL_DOWNLOAD_FW: + return nvme_download_firmware(ns, (void __user *)arg); + case NVME_IOCTL_ACTIVATE_FW: + return nvme_activate_firmware(ns, arg); default: return -ENOTTY; } @@ -802,6 +1151,45 @@ static const struct block_device_operations nvme_fops = { .ioctl = nvme_ioctl, }; +static void nvme_resubmit_bios(struct nvme_queue *nvmeq) +{ + while (bio_list_peek(&nvmeq->sq_cong)) { + struct bio *bio = bio_list_pop(&nvmeq->sq_cong); + struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data; + if (nvme_submit_bio_queue(nvmeq, ns, bio)) { + bio_list_add_head(&nvmeq->sq_cong, bio); + break; + } + } +} + +static int nvme_kthread(void *data) +{ + struct nvme_dev *dev; + + while (!kthread_should_stop()) { + __set_current_state(TASK_RUNNING); + spin_lock(&dev_list_lock); + list_for_each_entry(dev, &dev_list, node) { + int i; + for (i = 0; i < dev->queue_count; i++) { + struct nvme_queue *nvmeq = dev->queues[i]; + if (!nvmeq) + continue; + spin_lock_irq(&nvmeq->q_lock); + if (nvme_process_cq(nvmeq)) + printk("process_cq did something\n"); + nvme_resubmit_bios(nvmeq); + spin_unlock_irq(&nvmeq->q_lock); + } + } + spin_unlock(&dev_list_lock); + set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(HZ); + } + return 0; +} + static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int index, struct nvme_id_ns *id, struct nvme_lba_range_type *rt) { @@ -838,6 +1226,7 @@ static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int index, disk->fops = &nvme_fops; disk->private_data = ns; disk->queue = ns->queue; + disk->driverfs_dev = &dev->pci_dev->dev; sprintf(disk->disk_name, "nvme%dn%d", dev->instance, index); set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); @@ -877,29 +1266,30 @@ static int set_queue_count(struct nvme_dev *dev, int count) static int __devinit nvme_setup_io_queues(struct nvme_dev *dev) { - int result, cpu, i, nr_queues; + int result, cpu, i, nr_io_queues; - nr_queues = num_online_cpus(); - result = set_queue_count(dev, nr_queues); + nr_io_queues = num_online_cpus(); + result = set_queue_count(dev, nr_io_queues); if (result < 0) return result; - if (result < nr_queues) - nr_queues = result; + if (result < nr_io_queues) + nr_io_queues = result; /* Deregister the admin queue's interrupt */ free_irq(dev->entry[0].vector, dev->queues[0]); - for (i = 0; i < nr_queues; i++) + for (i = 0; i < nr_io_queues; i++) dev->entry[i].entry = i; for (;;) { - result = pci_enable_msix(dev->pci_dev, dev->entry, nr_queues); + result = pci_enable_msix(dev->pci_dev, dev->entry, + nr_io_queues); if (result == 0) { break; } else if (result > 0) { - nr_queues = result; + nr_io_queues = result; continue; } else { - nr_queues = 1; + nr_io_queues = 1; break; } } @@ -908,12 +1298,12 @@ static int __devinit nvme_setup_io_queues(struct nvme_dev *dev) /* XXX: handle failure here */ cpu = cpumask_first(cpu_online_mask); - for (i = 0; i < nr_queues; i++) { + for (i = 0; i < nr_io_queues; i++) { irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu)); cpu = cpumask_next(cpu, cpu_online_mask); } - for (i = 0; i < nr_queues; i++) { + for (i = 0; i < nr_io_queues; i++) { dev->queues[i + 1] = nvme_create_queue(dev, i + 1, NVME_Q_DEPTH, i); if (!dev->queues[i + 1]) @@ -936,6 +1326,7 @@ static int __devinit nvme_dev_add(struct nvme_dev *dev) { int res, nn, i; struct nvme_ns *ns, *next; + struct nvme_id_ctrl *ctrl; void *id; dma_addr_t dma_addr; struct nvme_command cid, crt; @@ -960,7 +1351,11 @@ static int __devinit nvme_dev_add(struct nvme_dev *dev) goto out_free; } - nn = le32_to_cpup(&((struct nvme_id_ctrl *)id)->nn); + ctrl = id; + nn = le32_to_cpup(&ctrl->nn); + memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn)); + memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn)); + memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr)); cid.identify.cns = 0; memset(&crt, 0, sizeof(crt)); @@ -1006,6 +1401,10 @@ static int nvme_dev_remove(struct nvme_dev *dev) { struct nvme_ns *ns, *next; + spin_lock(&dev_list_lock); + list_del(&dev->node); + spin_unlock(&dev_list_lock); + /* TODO: wait all I/O finished or cancel them */ list_for_each_entry_safe(ns, next, &dev->namespaces, list) { @@ -1019,6 +1418,30 @@ static int nvme_dev_remove(struct nvme_dev *dev) return 0; } +static int nvme_setup_prp_pools(struct nvme_dev *dev) +{ + struct device *dmadev = &dev->pci_dev->dev; + dev->prp_page_pool = dma_pool_create("prp list page", dmadev, + PAGE_SIZE, PAGE_SIZE, 0); + if (!dev->prp_page_pool) + return -ENOMEM; + + /* Optimisation for I/Os between 4k and 128k */ + dev->prp_small_pool = dma_pool_create("prp list 256", dmadev, + 256, 256, 0); + if (!dev->prp_small_pool) { + dma_pool_destroy(dev->prp_page_pool); + return -ENOMEM; + } + return 0; +} + +static void nvme_release_prp_pools(struct nvme_dev *dev) +{ + dma_pool_destroy(dev->prp_page_pool); + dma_pool_destroy(dev->prp_small_pool); +} + /* XXX: Use an ida or something to let remove / add work correctly */ static void nvme_set_instance(struct nvme_dev *dev) { @@ -1033,7 +1456,7 @@ static void nvme_release_instance(struct nvme_dev *dev) static int __devinit nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - int result = -ENOMEM; + int bars, result = -ENOMEM; struct nvme_dev *dev; dev = kzalloc(sizeof(*dev), GFP_KERNEL); @@ -1048,17 +1471,29 @@ static int __devinit nvme_probe(struct pci_dev *pdev, if (!dev->queues) goto free; + if (pci_enable_device_mem(pdev)) + goto free; + pci_set_master(pdev); + bars = pci_select_bars(pdev, IORESOURCE_MEM); + if (pci_request_selected_regions(pdev, bars, "nvme")) + goto disable; + INIT_LIST_HEAD(&dev->namespaces); dev->pci_dev = pdev; pci_set_drvdata(pdev, dev); - dma_set_mask(&dev->pci_dev->dev, DMA_BIT_MASK(64)); + dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); + dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); nvme_set_instance(dev); dev->entry[0].vector = pdev->irq; + result = nvme_setup_prp_pools(dev); + if (result) + goto disable_msix; + dev->bar = ioremap(pci_resource_start(pdev, 0), 8192); if (!dev->bar) { result = -ENOMEM; - goto disable; + goto disable_msix; } result = nvme_configure_admin_queue(dev); @@ -1066,18 +1501,31 @@ static int __devinit nvme_probe(struct pci_dev *pdev, goto unmap; dev->queue_count++; + spin_lock(&dev_list_lock); + list_add(&dev->node, &dev_list); + spin_unlock(&dev_list_lock); + result = nvme_dev_add(dev); if (result) goto delete; + return 0; delete: + spin_lock(&dev_list_lock); + list_del(&dev->node); + spin_unlock(&dev_list_lock); + nvme_free_queues(dev); unmap: iounmap(dev->bar); - disable: + disable_msix: pci_disable_msix(pdev); nvme_release_instance(dev); + nvme_release_prp_pools(dev); + disable: + pci_disable_device(pdev); + pci_release_regions(pdev); free: kfree(dev->queues); kfree(dev->entry); @@ -1092,6 +1540,9 @@ static void __devexit nvme_remove(struct pci_dev *pdev) pci_disable_msix(pdev); iounmap(dev->bar); nvme_release_instance(dev); + nvme_release_prp_pools(dev); + pci_disable_device(pdev); + pci_release_regions(pdev); kfree(dev->queues); kfree(dev->entry); kfree(dev); @@ -1135,17 +1586,25 @@ static struct pci_driver nvme_driver = { static int __init nvme_init(void) { - int result; + int result = -EBUSY; + + nvme_thread = kthread_run(nvme_kthread, NULL, "nvme"); + if (IS_ERR(nvme_thread)) + return PTR_ERR(nvme_thread); nvme_major = register_blkdev(nvme_major, "nvme"); if (nvme_major <= 0) - return -EBUSY; + goto kill_kthread; result = pci_register_driver(&nvme_driver); - if (!result) - return 0; + if (result) + goto unregister_blkdev; + return 0; + unregister_blkdev: unregister_blkdev(nvme_major, "nvme"); + kill_kthread: + kthread_stop(nvme_thread); return result; } @@ -1153,10 +1612,11 @@ static void __exit nvme_exit(void) { pci_unregister_driver(&nvme_driver); unregister_blkdev(nvme_major, "nvme"); + kthread_stop(nvme_thread); } MODULE_AUTHOR("Matthew Wilcox "); MODULE_LICENSE("GPL"); -MODULE_VERSION("0.1"); +MODULE_VERSION("0.4"); module_init(nvme_init); module_exit(nvme_exit);