1 #include <linux/module.h>
2 #include <linux/moduleparam.h>
3 #include <linux/sched.h>
5 #include <linux/blkdev.h>
6 #include <linux/init.h>
7 #include <linux/slab.h>
8 #include <linux/blk-mq.h>
9 #include <linux/hrtimer.h>
12 struct list_head list;
13 struct llist_node ll_list;
14 struct call_single_data csd;
18 struct nullb_queue *nq;
22 unsigned long *tag_map;
23 wait_queue_head_t wait;
24 unsigned int queue_depth;
26 struct nullb_cmd *cmds;
30 struct list_head list;
32 struct request_queue *q;
35 unsigned int queue_depth;
38 struct nullb_queue *queues;
39 unsigned int nr_queues;
42 static LIST_HEAD(nullb_list);
43 static struct mutex lock;
44 static int null_major;
45 static int nullb_indexes;
47 struct completion_queue {
48 struct llist_head list;
53 * These are per-cpu for now, they will need to be configured by the
54 * complete_queues parameter and appropriately mapped.
56 static DEFINE_PER_CPU(struct completion_queue, completion_queues);
68 static int submit_queues;
69 module_param(submit_queues, int, S_IRUGO);
70 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
72 static int home_node = NUMA_NO_NODE;
73 module_param(home_node, int, S_IRUGO);
74 MODULE_PARM_DESC(home_node, "Home node for the device");
76 static int queue_mode = NULL_Q_MQ;
77 module_param(queue_mode, int, S_IRUGO);
78 MODULE_PARM_DESC(use_mq, "Use blk-mq interface (0=bio,1=rq,2=multiqueue)");
81 module_param(gb, int, S_IRUGO);
82 MODULE_PARM_DESC(gb, "Size in GB");
85 module_param(bs, int, S_IRUGO);
86 MODULE_PARM_DESC(bs, "Block size (in bytes)");
88 static int nr_devices = 2;
89 module_param(nr_devices, int, S_IRUGO);
90 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
92 static int irqmode = NULL_IRQ_SOFTIRQ;
93 module_param(irqmode, int, S_IRUGO);
94 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
96 static int completion_nsec = 10000;
97 module_param(completion_nsec, int, S_IRUGO);
98 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
100 static int hw_queue_depth = 64;
101 module_param(hw_queue_depth, int, S_IRUGO);
102 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
104 static bool use_per_node_hctx = true;
105 module_param(use_per_node_hctx, bool, S_IRUGO);
106 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: true");
108 static void put_tag(struct nullb_queue *nq, unsigned int tag)
110 clear_bit_unlock(tag, nq->tag_map);
112 if (waitqueue_active(&nq->wait))
116 static unsigned int get_tag(struct nullb_queue *nq)
121 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
122 if (tag >= nq->queue_depth)
124 } while (test_and_set_bit_lock(tag, nq->tag_map));
129 static void free_cmd(struct nullb_cmd *cmd)
131 put_tag(cmd->nq, cmd->tag);
134 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
136 struct nullb_cmd *cmd;
141 cmd = &nq->cmds[tag];
150 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
152 struct nullb_cmd *cmd;
155 cmd = __alloc_cmd(nq);
156 if (cmd || !can_wait)
160 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
161 cmd = __alloc_cmd(nq);
168 finish_wait(&nq->wait, &wait);
172 static void end_cmd(struct nullb_cmd *cmd)
175 if (queue_mode == NULL_Q_MQ)
176 blk_mq_end_io(cmd->rq, 0);
178 INIT_LIST_HEAD(&cmd->rq->queuelist);
179 blk_end_request_all(cmd->rq, 0);
182 bio_endio(cmd->bio, 0);
184 if (queue_mode != NULL_Q_MQ)
188 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
190 struct completion_queue *cq;
191 struct llist_node *entry;
192 struct nullb_cmd *cmd;
194 cq = &per_cpu(completion_queues, smp_processor_id());
196 while ((entry = llist_del_all(&cq->list)) != NULL) {
198 cmd = container_of(entry, struct nullb_cmd, ll_list);
204 return HRTIMER_NORESTART;
207 static void null_cmd_end_timer(struct nullb_cmd *cmd)
209 struct completion_queue *cq = &per_cpu(completion_queues, get_cpu());
211 cmd->ll_list.next = NULL;
212 if (llist_add(&cmd->ll_list, &cq->list)) {
213 ktime_t kt = ktime_set(0, completion_nsec);
215 hrtimer_start(&cq->timer, kt, HRTIMER_MODE_REL);
221 static void null_softirq_done_fn(struct request *rq)
223 blk_end_request_all(rq, 0);
228 static void null_ipi_cmd_end_io(void *data)
230 struct completion_queue *cq;
231 struct llist_node *entry, *next;
232 struct nullb_cmd *cmd;
234 cq = &per_cpu(completion_queues, smp_processor_id());
236 entry = llist_del_all(&cq->list);
240 cmd = llist_entry(entry, struct nullb_cmd, ll_list);
246 static void null_cmd_end_ipi(struct nullb_cmd *cmd)
248 struct call_single_data *data = &cmd->csd;
250 struct completion_queue *cq = &per_cpu(completion_queues, cpu);
252 cmd->ll_list.next = NULL;
254 if (llist_add(&cmd->ll_list, &cq->list)) {
255 data->func = null_ipi_cmd_end_io;
257 __smp_call_function_single(cpu, data, 0);
263 #endif /* CONFIG_SMP */
265 static inline void null_handle_cmd(struct nullb_cmd *cmd)
267 /* Complete IO by inline, softirq or timer */
272 case NULL_IRQ_SOFTIRQ:
274 null_cmd_end_ipi(cmd);
280 null_cmd_end_timer(cmd);
285 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
289 if (nullb->nr_queues != 1)
290 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
292 return &nullb->queues[index];
295 static void null_queue_bio(struct request_queue *q, struct bio *bio)
297 struct nullb *nullb = q->queuedata;
298 struct nullb_queue *nq = nullb_to_queue(nullb);
299 struct nullb_cmd *cmd;
301 cmd = alloc_cmd(nq, 1);
304 null_handle_cmd(cmd);
307 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
309 struct nullb *nullb = q->queuedata;
310 struct nullb_queue *nq = nullb_to_queue(nullb);
311 struct nullb_cmd *cmd;
313 cmd = alloc_cmd(nq, 0);
320 return BLKPREP_DEFER;
323 static void null_request_fn(struct request_queue *q)
327 while ((rq = blk_fetch_request(q)) != NULL) {
328 struct nullb_cmd *cmd = rq->special;
330 spin_unlock_irq(q->queue_lock);
331 null_handle_cmd(cmd);
332 spin_lock_irq(q->queue_lock);
336 static int null_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq)
338 struct nullb_cmd *cmd = rq->special;
341 cmd->nq = hctx->driver_data;
343 null_handle_cmd(cmd);
344 return BLK_MQ_RQ_QUEUE_OK;
347 static struct blk_mq_hw_ctx *null_alloc_hctx(struct blk_mq_reg *reg, unsigned int hctx_index)
349 return kzalloc_node(sizeof(struct blk_mq_hw_ctx), GFP_KERNEL,
353 static void null_free_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_index)
358 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
363 init_waitqueue_head(&nq->wait);
364 nq->queue_depth = nullb->queue_depth;
367 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
370 struct nullb *nullb = data;
371 struct nullb_queue *nq = &nullb->queues[index];
373 hctx->driver_data = nq;
374 null_init_queue(nullb, nq);
380 static struct blk_mq_ops null_mq_ops = {
381 .queue_rq = null_queue_rq,
382 .map_queue = blk_mq_map_queue,
383 .init_hctx = null_init_hctx,
386 static struct blk_mq_reg null_mq_reg = {
389 .cmd_size = sizeof(struct nullb_cmd),
390 .flags = BLK_MQ_F_SHOULD_MERGE,
393 static void null_del_dev(struct nullb *nullb)
395 list_del_init(&nullb->list);
397 del_gendisk(nullb->disk);
398 if (queue_mode == NULL_Q_MQ)
399 blk_mq_free_queue(nullb->q);
401 blk_cleanup_queue(nullb->q);
402 put_disk(nullb->disk);
406 static int null_open(struct block_device *bdev, fmode_t mode)
411 static void null_release(struct gendisk *disk, fmode_t mode)
415 static const struct block_device_operations null_fops = {
416 .owner = THIS_MODULE,
418 .release = null_release,
421 static int setup_commands(struct nullb_queue *nq)
423 struct nullb_cmd *cmd;
426 nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
430 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
431 nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
437 for (i = 0; i < nq->queue_depth; i++) {
439 INIT_LIST_HEAD(&cmd->list);
440 cmd->ll_list.next = NULL;
447 static void cleanup_queue(struct nullb_queue *nq)
453 static void cleanup_queues(struct nullb *nullb)
457 for (i = 0; i < nullb->nr_queues; i++)
458 cleanup_queue(&nullb->queues[i]);
460 kfree(nullb->queues);
463 static int setup_queues(struct nullb *nullb)
465 nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
470 nullb->nr_queues = 0;
471 nullb->queue_depth = hw_queue_depth;
476 static int init_driver_queues(struct nullb *nullb)
478 struct nullb_queue *nq;
481 for (i = 0; i < submit_queues; i++) {
482 nq = &nullb->queues[i];
484 null_init_queue(nullb, nq);
486 ret = setup_commands(nq);
494 cleanup_queues(nullb);
498 static int null_add_dev(void)
500 struct gendisk *disk;
504 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
508 spin_lock_init(&nullb->lock);
510 if (setup_queues(nullb))
513 if (queue_mode == NULL_Q_MQ) {
514 null_mq_reg.numa_node = home_node;
515 null_mq_reg.queue_depth = hw_queue_depth;
516 null_mq_reg.nr_hw_queues = submit_queues;
518 if (use_per_node_hctx) {
519 null_mq_reg.ops->alloc_hctx = null_alloc_hctx;
520 null_mq_reg.ops->free_hctx = null_free_hctx;
522 null_mq_reg.ops->alloc_hctx = blk_mq_alloc_single_hw_queue;
523 null_mq_reg.ops->free_hctx = blk_mq_free_single_hw_queue;
526 nullb->q = blk_mq_init_queue(&null_mq_reg, nullb);
527 } else if (queue_mode == NULL_Q_BIO) {
528 nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
529 blk_queue_make_request(nullb->q, null_queue_bio);
530 init_driver_queues(nullb);
532 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
533 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
535 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
536 init_driver_queues(nullb);
542 nullb->q->queuedata = nullb;
543 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
545 disk = nullb->disk = alloc_disk_node(1, home_node);
548 if (queue_mode == NULL_Q_MQ)
549 blk_mq_free_queue(nullb->q);
551 blk_cleanup_queue(nullb->q);
552 cleanup_queues(nullb);
559 list_add_tail(&nullb->list, &nullb_list);
560 nullb->index = nullb_indexes++;
563 blk_queue_logical_block_size(nullb->q, bs);
564 blk_queue_physical_block_size(nullb->q, bs);
566 size = gb * 1024 * 1024 * 1024ULL;
567 sector_div(size, bs);
568 set_capacity(disk, size);
570 disk->flags |= GENHD_FL_EXT_DEVT;
571 disk->major = null_major;
572 disk->first_minor = nullb->index;
573 disk->fops = &null_fops;
574 disk->private_data = nullb;
575 disk->queue = nullb->q;
576 sprintf(disk->disk_name, "nullb%d", nullb->index);
581 static int __init null_init(void)
585 #if !defined(CONFIG_SMP)
586 if (irqmode == NULL_IRQ_SOFTIRQ) {
587 pr_warn("null_blk: softirq completions not available.\n");
588 pr_warn("null_blk: using direct completions.\n");
589 irqmode = NULL_IRQ_NONE;
593 if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
594 if (submit_queues > 0)
595 pr_warn("null_blk: submit_queues param is set to %u.",
597 submit_queues = nr_online_nodes;
598 } else if (submit_queues > nr_cpu_ids)
599 submit_queues = nr_cpu_ids;
600 else if (!submit_queues)
605 /* Initialize a separate list for each CPU for issuing softirqs */
606 for_each_possible_cpu(i) {
607 struct completion_queue *cq = &per_cpu(completion_queues, i);
609 init_llist_head(&cq->list);
611 if (irqmode != NULL_IRQ_TIMER)
614 hrtimer_init(&cq->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
615 cq->timer.function = null_cmd_timer_expired;
618 null_major = register_blkdev(0, "nullb");
622 for (i = 0; i < nr_devices; i++) {
623 if (null_add_dev()) {
624 unregister_blkdev(null_major, "nullb");
629 pr_info("null: module loaded\n");
633 static void __exit null_exit(void)
637 unregister_blkdev(null_major, "nullb");
640 while (!list_empty(&nullb_list)) {
641 nullb = list_entry(nullb_list.next, struct nullb, list);
647 module_init(null_init);
648 module_exit(null_exit);
650 MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
651 MODULE_LICENSE("GPL");