Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mason/linux...

[~andy/linux] / block / elevator.c

diff --git a/block/elevator.c b/block/elevator.c
index f016855a46b094628190f68e698ae21a936912e6..6a55d418896f5ceee0042da69c0177c495219cbc 100644 (file)
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -38,6 +38,7 @@
 #include <trace/events/block.h>
 
 #include "blk.h"
+#include "blk-cgroup.h"
 
 static DEFINE_SPINLOCK(elv_list_lock);
 static LIST_HEAD(elv_list);
@@ -121,15 +122,6 @@ static struct elevator_type *elevator_get(const char *name)
        return e;
 }
 
-static int elevator_init_queue(struct request_queue *q,
-                              struct elevator_queue *eq)
-{
-       eq->elevator_data = eq->type->ops.elevator_init_fn(q);
-       if (eq->elevator_data)
-               return 0;
-       return -ENOMEM;
-}
-
 static char chosen_elevator[ELV_NAME_MAX];
 
 static int __init elevator_setup(char *str)
@@ -188,7 +180,6 @@ static void elevator_release(struct kobject *kobj)
 int elevator_init(struct request_queue *q, char *name)
 {
        struct elevator_type *e = NULL;
-       struct elevator_queue *eq;
        int err;
 
        if (unlikely(q->elevator))
@@ -222,17 +213,16 @@ int elevator_init(struct request_queue *q, char *name)
                }
        }
 
-       eq = elevator_alloc(q, e);
-       if (!eq)
+       q->elevator = elevator_alloc(q, e);
+       if (!q->elevator)
                return -ENOMEM;
 
-       err = elevator_init_queue(q, eq);
+       err = e->ops.elevator_init_fn(q);
        if (err) {
-               kobject_put(&eq->kobj);
+               kobject_put(&q->elevator->kobj);
                return err;
        }
 
-       q->elevator = eq;
        return 0;
 }
 EXPORT_SYMBOL(elevator_init);
@@ -564,25 +554,6 @@ void elv_drain_elevator(struct request_queue *q)
        }
 }
 
-void elv_quiesce_start(struct request_queue *q)
-{
-       if (!q->elevator)
-               return;
-
-       spin_lock_irq(q->queue_lock);
-       queue_flag_set(QUEUE_FLAG_ELVSWITCH, q);
-       spin_unlock_irq(q->queue_lock);
-
-       blk_drain_queue(q, false);
-}
-
-void elv_quiesce_end(struct request_queue *q)
-{
-       spin_lock_irq(q->queue_lock);
-       queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
-       spin_unlock_irq(q->queue_lock);
-}
-
 void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 {
        trace_block_rq_insert(q, rq);
@@ -692,12 +663,13 @@ struct request *elv_former_request(struct request_queue *q, struct request *rq)
        return NULL;
 }
 
-int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
+int elv_set_request(struct request_queue *q, struct request *rq,
+                   struct bio *bio, gfp_t gfp_mask)
 {
        struct elevator_queue *e = q->elevator;
 
        if (e->type->ops.elevator_set_req_fn)
-               return e->type->ops.elevator_set_req_fn(q, rq, gfp_mask);
+               return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask);
        return 0;
 }
 
@@ -801,8 +773,9 @@ static struct kobj_type elv_ktype = {
        .release        = elevator_release,
 };
 
-int __elv_register_queue(struct request_queue *q, struct elevator_queue *e)
+int elv_register_queue(struct request_queue *q)
 {
+       struct elevator_queue *e = q->elevator;
        int error;
 
        error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
@@ -820,11 +793,6 @@ int __elv_register_queue(struct request_queue *q, struct elevator_queue *e)
        }
        return error;
 }
-
-int elv_register_queue(struct request_queue *q)
-{
-       return __elv_register_queue(q, q->elevator);
-}
 EXPORT_SYMBOL(elv_register_queue);
 
 void elv_unregister_queue(struct request_queue *q)
@@ -907,53 +875,60 @@ EXPORT_SYMBOL_GPL(elv_unregister);
  */
 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 {
-       struct elevator_queue *old_elevator, *e;
+       struct elevator_queue *old = q->elevator;
+       bool registered = old->registered;
        int err;
 
-       /* allocate new elevator */
-       e = elevator_alloc(q, new_e);
-       if (!e)
-               return -ENOMEM;
+       /*
+        * Turn on BYPASS and drain all requests w/ elevator private data.
+        * Block layer doesn't call into a quiesced elevator - all requests
+        * are directly put on the dispatch list without elevator data
+        * using INSERT_BACK.  All requests have SOFTBARRIER set and no
+        * merge happens either.
+        */
+       blk_queue_bypass_start(q);
+
+       /* unregister and clear all auxiliary data of the old elevator */
+       if (registered)
+               elv_unregister_queue(q);
+
+       spin_lock_irq(q->queue_lock);
+       ioc_clear_queue(q);
+       spin_unlock_irq(q->queue_lock);
 
-       err = elevator_init_queue(q, e);
+       /* allocate, init and register new elevator */
+       err = -ENOMEM;
+       q->elevator = elevator_alloc(q, new_e);
+       if (!q->elevator)
+               goto fail_init;
+
+       err = new_e->ops.elevator_init_fn(q);
        if (err) {
-               kobject_put(&e->kobj);
-               return err;
+               kobject_put(&q->elevator->kobj);
+               goto fail_init;
        }
 
-       /* turn on BYPASS and drain all requests w/ elevator private data */
-       elv_quiesce_start(q);
-
-       /* unregister old queue, register new one and kill old elevator */
-       if (q->elevator->registered) {
-               elv_unregister_queue(q);
-               err = __elv_register_queue(q, e);
+       if (registered) {
+               err = elv_register_queue(q);
                if (err)
                        goto fail_register;
        }
 
-       /* done, clear io_cq's, switch elevators and turn off BYPASS */
-       spin_lock_irq(q->queue_lock);
-       ioc_clear_queue(q);
-       old_elevator = q->elevator;
-       q->elevator = e;
-       spin_unlock_irq(q->queue_lock);
-
-       elevator_exit(old_elevator);
-       elv_quiesce_end(q);
+       /* done, kill the old one and finish */
+       elevator_exit(old);
+       blk_queue_bypass_end(q);
 
-       blk_add_trace_msg(q, "elv switch: %s", e->type->elevator_name);
+       blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
 
        return 0;
 
 fail_register:
-       /*
-        * switch failed, exit the new io scheduler and reattach the old
-        * one again (along with re-adding the sysfs dir)
-        */
-       elevator_exit(e);
+       elevator_exit(q->elevator);
+fail_init:
+       /* switch failed, restore and re-register old elevator */
+       q->elevator = old;
        elv_register_queue(q);
-       elv_quiesce_end(q);
+       blk_queue_bypass_end(q);
 
        return err;
 }