block: Generic bio chaining

[~andy/linux] / fs / bio.c

diff --git a/fs/bio.c b/fs/bio.c
index e6dfa06773ac6ae7d4f7265c271a94196eafdab3..b0a16dbc71efba708d9922fc4258e6b95d1e3068 100644 (file)
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -273,6 +273,7 @@ void bio_init(struct bio *bio)
 {
        memset(bio, 0, sizeof(*bio));
        bio->bi_flags = 1 << BIO_UPTODATE;
+       atomic_set(&bio->bi_remaining, 1);
        atomic_set(&bio->bi_cnt, 1);
 }
 EXPORT_SYMBOL(bio_init);
@@ -295,9 +296,35 @@ void bio_reset(struct bio *bio)
 
        memset(bio, 0, BIO_RESET_BYTES);
        bio->bi_flags = flags|(1 << BIO_UPTODATE);
+       atomic_set(&bio->bi_remaining, 1);
 }
 EXPORT_SYMBOL(bio_reset);
 
+static void bio_chain_endio(struct bio *bio, int error)
+{
+       bio_endio(bio->bi_private, error);
+       bio_put(bio);
+}
+
+/**
+ * bio_chain - chain bio completions
+ *
+ * The caller won't have a bi_end_io called when @bio completes - instead,
+ * @parent's bi_end_io won't be called until both @parent and @bio have
+ * completed; the chained bio will also be freed when it completes.
+ *
+ * The caller must not set bi_private or bi_end_io in @bio.
+ */
+void bio_chain(struct bio *bio, struct bio *parent)
+{
+       BUG_ON(bio->bi_private || bio->bi_end_io);
+
+       bio->bi_private = parent;
+       bio->bi_end_io  = bio_chain_endio;
+       atomic_inc(&parent->bi_remaining);
+}
+EXPORT_SYMBOL(bio_chain);
+
 static void bio_alloc_rescue(struct work_struct *work)
 {
        struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
@@ -1719,16 +1746,53 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
  **/
 void bio_endio(struct bio *bio, int error)
 {
-       if (error)
-               clear_bit(BIO_UPTODATE, &bio->bi_flags);
-       else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
-               error = -EIO;
+       while (bio) {
+               BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
+
+               if (error)
+                       clear_bit(BIO_UPTODATE, &bio->bi_flags);
+               else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
+                       error = -EIO;
+
+               if (!atomic_dec_and_test(&bio->bi_remaining))
+                       return;
 
-       if (bio->bi_end_io)
-               bio->bi_end_io(bio, error);
+               /*
+                * Need to have a real endio function for chained bios,
+                * otherwise various corner cases will break (like stacking
+                * block devices that save/restore bi_end_io) - however, we want
+                * to avoid unbounded recursion and blowing the stack. Tail call
+                * optimization would handle this, but compiling with frame
+                * pointers also disables gcc's sibling call optimization.
+                */
+               if (bio->bi_end_io == bio_chain_endio) {
+                       struct bio *parent = bio->bi_private;
+                       bio_put(bio);
+                       bio = parent;
+               } else {
+                       if (bio->bi_end_io)
+                               bio->bi_end_io(bio, error);
+                       bio = NULL;
+               }
+       }
 }
 EXPORT_SYMBOL(bio_endio);
 
+/**
+ * bio_endio_nodec - end I/O on a bio, without decrementing bi_remaining
+ * @bio:       bio
+ * @error:     error, if any
+ *
+ * For code that has saved and restored bi_end_io; thing hard before using this
+ * function, probably you should've cloned the entire bio.
+ **/
+void bio_endio_nodec(struct bio *bio, int error)
+{
+       atomic_inc(&bio->bi_remaining);
+       bio_endio(bio, error);
+}
+EXPORT_SYMBOL(bio_endio_nodec);
+
 void bio_pair_release(struct bio_pair *bp)
 {
        if (atomic_dec_and_test(&bp->cnt)) {