]> Pileus Git - ~andy/linux/blobdiff - fs/ext4/page-io.c
ext4: don't wait for extent conversion in ext4_punch_hole()
[~andy/linux] / fs / ext4 / page-io.c
index 5f20bc48104185e686727081a9debe855601182a..6ee5bd389405c7faf540c5780fa97f71498553d7 100644 (file)
@@ -58,18 +58,89 @@ void ext4_ioend_shutdown(struct inode *inode)
         * We need to make sure the work structure is finished being
         * used before we let the inode get destroyed.
         */
-       if (work_pending(&EXT4_I(inode)->i_unwritten_work))
-               cancel_work_sync(&EXT4_I(inode)->i_unwritten_work);
+       if (work_pending(&EXT4_I(inode)->i_rsv_conversion_work))
+               cancel_work_sync(&EXT4_I(inode)->i_rsv_conversion_work);
+       if (work_pending(&EXT4_I(inode)->i_unrsv_conversion_work))
+               cancel_work_sync(&EXT4_I(inode)->i_unrsv_conversion_work);
+}
+
+/*
+ * Print an buffer I/O error compatible with the fs/buffer.c.  This
+ * provides compatibility with dmesg scrapers that look for a specific
+ * buffer I/O error message.  We really need a unified error reporting
+ * structure to userspace ala Digital Unix's uerf system, but it's
+ * probably not going to happen in my lifetime, due to LKML politics...
+ */
+static void buffer_io_error(struct buffer_head *bh)
+{
+       char b[BDEVNAME_SIZE];
+       printk(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n",
+                       bdevname(bh->b_bdev, b),
+                       (unsigned long long)bh->b_blocknr);
+}
+
+static void ext4_finish_bio(struct bio *bio)
+{
+       int i;
+       int error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
+
+       for (i = 0; i < bio->bi_vcnt; i++) {
+               struct bio_vec *bvec = &bio->bi_io_vec[i];
+               struct page *page = bvec->bv_page;
+               struct buffer_head *bh, *head;
+               unsigned bio_start = bvec->bv_offset;
+               unsigned bio_end = bio_start + bvec->bv_len;
+               unsigned under_io = 0;
+               unsigned long flags;
+
+               if (!page)
+                       continue;
+
+               if (error) {
+                       SetPageError(page);
+                       set_bit(AS_EIO, &page->mapping->flags);
+               }
+               bh = head = page_buffers(page);
+               /*
+                * We check all buffers in the page under BH_Uptodate_Lock
+                * to avoid races with other end io clearing async_write flags
+                */
+               local_irq_save(flags);
+               bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
+               do {
+                       if (bh_offset(bh) < bio_start ||
+                           bh_offset(bh) + bh->b_size > bio_end) {
+                               if (buffer_async_write(bh))
+                                       under_io++;
+                               continue;
+                       }
+                       clear_buffer_async_write(bh);
+                       if (error)
+                               buffer_io_error(bh);
+               } while ((bh = bh->b_this_page) != head);
+               bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
+               local_irq_restore(flags);
+               if (!under_io)
+                       end_page_writeback(page);
+       }
 }
 
 static void ext4_release_io_end(ext4_io_end_t *io_end)
 {
+       struct bio *bio, *next_bio;
+
        BUG_ON(!list_empty(&io_end->list));
        BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
        WARN_ON(io_end->handle);
 
        if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count))
                wake_up_all(ext4_ioend_wq(io_end->inode));
+
+       for (bio = io_end->bio; bio; bio = next_bio) {
+               next_bio = bio->bi_private;
+               ext4_finish_bio(bio);
+               bio_put(bio);
+       }
        if (io_end->flag & EXT4_IO_END_DIRECT)
                inode_dio_done(io_end->inode);
        if (io_end->iocb)
@@ -87,7 +158,14 @@ static void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end)
                wake_up_all(ext4_ioend_wq(inode));
 }
 
-/* check a range of space and convert unwritten extents to written. */
+/*
+ * Check a range of space and convert unwritten extents to written. Note that
+ * we are protected from truncate touching same part of extent tree by the
+ * fact that truncate code waits for all DIO to finish (thus exclusion from
+ * direct IO is achieved) and also waits for PageWriteback bits. Thus we
+ * cannot get to ext4_ext_truncate() before all IOs overlapping that range are
+ * completed (happens from ext4_free_ioend()).
+ */
 static int ext4_end_io(ext4_io_end_t *io)
 {
        struct inode *inode = io->inode;
@@ -114,20 +192,17 @@ static int ext4_end_io(ext4_io_end_t *io)
        return ret;
 }
 
-static void dump_completed_IO(struct inode *inode)
+static void dump_completed_IO(struct inode *inode, struct list_head *head)
 {
 #ifdef EXT4FS_DEBUG
        struct list_head *cur, *before, *after;
        ext4_io_end_t *io, *io0, *io1;
 
-       if (list_empty(&EXT4_I(inode)->i_completed_io_list)) {
-               ext4_debug("inode %lu completed_io list is empty\n",
-                          inode->i_ino);
+       if (list_empty(head))
                return;
-       }
 
-       ext4_debug("Dump inode %lu completed_io list\n", inode->i_ino);
-       list_for_each_entry(io, &EXT4_I(inode)->i_completed_io_list, list) {
+       ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
+       list_for_each_entry(io, head, list) {
                cur = &io->list;
                before = cur->prev;
                io0 = container_of(before, ext4_io_end_t, list);
@@ -148,16 +223,23 @@ static void ext4_add_complete_io(ext4_io_end_t *io_end)
        unsigned long flags;
 
        BUG_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
-       wq = EXT4_SB(io_end->inode->i_sb)->dio_unwritten_wq;
-
        spin_lock_irqsave(&ei->i_completed_io_lock, flags);
-       if (list_empty(&ei->i_completed_io_list))
-               queue_work(wq, &ei->i_unwritten_work);
-       list_add_tail(&io_end->list, &ei->i_completed_io_list);
+       if (io_end->handle) {
+               wq = EXT4_SB(io_end->inode->i_sb)->rsv_conversion_wq;
+               if (list_empty(&ei->i_rsv_conversion_list))
+                       queue_work(wq, &ei->i_rsv_conversion_work);
+               list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
+       } else {
+               wq = EXT4_SB(io_end->inode->i_sb)->unrsv_conversion_wq;
+               if (list_empty(&ei->i_unrsv_conversion_list))
+                       queue_work(wq, &ei->i_unrsv_conversion_work);
+               list_add_tail(&io_end->list, &ei->i_unrsv_conversion_list);
+       }
        spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
 }
 
-static int ext4_do_flush_completed_IO(struct inode *inode)
+static int ext4_do_flush_completed_IO(struct inode *inode,
+                                     struct list_head *head)
 {
        ext4_io_end_t *io;
        struct list_head unwritten;
@@ -166,8 +248,8 @@ static int ext4_do_flush_completed_IO(struct inode *inode)
        int err, ret = 0;
 
        spin_lock_irqsave(&ei->i_completed_io_lock, flags);
-       dump_completed_IO(inode);
-       list_replace_init(&ei->i_completed_io_list, &unwritten);
+       dump_completed_IO(inode, head);
+       list_replace_init(head, &unwritten);
        spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
 
        while (!list_empty(&unwritten)) {
@@ -183,23 +265,20 @@ static int ext4_do_flush_completed_IO(struct inode *inode)
 }
 
 /*
- * work on completed aio dio IO, to convert unwritten extents to extents
+ * work on completed IO, to convert unwritten extents to extents
  */
-void ext4_end_io_work(struct work_struct *work)
+void ext4_end_io_rsv_work(struct work_struct *work)
 {
        struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
-                                                 i_unwritten_work);
-       ext4_do_flush_completed_IO(&ei->vfs_inode);
+                                                 i_rsv_conversion_work);
+       ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
 }
 
-int ext4_flush_unwritten_io(struct inode *inode)
+void ext4_end_io_unrsv_work(struct work_struct *work)
 {
-       int ret;
-       WARN_ON_ONCE(!mutex_is_locked(&inode->i_mutex) &&
-                    !(inode->i_state & I_FREEING));
-       ret = ext4_do_flush_completed_IO(inode);
-       ext4_unwritten_wait(inode);
-       return ret;
+       struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
+                                                 i_unrsv_conversion_work);
+       ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_unrsv_conversion_list);
 }
 
 ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags)
@@ -248,79 +327,31 @@ ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
        return io_end;
 }
 
-/*
- * Print an buffer I/O error compatible with the fs/buffer.c.  This
- * provides compatibility with dmesg scrapers that look for a specific
- * buffer I/O error message.  We really need a unified error reporting
- * structure to userspace ala Digital Unix's uerf system, but it's
- * probably not going to happen in my lifetime, due to LKML politics...
- */
-static void buffer_io_error(struct buffer_head *bh)
-{
-       char b[BDEVNAME_SIZE];
-       printk(KERN_ERR "Buffer I/O error on device %s, logical block %llu\n",
-                       bdevname(bh->b_bdev, b),
-                       (unsigned long long)bh->b_blocknr);
-}
-
 static void ext4_end_bio(struct bio *bio, int error)
 {
        ext4_io_end_t *io_end = bio->bi_private;
-       struct inode *inode;
-       int i;
-       int blocksize;
        sector_t bi_sector = bio->bi_sector;
 
        BUG_ON(!io_end);
-       inode = io_end->inode;
-       blocksize = 1 << inode->i_blkbits;
-       bio->bi_private = NULL;
        bio->bi_end_io = NULL;
        if (test_bit(BIO_UPTODATE, &bio->bi_flags))
                error = 0;
-       for (i = 0; i < bio->bi_vcnt; i++) {
-               struct bio_vec *bvec = &bio->bi_io_vec[i];
-               struct page *page = bvec->bv_page;
-               struct buffer_head *bh, *head;
-               unsigned bio_start = bvec->bv_offset;
-               unsigned bio_end = bio_start + bvec->bv_len;
-               unsigned under_io = 0;
-               unsigned long flags;
-
-               if (!page)
-                       continue;
 
-               if (error) {
-                       SetPageError(page);
-                       set_bit(AS_EIO, &page->mapping->flags);
-               }
-               bh = head = page_buffers(page);
+       if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
                /*
-                * We check all buffers in the page under BH_Uptodate_Lock
-                * to avoid races with other end io clearing async_write flags
+                * Link bio into list hanging from io_end. We have to do it
+                * atomically as bio completions can be racing against each
+                * other.
                 */
-               local_irq_save(flags);
-               bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
-               do {
-                       if (bh_offset(bh) < bio_start ||
-                           bh_offset(bh) + blocksize > bio_end) {
-                               if (buffer_async_write(bh))
-                                       under_io++;
-                               continue;
-                       }
-                       clear_buffer_async_write(bh);
-                       if (error)
-                               buffer_io_error(bh);
-               } while ((bh = bh->b_this_page) != head);
-               bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
-               local_irq_restore(flags);
-               if (!under_io)
-                       end_page_writeback(page);
+               bio->bi_private = xchg(&io_end->bio, bio);
+       } else {
+               ext4_finish_bio(bio);
+               bio_put(bio);
        }
-       bio_put(bio);
 
        if (error) {
-               io_end->flag |= EXT4_IO_END_ERROR;
+               struct inode *inode = io_end->inode;
+
                ext4_warning(inode->i_sb, "I/O error writing to inode %lu "
                             "(offset %llu size %ld starting block %llu)",
                             inode->i_ino,
@@ -329,7 +360,6 @@ static void ext4_end_bio(struct bio *bio, int error)
                             (unsigned long long)
                             bi_sector >> (inode->i_blkbits - 9));
        }
-
        ext4_put_io_end_defer(io_end);
 }