From: Zach Brown Date: Fri, 14 Jun 2013 20:54:19 +0000 (-0500) Subject: loop: use aio to perform io on the underlying file X-Git-Tag: next-20130822~107^2~16 X-Git-Url: https://git.karo-electronics.de/?a=commitdiff_plain;h=60360803f553be66708fd06565d9b45388a3b33e;p=karo-tx-linux.git loop: use aio to perform io on the underlying file This uses the new kernel aio interface to process loopback IO by submitting concurrent direct aio. Previously loop's IO was serialized by synchronous processing in a thread. The aio operations specify the memory for the IO with the bio_vec arrays directly instead of mappings of the pages. The use of aio operations is enabled when the backing file supports the read_iter, write_iter and direct_IO methods. Signed-off-by: Dave Kleikamp Cc: Zach Brown --- diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 40e715531aa6..cef47ce077a9 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -75,6 +75,7 @@ #include #include #include +#include #include "loop.h" #include @@ -218,6 +219,48 @@ lo_do_transfer(struct loop_device *lo, int cmd, return lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); } +#ifdef CONFIG_AIO +static void lo_rw_aio_complete(u64 data, long res) +{ + struct bio *bio = (struct bio *)(uintptr_t)data; + + if (res > 0) + res = 0; + else if (res < 0) + res = -EIO; + + bio_endio(bio, res); +} + +static int lo_rw_aio(struct loop_device *lo, struct bio *bio) +{ + struct file *file = lo->lo_backing_file; + struct kiocb *iocb; + unsigned short op; + struct iov_iter iter; + struct bio_vec *bvec; + size_t nr_segs; + loff_t pos = ((loff_t) bio->bi_sector << 9) + lo->lo_offset; + + iocb = aio_kernel_alloc(GFP_NOIO); + if (!iocb) + return -ENOMEM; + + if (bio_rw(bio) & WRITE) + op = IOCB_CMD_WRITE_ITER; + else + op = IOCB_CMD_READ_ITER; + + bvec = bio_iovec_idx(bio, bio->bi_idx); + nr_segs = bio_segments(bio); + iov_iter_init_bvec(&iter, bvec, nr_segs, bvec_length(bvec, nr_segs), 0); + aio_kernel_init_rw(iocb, file, iov_iter_count(&iter), pos); + aio_kernel_init_callback(iocb, lo_rw_aio_complete, (u64)(uintptr_t)bio); + + return aio_kernel_submit(iocb, op, &iter); +} +#endif /* CONFIG_AIO */ + /** * __do_lo_send_write - helper for writing data to a loop device * @@ -418,50 +461,33 @@ static int do_bio_filebacked(struct loop_device *lo, struct bio *bio) pos = ((loff_t) bio->bi_sector << 9) + lo->lo_offset; if (bio_rw(bio) == WRITE) { - struct file *file = lo->lo_backing_file; - - if (bio->bi_rw & REQ_FLUSH) { - ret = vfs_fsync(file, 0); - if (unlikely(ret && ret != -EINVAL)) { - ret = -EIO; - goto out; - } - } + ret = lo_send(lo, bio, pos); + } else + ret = lo_receive(lo, bio, lo->lo_blocksize, pos); - /* - * We use punch hole to reclaim the free space used by the - * image a.k.a. discard. However we do not support discard if - * encryption is enabled, because it may give an attacker - * useful information. - */ - if (bio->bi_rw & REQ_DISCARD) { - struct file *file = lo->lo_backing_file; - int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; + return ret; +} - if ((!file->f_op->fallocate) || - lo->lo_encrypt_key_size) { - ret = -EOPNOTSUPP; - goto out; - } - ret = file->f_op->fallocate(file, mode, pos, - bio->bi_size); - if (unlikely(ret && ret != -EINVAL && - ret != -EOPNOTSUPP)) - ret = -EIO; - goto out; - } +static int lo_discard(struct loop_device *lo, struct bio *bio) +{ + struct file *file = lo->lo_backing_file; + int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; + loff_t pos = ((loff_t) bio->bi_sector << 9) + lo->lo_offset; + int ret; - ret = lo_send(lo, bio, pos); + /* + * We use punch hole to reclaim the free space used by the + * image a.k.a. discard. However we do not support discard if + * encryption is enabled, because it may give an attacker + * useful information. + */ - if ((bio->bi_rw & REQ_FUA) && !ret) { - ret = vfs_fsync(file, 0); - if (unlikely(ret && ret != -EINVAL)) - ret = -EIO; - } - } else - ret = lo_receive(lo, bio, lo->lo_blocksize, pos); + if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) + return -EOPNOTSUPP; -out: + ret = file->f_op->fallocate(file, mode, pos, bio->bi_size); + if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) + ret = -EIO; return ret; } @@ -525,7 +551,35 @@ static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio) do_loop_switch(lo, bio->bi_private); bio_put(bio); } else { - int ret = do_bio_filebacked(lo, bio); + int ret; + + if (bio_rw(bio) == WRITE) { + if (bio->bi_rw & REQ_FLUSH) { + ret = vfs_fsync(lo->lo_backing_file, 1); + if (unlikely(ret && ret != -EINVAL)) + goto out; + } + if (bio->bi_rw & REQ_DISCARD) { + ret = lo_discard(lo, bio); + goto out; + } + } +#ifdef CONFIG_AIO + if (lo->lo_flags & LO_FLAGS_USE_AIO && + lo->transfer == transfer_none) { + ret = lo_rw_aio(lo, bio); + if (ret == 0) + return; + } else +#endif + ret = do_bio_filebacked(lo, bio); + + if ((bio_rw(bio) == WRITE) && bio->bi_rw & REQ_FUA && !ret) { + ret = vfs_fsync(lo->lo_backing_file, 0); + if (unlikely(ret && ret != -EINVAL)) + ret = -EIO; + } +out: bio_endio(bio, ret); } } @@ -547,6 +601,12 @@ static int loop_thread(void *data) struct loop_device *lo = data; struct bio *bio; + /* + * In cases where the underlying filesystem calls balance_dirty_pages() + * we want less throttling to avoid lock ups trying to write dirty + * pages through the loop device + */ + current->flags |= PF_LESS_THROTTLE; set_user_nice(current, -20); while (!kthread_should_stop() || !bio_list_empty(&lo->lo_bio_list)) { @@ -869,6 +929,14 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, !file->f_op->write) lo_flags |= LO_FLAGS_READ_ONLY; +#ifdef CONFIG_AIO + if (file->f_op->write_iter && file->f_op->read_iter && + mapping->a_ops->direct_IO) { + file->f_flags |= O_DIRECT; + lo_flags |= LO_FLAGS_USE_AIO; + } +#endif + lo_blocksize = S_ISBLK(inode->i_mode) ? inode->i_bdev->bd_block_size : PAGE_SIZE; diff --git a/include/uapi/linux/loop.h b/include/uapi/linux/loop.h index e0cecd2eabdc..6edc6b68badd 100644 --- a/include/uapi/linux/loop.h +++ b/include/uapi/linux/loop.h @@ -21,6 +21,7 @@ enum { LO_FLAGS_READ_ONLY = 1, LO_FLAGS_AUTOCLEAR = 4, LO_FLAGS_PARTSCAN = 8, + LO_FLAGS_USE_AIO = 16, }; #include /* for __kernel_old_dev_t */