kvm tools: Use native vectored AIO in virtio-blk

This patch hooks AIO support into virtio-blk, allowing for faster IO.

Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
[ penberg@kernel.org: wrap libaio include with CONFIG_HAS_AIO ]
Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/tools/kvm/disk/core.c b/tools/kvm/disk/core.c
index a5e671391b29da56d3902cfb251ee4c0587ddcde..cc2a1f67a896f10423e5ded80a83496cd5fa3808 100644 (file)
--- a/tools/kvm/disk/core.c
+++ b/tools/kvm/disk/core.c
@@ -1,8 +1,32 @@
 #include "kvm/disk-image.h"
 #include "kvm/qcow.h"
+#include "kvm/virtio-blk.h"
+
+#include <sys/eventfd.h>
+#include <sys/poll.h>
+
+#define AIO_MAX 32
 
 int debug_iodelay;
 
+#ifdef CONFIG_HAS_AIO
+static void *disk_image__thread(void *param)
+{
+       struct disk_image *disk = param;
+       u64 dummy;
+
+       while (read(disk->evt, &dummy, sizeof(dummy)) > 0) {
+               struct io_event event;
+               struct timespec notime = {0};
+
+               while (io_getevents(disk->ctx, 1, 1, &event, &notime) > 0)
+                       disk->disk_req_cb(event.data, event.res);
+       }
+
+       return NULL;
+}
+#endif
+
 struct disk_image *disk_image__new(int fd, u64 size, struct disk_image_operations *ops, int use_mmap)
 {
        struct disk_image *disk;
@@ -26,6 +50,16 @@ struct disk_image *disk_image__new(int fd, u64 size, struct disk_image_operation
                }
        }
 
+#ifdef CONFIG_HAS_AIO
+       if (disk) {
+               pthread_t thread;
+
+               disk->evt = eventfd(0, 0);
+               io_setup(AIO_MAX, &disk->ctx);
+               if (pthread_create(&thread, NULL, disk_image__thread, disk) != 0)
+                       die("Failed starting IO thread");
+       }
+#endif
        return disk;
 }
 
@@ -87,6 +121,7 @@ struct disk_image **disk_image__open_all(const char **filenames, bool *readonly,
                        goto error;
                }
        }
+
        return disks;
 error:
        for (i = 0; i < count; i++)
@@ -151,7 +186,7 @@ ssize_t disk_image__read(struct disk_image *disk, u64 sector, const struct iovec
                /* Do nothing */
        }
 
-       if (disk->disk_req_cb)
+       if (!disk->async && disk->disk_req_cb)
                disk->disk_req_cb(param, total);
 
        return total;
@@ -183,7 +218,7 @@ ssize_t disk_image__write(struct disk_image *disk, u64 sector, const struct iove
                /* Do nothing */
        }
 
-       if (disk->disk_req_cb)
+       if (!disk->async && disk->disk_req_cb)
                disk->disk_req_cb(param, total);
 
        return total;

diff --git a/tools/kvm/disk/qcow.c b/tools/kvm/disk/qcow.c
index f9598ded4477bd679fb3672dac3ead4212ec10d2..87385df721e5d954c6a2f6a0f2aa1d071aea3092 100644 (file)
--- a/tools/kvm/disk/qcow.c
+++ b/tools/kvm/disk/qcow.c
@@ -1145,6 +1145,8 @@ static struct disk_image *qcow2_probe(int fd, bool readonly)
 
        if (!disk_image)
                goto free_refcount_table;
+
+       disk_image->async = 1;
        disk_image->priv = q;
 
        return disk_image;
@@ -1276,6 +1278,8 @@ static struct disk_image *qcow1_probe(int fd, bool readonly)
 
        if (!disk_image)
                goto free_l1_table;
+
+       disk_image->async = 1;
        disk_image->priv = q;
 
        return disk_image;

diff --git a/tools/kvm/disk/raw.c b/tools/kvm/disk/raw.c
index 51e83186267d8f3f87b8536bec69268e99f574e1..de2092ab7c97d8c0dad7c2e157e5ea19ae0f3245 100644 (file)
--- a/tools/kvm/disk/raw.c
+++ b/tools/kvm/disk/raw.c
@@ -1,11 +1,22 @@
 #include "kvm/disk-image.h"
 
+#ifdef CONFIG_HAS_AIO
+#include <libaio.h>
+#endif
+
 ssize_t raw_image__read_sector(struct disk_image *disk, u64 sector, const struct iovec *iov,
                                int iovcount, void *param)
 {
        u64 offset = sector << SECTOR_SHIFT;
 
+#ifdef CONFIG_HAS_AIO
+       struct iocb iocb;
+
+       return aio_preadv(disk->ctx, &iocb, disk->fd, iov, iovcount, offset,
+                               disk->evt, param);
+#else
        return preadv_in_full(disk->fd, iov, iovcount, offset);
+#endif
 }
 
 ssize_t raw_image__write_sector(struct disk_image *disk, u64 sector, const struct iovec *iov,
@@ -13,7 +24,14 @@ ssize_t raw_image__write_sector(struct disk_image *disk, u64 sector, const struc
 {
        u64 offset = sector << SECTOR_SHIFT;
 
+#ifdef CONFIG_HAS_AIO
+       struct iocb iocb;
+
+       return aio_pwritev(disk->ctx, &iocb, disk->fd, iov, iovcount, offset,
+                               disk->evt, param);
+#else
        return pwritev_in_full(disk->fd, iov, iovcount, offset);
+#endif
 }
 
 ssize_t raw_image__read_sector_mmap(struct disk_image *disk, u64 sector, const struct iovec *iov,
@@ -59,6 +77,12 @@ int raw_image__close(struct disk_image *disk)
        if (disk->priv != MAP_FAILED)
                ret = munmap(disk->priv, disk->size);
 
+       close(disk->evt);
+
+#ifdef CONFIG_HAS_VIRTIO
+       io_destroy(disk->ctx);
+#endif
+
        return ret;
 }
 
@@ -83,6 +107,7 @@ struct disk_image_operations ro_ops_nowrite = {
 
 struct disk_image *raw_image__probe(int fd, struct stat *st, bool readonly)
 {
+       struct disk_image *disk;
 
        if (readonly) {
                /*
@@ -94,8 +119,12 @@ struct disk_image *raw_image__probe(int fd, struct stat *st, bool readonly)
                disk = disk_image__new(fd, st->st_size, &ro_ops, DISK_IMAGE_MMAP);
                if (disk == NULL) {
                        ro_ops = raw_image_regular_ops;
-                       ro_ops.write_sector = NULL;
+
                        disk = disk_image__new(fd, st->st_size, &ro_ops_nowrite, DISK_IMAGE_REGULAR);
+#ifdef CONFIG_HAS_VIRTIO
+                       if (disk)
+                               disk->async = 1;
+#endif
                }
 
                return disk;
@@ -103,6 +132,11 @@ struct disk_image *raw_image__probe(int fd, struct stat *st, bool readonly)
                /*
                 * Use read/write instead of mmap
                 */
-               return disk_image__new(fd, st->st_size, &raw_image_regular_ops, DISK_IMAGE_REGULAR);
+               disk = disk_image__new(fd, st->st_size, &raw_image_regular_ops, DISK_IMAGE_REGULAR);
+#ifdef CONFIG_HAS_VIRTIO
+               if (disk)
+                       disk->async = 1;
+#endif
+               return disk;
        }
 }

diff --git a/tools/kvm/include/kvm/disk-image.h b/tools/kvm/include/kvm/disk-image.h
index 632e54dc013e205e139efb09953bf6af467b09ea..56c08da80b485f8ddd4695dca3cee019ccc7c509 100644 (file)
--- a/tools/kvm/include/kvm/disk-image.h
+++ b/tools/kvm/include/kvm/disk-image.h
@@ -46,6 +46,11 @@ struct disk_image {
        void                            *priv;
        void                            *disk_req_cb_param;
        void                            (*disk_req_cb)(void *param, long len);
+       bool                            async;
+       int                             evt;
+#ifdef CONFIG_HAS_AIO
+       io_context_t                    ctx;
+#endif
 };
 
 struct disk_image *disk_image__open(const char *filename, bool readonly);

diff --git a/tools/kvm/virtio/blk.c b/tools/kvm/virtio/blk.c
index 5969d27a6c1edc6c25b01b77920ba3c80b97ef57..223e79f68bd231495f4e9194b31f1acb2ad1fe1e 100644 (file)
--- a/tools/kvm/virtio/blk.c
+++ b/tools/kvm/virtio/blk.c
@@ -2,7 +2,6 @@
 
 #include "kvm/virtio-pci-dev.h"
 #include "kvm/disk-image.h"
-#include "kvm/virtio.h"
 #include "kvm/mutex.h"
 #include "kvm/util.h"
 #include "kvm/kvm.h"
@@ -11,23 +10,23 @@
 #include "kvm/ioeventfd.h"
 #include "kvm/guest_compat.h"
 #include "kvm/virtio-pci.h"
+#include "kvm/virtio.h"
 
 #include <linux/virtio_ring.h>
 #include <linux/virtio_blk.h>
-
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/types.h>
 #include <pthread.h>
 
 #define VIRTIO_BLK_MAX_DEV             4
-#define NUM_VIRT_QUEUES                        1
 
-#define VIRTIO_BLK_QUEUE_SIZE          128
 /*
  * the header and status consume too entries
  */
 #define DISK_SEG_MAX                   (VIRTIO_BLK_QUEUE_SIZE - 2)
+#define VIRTIO_BLK_QUEUE_SIZE          128
+#define NUM_VIRT_QUEUES                        1
 
 struct blk_dev_req {
        struct list_head                list;
@@ -86,8 +85,8 @@ void virtio_blk_complete(void *param, long len)
        u8 *status;
 
        /* status */
-       status                  = req->iov[req->out + req->in - 1].iov_base;
-       *status                 = (len < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
+       status  = req->iov[req->out + req->in - 1].iov_base;
+       *status = (len < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
 
        mutex_lock(&bdev->mutex);
        virt_queue__set_used_elem(req->vq, req->head, len);
@@ -103,17 +102,14 @@ static void virtio_blk_do_io_request(struct kvm *kvm, struct blk_dev_req *req)
        struct virtio_blk_outhdr *req_hdr;
        ssize_t block_cnt;
        struct blk_dev *bdev;
-       struct virt_queue *queue;
        struct iovec *iov;
-       u16 out, in, head;
+       u16 out, in;
 
        block_cnt       = -1;
        bdev            = req->bdev;
-       queue           = req->vq;
        iov             = req->iov;
        out             = req->out;
        in              = req->in;
-       head            = req->head;
        req_hdr         = iov[0].iov_base;
 
        switch (req_hdr->type) {