From 4b0ee1b76a3fa1b323503a1cd72b271b418ec238 Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Wed, 16 Nov 2011 14:24:36 +0200 Subject: [PATCH] kvm tools: Add vhost-net support This patch adds support to using the vhost-net device when using a tap backed virtio-net device. Activating vhost-net is done by appending a 'vhost=1' flag to the net device configuration. For example: 'kvm run -n mode=tap,vhost=1' This improves performance as follows: Short version ------------------ TCP Throughput: +29% UDP Throughput: +10% TCP Latency: -15% UDP Latency: -12% Long version ------------------ MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET Recv Send Send Socket Socket Message Elapsed Size Size Size Time Throughput bytes bytes bytes secs. 10^6bits/sec 87380 16384 16384 10.00 4895.04 MIGRATED UDP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET Socket Message Elapsed Messages Size Size Time Okay Errors Throughput bytes bytes secs # # 10^6bits/sec 229376 65507 10.00 125287 0 6565.60 229376 10.00 106910 5602.57 MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0 Local /Remote Socket Size Request Resp. Elapsed Trans. Send Recv Size Size Time Rate bytes Bytes bytes bytes secs. per sec 16384 87380 1 1 10.00 14811.55 MIGRATED UDP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0 Local /Remote Socket Size Request Resp. Elapsed Trans. Send Recv Size Size Time Rate bytes Bytes bytes bytes secs. per sec 229376 229376 1 1 10.00 16000.44 229376 229376 After: MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET Recv Send Send Socket Socket Message Elapsed Size Size Size Time Throughput bytes bytes bytes secs. 10^6bits/sec 87380 16384 16384 10.00 6340.74 MIGRATED UDP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET Socket Message Elapsed Messages Size Size Time Okay Errors Throughput bytes bytes secs # # 10^6bits/sec 229376 65507 10.00 131478 0 6890.09 229376 10.00 118136 6190.90 MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0 Local /Remote Socket Size Request Resp. Elapsed Trans. Send Recv Size Size Time Rate bytes Bytes bytes bytes secs. per sec 16384 87380 1 1 10.00 17126.10 MIGRATED UDP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0 Local /Remote Socket Size Request Resp. Elapsed Trans. Send Recv Size Size Time Rate bytes Bytes bytes bytes secs. per sec 229376 229376 1 1 10.00 17944.51 Cc: Michael S. Tsirkin Signed-off-by: Sasha Levin Signed-off-by: Pekka Enberg --- tools/kvm/builtin-run.c | 2 + tools/kvm/include/kvm/virtio-net.h | 1 + tools/kvm/virtio/net.c | 120 ++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 1 deletion(-) diff --git a/tools/kvm/builtin-run.c b/tools/kvm/builtin-run.c index 13025db32a75..3b00bf059ae8 100644 --- a/tools/kvm/builtin-run.c +++ b/tools/kvm/builtin-run.c @@ -217,6 +217,8 @@ static int set_net_param(struct virtio_net_params *p, const char *param, p->guest_ip = strdup(val); } else if (strcmp(param, "host_ip") == 0) { p->host_ip = strdup(val); + } else if (strcmp(param, "vhost") == 0) { + p->vhost = atoi(val); } return 0; diff --git a/tools/kvm/include/kvm/virtio-net.h b/tools/kvm/include/kvm/virtio-net.h index 58ae162b2a40..dade8cb65359 100644 --- a/tools/kvm/include/kvm/virtio-net.h +++ b/tools/kvm/include/kvm/virtio-net.h @@ -11,6 +11,7 @@ struct virtio_net_params { char host_mac[6]; struct kvm *kvm; int mode; + int vhost; }; void virtio_net__init(const struct virtio_net_params *params); diff --git a/tools/kvm/virtio/net.c b/tools/kvm/virtio/net.c index cee2b5bd14f3..58ca4ed2c6d2 100644 --- a/tools/kvm/virtio/net.c +++ b/tools/kvm/virtio/net.c @@ -10,6 +10,7 @@ #include "kvm/guest_compat.h" #include "kvm/virtio-trans.h" +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #define VIRTIO_NET_QUEUE_SIZE 128 #define VIRTIO_NET_NUM_QUEUES 2 @@ -57,6 +59,7 @@ struct net_dev { pthread_mutex_t io_tx_lock; pthread_cond_t io_tx_cond; + int vhost_fd; int tap_fd; char tap_name[IFNAMSIZ]; @@ -323,9 +326,12 @@ static void set_guest_features(struct kvm *kvm, void *dev, u32 features) static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn) { + struct vhost_vring_state state = { .index = vq }; + struct vhost_vring_addr addr; struct net_dev *ndev = dev; struct virt_queue *queue; void *p; + int r; compat__remove_message(compat_id); @@ -335,9 +341,82 @@ static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn) vring_init(&queue->vring, VIRTIO_NET_QUEUE_SIZE, p, VIRTIO_PCI_VRING_ALIGN); + if (ndev->vhost_fd == 0) + return 0; + + state.num = queue->vring.num; + r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_NUM, &state); + if (r < 0) + die_perror("VHOST_SET_VRING_NUM failed"); + state.num = 0; + r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_BASE, &state); + if (r < 0) + die_perror("VHOST_SET_VRING_BASE failed"); + + addr = (struct vhost_vring_addr) { + .index = vq, + .desc_user_addr = (u64)(unsigned long)queue->vring.desc, + .avail_user_addr = (u64)(unsigned long)queue->vring.avail, + .used_user_addr = (u64)(unsigned long)queue->vring.used, + }; + + r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_ADDR, &addr); + if (r < 0) + die_perror("VHOST_SET_VRING_ADDR failed"); + return 0; } +static void notify_vq_gsi(struct kvm *kvm, void *dev, u32 vq, u32 gsi) +{ + struct net_dev *ndev = dev; + struct kvm_irqfd irq; + struct vhost_vring_file file; + int r; + + if (ndev->vhost_fd == 0) + return; + + irq = (struct kvm_irqfd) { + .gsi = gsi, + .fd = eventfd(0, 0), + }; + file = (struct vhost_vring_file) { + .index = vq, + .fd = irq.fd, + }; + + r = ioctl(kvm->vm_fd, KVM_IRQFD, &irq); + if (r < 0) + die_perror("KVM_IRQFD failed"); + + r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_CALL, &file); + if (r < 0) + die_perror("VHOST_SET_VRING_CALL failed"); + file.fd = ndev->tap_fd; + r = ioctl(ndev->vhost_fd, VHOST_NET_SET_BACKEND, &file); + if (r != 0) + die("VHOST_NET_SET_BACKEND failed %d", errno); + +} + +static void notify_vq_eventfd(struct kvm *kvm, void *dev, u32 vq, u32 efd) +{ + struct net_dev *ndev = dev; + struct vhost_vring_file file = { + .index = vq, + .fd = efd, + }; + int r; + + if (ndev->vhost_fd == 0) + return; + + r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_KICK, &file); + if (r < 0) + die_perror("VHOST_SET_VRING_KICK failed"); +} + static int notify_vq(struct kvm *kvm, void *dev, u32 vq) { struct net_dev *ndev = dev; @@ -368,8 +447,44 @@ static struct virtio_ops net_dev_virtio_ops = (struct virtio_ops) { .notify_vq = notify_vq, .get_pfn_vq = get_pfn_vq, .get_size_vq = get_size_vq, + .notify_vq_gsi = notify_vq_gsi, + .notify_vq_eventfd = notify_vq_eventfd, }; +static void virtio_net__vhost_init(struct kvm *kvm, struct net_dev *ndev) +{ + u64 features = 0; + struct vhost_memory *mem; + int r; + + ndev->vhost_fd = open("/dev/vhost-net", O_RDWR); + if (ndev->vhost_fd < 0) + die_perror("Failed openning vhost-net device"); + + mem = malloc(sizeof(*mem) + sizeof(struct vhost_memory_region)); + if (mem == NULL) + die("Failed allocating memory for vhost memory map"); + + mem->nregions = 1; + mem->regions[0] = (struct vhost_memory_region) { + .guest_phys_addr = 0, + .memory_size = kvm->ram_size, + .userspace_addr = (u64)kvm->ram_start, + }; + + r = ioctl(ndev->vhost_fd, VHOST_SET_OWNER); + if (r != 0) + die_perror("VHOST_SET_OWNER failed"); + + r = ioctl(ndev->vhost_fd, VHOST_SET_FEATURES, &features); + if (r != 0) + die_perror("VHOST_SET_FEATURES failed"); + r = ioctl(ndev->vhost_fd, VHOST_SET_MEM_TABLE, mem); + if (r != 0) + die_perror("VHOST_SET_MEM_TABLE failed"); + free(mem); +} + void virtio_net__init(const struct virtio_net_params *params) { int i; @@ -415,7 +530,10 @@ void virtio_net__init(const struct virtio_net_params *params) VIRTIO_ID_NET, PCI_CLASS_NET); ndev->vtrans.virtio_ops = &net_dev_virtio_ops; - virtio_net__io_thread_init(params->kvm, ndev); + if (params->vhost) + virtio_net__vhost_init(params->kvm, ndev); + else + virtio_net__io_thread_init(params->kvm, ndev); if (compat_id != -1) compat_id = compat__add_message("virtio-net device was not detected", -- 2.39.5