vhost_net: basic polling support

[karo-tx-linux.git] / drivers / vhost / net.c

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 7bd75ff8be260695b4135979ca4a91df42b58f3f..f744eeb3e2b47ec0abbfa908e81e4cc108097145 100644 (file)
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -287,6 +287,43 @@ static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success)
        rcu_read_unlock_bh();
 }
 
+static inline unsigned long busy_clock(void)
+{
+       return local_clock() >> 10;
+}
+
+static bool vhost_can_busy_poll(struct vhost_dev *dev,
+                               unsigned long endtime)
+{
+       return likely(!need_resched()) &&
+              likely(!time_after(busy_clock(), endtime)) &&
+              likely(!signal_pending(current)) &&
+              !vhost_has_work(dev);
+}
+
+static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
+                                   struct vhost_virtqueue *vq,
+                                   struct iovec iov[], unsigned int iov_size,
+                                   unsigned int *out_num, unsigned int *in_num)
+{
+       unsigned long uninitialized_var(endtime);
+       int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
+                                   out_num, in_num, NULL, NULL);
+
+       if (r == vq->num && vq->busyloop_timeout) {
+               preempt_disable();
+               endtime = busy_clock() + vq->busyloop_timeout;
+               while (vhost_can_busy_poll(vq->dev, endtime) &&
+                      vhost_vq_avail_empty(vq->dev, vq))
+                       cpu_relax_lowlatency();
+               preempt_enable();
+               r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
+                                       out_num, in_num, NULL, NULL);
+       }
+
+       return r;
+}
+
 /* Expects to be always run from workqueue - which acts as
  * read-size critical section for our kind of RCU. */
 static void handle_tx(struct vhost_net *net)
@@ -331,10 +368,9 @@ static void handle_tx(struct vhost_net *net)
                              % UIO_MAXIOV == nvq->done_idx))
                        break;
 
-               head = vhost_get_vq_desc(vq, vq->iov,
-                                        ARRAY_SIZE(vq->iov),
-                                        &out, &in,
-                                        NULL, NULL);
+               head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
+                                               ARRAY_SIZE(vq->iov),
+                                               &out, &in);
                /* On error, stop handling until the next kick. */
                if (unlikely(head < 0))
                        break;
@@ -435,6 +471,38 @@ static int peek_head_len(struct sock *sk)
        return len;
 }
 
+static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
+{
+       struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
+       struct vhost_virtqueue *vq = &nvq->vq;
+       unsigned long uninitialized_var(endtime);
+       int len = peek_head_len(sk);
+
+       if (!len && vq->busyloop_timeout) {
+               /* Both tx vq and rx socket were polled here */
+               mutex_lock(&vq->mutex);
+               vhost_disable_notify(&net->dev, vq);
+
+               preempt_disable();
+               endtime = busy_clock() + vq->busyloop_timeout;
+
+               while (vhost_can_busy_poll(&net->dev, endtime) &&
+                      skb_queue_empty(&sk->sk_receive_queue) &&
+                      vhost_vq_avail_empty(&net->dev, vq))
+                       cpu_relax_lowlatency();
+
+               preempt_enable();
+
+               if (vhost_enable_notify(&net->dev, vq))
+                       vhost_poll_queue(&vq->poll);
+               mutex_unlock(&vq->mutex);
+
+               len = peek_head_len(sk);
+       }
+
+       return len;
+}
+
 /* This is a multi-buffer version of vhost_get_desc, that works if
  *     vq has read descriptors only.
  * @vq         - the relevant virtqueue
@@ -553,7 +621,7 @@ static void handle_rx(struct vhost_net *net)
                vq->log : NULL;
        mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF);
 
-       while ((sock_len = peek_head_len(sock->sk))) {
+       while ((sock_len = vhost_net_rx_peek_head_len(net, sock->sk))) {
                sock_len += sock_hlen;
                vhost_len = sock_len + vhost_hlen;
                headcount = get_rx_bufs(vq, vq->heads, vhost_len,