Provide a zero-copy method on KVM virtio-net.
Subsystems (5)
cover Provide a zero-copy method on KVM virtio-net.
The idea is simple, just to pin the guest VM user space and then let host NIC driver has the chance to directly DMA to it. The patches are based on vhost-net backend driver. We add a device which provides proto_ops as sendmsg/recvmsg to vhost-net to send/recv directly to/from the NIC driver. KVM guest who use the vhost-net backend may bind any ethX interface in the host side to get copyless data transfer thru guest virtio-net frontend.
We provide multiple submits and asynchronous notifiicaton to vhost-net too.
Our goal is to improve the bandwidth and reduce the CPU usage. Exact performance data will be provided later. But for simple test with netperf, we found bindwidth up and CPU % up too, but the bindwidth up ratio is much more than CPU % up ratio.
What we have not done yet:
To support GRO
Performance tuning
PATCH 1/3 A device for zero-copy based on KVM virtio-net.
Add a device to utilize the vhost-net backend driver for copy-less data transfer between guest FE and host NIC. It pins the guest user space to the host memory and provides proto_ops as sendmsg/recvmsg to vhost-net.
--- drivers/vhost/Kconfig | 5 + drivers/vhost/Makefile | 2 + drivers/vhost/mpassthru.c | 1178 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/miscdevice.h | 1 + include/linux/mpassthru.h | 17 + 5 files changed, 1203 insertions(+), 0 deletions(-) create mode 100644 drivers/vhost/mpassthru.c create mode 100644 include/linux/mpassthru.h
drivers/vhost/Kconfig +5 -0
@@ -9,3 +9,8 @@ config VHOST_NET To compile this driver as a module, choose M here: the module will be called vhost_net. +config VHOST_PASSTHRU + tristate "Zerocopy network driver (EXPERIMENTAL)" + depends on VHOST_NET + ---help--- + zerocopy network I/O support
drivers/vhost/Makefile +2 -0
@@ -1,2 +1,4 @@ obj-$(CONFIG_VHOST_NET) += vhost_net.o vhost_net-y := vhost.o net.o + +obj-$(CONFIG_VHOST_PASSTHRU) += mpassthru.o
drivers/vhost/mpassthru.c new file +1178 -0
1,178 lines not on this page · raw message
include/linux/miscdevice.h +1 -0
@@ -31,6 +31,7 @@#define FUSE_MINOR 229#define KVM_MINOR 232#define VHOST_NET_MINOR 233 +#define MPASSTHRU_MINOR 234#define MISC_DYNAMIC_MINOR 255struct device ;
include/linux/mpassthru.h new file +17 -0
@@ -0,0 +1,17 @@ +#ifndef __MPASSTHRU_H +#define __MPASSTHRU_H + +#include <linux/types.h> +#include <linux/if_ether.h> + +/* ioctl defines */ +#define MPASSTHRU_BINDDEV _IOW('M', 213, int) +#define MPASSTHRU_UNBINDDEV _IOW('M', 214, int) + +/* MPASSTHRU ifc flags */ +#define IFF_MPASSTHRU 0x0001 +#define IFF_MPASSTHRU_EXCL 0x0002 + +struct socket *mp_get_socket (struct file *); + +#endif /* __MPASSTHRU_H */
Signature · 1 line
1.5.4.4
PATCH 2/3 Provides multiple submits and asynchronous notifications.
The vhost-net backend now only supports synchronous send/recv operations. The patch provides multiple submits and asynchronous notifications. This is needed for zero-copy case.
--- drivers/vhost/net.c | 145 +++++++++++++++++++++++++++++++++++++++++++++++-- drivers/vhost/vhost.h | 23 ++++++++ 2 files changed, 164 insertions(+), 4 deletions(-)
drivers/vhost/net.c +141 -4
@@ -22,6 +22,7 @@#include <linux/if_packet.h> #include <linux/if_arp.h> #include <linux/if_tun.h> +#include <linux/mpassthru.h> #include <net/sock.h>
@@ -91,6 +92,10 @@ static void tx_poll_start(struct vhost_net *net, struct socket *sock) net->tx_poll_state =VHOST_NET_POLL_STARTED ; } +static void handle_async_rx_events_notify (struct vhost_net *net, +struct vhost_virtqueue *vq); +static void handle_async_tx_events_notify (struct vhost_net *net, +struct vhost_virtqueue *vq);/* Expects to be always run from workqueue - which acts as * read-size critical section for our kind of RCU. */ static void handle_tx (struct vhost_net *net)
@@ -124,6 +129,8 @@ static void handle_tx(struct vhost_net *net)tx_poll_stop (net); hdr_size = vq->hdr_size; +handle_async_tx_events_notify (net, vq); +for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov),
@@ -151,6 +158,12 @@ static void handle_tx(struct vhost_net *net)/* Skip header. TODO: support TSO. */ s =move_iovec_hdr (vq->iov, vq->hdr, hdr_size, out); msg.msg_iovlen = out; + +if (vq->link_state ==VHOST_VQ_LINK_ASYNC ) { + vq->head = head; + msg.msg_control = (void *)vq; + } + len =iov_length (vq->iov, out);/* Sanity check */ if (!len) {
@@ -166,6 +179,10 @@ static void handle_tx(struct vhost_net *net)tx_poll_start (net, sock);break ; } + +if (vq->link_state ==VHOST_VQ_LINK_ASYNC ) +continue ; +if (err != len)pr_err ("Truncated TX packet: " " len %d != %zd\n" , err, len);
@@ -177,6 +194,8 @@ static void handle_tx(struct vhost_net *net) } } +handle_async_tx_events_notify (net, vq); +mutex_unlock (&vq->mutex);unuse_mm (net->dev.mm); }
@@ -206,7 +225,8 @@ static void handle_rx(struct vhost_net *net)int err;size_t hdr_size;struct socket *sock =rcu_dereference (vq->private_data); -if (!sock ||skb_queue_empty (&sock->sk->sk_receive_queue)) +if (!sock || (skb_queue_empty (&sock->sk->sk_receive_queue) && + vq->link_state ==VHOST_VQ_LINK_SYNC ))return ;use_mm (net->dev.mm);
@@ -217,6 +237,8 @@ static void handle_rx(struct vhost_net *net) vq_log =unlikely (vhost_has_feature (&net->dev,VHOST_F_LOG_ALL )) ? vq->log :NULL ; +handle_async_rx_events_notify (net, vq); +for (;;) { head = vhost_get_vq_desc(&net->dev, vq, vq->iov, ARRAY_SIZE(vq->iov),
@@ -245,6 +267,11 @@ static void handle_rx(struct vhost_net *net) s =move_iovec_hdr (vq->iov, vq->hdr, hdr_size, in); msg.msg_iovlen = in; len =iov_length (vq->iov, in); +if (vq->link_state ==VHOST_VQ_LINK_ASYNC ) { + vq->head = head; + vq->_log = log; + msg.msg_control = (void *)vq; + }/* Sanity check */ if (!len) { vq_err(vq, "Unexpected header len for RX: "
@@ -259,6 +286,10 @@ static void handle_rx(struct vhost_net *net)vhost_discard_vq_desc (vq);break ; } + +if (vq->link_state ==VHOST_VQ_LINK_ASYNC ) +continue ; +/* TODO: Should check and handle checksum. */ if (err > len) { pr_err("Discarded truncated rx packet: "
@@ -284,10 +315,83 @@ static void handle_rx(struct vhost_net *net) } } +handle_async_rx_events_notify (net, vq); +mutex_unlock (&vq->mutex);unuse_mm (net->dev.mm); } +struct vhost_notifier *notify_dequeue (struct vhost_virtqueue *vq) +{ +struct vhost_notifier *vnotify =NULL ; +unsigned long flags; + +spin_lock_irqsave (&vq->notify_lock, flags); +if (!list_empty (&vq->notifier)) { + vnotify =list_first_entry (&vq->notifier, + struct vhost_notifier, list); +list_del (&vnotify->list); + } +spin_unlock_irqrestore (&vq->notify_lock, flags); +return vnotify; +} + +static void handle_async_rx_events_notify (struct vhost_net *net, +struct vhost_virtqueue *vq) +{ +struct vhost_notifier *vnotify =NULL ; +struct vhost_log *vq_log =NULL ; +int rx_total_len =0 ; +int log, size; + +if (vq->link_state !=VHOST_VQ_LINK_ASYNC ) +return ; +if (vq != &net->dev.vqs[VHOST_NET_VQ_RX ]) +return ; + +if (vq->receiver) + vq->receiver (vq); + vq_log =unlikely (vhost_has_feature ( + &net->dev,VHOST_F_LOG_ALL )) ? vq->log :NULL ; +while ((vnotify =notify_dequeue (vq)) !=NULL ) { +vhost_add_used_and_signal (&net->dev, vq, + vnotify->head, vnotify->size); + log = vnotify->log; + size = vnotify->size; + rx_total_len += vnotify->size; + vnotify->dtor (vnotify); +if (unlikely (vq_log)) +vhost_log_write (vq, vq_log, log, size); +if (unlikely (rx_total_len >=VHOST_NET_WEIGHT )) { +vhost_poll_queue (&vq->poll); +break ; + } + } +} + +static void handle_async_tx_events_notify (struct vhost_net *net, +struct vhost_virtqueue *vq) +{ +struct vhost_notifier *vnotify =NULL ; +int tx_total_len =0 ; + +if (vq->link_state !=VHOST_VQ_LINK_ASYNC ) +return ; +if (vq != &net->dev.vqs[VHOST_NET_VQ_TX ]) +return ; + +while ((vnotify =notify_dequeue (vq)) !=NULL ) { +vhost_add_used_and_signal (&net->dev, vq, + vnotify->head,0 ); + tx_total_len += vnotify->size; + vnotify->dtor (vnotify); +if (unlikely (tx_total_len >=VHOST_NET_WEIGHT )) { +vhost_poll_queue (&vq->poll); +break ; + } + } +} +static void handle_tx_kick (struct work_struct *work) {struct vhost_virtqueue *vq;
@@ -462,7 +566,19 @@ static struct socket *get_tun_socket(int fd)return sock; } -static struct socket *get_socket (int fd) +static struct socket *get_mp_socket (int fd) +{ +struct file *file =fget (fd); +struct socket *sock; +if (!file) +return ERR_PTR (-EBADF ); + sock =mp_get_socket (file); +if (IS_ERR (sock)) +fput (file); +return sock; +} + +static struct socket *get_socket (struct vhost_virtqueue *vq,int fd) {struct socket *sock;if (fd ==-1 )
@@ -473,9 +589,26 @@ static struct socket *get_socket(int fd) sock =get_tun_socket (fd);if (!IS_ERR (sock))return sock; + sock =get_mp_socket (fd); +if (!IS_ERR (sock)) { + vq->link_state =VHOST_VQ_LINK_ASYNC ; +return sock; + }return ERR_PTR (-ENOTSOCK ); } +static void vhost_init_link_state (struct vhost_net *n,int index) +{ +struct vhost_virtqueue *vq = n->vqs + index; + +WARN_ON (!mutex_is_locked (&vq->mutex)); +if (vq->link_state ==VHOST_VQ_LINK_ASYNC ) { + vq->receiver =NULL ; +INIT_LIST_HEAD (&vq->notifier); +spin_lock_init (&vq->notify_lock); + } +} +static long vhost_net_set_backend (struct vhost_net *n,unsigned index,int fd) {struct socket *sock, *oldsock;
@@ -493,12 +626,15 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd) } vq = n->vqs + index;mutex_lock (&vq->mutex); - sock =get_socket (fd); + vq->link_state =VHOST_VQ_LINK_SYNC ; + sock =get_socket (vq, fd);if (IS_ERR (sock)) { r =PTR_ERR (sock); goto err; } +vhost_init_link_state (n, index); +/* start polling new socket */ oldsock = vq->private_data;if (sock == oldsock)
@@ -507,8 +643,8 @@ static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)vhost_net_disable_vq (n, vq);rcu_assign_pointer (vq->private_data, sock);vhost_net_enable_vq (n, vq); -mutex_unlock (&vq->mutex); done: +mutex_unlock (&vq->mutex);mutex_unlock (&n->dev.mutex);if (oldsock) {vhost_net_flush_vq (n, index);
@@ -516,6 +652,7 @@ done: }return r; err: +mutex_unlock (&vq->mutex);mutex_unlock (&n->dev.mutex);return r; }
drivers/vhost/vhost.h +23 -0
@@ -43,6 +43,22 @@ struct vhost_log {u64 len; }; +enum vhost_vq_link_state { +VHOST_VQ_LINK_SYNC =0 , +VHOST_VQ_LINK_ASYNC =1 , +}; + +/* The structure to notify the virtqueue for async socket */ +struct vhost_notifier { +struct list_head list; +struct vhost_virtqueue *vq; +int head; +int size; +int log; +void *ctrl; +void (*dtor)(struct vhost_notifier *); +}; +/* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue {struct vhost_dev *dev;
@@ -96,6 +112,13 @@ struct vhost_virtqueue {/* Log write descriptors */ void __user *log_base;struct vhost_log log[VHOST_NET_MAX_SG ]; +/*Differiate async socket for 0-copy from normal*/ +enum vhost_vq_link_state link_state; +int head; +int _log; +struct list_head notifier; +spinlock_t notify_lock; +void (*receiver)(struct vhost_virtqueue *); };struct vhost_dev {
Signature · 1 line
1.5.4.4
PATCH 3/3 Let host NIC driver to DMA to guest user space.
The patch let host NIC driver to receive user space skb, then the driver has chance to directly DMA to guest user space buffers thru single ethX interface.
--- include/linux/netdevice.h | 72 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/skbuff.h | 32 ++++++++++++++++++-- net/core/dev.c | 27 +++++++++++++++++ net/core/skbuff.c | 62 +++++++++++++++++++++++++++++++++++---- 4 files changed, 184 insertions(+), 9 deletions(-)
include/linux/netdevice.h +72 -0
@@ -486,6 +486,16 @@ struct netdev_queue { }____cacheline_aligned_in_smp ; +struct netdev_page_ctor { +int hdr_len; +int data_len; +int npages; +unsigned flags; +struct socket *sock; +struct skb_user_page *(*ctor)(struct netdev_page_ctor *, +struct sk_buff *,int ); +}; +/* * This structure defines the management hooks for network devices. * The following hooks can be defined; unless noted otherwise, they are
@@ -636,6 +646,8 @@ struct net_device_ops {int (*ndo_fcoe_ddp_done)(struct net_device *dev,u16 xid);#endif +int (*ndo_page_ctor_prep)(struct net_device *dev, +struct netdev_page_ctor *ctor); };/*
@@ -916,6 +928,7 @@ struct net_device/* max exchange id for FCoE LRO by ddp */ unsigned int fcoe_ddp_xid;#endif +struct netdev_page_ctor *page_ctor; };#define to_net_dev (d) container_of(d, struct net_device, dev)
@@ -2013,6 +2026,65 @@ static inline u32 dev_ethtool_get_flags(struct net_device *dev)return 0 ;return dev->ethtool_ops->get_flags (dev); } + +static inline int netdev_page_ctor_prep (struct net_device *dev, +struct netdev_page_ctor *ctor) +{ +int rc; +int npages, data_len; +const struct net_device_ops *ops = dev->netdev_ops; + +/* needed by packet split */ +if (ops->ndo_page_ctor_prep) { + rc = ops->ndo_page_ctor_prep (dev, ctor); +if (rc) +return rc; + }else {/* should be temp */ + ctor->hdr_len =128 ; + ctor->data_len =2048 ; + ctor->npages =1 ; + } + +if (ctor->hdr_len <=0 ) + goto err; + + npages = ctor->npages; + data_len = ctor->data_len; +if (npages <=0 || npages >MAX_SKB_FRAGS || + (data_len <PAGE_SIZE * (npages -1 ) || + data_len >PAGE_SIZE * npages)) + goto err; + +return 0 ; +err: +dev_warn (&dev->dev,"invalid page constructor parameters\n" ); + +return -EINVAL ; +} + +static inline int netdev_page_ctor_attach (struct net_device *dev, +struct netdev_page_ctor *ctor) +{ +if (dev->flags &IFF_UP ) +return -EBUSY ; + +if (rcu_dereference (dev->page_ctor)) +return -EBUSY ; + +rcu_assign_pointer (dev->page_ctor, ctor); + +return 0 ; +} + +static inline void netdev_page_ctor_detach (struct net_device *dev) +{ +if (!rcu_dereference (dev->page_ctor)) +return ; + +rcu_assign_pointer (dev->page_ctor,NULL ); +synchronize_rcu (); +} +#endif /* __KERNEL__ */ #endif /* _LINUX_NETDEVICE_H */
include/linux/skbuff.h +29 -3
@@ -209,6 +209,13 @@ struct skb_shared_info {void * destructor_arg; }; +struct skb_user_page { +u8 *start; +int size; +struct skb_frag_struct *frags; +struct skb_shared_info *ushinfo; +void (*dtor)(struct skb_user_page *); +};/* We divide dataref into two halves. The higher 16 bits hold references * to the payload part of skb->data. The lower 16 bits hold references to * the entire skb->data. A clone of a headerless skb holds the length of
@@ -441,17 +448,18 @@ extern void kfree_skb(struct sk_buff *skb);extern void consume_skb (struct sk_buff *skb);extern void __kfree_skb (struct sk_buff *skb);extern struct sk_buff *__alloc_skb (unsigned int size, -gfp_t priority,int fclone,int node); +gfp_t priority,int fclone, +int node,struct net_device *dev);static inline struct sk_buff *alloc_skb (unsigned int size,gfp_t priority) { -return __alloc_skb (size, priority,0 ,-1 ); +return __alloc_skb (size, priority,0 ,-1 ,NULL ); }static inline struct sk_buff *alloc_skb_fclone (unsigned int size,gfp_t priority) { -return __alloc_skb (size, priority,1 ,-1 ); +return __alloc_skb (size, priority,1 ,-1 ,NULL ); }extern int skb_recycle_check (struct sk_buff *skb,int skb_size);
@@ -1509,6 +1517,24 @@ static inline void netdev_free_page(struct net_device *dev, struct page *page)__free_page (page); } +extern struct skb_user_page *netdev_alloc_user_pages (struct net_device *dev, +struct sk_buff *skb,int npages); + +extern int netdev_use_ps_feature (struct net_device *dev); + +static inline struct skb_user_page *netdev_alloc_user_page ( +struct net_device *dev, +struct sk_buff *skb,unsigned int size) +{ +struct skb_user_page *user; +int npages = (size <PAGE_SIZE ) ?1 : (size /PAGE_SIZE ); + + user =netdev_alloc_user_pages (dev, skb, npages); +if (likely (user)) +return user; +return NULL ; +} +/** * skb_clone_writable - is the header of a clone writable * @skb: buffer to check
net/core/dev.c +27 -0
@@ -2265,6 +2265,27 @@ void netif_nit_deliver(struct sk_buff *skb)rcu_read_unlock (); } +static inline struct sk_buff *handle_user_space_buf (struct sk_buff *skb, +struct packet_type **pt_prev, +int *ret,struct net_device *orig_dev) +{ +struct netdev_page_ctor *ctor =NULL ; +struct sock *sk =NULL ; + +if (skb->dev) + ctor = skb->dev->page_ctor; +if (!ctor) +return skb; + + sk = ctor->sock->sk; + +skb_queue_tail (&sk->sk_receive_queue, skb); + + sk->sk_data_ready (sk, skb->len); +return NULL ; +} + +/** * netif_receive_skb - process receive buffer from network * @skb: buffer to process
@@ -2342,6 +2363,9 @@ int netif_receive_skb(struct sk_buff *skb) goto out; ncls:#endif + skb =handle_user_space_buf (skb, &pt_prev, &ret, orig_dev); +if (!skb) + goto out; skb =handle_bridge (skb, &pt_prev, &ret, orig_dev);if (!skb)
@@ -2455,6 +2479,9 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)if (skb_is_gso (skb) ||skb_has_frags (skb)) goto normal; +if (skb->dev && skb->dev->page_ctor) + goto normal; +rcu_read_lock ();list_for_each_entry_rcu (ptype, head, list) {if (ptype->type != type || ptype->dev || !ptype->gro_receive)
net/core/skbuff.c +56 -6
122 lines not on this page · raw message
Signature · 1 line
1.5.4.4
Re: [PATCH 1/3] A device for zero-copy based on KVM virtio-net.
In reply to Xin XiaohuiQuoted 12 lines · Xin Xiaohui
Add a device to utilize the vhost-net backend driver for copy-less data transfer between guest FE and host NIC. It pins the guest user space to the host memory and provides
proto_opsas sendmsg/recvmsg to vhost-net.Signed-off-by Xin Xiaohui <[email protected]>Signed-off-by Zhao Yu <[email protected]>Sigend-off-by Jeff Dike <[email protected]>drivers/vhost/mpassthru.c new file +2
-0+static int page_ctor_attach (struct mp_struct *mp) +{
drivers/vhost/mpassthru.c line 283
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+int rc; +struct page_ctor *ctor; +struct net_device *dev = mp->dev; + +rcu_read_lock (); +if (rcu_dereference (mp->ctor)) { +rcu_read_unlock (); +return -EBUSY ; + } +rcu_read_unlock ();
Strange read locking here, for an obvious writer role. What do you really want to do ?
If writer are serialized by mp_mutex, you dont need this recu_read_lock()/rcu_read_unlock() stuff.
drivers/vhost/mpassthru.c line 294
Quoted 11 lines
drivers/vhost/mpassthru.c new file +11
-0+ + ctor =kzalloc (sizeof (*ctor),GFP_KERNEL ); +if (!ctor) +return -ENOMEM ; + rc =netdev_page_ctor_prep (dev, &ctor->ctor); +if (rc) + goto fail; + + ctor->cache =kmem_cache_create ("skb_page_info" , +sizeof (struct page_info ),0 , +SLAB_HWCACHE_ALIGN |SLAB_PANIC ,NULL );
SLAB_PANIC here means : crash whole system in case of error. This is not what you want in a driver.
Quoted 36 lines
drivers/vhost/mpassthru.c new file +36
-0+ +if (!ctor->cache) + goto cache_fail; + +INIT_LIST_HEAD (&ctor->readq); +spin_lock_init (&ctor->read_lock); + + ctor->w_len =0 ; + ctor->r_len =0 ; + +dev_hold (dev); + ctor->dev = dev; + ctor->ctor.ctor = page_ctor; + ctor->ctor.sock = &mp->socket; +atomic_set (&ctor->refcnt,1 ); + + rc =netdev_page_ctor_attach (dev, &ctor->ctor); +if (rc) + goto fail; + +/* locked by mp_mutex */ +rcu_assign_pointer (mp->ctor, ctor); + +/* XXX:Need we do set_offload here ? */ + +return 0 ; + +fail: +kmem_cache_destroy (ctor->cache); +cache_fail: +kfree (ctor); +dev_put (dev); + +return rc; +} +
drivers/vhost/mpassthru.c line 340
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+ +static inline void get_page_ctor (struct page_ctor *ctor) +{ +atomic_inc (&ctor->refcnt); +} + +static inline void put_page_ctor (struct page_ctor *ctor) +{ +if (atomic_dec_and_test (&ctor->refcnt)) +kfree (ctor);
Are you sure a RCU grace period is not needed before freeing ?
drivers/vhost/mpassthru.c line 365
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+ +static int page_ctor_detach (struct mp_struct *mp) +{ +struct page_ctor *ctor; +struct page_info *info; +int i; + +rcu_read_lock (); + ctor =rcu_dereference (mp->ctor); +rcu_read_unlock ();
Strange locking again here
Quoted 20 lines
drivers/vhost/mpassthru.c new file +20
-0+ +if (!ctor) +return -ENODEV ; + +while ((info =info_dequeue (ctor))) { +for (i =0 ; i < info->pnum; i++) +if (info->pages[i]) +put_page (info->pages[i]); +kmem_cache_free (ctor->cache, info); + } +kmem_cache_destroy (ctor->cache); +netdev_page_ctor_detach (ctor->dev); +dev_put (ctor->dev); + +/* locked by mp_mutex */ +rcu_assign_pointer (mp->ctor,NULL ); +synchronize_rcu (); + +put_page_ctor (ctor); +
drivers/vhost/mpassthru.c line 395
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+return 0 ; +} + +/* For small user space buffers transmit, we don't need to call +* get_user_pages(). +*/ +static struct page_info *alloc_small_page_info (struct page_ctor *ctor, +int total) +{ +struct page_info *info =kmem_cache_alloc (ctor->cache,GFP_KERNEL );
kmem_cache_zalloc() ?
drivers/vhost/mpassthru.c line 400
Quoted 5 lines
drivers/vhost/mpassthru.c new file +5
-0+ +if (!info) +return NULL ; +memset (info,0 ,sizeof (struct page_info )); +memset (info->pages,0 ,sizeof (info->pages));
redundant memset() whole structure already cleared one line above
Review
Quoted 2 lines
(quoted diff) +2
-0+ + info->header = 0;
already cleared
Review
Quoted 2 lines
(quoted diff) +2
-0+ info->total = total; + info->skb = NULL;
already cleared
drivers/vhost/mpassthru.c line 408
Quoted 5 lines
drivers/vhost/mpassthru.c new file +4
-0+ info->user.dtor = page_dtor; + info->ctor = ctor; + info->flags =INFO_WRITE ; + info->pnum =0 ;
already cleared
Quoted 7 lines
drivers/vhost/mpassthru.c new file +6
-0+return info; +} + +/* The main function to transform the guest user space address +* to host kernel address via get_user_pages(). Thus the hardware +* can do DMA directly to the user space address.
drivers/vhost/mpassthru.c line 424
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+*/ +static struct page_info *alloc_page_info (struct page_ctor *ctor, +struct iovec *iov,int count,struct frag *frags, +int npages,int total) +{ +int rc; +int i, j, n =0 ; +int len; +unsigned long base; +struct page_info *info =kmem_cache_alloc (ctor->cache,GFP_KERNEL );
kmem_cache_zalloc() ?
drivers/vhost/mpassthru.c line 399
Quoted 5 lines
drivers/vhost/mpassthru.c new file +4
-0+ +if (!info) +return NULL ; +memset (info,0 ,sizeof (struct page_info ));
kmem_cache_zalloc() ?
Review
Quoted 1 line
(quoted diff) +1
-0+ memset(info->pages, 0, sizeof(info->pages));
already cleared
Quoted 61 lines
drivers/vhost/mpassthru.c new file +60
-0+ +down_read (¤t->mm->mmap_sem); +for (i = j =0 ; i < count; i++) { + base = (unsigned long )iov[i].iov_base; + len = iov[i].iov_len; + +if (!len) +continue ; + n = ((base & ~PAGE_MASK ) + len + ~PAGE_MASK ) >>PAGE_SHIFT ; + + rc =get_user_pages (current, current->mm, base, n, + npages ?1 :0 ,0 , &info->pages[j],NULL ); +if (rc != n) { +up_read (¤t->mm->mmap_sem); + goto failed; + } + +while (n--) { + frags[j].offset = base & ~PAGE_MASK ; + frags[j].size =min_t (int, len, +PAGE_SIZE - frags[j].offset); + len -= frags[j].size; + base += frags[j].size; + j++; + } + } +up_read (¤t->mm->mmap_sem); + +#ifdef CONFIG_HIGHMEM +if (npages && !(dev->features &NETIF_F_HIGHDMA )) { +for (i =0 ; i < j; i++) { +if (PageHighMem (info->pages[i])) + goto failed; + } + } +#endif + + info->header =0 ; + info->total = total; + info->skb =NULL ; + info->user.dtor = page_dtor; + info->ctor = ctor; + info->pnum = j; + +if (!npages) + info->flags =INFO_WRITE ; +if (info->flags ==INFO_READ ) { + info->user.start = (u8 *)(((unsigned long ) + (pfn_to_kaddr (page_to_pfn (info->pages[0 ]))) + + frags[0 ].offset) -NET_IP_ALIGN -NET_SKB_PAD ); + info->user.size = iov[0 ].iov_len +NET_IP_ALIGN +NET_SKB_PAD ; + } +return info; + +failed: +for (i =0 ; i < j; i++) +put_page (info->pages[i]); + +kmem_cache_free (ctor->cache, info); +
drivers/vhost/mpassthru.c line 499
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+return NULL ; +} + +struct page_ctor *mp_rcu_get_ctor (struct page_ctor *ctor) +{ +struct page_ctor *_ctor =NULL ; + +rcu_read_lock (); + _ctor =rcu_dereference (ctor); +rcu_read_unlock ();
strange locking. After rcu_read_unlock() you have no guarantee _ctor points to something not freed.
drivers/vhost/mpassthru.c line 506
Quoted 7 lines
drivers/vhost/mpassthru.c new file +7
-0+ +if (!_ctor) { +DBG (KERN_INFO "Device %s cannot do mediate passthru.\n" , + ctor->dev->name); +return NULL ; + } +if (_ctor)
redundant test
drivers/vhost/mpassthru.c line 510
Quoted 4 lines
drivers/vhost/mpassthru.c new file +4
-0+get_page_ctor (_ctor); +return _ctor; +} +
I stopped my review at this point. Please check your RCU usages. It is not sufficient to hold rcu read lock just to fetch the pointer, you also must hold the lock while using the object itself, or get a reference on object before release RCU lock, to make sure object wont disappear under you...
for example :
rcu_read_lock();
ptr = rcu_dereference(...);
if (ptr)
atomic_inc(&ptr->refcnt);
rcu_read_unlock();
RE: [PATCH 1/3] A device for zero-copy based on KVM virtio-net.
Eric,
Thanks. I will look into that. But don't stop there. Please comments more. :-)
Thanks
Xiaohui
-----Original Message-----
From: Eric Dumazet [mailto:[email protected]]
Sent: Wednesday, February 10, 2010 11:18 PM
To: Xin, Xiaohui
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Zhao Yu
Subject: Re: [PATCH 1/3] A device for zero-copy based on KVM virtio-net.
Quoted 12 lines · Xin Xiaohui
Add a device to utilize the vhost-net backend driver for copy-less data transfer between guest FE and host NIC. It pins the guest user space to the host memory and provides
proto_opsas sendmsg/recvmsg to vhost-net.Signed-off-by Xin Xiaohui <[email protected]>Signed-off-by Zhao Yu <[email protected]>Sigend-off-by Jeff Dike <[email protected]>drivers/vhost/mpassthru.c new file +2
-0+static int page_ctor_attach (struct mp_struct *mp) +{
drivers/vhost/mpassthru.c line 283
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+int rc; +struct page_ctor *ctor; +struct net_device *dev = mp->dev; + +rcu_read_lock (); +if (rcu_dereference (mp->ctor)) { +rcu_read_unlock (); +return -EBUSY ; + } +rcu_read_unlock ();
Strange read locking here, for an obvious writer role. What do you really want to do ?
If writer are serialized by mp_mutex, you dont need this recu_read_lock()/rcu_read_unlock() stuff.
drivers/vhost/mpassthru.c line 294
Quoted 11 lines
drivers/vhost/mpassthru.c new file +11
-0+ + ctor =kzalloc (sizeof (*ctor),GFP_KERNEL ); +if (!ctor) +return -ENOMEM ; + rc =netdev_page_ctor_prep (dev, &ctor->ctor); +if (rc) + goto fail; + + ctor->cache =kmem_cache_create ("skb_page_info" , +sizeof (struct page_info ),0 , +SLAB_HWCACHE_ALIGN |SLAB_PANIC ,NULL );
SLAB_PANIC here means : crash whole system in case of error. This is not what you want in a driver.
Quoted 36 lines
drivers/vhost/mpassthru.c new file +36
-0+ +if (!ctor->cache) + goto cache_fail; + +INIT_LIST_HEAD (&ctor->readq); +spin_lock_init (&ctor->read_lock); + + ctor->w_len =0 ; + ctor->r_len =0 ; + +dev_hold (dev); + ctor->dev = dev; + ctor->ctor.ctor = page_ctor; + ctor->ctor.sock = &mp->socket; +atomic_set (&ctor->refcnt,1 ); + + rc =netdev_page_ctor_attach (dev, &ctor->ctor); +if (rc) + goto fail; + +/* locked by mp_mutex */ +rcu_assign_pointer (mp->ctor, ctor); + +/* XXX:Need we do set_offload here ? */ + +return 0 ; + +fail: +kmem_cache_destroy (ctor->cache); +cache_fail: +kfree (ctor); +dev_put (dev); + +return rc; +} +
drivers/vhost/mpassthru.c line 340
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+ +static inline void get_page_ctor (struct page_ctor *ctor) +{ +atomic_inc (&ctor->refcnt); +} + +static inline void put_page_ctor (struct page_ctor *ctor) +{ +if (atomic_dec_and_test (&ctor->refcnt)) +kfree (ctor);
Are you sure a RCU grace period is not needed before freeing ?
drivers/vhost/mpassthru.c line 365
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+ +static int page_ctor_detach (struct mp_struct *mp) +{ +struct page_ctor *ctor; +struct page_info *info; +int i; + +rcu_read_lock (); + ctor =rcu_dereference (mp->ctor); +rcu_read_unlock ();
Strange locking again here
Quoted 20 lines
drivers/vhost/mpassthru.c new file +20
-0+ +if (!ctor) +return -ENODEV ; + +while ((info =info_dequeue (ctor))) { +for (i =0 ; i < info->pnum; i++) +if (info->pages[i]) +put_page (info->pages[i]); +kmem_cache_free (ctor->cache, info); + } +kmem_cache_destroy (ctor->cache); +netdev_page_ctor_detach (ctor->dev); +dev_put (ctor->dev); + +/* locked by mp_mutex */ +rcu_assign_pointer (mp->ctor,NULL ); +synchronize_rcu (); + +put_page_ctor (ctor); +
drivers/vhost/mpassthru.c line 395
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+return 0 ; +} + +/* For small user space buffers transmit, we don't need to call +* get_user_pages(). +*/ +static struct page_info *alloc_small_page_info (struct page_ctor *ctor, +int total) +{ +struct page_info *info =kmem_cache_alloc (ctor->cache,GFP_KERNEL );
kmem_cache_zalloc() ?
drivers/vhost/mpassthru.c line 400
Quoted 5 lines
drivers/vhost/mpassthru.c new file +5
-0+ +if (!info) +return NULL ; +memset (info,0 ,sizeof (struct page_info )); +memset (info->pages,0 ,sizeof (info->pages));
redundant memset() whole structure already cleared one line above
Review
Quoted 2 lines
(quoted diff) +2
-0+ + info->header = 0;
already cleared
Review
Quoted 2 lines
(quoted diff) +2
-0+ info->total = total; + info->skb = NULL;
already cleared
drivers/vhost/mpassthru.c line 408
Quoted 5 lines
drivers/vhost/mpassthru.c new file +4
-0+ info->user.dtor = page_dtor; + info->ctor = ctor; + info->flags =INFO_WRITE ; + info->pnum =0 ;
already cleared
Quoted 7 lines
drivers/vhost/mpassthru.c new file +6
-0+return info; +} + +/* The main function to transform the guest user space address +* to host kernel address via get_user_pages(). Thus the hardware +* can do DMA directly to the user space address.
drivers/vhost/mpassthru.c line 424
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+*/ +static struct page_info *alloc_page_info (struct page_ctor *ctor, +struct iovec *iov,int count,struct frag *frags, +int npages,int total) +{ +int rc; +int i, j, n =0 ; +int len; +unsigned long base; +struct page_info *info =kmem_cache_alloc (ctor->cache,GFP_KERNEL );
kmem_cache_zalloc() ?
drivers/vhost/mpassthru.c line 399
Quoted 5 lines
drivers/vhost/mpassthru.c new file +4
-0+ +if (!info) +return NULL ; +memset (info,0 ,sizeof (struct page_info ));
kmem_cache_zalloc() ?
Review
Quoted 1 line
(quoted diff) +1
-0+ memset(info->pages, 0, sizeof(info->pages));
already cleared
Quoted 61 lines
drivers/vhost/mpassthru.c new file +60
-0+ +down_read (¤t->mm->mmap_sem); +for (i = j =0 ; i < count; i++) { + base = (unsigned long )iov[i].iov_base; + len = iov[i].iov_len; + +if (!len) +continue ; + n = ((base & ~PAGE_MASK ) + len + ~PAGE_MASK ) >>PAGE_SHIFT ; + + rc =get_user_pages (current, current->mm, base, n, + npages ?1 :0 ,0 , &info->pages[j],NULL ); +if (rc != n) { +up_read (¤t->mm->mmap_sem); + goto failed; + } + +while (n--) { + frags[j].offset = base & ~PAGE_MASK ; + frags[j].size =min_t (int, len, +PAGE_SIZE - frags[j].offset); + len -= frags[j].size; + base += frags[j].size; + j++; + } + } +up_read (¤t->mm->mmap_sem); + +#ifdef CONFIG_HIGHMEM +if (npages && !(dev->features &NETIF_F_HIGHDMA )) { +for (i =0 ; i < j; i++) { +if (PageHighMem (info->pages[i])) + goto failed; + } + } +#endif + + info->header =0 ; + info->total = total; + info->skb =NULL ; + info->user.dtor = page_dtor; + info->ctor = ctor; + info->pnum = j; + +if (!npages) + info->flags =INFO_WRITE ; +if (info->flags ==INFO_READ ) { + info->user.start = (u8 *)(((unsigned long ) + (pfn_to_kaddr (page_to_pfn (info->pages[0 ]))) + + frags[0 ].offset) -NET_IP_ALIGN -NET_SKB_PAD ); + info->user.size = iov[0 ].iov_len +NET_IP_ALIGN +NET_SKB_PAD ; + } +return info; + +failed: +for (i =0 ; i < j; i++) +put_page (info->pages[i]); + +kmem_cache_free (ctor->cache, info); +
drivers/vhost/mpassthru.c line 499
Quoted 10 lines
drivers/vhost/mpassthru.c new file +10
-0+return NULL ; +} + +struct page_ctor *mp_rcu_get_ctor (struct page_ctor *ctor) +{ +struct page_ctor *_ctor =NULL ; + +rcu_read_lock (); + _ctor =rcu_dereference (ctor); +rcu_read_unlock ();
strange locking. After rcu_read_unlock() you have no guarantee _ctor points to something not freed.
drivers/vhost/mpassthru.c line 506
Quoted 7 lines
drivers/vhost/mpassthru.c new file +7
-0+ +if (!_ctor) { +DBG (KERN_INFO "Device %s cannot do mediate passthru.\n" , + ctor->dev->name); +return NULL ; + } +if (_ctor)
redundant test
drivers/vhost/mpassthru.c line 510
Quoted 4 lines
drivers/vhost/mpassthru.c new file +4
-0+get_page_ctor (_ctor); +return _ctor; +} +
I stopped my review at this point. Please check your RCU usages. It is not sufficient to hold rcu read lock just to fetch the pointer, you also must hold the lock while using the object itself, or get a reference on object before release RCU lock, to make sure object wont disappear under you...
for example :
rcu_read_lock();
ptr = rcu_dereference(...);
if (ptr)
atomic_inc(&ptr->refcnt);
rcu_read_unlock();
Arnd Bergmann
maintainerQuoted 10 lines · Xin Xiaohui
The idea is simple, just to pin the guest VM user space and then let host NIC driver has the chance to directly DMA to it. The patches are based on vhost-net backend driver. We add a device which provides
proto_opsas sendmsg/recvmsg to vhost-net to send/recv directly to/from the NIC driver. KVM guest who use the vhost-net backend may bind any ethX interface in the host side to get copyless data transfer thru guest virtio-net frontend.We provide multiple submits and asynchronous notifiicaton to vhost-net too.
This does a lot of things that I had planned for macvtap. It's great to hear that you have made this much progress.
However, I'd hope that we could combine this with the macvtap driver, which would give us zero-copy transfer capability both with and without vhost, as well as (tx at least) when using multiple guests on a macvlan setup.
For transmit, it should be fairly straightforward to hook up your zero-copy method and the vhost-net interface into the macvtap driver.
You have simplified the receiv path significantly by assuming that the entire netdev can receive into a single guest, right? I'm assuming that the idea is to allow VMDq adapters to simply show up as separate adapters and have the driver handle this in a hardware specific way.
My plan for this was to instead move support for VMDq into the macvlan driver so we can transparently use VMDq on hardware where available, including zero-copy receives, but fall back to software operation on non-VMDq hardware.
Arnd
Quoted 12 lines
Quoted 11 lines · Xin Xiaohui
The idea is simple, just to pin the guest VM user space and then let host NIC driver has the chance to directly DMA to it. The patches are based on vhost-net backend driver. We add a device which provides
proto_opsas sendmsg/recvmsg to vhost-net to send/recv directly to/from the NIC driver. KVM guest who use the vhost-net backend may bind any ethX interface in the host side to get copyless data transfer thru guest virtio-net frontend.We provide multiple submits and asynchronous notifiicaton to vhost-net too.
Quoted 7 lines
This does a lot of things that I had planned for macvtap. It's great to hear that you have made this much progress.
However, I'd hope that we could combine this with the macvtap driver, which would give us zero-copy transfer capability both with and without vhost, as well as (tx at least) when using multiple guests on a macvlan setup.
You mean the zero-copy can work with macvtap driver without vhost. May you give me some detailed info about your macvtap driver and the relationship between vhost and macvtap to make me have a clear picture then?
Quoted 6 lines
For transmit, it should be fairly straightforward to hook up your zero-copy method and the vhost-net interface into the macvtap driver.
You have simplified the receiv path significantly by assuming that the entire netdev can receive into a single guest, right?
Yes.
Quoted 3 lines
I'm assuming that the idea is to allow VMDq adapters to simply show up as separate adapters and have the driver handle this in a hardware specific way.
Does the VMDq driver do so now?
Quoted 4 lines
My plan for this was to instead move support for VMDq into the macvlan driver so we can transparently use VMDq on hardware where available, including zero-copy receives, but fall back to software operation on non-VMDq hardware.
Arnd
Arnd Bergmann
maintainerQuoted 11 lines · Xiaohui
Quoted 7 lines
This does a lot of things that I had planned for macvtap. It's great to hear that you have made this much progress.
However, I'd hope that we could combine this with the macvtap driver, which would give us zero-copy transfer capability both with and without vhost, as well as (tx at least) when using multiple guests on a macvlan setup.
You mean the zero-copy can work with macvtap driver without vhost. May you give me some detailed info about your macvtap driver and the relationship between vhost and macvtap to make me have a clear picture then?
macvtap provides a user interface that is largely compatible with the tun/tap driver, and can be used in place of that from qemu. Vhost-net currently interfaces with tun/tap, but not yet with macvtap, which is easy enough to add and already on my list.
The underlying code is macvlan, which is a driver that virtualizes network adapters in software, giving you multiple net_device instances for a real NIC, each of them with their own MAC address.
In order to do zero-copy transmit with macvtap, the idea is to add a nonblocking version of the aio_write() function that works a lot like your transmit function.
For receive, the hardware does not currently know which guest is supposed to get any frame coming in from the outside. Adding zero-copy receive requires interaction with the device driver and hardware capabilities to separate traffic by inbound MAC address into separate buffers per VM.
Quoted 5 lines
Quoted 3 lines
I'm assuming that the idea is to allow VMDq adapters to simply show up as separate adapters and have the driver handle this in a hardware specific way.
Does the VMDq driver do so now?
I don't think anyone has published a VMDq capable driver so far. I was just assuming that you were working on one.
Arnd
Will be in a vacation during 2/13~2/20, so email may be very slow or no replied for your comments. But please don't hesitate to comment more, and I will address them after the vacation. :-)
Thanks
Xiaohui
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Xin Xiaohui
Sent: Wednesday, February 10, 2010 7:49 PM
To: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: [PATCH 0/3] Provide a zero-copy method on KVM virtio-net.
The idea is simple, just to pin the guest VM user space and then let host NIC driver has the chance to directly DMA to it. The patches are based on vhost-net backend driver. We add a device which provides proto_ops as sendmsg/recvmsg to vhost-net to send/recv directly to/from the NIC driver. KVM guest who use the vhost-net backend may bind any ethX interface in the host side to get copyless data transfer thru guest virtio-net frontend.
We provide multiple submits and asynchronous notifiicaton to vhost-net too.
Our goal is to improve the bandwidth and reduce the CPU usage. Exact performance data will be provided later. But for simple test with netperf, we found bindwidth up and CPU % up too, but the bindwidth up ratio is much more than CPU % up ratio.
What we have not done yet:
To support GRO
Performance tuning
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html