vhost: ring format independence
Subsystems (5)
Michael S. Tsirkin
maintainerRFC PATCH v4 1/5 vhost: option to fetch descriptors through an independent struct
In reply to Michael S. TsirkinThe idea is to support multiple ring formats by converting to a format-independent array of descriptors.
This costs extra cycles, but we gain in ability
to fetch a batch of descriptors in one go, which
is good for code cache locality.
When used, this causes a minor performance degradation, it's been kept as simple as possible for ease of review. A follow-up patch gets us back the performance by adding batching.
To simplify benchmarking, I kept the old code around so one can switch back and forth between old and new code. This will go away in the final submission.
--- drivers/vhost/vhost.c | 294 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 16 +++ 2 files changed, 309 insertions(+), 1 deletion(-)
drivers/vhost/vhost.c +293 -1
@@ -301,6 +301,7 @@ static void vhost_vq_reset(struct vhost_dev *dev, struct vhost_virtqueue *vq) { vq->num =1 ; + vq->ndescs =0 ; vq->desc =NULL ; vq->avail =NULL ; vq->used =NULL ;
@@ -369,6 +370,9 @@ static int vhost_worker(void *data)static void vhost_vq_free_iovecs (struct vhost_virtqueue *vq) { +kfree (vq->descs); + vq->descs =NULL ; + vq->max_descs =0 ;kfree (vq->indirect); vq->indirect =NULL ;kfree (vq->log);
@@ -385,6 +389,10 @@ static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)for (i =0 ; i < dev->nvqs; ++i) { vq = dev->vqs[i]; + vq->max_descs = dev->iov_limit; + vq->descs =kmalloc_array (vq->max_descs, +sizeof (*vq->descs), +GFP_KERNEL ); vq->indirect =kmalloc_array (UIO_MAXIOV ,sizeof (*vq->indirect),GFP_KERNEL );
@@ -392,7 +400,7 @@ static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) GFP_KERNEL); vq->heads =kmalloc_array (dev->iov_limit,sizeof (*vq->heads),GFP_KERNEL ); -if (!vq->indirect || !vq->log || !vq->heads) +if (!vq->indirect || !vq->log || !vq->heads || !vq->descs) goto err_nomem; }return 0 ;
@@ -2346,6 +2354,290 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, }EXPORT_SYMBOL_GPL (vhost_get_vq_desc); +static struct vhost_desc *peek_split_desc (struct vhost_virtqueue *vq) +{ +BUG_ON (!vq->ndescs); +return &vq->descs[vq->ndescs -1 ]; +} + +static void pop_split_desc (struct vhost_virtqueue *vq) +{ +BUG_ON (!vq->ndescs); + --vq->ndescs; +} + +static int push_split_desc (struct vhost_virtqueue *vq,struct vring_desc *desc,u16 id) +{ +struct vhost_desc *h; + +if (unlikely (vq->ndescs >= vq->max_descs)) +return -EINVAL ; + h = &vq->descs[vq->ndescs++]; + h->addr =vhost64_to_cpu (vq, desc->addr); + h->len =vhost32_to_cpu (vq, desc->len); + h->flags =vhost16_to_cpu (vq, desc->flags); + h->id = id; + +return 0 ; +} + +static int fetch_indirect_descs (struct vhost_virtqueue *vq, +struct vhost_desc *indirect, +u16 head) +{ +struct vring_desc desc; +unsigned int i =0 , count, found =0 ; +u32 len = indirect->len; +struct iov_iter from; +int ret; + +/* Sanity check */ +if (unlikely (len %sizeof desc)) { +vq_err (vq,"Invalid length in indirect descriptor: " +"len 0x%llx not multiple of 0x%zx\n" , + (unsigned long long )len, +sizeof desc); +return -EINVAL ; + } + + ret =translate_desc (vq, indirect->addr, len, vq->indirect, +UIO_MAXIOV ,VHOST_ACCESS_RO ); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Translation failure %d in indirect.\n" , ret); +return ret; + } +iov_iter_init (&from,READ , vq->indirect, ret, len); + +/* We will use the result as an address to read from, so most +* architectures only need a compiler barrier here. */ +read_barrier_depends (); + + count = len /sizeof desc; +/* Buffers are chained via a 16 bit next field, so +* we can have at most 2^16 of these. */ +if (unlikely (count >USHRT_MAX +1 )) { +vq_err (vq,"Indirect buffer length too big: %d\n" , + indirect->len); +return -E2BIG ; + } +if (unlikely (vq->ndescs + count > vq->max_descs)) { +vq_err (vq,"Too many indirect + direct descs: %d + %d\n" , + vq->ndescs, indirect->len); +return -E2BIG ; + } + +do { +if (unlikely (++found > count)) { +vq_err (vq,"Loop detected: last one at %u " +"indirect size %u\n" , + i, count); +return -EINVAL ; + } +if (unlikely (!copy_from_iter_full (&desc,sizeof (desc), &from))) { +vq_err (vq,"Failed indirect descriptor: idx %d, %zx\n" , + i, (size_t )indirect->addr + i *sizeof desc); +return -EINVAL ; + } +if (unlikely (desc.flags &cpu_to_vhost16 (vq,VRING_DESC_F_INDIRECT ))) { +vq_err (vq,"Nested indirect descriptor: idx %d, %zx\n" , + i, (size_t )indirect->addr + i *sizeof desc); +return -EINVAL ; + } + +push_split_desc (vq, &desc, head); + }while ((i =next_desc (vq, &desc)) !=-1 ); +return 0 ; +} + +static int fetch_descs (struct vhost_virtqueue *vq) +{ +unsigned int i, head, found =0 ; +struct vhost_desc *last; +struct vring_desc desc; +__virtio16 avail_idx; +__virtio16 ring_head; +u16 last_avail_idx; +int ret; + +/* Check it isn't doing very strange things with descriptor numbers. */ + last_avail_idx = vq->last_avail_idx; + +if (vq->avail_idx == vq->last_avail_idx) { +if (unlikely (vhost_get_avail_idx (vq, &avail_idx))) { +vq_err (vq,"Failed to access avail idx at %p\n" , + &vq->avail->idx); +return -EFAULT ; + } + vq->avail_idx =vhost16_to_cpu (vq, avail_idx); + +if (unlikely ((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { +vq_err (vq,"Guest moved used index from %u to %u" , + last_avail_idx, vq->avail_idx); +return -EFAULT ; + } + +/* If there's nothing new since last we looked, return +* invalid. +*/ +if (vq->avail_idx == last_avail_idx) +return vq->num; + +/* Only get avail ring entries after they have been +* exposed by guest. +*/ +smp_rmb (); + } + +/* Grab the next descriptor number they're advertising */ +if (unlikely (vhost_get_avail_head (vq, &ring_head, last_avail_idx))) { +vq_err (vq,"Failed to read head: idx %d address %p\n" , + last_avail_idx, + &vq->avail->ring[last_avail_idx % vq->num]); +return -EFAULT ; + } + + head =vhost16_to_cpu (vq, ring_head); + +/* If their number is silly, that's an error. */ +if (unlikely (head >= vq->num)) { +vq_err (vq,"Guest says index %u > %u is available" , + head, vq->num); +return -EINVAL ; + } + + i = head; +do { +if (unlikely (i >= vq->num)) { +vq_err (vq,"Desc index is %u > %u, head = %u" , + i, vq->num, head); +return -EINVAL ; + } +if (unlikely (++found > vq->num)) { +vq_err (vq,"Loop detected: last one at %u " +"vq size %u head %u\n" , + i, vq->num, head); +return -EINVAL ; + } + ret =vhost_get_desc (vq, &desc, i); +if (unlikely (ret)) { +vq_err (vq,"Failed to get descriptor: idx %d addr %p\n" , + i, vq->desc + i); +return -EFAULT ; + } + ret =push_split_desc (vq, &desc, head); +if (unlikely (ret)) { +vq_err (vq,"Failed to save descriptor: idx %d\n" , i); +return -EINVAL ; + } + }while ((i =next_desc (vq, &desc)) !=-1 ); + + last =peek_split_desc (vq); +if (unlikely (last->flags &VRING_DESC_F_INDIRECT )) { +pop_split_desc (vq); + ret =fetch_indirect_descs (vq, last, head); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Failure detected " +"in indirect descriptor at idx %d\n" , head); +return ret; + } + } + +/* Assume notifications from guest are disabled at this point, +* if they aren't we would need to update avail_event index. */ +BUG_ON (!(vq->used_flags &VRING_USED_F_NO_NOTIFY )); + +/* On success, increment avail index. */ + vq->last_avail_idx++; + +return 0 ; +} + +/* This looks in the virtqueue and for the first available buffer, and converts +* it to an iovec for convenient access. Since descriptors consist of some +* number of output then some number of input descriptors, it's actually two +* iovecs, but we pack them into one and note how many of each there were. +* +* This function returns the descriptor number found, or vq->num (which is +* never a valid descriptor number) if none was found. A negative code is +* returned on error. */ +int vhost_get_vq_desc_batch (struct vhost_virtqueue *vq, +struct iovec iov[],unsigned int iov_size, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num) +{ +int ret =fetch_descs (vq); +int i; + +if (ret) +return ret; + +/* Now convert to IOV */ +/* When we start there are none of either input nor output. */ + *out_num = *in_num =0 ; +if (unlikely (log)) + *log_num =0 ; + +for (i =0 ; i < vq->ndescs; ++i) { +unsigned iov_count = *in_num + *out_num; +struct vhost_desc *desc = &vq->descs[i]; +int access; + +if (desc->flags & ~(VRING_DESC_F_INDIRECT |VRING_DESC_F_WRITE )) { +vq_err (vq,"Unexpected flags: 0x%x at descriptor id 0x%x\n" , + desc->flags, desc->id); + ret = -EINVAL ; + goto err; + } +if (desc->flags &VRING_DESC_F_WRITE ) + access =VHOST_ACCESS_WO ; +else + access =VHOST_ACCESS_RO ; + ret =translate_desc (vq, desc->addr, + desc->len, iov + iov_count, + iov_size - iov_count, access); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Translation failure %d descriptor idx %d\n" , + ret, i); + goto err; + } +if (access ==VHOST_ACCESS_WO ) { +/* If this is an input descriptor, +* increment that count. */ + *in_num += ret; +if (unlikely (log && ret)) { + log[*log_num].addr = desc->addr; + log[*log_num].len = desc->len; + ++*log_num; + } + }else { +/* If it's an output descriptor, they're all supposed +* to come before any input descriptors. */ +if (unlikely (*in_num)) { +vq_err (vq,"Descriptor has out after in: " +"idx %d\n" , i); + ret = -EINVAL ; + goto err; + } + *out_num += ret; + } + } + + ret = id; + vq->ndescs =0 ; + +return ret; + +err: +vhost_discard_vq_desc (vq,1 ); + vq->ndescs =0 ; + +return ret; +} +EXPORT_SYMBOL_GPL (vhost_get_vq_desc_batch); +/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ void vhost_discard_vq_desc (struct vhost_virtqueue *vq,int n) {
drivers/vhost/vhost.h +16 -0
@@ -80,6 +80,13 @@ enum vhost_uaddr_type { VHOST_NUM_ADDRS = 3, }; +struct vhost_desc { +u64 addr; +u32 len; +u16 flags;/* VRING_DESC_F_WRITE, VRING_DESC_F_NEXT */ +u16 id; +}; +/* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue {struct vhost_dev *dev;
@@ -90,6 +97,11 @@ struct vhost_virtqueue {struct vring_desc __user *desc;struct vring_avail __user *avail;struct vring_used __user *used; + +struct vhost_desc *descs; +int ndescs; +int max_descs; +const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;struct eventfd_ctx *call_ctx;
@@ -190,6 +202,10 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argbool vhost_vq_access_ok (struct vhost_virtqueue *vq);bool vhost_log_access_ok (struct vhost_dev *); +int vhost_get_vq_desc_batch (struct vhost_virtqueue *, +struct iovec iov[],unsigned int iov_count, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num); int vhost_get_vq_desc(struct vhost_virtqueue *, struct iovec iov[], unsigned int iov_count, unsigned int *out_num, unsigned int *in_num,
Signature · 1 line
MST
Michael S. Tsirkin
maintainerv4 cover vhost: ring format independence
This adds infrastructure required for supporting multiple ring formats.
The idea is as follows: we convert descriptors to an independent format first, and process that converting to iov later.
The point is that we have a tight loop that fetches descriptors, which is good for cache utilization. This will also allow all kind of batching tricks - e.g. it seems possible to keep SMAP disabled while we are fetching multiple descriptors.
This seems to perform exactly the same as the original code already based on a microbenchmark. Lightly tested.
More testing would be very much appreciated.
To use new code:
echo 1 > /sys/module/vhost_test/parameters/newcode
or
echo 1 > /sys/module/vhost_net/parameters/newcode
changes from v3:
- fixed error handling in case of indirect descriptors
- add BUG_ON to detect buffer overflow in case of bugs in response to comment by Jason Wang
- minor code tweaks
Changes from v2:
- fixed indirect descriptor batching reported by Jason Wang
Changes from v1:
- typo fixes
Michael S. Tsirkin (5):
vhost: option to fetch descriptors through an independent struct vhost/test: add an option to test new code
vhost: batching fetches
vhost/net: add an option to test new code
vhost: last descriptor must have NEXT clear
drivers/vhost/net.c | 32 ++++- drivers/vhost/test.c | 19 ++- drivers/vhost/vhost.c | 328 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 20 ++- 4 files changed, 385 insertions(+), 14 deletions(-)
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 2/5 vhost/test: add an option to test new code
Add a writeable module parameter that tests the new code. Note: no effort was made to ensure things work correctly if the parameter is changed while the device is open. Make sure to close the device before changing its value.
--- drivers/vhost/test.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
drivers/vhost/test.c +13 -4
@@ -18,6 +18,9 @@#include "test.h" #include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +/* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ #define VHOST_TEST_WEIGHT 0x80000
@@ -58,10 +61,16 @@ static void handle_vq(struct vhost_test *n)vhost_disable_notify (&n->dev, vq);for (;;) { - head =vhost_get_vq_desc (vq, vq->iov, -ARRAY_SIZE (vq->iov), - &out, &in, -NULL ,NULL ); +if (newcode) + head =vhost_get_vq_desc_batch (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL ); +else + head =vhost_get_vq_desc (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL );/* On error, stop handling until the next kick. */ if (unlikely (head <0 ))break ;
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 1/5 vhost: option to fetch descriptors through an independent struct
In reply to Michael S. TsirkinThe idea is to support multiple ring formats by converting to a format-independent array of descriptors.
This costs extra cycles, but we gain in ability
to fetch a batch of descriptors in one go, which
is good for code cache locality.
When used, this causes a minor performance degradation, it's been kept as simple as possible for ease of review. A follow-up patch gets us back the performance by adding batching.
To simplify benchmarking, I kept the old code around so one can switch back and forth between old and new code. This will go away in the final submission.
--- drivers/vhost/vhost.c | 294 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 16 +++ 2 files changed, 309 insertions(+), 1 deletion(-)
drivers/vhost/vhost.c +293 -1
@@ -301,6 +301,7 @@ static void vhost_vq_reset(struct vhost_dev *dev, struct vhost_virtqueue *vq) { vq->num =1 ; + vq->ndescs =0 ; vq->desc =NULL ; vq->avail =NULL ; vq->used =NULL ;
@@ -369,6 +370,9 @@ static int vhost_worker(void *data)static void vhost_vq_free_iovecs (struct vhost_virtqueue *vq) { +kfree (vq->descs); + vq->descs =NULL ; + vq->max_descs =0 ;kfree (vq->indirect); vq->indirect =NULL ;kfree (vq->log);
@@ -385,6 +389,10 @@ static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)for (i =0 ; i < dev->nvqs; ++i) { vq = dev->vqs[i]; + vq->max_descs = dev->iov_limit; + vq->descs =kmalloc_array (vq->max_descs, +sizeof (*vq->descs), +GFP_KERNEL ); vq->indirect =kmalloc_array (UIO_MAXIOV ,sizeof (*vq->indirect),GFP_KERNEL );
@@ -392,7 +400,7 @@ static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) GFP_KERNEL); vq->heads =kmalloc_array (dev->iov_limit,sizeof (*vq->heads),GFP_KERNEL ); -if (!vq->indirect || !vq->log || !vq->heads) +if (!vq->indirect || !vq->log || !vq->heads || !vq->descs) goto err_nomem; }return 0 ;
@@ -2346,6 +2354,290 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq, }EXPORT_SYMBOL_GPL (vhost_get_vq_desc); +static struct vhost_desc *peek_split_desc (struct vhost_virtqueue *vq) +{ +BUG_ON (!vq->ndescs); +return &vq->descs[vq->ndescs -1 ]; +} + +static void pop_split_desc (struct vhost_virtqueue *vq) +{ +BUG_ON (!vq->ndescs); + --vq->ndescs; +} + +static int push_split_desc (struct vhost_virtqueue *vq,struct vring_desc *desc,u16 id) +{ +struct vhost_desc *h; + +if (unlikely (vq->ndescs >= vq->max_descs)) +return -EINVAL ; + h = &vq->descs[vq->ndescs++]; + h->addr =vhost64_to_cpu (vq, desc->addr); + h->len =vhost32_to_cpu (vq, desc->len); + h->flags =vhost16_to_cpu (vq, desc->flags); + h->id = id; + +return 0 ; +} + +static int fetch_indirect_descs (struct vhost_virtqueue *vq, +struct vhost_desc *indirect, +u16 head) +{ +struct vring_desc desc; +unsigned int i =0 , count, found =0 ; +u32 len = indirect->len; +struct iov_iter from; +int ret; + +/* Sanity check */ +if (unlikely (len %sizeof desc)) { +vq_err (vq,"Invalid length in indirect descriptor: " +"len 0x%llx not multiple of 0x%zx\n" , + (unsigned long long )len, +sizeof desc); +return -EINVAL ; + } + + ret =translate_desc (vq, indirect->addr, len, vq->indirect, +UIO_MAXIOV ,VHOST_ACCESS_RO ); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Translation failure %d in indirect.\n" , ret); +return ret; + } +iov_iter_init (&from,READ , vq->indirect, ret, len); + +/* We will use the result as an address to read from, so most +* architectures only need a compiler barrier here. */ +read_barrier_depends (); + + count = len /sizeof desc; +/* Buffers are chained via a 16 bit next field, so +* we can have at most 2^16 of these. */ +if (unlikely (count >USHRT_MAX +1 )) { +vq_err (vq,"Indirect buffer length too big: %d\n" , + indirect->len); +return -E2BIG ; + } +if (unlikely (vq->ndescs + count > vq->max_descs)) { +vq_err (vq,"Too many indirect + direct descs: %d + %d\n" , + vq->ndescs, indirect->len); +return -E2BIG ; + } + +do { +if (unlikely (++found > count)) { +vq_err (vq,"Loop detected: last one at %u " +"indirect size %u\n" , + i, count); +return -EINVAL ; + } +if (unlikely (!copy_from_iter_full (&desc,sizeof (desc), &from))) { +vq_err (vq,"Failed indirect descriptor: idx %d, %zx\n" , + i, (size_t )indirect->addr + i *sizeof desc); +return -EINVAL ; + } +if (unlikely (desc.flags &cpu_to_vhost16 (vq,VRING_DESC_F_INDIRECT ))) { +vq_err (vq,"Nested indirect descriptor: idx %d, %zx\n" , + i, (size_t )indirect->addr + i *sizeof desc); +return -EINVAL ; + } + +push_split_desc (vq, &desc, head); + }while ((i =next_desc (vq, &desc)) !=-1 ); +return 0 ; +} + +static int fetch_descs (struct vhost_virtqueue *vq) +{ +unsigned int i, head, found =0 ; +struct vhost_desc *last; +struct vring_desc desc; +__virtio16 avail_idx; +__virtio16 ring_head; +u16 last_avail_idx; +int ret; + +/* Check it isn't doing very strange things with descriptor numbers. */ + last_avail_idx = vq->last_avail_idx; + +if (vq->avail_idx == vq->last_avail_idx) { +if (unlikely (vhost_get_avail_idx (vq, &avail_idx))) { +vq_err (vq,"Failed to access avail idx at %p\n" , + &vq->avail->idx); +return -EFAULT ; + } + vq->avail_idx =vhost16_to_cpu (vq, avail_idx); + +if (unlikely ((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { +vq_err (vq,"Guest moved used index from %u to %u" , + last_avail_idx, vq->avail_idx); +return -EFAULT ; + } + +/* If there's nothing new since last we looked, return +* invalid. +*/ +if (vq->avail_idx == last_avail_idx) +return vq->num; + +/* Only get avail ring entries after they have been +* exposed by guest. +*/ +smp_rmb (); + } + +/* Grab the next descriptor number they're advertising */ +if (unlikely (vhost_get_avail_head (vq, &ring_head, last_avail_idx))) { +vq_err (vq,"Failed to read head: idx %d address %p\n" , + last_avail_idx, + &vq->avail->ring[last_avail_idx % vq->num]); +return -EFAULT ; + } + + head =vhost16_to_cpu (vq, ring_head); + +/* If their number is silly, that's an error. */ +if (unlikely (head >= vq->num)) { +vq_err (vq,"Guest says index %u > %u is available" , + head, vq->num); +return -EINVAL ; + } + + i = head; +do { +if (unlikely (i >= vq->num)) { +vq_err (vq,"Desc index is %u > %u, head = %u" , + i, vq->num, head); +return -EINVAL ; + } +if (unlikely (++found > vq->num)) { +vq_err (vq,"Loop detected: last one at %u " +"vq size %u head %u\n" , + i, vq->num, head); +return -EINVAL ; + } + ret =vhost_get_desc (vq, &desc, i); +if (unlikely (ret)) { +vq_err (vq,"Failed to get descriptor: idx %d addr %p\n" , + i, vq->desc + i); +return -EFAULT ; + } + ret =push_split_desc (vq, &desc, head); +if (unlikely (ret)) { +vq_err (vq,"Failed to save descriptor: idx %d\n" , i); +return -EINVAL ; + } + }while ((i =next_desc (vq, &desc)) !=-1 ); + + last =peek_split_desc (vq); +if (unlikely (last->flags &VRING_DESC_F_INDIRECT )) { +pop_split_desc (vq); + ret =fetch_indirect_descs (vq, last, head); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Failure detected " +"in indirect descriptor at idx %d\n" , head); +return ret; + } + } + +/* Assume notifications from guest are disabled at this point, +* if they aren't we would need to update avail_event index. */ +BUG_ON (!(vq->used_flags &VRING_USED_F_NO_NOTIFY )); + +/* On success, increment avail index. */ + vq->last_avail_idx++; + +return 0 ; +} + +/* This looks in the virtqueue and for the first available buffer, and converts +* it to an iovec for convenient access. Since descriptors consist of some +* number of output then some number of input descriptors, it's actually two +* iovecs, but we pack them into one and note how many of each there were. +* +* This function returns the descriptor number found, or vq->num (which is +* never a valid descriptor number) if none was found. A negative code is +* returned on error. */ +int vhost_get_vq_desc_batch (struct vhost_virtqueue *vq, +struct iovec iov[],unsigned int iov_size, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num) +{ +int ret =fetch_descs (vq); +int i; + +if (ret) +return ret; + +/* Now convert to IOV */ +/* When we start there are none of either input nor output. */ + *out_num = *in_num =0 ; +if (unlikely (log)) + *log_num =0 ; + +for (i =0 ; i < vq->ndescs; ++i) { +unsigned iov_count = *in_num + *out_num; +struct vhost_desc *desc = &vq->descs[i]; +int access; + +if (desc->flags & ~(VRING_DESC_F_INDIRECT |VRING_DESC_F_WRITE )) { +vq_err (vq,"Unexpected flags: 0x%x at descriptor id 0x%x\n" , + desc->flags, desc->id); + ret = -EINVAL ; + goto err; + } +if (desc->flags &VRING_DESC_F_WRITE ) + access =VHOST_ACCESS_WO ; +else + access =VHOST_ACCESS_RO ; + ret =translate_desc (vq, desc->addr, + desc->len, iov + iov_count, + iov_size - iov_count, access); +if (unlikely (ret <0 )) { +if (ret != -EAGAIN ) +vq_err (vq,"Translation failure %d descriptor idx %d\n" , + ret, i); + goto err; + } +if (access ==VHOST_ACCESS_WO ) { +/* If this is an input descriptor, +* increment that count. */ + *in_num += ret; +if (unlikely (log && ret)) { + log[*log_num].addr = desc->addr; + log[*log_num].len = desc->len; + ++*log_num; + } + }else { +/* If it's an output descriptor, they're all supposed +* to come before any input descriptors. */ +if (unlikely (*in_num)) { +vq_err (vq,"Descriptor has out after in: " +"idx %d\n" , i); + ret = -EINVAL ; + goto err; + } + *out_num += ret; + } + } + + ret = id; + vq->ndescs =0 ; + +return ret; + +err: +vhost_discard_vq_desc (vq,1 ); + vq->ndescs =0 ; + +return ret; +} +EXPORT_SYMBOL_GPL (vhost_get_vq_desc_batch); +/* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ void vhost_discard_vq_desc (struct vhost_virtqueue *vq,int n) {
drivers/vhost/vhost.h +16 -0
@@ -80,6 +80,13 @@ enum vhost_uaddr_type { VHOST_NUM_ADDRS = 3, }; +struct vhost_desc { +u64 addr; +u32 len; +u16 flags;/* VRING_DESC_F_WRITE, VRING_DESC_F_NEXT */ +u16 id; +}; +/* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue {struct vhost_dev *dev;
@@ -90,6 +97,11 @@ struct vhost_virtqueue {struct vring_desc __user *desc;struct vring_avail __user *avail;struct vring_used __user *used; + +struct vhost_desc *descs; +int ndescs; +int max_descs; +const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;struct eventfd_ctx *call_ctx;
@@ -190,6 +202,10 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argbool vhost_vq_access_ok (struct vhost_virtqueue *vq);bool vhost_log_access_ok (struct vhost_dev *); +int vhost_get_vq_desc_batch (struct vhost_virtqueue *, +struct iovec iov[],unsigned int iov_count, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num); int vhost_get_vq_desc(struct vhost_virtqueue *, struct iovec iov[], unsigned int iov_count, unsigned int *out_num, unsigned int *in_num,
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 3/5 vhost: batching fetches
In reply to Michael S. TsirkinWith this patch applied, new and old code perform identically.
Lots of extra optimizations are now possible, e.g. we can fetch multiple heads with copy_from/to_user now. We can get rid of maintaining the log array. Etc etc.
--- drivers/vhost/test.c | 2 +- drivers/vhost/vhost.c | 48 +++++++++++++++++++++++++++++++++++-------- drivers/vhost/vhost.h | 4 +++- 3 files changed, 44 insertions(+), 10 deletions(-)
drivers/vhost/test.c +1 -1
@@ -128,7 +128,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) dev = &n->dev; vqs[VHOST_TEST_VQ ] = &n->vqs[VHOST_TEST_VQ ]; n->vqs[VHOST_TEST_VQ ].handle_kick = handle_vq_kick; -vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV , +vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV +64 ,VHOST_TEST_PKT_WEIGHT ,VHOST_TEST_WEIGHT ); f->private_data = n;
drivers/vhost/vhost.c +40 -8
112 lines not on this page · raw message
drivers/vhost/vhost.h +3 -1
@@ -100,7 +100,9 @@ struct vhost_virtqueue {struct vhost_desc *descs;int ndescs; +int first_desc;int max_descs; +int batch_descs;const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;
@@ -245,7 +247,7 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev,int vhost_init_device_iotlb (struct vhost_dev *d,bool enabled);#define vq_err (vq, fmt, ...) do { \ - pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ + pr_err(pr_fmt(fmt), ##__VA_ARGS__); \ if ((vq)->error_ctx) \ eventfd_signal((vq)->error_ctx, 1);\ } while (0)
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 2/5 vhost/test: add an option to test new code
In reply to Michael S. TsirkinAdd a writeable module parameter that tests the new code. Note: no effort was made to ensure things work correctly if the parameter is changed while the device is open. Make sure to close the device before changing its value.
--- drivers/vhost/test.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
drivers/vhost/test.c +13 -4
@@ -18,6 +18,9 @@#include "test.h" #include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +/* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ #define VHOST_TEST_WEIGHT 0x80000
@@ -58,10 +61,16 @@ static void handle_vq(struct vhost_test *n)vhost_disable_notify (&n->dev, vq);for (;;) { - head =vhost_get_vq_desc (vq, vq->iov, -ARRAY_SIZE (vq->iov), - &out, &in, -NULL ,NULL ); +if (newcode) + head =vhost_get_vq_desc_batch (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL ); +else + head =vhost_get_vq_desc (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL );/* On error, stop handling until the next kick. */ if (unlikely (head <0 ))break ;
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 4/5 vhost/net: add an option to test new code
In reply to Michael S. TsirkinAdd a writeable module parameter that tests the new code. Note: no effort was made to ensure things work correctly if the parameter is changed while the device is open. Make sure to close the device before changing its value.
--- drivers/vhost/net.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-)
drivers/vhost/net.c +25 -7
@@ -35,6 +35,9 @@#include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +static int experimental_zcopytx =0 ;module_param (experimental_zcopytx, int,0444 ); MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
@@ -565,8 +568,14 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net,struct vhost_virtqueue *rvq = &rnvq->vq;struct vhost_virtqueue *tvq = &tnvq->vq; -int r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), - out_num, in_num,NULL ,NULL ); +int r; + +if (newcode) + r =vhost_get_vq_desc_batch (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); +else + r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL );if (r == tvq->num && tvq->busyloop_timeout) {/* Flush batched packets first */
@@ -575,8 +584,12 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net,vhost_net_busy_poll (net, rvq, tvq, busyloop_intr, false); - r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), - out_num, in_num,NULL ,NULL ); +if (newcode) + r =vhost_get_vq_desc_batch (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); +else + r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); }return r;
@@ -1046,9 +1059,14 @@ static int get_rx_bufs(struct vhost_virtqueue *vq, r = -ENOBUFS ; goto err; } - r =vhost_get_vq_desc (vq, vq->iov + seg, -ARRAY_SIZE (vq->iov) - seg, &out, - &in, log, log_num); +if (newcode) + r =vhost_get_vq_desc_batch (vq, vq->iov + seg, +ARRAY_SIZE (vq->iov) - seg, &out, + &in, log, log_num); +else + r =vhost_get_vq_desc (vq, vq->iov + seg, +ARRAY_SIZE (vq->iov) - seg, &out, + &in, log, log_num);if (unlikely (r <0 )) goto err;
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 3/5 vhost: batching fetches
In reply to Michael S. TsirkinWith this patch applied, new and old code perform identically.
Lots of extra optimizations are now possible, e.g. we can fetch multiple heads with copy_from/to_user now. We can get rid of maintaining the log array. Etc etc.
--- drivers/vhost/test.c | 2 +- drivers/vhost/vhost.c | 48 +++++++++++++++++++++++++++++++++++-------- drivers/vhost/vhost.h | 4 +++- 3 files changed, 44 insertions(+), 10 deletions(-)
drivers/vhost/test.c +1 -1
@@ -128,7 +128,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) dev = &n->dev; vqs[VHOST_TEST_VQ ] = &n->vqs[VHOST_TEST_VQ ]; n->vqs[VHOST_TEST_VQ ].handle_kick = handle_vq_kick; -vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV , +vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV +64 ,VHOST_TEST_PKT_WEIGHT ,VHOST_TEST_WEIGHT ); f->private_data = n;
drivers/vhost/vhost.c +40 -8
112 lines not on this page · raw message
drivers/vhost/vhost.h +3 -1
@@ -100,7 +100,9 @@ struct vhost_virtqueue {struct vhost_desc *descs;int ndescs; +int first_desc;int max_descs; +int batch_descs;const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;
@@ -245,7 +247,7 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev,int vhost_init_device_iotlb (struct vhost_dev *d,bool enabled);#define vq_err (vq, fmt, ...) do { \ - pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ + pr_err(pr_fmt(fmt), ##__VA_ARGS__); \ if ((vq)->error_ctx) \ eventfd_signal((vq)->error_ctx, 1);\ } while (0)
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 5/5 vhost: last descriptor must have NEXT clear
In reply to Michael S. Tsirkinfetch_buf already guarantees we exit on a descriptor without a NEXT flag. Add a BUG_ON statement to make sure we don't overflow the buffer in case of a bug.
--- drivers/vhost/vhost.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/vhost/vhost.c +2 -0
@@ -2656,6 +2656,8 @@ int vhost_get_vq_desc_batch(struct vhost_virtqueue *vq,break ; } +BUG_ON (i >= vq->ndescs); + vq->first_desc = i +1 ;return ret;
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 4/5 vhost/net: add an option to test new code
In reply to Michael S. TsirkinAdd a writeable module parameter that tests the new code. Note: no effort was made to ensure things work correctly if the parameter is changed while the device is open. Make sure to close the device before changing its value.
--- drivers/vhost/net.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-)
drivers/vhost/net.c +25 -7
@@ -35,6 +35,9 @@#include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +static int experimental_zcopytx =0 ;module_param (experimental_zcopytx, int,0444 ); MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;"
@@ -565,8 +568,14 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net,struct vhost_virtqueue *rvq = &rnvq->vq;struct vhost_virtqueue *tvq = &tnvq->vq; -int r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), - out_num, in_num,NULL ,NULL ); +int r; + +if (newcode) + r =vhost_get_vq_desc_batch (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); +else + r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL );if (r == tvq->num && tvq->busyloop_timeout) {/* Flush batched packets first */
@@ -575,8 +584,12 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net,vhost_net_busy_poll (net, rvq, tvq, busyloop_intr, false); - r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), - out_num, in_num,NULL ,NULL ); +if (newcode) + r =vhost_get_vq_desc_batch (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); +else + r =vhost_get_vq_desc (tvq, tvq->iov,ARRAY_SIZE (tvq->iov), + out_num, in_num,NULL ,NULL ); }return r;
@@ -1046,9 +1059,14 @@ static int get_rx_bufs(struct vhost_virtqueue *vq, r = -ENOBUFS ; goto err; } - r =vhost_get_vq_desc (vq, vq->iov + seg, -ARRAY_SIZE (vq->iov) - seg, &out, - &in, log, log_num); +if (newcode) + r =vhost_get_vq_desc_batch (vq, vq->iov + seg, +ARRAY_SIZE (vq->iov) - seg, &out, + &in, log, log_num); +else + r =vhost_get_vq_desc (vq, vq->iov + seg, +ARRAY_SIZE (vq->iov) - seg, &out, + &in, log, log_num);if (unlikely (r <0 )) goto err;
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v4 5/5 vhost: last descriptor must have NEXT clear
In reply to Michael S. Tsirkinfetch_buf already guarantees we exit on a descriptor without a NEXT flag. Add a BUG_ON statement to make sure we don't overflow the buffer in case of a bug.
--- drivers/vhost/vhost.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/vhost/vhost.c +2 -0
@@ -2656,6 +2656,8 @@ int vhost_get_vq_desc_batch(struct vhost_virtqueue *vq,break ; } +BUG_ON (i >= vq->ndescs); + vq->first_desc = i +1 ;return ret;
Signature · 1 line
MST
Quoted 29 lines · Michael S. Tsirkin
This adds infrastructure required for supporting multiple ring formats.
The idea is as follows: we convert descriptors to an independent format first, and process that converting to iov later.
The point is that we have a tight loop that fetches descriptors, which is good for cache utilization. This will also allow all kind of batching tricks - e.g. it seems possible to keep SMAP disabled while we are fetching multiple descriptors.
This seems to perform exactly the same as the original code already based on a microbenchmark. Lightly tested.
More testing would be very much appreciated.To use new code:
echo 1 >/sys/module/vhost_test/parameters/newcode
or
echo 1 >/sys/module/vhost_net/parameters/newcodechanges from v3:
- fixed error handling in case of indirect descriptors
- addBUG_ONto detect buffer overflow in case of bugs in response to comment by Jason Wang
- minor code tweaks
Quoted 6 lines
Changes from v2:
- fixed indirect descriptor batching reported by Jason WangChanges from v1:
- typo fixes
I've just done some quick benchmark with testpmd + vhost_net txonly.
With 256 queue size, no difference but in 1024 queue size 1% regression of PPS were found.
Thanks
Quoted 9 lines
Michael S. Tsirkin (5):
vhost: option to fetch descriptors through an independent struct vhost/test: add an option to test new code
vhost: batching fetches
vhost/net: add an option to test new code
vhost: last descriptor must have NEXT clear
Quoted 6 lines
drivers/vhost/net.c | 32 ++++- drivers/vhost/test.c | 19 ++- drivers/vhost/vhost.c | 328 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 20 ++- 4 files changed, 385 insertions(+), 14 deletions(-)
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Quoted 29 lines · Michael S. Tsirkin
This adds infrastructure required for supporting multiple ring formats.
The idea is as follows: we convert descriptors to an independent format first, and process that converting to iov later.
The point is that we have a tight loop that fetches descriptors, which is good for cache utilization. This will also allow all kind of batching tricks - e.g. it seems possible to keep SMAP disabled while we are fetching multiple descriptors.
This seems to perform exactly the same as the original code already based on a microbenchmark. Lightly tested.
More testing would be very much appreciated.To use new code:
echo 1 >/sys/module/vhost_test/parameters/newcode
or
echo 1 >/sys/module/vhost_net/parameters/newcodechanges from v3:
- fixed error handling in case of indirect descriptors
- addBUG_ONto detect buffer overflow in case of bugs in response to comment by Jason Wang
- minor code tweaks
Quoted 6 lines
Changes from v2:
- fixed indirect descriptor batching reported by Jason WangChanges from v1:
- typo fixes
I've just done some quick benchmark with testpmd + vhost_net txonly.
With 256 queue size, no difference but in 1024 queue size 1% regression of PPS were found.
Thanks
Quoted 15 lines
Michael S. Tsirkin (5):
vhost: option to fetch descriptors through an independent struct vhost/test: add an option to test new code
vhost: batching fetches
vhost/net: add an option to test new code
vhost: last descriptor must have NEXT cleardrivers/vhost/net.c | 32 ++++- drivers/vhost/test.c | 19 ++- drivers/vhost/vhost.c | 328 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 20 ++- 4 files changed, 385 insertions(+), 14 deletions(-)