vhost: ring format independence
Subsystems (5)
Michael S. Tsirkin
maintainerv2 cover vhost: ring format independence
This adds infrastructure required for supporting multiple ring formats.
The idea is as follows: we convert descriptors to an independent format first, and process that converting to iov later.
The point is that we have a tight loop that fetches descriptors, which is good for cache utilization. This will also allow all kind of batching tricks - e.g. it seems possible to keep SMAP disabled while we are fetching multiple descriptors.
This seems to perform exactly the same as the original code already based on a microbenchmark.
More testing would be very much appreciated.
Biggest TODO before this first step is ready to go in is to batch indirect descriptors as well.
Integrating into vhost-net is basically s/vhost_get_vq_desc/vhost_get_vq_desc_batch/ - or add a module parameter like I did in the test module.
Changes from v1:
- typo fixes
Michael S. Tsirkin (2):
vhost: option to fetch descriptors through an independent struct
vhost: batching fetches
drivers/vhost/test.c | 19 ++- drivers/vhost/vhost.c | 333 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 20 ++- 3 files changed, 365 insertions(+), 7 deletions(-)
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v2 1/2 vhost: option to fetch descriptors through an independent struct
The idea is to support multiple ring formats by converting to a format-independent array of descriptors.
This costs extra cycles, but we gain in ability to fetch a batch of descriptors in one go, which is good for code cache locality.
To simplify benchmarking, I kept the old code
around so one can switch back and forth by
writing into a module parameter.
This will go away in the final submission.
This patch causes a minor performance degradation, it's been kept as simple as possible for ease of review. Next patch gets us back the performance by adding batching.
--- drivers/vhost/test.c | 17 ++- drivers/vhost/vhost.c | 299 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 16 +++ 3 files changed, 327 insertions(+), 5 deletions(-)
drivers/vhost/test.c +13 -4
@@ -18,6 +18,9 @@#include "test.h" #include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +/* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ #define VHOST_TEST_WEIGHT 0x80000
@@ -58,10 +61,16 @@ static void handle_vq(struct vhost_test *n)vhost_disable_notify (&n->dev, vq);for (;;) { - head =vhost_get_vq_desc (vq, vq->iov, -ARRAY_SIZE (vq->iov), - &out, &in, -NULL ,NULL ); +if (newcode) + head =vhost_get_vq_desc_batch (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL ); +else + head =vhost_get_vq_desc (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL );/* On error, stop handling until the next kick. */ if (unlikely (head <0 ))break ;
drivers/vhost/vhost.c +298 -1
329 lines not on this page · raw message
drivers/vhost/vhost.h +16 -0
@@ -80,6 +80,13 @@ enum vhost_uaddr_type { VHOST_NUM_ADDRS = 3, }; +struct vhost_desc { +u64 addr; +u32 len; +u16 flags;/* VRING_DESC_F_WRITE, VRING_DESC_F_NEXT */ +u16 id; +}; +/* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue {struct vhost_dev *dev;
@@ -90,6 +97,11 @@ struct vhost_virtqueue {struct vring_desc __user *desc;struct vring_avail __user *avail;struct vring_used __user *used; + +struct vhost_desc *descs; +int ndescs; +int max_descs; +const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;struct eventfd_ctx *call_ctx;
@@ -190,6 +202,10 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argbool vhost_vq_access_ok (struct vhost_virtqueue *vq);bool vhost_log_access_ok (struct vhost_dev *); +int vhost_get_vq_desc_batch (struct vhost_virtqueue *, +struct iovec iov[],unsigned int iov_count, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num); int vhost_get_vq_desc(struct vhost_virtqueue *, struct iovec iov[], unsigned int iov_count, unsigned int *out_num, unsigned int *in_num,
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v2 1/2 vhost: option to fetch descriptors through an independent struct
In reply to Michael S. TsirkinThe idea is to support multiple ring formats by converting to a format-independent array of descriptors.
This costs extra cycles, but we gain in ability to fetch a batch of descriptors in one go, which is good for code cache locality.
To simplify benchmarking, I kept the old code
around so one can switch back and forth by
writing into a module parameter.
This will go away in the final submission.
This patch causes a minor performance degradation, it's been kept as simple as possible for ease of review. Next patch gets us back the performance by adding batching.
--- drivers/vhost/test.c | 17 ++- drivers/vhost/vhost.c | 299 +++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 16 +++ 3 files changed, 327 insertions(+), 5 deletions(-)
drivers/vhost/test.c +13 -4
@@ -18,6 +18,9 @@#include "test.h" #include "vhost.h" +static int newcode =0 ; +module_param (newcode, int,0644 ); +/* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ #define VHOST_TEST_WEIGHT 0x80000
@@ -58,10 +61,16 @@ static void handle_vq(struct vhost_test *n)vhost_disable_notify (&n->dev, vq);for (;;) { - head =vhost_get_vq_desc (vq, vq->iov, -ARRAY_SIZE (vq->iov), - &out, &in, -NULL ,NULL ); +if (newcode) + head =vhost_get_vq_desc_batch (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL ); +else + head =vhost_get_vq_desc (vq, vq->iov, +ARRAY_SIZE (vq->iov), + &out, &in, +NULL ,NULL );/* On error, stop handling until the next kick. */ if (unlikely (head <0 ))break ;
drivers/vhost/vhost.c +298 -1
329 lines not on this page · raw message
drivers/vhost/vhost.h +16 -0
@@ -80,6 +80,13 @@ enum vhost_uaddr_type { VHOST_NUM_ADDRS = 3, }; +struct vhost_desc { +u64 addr; +u32 len; +u16 flags;/* VRING_DESC_F_WRITE, VRING_DESC_F_NEXT */ +u16 id; +}; +/* The virtqueue structure describes a queue attached to a device. */ struct vhost_virtqueue {struct vhost_dev *dev;
@@ -90,6 +97,11 @@ struct vhost_virtqueue {struct vring_desc __user *desc;struct vring_avail __user *avail;struct vring_used __user *used; + +struct vhost_desc *descs; +int ndescs; +int max_descs; +const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;struct eventfd_ctx *call_ctx;
@@ -190,6 +202,10 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argbool vhost_vq_access_ok (struct vhost_virtqueue *vq);bool vhost_log_access_ok (struct vhost_dev *); +int vhost_get_vq_desc_batch (struct vhost_virtqueue *, +struct iovec iov[],unsigned int iov_count, +unsigned int *out_num,unsigned int *in_num, +struct vhost_log *log,unsigned int *log_num); int vhost_get_vq_desc(struct vhost_virtqueue *, struct iovec iov[], unsigned int iov_count, unsigned int *out_num, unsigned int *in_num,
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v2 2/2 vhost: batching fetches
In reply to Michael S. TsirkinWith this patch applied, new and old code perform identically.
Lots of extra optimizations are now possible, e.g. we can fetch multiple heads with copy_from/to_user now. We can get rid of maintaining the log array. Etc etc.
--- drivers/vhost/test.c | 2 +- drivers/vhost/vhost.c | 50 ++++++++++++++++++++++++++++++++++++------- drivers/vhost/vhost.h | 4 +++- 3 files changed, 46 insertions(+), 10 deletions(-)
drivers/vhost/test.c +1 -1
@@ -128,7 +128,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) dev = &n->dev; vqs[VHOST_TEST_VQ ] = &n->vqs[VHOST_TEST_VQ ]; n->vqs[VHOST_TEST_VQ ].handle_kick = handle_vq_kick; -vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV , +vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV +64 ,VHOST_TEST_PKT_WEIGHT ,VHOST_TEST_WEIGHT ); f->private_data = n;
drivers/vhost/vhost.c +42 -8
120 lines not on this page · raw message
drivers/vhost/vhost.h +3 -1
@@ -100,7 +100,9 @@ struct vhost_virtqueue {struct vhost_desc *descs;int ndescs; +int first_desc;int max_descs; +int batch_descs;const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;
@@ -245,7 +247,7 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev,int vhost_init_device_iotlb (struct vhost_dev *d,bool enabled);#define vq_err (vq, fmt, ...) do { \ - pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ + pr_err(pr_fmt(fmt), ##__VA_ARGS__); \ if ((vq)->error_ctx) \ eventfd_signal((vq)->error_ctx, 1);\ } while (0)
Signature · 1 line
MST
Michael S. Tsirkin
maintainerRFC PATCH v2 2/2 vhost: batching fetches
In reply to Michael S. TsirkinWith this patch applied, new and old code perform identically.
Lots of extra optimizations are now possible, e.g. we can fetch multiple heads with copy_from/to_user now. We can get rid of maintaining the log array. Etc etc.
--- drivers/vhost/test.c | 2 +- drivers/vhost/vhost.c | 50 ++++++++++++++++++++++++++++++++++++------- drivers/vhost/vhost.h | 4 +++- 3 files changed, 46 insertions(+), 10 deletions(-)
drivers/vhost/test.c +1 -1
@@ -128,7 +128,7 @@ static int vhost_test_open(struct inode *inode, struct file *f) dev = &n->dev; vqs[VHOST_TEST_VQ ] = &n->vqs[VHOST_TEST_VQ ]; n->vqs[VHOST_TEST_VQ ].handle_kick = handle_vq_kick; -vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV , +vhost_dev_init (dev, vqs,VHOST_TEST_VQ_MAX ,UIO_MAXIOV +64 ,VHOST_TEST_PKT_WEIGHT ,VHOST_TEST_WEIGHT ); f->private_data = n;
drivers/vhost/vhost.c +42 -8
120 lines not on this page · raw message
drivers/vhost/vhost.h +3 -1
@@ -100,7 +100,9 @@ struct vhost_virtqueue {struct vhost_desc *descs;int ndescs; +int first_desc;int max_descs; +int batch_descs;const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS ];struct file *kick;
@@ -245,7 +247,7 @@ ssize_t vhost_chr_write_iter(struct vhost_dev *dev,int vhost_init_device_iotlb (struct vhost_dev *d,bool enabled);#define vq_err (vq, fmt, ...) do { \ - pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \ + pr_err(pr_fmt(fmt), ##__VA_ARGS__); \ if ((vq)->error_ctx) \ eventfd_signal((vq)->error_ctx, 1);\ } while (0)
Signature · 1 line
MST