virtio: skip avail/used index reads
Subsystems (6)
Michael S. Tsirkin
maintainerRFC PATCH 1/1 virtio: skip avail/used index reads
This adds a new vring feature bit: when enabled, host and guest poll the available/used ring directly instead of looking at the index field first.
To guarantee it is possible to detect updates, the high bits (above vring.num - 1) in the ring head ID value are modified to match the index bits - these change on each wrap-around. Writer also XORs this with 0x8000 such that rings can be zero-initialized.
Reader is modified to ignore these high bits when looking up descriptors.
The point is to reduce the number of cacheline misses for both reads and writes.
I see a speedup of about 20% on a multithreaded micro-benchmark (virtio-test), but regression of about 2% on a single-threaded one (vring_bench). I think this has to do with the fact that complete_multi_user is implemented suboptimally.
TODO:
investigate single-threaded regression better name for a feature flag split the patch to make it easier to review look at more aggressive ring layout changes write a spec patch
This is on top of the following patches in my tree:
virtio_ring: Shadow available ring flags & index
vhost: replace % with & on data path tools/virtio: fix byteswap logic tools/virtio: move list macro stubs
--- drivers/vhost/vhost.h | 3 +- include/linux/vringh.h | 3 + include/uapi/linux/virtio_ring.h | 3 + drivers/vhost/vhost.c | 104 ++++++++++++++++++-------- drivers/vhost/vringh.c | 153 +++++++++++++++++++++++++++++++++------ drivers/virtio/virtio_ring.c | 40 ++++++++-- tools/virtio/virtio_test.c | 14 +++- 7 files changed, 255 insertions(+), 65 deletions(-)
drivers/vhost/vhost.h +2 -1
@@ -175,7 +175,8 @@ enum { (1ULL <<VIRTIO_RING_F_EVENT_IDX ) | (1ULL <<VHOST_F_LOG_ALL ) | (1ULL <<VIRTIO_F_ANY_LAYOUT ) | - (1ULL << VIRTIO_F_VERSION_1) + (1ULL <<VIRTIO_F_VERSION_1 ) | + (1ULL << VIRTIO_RING_F_POLL) };static inline bool vhost_has_feature (struct vhost_virtqueue *vq,int bit)
include/linux/vringh.h +3 -0
@@ -40,6 +40,9 @@ struct vringh {/* Can we get away with weak barriers? */ bool weak_barriers; +/* Poll ring directly */ +bool poll; +/* Last available index we saw (ie. where we're up to). */ u16 last_avail_idx;
include/uapi/linux/virtio_ring.h +3 -0
@@ -62,6 +62,9 @@* at the end of the used ring. Guest should ignore the used->flags field. */ #define VIRTIO_RING_F_EVENT_IDX 29 +/* Support ring polling */ +#define VIRTIO_RING_F_POLL 33 +/* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ struct vring_desc {/* Address (guest-physical). */
drivers/vhost/vhost.c +73 -31
134 lines not on this page · raw message
drivers/vhost/vringh.c +129 -24
218 lines not on this page · raw message
drivers/virtio/virtio_ring.c +32 -8
@@ -80,6 +80,9 @@ struct vring_virtqueue {/* Last used index we've seen. */ u16 last_used_idx; +/* Poll ring instead of the index */ +bool poll; +/* Last written value to avail->flags */ u16 avail_flags_shadow;
@@ -239,9 +242,14 @@ static inline int virtqueue_add(struct virtqueue *_vq,/* Set token. */ vq->data[head] = data; + avail = vq->avail_idx_shadow & (vq->vring.num -1 ); + +if (vq->poll) { +virtio_wmb (vq->weak_barriers); + head ^= ((vq->avail_idx_shadow ^0x8000 ) & ~(vq->vring.num -1 )); + }/* Put entry in available array (but don't update avail->idx until they * do sync). */ - avail = vq->avail_idx_shadow & (vq->vring.num -1 ); vq->vring.avail->ring[avail] =cpu_to_virtio16 (_vq->vdev, head);/* Descriptors and available array need to be set before we expose the
@@ -488,17 +496,32 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)return NULL ; } -if (!more_used (vq)) { -pr_debug ("No more buffers in queue\n" ); -END_USE (vq); -return NULL ; - } +if (!vq->poll) { +if (!more_used (vq)) { +pr_debug ("No more buffers in queue\n" ); +END_USE (vq); +return NULL ; + } -/* Only get used array entries after they have been exposed by host. */ -virtio_rmb (vq->weak_barriers); +/* Only get used array entries after they have been exposed by host. */ +virtio_rmb (vq->weak_barriers); + } last_used = (vq->last_used_idx & (vq->vring.num -1 )); i =virtio32_to_cpu (_vq->vdev, vq->vring.used->ring[last_used].id); + +if (vq->poll) { +if ((i ^ vq->last_used_idx ^0x8000 ) & ~(vq->vring.num -1 )) { +pr_debug ("No more buffers in queue\n" ); +END_USE (vq); +return NULL ; + } + i &= vq->vring.num -1 ; + +/* Only get used array entries after they have been exposed by host. */ +virtio_rmb (vq->weak_barriers); + } + *len =virtio32_to_cpu (_vq->vdev, vq->vring.used->ring[last_used].len);if (unlikely (i >= vq->vring.num)) {
@@ -764,6 +787,7 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, vq->indirect =virtio_has_feature (vdev,VIRTIO_RING_F_INDIRECT_DESC ); vq->event =virtio_has_feature (vdev,VIRTIO_RING_F_EVENT_IDX ); + vq->poll =virtio_has_feature (vdev,VIRTIO_RING_F_POLL );/* No callback? Tell other side not to bother us. */ if (!callback) {
tools/virtio/virtio_test.c +13 -1
@@ -220,6 +220,14 @@ const struct option longopts[] = { .val = 'e', }, { + .name = "ring-poll", + .val = 'R', + }, + { + .name = "no-ring-poll", + .val = 'r', + }, + { .name = "indirect", .val = 'I', },
@@ -261,7 +269,8 @@ int main(int argc, char **argv) {struct vdev_info dev;unsigned long long features = (1ULL <<VIRTIO_RING_F_INDIRECT_DESC ) | - (1ULL <<VIRTIO_RING_F_EVENT_IDX ) | (1ULL <<VIRTIO_F_VERSION_1 ); + (1ULL <<VIRTIO_RING_F_EVENT_IDX ) | (1ULL <<VIRTIO_F_VERSION_1 ) | + (1ULL <<VIRTIO_RING_F_POLL );int o;bool delayed = false;
@@ -276,6 +285,9 @@ int main(int argc, char **argv)case 'e' : features &= ~(1ULL <<VIRTIO_RING_F_EVENT_IDX );break ; +case 'r' : + features &= ~(1ULL <<VIRTIO_RING_F_POLL ); +break ;case 'h' :help (); goto done;
Signature · 1 line
MST