Virtio-over-PCIe loopback
v3 cover Virtio-over-PCIe loopback
This adds a loopback driver to be able to use/test/evaluate the drivers/misc/mic/vop/ code without special hardware.
See v1 for more info:
https://lore.kernel.org/lkml/[email protected]/
v3:
- Rebase on top of merged patches
- Resend since last posting had duplicate patches
v2:
- Rebase on top of the patches which have merged
- Use unsigned long instead of uintptr_t
- Drop "Use consistent DMA" for now since it's not needed for loopback
- Add race fix
- Build on all architectures, not just ARM and x86
- Add sample userspace code
Vincent Whitchurch (4):
mic: vop: Fix init race with shared interrupts
mic: vop: Add loopback driver
samples: mic: Split out vop code from mpssd
samples: mic: Add sample VOP userspace
drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++ drivers/misc/mic/vop/vop_main.c | 42 +- samples/mic/mpssd/.gitignore | 1 + samples/mic/mpssd/Makefile | 9 +- samples/mic/mpssd/mpssd.c | 1317 +------------------------- samples/mic/mpssd/mpssd.h | 1 + samples/mic/mpssd/vop.c | 1357 +++++++++++++++++++++++++++ samples/mic/mpssd/vopd.c | 25 + 10 files changed, 1827 insertions(+), 1319 deletions(-) create mode 100644 drivers/misc/mic/vop/vop_loopback.c create mode 100644 samples/mic/mpssd/vop.c create mode 100644 samples/mic/mpssd/vopd.c
Signature · 1 line
2.20.0
PATCH v3 1/4 mic: vop: Fix init race with shared interrupts
If a vop virtio device's interrupt is shared with others, there is a window where the interrupt can hit and the ->vqs list can be attempted to be traversed before register_virtio_device() has initialized it, leading to a NULL pointer dereference in vop_virtio_intr_handler().
Fix this by keeping a local list of virtqueues in this driver and using that instead of the list inside the struct virtio_device, similar to how virtio-pci handles this.
--- drivers/misc/mic/vop/vop_main.c | 42 ++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-)
drivers/misc/mic/vop/vop_main.c +39 -3
@@ -40,6 +40,11 @@#define VOP_MAX_VRINGS 4 +struct vop_vq { +struct virtqueue *vq; +struct list_head list; +}; +/* * _vop_vdev - Allocated per virtio device instance injected by the peer. *
@@ -68,6 +73,9 @@ struct _vop_vdev {int used_size[VOP_MAX_VRINGS ];struct completion reset_done;struct mic_irq *virtio_cookie; +struct vop_vq *vqs; +struct list_head virtqueues; +spinlock_t virtqueues_lock;int c2h_vdev_db;int h2c_vdev_db;int dnode;
@@ -264,6 +272,12 @@ static void vop_del_vq(struct virtqueue *vq, int n) {struct _vop_vdev *vdev =to_vopvdev (vq->vdev);struct vop_device *vpdev = vdev->vpdev; +struct vop_vq *vopvq = &vdev->vqs[n]; +unsigned long flags; + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_del (&vopvq->list); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags);dma_unmap_single (&vpdev->dev, vdev->used[n], vdev->used_size[n],DMA_BIDIRECTIONAL );
@@ -284,6 +298,9 @@ static void vop_del_vqs(struct virtio_device *dev)list_for_each_entry_safe (vq, n, &dev->vqs, list)vop_del_vq (vq, idx++); + +kfree (vdev->vqs); + vdev->vqs =NULL ; } static struct virtqueue *vop_new_virtqueue(unsigned int index,
@@ -411,7 +428,13 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,if (nvqs >ioread8 (&vdev->desc->num_vq))return -ENOENT ; + vdev->vqs =kcalloc (nvqs,sizeof (*vdev->vqs),GFP_KERNEL ); +if (!vdev->vqs) +return -ENOMEM ; +for (i =0 ; i < nvqs; ++i) { +unsigned long flags; +if (!names[i]) { vqs[i] =NULL ;continue ;
@@ -425,6 +448,12 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs, err =PTR_ERR (vqs[i]); goto error; } + + vdev->vqs[i].vq = vqs[i]; + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_add (&vdev->vqs[i].list, &vdev->virtqueues); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags); }iowrite8 (1 , &dc->used_address_updated);
@@ -468,13 +497,17 @@ static struct virtio_config_ops vop_vq_config_ops = {static irqreturn_t vop_virtio_intr_handler (int irq,void *data) { +unsigned long flags;struct _vop_vdev *vdev = data;struct vop_device *vpdev = vdev->vpdev; -struct virtqueue *vq; +struct vop_vq *vopvq; vpdev->hw_ops->ack_interrupt (vpdev, vdev->h2c_vdev_db); -list_for_each_entry (vq, &vdev->vdev.vqs, list) -vring_interrupt (0 , vq); + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_for_each_entry (vopvq, &vdev->virtqueues, list) +vring_interrupt (0 , vopvq->vq); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags);return IRQ_HANDLED ; }
@@ -516,6 +549,9 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, vdev->vdev.priv = (void *)(unsigned long )dnode;init_completion (&vdev->reset_done); +INIT_LIST_HEAD (&vdev->virtqueues); +spin_lock_init (&vdev->virtqueues_lock); + vdev->h2c_vdev_db = vpdev->hw_ops->next_db (vpdev); vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev, vop_virtio_intr_handler, "virtio intr",
Signature · 1 line
2.20.0
PATCH v3 2/4 mic: vop: Add loopback driver
In reply to Vincent WhitchurchAdd a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.
--- drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
@@ -148,6 +148,16 @@ config VOP OS and tools for MIC to use with this driver are available from <http://software.intel.com/en-us/mic-developer>. +config VOP_LOOPBACK + tristate "VOP loopback driver" + depends on VOP + help + This enables a loopback driver to test and evaluate the VOP + infrastructure without actual PCIe hardware. The host and the guest + sides run under the same kernel. + + If unsure, say N. + if VOP source "drivers/vhost/Kconfig.vringh" endif
drivers/misc/mic/vop/Makefile +2 -0
@@ -7,3 +7,5 @@ obj-m := vop.o vop-objs += vop_main.o vop-objs += vop_debugfs.o vop-objs += vop_vringh.o + +obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0
382 lines not on this page · raw message
Signature · 1 line
2.20.0
PATCH v3 3/4 samples: mic: Split out vop code from mpssd
In reply to Vincent Whitchurchmpssd.c contains both code to boot the MIC hardware and code to serve virtio devices over vop. Split the vop-related code to a separate file so that it can be used by other hardware and vop-loopback.
Except for changes to change_virtblk_backend() and the introduction of serve_virtio(), the rest is just movement of unmodified code between files.
--- samples/mic/mpssd/Makefile | 4 +- samples/mic/mpssd/mpssd.c | 1317 +--------------------------------- samples/mic/mpssd/mpssd.h | 1 + samples/mic/mpssd/vop.c | 1357 ++++++++++++++++++++++++++++++++++++ 4 files changed, 1364 insertions(+), 1315 deletions(-) create mode 100644 samples/mic/mpssd/vop.c
samples/mic/mpssd/Makefile +2 -2
@@ -14,8 +14,8 @@ CFLAGS += -DDEBUG=$(DEBUG) endif all: $(PROGS) -mpssd: mpssd.c sysfs.c - $(CC) $(CFLAGS) mpssd.c sysfs.c -o mpssd -lpthread +mpssd: mpssd.c sysfs.c vop.o + $(CC) $(CFLAGS) $^ -o $@ -lpthread install: install mpssd /usr/sbin/mpssd
samples/mic/mpssd/mpssd.c +4 -1313
1,351 lines not on this page · raw message
samples/mic/mpssd/mpssd.h +1 -0
@@ -97,4 +97,5 @@ __attribute__((format(printf, 1, 2)))void mpsslog (char *format, ...);char *readsysfs (char *dir,char *entry);int setsysfs (char *dir,char *entry,char *value); +void serve_virtio (struct mic_info *mic,struct mic_info *list);#endif
samples/mic/mpssd/vop.c new file +1357 -0
1,357 lines not on this page · raw message
Signature · 1 line
2.20.0
PATCH v3 4/4 samples: mic: Add sample VOP userspace
In reply to Vincent WhitchurchAdd a sample userspace program which can be used with the VOP framework with vop-loopback and non-MIC hardware.
For example, the following commands can be used to test networking over vop-loopback. (IPv6 is used with zone indices to force traffic through the interfaces instead of short-circuit delivery, since both interfaces are on the same system.)
# modprobe vop_loopback # ./vopd &
Setup host interface:
# ip a a dev mic0 scope link fe80::0 # ip link set dev mic0 up
Setup guest interface:
# ip a a dev eth1 scope link fe80::1 # ip link set dev eth1 up
Ping from host to guest:
# ping6 fe80::1%mic0
Ping from guest to host:
# ping6 fe80::0%eth1
--- samples/mic/mpssd/.gitignore | 1 + samples/mic/mpssd/Makefile | 5 ++++- samples/mic/mpssd/vopd.c | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 samples/mic/mpssd/vopd.c
samples/mic/mpssd/.gitignore +1 -0
@@ -1 +1,2 @@ mpssd +vopd
samples/mic/mpssd/Makefile +4 -1
@@ -5,7 +5,7 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) ifeq ($(ARCH),x86) -PROGS := mpssd +PROGS := mpssd vopd CC = $(CROSS_COMPILE)gcc CFLAGS := -I../../../usr/include -I../../../tools/include
@@ -17,6 +17,9 @@ all: $(PROGS) mpssd: mpssd.c sysfs.c vop.o $(CC) $(CFLAGS) $^ -o $@ -lpthread +vopd: vopd.c vop.o + $(CC) $(CFLAGS) $^ -o $@ -lpthread + install: install mpssd /usr/sbin/mpssd install micctrl /usr/sbin/micctrl
samples/mic/mpssd/vopd.c new file +25 -0
@@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <stdio.h> + +#include "mpssd.h" + +void mpsslog (char *format, ...) +{ +va_list args; + +va_start (args, format); +vprintf (format, args); +va_end (args); +} + +int main (int argc,char *argv[]) +{ +struct mic_info themic = { + .name ="mic" , + }; + +serve_virtio (&themic, &themic); + +return 0 ; +}
Signature · 1 line
2.20.0