Skip to contentSkip to conversation

Virtio-over-PCIe loopback

PATCH New Vincent Whitchurch 1 participant 5 messages Apr 3, 2019 Char and misc drivers

lore
Share & tools
lore
#0 lore raw

v3 cover Virtio-over-PCIe loopback

This adds a loopback driver to be able to use/test/evaluate the drivers/misc/mic/vop/ code without special hardware.

See v1 for more info:
https://lore.kernel.org/lkml/[email protected]/

v3:
- Rebase on top of merged patches
- Resend since last posting had duplicate patches

v2:
- Rebase on top of the patches which have merged
- Use unsigned long instead of uintptr_t
- Drop "Use consistent DMA" for now since it's not needed for loopback
- Add race fix
- Build on all architectures, not just ARM and x86
- Add sample userspace code

Vincent Whitchurch (4):
mic: vop: Fix init race with shared interrupts
mic: vop: Add loopback driver
samples: mic: Split out vop code from mpssd
samples: mic: Add sample VOP userspace

 drivers/misc/mic/Kconfig            |   10 +
 drivers/misc/mic/vop/Makefile       |    2 +
 drivers/misc/mic/vop/vop_loopback.c |  382 ++++++++
 drivers/misc/mic/vop/vop_main.c     |   42 +-
 samples/mic/mpssd/.gitignore        |    1 +
 samples/mic/mpssd/Makefile          |    9 +-
 samples/mic/mpssd/mpssd.c           | 1317 +-------------------------
 samples/mic/mpssd/mpssd.h           |    1 +
 samples/mic/mpssd/vop.c             | 1357 +++++++++++++++++++++++++++
 samples/mic/mpssd/vopd.c            |   25 +
 10 files changed, 1827 insertions(+), 1319 deletions(-)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c
 create mode 100644 samples/mic/mpssd/vop.c
 create mode 100644 samples/mic/mpssd/vopd.c
Signature · 1 line
2.20.0
#1 lore raw

PATCH v3 1/4 mic: vop: Fix init race with shared interrupts

If a vop virtio device's interrupt is shared with others, there is a window where the interrupt can hit and the ->vqs list can be attempted to be traversed before register_virtio_device() has initialized it, leading to a NULL pointer dereference in vop_virtio_intr_handler().

Fix this by keeping a local list of virtqueues in this driver and using that instead of the list inside the struct virtio_device, similar to how virtio-pci handles this.

---
 drivers/misc/mic/vop/vop_main.c | 42 ++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
drivers/misc/mic/vop/vop_main.c +39 -3
index e37b2c2152a2..6764feea6f55 100644
@@ -40,6 +40,11 @@
 
 #define VOP_MAX_VRINGS 4
 
+struct vop_vq {
+	struct virtqueue *vq;
+	struct list_head list;
+};
+
 /*
  * _vop_vdev - Allocated per virtio device instance injected by the peer.
  *
@@ -68,6 +73,9 @@ struct _vop_vdev {
 	int used_size[VOP_MAX_VRINGS];
 	struct completion reset_done;
 	struct mic_irq *virtio_cookie;
+	struct vop_vq *vqs;
+	struct list_head virtqueues;
+	spinlock_t virtqueues_lock;
 	int c2h_vdev_db;
 	int h2c_vdev_db;
 	int dnode;
@@ -264,6 +272,12 @@ static void vop_del_vq(struct virtqueue *vq, int n)
 {
 	struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
 	struct vop_device *vpdev = vdev->vpdev;
+	struct vop_vq *vopvq = &vdev->vqs[n];
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+	list_del(&vopvq->list);
+	spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 
 	dma_unmap_single(&vpdev->dev, vdev->used[n],
 			 vdev->used_size[n], DMA_BIDIRECTIONAL);
@@ -284,6 +298,9 @@ static void vop_del_vqs(struct virtio_device *dev)
 
 	list_for_each_entry_safe(vq, n, &dev->vqs, list)
 		vop_del_vq(vq, idx++);
+
+	kfree(vdev->vqs);
+	vdev->vqs = NULL;
 }
 
 static struct virtqueue *vop_new_virtqueue(unsigned int index,
@@ -411,7 +428,13 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 	if (nvqs > ioread8(&vdev->desc->num_vq))
 		return -ENOENT;
 
+	vdev->vqs = kcalloc(nvqs, sizeof(*vdev->vqs), GFP_KERNEL);
+	if (!vdev->vqs)
+		return -ENOMEM;
+
 	for (i = 0; i < nvqs; ++i) {
+		unsigned long flags;
+
 		if (!names[i]) {
 			vqs[i] = NULL;
 			continue;
@@ -425,6 +448,12 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 			err = PTR_ERR(vqs[i]);
 			goto error;
 		}
+
+		vdev->vqs[i].vq = vqs[i];
+
+		spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+		list_add(&vdev->vqs[i].list, &vdev->virtqueues);
+		spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 	}
 
 	iowrite8(1, &dc->used_address_updated);
@@ -468,13 +497,17 @@ static struct virtio_config_ops vop_vq_config_ops = {
 
 static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
 {
+	unsigned long flags;
 	struct _vop_vdev *vdev = data;
 	struct vop_device *vpdev = vdev->vpdev;
-	struct virtqueue *vq;
+	struct vop_vq *vopvq;
 
 	vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
-	list_for_each_entry(vq, &vdev->vdev.vqs, list)
-		vring_interrupt(0, vq);
+
+	spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+	list_for_each_entry(vopvq, &vdev->virtqueues, list)
+		vring_interrupt(0, vopvq->vq);
+	spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 
 	return IRQ_HANDLED;
 }
@@ -516,6 +549,9 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 	vdev->vdev.priv = (void *)(unsigned long)dnode;
 	init_completion(&vdev->reset_done);
 
+	INIT_LIST_HEAD(&vdev->virtqueues);
+	spin_lock_init(&vdev->virtqueues_lock);
+
 	vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
 	vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
 			vop_virtio_intr_handler, "virtio intr",
Signature · 1 line
2.20.0
#2 lore raw

PATCH v3 2/4 mic: vop: Add loopback driver

In reply to Vincent Whitchurch

Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.

---
 drivers/misc/mic/Kconfig            |  10 +
 drivers/misc/mic/vop/Makefile       |   2 +
 drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
index 242dcee14689..2e2f745afb3a 100644
@@ -148,6 +148,16 @@ config VOP
 	  OS and tools for MIC to use with this driver are available from
 	  <http://software.intel.com/en-us/mic-developer>.
 
+config VOP_LOOPBACK
+	tristate "VOP loopback driver"
+	depends on VOP
+	help
+	  This enables a loopback driver to test and evaluate the VOP
+	  infrastructure without actual PCIe hardware.  The host and the guest
+	  sides run under the same kernel.
+
+	  If unsure, say N.
+
 if VOP
 source "drivers/vhost/Kconfig.vringh"
 endif
drivers/misc/mic/vop/Makefile +2 -0
index 78819c8999f1..a6ead25c4418 100644
@@ -7,3 +7,5 @@ obj-m := vop.o
 vop-objs += vop_main.o
 vop-objs += vop_debugfs.o
 vop-objs += vop_vringh.o
+
+obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0

382 lines not on this page · raw message

Signature · 1 line
2.20.0
#3 lore raw

PATCH v3 3/4 samples: mic: Split out vop code from mpssd

In reply to Vincent Whitchurch

mpssd.c contains both code to boot the MIC hardware and code to serve virtio devices over vop. Split the vop-related code to a separate file so that it can be used by other hardware and vop-loopback.

Except for changes to change_virtblk_backend() and the introduction of serve_virtio(), the rest is just movement of unmodified code between files.

---
 samples/mic/mpssd/Makefile |    4 +-
 samples/mic/mpssd/mpssd.c  | 1317 +---------------------------------
 samples/mic/mpssd/mpssd.h  |    1 +
 samples/mic/mpssd/vop.c    | 1357 ++++++++++++++++++++++++++++++++++++
 4 files changed, 1364 insertions(+), 1315 deletions(-)
 create mode 100644 samples/mic/mpssd/vop.c
samples/mic/mpssd/Makefile +2 -2
index a7a6e0c70424..bc94054dbe2b 100644
@@ -14,8 +14,8 @@ CFLAGS += -DDEBUG=$(DEBUG)
 endif
 
 all: $(PROGS)
-mpssd: mpssd.c sysfs.c
-	$(CC) $(CFLAGS) mpssd.c sysfs.c -o mpssd -lpthread
+mpssd: mpssd.c sysfs.c vop.o
+	$(CC) $(CFLAGS) $^ -o $@ -lpthread
 
 install:
 	install mpssd /usr/sbin/mpssd
samples/mic/mpssd/mpssd.c +4 -1313

1,351 lines not on this page · raw message

samples/mic/mpssd/mpssd.h +1 -0
index 82d3b519f0e7..7c2caf6f9a3b 100644
@@ -97,4 +97,5 @@ __attribute__((format(printf, 1, 2)))
 void mpsslog(char *format, ...);
 char *readsysfs(char *dir, char *entry);
 int setsysfs(char *dir, char *entry, char *value);
+void serve_virtio(struct mic_info *mic, struct mic_info *list);
 #endif
samples/mic/mpssd/vop.c new file +1357 -0

1,357 lines not on this page · raw message

Signature · 1 line
2.20.0
#4 lore raw

PATCH v3 4/4 samples: mic: Add sample VOP userspace

In reply to Vincent Whitchurch

Add a sample userspace program which can be used with the VOP framework with vop-loopback and non-MIC hardware.

For example, the following commands can be used to test networking over vop-loopback. (IPv6 is used with zone indices to force traffic through the interfaces instead of short-circuit delivery, since both interfaces are on the same system.)

 # modprobe vop_loopback
 # ./vopd &

Setup host interface:

 # ip a a dev mic0 scope link fe80::0
 # ip link set dev mic0 up

Setup guest interface:

 # ip a a dev eth1 scope link fe80::1
 # ip link set dev eth1 up

Ping from host to guest:

 # ping6 fe80::1%mic0

Ping from guest to host:

 # ping6 fe80::0%eth1
---
 samples/mic/mpssd/.gitignore |  1 +
 samples/mic/mpssd/Makefile   |  5 ++++-
 samples/mic/mpssd/vopd.c     | 25 +++++++++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 samples/mic/mpssd/vopd.c
samples/mic/mpssd/.gitignore +1 -0
index 8b7c72f07c92..34c001135abb 100644
@@ -1 +1,2 @@
 mpssd
+vopd
samples/mic/mpssd/Makefile +4 -1
index bc94054dbe2b..b94b18de5a43 100644
@@ -5,7 +5,7 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/)
 
 ifeq ($(ARCH),x86)
 
-PROGS := mpssd
+PROGS := mpssd vopd
 CC = $(CROSS_COMPILE)gcc
 CFLAGS := -I../../../usr/include -I../../../tools/include
 
@@ -17,6 +17,9 @@ all: $(PROGS)
 mpssd: mpssd.c sysfs.c vop.o
 	$(CC) $(CFLAGS) $^ -o $@ -lpthread
 
+vopd: vopd.c vop.o
+	$(CC) $(CFLAGS) $^ -o $@ -lpthread
+
 install:
 	install mpssd /usr/sbin/mpssd
 	install micctrl /usr/sbin/micctrl
samples/mic/mpssd/vopd.c new file +25 -0
new file mode 100644
index 000000000000..48c1877fed28
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <stdio.h>
+
+#include "mpssd.h"
+
+void mpsslog(char *format, ...)
+{
+	va_list args;
+
+	va_start(args, format);
+	vprintf(format, args);
+	va_end(args);
+}
+
+int main(int argc, char *argv[])
+{
+	struct mic_info themic = {
+		.name = "mic",
+	};
+
+	serve_virtio(&themic, &themic);
+
+	return 0;
+}
Signature · 1 line
2.20.0