Skip to contentSkip to conversation

Virtio-over-PCIe on non-MIC

PATCH In review Vincent Whitchurch 2 participants 13 messages Feb 22, 2019 – Feb 26, 2019

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v2 cover Virtio-over-PCIe on non-MIC

This allows the drivers/misc/mic/vop/ code to be used on other architectures and adds a loopback driver to be able to use/test/evaluate it without special hardware.

See v1 for more info:
https://lore.kernel.org/lkml/[email protected]/

v2:
- Rebase on top of the patches which have merged
- Use unsigned long instead of uintptr_t
- Drop "Use consistent DMA" for now since it's not needed for loopback
- Add race fix
- Build on all architectures, not just ARM and x86
- Add sample userspace code

Vincent Whitchurch (7):
mic: vop: Cast pointers to unsigned long
mic: Rename ioremap pointer to remap
mic: vop: Allow building on more systems
mic: vop: Add loopback driver
mic: vop: Fix init race with shared interrupts
samples: mic: Split out vop code from mpssd
samples: mic: Add sample VOP userspace

 drivers/misc/mic/Kconfig            |   13 +-
 drivers/misc/mic/bus/scif_bus.h     |    8 +-
 drivers/misc/mic/bus/vop_bus.h      |    8 +-
 drivers/misc/mic/card/mic_device.c  |    8 +-
 drivers/misc/mic/host/mic_boot.c    |    8 +-
 drivers/misc/mic/scif/scif_map.h    |    4 +-
 drivers/misc/mic/vop/Makefile       |    2 +
 drivers/misc/mic/vop/vop_loopback.c |  382 ++++++++
 drivers/misc/mic/vop/vop_main.c     |   62 +-
 drivers/misc/mic/vop/vop_vringh.c   |   15 +-
 samples/mic/mpssd/.gitignore        |    1 +
 samples/mic/mpssd/Makefile          |    9 +-
 samples/mic/mpssd/mpssd.c           | 1317 +-------------------------
 samples/mic/mpssd/mpssd.h           |    1 +
 samples/mic/mpssd/vop.c             | 1357 +++++++++++++++++++++++++++
 samples/mic/mpssd/vopd.c            |   25 +
 16 files changed, 1866 insertions(+), 1354 deletions(-)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c
 create mode 100644 samples/mic/mpssd/vop.c
 create mode 100644 samples/mic/mpssd/vopd.c
Signature · 1 line
2.20.0
#1 lore raw

PATCH v2 1/7 vop: Cast pointers to unsigned long

Fix these on 32-bit:

vop_vringh.c:711:13: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]

---
 drivers/misc/mic/vop/vop_main.c   | 13 +++++++++----
 drivers/misc/mic/vop/vop_vringh.c |  5 +++--
 2 files changed, 12 insertions(+), 6 deletions(-)
drivers/misc/mic/vop/vop_main.c +9 -4
index 3f96e19e57eb..263e0c3fa7c6 100644
@@ -514,7 +514,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 	vdev->desc = d;
 	vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 	vdev->dnode = dnode;
-	vdev->vdev.priv = (void *)(u64)dnode;
+	vdev->vdev.priv = (void *)(unsigned long)dnode;
 	init_completion(&vdev->reset_done);
 
 	vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
@@ -536,7 +536,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 			offset, type);
 		goto free_irq;
 	}
-	writeq((u64)vdev, &vdev->dc->vdev);
+	writeq((unsigned long)vdev, &vdev->dc->vdev);
 	dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
 		__func__, offset, type, vdev);
 
@@ -563,13 +563,18 @@ static int vop_match_desc(struct device *dev, void *data)
 	return vdev->desc == (void __iomem *)data;
 }
 
+static struct _vop_vdev *vop_dc_to_vdev(struct mic_device_ctrl *dc)
+{
+	return (struct _vop_vdev *)(unsigned long)readq(&dc->vdev);
+}
+
 static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
 				      unsigned int offset,
 				      struct vop_device *vpdev)
 {
 	struct mic_device_ctrl __iomem *dc
 		= (void __iomem *)d + _vop_aligned_desc_size(d);
-	struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
+	struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 
 	if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
 		return;
@@ -588,7 +593,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d,
 {
 	struct mic_device_ctrl __iomem *dc
 		= (void __iomem *)d + _vop_aligned_desc_size(d);
-	struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
+	struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 	u8 status;
 	int ret = -1;
 
drivers/misc/mic/vop/vop_vringh.c +3 -2
index 0bac8bce933c..73d9fe6bc18d 100644
@@ -712,16 +712,17 @@ static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,
 
 	while (len && iov->i < iov->used) {
 		struct kvec *kiov = &iov->iov[iov->i];
+		unsigned long daddr = (unsigned long)kiov->iov_base;
 
 		partlen = min(kiov->iov_len, len);
 		if (read)
 			ret = vop_virtio_copy_to_user(vdev, ubuf, partlen,
-						      (u64)kiov->iov_base,
+						      daddr,
 						      kiov->iov_len,
 						      vr_idx);
 		else
 			ret = vop_virtio_copy_from_user(vdev, ubuf, partlen,
-							(u64)kiov->iov_base,
+							daddr,
 							kiov->iov_len,
 							vr_idx);
 		if (ret) {
Signature · 1 line
2.20.0
#2 lore raw

PATCH v2 1/7 mic: vop: Cast pointers to unsigned long

In reply to Vincent Whitchurch

Fix these on 32-bit:

vop_vringh.c:711:13: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]

---
 drivers/misc/mic/vop/vop_main.c   | 13 +++++++++----
 drivers/misc/mic/vop/vop_vringh.c |  5 +++--
 2 files changed, 12 insertions(+), 6 deletions(-)
drivers/misc/mic/vop/vop_main.c +9 -4
index 3f96e19e57eb..263e0c3fa7c6 100644
@@ -514,7 +514,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 	vdev->desc = d;
 	vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 	vdev->dnode = dnode;
-	vdev->vdev.priv = (void *)(u64)dnode;
+	vdev->vdev.priv = (void *)(unsigned long)dnode;
 	init_completion(&vdev->reset_done);
 
 	vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
@@ -536,7 +536,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 			offset, type);
 		goto free_irq;
 	}
-	writeq((u64)vdev, &vdev->dc->vdev);
+	writeq((unsigned long)vdev, &vdev->dc->vdev);
 	dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
 		__func__, offset, type, vdev);
 
@@ -563,13 +563,18 @@ static int vop_match_desc(struct device *dev, void *data)
 	return vdev->desc == (void __iomem *)data;
 }
 
+static struct _vop_vdev *vop_dc_to_vdev(struct mic_device_ctrl *dc)
+{
+	return (struct _vop_vdev *)(unsigned long)readq(&dc->vdev);
+}
+
 static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
 				      unsigned int offset,
 				      struct vop_device *vpdev)
 {
 	struct mic_device_ctrl __iomem *dc
 		= (void __iomem *)d + _vop_aligned_desc_size(d);
-	struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
+	struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 
 	if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
 		return;
@@ -588,7 +593,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d,
 {
 	struct mic_device_ctrl __iomem *dc
 		= (void __iomem *)d + _vop_aligned_desc_size(d);
-	struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
+	struct _vop_vdev *vdev = vop_dc_to_vdev(dc);
 	u8 status;
 	int ret = -1;
 
drivers/misc/mic/vop/vop_vringh.c +3 -2
index 0bac8bce933c..73d9fe6bc18d 100644
@@ -712,16 +712,17 @@ static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,
 
 	while (len && iov->i < iov->used) {
 		struct kvec *kiov = &iov->iov[iov->i];
+		unsigned long daddr = (unsigned long)kiov->iov_base;
 
 		partlen = min(kiov->iov_len, len);
 		if (read)
 			ret = vop_virtio_copy_to_user(vdev, ubuf, partlen,
-						      (u64)kiov->iov_base,
+						      daddr,
 						      kiov->iov_len,
 						      vr_idx);
 		else
 			ret = vop_virtio_copy_from_user(vdev, ubuf, partlen,
-							(u64)kiov->iov_base,
+							daddr,
 							kiov->iov_len,
 							vr_idx);
 		if (ret) {
Signature · 1 line
2.20.0
#3 lore raw

PATCH v2 2/7 mic: Rename ioremap pointer to remap

In reply to Vincent Whitchurch

Some architectures (like MIPS) implement ioremap as a macro, and this leads to conflicts with the ioremap function pointer in various mic structures.

drivers/misc/mic/vop/vop_vringh.c:
In function 'vop_virtio_init_post':
drivers/misc/mic/vop/vop_vringh.c:86:13:
error: macro "ioremap" passed 3 arguments, but takes just 2

Rename ioremap to remap to fix this. Likewise for iounmap.

---
 drivers/misc/mic/bus/scif_bus.h    |  8 ++++----
 drivers/misc/mic/bus/vop_bus.h     |  8 ++++----
 drivers/misc/mic/card/mic_device.c |  8 ++++----
 drivers/misc/mic/host/mic_boot.c   |  8 ++++----
 drivers/misc/mic/scif/scif_map.h   |  4 ++--
 drivers/misc/mic/vop/vop_main.c    |  7 +++----
 drivers/misc/mic/vop/vop_vringh.c  | 10 +++++-----
 7 files changed, 26 insertions(+), 27 deletions(-)
drivers/misc/mic/bus/scif_bus.h +4 -4
index ff59568219ad..377a4f38cd7e 100644
@@ -88,8 +88,8 @@ struct scif_driver {
  * @send_intr: Send an interrupt to the remote node on a specified doorbell.
  * @send_p2p_intr: Send an interrupt to the peer node on a specified doorbell
  * which is specifically targeted for a peer to peer node.
- * @ioremap: Map a buffer with the specified physical address and length.
- * @iounmap: Unmap a buffer previously mapped.
+ * @remap: Map a buffer with the specified physical address and length.
+ * @unmap: Unmap a buffer previously mapped.
  */
 struct scif_hw_ops {
 	int (*next_db)(struct scif_hw_dev *sdev);
@@ -104,9 +104,9 @@ struct scif_hw_ops {
 	void (*send_intr)(struct scif_hw_dev *sdev, int db);
 	void (*send_p2p_intr)(struct scif_hw_dev *sdev, int db,
 			      struct mic_mw *mw);
-	void __iomem * (*ioremap)(struct scif_hw_dev *sdev,
+	void __iomem * (*remap)(struct scif_hw_dev *sdev,
 				  phys_addr_t pa, size_t len);
-	void (*iounmap)(struct scif_hw_dev *sdev, void __iomem *va);
+	void (*unmap)(struct scif_hw_dev *sdev, void __iomem *va);
 };
 
 int scif_register_driver(struct scif_driver *driver);
drivers/misc/mic/bus/vop_bus.h +4 -4
index fff7a865d721..cf5f3fae573c 100644
@@ -87,8 +87,8 @@ struct vop_driver {
  * @get_dp: Get access to the virtio device page used by the self
  *          node to add/remove/configure virtio devices.
  * @send_intr: Send an interrupt to the peer node on a specified doorbell.
- * @ioremap: Map a buffer with the specified DMA address and length.
- * @iounmap: Unmap a buffer previously mapped.
+ * @remap: Map a buffer with the specified DMA address and length.
+ * @unmap: Unmap a buffer previously mapped.
  * @dma_filter: The DMA filter function to use for obtaining access to
  *		a DMA channel on the peer node.
  */
@@ -104,9 +104,9 @@ struct vop_hw_ops {
 	void __iomem * (*get_remote_dp)(struct vop_device *vpdev);
 	void * (*get_dp)(struct vop_device *vpdev);
 	void (*send_intr)(struct vop_device *vpdev, int db);
-	void __iomem * (*ioremap)(struct vop_device *vpdev,
+	void __iomem * (*remap)(struct vop_device *vpdev,
 				  dma_addr_t pa, size_t len);
-	void (*iounmap)(struct vop_device *vpdev, void __iomem *va);
+	void (*unmap)(struct vop_device *vpdev, void __iomem *va);
 };
 
 struct vop_device *
drivers/misc/mic/card/mic_device.c +4 -4
index e749af48f736..dcd07ef29801 100644
@@ -245,8 +245,8 @@ static struct scif_hw_ops scif_hw_ops = {
 	.next_db = ___mic_next_db,
 	.send_intr = ___mic_send_intr,
 	.send_p2p_intr = ___mic_send_p2p_intr,
-	.ioremap = ___mic_ioremap,
-	.iounmap = ___mic_iounmap,
+	.remap = ___mic_ioremap,
+	.unmap = ___mic_iounmap,
 };
 
 static inline struct mic_driver *vpdev_to_mdrv(struct vop_device *vpdev)
@@ -316,8 +316,8 @@ static struct vop_hw_ops vop_hw_ops = {
 	.next_db = __mic_next_db,
 	.get_remote_dp = __mic_get_remote_dp,
 	.send_intr = __mic_send_intr,
-	.ioremap = __mic_ioremap,
-	.iounmap = __mic_iounmap,
+	.remap = __mic_ioremap,
+	.unmap = __mic_iounmap,
 };
 
 static int mic_request_dma_chans(struct mic_driver *mdrv)
drivers/misc/mic/host/mic_boot.c +4 -4
index 6479435ac96b..079c36f0ce6e 100644
@@ -133,8 +133,8 @@ static struct vop_hw_ops vop_hw_ops = {
 	.get_dp = __mic_get_dp,
 	.get_remote_dp = __mic_get_remote_dp,
 	.send_intr = __mic_send_intr,
-	.ioremap = __mic_ioremap,
-	.iounmap = __mic_iounmap,
+	.remap = __mic_ioremap,
+	.unmap = __mic_iounmap,
 };
 
 static inline struct mic_device *scdev_to_mdev(struct scif_hw_dev *scdev)
@@ -315,8 +315,8 @@ static struct scif_hw_ops scif_hw_ops = {
 	.ack_interrupt = ___mic_ack_interrupt,
 	.next_db = ___mic_next_db,
 	.send_intr = ___mic_send_intr,
-	.ioremap = ___mic_ioremap,
-	.iounmap = ___mic_iounmap,
+	.remap = ___mic_ioremap,
+	.unmap = ___mic_iounmap,
 };
 
 static inline struct mic_device *mbdev_to_mdev(struct mbus_device *mbdev)
drivers/misc/mic/scif/scif_map.h +2 -2
index 3e86360ba5a6..7b380534eba1 100644
@@ -97,7 +97,7 @@ scif_ioremap(dma_addr_t phys, size_t size, struct scif_dev *scifdev)
 		out_virt = phys_to_virt(phys);
 	else
 		out_virt = (void __force *)
-			   sdev->hw_ops->ioremap(sdev, phys, size);
+			   sdev->hw_ops->remap(sdev, phys, size);
 	return out_virt;
 }
 
@@ -107,7 +107,7 @@ scif_iounmap(void *virt, size_t len, struct scif_dev *scifdev)
 	if (!scifdev_self(scifdev)) {
 		struct scif_hw_dev *sdev = scifdev->sdev;
 
-		sdev->hw_ops->iounmap(sdev, (void __force __iomem *)virt);
+		sdev->hw_ops->unmap(sdev, (void __force __iomem *)virt);
 	}
 }
 
drivers/misc/mic/vop/vop_main.c +3 -4
index 263e0c3fa7c6..e37b2c2152a2 100644
@@ -270,7 +270,7 @@ static void vop_del_vq(struct virtqueue *vq, int n)
 	free_pages((unsigned long)vdev->used_virt[n],
 		   get_order(vdev->used_size[n]));
 	vring_del_virtqueue(vq);
-	vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]);
+	vpdev->hw_ops->unmap(vpdev, vdev->vr[n]);
 	vdev->vr[n] = NULL;
 }
 
@@ -338,8 +338,7 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 	memcpy_fromio(&config, vqconfig, sizeof(config));
 	_vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
 	vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
-	va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address),
-			vr_size);
+	va = vpdev->hw_ops->remap(vpdev, le64_to_cpu(config.address), vr_size);
 	if (!va)
 		return ERR_PTR(-ENOMEM);
 	vdev->vr[index] = va;
@@ -393,7 +392,7 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 	free_pages((unsigned long)used,
 		   get_order(vdev->used_size[index]));
 unmap:
-	vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]);
+	vpdev->hw_ops->unmap(vpdev, vdev->vr[index]);
 	return ERR_PTR(err);
 }
 
drivers/misc/mic/vop/vop_vringh.c +5 -5
index 73d9fe6bc18d..3632fce40590 100644
@@ -80,7 +80,7 @@ static void vop_virtio_init_post(struct vop_vdev *vdev)
 			continue;
 		}
 		vdev->vvr[i].vrh.vring.used =
-			(void __force *)vpdev->hw_ops->ioremap(
+			(void __force *)vpdev->hw_ops->remap(
 			vpdev,
 			le64_to_cpu(vqconfig[i].used_address),
 			used_size);
@@ -528,7 +528,7 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf,
 				   int vr_idx)
 {
 	struct vop_device *vpdev = vdev->vpdev;
-	void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len);
+	void __iomem *dbuf = vpdev->hw_ops->remap(vpdev, daddr, len);
 	struct vop_vringh *vvr = &vdev->vvr[vr_idx];
 	struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
 	size_t dma_alignment;
@@ -588,7 +588,7 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf,
 	}
 	err = 0;
 err:
-	vpdev->hw_ops->iounmap(vpdev, dbuf);
+	vpdev->hw_ops->unmap(vpdev, dbuf);
 	dev_dbg(vop_dev(vdev),
 		"%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n",
 		__func__, ubuf, dbuf, len, vr_idx);
@@ -606,7 +606,7 @@ static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf,
 				     int vr_idx)
 {
 	struct vop_device *vpdev = vdev->vpdev;
-	void __iomem *dbuf = vpdev->hw_ops->ioremap(vpdev, daddr, len);
+	void __iomem *dbuf = vpdev->hw_ops->remap(vpdev, daddr, len);
 	struct vop_vringh *vvr = &vdev->vvr[vr_idx];
 	struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev);
 	size_t dma_alignment;
@@ -676,7 +676,7 @@ static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf,
 	vdev->out_bytes += len;
 	err = 0;
 err:
-	vpdev->hw_ops->iounmap(vpdev, dbuf);
+	vpdev->hw_ops->unmap(vpdev, dbuf);
 	dev_dbg(vop_dev(vdev),
 		"%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n",
 		__func__, ubuf, dbuf, len, vr_idx);
Signature · 1 line
2.20.0
#4 lore raw

PATCH v2 3/7 mic: vop: Allow building on more systems

In reply to Vincent Whitchurch

VOP_BUS does not actually depend on x86-64 or PCI or X86_DEV_DMA_OPS.

The dependency on X86_DEV_DMA_OPS has been unnecessary since commit 5657933dbb6e25fe ("treewide: Move dma_ops from struct dev_archdata into struct device").

---
 drivers/misc/mic/Kconfig | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/misc/mic/Kconfig +1 -2
index 227cc7443671..242dcee14689 100644
@@ -38,7 +38,6 @@ comment "VOP Bus Driver"
 
 config VOP_BUS
 	tristate "VOP Bus Driver"
-	depends on 64BIT && PCI && X86 && X86_DEV_DMA_OPS
 	help
 	  This option is selected by any driver which registers a
 	  device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -132,7 +131,7 @@ comment "VOP Driver"
 
 config VOP
 	tristate "VOP Driver"
-	depends on 64BIT && PCI && X86 && VOP_BUS
+	depends on VOP_BUS
 	select VHOST_RING
 	select VIRTIO
 	help
Signature · 1 line
2.20.0
#5 lore raw

PATCH v2 3/7 vop: Allow building on more systems

In reply to Vincent Whitchurch

VOP_BUS does not actually depend on x86-64 or PCI or X86_DEV_DMA_OPS.

The dependency on X86_DEV_DMA_OPS has been unnecessary since commit 5657933dbb6e25fe ("treewide: Move dma_ops from struct dev_archdata into struct device").

---
 drivers/misc/mic/Kconfig | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/misc/mic/Kconfig +1 -2
index 227cc7443671..242dcee14689 100644
@@ -38,7 +38,6 @@ comment "VOP Bus Driver"
 
 config VOP_BUS
 	tristate "VOP Bus Driver"
-	depends on 64BIT && PCI && X86 && X86_DEV_DMA_OPS
 	help
 	  This option is selected by any driver which registers a
 	  device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -132,7 +131,7 @@ comment "VOP Driver"
 
 config VOP
 	tristate "VOP Driver"
-	depends on 64BIT && PCI && X86 && VOP_BUS
+	depends on VOP_BUS
 	select VHOST_RING
 	select VIRTIO
 	help
Signature · 1 line
2.20.0
#6 lore raw

PATCH v2 4/7 mic: vop: Add loopback driver

In reply to Vincent Whitchurch

Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.

---
 drivers/misc/mic/Kconfig            |  10 +
 drivers/misc/mic/vop/Makefile       |   2 +
 drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
index 242dcee14689..2e2f745afb3a 100644
@@ -148,6 +148,16 @@ config VOP
 	  OS and tools for MIC to use with this driver are available from
 	  <http://software.intel.com/en-us/mic-developer>.
 
+config VOP_LOOPBACK
+	tristate "VOP loopback driver"
+	depends on VOP
+	help
+	  This enables a loopback driver to test and evaluate the VOP
+	  infrastructure without actual PCIe hardware.  The host and the guest
+	  sides run under the same kernel.
+
+	  If unsure, say N.
+
 if VOP
 source "drivers/vhost/Kconfig.vringh"
 endif
drivers/misc/mic/vop/Makefile +2 -0
index 78819c8999f1..a6ead25c4418 100644
@@ -7,3 +7,5 @@ obj-m := vop.o
 vop-objs += vop_main.o
 vop-objs += vop_debugfs.o
 vop-objs += vop_vringh.o
+
+obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0

382 lines not on this page · raw message

Signature · 1 line
2.20.0
maintainer
#7 lore raw

Re: [PATCH v2 char-misc-next 4/7] mic: vop: Add loopback driver

Quoted 11 lines · Vincent Whitchurch

Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.

---
 drivers/misc/mic/Kconfig            |  10 +
 drivers/misc/mic/vop/Makefile       |   2 +
 drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c

You sent 2 4/7 patches in this series, is that intentional? Are they different?

I've applied the first 3 here, can you rebase and resend the rest of these, without any duplicated patches?

thanks,

greg k-h

maintainer
#8 lore raw

Re: [PATCH v2 char-misc-next 4/7] mic: vop: Add loopback driver

In reply to Vincent Whitchurch
Quoted 11 lines · Vincent Whitchurch

Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.

---
 drivers/misc/mic/Kconfig            |  10 +
 drivers/misc/mic/vop/Makefile       |   2 +
 drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c

You sent 2 4/7 patches in this series, is that intentional? Are they different?

I've applied the first 3 here, can you rebase and resend the rest of these, without any duplicated patches?

thanks,

greg k-h

#9 lore raw

PATCH v2 4/7 vop: Add loopback driver

In reply to Vincent Whitchurch

Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.

---
 drivers/misc/mic/Kconfig            |  10 +
 drivers/misc/mic/vop/Makefile       |   2 +
 drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++
 3 files changed, 394 insertions(+)
 create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
index 242dcee14689..2e2f745afb3a 100644
@@ -148,6 +148,16 @@ config VOP
 	  OS and tools for MIC to use with this driver are available from
 	  <http://software.intel.com/en-us/mic-developer>.
 
+config VOP_LOOPBACK
+	tristate "VOP loopback driver"
+	depends on VOP
+	help
+	  This enables a loopback driver to test and evaluate the VOP
+	  infrastructure without actual PCIe hardware.  The host and the guest
+	  sides run under the same kernel.
+
+	  If unsure, say N.
+
 if VOP
 source "drivers/vhost/Kconfig.vringh"
 endif
drivers/misc/mic/vop/Makefile +2 -0
index 78819c8999f1..a6ead25c4418 100644
@@ -7,3 +7,5 @@ obj-m := vop.o
 vop-objs += vop_main.o
 vop-objs += vop_debugfs.o
 vop-objs += vop_vringh.o
+
+obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0

382 lines not on this page · raw message

Signature · 1 line
2.20.0
#10 lore raw

PATCH v2 5/7 mic: vop: Fix init race with shared interrupts

In reply to Vincent Whitchurch

If a vop virtio device's interrupt is shared with others, there is a window where the interrupt can hit and the ->vqs list can be attempted to be traversed before register_virtio_device() has initialized it, leading to a NULL pointer dereference in vop_virtio_intr_handler().

Fix this by keeping a local list of virtqueues in this driver and using that instead of the list inside the struct virtio_device, similar to how virtio-pci handles this.

---
 drivers/misc/mic/vop/vop_main.c | 42 ++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
drivers/misc/mic/vop/vop_main.c +39 -3
index e37b2c2152a2..6764feea6f55 100644
@@ -40,6 +40,11 @@
 
 #define VOP_MAX_VRINGS 4
 
+struct vop_vq {
+	struct virtqueue *vq;
+	struct list_head list;
+};
+
 /*
  * _vop_vdev - Allocated per virtio device instance injected by the peer.
  *
@@ -68,6 +73,9 @@ struct _vop_vdev {
 	int used_size[VOP_MAX_VRINGS];
 	struct completion reset_done;
 	struct mic_irq *virtio_cookie;
+	struct vop_vq *vqs;
+	struct list_head virtqueues;
+	spinlock_t virtqueues_lock;
 	int c2h_vdev_db;
 	int h2c_vdev_db;
 	int dnode;
@@ -264,6 +272,12 @@ static void vop_del_vq(struct virtqueue *vq, int n)
 {
 	struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
 	struct vop_device *vpdev = vdev->vpdev;
+	struct vop_vq *vopvq = &vdev->vqs[n];
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+	list_del(&vopvq->list);
+	spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 
 	dma_unmap_single(&vpdev->dev, vdev->used[n],
 			 vdev->used_size[n], DMA_BIDIRECTIONAL);
@@ -284,6 +298,9 @@ static void vop_del_vqs(struct virtio_device *dev)
 
 	list_for_each_entry_safe(vq, n, &dev->vqs, list)
 		vop_del_vq(vq, idx++);
+
+	kfree(vdev->vqs);
+	vdev->vqs = NULL;
 }
 
 static struct virtqueue *vop_new_virtqueue(unsigned int index,
@@ -411,7 +428,13 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 	if (nvqs > ioread8(&vdev->desc->num_vq))
 		return -ENOENT;
 
+	vdev->vqs = kcalloc(nvqs, sizeof(*vdev->vqs), GFP_KERNEL);
+	if (!vdev->vqs)
+		return -ENOMEM;
+
 	for (i = 0; i < nvqs; ++i) {
+		unsigned long flags;
+
 		if (!names[i]) {
 			vqs[i] = NULL;
 			continue;
@@ -425,6 +448,12 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 			err = PTR_ERR(vqs[i]);
 			goto error;
 		}
+
+		vdev->vqs[i].vq = vqs[i];
+
+		spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+		list_add(&vdev->vqs[i].list, &vdev->virtqueues);
+		spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 	}
 
 	iowrite8(1, &dc->used_address_updated);
@@ -468,13 +497,17 @@ static struct virtio_config_ops vop_vq_config_ops = {
 
 static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
 {
+	unsigned long flags;
 	struct _vop_vdev *vdev = data;
 	struct vop_device *vpdev = vdev->vpdev;
-	struct virtqueue *vq;
+	struct vop_vq *vopvq;
 
 	vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
-	list_for_each_entry(vq, &vdev->vdev.vqs, list)
-		vring_interrupt(0, vq);
+
+	spin_lock_irqsave(&vdev->virtqueues_lock, flags);
+	list_for_each_entry(vopvq, &vdev->virtqueues, list)
+		vring_interrupt(0, vopvq->vq);
+	spin_unlock_irqrestore(&vdev->virtqueues_lock, flags);
 
 	return IRQ_HANDLED;
 }
@@ -516,6 +549,9 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 	vdev->vdev.priv = (void *)(unsigned long)dnode;
 	init_completion(&vdev->reset_done);
 
+	INIT_LIST_HEAD(&vdev->virtqueues);
+	spin_lock_init(&vdev->virtqueues_lock);
+
 	vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
 	vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
 			vop_virtio_intr_handler, "virtio intr",
Signature · 1 line
2.20.0
#11 lore raw

PATCH v2 6/7 samples: mic: Split out vop code from mpssd

In reply to Vincent Whitchurch

mpssd.c contains both code to boot the MIC hardware and code to serve virtio devices over vop. Split the vop-related code to a separate file so that it can be used by other hardware and vop-loopback.

Except for changes to change_virtblk_backend() and the introduction of serve_virtio(), the rest is just movement of unmodified code between files.

---
 samples/mic/mpssd/Makefile |    4 +-
 samples/mic/mpssd/mpssd.c  | 1317 +---------------------------------
 samples/mic/mpssd/mpssd.h  |    1 +
 samples/mic/mpssd/vop.c    | 1357 ++++++++++++++++++++++++++++++++++++
 4 files changed, 1364 insertions(+), 1315 deletions(-)
 create mode 100644 samples/mic/mpssd/vop.c
samples/mic/mpssd/Makefile +2 -2
index a7a6e0c70424..bc94054dbe2b 100644
@@ -14,8 +14,8 @@ CFLAGS += -DDEBUG=$(DEBUG)
 endif
 
 all: $(PROGS)
-mpssd: mpssd.c sysfs.c
-	$(CC) $(CFLAGS) mpssd.c sysfs.c -o mpssd -lpthread
+mpssd: mpssd.c sysfs.c vop.o
+	$(CC) $(CFLAGS) $^ -o $@ -lpthread
 
 install:
 	install mpssd /usr/sbin/mpssd
samples/mic/mpssd/mpssd.c +4 -1313

1,351 lines not on this page · raw message

samples/mic/mpssd/mpssd.h +1 -0
index 8bd64944aacc..f3e8c13e3948 100644
@@ -100,4 +100,5 @@ __attribute__((format(printf, 1, 2)))
 void mpsslog(char *format, ...);
 char *readsysfs(char *dir, char *entry);
 int setsysfs(char *dir, char *entry, char *value);
+void serve_virtio(struct mic_info *mic, struct mic_info *list);
 #endif
samples/mic/mpssd/vop.c new file +1357 -0

1,357 lines not on this page · raw message

Signature · 1 line
2.20.0
#12 lore raw

PATCH v2 7/7 samples: mic: Add sample VOP userspace

In reply to Vincent Whitchurch

Add a sample userspace program which can be used with the VOP framework with vop-loopback and non-MIC hardware.

For example, the following commands can be used to test networking over vop-loopback. (IPv6 is used with zone indices to force traffic through the interfaces instead of short-circuit delivery, since both interfaces are on the same system.)

 # modprobe vop_loopback
 # ./vopd &

Setup host interface:

 # ip a a dev mic0 scope link fe80::0
 # ip link set dev mic0 up

Setup guest interface:

 # ip a a dev eth1 scope link fe80::1
 # ip link set dev eth1 up

Ping from host to guest:

 # ping6 fe80::1%mic0

Ping from guest to host:

 # ping6 fe80::0%eth1
---
 samples/mic/mpssd/.gitignore |  1 +
 samples/mic/mpssd/Makefile   |  5 ++++-
 samples/mic/mpssd/vopd.c     | 25 +++++++++++++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 samples/mic/mpssd/vopd.c
samples/mic/mpssd/.gitignore +1 -0
index 8b7c72f07c92..34c001135abb 100644
@@ -1 +1,2 @@
 mpssd
+vopd
samples/mic/mpssd/Makefile +4 -1
index bc94054dbe2b..b94b18de5a43 100644
@@ -5,7 +5,7 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/)
 
 ifeq ($(ARCH),x86)
 
-PROGS := mpssd
+PROGS := mpssd vopd
 CC = $(CROSS_COMPILE)gcc
 CFLAGS := -I../../../usr/include -I../../../tools/include
 
@@ -17,6 +17,9 @@ all: $(PROGS)
 mpssd: mpssd.c sysfs.c vop.o
 	$(CC) $(CFLAGS) $^ -o $@ -lpthread
 
+vopd: vopd.c vop.o
+	$(CC) $(CFLAGS) $^ -o $@ -lpthread
+
 install:
 	install mpssd /usr/sbin/mpssd
 	install micctrl /usr/sbin/micctrl
samples/mic/mpssd/vopd.c new file +25 -0
new file mode 100644
index 000000000000..48c1877fed28
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <stdio.h>
+
+#include "mpssd.h"
+
+void mpsslog(char *format, ...)
+{
+	va_list args;
+
+	va_start(args, format);
+	vprintf(format, args);
+	va_end(args);
+}
+
+int main(int argc, char *argv[])
+{
+	struct mic_info themic = {
+		.name = "mic",
+	};
+
+	serve_virtio(&themic, &themic);
+
+	return 0;
+}
Signature · 1 line
2.20.0