Virtio-over-PCIe on non-MIC
Subsystems (2)
v2 cover Virtio-over-PCIe on non-MIC
This allows the drivers/misc/mic/vop/ code to be used on other architectures and adds a loopback driver to be able to use/test/evaluate it without special hardware.
See v1 for more info:
https://lore.kernel.org/lkml/[email protected]/
v2:
- Rebase on top of the patches which have merged
- Use unsigned long instead of uintptr_t
- Drop "Use consistent DMA" for now since it's not needed for loopback
- Add race fix
- Build on all architectures, not just ARM and x86
- Add sample userspace code
Vincent Whitchurch (7):
mic: vop: Cast pointers to unsigned long
mic: Rename ioremap pointer to remap
mic: vop: Allow building on more systems
mic: vop: Add loopback driver
mic: vop: Fix init race with shared interrupts
samples: mic: Split out vop code from mpssd
samples: mic: Add sample VOP userspace
drivers/misc/mic/Kconfig | 13 +- drivers/misc/mic/bus/scif_bus.h | 8 +- drivers/misc/mic/bus/vop_bus.h | 8 +- drivers/misc/mic/card/mic_device.c | 8 +- drivers/misc/mic/host/mic_boot.c | 8 +- drivers/misc/mic/scif/scif_map.h | 4 +- drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++ drivers/misc/mic/vop/vop_main.c | 62 +- drivers/misc/mic/vop/vop_vringh.c | 15 +- samples/mic/mpssd/.gitignore | 1 + samples/mic/mpssd/Makefile | 9 +- samples/mic/mpssd/mpssd.c | 1317 +------------------------- samples/mic/mpssd/mpssd.h | 1 + samples/mic/mpssd/vop.c | 1357 +++++++++++++++++++++++++++ samples/mic/mpssd/vopd.c | 25 + 16 files changed, 1866 insertions(+), 1354 deletions(-) create mode 100644 drivers/misc/mic/vop/vop_loopback.c create mode 100644 samples/mic/mpssd/vop.c create mode 100644 samples/mic/mpssd/vopd.c
Signature · 1 line
2.20.0
PATCH v2 1/7 vop: Cast pointers to unsigned long
Fix these on 32-bit:
vop_vringh.c:711:13: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]
--- drivers/misc/mic/vop/vop_main.c | 13 +++++++++---- drivers/misc/mic/vop/vop_vringh.c | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-)
drivers/misc/mic/vop/vop_main.c +9 -4
@@ -514,7 +514,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, vdev->desc = d; vdev->dc = (void __iomem *)d +_vop_aligned_desc_size (d); vdev->dnode = dnode; - vdev->vdev.priv = (void *)(u64 )dnode; + vdev->vdev.priv = (void *)(unsigned long )dnode;init_completion (&vdev->reset_done); vdev->h2c_vdev_db = vpdev->hw_ops->next_db (vpdev);
@@ -536,7 +536,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, offset, type); goto free_irq; } -writeq ((u64 )vdev, &vdev->dc->vdev); +writeq ((unsigned long )vdev, &vdev->dc->vdev);dev_dbg (_vop_dev (vdev),"%s: registered vop device %u type %u vdev %p\n" , __func__, offset, type, vdev);
@@ -563,13 +563,18 @@ static int vop_match_desc(struct device *dev, void *data)return vdev->desc == (void __iomem *)data; } +static struct _vop_vdev *vop_dc_to_vdev (struct mic_device_ctrl *dc) +{ +return (struct _vop_vdev *)(unsigned long )readq (&dc->vdev); +} +static void _vop_handle_config_change (struct mic_device_desc __iomem *d,unsigned int offset,struct vop_device *vpdev) {struct mic_device_ctrl __iomem *dc = (void __iomem *)d +_vop_aligned_desc_size (d); -struct _vop_vdev *vdev = (struct _vop_vdev *)readq (&dc->vdev); +struct _vop_vdev *vdev =vop_dc_to_vdev (dc);if (ioread8 (&dc->config_change) !=MIC_VIRTIO_PARAM_CONFIG_CHANGED )return ;
@@ -588,7 +593,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d, {struct mic_device_ctrl __iomem *dc = (void __iomem *)d +_vop_aligned_desc_size (d); -struct _vop_vdev *vdev = (struct _vop_vdev *)readq (&dc->vdev); +struct _vop_vdev *vdev =vop_dc_to_vdev (dc);u8 status;int ret =-1 ;
drivers/misc/mic/vop/vop_vringh.c +3 -2
@@ -712,16 +712,17 @@ static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,while (len && iov->i < iov->used) {struct kvec *kiov = &iov->iov[iov->i]; +unsigned long daddr = (unsigned long )kiov->iov_base; partlen =min (kiov->iov_len, len);if (read) ret =vop_virtio_copy_to_user (vdev, ubuf, partlen, - (u64 )kiov->iov_base, + daddr, kiov->iov_len, vr_idx);else ret =vop_virtio_copy_from_user (vdev, ubuf, partlen, - (u64 )kiov->iov_base, + daddr, kiov->iov_len, vr_idx);if (ret) {
Signature · 1 line
2.20.0
PATCH v2 1/7 mic: vop: Cast pointers to unsigned long
In reply to Vincent WhitchurchFix these on 32-bit:
vop_vringh.c:711:13: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast]
--- drivers/misc/mic/vop/vop_main.c | 13 +++++++++---- drivers/misc/mic/vop/vop_vringh.c | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-)
drivers/misc/mic/vop/vop_main.c +9 -4
@@ -514,7 +514,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, vdev->desc = d; vdev->dc = (void __iomem *)d +_vop_aligned_desc_size (d); vdev->dnode = dnode; - vdev->vdev.priv = (void *)(u64 )dnode; + vdev->vdev.priv = (void *)(unsigned long )dnode;init_completion (&vdev->reset_done); vdev->h2c_vdev_db = vpdev->hw_ops->next_db (vpdev);
@@ -536,7 +536,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, offset, type); goto free_irq; } -writeq ((u64 )vdev, &vdev->dc->vdev); +writeq ((unsigned long )vdev, &vdev->dc->vdev);dev_dbg (_vop_dev (vdev),"%s: registered vop device %u type %u vdev %p\n" , __func__, offset, type, vdev);
@@ -563,13 +563,18 @@ static int vop_match_desc(struct device *dev, void *data)return vdev->desc == (void __iomem *)data; } +static struct _vop_vdev *vop_dc_to_vdev (struct mic_device_ctrl *dc) +{ +return (struct _vop_vdev *)(unsigned long )readq (&dc->vdev); +} +static void _vop_handle_config_change (struct mic_device_desc __iomem *d,unsigned int offset,struct vop_device *vpdev) {struct mic_device_ctrl __iomem *dc = (void __iomem *)d +_vop_aligned_desc_size (d); -struct _vop_vdev *vdev = (struct _vop_vdev *)readq (&dc->vdev); +struct _vop_vdev *vdev =vop_dc_to_vdev (dc);if (ioread8 (&dc->config_change) !=MIC_VIRTIO_PARAM_CONFIG_CHANGED )return ;
@@ -588,7 +593,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d, {struct mic_device_ctrl __iomem *dc = (void __iomem *)d +_vop_aligned_desc_size (d); -struct _vop_vdev *vdev = (struct _vop_vdev *)readq (&dc->vdev); +struct _vop_vdev *vdev =vop_dc_to_vdev (dc);u8 status;int ret =-1 ;
drivers/misc/mic/vop/vop_vringh.c +3 -2
@@ -712,16 +712,17 @@ static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,while (len && iov->i < iov->used) {struct kvec *kiov = &iov->iov[iov->i]; +unsigned long daddr = (unsigned long )kiov->iov_base; partlen =min (kiov->iov_len, len);if (read) ret =vop_virtio_copy_to_user (vdev, ubuf, partlen, - (u64 )kiov->iov_base, + daddr, kiov->iov_len, vr_idx);else ret =vop_virtio_copy_from_user (vdev, ubuf, partlen, - (u64 )kiov->iov_base, + daddr, kiov->iov_len, vr_idx);if (ret) {
Signature · 1 line
2.20.0
PATCH v2 2/7 mic: Rename ioremap pointer to remap
In reply to Vincent WhitchurchSome architectures (like MIPS) implement ioremap as a macro, and this leads to conflicts with the ioremap function pointer in various mic structures.
drivers/misc/mic/vop/vop_vringh.c:
In function 'vop_virtio_init_post':
drivers/misc/mic/vop/vop_vringh.c:86:13:
error: macro "ioremap" passed 3 arguments, but takes just 2
Rename ioremap to remap to fix this. Likewise for iounmap.
--- drivers/misc/mic/bus/scif_bus.h | 8 ++++---- drivers/misc/mic/bus/vop_bus.h | 8 ++++---- drivers/misc/mic/card/mic_device.c | 8 ++++---- drivers/misc/mic/host/mic_boot.c | 8 ++++---- drivers/misc/mic/scif/scif_map.h | 4 ++-- drivers/misc/mic/vop/vop_main.c | 7 +++---- drivers/misc/mic/vop/vop_vringh.c | 10 +++++----- 7 files changed, 26 insertions(+), 27 deletions(-)
drivers/misc/mic/bus/scif_bus.h +4 -4
@@ -88,8 +88,8 @@ struct scif_driver {* @send_intr: Send an interrupt to the remote node on a specified doorbell. * @send_p2p_intr: Send an interrupt to the peer node on a specified doorbell * which is specifically targeted for a peer to peer node. -* @ioremap: Map a buffer with the specified physical address and length. -* @iounmap: Unmap a buffer previously mapped. +* @remap: Map a buffer with the specified physical address and length. +* @unmap: Unmap a buffer previously mapped. */ struct scif_hw_ops {int (*next_db)(struct scif_hw_dev *sdev);
@@ -104,9 +104,9 @@ struct scif_hw_ops {void (*send_intr)(struct scif_hw_dev *sdev,int db);void (*send_p2p_intr)(struct scif_hw_dev *sdev,int db,struct mic_mw *mw); -void __iomem * (*ioremap)(struct scif_hw_dev *sdev, +void __iomem * (*remap)(struct scif_hw_dev *sdev,phys_addr_t pa,size_t len); -void (*iounmap)(struct scif_hw_dev *sdev,void __iomem *va); +void (*unmap)(struct scif_hw_dev *sdev,void __iomem *va); };int scif_register_driver (struct scif_driver *driver);
drivers/misc/mic/bus/vop_bus.h +4 -4
@@ -87,8 +87,8 @@ struct vop_driver {* @get_dp: Get access to the virtio device page used by the self * node to add/remove/configure virtio devices. * @send_intr: Send an interrupt to the peer node on a specified doorbell. -* @ioremap: Map a buffer with the specified DMA address and length. -* @iounmap: Unmap a buffer previously mapped. +* @remap: Map a buffer with the specified DMA address and length. +* @unmap: Unmap a buffer previously mapped. * @dma_filter: The DMA filter function to use for obtaining access to * a DMA channel on the peer node. */
@@ -104,9 +104,9 @@ struct vop_hw_ops {void __iomem * (*get_remote_dp)(struct vop_device *vpdev);void * (*get_dp)(struct vop_device *vpdev);void (*send_intr)(struct vop_device *vpdev,int db); -void __iomem * (*ioremap)(struct vop_device *vpdev, +void __iomem * (*remap)(struct vop_device *vpdev,dma_addr_t pa,size_t len); -void (*iounmap)(struct vop_device *vpdev,void __iomem *va); +void (*unmap)(struct vop_device *vpdev,void __iomem *va); }; struct vop_device *
drivers/misc/mic/card/mic_device.c +4 -4
@@ -245,8 +245,8 @@ static struct scif_hw_ops scif_hw_ops = { .next_db = ___mic_next_db, .send_intr = ___mic_send_intr, .send_p2p_intr = ___mic_send_p2p_intr, - .ioremap = ___mic_ioremap, - .iounmap = ___mic_iounmap, + .remap = ___mic_ioremap, + .unmap = ___mic_iounmap, };static inline struct mic_driver *vpdev_to_mdrv (struct vop_device *vpdev)
@@ -316,8 +316,8 @@ static struct vop_hw_ops vop_hw_ops = { .next_db = __mic_next_db, .get_remote_dp = __mic_get_remote_dp, .send_intr = __mic_send_intr, - .ioremap = __mic_ioremap, - .iounmap = __mic_iounmap, + .remap = __mic_ioremap, + .unmap = __mic_iounmap, };static int mic_request_dma_chans (struct mic_driver *mdrv)
drivers/misc/mic/host/mic_boot.c +4 -4
@@ -133,8 +133,8 @@ static struct vop_hw_ops vop_hw_ops = { .get_dp = __mic_get_dp, .get_remote_dp = __mic_get_remote_dp, .send_intr = __mic_send_intr, - .ioremap = __mic_ioremap, - .iounmap = __mic_iounmap, + .remap = __mic_ioremap, + .unmap = __mic_iounmap, };static inline struct mic_device *scdev_to_mdev (struct scif_hw_dev *scdev)
@@ -315,8 +315,8 @@ static struct scif_hw_ops scif_hw_ops = { .ack_interrupt = ___mic_ack_interrupt, .next_db = ___mic_next_db, .send_intr = ___mic_send_intr, - .ioremap = ___mic_ioremap, - .iounmap = ___mic_iounmap, + .remap = ___mic_ioremap, + .unmap = ___mic_iounmap, };static inline struct mic_device *mbdev_to_mdev (struct mbus_device *mbdev)
drivers/misc/mic/scif/scif_map.h +2 -2
@@ -97,7 +97,7 @@ scif_ioremap(dma_addr_t phys, size_t size, struct scif_dev *scifdev) out_virt =phys_to_virt (phys); else out_virt = (void __force *) - sdev->hw_ops->ioremap (sdev, phys, size); + sdev->hw_ops->remap (sdev, phys, size);return out_virt; }
@@ -107,7 +107,7 @@ scif_iounmap(void *virt, size_t len, struct scif_dev *scifdev)if (!scifdev_self (scifdev)) {struct scif_hw_dev *sdev = scifdev->sdev; - sdev->hw_ops->iounmap (sdev, (void __force __iomem *)virt); + sdev->hw_ops->unmap (sdev, (void __force __iomem *)virt); } }
drivers/misc/mic/vop/vop_main.c +3 -4
@@ -270,7 +270,7 @@ static void vop_del_vq(struct virtqueue *vq, int n)free_pages ((unsigned long )vdev->used_virt[n],get_order (vdev->used_size[n]));vring_del_virtqueue (vq); - vpdev->hw_ops->iounmap (vpdev, vdev->vr[n]); + vpdev->hw_ops->unmap (vpdev, vdev->vr[n]); vdev->vr[n] =NULL ; }
@@ -338,8 +338,7 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,memcpy_fromio (&config, vqconfig,sizeof (config)); _vr_size =vring_size (le16_to_cpu (config.num),MIC_VIRTIO_RING_ALIGN ); vr_size =PAGE_ALIGN (_vr_size +sizeof (struct _mic_vring_info )); - va = vpdev->hw_ops->ioremap (vpdev,le64_to_cpu (config.address), - vr_size); + va = vpdev->hw_ops->remap (vpdev,le64_to_cpu (config.address), vr_size);if (!va)return ERR_PTR (-ENOMEM ); vdev->vr[index] = va;
@@ -393,7 +392,7 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,free_pages ((unsigned long )used,get_order (vdev->used_size[index])); unmap: - vpdev->hw_ops->iounmap (vpdev, vdev->vr[index]); + vpdev->hw_ops->unmap (vpdev, vdev->vr[index]);return ERR_PTR (err); }
drivers/misc/mic/vop/vop_vringh.c +5 -5
@@ -80,7 +80,7 @@ static void vop_virtio_init_post(struct vop_vdev *vdev)continue ; } vdev->vvr[i].vrh.vring.used = - (void __force *)vpdev->hw_ops->ioremap ( + (void __force *)vpdev->hw_ops->remap ( vpdev,le64_to_cpu (vqconfig[i].used_address), used_size);
@@ -528,7 +528,7 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, int vr_idx) {struct vop_device *vpdev = vdev->vpdev; -void __iomem *dbuf = vpdev->hw_ops->ioremap (vpdev, daddr, len); +void __iomem *dbuf = vpdev->hw_ops->remap (vpdev, daddr, len);struct vop_vringh *vvr = &vdev->vvr[vr_idx];struct vop_info *vi =dev_get_drvdata (&vpdev->dev);size_t dma_alignment;
@@ -588,7 +588,7 @@ static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf, } err =0 ; err: - vpdev->hw_ops->iounmap (vpdev, dbuf); + vpdev->hw_ops->unmap (vpdev, dbuf);dev_dbg (vop_dev (vdev),"%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n" , __func__, ubuf, dbuf, len, vr_idx);
@@ -606,7 +606,7 @@ static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf, int vr_idx) {struct vop_device *vpdev = vdev->vpdev; -void __iomem *dbuf = vpdev->hw_ops->ioremap (vpdev, daddr, len); +void __iomem *dbuf = vpdev->hw_ops->remap (vpdev, daddr, len);struct vop_vringh *vvr = &vdev->vvr[vr_idx];struct vop_info *vi =dev_get_drvdata (&vdev->vpdev->dev);size_t dma_alignment;
@@ -676,7 +676,7 @@ static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf, vdev->out_bytes += len; err =0 ; err: - vpdev->hw_ops->iounmap (vpdev, dbuf); + vpdev->hw_ops->unmap (vpdev, dbuf);dev_dbg (vop_dev (vdev),"%s: ubuf %p dbuf %p len 0x%zx vr_idx 0x%x\n" , __func__, ubuf, dbuf, len, vr_idx);
Signature · 1 line
2.20.0
PATCH v2 3/7 mic: vop: Allow building on more systems
In reply to Vincent WhitchurchVOP_BUS does not actually depend on x86-64 or PCI or X86_DEV_DMA_OPS.
The dependency on X86_DEV_DMA_OPS has been unnecessary since commit 5657933dbb6e25fe ("treewide: Move dma_ops from struct dev_archdata into struct device").
--- drivers/misc/mic/Kconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/misc/mic/Kconfig +1 -2
@@ -38,7 +38,6 @@ comment "VOP Bus Driver" config VOP_BUS tristate "VOP Bus Driver" - depends on 64BIT && PCI && X86 && X86_DEV_DMA_OPS help This option is selected by any driver which registers a device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -132,7 +131,7 @@ comment "VOP Driver" config VOP tristate "VOP Driver" - depends on 64BIT && PCI && X86 && VOP_BUS + depends on VOP_BUS select VHOST_RING select VIRTIO help
Signature · 1 line
2.20.0
PATCH v2 3/7 vop: Allow building on more systems
In reply to Vincent WhitchurchVOP_BUS does not actually depend on x86-64 or PCI or X86_DEV_DMA_OPS.
The dependency on X86_DEV_DMA_OPS has been unnecessary since commit 5657933dbb6e25fe ("treewide: Move dma_ops from struct dev_archdata into struct device").
--- drivers/misc/mic/Kconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/misc/mic/Kconfig +1 -2
@@ -38,7 +38,6 @@ comment "VOP Bus Driver" config VOP_BUS tristate "VOP Bus Driver" - depends on 64BIT && PCI && X86 && X86_DEV_DMA_OPS help This option is selected by any driver which registers a device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -132,7 +131,7 @@ comment "VOP Driver" config VOP tristate "VOP Driver" - depends on 64BIT && PCI && X86 && VOP_BUS + depends on VOP_BUS select VHOST_RING select VIRTIO help
Signature · 1 line
2.20.0
PATCH v2 4/7 mic: vop: Add loopback driver
In reply to Vincent WhitchurchAdd a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.
--- drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
@@ -148,6 +148,16 @@ config VOP OS and tools for MIC to use with this driver are available from <http://software.intel.com/en-us/mic-developer>. +config VOP_LOOPBACK + tristate "VOP loopback driver" + depends on VOP + help + This enables a loopback driver to test and evaluate the VOP + infrastructure without actual PCIe hardware. The host and the guest + sides run under the same kernel. + + If unsure, say N. + if VOP source "drivers/vhost/Kconfig.vringh" endif
drivers/misc/mic/vop/Makefile +2 -0
@@ -7,3 +7,5 @@ obj-m := vop.o vop-objs += vop_main.o vop-objs += vop_debugfs.o vop-objs += vop_vringh.o + +obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0
382 lines not on this page · raw message
Signature · 1 line
2.20.0
Greg KH
maintainerRe: [PATCH v2 char-misc-next 4/7] mic: vop: Add loopback driver
Quoted 11 lines · Vincent Whitchurch
Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.
Signed-off-by Vincent Whitchurch <[email protected]>--- drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 drivers/misc/mic/vop/vop_loopback.c
You sent 2 4/7 patches in this series, is that intentional? Are they different?
I've applied the first 3 here, can you rebase and resend the rest of these, without any duplicated patches?
thanks,
greg k-h
Greg KH
maintainerRe: [PATCH v2 char-misc-next 4/7] mic: vop: Add loopback driver
In reply to Vincent WhitchurchQuoted 11 lines · Vincent Whitchurch
Add a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.
Signed-off-by Vincent Whitchurch <[email protected]>--- drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 drivers/misc/mic/vop/vop_loopback.c
You sent 2 4/7 patches in this series, is that intentional? Are they different?
I've applied the first 3 here, can you rebase and resend the rest of these, without any duplicated patches?
thanks,
greg k-h
PATCH v2 4/7 vop: Add loopback driver
In reply to Vincent WhitchurchAdd a loopback driver to allow testing and evaluation of the VOP framework without special hardware. The host and the guest will run under the same kernel.
--- drivers/misc/mic/Kconfig | 10 + drivers/misc/mic/vop/Makefile | 2 + drivers/misc/mic/vop/vop_loopback.c | 382 ++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 drivers/misc/mic/vop/vop_loopback.c
drivers/misc/mic/Kconfig +10 -0
@@ -148,6 +148,16 @@ config VOP OS and tools for MIC to use with this driver are available from <http://software.intel.com/en-us/mic-developer>. +config VOP_LOOPBACK + tristate "VOP loopback driver" + depends on VOP + help + This enables a loopback driver to test and evaluate the VOP + infrastructure without actual PCIe hardware. The host and the guest + sides run under the same kernel. + + If unsure, say N. + if VOP source "drivers/vhost/Kconfig.vringh" endif
drivers/misc/mic/vop/Makefile +2 -0
@@ -7,3 +7,5 @@ obj-m := vop.o vop-objs += vop_main.o vop-objs += vop_debugfs.o vop-objs += vop_vringh.o + +obj-$(CONFIG_VOP_LOOPBACK) += vop_loopback.o
drivers/misc/mic/vop/vop_loopback.c new file +382 -0
382 lines not on this page · raw message
Signature · 1 line
2.20.0
PATCH v2 5/7 mic: vop: Fix init race with shared interrupts
In reply to Vincent WhitchurchIf a vop virtio device's interrupt is shared with others, there is a window where the interrupt can hit and the ->vqs list can be attempted to be traversed before register_virtio_device() has initialized it, leading to a NULL pointer dereference in vop_virtio_intr_handler().
Fix this by keeping a local list of virtqueues in this driver and using that instead of the list inside the struct virtio_device, similar to how virtio-pci handles this.
--- drivers/misc/mic/vop/vop_main.c | 42 ++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-)
drivers/misc/mic/vop/vop_main.c +39 -3
@@ -40,6 +40,11 @@#define VOP_MAX_VRINGS 4 +struct vop_vq { +struct virtqueue *vq; +struct list_head list; +}; +/* * _vop_vdev - Allocated per virtio device instance injected by the peer. *
@@ -68,6 +73,9 @@ struct _vop_vdev {int used_size[VOP_MAX_VRINGS ];struct completion reset_done;struct mic_irq *virtio_cookie; +struct vop_vq *vqs; +struct list_head virtqueues; +spinlock_t virtqueues_lock;int c2h_vdev_db;int h2c_vdev_db;int dnode;
@@ -264,6 +272,12 @@ static void vop_del_vq(struct virtqueue *vq, int n) {struct _vop_vdev *vdev =to_vopvdev (vq->vdev);struct vop_device *vpdev = vdev->vpdev; +struct vop_vq *vopvq = &vdev->vqs[n]; +unsigned long flags; + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_del (&vopvq->list); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags);dma_unmap_single (&vpdev->dev, vdev->used[n], vdev->used_size[n],DMA_BIDIRECTIONAL );
@@ -284,6 +298,9 @@ static void vop_del_vqs(struct virtio_device *dev)list_for_each_entry_safe (vq, n, &dev->vqs, list)vop_del_vq (vq, idx++); + +kfree (vdev->vqs); + vdev->vqs =NULL ; } static struct virtqueue *vop_new_virtqueue(unsigned int index,
@@ -411,7 +428,13 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,if (nvqs >ioread8 (&vdev->desc->num_vq))return -ENOENT ; + vdev->vqs =kcalloc (nvqs,sizeof (*vdev->vqs),GFP_KERNEL ); +if (!vdev->vqs) +return -ENOMEM ; +for (i =0 ; i < nvqs; ++i) { +unsigned long flags; +if (!names[i]) { vqs[i] =NULL ;continue ;
@@ -425,6 +448,12 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs, err =PTR_ERR (vqs[i]); goto error; } + + vdev->vqs[i].vq = vqs[i]; + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_add (&vdev->vqs[i].list, &vdev->virtqueues); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags); }iowrite8 (1 , &dc->used_address_updated);
@@ -468,13 +497,17 @@ static struct virtio_config_ops vop_vq_config_ops = {static irqreturn_t vop_virtio_intr_handler (int irq,void *data) { +unsigned long flags;struct _vop_vdev *vdev = data;struct vop_device *vpdev = vdev->vpdev; -struct virtqueue *vq; +struct vop_vq *vopvq; vpdev->hw_ops->ack_interrupt (vpdev, vdev->h2c_vdev_db); -list_for_each_entry (vq, &vdev->vdev.vqs, list) -vring_interrupt (0 , vq); + +spin_lock_irqsave (&vdev->virtqueues_lock, flags); +list_for_each_entry (vopvq, &vdev->virtqueues, list) +vring_interrupt (0 , vopvq->vq); +spin_unlock_irqrestore (&vdev->virtqueues_lock, flags);return IRQ_HANDLED ; }
@@ -516,6 +549,9 @@ static int _vop_add_device(struct mic_device_desc __iomem *d, vdev->vdev.priv = (void *)(unsigned long )dnode;init_completion (&vdev->reset_done); +INIT_LIST_HEAD (&vdev->virtqueues); +spin_lock_init (&vdev->virtqueues_lock); + vdev->h2c_vdev_db = vpdev->hw_ops->next_db (vpdev); vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev, vop_virtio_intr_handler, "virtio intr",
Signature · 1 line
2.20.0
PATCH v2 6/7 samples: mic: Split out vop code from mpssd
In reply to Vincent Whitchurchmpssd.c contains both code to boot the MIC hardware and code to serve virtio devices over vop. Split the vop-related code to a separate file so that it can be used by other hardware and vop-loopback.
Except for changes to change_virtblk_backend() and the introduction of serve_virtio(), the rest is just movement of unmodified code between files.
--- samples/mic/mpssd/Makefile | 4 +- samples/mic/mpssd/mpssd.c | 1317 +--------------------------------- samples/mic/mpssd/mpssd.h | 1 + samples/mic/mpssd/vop.c | 1357 ++++++++++++++++++++++++++++++++++++ 4 files changed, 1364 insertions(+), 1315 deletions(-) create mode 100644 samples/mic/mpssd/vop.c
samples/mic/mpssd/Makefile +2 -2
@@ -14,8 +14,8 @@ CFLAGS += -DDEBUG=$(DEBUG) endif all: $(PROGS) -mpssd: mpssd.c sysfs.c - $(CC) $(CFLAGS) mpssd.c sysfs.c -o mpssd -lpthread +mpssd: mpssd.c sysfs.c vop.o + $(CC) $(CFLAGS) $^ -o $@ -lpthread install: install mpssd /usr/sbin/mpssd
samples/mic/mpssd/mpssd.c +4 -1313
1,351 lines not on this page · raw message
samples/mic/mpssd/mpssd.h +1 -0
@@ -100,4 +100,5 @@ __attribute__((format(printf, 1, 2)))void mpsslog (char *format, ...);char *readsysfs (char *dir,char *entry);int setsysfs (char *dir,char *entry,char *value); +void serve_virtio (struct mic_info *mic,struct mic_info *list);#endif
samples/mic/mpssd/vop.c new file +1357 -0
1,357 lines not on this page · raw message
Signature · 1 line
2.20.0
PATCH v2 7/7 samples: mic: Add sample VOP userspace
In reply to Vincent WhitchurchAdd a sample userspace program which can be used with the VOP framework with vop-loopback and non-MIC hardware.
For example, the following commands can be used to test networking over vop-loopback. (IPv6 is used with zone indices to force traffic through the interfaces instead of short-circuit delivery, since both interfaces are on the same system.)
# modprobe vop_loopback # ./vopd &
Setup host interface:
# ip a a dev mic0 scope link fe80::0 # ip link set dev mic0 up
Setup guest interface:
# ip a a dev eth1 scope link fe80::1 # ip link set dev eth1 up
Ping from host to guest:
# ping6 fe80::1%mic0
Ping from guest to host:
# ping6 fe80::0%eth1
--- samples/mic/mpssd/.gitignore | 1 + samples/mic/mpssd/Makefile | 5 ++++- samples/mic/mpssd/vopd.c | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 samples/mic/mpssd/vopd.c
samples/mic/mpssd/.gitignore +1 -0
@@ -1 +1,2 @@ mpssd +vopd
samples/mic/mpssd/Makefile +4 -1
@@ -5,7 +5,7 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) ifeq ($(ARCH),x86) -PROGS := mpssd +PROGS := mpssd vopd CC = $(CROSS_COMPILE)gcc CFLAGS := -I../../../usr/include -I../../../tools/include
@@ -17,6 +17,9 @@ all: $(PROGS) mpssd: mpssd.c sysfs.c vop.o $(CC) $(CFLAGS) $^ -o $@ -lpthread +vopd: vopd.c vop.o + $(CC) $(CFLAGS) $^ -o $@ -lpthread + install: install mpssd /usr/sbin/mpssd install micctrl /usr/sbin/micctrl
samples/mic/mpssd/vopd.c new file +25 -0
@@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <stdio.h> + +#include "mpssd.h" + +void mpsslog (char *format, ...) +{ +va_list args; + +va_start (args, format); +vprintf (format, args); +va_end (args); +} + +int main (int argc,char *argv[]) +{ +struct mic_info themic = { + .name ="mic" , + }; + +serve_virtio (&themic, &themic); + +return 0 ; +}
Signature · 1 line
2.20.0