Skip to contentSkip to conversation

vfio/cdx: add support for CDX bus

PATCH In review Nipun Gupta 3 participants 6 messages Apr 3, 2023 – Apr 13, 2023

Subsystems (3)
lore
Share & tools
lore

Nipun Gupta

author maintainer
#0 lore raw

PATCH 1/1 vfio/cdx: add support for CDX bus

vfio-cdx driver enables IOCTLs for user space to query MMIO regions for CDX devices and mmap them. This change also adds support for reset of CDX devices.

This change adds the VFIO CDX driver and enables the following ioctls for CDX devices:
- VFIO_DEVICE_GET_INFO:
- VFIO_DEVICE_GET_REGION_INFO
- VFIO_DEVICE_RESET

---

CDX bus is now merged on linux-next tree. So sending VFIO driver patch for CDX bus.

 MAINTAINERS                         |   1 +
 drivers/vfio/Kconfig                |   1 +
 drivers/vfio/Makefile               |   1 +
 drivers/vfio/cdx/Kconfig            |  17 ++
 drivers/vfio/cdx/Makefile           |   8 +
 drivers/vfio/cdx/vfio_cdx.c         | 314 ++++++++++++++++++++++++++++
 drivers/vfio/cdx/vfio_cdx_private.h |  32 +++
 7 files changed, 374 insertions(+)
 create mode 100644 drivers/vfio/cdx/Kconfig
 create mode 100644 drivers/vfio/cdx/Makefile
 create mode 100644 drivers/vfio/cdx/vfio_cdx.c
 create mode 100644 drivers/vfio/cdx/vfio_cdx_private.h
MAINTAINERS +1 -0
index 99b665e85f0a..2acb3dbff174 100644
@@ -970,6 +970,7 @@ M:	Nikhil Agarwal <[email protected]>
 S:	Maintained
 F:	Documentation/devicetree/bindings/bus/xlnx,versal-net-cdx.yaml
 F:	drivers/cdx/*
+F:	drivers/vfio/cdx/*
 F:	include/linux/cdx/*
 
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER
drivers/vfio/Kconfig +1 -0
index 89e06c981e43..aba36f5be4ec 100644
@@ -57,6 +57,7 @@ source "drivers/vfio/pci/Kconfig"
 source "drivers/vfio/platform/Kconfig"
 source "drivers/vfio/mdev/Kconfig"
 source "drivers/vfio/fsl-mc/Kconfig"
+source "drivers/vfio/cdx/Kconfig"
 endif
 
 source "virt/lib/Kconfig"
drivers/vfio/Makefile +1 -0
index 70e7dcb302ef..1a27b2516612 100644
@@ -14,3 +14,4 @@ obj-$(CONFIG_VFIO_PCI) += pci/
 obj-$(CONFIG_VFIO_PLATFORM) += platform/
 obj-$(CONFIG_VFIO_MDEV) += mdev/
 obj-$(CONFIG_VFIO_FSL_MC) += fsl-mc/
+obj-$(CONFIG_VFIO_CDX) += cdx/
drivers/vfio/cdx/Kconfig new file +17 -0
new file mode 100644
index 000000000000..e6de0a0caa32
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# VFIO CDX configuration
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+config VFIO_CDX
+	tristate "VFIO support for CDX bus devices"
+	depends on CDX_BUS
+	select EVENTFD
+	help
+	  Driver to enable VFIO support for the devices on CDX bus.
+	  This is required to make use of CDX devices present in
+	  the system using the VFIO framework.
+
+	  If you don't know what to do here, say N.
drivers/vfio/cdx/Makefile new file +8 -0
new file mode 100644
index 000000000000..82e4ef412c0f
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_VFIO_CDX) += vfio-cdx.o
+
+vfio-cdx-objs := vfio_cdx.o
drivers/vfio/cdx/vfio_cdx.c new file +314 -0

314 lines not on this page · raw message

drivers/vfio/cdx/vfio_cdx_private.h new file +32 -0
new file mode 100644
index 000000000000..8b6f1ee3f5cd
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef VFIO_CDX_PRIVATE_H
+#define VFIO_CDX_PRIVATE_H
+
+#define VFIO_CDX_OFFSET_SHIFT    40
+#define VFIO_CDX_OFFSET_MASK (((u64)(1) << VFIO_CDX_OFFSET_SHIFT) - 1)
+
+#define VFIO_CDX_OFFSET_TO_INDEX(off) ((off) >> VFIO_CDX_OFFSET_SHIFT)
+
+#define VFIO_CDX_INDEX_TO_OFFSET(index)	\
+	((u64)(index) << VFIO_CDX_OFFSET_SHIFT)
+
+struct vfio_cdx_region {
+	u32			flags;
+	u32			type;
+	u64			addr;
+	resource_size_t		size;
+	void __iomem		*ioaddr;
+};
+
+struct vfio_cdx_device {
+	struct vfio_device	vdev;
+	struct cdx_device	*cdx_dev;
+	struct device		*dev;
+	struct vfio_cdx_region	*regions;
+};
+
+#endif /* VFIO_CDX_PRIVATE_H */
Signature · 1 line
2.17.1
#1 lore raw
Quoted 5 lines · On Mon, 3 Apr 2023 19:55:25 +0530 Nipun Gupta <[email protected]> wrote:
drivers/vfio/cdx/Makefile new file +0 -0
new file mode 100644
index 000000000000..82e4ef412c0f

...

Quoted 46 lines
drivers/vfio/cdx/Makefile new file +46 -0
+static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
+			      struct vm_area_struct *vma)
+{
+	u64 size = vma->vm_end - vma->vm_start;
+	u64 pgoff, base;
+
+	pgoff = vma->vm_pgoff &
+		((1U << (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+	base = pgoff << PAGE_SHIFT;
+
+	if (region.size < PAGE_SIZE || base + size > region.size)
+		return -EINVAL;
+
+	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+			       size, vma->vm_page_prot);
+}
+
+static int vfio_cdx_mmap(struct vfio_device *core_vdev,
+			 struct vm_area_struct *vma)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	struct cdx_device *cdx_dev = vdev->cdx_dev;
+	unsigned int index;
+
+	index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
+
+	if (vma->vm_end < vma->vm_start)
+		return -EINVAL;
+	if (vma->vm_start & ~PAGE_MASK)
+		return -EINVAL;
+	if (vma->vm_end & ~PAGE_MASK)
+		return -EINVAL;
+	if (!(vma->vm_flags & VM_SHARED))
+		return -EINVAL;
+	if (index >= cdx_dev->res_count)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) &&
+	    (vma->vm_flags & VM_READ))

drivers/vfio/cdx/Makefile

Quoted 10 lines
drivers/vfio/cdx/Makefile new file +10 -0
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) &&
+	    (vma->vm_flags & VM_WRITE))
+		return -EINVAL;
+
+	vma->vm_private_data = cdx_dev;
+
+	return vfio_cdx_mmap_mmio(vdev->regions[index], vma);
+}

I see discussion of MMIO_REGIONS_ENABLE controlling host access to the device in mc_cdx_pcol.h. Is a user of vfio-cdx able to manipulate whether MMIO space of the device is enabled? If so, what's the system response to accessing MMIO through the mmap while disabled? Is MMIO space accessible even through calling the RESET ioctl? Is there a public specification somewhere for CDX? Thanks,

Alex

author maintainer
#2 lore raw
Quoted 64 lines · Alex Williamson

Caution: This message originated from an External Source. Use proper caution when opening attachments, clicking links, or responding.

Quoted 5 lines · On Mon, 3 Apr 2023 19:55:25 +0530 Nipun Gupta <[email protected]> wrote:
drivers/vfio/cdx/Makefile new file +0 -0
new file mode 100644
index 000000000000..82e4ef412c0f

...

Quoted 53 lines
drivers/vfio/cdx/Makefile new file +53 -0
+static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
+                           struct vm_area_struct *vma)
+{
+     u64 size = vma->vm_end - vma->vm_start;
+     u64 pgoff, base;
+
+     pgoff = vma->vm_pgoff &
+             ((1U << (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+     base = pgoff << PAGE_SHIFT;
+
+     if (region.size < PAGE_SIZE || base + size > region.size)
+             return -EINVAL;
+
+     vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+     vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+     return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+                            size, vma->vm_page_prot);
+}
+
+static int vfio_cdx_mmap(struct vfio_device *core_vdev,
+                      struct vm_area_struct *vma)
+{
+     struct vfio_cdx_device *vdev =
+             container_of(core_vdev, struct vfio_cdx_device, vdev);
+     struct cdx_device *cdx_dev = vdev->cdx_dev;
+     unsigned int index;
+
+     index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
+
+     if (vma->vm_end < vma->vm_start)
+             return -EINVAL;
+     if (vma->vm_start & ~PAGE_MASK)
+             return -EINVAL;
+     if (vma->vm_end & ~PAGE_MASK)
+             return -EINVAL;
+     if (!(vma->vm_flags & VM_SHARED))
+             return -EINVAL;
+     if (index >= cdx_dev->res_count)
+             return -EINVAL;
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+             return -EINVAL;
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) &&
+         (vma->vm_flags & VM_READ))
+             return -EINVAL;
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) &&
+         (vma->vm_flags & VM_WRITE))
+             return -EINVAL;
+
+     vma->vm_private_data = cdx_dev;
Quoted 8 lines

drivers/vfio/cdx/Makefile

Quoted 3 lines
drivers/vfio/cdx/Makefile new file +3 -0
+
+     return vfio_cdx_mmap_mmio(vdev->regions[index], vma);
+}

I see discussion of MMIO_REGIONS_ENABLE controlling host access to the device in mc_cdx_pcol.h. Is a user of vfio-cdx able to manipulate whether MMIO space of the device is enabled? If so, what's the system response to accessing MMIO through the mmap while disabled?

The MMIO enable/disable has been added in mc_cdx_pcol.h from the future support perspective, but it is not currently supported, as all the CDX devices have the MMIO enable flag permanently set which cannot be disabled. Due to this we have not added any interface/API in the CDX bus to disable MMIO for the device. This is still under discussion and future patches will add complete support for this.

That said, if required we can add a flag currently in the "cdx_device" which will be set when for MMIO enabled (it would be by default enabled for now), and depending on this flag VFIO can return error during mmap, but we would prefer to defer it to be added along with the complete support for MMIO enable/disable in the CDX bus.

Quoted 2 lines

Is MMIO
space accessible even through calling the RESET ioctl?

Yes, MMIO region would be accessible even through calling reset, but user may not see the correct values as the device is being reset.

Quoted 2 lines

Is there a
public specification somewhere for CDX? Thanks,

I am afraid there is no public specification for CDX as of now.

Thanks,
Nipun

Quoted 3 lines

Alex

#3 lore raw
In reply to Nipun Gupta

drivers/vfio/cdx/vfio_cdx.c line 22

Quoted 3 lines · Nipun Gupta
drivers/vfio/cdx/vfio_cdx.c new file +3 -0
+enum {
+	CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1,
+};

This seems to be missing the file2alias part.

drivers/vfio/cdx/vfio_cdx.c line 101

Quoted 9 lines
drivers/vfio/cdx/vfio_cdx.c new file +9 -0
+static void vfio_cdx_regions_cleanup(struct vfio_cdx_device *vdev)
+{
+	kfree(vdev->regions);
+}
+
+static int vfio_cdx_reset_device(struct vfio_cdx_device *vdev)
+{
+	return cdx_dev_reset(&vdev->cdx_dev->dev);
+}

Wrapper functions should be avoided.

drivers/vfio/cdx/vfio_cdx.c line 115

Quoted 13 lines
drivers/vfio/cdx/vfio_cdx.c new file +13 -0
+static void vfio_cdx_close_device(struct vfio_device *core_vdev)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	int ret;
+
+	vfio_cdx_regions_cleanup(vdev);
+
+	/* reset the device before cleaning up the interrupts */
+	ret = vfio_cdx_reset_device(vdev);
+	if (WARN_ON(ret))
+		dev_warn(core_vdev->dev,
+			 "VFIO_CDX: reset device has failed (%d)\n", ret);

This is pretty problematic.. if the reset can fail the device is returned to the system in an unknown state and it seems pretty likely that it can be a way to attack the kernel.

drivers/vfio/cdx/vfio_cdx.c line 174

Quoted 4 lines
drivers/vfio/cdx/vfio_cdx.c new file +4 -0
+	case VFIO_DEVICE_RESET:
+	{
+		return vfio_cdx_reset_device(vdev);
+	}

What happens to MMIO access during this reset?

Quoted 5 lines
drivers/vfio/cdx/vfio_cdx.c new file +5 -0
+static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
+			      struct vm_area_struct *vma)
+{
+	u64 size = vma->vm_end - vma->vm_start;
+	u64 pgoff, base;

drivers/vfio/cdx/vfio_cdx.c line 194

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +10 -0
+
+	pgoff = vma->vm_pgoff &
+		((1U << (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+	base = pgoff << PAGE_SHIFT;
+
+	if (region.size < PAGE_SIZE || base + size > region.size)
+		return -EINVAL;
+
+	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

pgprot_device

drivers/vfio/cdx/vfio_cdx.c line 197

Quoted 2 lines
drivers/vfio/cdx/vfio_cdx.c new file +2 -0
+	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+			       size, vma->vm_page_prot);

io_remap_pfn_range

Quoted 6 lines
drivers/vfio/cdx/vfio_cdx.c new file +6 -0
+static int vfio_cdx_mmap(struct vfio_device *core_vdev,
+			 struct vm_area_struct *vma)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	struct cdx_device *cdx_dev = vdev->cdx_dev;

drivers/vfio/cdx/vfio_cdx.c line 215

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +10 -0
+	unsigned int index;
+
+	index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
+
+	if (vma->vm_end < vma->vm_start)
+		return -EINVAL;
+	if (vma->vm_start & ~PAGE_MASK)
+		return -EINVAL;
+	if (vma->vm_end & ~PAGE_MASK)
+		return -EINVAL;

The core code already assures these checks.

Quoted 7 lines
drivers/vfio/cdx/vfio_cdx.c new file +7 -0
+	if (!(vma->vm_flags & VM_SHARED))
+		return -EINVAL;
+	if (index >= cdx_dev->res_count)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+		return -EINVAL;

drivers/vfio/cdx/vfio_cdx.c line 232

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +10 -0
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) &&
+	    (vma->vm_flags & VM_READ))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) &&
+	    (vma->vm_flags & VM_WRITE))
+		return -EINVAL;
+
+	vma->vm_private_data = cdx_dev;

not needed

Quoted 11 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +5 -0
new file mode 100644
index 000000000000..8b6f1ee3f5cd
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+

drivers/vfio/cdx/vfio_cdx_private.h line 15

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +10 -0
+#ifndef VFIO_CDX_PRIVATE_H
+#define VFIO_CDX_PRIVATE_H
+
+#define VFIO_CDX_OFFSET_SHIFT    40
+#define VFIO_CDX_OFFSET_MASK (((u64)(1) << VFIO_CDX_OFFSET_SHIFT) - 1)
+
+#define VFIO_CDX_OFFSET_TO_INDEX(off) ((off) >> VFIO_CDX_OFFSET_SHIFT)
+
+#define VFIO_CDX_INDEX_TO_OFFSET(index)	\
+	((u64)(index) << VFIO_CDX_OFFSET_SHIFT)

use static inlines for function-line macros

Quoted 4 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +4 -0
+struct vfio_cdx_region {
+	u32			flags;
+	u32			type;
+	u64			addr;

drivers/vfio/cdx/vfio_cdx_private.h line 30

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +10 -0
+	resource_size_t		size;
+	void __iomem		*ioaddr;
+};
+
+struct vfio_cdx_device {
+	struct vfio_device	vdev;
+	struct cdx_device	*cdx_dev;
+	struct device		*dev;
+	struct vfio_cdx_region	*regions;
+};

This header file does not seem necessary right now

Jason

author maintainer
#4 lore raw
Quoted 10 lines · Jason Gunthorpe

Caution: This message originated from an External Source. Use proper caution when opening attachments, clicking links, or responding.

drivers/vfio/cdx/vfio_cdx.c line 22

Quoted 3 lines · Nipun Gupta
drivers/vfio/cdx/vfio_cdx.c new file +3 -0
+enum {
+     CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1,
+};

This seems to be missing the file2alias part.

Sure. Will add this.

Quoted 12 lines

drivers/vfio/cdx/vfio_cdx.c line 101

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +9 -0
+static void vfio_cdx_regions_cleanup(struct vfio_cdx_device *vdev)
+{
+     kfree(vdev->regions);
+}
+
+static int vfio_cdx_reset_device(struct vfio_cdx_device *vdev)
+{
+     return cdx_dev_reset(&vdev->cdx_dev->dev);
+}

Wrapper functions should be avoided.

Agree

Quoted 10 lines
Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +9 -0
+static void vfio_cdx_close_device(struct vfio_device *core_vdev)
+{
+     struct vfio_cdx_device *vdev =
+             container_of(core_vdev, struct vfio_cdx_device, vdev);
+     int ret;
+
+     vfio_cdx_regions_cleanup(vdev);
+
+     /* reset the device before cleaning up the interrupts */
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx.c line 115

Quoted 4 lines
drivers/vfio/cdx/vfio_cdx.c new file +4 -0
+     ret = vfio_cdx_reset_device(vdev);
+     if (WARN_ON(ret))
+             dev_warn(core_vdev->dev,
+                      "VFIO_CDX: reset device has failed (%d)\n", ret);

This is pretty problematic.. if the reset can fail the device is returned to the system in an unknown state and it seems pretty likely that it can be a way to attack the kernel.

We will update the code to disable the device in case of failures.

Quoted 7 lines

drivers/vfio/cdx/vfio_cdx.c line 174

Quoted 5 lines
drivers/vfio/cdx/vfio_cdx.c new file +4 -0
+     case VFIO_DEVICE_RESET:
+     {
+             return vfio_cdx_reset_device(vdev);
+     }

What happens to MMIO access during this reset?

There is no config space here; and access to mmap space will go as usual but user may not see the expected behavior as the device is being reset. It is responsibility of user of VFIO device to serialize such access and reset.

Quoted 10 lines
Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +9 -0
+static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
+                           struct vm_area_struct *vma)
+{
+     u64 size = vma->vm_end - vma->vm_start;
+     u64 pgoff, base;
+
+     pgoff = vma->vm_pgoff &
+             ((1U << (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+     base = pgoff << PAGE_SHIFT;
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx.c line 194

Quoted 6 lines
drivers/vfio/cdx/vfio_cdx.c new file +6 -0
+
+     if (region.size < PAGE_SIZE || base + size > region.size)
+             return -EINVAL;
+
+     vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+     vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

pgprot_device

Will update..

Quoted 15 lines

drivers/vfio/cdx/vfio_cdx.c line 197

Quoted 3 lines
drivers/vfio/cdx/vfio_cdx.c new file +2 -0
+     return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+                            size, vma->vm_page_prot);

io_remap_pfn_range

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx.c new file +10 -0
+static int vfio_cdx_mmap(struct vfio_device *core_vdev,
+                      struct vm_area_struct *vma)
+{
+     struct vfio_cdx_device *vdev =
+             container_of(core_vdev, struct vfio_cdx_device, vdev);
+     struct cdx_device *cdx_dev = vdev->cdx_dev;
+     unsigned int index;
+
+     index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
+
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx.c line 215

Quoted 6 lines
drivers/vfio/cdx/vfio_cdx.c new file +6 -0
+     if (vma->vm_end < vma->vm_start)
+             return -EINVAL;
+     if (vma->vm_start & ~PAGE_MASK)
+             return -EINVAL;
+     if (vma->vm_end & ~PAGE_MASK)
+             return -EINVAL;

The core code already assures these checks.

Sure will remove these unnecessary checks.

Quoted 12 lines
Quoted 12 lines
drivers/vfio/cdx/vfio_cdx.c new file +11 -0
+     if (!(vma->vm_flags & VM_SHARED))
+             return -EINVAL;
+     if (index >= cdx_dev->res_count)
+             return -EINVAL;
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+             return -EINVAL;
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ) &&
+         (vma->vm_flags & VM_READ))
+             return -EINVAL;
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx.c line 232

Quoted 6 lines
drivers/vfio/cdx/vfio_cdx.c new file +6 -0
+
+     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) &&
+         (vma->vm_flags & VM_WRITE))
+             return -EINVAL;
+
+     vma->vm_private_data = cdx_dev;

not needed

Okay

Quoted 16 lines
Quoted 16 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +9 -0
new file mode 100644
index 000000000000..8b6f1ee3f5cd
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
+ */
+
+#ifndef VFIO_CDX_PRIVATE_H
+#define VFIO_CDX_PRIVATE_H
+
+#define VFIO_CDX_OFFSET_SHIFT    40
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx_private.h line 15

Quoted 6 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +6 -0
+#define VFIO_CDX_OFFSET_MASK (((u64)(1) << VFIO_CDX_OFFSET_SHIFT) - 1)
+
+#define VFIO_CDX_OFFSET_TO_INDEX(off) ((off) >> VFIO_CDX_OFFSET_SHIFT)
+
+#define VFIO_CDX_INDEX_TO_OFFSET(index)      \
+     ((u64)(index) << VFIO_CDX_OFFSET_SHIFT)

use static inlines for function-line macros

Okay. Will update the code.

Quoted 17 lines
Quoted 5 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +4 -0
+struct vfio_cdx_region {
+     u32                     flags;
+     u32                     type;
+     u64                     addr;

drivers/vfio/cdx/vfio_cdx_private.h line 30

Quoted 10 lines
drivers/vfio/cdx/vfio_cdx_private.h new file +10 -0
+     resource_size_t         size;
+     void __iomem            *ioaddr;
+};
+
+struct vfio_cdx_device {
+     struct vfio_device      vdev;
+     struct cdx_device       *cdx_dev;
+     struct device           *dev;
+     struct vfio_cdx_region  *regions;
+};

This header file does not seem necessary right now

There is work in progress which would add support of MSI in VFIO-CDX driver, and where would be additional file/s. So we would like to keep the header file separate.

Thanks,
Nipun

Quoted 2 lines

Jason

author maintainer
#5 lore raw
Quoted 13 lines

-----Original Message-----
From: Gupta, Nipun
Sent: Friday, April 7, 2023 10:35 AM
To: Jason Gunthorpe <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; git (AMD-Xilinx) <[email protected]>; Anand, Harpreet <[email protected]>; Simek, Michal <[email protected]>; Agarwal, Nikhil <[email protected]>; Jansen Van Vuuren, Pieter <[email protected]>; [email protected]
Subject: Re: [PATCH] vfio/cdx: add support for CDX bus

<snip>

Quoted 20 lines
Quoted 11 lines
Quoted 11 lines
drivers/vfio/cdx/vfio_cdx.c new file +9 -0
+static void vfio_cdx_close_device(struct vfio_device *core_vdev)
+{
+     struct vfio_cdx_device *vdev =
+             container_of(core_vdev, struct vfio_cdx_device, vdev);
+     int ret;
+
+     vfio_cdx_regions_cleanup(vdev);
+
+     /* reset the device before cleaning up the interrupts */
Quoted 8 lines

drivers/vfio/cdx/vfio_cdx.c line 115

Quoted 4 lines
drivers/vfio/cdx/vfio_cdx.c new file +4 -0
+     ret = vfio_cdx_reset_device(vdev);
+     if (WARN_ON(ret))
+             dev_warn(core_vdev->dev,
+                      "VFIO_CDX: reset device has failed (%d)\n", ret);

This is pretty problematic.. if the reset can fail the device is returned to the system in an unknown state and it seems pretty likely that it can be a way to attack the kernel.

Quoted 1 line

We will update the code to disable the device in case of failures.

We double checked with firmware/hardware team, when driver sends CDX device reset command to firmware, it actually quiesce and then reset the device. So, the device remains disabled in case of failures.

Regards,
Nipun