Skip to contentSkip to conversation

vfio: Add support for platform devices

PATCH New Mostafa Saleh 1 participant 10 messages Oct 24

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover vfio: Add support for platform devices

This patch series extend VFIO support to platform devices by using VFIO-platform.

One of the main difference is how the guest discover the device, which is done through the device tree.
kvmtool can only describe:
- regs
- interrupts

However, the guest still needs a compatible string and other per device bindings.
To avoid having any hardcoded DT nodes in the source code, kvmtool will accept a dtb argument with each assigned device that will be copied to the guest node.

Patches
=======
1-4: Fixes, code refactoring and improving interrupt support. 5: DT common function.
6-8: VFIO-platform support.
9: Custom device tree support.

Patch 2 is better viewed with --color-moved

Testing
=======
Tested on Lenovo Mini-x Gen 10 (Snapdragon-X) by assigning 2 devices, QCE and BAM as follows:

echo 1dfa000.crypto > /sys/bus/platform/drivers/qcrypto/unbind echo 1dc4000.dma-controller > /sys/bus/platform/drivers/bam-dma-engine/unbind

echo "vfio-platform" > /sys/bus/platform/devices/1dfa000.crypto/driver_override echo "vfio-platform" > /sys/bus/platform/devices/1dc4000.dma-controller/driver_override

echo 1dc4000.dma-controller > /sys/bus/platform/drivers/vfio-platform/bind echo 1dfa000.crypto > /sys/bus/platform/drivers/vfio-platform/bind

./lkvm-static run --irqchip gicv3 -k Image -p "nokaslr" -d rootfs.ext2 --force-pci -c 1 --debug --vfio- platform 1dc4000.dma-controller,purwa-cryptobam.dtb --vfio-platform 1dfa000.crypto,purwa-qce.dtb

There are some fixes on the list that are needed to be able to unbind the bam device [1]

Mostafa Saleh (9):
vfio: Fix fd leak in vfio_group_get_for_dev()
vfio: Factor out wired IRQ setup and teardown
vfio: Support edge-triggered wired IRQs
fdt: Allow fdt_irq_fn to generate multiple interrupts
fdt: Add helper to copy an FDT node
vfio: Reserve guest address space for VFIO platform
vfio: Add basic support for vfio-platform
vfio: platform: Add interrupts support
vfio: platform: Take the guest node from a device tree template

 Makefile                       |   1 +
 arm64/fdt.c                    |  36 ++--
 arm64/include/kvm/kvm-arch.h   |  10 +-
 builtin-run.c                  |   3 +
 hw/cfi_flash.c                 |   4 +-
 hw/rtc.c                       |   4 +-
 hw/serial.c                    |   3 +-
 include/kvm/fdt.h              |  32 +++-
 include/kvm/vfio.h             |  41 ++++-
 mips/include/kvm/kvm-arch.h    |   2 +
 powerpc/include/kvm/kvm-arch.h |   2 +
 riscv/fdt.c                    |   2 +-
 riscv/include/kvm/kvm-arch.h   |   7 +-
 riscv/irq.c                    |  27 ++-
 vfio/core.c                    | 152 +++++++++++++++-
 vfio/pci.c                     | 111 +-----------
 vfio/platform.c                | 317 +++++++++++++++++++++++++++++++++
 virtio/mmio.c                  |  10 +-
 x86/include/kvm/kvm-arch.h     |   2 +
 19 files changed, 619 insertions(+), 147 deletions(-)
 create mode 100644 vfio/platform.c
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#1 lore raw

PATCH 1/9 vfio: Fix fd leak in vfio_group_get_for_dev()

The fd opened to read the iommu_group link of the device is closed on every path except the one that finds the group already open, so every device after the first one in a group leaked an fd. Return through the goto on that path too.

---
 vfio/core.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
vfio/core.c +1 -1
index fbe0098..e0c56ab 100644
@@ -544,7 +544,7 @@ vfio_group_get_for_dev(struct kvm *kvm, struct vfio_device *vdev)
 	list_for_each_entry(group, &vfio_groups, list) {
 		if (group->id == group_id) {
 			group->refs++;
-			return group;
+			goto out_close;
 		}
 	}
 
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#2 lore raw

PATCH 2/9 vfio: Factor out wired IRQ setup and teardown

In reply to Mostafa Saleh

Factor the wired IRQ eventfd and KVM irqfd setup and teardown out of vfio_pci_enable_intx() and vfio_pci_disable_intx() into vfio_irq_enable() and vfio_irq_disable() in vfio/core.c, and move union vfio_irq_eventfd to vfio.h so both PCI INTx and VFIO platform devices can share them.

The is a mechanical code move, no functional change.

---
 include/kvm/vfio.h |  27 +++++++++--
 vfio/core.c        | 107 +++++++++++++++++++++++++++++++++++++++++++
 vfio/pci.c         | 110 +++------------------------------------------
 3 files changed, 137 insertions(+), 107 deletions(-)
include/kvm/vfio.h +24 -3
index ac7b622..9a5783e 100644
@@ -6,6 +6,7 @@
 #include "kvm/pci.h"
 
 #include <linux/vfio.h>
+#include <string.h>
 
 #define vfio_dev_err(vdev, fmt, ...) \
 	pr_err("%s: " fmt, (vdev)->params->name, ##__VA_ARGS__)
@@ -70,13 +71,17 @@ struct vfio_pci_msi_common {
 #define VFIO_PCI_IRQ_MODE_MSI		(1 << 1)
 #define VFIO_PCI_IRQ_MODE_MSIX		(1 << 2)
 
+struct vfio_irq {
+	int				gsi;
+	int				trigger_fd;
+	int				unmask_fd;
+};
+
 struct vfio_pci_device {
 	struct pci_device_header	hdr;
 
 	unsigned long			irq_modes;
-	int				intx_fd;
-	int				unmask_fd;
-	unsigned int			intx_gsi;
+	struct vfio_irq			intx;
 	struct vfio_pci_msi_common	msi;
 	struct vfio_pci_msi_common	msix;
 	struct vfio_pci_msix_table	msix_table;
@@ -119,10 +124,26 @@ struct vfio_group {
 	struct list_head		list;
 };
 
+/* Wrapper around UAPI vfio_irq_set */
+union vfio_irq_eventfd {
+	struct vfio_irq_set	irq;
+	u8 buffer[sizeof(struct vfio_irq_set) + sizeof(int)];
+};
+
+static inline void set_vfio_irq_eventd_payload(union vfio_irq_eventfd *evfd,
+					       int fd)
+{
+	memcpy(&evfd->irq.data, &fd, sizeof(fd));
+}
+
 int vfio_device_parser(const struct option *opt, const char *arg, int unset);
 int vfio_map_region(struct kvm *kvm, struct vfio_device *vdev,
 		    struct vfio_region *region);
 void vfio_unmap_region(struct kvm *kvm, struct vfio_region *region);
+int vfio_irq_enable(struct kvm *kvm, struct vfio_device *vdev,
+		    unsigned int index, struct vfio_irq *irq);
+void vfio_irq_disable(struct kvm *kvm, struct vfio_device *vdev,
+		      unsigned int index, struct vfio_irq *irq);
 int vfio_pci_setup_device(struct kvm *kvm, struct vfio_device *device);
 void vfio_pci_teardown_device(struct kvm *kvm, struct vfio_device *vdev);
 
vfio/core.c +107 -0

121 lines not on this page · raw message

vfio/pci.c +6 -104

140 lines not on this page · raw message

Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#3 lore raw

PATCH 3/9 vfio: Support edge-triggered wired IRQs

In reply to Mostafa Saleh

PCI INTx is are level-triggered and automasked, but platform device interrupts can also be edge-triggered.

Pass an automasked boolean to vfio_irq_enable() and only create and register unmask_fd when the interrupt is automasked.

---
 include/kvm/vfio.h |  2 +-
 vfio/core.c        | 51 ++++++++++++++++++++++++++--------------------
 vfio/pci.c         |  3 ++-
 3 files changed, 32 insertions(+), 24 deletions(-)
include/kvm/vfio.h +1 -1
index 9a5783e..0b2b105 100644
@@ -141,7 +141,7 @@ int vfio_map_region(struct kvm *kvm, struct vfio_device *vdev,
 		    struct vfio_region *region);
 void vfio_unmap_region(struct kvm *kvm, struct vfio_region *region);
 int vfio_irq_enable(struct kvm *kvm, struct vfio_device *vdev,
-		    unsigned int index, struct vfio_irq *irq);
+		    unsigned int index, bool automasked, struct vfio_irq *irq);
 void vfio_irq_disable(struct kvm *kvm, struct vfio_device *vdev,
 		      unsigned int index, struct vfio_irq *irq);
 int vfio_pci_setup_device(struct kvm *kvm, struct vfio_device *device);
vfio/core.c +29 -22

88 lines not on this page · raw message

vfio/pci.c +2 -1
index fb98435..9bf2f9e 100644
@@ -1216,7 +1216,8 @@ static int vfio_pci_enable_intx(struct kvm *kvm, struct vfio_device *vdev)
 {
 	struct vfio_pci_device *pdev = &vdev->pci;
 
-	return vfio_irq_enable(kvm, vdev, VFIO_PCI_INTX_IRQ_INDEX, &pdev->intx);
+	return vfio_irq_enable(kvm, vdev, VFIO_PCI_INTX_IRQ_INDEX, true,
+			       &pdev->intx);
 }
 
 static int vfio_pci_init_intx(struct kvm *kvm, struct vfio_device *vdev)
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#4 lore raw

PATCH 4/9 fdt: Allow fdt_irq_fn to generate multiple interrupts

In reply to Mostafa Saleh

fdt_irq_fn currently accepts a single interrupt line and trigger type, which is sufficient for emulated MMIO devices with one interrupt. Assigned platform devices can expose multiple interrupts that must be written into a single "interrupts" property.

Update fdt_irq_fn and its arm64 and riscv implementations to accept an array of interrupt lines and trigger types, and convert existing callers to use the fdt_irq_fn typedef.

---
 arm64/fdt.c                  | 36 ++++++++++++++++++++++++++----------
 hw/cfi_flash.c               |  4 +---
 hw/rtc.c                     |  4 +---
 hw/serial.c                  |  3 ++-
 include/kvm/fdt.h            |  4 +++-
 riscv/fdt.c                  |  2 +-
 riscv/include/kvm/kvm-arch.h |  4 +++-
 riscv/irq.c                  | 27 +++++++++++++++++++--------
 virtio/mmio.c                | 10 ++++------
 9 files changed, 60 insertions(+), 34 deletions(-)
arm64/fdt.c +26 -10
index 3cbd36e..e2c37ad 100644
@@ -60,15 +60,29 @@ static void generate_cpu_nodes(void *fdt, struct kvm *kvm)
 	_FDT(fdt_end_node(fdt));
 }
 
-static void generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
+static void generate_irq_prop(void *fdt, const u8 *irqs,
+			      const enum irq_type *irq_types,
+			      unsigned int num_irqs)
 {
-	u32 irq_prop[] = {
-		cpu_to_fdt32(GIC_FDT_IRQ_TYPE_SPI),
-		cpu_to_fdt32(irq - GIC_SPI_IRQ_BASE),
-		cpu_to_fdt32(irq_type)
-	};
+	unsigned int i;
+	u32 *irq_prop;
 
-	_FDT(fdt_property(fdt, "interrupts", irq_prop, sizeof(irq_prop)));
+	if (!num_irqs)
+		return;
+
+	irq_prop = calloc(num_irqs, 3 * sizeof(*irq_prop));
+	if (!irq_prop)
+		die("Failed to allocate the interrupts property");
+
+	for (i = 0; i < num_irqs; i++) {
+		irq_prop[3 * i] = cpu_to_fdt32(GIC_FDT_IRQ_TYPE_SPI);
+		irq_prop[3 * i + 1] = cpu_to_fdt32(irqs[i] - GIC_SPI_IRQ_BASE);
+		irq_prop[3 * i + 2] = cpu_to_fdt32(irq_types[i]);
+	}
+
+	_FDT(fdt_property(fdt, "interrupts", irq_prop,
+			  3 * sizeof(*irq_prop) * num_irqs));
+	free(irq_prop);
 }
 
 static bool emit_dma_regions;
@@ -78,9 +92,11 @@ void generate_dma_region_prop(void *fdt)
 		_FDT(fdt_property_cell(fdt, "memory-region", PHANDLE_DMA));
 }
 
-static void generate_aux_props(void *fdt, u8 irq, enum irq_type irq_type)
+static void generate_aux_props(void *fdt, const u8 *irqs,
+			       const enum irq_type *irq_types,
+			       unsigned int num_irqs)
 {
-	generate_irq_prop(fdt, irq, irq_type);
+	generate_irq_prop(fdt, irqs, irq_types, num_irqs);
 	generate_dma_region_prop(fdt);
 }
 
@@ -125,7 +141,7 @@ static int setup_fdt(struct kvm *kvm)
 	void *fdt_dest		= guest_flat_to_host(kvm,
 						     kvm->arch.dtb_guest_start);
 	void (*generate_mmio_fdt_nodes)(void *, struct device_header *,
-					void (*)(void *, u8, enum irq_type));
+					fdt_irq_fn);
 	void (*generate_cpu_peripheral_fdt_nodes)(void *, struct kvm *)
 					= kvm->cpus[0]->generate_fdt_nodes;
 
hw/cfi_flash.c +1 -3
index 4413955..5641af6 100644
@@ -518,9 +518,7 @@ static void cfi_flash_mmio(struct kvm_cpu *vcpu,
 #ifdef CONFIG_HAS_LIBFDT
 static void generate_cfi_flash_fdt_node(void *fdt,
 					struct device_header *dev_hdr,
-					void (*generate_irq_prop)(void *fdt,
-								  u8 irq,
-								enum irq_type))
+					fdt_irq_fn generate_irq_prop)
 {
 	struct cfi_flash_device *sfdev;
 	u64 reg_prop[2];
hw/rtc.c +1 -3
index e3523a1..fc6533a 100644
@@ -136,9 +136,7 @@ static void cmos_ram_io(struct kvm_cpu *vcpu, u64 addr, u8 *data,
 #ifdef CONFIG_HAS_LIBFDT
 static void generate_rtc_fdt_node(void *fdt,
 				  struct device_header *dev_hdr,
-				  void (*generate_irq_prop)(void *fdt,
-							    u8 irq,
-							    enum irq_type))
+				  fdt_irq_fn generate_irq_prop)
 {
 	u64 reg_prop[2] = { cpu_to_fdt64(RTC_BASE_ADDRESS), cpu_to_fdt64(2) };
 
hw/serial.c +2 -1
index 68e6186..8e26837 100644
@@ -434,6 +434,7 @@ void serial8250_generate_fdt_node(void *fdt, struct device_header *dev_hdr,
 		cpu_to_fdt64(addr),
 		cpu_to_fdt64(8),
 	};
+	enum irq_type irq_type = IRQ_TYPE_LEVEL_HIGH;
 
 	snprintf(dev_name, DEVICE_NAME_MAX_LEN, "U6_16550A@%llx", addr);
 
@@ -446,7 +447,7 @@ void serial8250_generate_fdt_node(void *fdt, struct device_header *dev_hdr,
 	_FDT(fdt_begin_node(fdt, dev_name));
 	_FDT(fdt_property_string(fdt, "compatible", "ns16550a"));
 	_FDT(fdt_property(fdt, "reg", reg_prop, sizeof(reg_prop)));
-	irq_fn(fdt, dev->irq, IRQ_TYPE_LEVEL_HIGH);
+	irq_fn(fdt, &dev->irq, &irq_type, 1);
 	_FDT(fdt_property_cell(fdt, "clock-frequency", 1843200));
 	_FDT(fdt_end_node(fdt));
 }
include/kvm/fdt.h +3 -1
index 060c37b..e84c5c1 100644
@@ -25,7 +25,9 @@ enum irq_type {
 	IRQ_TYPE_LEVEL_MASK	= (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH),
 };
 
-typedef void (*fdt_irq_fn)(void *fdt, u8 irq, enum irq_type irq_type);
+typedef void (*fdt_irq_fn)(void *fdt, const u8 *irqs,
+			   const enum irq_type *irq_types,
+			   unsigned int num_irqs);
 
 extern char *fdt_stdout_path;
 
riscv/fdt.c +1 -1
index 1eb45fc..b571724 100644
@@ -351,7 +351,7 @@ static int setup_fdt(struct kvm *kvm)
 	void *fdt_dest		= guest_flat_to_host(kvm,
 						     kvm->arch.dtb_guest_start);
 	void (*generate_mmio_fdt_nodes)(void *, struct device_header *,
-					void (*)(void *, u8, enum irq_type));
+					fdt_irq_fn);
 
 	/* Create new tree without a reserve map */
 	_FDT(fdt_create(fdt, FDT_MAX_SIZE));
riscv/include/kvm/kvm-arch.h +3 -1
index 1bb2d32..460b90b 100644
@@ -117,7 +117,9 @@ void riscv__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd);
 
 int riscv__setup_irqfd_lines(struct kvm *kvm);
 
-void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+void riscv__generate_irq_prop(void *fdt, const u8 *irqs,
+			      const enum irq_type *irq_types,
+			      unsigned int num_irqs);
 
 void riscv__irqchip_create(struct kvm *kvm);
 
riscv/irq.c +19 -8
index 62dbaa8..2c35f25 100644
@@ -119,18 +119,29 @@ int riscv__setup_irqfd_lines(struct kvm *kvm)
 	return 0;
 }
 
-void riscv__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
+void riscv__generate_irq_prop(void *fdt, const u8 *irqs,
+			      const enum irq_type *irq_types,
+			      unsigned int num_irqs)
 {
-	u32 prop[2], size;
+	unsigned int i, cells = riscv_irqchip_line_sensing ? 2 : 1;
+	u32 *prop;
 
-	prop[0] = cpu_to_fdt32(irq);
-	size = sizeof(u32);
-	if (riscv_irqchip_line_sensing) {
-		prop[1] = cpu_to_fdt32(irq_type);
-		size += sizeof(u32);
+	if (!num_irqs)
+		return;
+
+	prop = calloc(num_irqs, cells * sizeof(*prop));
+	if (!prop)
+		die("Failed to allocate the interrupts property");
+
+	for (i = 0; i < num_irqs; i++) {
+		prop[cells * i] = cpu_to_fdt32(irqs[i]);
+		if (riscv_irqchip_line_sensing)
+			prop[cells * i + 1] = cpu_to_fdt32(irq_types[i]);
 	}
 
-	_FDT(fdt_property(fdt, "interrupts", prop, size));
+	_FDT(fdt_property(fdt, "interrupts", prop,
+			  cells * sizeof(*prop) * num_irqs));
+	free(prop);
 }
 
 static void (*riscv__irqchip_create_funcs[])(struct kvm *kvm) = {
virtio/mmio.c +4 -6
index fae73b5..0c66724 100644
@@ -113,9 +113,7 @@ int virtio_mmio_signal_config(struct kvm *kvm, struct virtio_device *vdev)
 static
 void generate_virtio_mmio_fdt_node(void *fdt,
 				   struct device_header *dev_hdr,
-				   void (*generate_irq_prop)(void *fdt,
-							     u8 irq,
-							     enum irq_type))
+				   fdt_irq_fn generate_irq_prop)
 {
 	char dev_name[DEVICE_NAME_MAX_LEN];
 	struct virtio_mmio *vmmio = container_of(dev_hdr,
@@ -126,6 +124,7 @@ void generate_virtio_mmio_fdt_node(void *fdt,
 		cpu_to_fdt64(addr),
 		cpu_to_fdt64(VIRTIO_MMIO_IO_SIZE),
 	};
+	enum irq_type irq_type = IRQ_TYPE_EDGE_RISING;
 
 	snprintf(dev_name, DEVICE_NAME_MAX_LEN, "virtio@%llx", addr);
 
@@ -133,14 +132,13 @@ void generate_virtio_mmio_fdt_node(void *fdt,
 	_FDT(fdt_property_string(fdt, "compatible", "virtio,mmio"));
 	_FDT(fdt_property(fdt, "reg", reg_prop, sizeof(reg_prop)));
 	_FDT(fdt_property(fdt, "dma-coherent", NULL, 0));
-	generate_irq_prop(fdt, vmmio->irq, IRQ_TYPE_EDGE_RISING);
+	generate_irq_prop(fdt, &vmmio->irq, &irq_type, 1);
 	_FDT(fdt_end_node(fdt));
 }
 #else
 static void generate_virtio_mmio_fdt_node(void *fdt,
 					  struct device_header *dev_hdr,
-					  void (*generate_irq_prop)(void *fdt,
-								    u8 irq))
+					  fdt_irq_fn generate_irq_prop)
 {
 	die("Unable to generate device tree nodes without libfdt\n");
 }
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#5 lore raw

PATCH 5/9 fdt: Add helper to copy an FDT node

In reply to Mostafa Saleh

Add fdt__copy_node() to recursively copy all properties and subnodes of a source FDT node into a destination FDT.

---
 include/kvm/fdt.h | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
include/kvm/fdt.h +28 -0
index e84c5c1..e7b65e8 100644
@@ -8,6 +8,7 @@
 #include <linux/types.h>
 
 #include "kvm/fdt-arch.h"
+#include "kvm/util.h"
 
 #define FDT_MAX_SIZE	0x10000
 
@@ -41,4 +42,31 @@ extern char *fdt_stdout_path;
 		}							\
 	} while (0)
 
+#ifdef CONFIG_HAS_LIBFDT
+static inline void fdt__copy_node(void *fdt_dest, const void *fdt_src,
+				  int offset)
+{
+	int prop_offset, subnode_offset, len;
+	const char *name;
+	const void *prop;
+
+	fdt_for_each_property_offset(prop_offset, fdt_src, offset) {
+		prop = fdt_getprop_by_offset(fdt_src, prop_offset, &name, &len);
+		if (!prop)
+			continue;
+
+		_FDT(fdt_property(fdt_dest, name, prop, len));
+	}
+
+	fdt_for_each_subnode(subnode_offset, fdt_src, offset) {
+		name = fdt_get_name(fdt_src, subnode_offset, NULL);
+		if (!name)
+			continue;
+		_FDT(fdt_begin_node(fdt_dest, name));
+		fdt__copy_node(fdt_dest, fdt_src, subnode_offset);
+		_FDT(fdt_end_node(fdt_dest));
+	}
+}
+#endif
+
 #endif /* KVM__FDT_H */
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#6 lore raw

PATCH 6/9 vfio: Reserve guest address space for VFIO platform

In reply to Mostafa Saleh

Assigned platform devices need guest physical addresses for their MMIO regions.

Reserve an area called KVM_VFIO_MMIO_AREA.

---
 arm64/include/kvm/kvm-arch.h   | 10 ++++++----
 mips/include/kvm/kvm-arch.h    |  2 ++
 powerpc/include/kvm/kvm-arch.h |  2 ++
 riscv/include/kvm/kvm-arch.h   |  3 +++
 x86/include/kvm/kvm-arch.h     |  2 ++
 5 files changed, 15 insertions(+), 4 deletions(-)
arm64/include/kvm/kvm-arch.h +6 -4
index e7dd526..cddb404 100644
@@ -19,8 +19,8 @@
  * +-------+----+-------+-------+--------+-----+---------+---......
  * |  PCI  |////| plat  |       |        |     |         |
  * |  I/O  |////| MMIO: | Flash | virtio | GIC |   PCI   |  DRAM
- * | space |////| UART, |       |  MMIO  |     |  (AXI)  |
- * |       |////| RTC,  |       |        |     |         |
+ * | space |////| UART, |       |  /VFIO |     |  (AXI)  |
+ * |       |////| RTC,  |       |  MMIO  |     |         |
  * |       |////| PVTIME|       |        |     |         |
  * +-------+----+-------+-------+--------+-----+---------+---......
  */
@@ -47,8 +47,10 @@
 #define KVM_FLASH_MAX_SIZE	0x1000000
 
 #define KVM_VIRTIO_MMIO_AREA	(KVM_FLASH_MMIO_BASE + KVM_FLASH_MAX_SIZE)
-#define ARM_VIRTIO_MMIO_SIZE	(ARM_AXI_AREA - \
-				(KVM_VIRTIO_MMIO_AREA + ARM_GIC_SIZE))
+#define ARM_VIRTIO_MMIO_SIZE	SZ_128M
+
+#define KVM_VFIO_MMIO_AREA	(KVM_VIRTIO_MMIO_AREA + ARM_VIRTIO_MMIO_SIZE)
+#define KVM_VFIO_MMIO_SIZE	SZ_32M
 
 #define ARM_GIC_DIST_BASE	(ARM_AXI_AREA - ARM_GIC_DIST_SIZE)
 #define ARM_GIC_CPUI_BASE	(ARM_GIC_DIST_BASE - ARM_GIC_CPUI_SIZE)
mips/include/kvm/kvm-arch.h +2 -0
index f5b9044..d9a9a64 100644
@@ -15,6 +15,8 @@
 #define KVM_PCI_CFG_AREA	KVM_MMIO_START
 #define KVM_PCI_MMIO_AREA	(KVM_MMIO_START + 0x1000000)
 #define KVM_VIRTIO_MMIO_AREA	(KVM_MMIO_START + 0x2000000)
+#define KVM_VFIO_MMIO_AREA	(KVM_MMIO_START + 0x3000000)
+#define KVM_VFIO_MMIO_SIZE	SZ_32M
 #define KVM_MMIO_SIZE		0x10000000
 
 /*
powerpc/include/kvm/kvm-arch.h +2 -0
index ff42c13..cebf73c 100644
@@ -42,6 +42,8 @@
 #define KVM_PCI_CFG_AREA		0x1000000
 #define KVM_PCI_MMIO_AREA		0x2000000
 #define KVM_VIRTIO_MMIO_AREA		0x3000000
+#define KVM_VFIO_MMIO_AREA		0x4000000
+#define KVM_VFIO_MMIO_SIZE		SZ_32M
 
 #define KVM_IRQ_OFFSET			16
 
riscv/include/kvm/kvm-arch.h +3 -0
index 460b90b..b347216 100644
@@ -45,6 +45,9 @@
 #define KVM_PCI_CFG_AREA	RISCV_PCI
 #define KVM_PCI_MMIO_AREA	(KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
 #define KVM_VIRTIO_MMIO_AREA	(RISCV_RTC_MMIO_BASE + RISCV_RTC_MMIO_SIZE)
+#define KVM_VIRTIO_MMIO_SIZE	SZ_128M
+#define KVM_VFIO_MMIO_AREA	(KVM_VIRTIO_MMIO_AREA + KVM_VIRTIO_MMIO_SIZE)
+#define KVM_VFIO_MMIO_SIZE	SZ_32M
 
 #define KVM_IOEVENTFD_HAS_PIO	0
 
x86/include/kvm/kvm-arch.h +2 -0
index 73a957b..cc2defa 100644
@@ -24,6 +24,8 @@
 #define KVM_PCI_CFG_AREA	(KVM_MMIO_START + 0x1000000)
 #define KVM_PCI_MMIO_AREA	(KVM_MMIO_START + 0x2000000)
 #define KVM_VIRTIO_MMIO_AREA	(KVM_MMIO_START + 0x3000000)
+#define KVM_VFIO_MMIO_AREA	(KVM_MMIO_START + 0x4000000)
+#define KVM_VFIO_MMIO_SIZE	SZ_32M
 
 #define KVM_IRQ_OFFSET		5
 
Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#7 lore raw

PATCH 7/9 vfio: Add basic support for vfio-platform

In reply to Mostafa Saleh

A VFIO device can be assigned to a guest with new cmdline --vfio-platform ${dev} , where ${dev} is the name of the device under /sys/bus/platform.

The device is expected to be bound to vfio-platform driver before running the tool.

The MMIO regions of the device are mapped into the guest address space reserved for VFIO, and described in a device tree node. The node only carries reg, so no guest driver binds to it yet. Interrupts and the rest of the node are added in the next patches.

---
 Makefile           |   1 +
 builtin-run.c      |   3 +
 include/kvm/vfio.h |   3 +
 vfio/core.c        |  22 +++++++
 vfio/platform.c    | 158 +++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 187 insertions(+)
 create mode 100644 vfio/platform.c
Makefile +1 -0
index 1c0e7f5..8d1c0d9 100644
@@ -65,6 +65,7 @@ OBJS	+= pci.o
 OBJS	+= term.o
 OBJS	+= vfio/core.o
 OBJS	+= vfio/pci.o
+OBJS	+= vfio/platform.o
 OBJS	+= virtio/blk.o
 OBJS	+= virtio/scsi.o
 OBJS	+= virtio/console.o
builtin-run.c +3 -0
index 81f255f..b09fc2e 100644
@@ -261,6 +261,9 @@ static int loglevel_parser(const struct option *opt, const char *arg, int unset)
 	OPT_GROUP("VFIO options:"),					\
 	OPT_CALLBACK('\0', "vfio-pci", NULL, "[domain:]bus:dev.fn",	\
 		     "Assign a PCI device to the virtual machine",	\
+		     vfio_device_parser, kvm),				\
+	OPT_CALLBACK('\0', "vfio-platform", NULL, "device",		\
+		     "Assign a platform device to the virtual machine",	\
 		     vfio_device_parser, kvm),				\
 									\
 	OPT_GROUP("Debug options:"),					\
include/kvm/vfio.h +3 -0
index 0b2b105..f641292 100644
@@ -24,6 +24,7 @@
 
 enum vfio_device_type {
 	VFIO_DEVICE_PCI,
+	VFIO_DEVICE_PLATFORM,
 };
 
 /* MSI/MSI-X capability enabled */
@@ -146,5 +147,7 @@ void vfio_irq_disable(struct kvm *kvm, struct vfio_device *vdev,
 		      unsigned int index, struct vfio_irq *irq);
 int vfio_pci_setup_device(struct kvm *kvm, struct vfio_device *device);
 void vfio_pci_teardown_device(struct kvm *kvm, struct vfio_device *vdev);
+int vfio_platform_setup_device(struct kvm *kvm, struct vfio_device *device);
+void vfio_platform_teardown_device(struct kvm *kvm, struct vfio_device *vdev);
 
 #endif /* KVM__VFIO_H */
vfio/core.c +22 -0
index aa124d2..6823a95 100644
@@ -41,6 +41,19 @@ static int vfio_device_pci_parser(const struct option *opt, char *arg,
 	return 0;
 }
 
+/* @arg is the name of the device under /sys/bus/platform/devices */
+static int vfio_device_platform_parser(const struct option *opt, char *arg,
+				       struct vfio_device_params *dev)
+{
+	dev->type = VFIO_DEVICE_PLATFORM;
+	dev->bus = "platform";
+	dev->name = strdup(arg);
+	if (!dev->name)
+		return -ENOMEM;
+
+	return 0;
+}
+
 int vfio_device_parser(const struct option *opt, const char *arg, int unset)
 {
 	int ret = -EINVAL;
@@ -72,6 +85,8 @@ int vfio_device_parser(const struct option *opt, const char *arg, int unset)
 
 	if (!strcmp(opt->long_name, "vfio-pci"))
 		ret = vfio_device_pci_parser(opt, cur, dev);
+	else if (!strcmp(opt->long_name, "vfio-platform"))
+		ret = vfio_device_platform_parser(opt, cur, dev);
 	else
 		ret = -EINVAL;
 
@@ -433,6 +448,10 @@ static int vfio_configure_device(struct kvm *kvm, struct vfio_device *vdev)
 		BUG_ON(!(vdev->info.flags & VFIO_DEVICE_FLAGS_PCI));
 		ret = vfio_pci_setup_device(kvm, vdev);
 		break;
+	case VFIO_DEVICE_PLATFORM:
+		BUG_ON(!(vdev->info.flags & VFIO_DEVICE_FLAGS_PLATFORM));
+		ret = vfio_platform_setup_device(kvm, vdev);
+		break;
 	default:
 		BUG_ON(1);
 		ret = -EINVAL;
@@ -703,6 +722,9 @@ static void vfio_device_exit(struct kvm *kvm, struct vfio_device *vdev)
 	case VFIO_DEVICE_PCI:
 		vfio_pci_teardown_device(kvm, vdev);
 		break;
+	case VFIO_DEVICE_PLATFORM:
+		vfio_platform_teardown_device(kvm, vdev);
+		break;
 	default:
 		vfio_dev_warn(vdev, "no teardown function for device");
 	}
vfio/platform.c new file +158 -0

158 lines not on this page · raw message

Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#8 lore raw

PATCH 8/9 vfio: platform: Add interrupts support

In reply to Mostafa Saleh

Configures interrupts of the device, relying on the new common VFIO interrupt code.

And generated the dt node for interrupts.

---
 include/kvm/vfio.h |   9 +++-
 vfio/platform.c    | 113 ++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 119 insertions(+), 3 deletions(-)
include/kvm/vfio.h +8 -1
index f641292..b3034a0 100644
@@ -89,6 +89,10 @@ struct vfio_pci_device {
 	struct vfio_pci_msix_pba	msix_pba;
 };
 
+struct vfio_platform_device {
+	struct vfio_irq			*irqs;
+};
+
 struct vfio_region {
 	struct vfio_region_info		info;
 	struct vfio_device		*vdev;
@@ -109,7 +113,10 @@ struct vfio_device {
 
 	char				*sysfs_path;
 
-	struct vfio_pci_device		pci;
+	union {
+		struct vfio_pci_device		pci;
+		struct vfio_platform_device	platform;
+	};
 };
 
 struct vfio_device_params {
vfio/platform.c +111 -2

158 lines not on this page · raw message

Signature · 1 line
2.56.0.307.g5fe7bd680c-goog
#9 lore raw

PATCH 9/9 vfio: platform: Take the guest node from a device tree template

In reply to Mostafa Saleh

At this point the device node has interrupts and regs, which is not enough to probe a driver in the guest, it at least needs a compatible string and other device bindings.
To avoid hardcoding those, let the user supply them as a compiled device tree blob:

--vfio-platform <device>[,<template.dtb>]

Every property of the template's root node, and every subnode, is copied into the generated node with fdt__copy_node().

---
 builtin-run.c      |  2 +-
 include/kvm/vfio.h |  2 ++
 vfio/core.c        | 16 ++++++++++++++-
 vfio/platform.c    | 50 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 2 deletions(-)
builtin-run.c +1 -1
index b09fc2e..a6fd8ec 100644
@@ -262,7 +262,7 @@ static int loglevel_parser(const struct option *opt, const char *arg, int unset)
 	OPT_CALLBACK('\0', "vfio-pci", NULL, "[domain:]bus:dev.fn",	\
 		     "Assign a PCI device to the virtual machine",	\
 		     vfio_device_parser, kvm),				\
-	OPT_CALLBACK('\0', "vfio-platform", NULL, "device",		\
+	OPT_CALLBACK('\0', "vfio-platform", NULL, "device[,template.dtb]",	\
 		     "Assign a platform device to the virtual machine",	\
 		     vfio_device_parser, kvm),				\
 									\
include/kvm/vfio.h +2 -0
index b3034a0..de24ba1 100644
@@ -121,6 +121,8 @@ struct vfio_device {
 
 struct vfio_device_params {
 	char				*name;
+	/* Platform devices: DTB whose root node is copied into the guest node */
+	char				*dt_template;
 	const char			*bus;
 	enum vfio_device_type		type;
 };
vfio/core.c +15 -1
index 6823a95..3ce1ffd 100644
@@ -41,16 +41,30 @@ static int vfio_device_pci_parser(const struct option *opt, char *arg,
 	return 0;
 }
 
-/* @arg is the name of the device under /sys/bus/platform/devices */
+/*
+ * @arg is the name of the device under /sys/bus/platform/devices, optionally
+ * followed by ",<template.dtb>".
+ */
 static int vfio_device_platform_parser(const struct option *opt, char *arg,
 				       struct vfio_device_params *dev)
 {
+	char *dt_template = strtok(NULL, ",");
+
 	dev->type = VFIO_DEVICE_PLATFORM;
 	dev->bus = "platform";
+	dev->dt_template = NULL;
 	dev->name = strdup(arg);
 	if (!dev->name)
 		return -ENOMEM;
 
+	if (dt_template) {
+		dev->dt_template = strdup(dt_template);
+		if (!dev->dt_template) {
+			free(dev->name);
+			return -ENOMEM;
+		}
+	}
+
 	return 0;
 }
 
vfio/platform.c +50 -0

73 lines not on this page · raw message

Signature · 1 line
2.56.0.307.g5fe7bd680c-goog